def test_create_and_load_meta_csv_df(self): # remove if exists if os.path.exists(os.path.join(self.dest, 'dataset_attr.csv')): os.remove(os.path.join(self.dest, 'dataset_attr.csv')) # run function df, train_df, test_df = dataset.create_and_load_meta_csv_df( self.dataset_path, destination_path=self.dest, randomize=self.randomize, split=self.split) self.assertEqual(len(train_df['label'].unique()), len(test_df['label'].unique())) # check df size self.assertEqual(len(df), len(train_df) + len(test_df)) # check if file generated and delete it if clear is true self.assertTrue( os.path.exists(os.path.join(self.dest, 'dataset_attr.csv'))) if os.path.exists(os.path.join(self.dest, 'dataset_attr.csv')): print("Successfully created 'dataset_attr.csv file'") if self.clear: print('Cleaning now...') os.remove(os.path.join(self.dest, 'dataset_attr.csv'))
def load_data(): ''' Argumensts : None Returns : Tuple of 4 values containg animal loader habitat loader animal DataFrame habitat DataFrame respectively Description : The function loads the data from dataset file ''' dataset_path = destination_path = 'Images/' animal_df, habitat_df = create_and_load_meta_csv_df(dataset_path, destination_path, randomize=True) animal_loader = ImageDataset(animal_df) habitat_loader = ImageDataset(habitat_df) animal_predict = torch.utils.data.DataLoader(animal_loader, batch_size=20, shuffle=False) habitat_predict = torch.utils.data.DataLoader(habitat_loader, batch_size=25, shuffle=False) return animal_predict, habitat_predict, animal_df, habitat_df
def train_model(dataset_path, debug=False, destination_path='', save=False, epochs=1): """Trains model with set hyper-parameters and provide an option to save the model. This function should contain necessary logic to load fruits dataset and train a CNN model on it. It should accept dataset_path which will be path to the dataset directory. You should also specify an option to save the trained model with all parameters. If debug option is specified, it'll print loss and accuracy for all iterations. Returns loss and accuracy for both train and validation sets. Args: dataset_path (str): Path to the dataset folder. For example, '../Data/fruits/'. debug (bool, optional): Prints train, validation loss and accuracy for every iteration. Defaults to False. destination_path (str, optional): Destination to save the model file. Defaults to ''. save (bool, optional): Saves model if True. Defaults to False. Returns: loss (torch.tensor): Train loss and validation loss. accuracy (torch.tensor): Train accuracy and validation accuracy. """ dframe, train_data, test_data = create_and_load_meta_csv_df( './Data/fruits', './Data/dataset', randomize=None, split=.8) train_data = DataLoader(dataset=ImageDataset( data=train_data, transform=transforms.ToTensor()), batch_size=32) test_data = DataLoader(dataset=ImageDataset(test_data, transforms.ToTensor()), batch_size=32) costFunction = nn.CrossEntropyLoss() optimizer = optim.SGD(net.parameters(), lr=.001, momentum=0.9) #print(train_data.dataset.data) acc, loss = train(epochs, train_data=train_data, costFunction=costFunction, optimizer=optimizer, test_data=test_data, debug=debug, destination_path=destination_path, save=save) return acc, loss
def train_model(dataset_path, debug=False, destination_path='./meta', save=False): """Trains model with set hyper-parameters and provide an option to save the model. This function should contain necessary logic to load fruits dataset and train a CNN model on it. It should accept dataset_path which will be path to the dataset directory. You should also specify an option to save the trained model with all parameters. If debug option is specified, it'll print loss and accuracy for all iterations. Returns loss and accuracy for both train and validation sets. Args: dataset_path (str): Path to the dataset folder. For example, '../Data/fruits/'. debug (bool, optional): Prints train, validation loss and accuracy for every iteration. Defaults to False. destination_path (str, optional): Destination to save the model file. Defaults to ''. save (bool, optional): Saves model if True. Defaults to False. Returns: loss (torch.tensor): Train loss and validation loss. accuracy (torch.tensor): Train accuracy and validation accuracy. """ df, traindf, testdf = create_and_load_meta_csv_df(dataset_path, destination_path, True, 0.7) traindataset = ImageDataset(traindf) testdataset = ImageDataset(testdf) trainloader = DataLoader(traindataset, batch_size=32, shuffle=True, num_workers=2) testloader = DataLoader(testdataset, batch_size=32, shuffle=True, num_workers=2) net = FNet() criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) for epoch in range(2): # loop over the dataset multiple times running_loss = 0.0 for i, data in enumerate(trainloader, 0): # get the inputs inputs, labels = data # zero the parameter gradients optimizer.zero_grad() # forward + backward + optimize outputs = net(inputs) loss = criterion(outputs, labels) loss.backward() optimizer.step() # print statistics running_loss += loss.item() if i % 2000 == 1999: # print every 2000 mini-batches print('[%d, %5d] loss: %.3f' % (epoch + 1, i + 1, running_loss / 2000)) running_loss = 0.0 print('Finished Training')
randomize = True # for spliting dataset into training and testing dataset split = 0.8 # Number of sample per second e.g. 16KHz sampling_rate = 20000 emotions = ["anger", "disgust", "fear", "happy", "neutral", "sad", "surprise"] """### Converting Dataset in CSV format it will cause easy operation on Dataset. """ # loading dataframes using dataset module from utils import dataset # To know more about "create_and_load_meta_csv_df" function and it's working, go to "./utils/dataset.py" script. df, train_df, test_df = dataset.create_and_load_meta_csv_df( dataset_path, destination_path, randomize, split) #print('Dataset samples : ', len(df),"\nTraining Samples : ", len(train_df),"\ntesting Samples : ", len(test_df)) """# 4. Data Visualization Let's understand what is our dataset. """ df.head() print("Actual Audio : ", df['path'][0]) print("Labels : ", df['label'][0]) """### Labels Assigned for emotions : - 0 : anger - 1 : disgust - 2 : fear