def test_create_and_load_meta_csv_df(self):
# remove if exists
if os.path.exists(os.path.join(self.dest, 'dataset_attr.csv')):
os.remove(os.path.join(self.dest, 'dataset_attr.csv'))
# run function
df, train_df, test_df = dataset.create_and_load_meta_csv_df(
self.dataset_path,
destination_path=self.dest,
randomize=self.randomize,
split=self.split)
self.assertEqual(len(train_df['label'].unique()),
len(test_df['label'].unique()))
# check df size
self.assertEqual(len(df), len(train_df) + len(test_df))
# check if file generated and delete it if clear is true
self.assertTrue(
os.path.exists(os.path.join(self.dest, 'dataset_attr.csv')))
if os.path.exists(os.path.join(self.dest, 'dataset_attr.csv')):
print("Successfully created 'dataset_attr.csv file'")
if self.clear:
print('Cleaning now...')
os.remove(os.path.join(self.dest, 'dataset_attr.csv'))
def load_data():
'''
Argumensts : None
Returns : Tuple of 4 values containg animal loader habitat loader animal DataFrame habitat DataFrame respectively
Description :
The function loads the data from dataset file
'''
dataset_path = destination_path = 'Images/'
animal_df, habitat_df = create_and_load_meta_csv_df(dataset_path,
destination_path,
randomize=True)
animal_loader = ImageDataset(animal_df)
habitat_loader = ImageDataset(habitat_df)
animal_predict = torch.utils.data.DataLoader(animal_loader,
batch_size=20,
shuffle=False)
habitat_predict = torch.utils.data.DataLoader(habitat_loader,
batch_size=25,
shuffle=False)
return animal_predict, habitat_predict, animal_df, habitat_df
def train_model(dataset_path,
debug=False,
destination_path='',
save=False,
epochs=1):
"""Trains model with set hyper-parameters and provide an option to save the model.
This function should contain necessary logic to load fruits dataset and train a CNN model on it. It should accept dataset_path which will be path to the dataset directory. You should also specify an option to save the trained model with all parameters. If debug option is specified, it'll print loss and accuracy for all iterations. Returns loss and accuracy for both train and validation sets.
Args:
dataset_path (str): Path to the dataset folder. For example, '../Data/fruits/'.
debug (bool, optional): Prints train, validation loss and accuracy for every iteration. Defaults to False.
destination_path (str, optional): Destination to save the model file. Defaults to ''.
save (bool, optional): Saves model if True. Defaults to False.
Returns:
loss (torch.tensor): Train loss and validation loss.
accuracy (torch.tensor): Train accuracy and validation accuracy.
"""
dframe, train_data, test_data = create_and_load_meta_csv_df(
'./Data/fruits', './Data/dataset', randomize=None, split=.8)
train_data = DataLoader(dataset=ImageDataset(
data=train_data, transform=transforms.ToTensor()),
batch_size=32)
test_data = DataLoader(dataset=ImageDataset(test_data,
transforms.ToTensor()),
batch_size=32)
costFunction = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=.001, momentum=0.9)
#print(train_data.dataset.data)
acc, loss = train(epochs,
train_data=train_data,
costFunction=costFunction,
optimizer=optimizer,
test_data=test_data,
debug=debug,
destination_path=destination_path,
save=save)
return acc, loss
def train_model(dataset_path,
debug=False,
destination_path='./meta',
save=False):
"""Trains model with set hyper-parameters and provide an option to save the model.
This function should contain necessary logic to load fruits dataset and train a CNN model on it. It should accept dataset_path which will be path to the dataset directory. You should also specify an option to save the trained model with all parameters. If debug option is specified, it'll print loss and accuracy for all iterations. Returns loss and accuracy for both train and validation sets.
Args:
dataset_path (str): Path to the dataset folder. For example, '../Data/fruits/'.
debug (bool, optional): Prints train, validation loss and accuracy for every iteration. Defaults to False.
destination_path (str, optional): Destination to save the model file. Defaults to ''.
save (bool, optional): Saves model if True. Defaults to False.
Returns:
loss (torch.tensor): Train loss and validation loss.
accuracy (torch.tensor): Train accuracy and validation accuracy.
"""
df, traindf, testdf = create_and_load_meta_csv_df(dataset_path,
destination_path, True,
0.7)
traindataset = ImageDataset(traindf)
testdataset = ImageDataset(testdf)
trainloader = DataLoader(traindataset,
batch_size=32,
shuffle=True,
num_workers=2)
testloader = DataLoader(testdataset,
batch_size=32,
shuffle=True,
num_workers=2)
net = FNet()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
for epoch in range(2): # loop over the dataset multiple times
running_loss = 0.0
for i, data in enumerate(trainloader, 0):
# get the inputs
inputs, labels = data
# zero the parameter gradients
optimizer.zero_grad()
# forward + backward + optimize
outputs = net(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
# print statistics
running_loss += loss.item()
if i % 2000 == 1999: # print every 2000 mini-batches
print('[%d, %5d] loss: %.3f' %
(epoch + 1, i + 1, running_loss / 2000))
running_loss = 0.0
print('Finished Training')
randomize = True
# for spliting dataset into training and testing dataset
split = 0.8
# Number of sample per second e.g. 16KHz
sampling_rate = 20000
emotions = ["anger", "disgust", "fear", "happy", "neutral", "sad", "surprise"]
"""### Converting Dataset in CSV format
it will cause easy operation on Dataset.
"""
# loading dataframes using dataset module
from utils import dataset
# To know more about "create_and_load_meta_csv_df" function and it's working, go to "./utils/dataset.py" script.
df, train_df, test_df = dataset.create_and_load_meta_csv_df(
dataset_path, destination_path, randomize, split)
#print('Dataset samples : ', len(df),"\nTraining Samples : ", len(train_df),"\ntesting Samples : ", len(test_df))
"""# 4. Data Visualization
Let's understand what is our dataset.
"""
df.head()
print("Actual Audio : ", df['path'][0])
print("Labels : ", df['label'][0])
"""### Labels Assigned for emotions :
- 0 : anger
- 1 : disgust
- 2 : fear