def loadData(self):
trainset = SimulationDataset("train", transforms=transforms.Compose([
utils.RandomCoose(['center']),
utils.Preprocess(self.input_shape),
# utils.RandomResizedCrop(self.input_shape),
# utils.RandomNoise(),
utils.RandomTranslate(10, 10),
# utils.RandomBrightness(),
# utils.RandomContrast(),
# utils.RandomHue(),
utils.RandomHorizontalFlip(),
utils.ToTensor(),
utils.Normalize([0.1, 0.4, 0.4], [0.9, 0.6, 0.5])
]))
# weights = utils.get_weights(trainset)
# sampler = torch.utils.data.sampler.WeightedRandomSampler(weights, len(weights), replacement=False)
# self.trainloader = torch.utils.data.DataLoader(trainset, batch_size=self.cfg.batch_size, sampler=sampler, num_workers=0, pin_memory=True)
self.trainloader = torch.utils.data.DataLoader(trainset, shuffle=True, batch_size=self.cfg.batch_size, num_workers=0, pin_memory=True)
testset = SimulationDataset("test", transforms=transforms.Compose([
utils.RandomCoose(['center']),
utils.Preprocess(self.input_shape),
utils.ToTensor(),
utils.Normalize([0.1, 0.4, 0.4], [0.9, 0.6, 0.5])
]))
self.testloader = torch.utils.data.DataLoader(testset, batch_size=self.cfg.batch_size, shuffle=False, num_workers=0, pin_memory=True)
def predict(self, image, preloaded=False):
# set test mode
self.net.eval()
if (not preloaded):
loadModel()
print('Loaded Model')
print('Starting Prediction')
composed = transforms.Compose([
utils.Preprocess(self.input_shape),
utils.ToTensor(),
utils.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
# Target gets discareded
sample = {'image': image, 'target': 0}
sample = composed(sample)
inputs = sample['image']
# Add single batch diemension
inputs = inputs.unsqueeze(0)
if (self.cfg.cuda):
inputs = Variable(inputs.cuda(async=True))
else:
inputs = Variable(inputs)
if (self.cfg.cuda):
outputs = self.net(inputs).cuda(async=True)
else:
outputs = self.net(inputs)
print('Finished Prediction')
print('Control tensor: %.6f ' % (outputs.item()))
# set train mode
self.net.train()
return outputs.item()
def main():
dataPath = "../../../Database/IEMOCAP_full_release/"
# hyperparams
tMax = 300
fMax = 400
batch_size = 64 * 2
# IEMOCAP
database = "IEMOCAP"
emotionsTest = ["Neutral", "Happiness", "Sadness", "Anger"]
num_classes = len(emotionsTest)
actTypeToUse = ["impro"]
emoTest = "".join([emo[0] for emo in actTypeToUse+emotionsTest])
dataname = "{}_{}".format(database, emoTest)
dataDf = utils.load_IEMOCAP(dataPath, actTypeToUse, emotionsTest)
# confusion matrix
cmV = np.zeros((num_classes, num_classes), dtype=int)
cmT = np.zeros((num_classes, num_classes), dtype=int)
for sp_test in range(10):
print("test speaker:", sp_test)
(x_train, y_train, y_train_raw,
x_val, y_val, y_val_raw,
x_test, y_test, y_test_raw) = utils.Preprocess(sp_test, emotionsTest,
dataDf, tMax, fMax)
model = load_model("models/CNN/{}.h5".format(sp_test))
y_val_p = model.predict(x_val,
batch_size=batch_size,
verbose=1)
y_val_p_C = utils.y_raw_transform(y_val_p, y_val_raw)
cmV += confusion_matrix(y_val_raw[:, 0], y_val_p_C)
y_test_p = model.predict(x_test,
batch_size=batch_size,
verbose=1)
y_test_p_C = utils.y_raw_transform(y_test_p, y_test_raw)
cmT += confusion_matrix(y_test_raw[:, 0], y_test_p_C)
print("speaker: " + str(sp_test))
print(cmV)
print(cmT)
del model
keras.backend.clear_session()
# Evaluation
waV, uaV = np.around(utils.waua(cmV), decimals=4)
waT, uaT = np.around(utils.waua(cmT), decimals=4)
cmpV = cmV / np.reshape(np.sum(cmV, 1), (4, 1))
cmpT = cmT / np.reshape(np.sum(cmT, 1), (4, 1))
imageName = "results/CNN/Test_{}_CNN_V.png".format(dataname)
title = "wa={}, ua={}".format(waV, uaV)
utils.plot_wauacm(title, cmpV, emotionsTest, imageName)
imageName = "results/CNN/Test_{}_CNN_T.png".format(dataname)
title = "wa={}, ua={}".format(waT, uaT)
utils.plot_wauacm(title, cmpT, emotionsTest, imageName)
print("waV: " + str(waV) + ", uaV: " + str(uaV))
print(cmpV)
print("waT: " + str(waT) + ", uaT: " + str(uaT))
print(cmpT)
def loadData(self):
trainset = SimulationDataset(
"train",
transforms=transforms.Compose([
utils.RandomCoose(['centre', 'left', 'right']),
utils.Preprocess(self.input_shape),
utils.RandomTranslate(100, 10),
utils.RandomBrightness(),
utils.RandomContrast(),
utils.RandomHue(),
utils.RandomHorizontalFlip(),
utils.ToTensor(),
utils.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]))
weights = utils.get_weights(trainset)
sampler = torch.utils.data.sampler.WeightedRandomSampler(
weights, len(weights), replacement=True)
# self.trainloader = torch.utils.data.DataLoader(trainset, batch_size=self.cfg.batch_size, sampler=sampler, num_workers=4)
self.trainloader = torch.utils.data.DataLoader(
trainset, batch_size=self.cfg.batch_size, num_workers=4)
testset = SimulationDataset("test",
transforms=transforms.Compose([
utils.RandomCoose(['center']),
utils.Preprocess(self.input_shape),
utils.ToTensor(),
utils.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
]))
self.testloader = torch.utils.data.DataLoader(
testset,
batch_size=self.cfg.batch_size,
shuffle=False,
num_workers=4)
def main(): dataname = 'testedois' mypath = '/home/luiza/Documents/autocone_dataset/' p = utils.Preprocess(path=mypath, dataname=dataname) all_images, data_list, n = p.return_data() m = Model() x_train, x_test, y_train, y_test, input_shape = m.train_test_split(all_images, data_list.iloc[:, 1:4]) print(input_shape, x_train.shape, y_train, y_train.shape) model = m.model(input_shape) m.model_fit(model, x_train, y_train, x_test, y_test, n)
def telemetry(sid, data):
if data:
# The current steering angle of the car
steering_angle = float(data["steering_angle"])
# The current throttle of the car
throttle = float(data["throttle"])
# The current speed of the car
speed = float(data["speed"])
# The current image from the center camera of the car
image = Image.open(BytesIO(base64.b64decode(data["image"])))
# save frame
if args.image_folder != '':
timestamp = datetime.utcnow().strftime('%Y_%m_%d_%H_%M_%S_%f')[:-3]
image_filename = os.path.join(args.image_folder, timestamp)
image.save('{}.jpg'.format(image_filename))
try:
image = np.asarray(image) # from PIL image to numpy array
image = utils.Preprocess(image) # apply the preprocessing
image = np.array([image]) # the model expects 4D array
# predict the steering angle for the image
steering_angle = float(model.predict(image, batch_size=1))
# lower the throttle as the speed increases
# if the speed is above the current speed limit, we are on a downhill.
# make sure we slow down first and then go back to the original max speed.
global speed_limit
if speed > speed_limit:
speed_limit = MIN_SPEED # slow down
else:
speed_limit = MAX_SPEED
throttle = 1.0 - steering_angle**2 - (speed / speed_limit)**2
print('{} {} {}'.format(steering_angle, throttle, speed))
send_control(steering_angle, throttle)
except Exception as e:
print(e)
else:
# NOTE: DON'T EDIT THIS.
sio.emit('manual', data={}, skip_sid=True)
def main():
dataPath = "../../../Database/IEMOCAP_full_release/"
if not os.path.exists("results"):
os.mkdir("results")
if not os.path.exists("results/CNN"):
os.mkdir("results/CNN")
if not os.path.exists("models"):
os.mkdir("models")
if not os.path.exists("models/CNN"):
os.mkdir("models/CNN")
if socket.getfqdn(socket.gethostname()) == "d8":
os.environ["CUDA_VISIBLE_DEVICES"] = "2,3"
# hyperparams
dr = 0.3
lr = 0.006
tMax = 300
fMax = 400
input_shape = (tMax, fMax, 1)
batch_size = 64 * 2
# IEMOCAP
database = "IEMOCAP"
emotionsTest = ["Neutral", "Happiness", "Sadness", "Anger"]
num_classes = len(emotionsTest)
actTypeToUse = ["impro"]
emoTest = "".join([emo[0] for emo in actTypeToUse + emotionsTest])
dataname = "{}_{}".format(database, emoTest)
dataDf = utils.load_IEMOCAP(dataPath, actTypeToUse, emotionsTest)
# confusion matrix
cmV = np.zeros((num_classes, num_classes), dtype=int)
cmT = np.zeros((num_classes, num_classes), dtype=int)
for sp_test in range(10):
print("test speaker:", sp_test)
(x_train, y_train, y_train_raw, x_val, y_val, y_val_raw, x_test,
y_test, y_test_raw) = utils.Preprocess(sp_test, emotionsTest, dataDf,
tMax, fMax)
# set training parameters
mc_cb = keras.callbacks.ModelCheckpoint('BestModel_CNN.h5',
monitor='val_acc',
verbose=1,
save_best_only=True,
save_weights_only=True)
mc_es = keras.callbacks.EarlyStopping(monitor="acc",
patience=10,
verbose=1)
# class weight
cw = class_weight.compute_class_weight('balanced',
np.unique(y_train_raw[:, 0]),
y_train_raw[:, 0])
cw2 = {0: cw[0], 1: cw[1], 2: cw[2], 3: cw[3]}
model = cnn_model(num_classes=num_classes,
input_shape=input_shape,
dr=dr,
lr=lr)
epochs = 16
model.fit(x_train,
y_train,
class_weight=cw2,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_val, y_val))
epochs = 50
model.fit(x_train,
y_train,
class_weight=cw2,
batch_size=batch_size,
epochs=epochs,
verbose=1,
callbacks=[mc_cb, mc_es],
validation_data=(x_val, y_val))
del mc_cb
model.load_weights("BestModel_CNN.h5")
y_val_p = model.predict(x_val, batch_size=batch_size, verbose=1)
y_val_p_C = utils.y_raw_transform(y_val_p, y_val_raw)
cmV += confusion_matrix(y_val_raw[:, 0], y_val_p_C)
y_test_p = model.predict(x_test, batch_size=batch_size, verbose=1)
y_test_p_C = utils.y_raw_transform(y_test_p, y_test_raw)
cmT += confusion_matrix(y_test_raw[:, 0], y_test_p_C)
print("speaker: " + str(sp_test))
print(cmV)
print(cmT)
model.save("models/CNN/{}.h5".format(sp_test))
del model
keras.backend.clear_session()
# Evaluation
waV, uaV = np.around(utils.waua(cmV), decimals=4)
waT, uaT = np.around(utils.waua(cmT), decimals=4)
cmpV = cmV / np.reshape(np.sum(cmV, 1), (4, 1))
cmpT = cmT / np.reshape(np.sum(cmT, 1), (4, 1))
imageName = "results/CNN/Train_{}_CNN_V.png".format(dataname)
title = "wa={}, ua={}".format(waV, uaV)
utils.plot_wauacm(title, cmpV, emotionsTest, imageName)
imageName = "results/CNN/Train_{}_CNN_T.png".format(dataname)
title = "wa={}, ua={}".format(waT, uaT)
utils.plot_wauacm(title, cmpT, emotionsTest, imageName)
print("waV: " + str(waV) + ", uaV: " + str(uaV))
print(cmpV)
print("waT: " + str(waT) + ", uaT: " + str(uaT))
print(cmpT)
# plt.imshow(F.to_pil_image(sample['image']))
# plt.title(str(sample['target']))
# plt.show()
return sample['image'], sample['target']
def __len__(self):
return len(self.image_paths)
if __name__ == '__main__':
input_shape = (utils.IMAGE_HEIGHT, utils.IMAGE_WIDTH)
dataset = SimulationDataset("train",
transforms=transforms.Compose([
utils.RandomCoose(['center']),
utils.Preprocess(input_shape),
utils.RandomHorizontalFlip(),
utils.ToTensor(),
utils.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
]))
print(dataset.__len__())
print(dataset.__getitem__(0)[0].size())
for c in range(3):
for i in range(dataset.__len__()):
print(dataset.__getitem__(i)[c].mean())
print(dataset.__getitem__(i)[c].std())
# print(dataset.__getitem__(0))
# print(len(dataset.__get_annotations__()))
import tensorflow as tf
import utils
from model import transformer
preprocessor = utils.Preprocess()
questions = preprocessor.wiki_questions + preprocessor.cornell_questions
answers = preprocessor.wiki_answers + preprocessor.cornell_answers
VOCAB_SIZE = preprocessor.vocab_size
questions, answers, = preprocessor.tokenize_and_filter(questions, answers)
BATCH_SIZE = 32
BUFFER_SIZE = 20000
# remove START_TOKEN from targets
dataset = tf.data.Dataset.from_tensor_slices((
{
'inputs': questions,
'dec_inputs': answers[:, :-1]
},
{
'outputs': answers[:, 1:]
},
))
dataset = dataset.cache()
dataset = dataset.shuffle(BUFFER_SIZE)
dataset = dataset.batch(BATCH_SIZE)
dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
"""
from scipy.io import wavfile
from python_speech_features import mfcc, logfbank
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import librosa
import multiprocessing
import time
import csv
import os
import wave, struct, math, random
import utils
prep = utils.Preprocess()
"""Upload and prepare dataset for usage"""
df = pd.read_csv(r'./one_hot_classes.csv')
df.set_index(df['Unnamed: 0'], inplace=True)
df.drop(columns=['Unnamed: 0'], inplace=True)
"""Upload WAV file, extract rate and signal"""
for f in df.index:
try:
rate, signal = wavfile.read('audio/' + f)
df.at[f, 'length'] = signal.shape[0] / rate
except:
df.drop(index=f, inplace=True)
df.to_csv('./whole_audio_l.csv')
"""Upload complete dataset"""
df = pd.read_csv(r'./whole_audio_l.csv')
from scipy.io import wavfile
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import librosa
import time
import csv
import os
import wave, struct, math, random
import utils
wavs = []
for route, dirs, files in os.walk('./audio'):
wavs = files
prep = utils.Preprocess(wavs)
"""Upload and prepare dataset for usage"""
df = pd.read_csv(r'./one_hot_classes.csv')
df.set_index(df['Unnamed: 0'], inplace=True)
df.drop(columns=['Unnamed: 0'], inplace=True)
"""Upload WAV file, extract rate and signal"""
for f in df.index:
try:
rate, signal = wavfile.read('audio/' + f)
df.at[f, 'length'] = signal.shape[0] / rate
except:
df.drop(index=f, inplace=True)
df.to_csv('./whole_audio_l.csv')
"""Upload complete dataset"""
df = pd.read_csv(r'./whole_audio_l.csv')
