def read_log(log_name, command):
"""Reads in a log and returns a list of all the command values"""
commands = []
with open(log_name, 'r') as log:
data = json.load(log)
for log in data:
# If "status" is present, a javascript title was sent
if "status" in log:
pass
else:
curr_command = log[command]
if curr_command is not None:
commands.append(curr_command)
if commands: # Check if there's actual commands to process
print(" mean: {}".format(utils.mean(commands)))
print(" max: {}".format(max(commands)))
print(" min: {}".format(min(commands)))
print(" total: {}".format(utils.total(commands)))
else:
print("There is no commands to be parsed")
def main(options):
DTYPE = torch.FloatTensor
# parse the input args
run_id = options['run_id']
epochs = options['epochs']
data_path = options['data_path']
model_path = options['model_path']
output_path = options['output_path']
signiture = options['signiture']
patience = options['patience']
output_dim = options['output_dim']
print("Training initializing... Setup ID is: {}".format(run_id))
# prepare the paths for storing models and outputs
model_path = os.path.join(model_path,
"model_{}_{}.pt".format(signiture, run_id))
output_path = os.path.join(output_path,
"results_{}_{}.csv".format(signiture, run_id))
print("Temp location for models: {}".format(model_path))
print("Grid search results are in: {}".format(output_path))
train_set, valid_set, test_set, input_dims = load_pom(data_path)
params = dict()
params['audio_hidden'] = [4, 8, 16]
params['video_hidden'] = [4, 8, 16]
params['text_hidden'] = [64, 128, 256]
params['audio_dropout'] = [0, 0.1, 0.15, 0.2, 0.3, 0.5]
params['video_dropout'] = [0, 0.1, 0.15, 0.2, 0.3, 0.5]
params['text_dropout'] = [0, 0.1, 0.15, 0.2, 0.3, 0.5]
params['factor_learning_rate'] = [0.0003, 0.0005, 0.001, 0.003]
params['learning_rate'] = [0.0003, 0.0005, 0.001, 0.003]
params['rank'] = [1, 4, 8, 16]
params['batch_size'] = [4, 8, 16, 32, 64, 128]
params['weight_decay'] = [0, 0.001, 0.002, 0.01]
total_settings = total(params)
print("There are {} different hyper-parameter settings in total.".format(
total_settings))
seen_settings = set()
with open(output_path, 'w+') as out:
writer = csv.writer(out)
writer.writerow([
"audio_hidden", "video_hidden", 'text_hidden', 'audio_dropout',
'video_dropout', 'text_dropout', 'factor_learning_rate',
'learning_rate', 'rank', 'batch_size', 'weight_decay',
'Best Validation MAE', 'Confidence accuracy',
'Passionate accuracy', 'Pleasant accuracy', 'Dominant accuracy',
'Credible accuracy', 'Vivid accuracy', 'Expertise accuracy',
'Entertaining accuracy', 'Reserved accuracy', 'Trusting accuracy',
'Relaxed accuracy', 'Outgoing accuracy', 'Thorough accuracy',
'Nervous accuracy', 'Persuasive accuracy', 'Humorous accuracy',
'Confidence MAE', 'Passionate MAE', 'Pleasant MAE', 'Dominant MAE',
'Credible MAE', 'Vivid MAE', 'Expertise MAE', 'Entertaining MAE',
'Reserved MAE', 'Trusting MAE', 'Relaxed MAE', 'Outgoing MAE',
'Thorough MAE', 'Nervous MAE', 'Persuasive MAE', 'Humorous MAE',
'Confidence corr', 'Passionate corr', 'Pleasant corr',
'Dominant corr', 'Credible corr', 'Vivid corr', 'Expertise corr',
'Entertaining corr', 'Reserved corr', 'Trusting corr',
'Relaxed corr', 'Outgoing corr', 'Thorough corr', 'Nervous corr',
'Persuasive corr', 'Humorous corr'
])
for i in range(total_settings):
ahid = random.choice(params['audio_hidden'])
vhid = random.choice(params['video_hidden'])
thid = random.choice(params['text_hidden'])
thid_2 = thid / 2
adr = random.choice(params['audio_dropout'])
vdr = random.choice(params['video_dropout'])
tdr = random.choice(params['text_dropout'])
factor_lr = random.choice(params['factor_learning_rate'])
lr = random.choice(params['learning_rate'])
r = random.choice(params['rank'])
batch_sz = random.choice(params['batch_size'])
decay = random.choice(params['weight_decay'])
# reject the setting if it has been tried
current_setting = (ahid, vhid, thid, adr, vdr, tdr, factor_lr, lr, r,
batch_sz, decay)
if current_setting in seen_settings:
continue
else:
seen_settings.add(current_setting)
model = LMF(input_dims, (ahid, vhid, thid), thid_2,
(adr, vdr, tdr, 0.5), output_dim, r)
if options['cuda']:
model = model.cuda()
DTYPE = torch.cuda.FloatTensor
print("Model initialized")
criterion = nn.L1Loss(size_average=False)
factors = list(model.parameters())[:3]
other = list(model.parameters())[5:]
optimizer = optim.Adam(
[{
"params": factors,
"lr": factor_lr
}, {
"params": other,
"lr": lr
}],
weight_decay=decay
) # don't optimize the first 2 params, they should be fixed (output_range and shift)
# setup training
complete = True
min_valid_loss = float('Inf')
train_iterator = DataLoader(train_set,
batch_size=batch_sz,
num_workers=4,
shuffle=True)
valid_iterator = DataLoader(valid_set,
batch_size=len(valid_set),
num_workers=4,
shuffle=True)
test_iterator = DataLoader(test_set,
batch_size=len(test_set),
num_workers=4,
shuffle=True)
curr_patience = patience
for e in range(epochs):
model.train()
model.zero_grad()
avg_train_loss = 0.0
for batch in train_iterator:
model.zero_grad()
x = batch[:-1]
x_a = Variable(x[0].float().type(DTYPE), requires_grad=False)
x_v = Variable(x[1].float().type(DTYPE), requires_grad=False)
x_t = Variable(x[2].float().type(DTYPE), requires_grad=False)
y = Variable(batch[-1].view(-1,
output_dim).float().type(DTYPE),
requires_grad=False)
output = model(x_a, x_v, x_t)
loss = criterion(output, y)
loss.backward()
avg_loss = loss.data[0] / float(output_dim)
avg_train_loss += avg_loss / len(train_set)
optimizer.step()
print("Epoch {} complete! Average Training loss: {}".format(
e, avg_train_loss))
# Terminate the training process if run into NaN
if np.isnan(avg_train_loss):
print("Training got into NaN values...\n\n")
complete = False
break
model.eval()
for batch in valid_iterator:
x = batch[:-1]
x_a = Variable(x[0].float().type(DTYPE), requires_grad=False)
x_v = Variable(x[1].float().type(DTYPE), requires_grad=False)
x_t = Variable(x[2].float().type(DTYPE), requires_grad=False)
y = Variable(batch[-1].view(-1,
output_dim).float().type(DTYPE),
requires_grad=False)
output = model(x_a, x_v, x_t)
valid_loss = criterion(output, y)
avg_valid_loss = valid_loss.data[0] / float(output_dim)
y = y.cpu().data.numpy().reshape(-1, output_dim)
if np.isnan(avg_valid_loss):
print("Training got into NaN values...\n\n")
complete = False
break
avg_valid_loss = avg_valid_loss / len(valid_set)
print("Validation loss is: {}".format(avg_valid_loss))
if (avg_valid_loss < min_valid_loss):
curr_patience = patience
min_valid_loss = avg_valid_loss
torch.save(model, model_path)
print("Found new best model, saving to disk...")
else:
curr_patience -= 1
if curr_patience <= 0:
break
print("\n\n")
if complete:
best_model = torch.load(model_path)
best_model.eval()
for batch in test_iterator:
x = batch[:-1]
x_a = Variable(x[0].float().type(DTYPE), requires_grad=False)
x_v = Variable(x[1].float().type(DTYPE), requires_grad=False)
x_t = Variable(x[2].float().type(DTYPE), requires_grad=False)
y = Variable(batch[-1].view(-1,
output_dim).float().type(DTYPE),
requires_grad=False)
output_test = best_model(x_a, x_v, x_t)
loss_test = criterion(output_test, y)
test_loss = loss_test.data[0]
avg_test_loss = test_loss / float(output_dim)
output_test = output_test.cpu().data.numpy().reshape(
-1, output_dim)
y = y.cpu().data.numpy().reshape(-1, output_dim)
# these are the needed metrics
mae = np.mean(np.absolute(output_test - y), axis=0)
mae = [round(a, 3) for a in mae]
corr = [
round(np.corrcoef(output_test[:, i], y[:, i])[0][1], 3)
for i in xrange(y.shape[1])
]
mult_acc = [
round(
sum(np.round(output_test[:, i]) == np.round(y[:, i])) /
float(len(y)), 3) for i in xrange(y.shape[1])
]
display(mae, corr, mult_acc)
results = [
ahid, vhid, thid, adr, vdr, tdr, factor_lr, lr, r, batch_sz,
decay,
min_valid_loss.cpu().data.numpy()
]
results.extend(mult_acc)
results.extend(mae)
results.extend(corr)
with open(output_path, 'a+') as out:
writer = csv.writer(out)
writer.writerow(results)
def main():
#print("hello")
#Networking things
s = socket.socket()
s.connect((cfg.HOST, cfg.PORT))
s.send("PASS {}\r\n".format(cfg.PASS).encode("utf-8"))
s.send("NICK {}\r\n".format(cfg.NICK).encode("utf-8"))
s.send("JOIN #{}\r\n".format(cfg.CHAN).encode("utf-8"))
CHAT_MSG = re.compile(r"^:\w+!\w+@\w+\.tmi\.twitch\.tv PRIVMSG #\w+ :")
utils.chat(s, "Hi everyone!")
thread.start_new_thread(utils.threatFillOpList, ())
#thread.start_new_thread(utils.constantGreeting(s), ()) #look into multiprocessing / multithreading (threding library?)
timeCount = 0
while True:
try:
response = s.recv(1024)
#if the connection to tmi.twitch.tv connects will get a pong
if not response: break
r = response.decode("utf-8")
if r == "PING :tmi.twitch.tv\r\n":
s.send("PONG :tmi.twtich.tv\r\n".encode("utf-8"))
else:
username = re.search(r"\w+", r).group(0)
message = CHAT_MSG.sub("", r)
print(r)
timeCount += 1
# Multithreading for all the greeting
# Hope this works and doesn't break
#t = threading.Thread(target=utils.constantGreeting(s))
#t.daemon = True
#t.start()
# Custom Commands
#Not a real timer sadly, This will increment each time a command is issued.
if timeCount == 500:
utils.chat(
s,
"Masc4Masc Mondays: Solo Stream with Jason! 8PM EST until 11P/12AM"
)
utils.chat(
s,
"Tabby Tuesdays: Solo Stream with Trent! 8PM EST until 11P/12AM"
)
utils.chat(
s,
"Thirsty Thursdays: Solo Stream with Matt! 10PM EST until 12AM"
)
utils.chat(
s,
"Festive Friday: Join the entire crew for party games! 7:30PM EST (ish) until we go to the bar (12A/1A)"
)
utils.chat(
s,
"Shady Saturday: Come talk shit and spill the T! 2PM EST until 8PM EST. WOOF!"
)
timecCount = 0
#if timeCount == 1000:
# utils.chat(s, "Remeber to follow and turn on the notification settings to know when we go on. To get more information about us, see the details portion of the stream.")
# timeCount = 0
print utils.isOp(username)
if message.strip() == "!schedule" and utils.isOp(
username) and cfg.CHAN == "supercubs":
utils.chat(
s,
"Masc4Masc Mondays: Solo Stream with Jason! 8PM EST until 11P/12AM"
)
utils.chat(
s,
"Tabby Tuesdays: Solo Stream with Trent! 8PM EST until 11P/12AM"
)
utils.chat(
s,
"Thirsty Thursdays: Solo Stream with Matt! 10PM EST until 12AM"
)
utils.chat(
s,
"Festive Friday: Join the entire crew for party games! 7:30PM EST (ish) until we go to the bar (12A/1A)"
)
utils.chat(
s,
"Shady Saturday Come talk shit and spill the T! 2PM EST (ish) until 8PM EST. WOOooF!"
)
#!commands whispers the command list NOT WORKING! -- needs a seperate connection to Group chat :/ Not sure..
if message.split()[0] == "!commands" and utils.isOp(username):
utils.whisper(s,
message.split(' ', 2)[1],
message.split(' ', 2)[2])
#!ban Hammer
if message.split()[0] == "!ban" and utils.isOp(username):
user = message.split()[1]
try:
utils.ban(s, message.split()[1])
except:
utils.chat(s, "No name")
#!timeout
if message.split()[0] == "!timeout" and utils.isOp(username):
user = message.split()[1]
try:
utils.timeout(s, message.split()[1])
except:
utils.chat(s, "No name")
#!eliminate <name>
if message.split()[0] == "!eliminate":
utils.eliminate(s, message.split()[1], username)
#!eliminated <name>
if message.split()[0] == "!eliminated" and utils.isOp(
username):
utils.eliminated(s, message.split()[1])
#!game (In testing)
if message.strip() == "!game":
#if len(message.strip()) > 1: #get requests package installed
# utils.chat(s, utils.currentPlaying())
utils.showGame(s)
#!create Command
if message.split()[0] == "!create" and utils.isOp(
username
): # I could make this safer by checking of the word has ! before the command else add it.
utils.createCommands(
message.split(' ', 2)[1],
message.split(' ', 2)[2])
#!useCommands created
if unicode(message.strip()) in cfg.commands:
utils.useCommands(s, message.strip())
#!delete commands
if message.split()[0] == "!delete" and utils.isOp(username):
utils.removeCommands(message.split()[1])
#!uptime
if message.strip() == "!uptime":
then = datetime.now()
utils.chat(
s, cfg.CHAN + " have been on for: " + (then - start))
#!valentine
if message.split()[0] == "!valentine":
utils.valentine(s, message.split(' ', 1)[1], username)
#!total
if message.strip() == "!total":
utils.total(s)
#!switch
if message.strip() == "!switch" and utils.isOp(username):
utils.chat(
s,
"Come join us at midinght release of the switch on March 3rd. We will be streaming for the release of the switch, more info to come!!!"
)
#if message.strip() == "!points":
# utils.points(s, username + " points are " + utils.points(username))
#if message.strip() == "!mods":
# print cfg.oplist
# if utils.isOp(username):
# utils.chat(s, username + " is a mod or higher")
except:
print "Main broke!"
sleep(1)
def main(options):
DTYPE = torch.FloatTensor
LONG = torch.LongTensor
# parse the input args
epochs = options['epochs']
data_path = options['data_path']
model_path = options['model_path']
output_path = options['output_path']
signiture = options['signiture']
patience = options['patience']
emotion = options['emotion']
output_dim = options['output_dim']
# prepare the paths for storing models and outputs
model_path = os.path.join(model_path,
"model_{}_{}.pt".format(signiture, emotion))
output_path = os.path.join(output_path,
"results_{}_{}.csv".format(signiture, emotion))
print("Temp location for models: {}".format(model_path))
print("Grid search results are in: {}".format(output_path))
os.makedirs(os.path.dirname(output_path), exist_ok=True)
os.makedirs(os.path.dirname(model_path), exist_ok=True)
train_set, valid_set, test_set, input_dims = load_iemocap(
data_path, emotion)
params = dict()
params['audio_hidden'] = [8, 16, 32]
params['video_hidden'] = [4, 8, 16]
params['text_hidden'] = [64, 128, 256]
params['audio_dropout'] = [0, 0.1, 0.15, 0.2, 0.3, 0.5]
params['video_dropout'] = [0, 0.1, 0.15, 0.2, 0.3, 0.5]
params['text_dropout'] = [0, 0.1, 0.15, 0.2, 0.3, 0.5]
params['factor_learning_rate'] = [0.0003, 0.0005, 0.001, 0.003]
params['learning_rate'] = [0.0003, 0.0005, 0.001, 0.003]
params['rank'] = [1, 4, 8, 16]
params['batch_size'] = [8, 16, 32, 64, 128]
params['weight_decay'] = [0, 0.001, 0.002, 0.01]
total_settings = total(params)
print("There are {} different hyper-parameter settings in total.".format(
total_settings))
seen_settings = set()
if not os.path.isfile(output_path):
with open(output_path, 'w+') as out:
writer = csv.writer(out)
writer.writerow([
"audio_hidden", "video_hidden", 'text_hidden', 'audio_dropout',
'video_dropout', 'text_dropout', 'factor_learning_rate',
'learning_rate', 'rank', 'batch_size', 'weight_decay',
'Best Validation CrossEntropyLoss', 'Test CrossEntropyLoss',
'Test F1-score', 'Test Accuracy Score'
])
for i in range(total_settings):
ahid = random.choice(params['audio_hidden'])
vhid = random.choice(params['video_hidden'])
thid = random.choice(params['text_hidden'])
thid_2 = thid // 2
adr = random.choice(params['audio_dropout'])
vdr = random.choice(params['video_dropout'])
tdr = random.choice(params['text_dropout'])
factor_lr = random.choice(params['factor_learning_rate'])
lr = random.choice(params['learning_rate'])
r = random.choice(params['rank'])
batch_sz = random.choice(params['batch_size'])
decay = random.choice(params['weight_decay'])
# reject the setting if it has been tried
current_setting = (ahid, vhid, thid, adr, vdr, tdr, factor_lr, lr, r,
batch_sz, decay)
if current_setting in seen_settings:
continue
else:
seen_settings.add(current_setting)
model = LMF(input_dims, (ahid, vhid, thid), thid_2,
(adr, vdr, tdr, 0.5), output_dim, r)
if options['cuda']:
model = model.cuda()
DTYPE = torch.cuda.FloatTensor
LONG = torch.cuda.LongTensor
print("Model initialized")
criterion = nn.CrossEntropyLoss(size_average=False)
factors = list(model.parameters())[:3]
other = list(model.parameters())[3:]
optimizer = optim.Adam([{
"params": factors,
"lr": factor_lr
}, {
"params": other,
"lr": lr
}],
weight_decay=decay)
# setup training
complete = True
min_valid_loss = float('Inf')
train_iterator = DataLoader(train_set,
batch_size=batch_sz,
num_workers=4,
shuffle=True)
valid_iterator = DataLoader(valid_set,
batch_size=len(valid_set),
num_workers=4,
shuffle=True)
test_iterator = DataLoader(test_set,
batch_size=len(test_set),
num_workers=4,
shuffle=True)
curr_patience = patience
for e in range(epochs):
model.train()
model.zero_grad()
avg_train_loss = 0.0
for batch in train_iterator:
model.zero_grad()
x = batch[:-1]
x_a = Variable(x[0].float().type(DTYPE), requires_grad=False)
x_v = Variable(x[1].float().type(DTYPE), requires_grad=False)
x_t = Variable(x[2].float().type(DTYPE), requires_grad=False)
y = Variable(batch[-1].view(-1, output_dim).float().type(LONG),
requires_grad=False)
try:
output = model(x_a, x_v, x_t)
except ValueError as e:
print(x_a.data.shape)
print(x_v.data.shape)
print(x_t.data.shape)
raise e
loss = criterion(output, torch.max(y, 1)[1])
loss.backward()
avg_loss = loss.data[0]
avg_train_loss += avg_loss / len(train_set)
optimizer.step()
print("Epoch {} complete! Average Training loss: {}".format(
e, avg_train_loss))
# Terminate the training process if run into NaN
if np.isnan(avg_train_loss):
print("Training got into NaN values...\n\n")
complete = False
break
model.eval()
for batch in valid_iterator:
x = batch[:-1]
x_a = Variable(x[0].float().type(DTYPE), requires_grad=False)
x_v = Variable(x[1].float().type(DTYPE), requires_grad=False)
x_t = Variable(x[2].float().type(DTYPE), requires_grad=False)
y = Variable(batch[-1].view(-1, output_dim).float().type(LONG),
requires_grad=False)
output = model(x_a, x_v, x_t)
valid_loss = criterion(output, torch.max(y, 1)[1])
avg_valid_loss = valid_loss.data[0]
y = y.cpu().data.numpy().reshape(-1, output_dim)
if np.isnan(avg_valid_loss):
print("Training got into NaN values...\n\n")
complete = False
break
avg_valid_loss = avg_valid_loss / len(valid_set)
print("Validation loss is: {}".format(avg_valid_loss))
if (avg_valid_loss < min_valid_loss):
curr_patience = patience
min_valid_loss = avg_valid_loss
torch.save(model, model_path)
print("Found new best model, saving to disk...")
else:
curr_patience -= 1
if curr_patience <= 0:
break
print("\n\n")
if complete:
best_model = torch.load(model_path)
best_model.eval()
for batch in test_iterator:
x = batch[:-1]
x_a = Variable(x[0].float().type(DTYPE), requires_grad=False)
x_v = Variable(x[1].float().type(DTYPE), requires_grad=False)
x_t = Variable(x[2].float().type(DTYPE), requires_grad=False)
y = Variable(batch[-1].view(-1, output_dim).float().type(LONG),
requires_grad=False)
output_test = model(x_a, x_v, x_t)
loss_test = criterion(output_test, torch.max(y, 1)[1])
test_loss = loss_test.data[0]
output_test = output_test.cpu().data.numpy().reshape(
-1, output_dim)
y = y.cpu().data.numpy().reshape(-1, output_dim)
test_loss = test_loss / len(test_set)
# these are the needed metrics
all_true_label = np.argmax(y, axis=1)
all_predicted_label = np.argmax(output_test, axis=1)
f1 = f1_score(all_true_label,
all_predicted_label,
average='weighted')
acc_score = accuracy_score(all_true_label, all_predicted_label)
display(f1, acc_score)
with open(output_path, 'a+') as out:
writer = csv.writer(out)
writer.writerow([
ahid, vhid, thid, adr, vdr, tdr, factor_lr, lr, r,
batch_sz, decay,
min_valid_loss.cpu().data.numpy(),
test_loss.cpu().data.numpy(), f1, acc_score
])
def main(opts):
data_type = torch.FloatTensor
# load option configs
poly_order = opts['poly_order']
poly_norm = opts['poly_norm']
euclid_norm = opts['euclid_norm']
audio_dim = opts['audio_dim']
video_dim = opts['video_dim']
text_dim = opts['text_dim']
input_dims = (audio_dim, video_dim, text_dim)
output_dim = opts['output_dim']
init_modal_len = opts['init_modal_len']
modal_wins = opts['modal_wins']
modal_pads = opts['modal_pads']
init_time_len = opts['init_time_len']
time_wins = opts['time_wins']
time_pads = opts['time_pads']
epochs = opts['epochs']
patience = opts['patience']
signature = opts['signature']
run_id = opts['run_id']
model_path = opts['model_path']
output_path = opts['output_path']
# set paths for storing models and outputs
model_path = os.path.join(model_path, 'model_{}_{}.pt'.format(signature, run_id))
output_path = os.path.join(output_path, 'result_{}_{}.csv'.format(signature, run_id))
# load data sets
train_set, label_train, valid_set, label_valid, test_set, label_test = load_saved_data(input_dims)
# set parameters
params = dict()
params['audio_hidden'] = [4, 8, 16]
params['video_hidden'] = [4, 8, 16]
params['text_hidden'] = [64, 128, 256]
params['audio_dropout'] = [0, 0.1, 0.15, 0.2, 0.3, 0.5]
params['video_dropout'] = [0, 0.1, 0.15, 0.2, 0.3, 0.5]
params['text_dropout'] = [0, 0.1, 0.15, 0.2, 0.3, 0.5]
params['factor_learning_rate'] = [0.0003, 0.0005, 0.001, 0.003]
params['learning_rate'] = [0.0003, 0.0005, 0.001, 0.003]
params['rank'] = [1, 4, 8, 16]
params['batch_size'] = [4, 8, 16, 32, 64, 128]
params['weight_decay'] = [0, 0.001, 0.002, 0.01]
# the window size in input layer
params['init_time_win'] = [1, 2, 4, 5, 10, 20]
# the output dimensions of the ptp block in first hidden layer
params['hid1_out_dim'] = [30, 40, 50, 60, 70]
total_settings = total(params)
seen_settings = set()
print('There are {} different hyper parameter settings in total.'.format(total_settings))
with open(output_path, 'w+') as out:
writer = csv.writer(out)
writer.writerow(['audio_hidden', 'video_hidden', 'text_hidden',
'audio_dropout', 'video_dropout', 'text_dropout',
'factor_learning_rate', 'learning_rate', 'rank', 'batch_size', 'weight_decay',
'best_valid_mae', 'test_mae', 'test_corr',
'test_multi_acc', 'test_binary_acc', 'test_f1',
'init_time_win', 'hid1_out_dim'])
for i in range(total_settings):
audio_hidden = random.choice(params['audio_hidden'])
video_hidden = random.choice(params['video_hidden'])
text_hidden = random.choice(params['text_hidden'])
audio_dropout = random.choice(params['audio_dropout'])
video_dropout = random.choice(params['video_dropout'])
text_dropout = random.choice(params['text_dropout'])
factor_lr = random.choice(params['factor_learning_rate'])
lr = random.choice(params['learning_rate'])
rank = random.choice(params['rank'])
batch_size = random.choice(params['batch_size'])
weight_decay = random.choice(params['weight_decay'])
init_time_win = random.choice(params['init_time_win'])
hid1_out_dim = random.choice(params['hid1_out_dim'])
# reject the setting if it has been tried
current_setting = (audio_hidden, video_hidden, text_hidden, audio_dropout, video_dropout, text_dropout,
factor_lr, lr, rank, batch_size, weight_decay, init_time_win, hid1_out_dim)
if current_setting in seen_settings:
continue
else:
seen_settings.add(current_setting)
# initialize the model
model = FNL2NO(poly_order, poly_norm, euclid_norm,
input_dims, (audio_hidden, video_hidden, text_hidden), (hid1_out_dim, output_dim),
init_time_len, init_time_win, time_wins, time_pads, init_modal_len, modal_wins, modal_pads,
(audio_dropout, video_dropout, text_dropout), rank)
win_factor_list = []
for n in range(model.inter_nodes[0]):
win_factor_list.append('l1_win_factor.' + str(n))
win_factor_list.append('l2_win_factor.0')
# split the parameters of the model into two parts
win_factors = []
other_params = []
for name, param in model.named_parameters():
if name in win_factor_list:
win_factors.append(param)
else:
other_params.append(param)
if torch.cuda.is_available():
model = model.cuda()
data_type = torch.cuda.FloatTensor
print('Model initialized')
# loss and optimizer
criterion = nn.L1Loss(size_average=False)
optimizer = optim.Adam([{'params': win_factors, 'lr': factor_lr}, {'params': other_params, 'lr': lr}],
weight_decay=weight_decay)
complete = True
min_valid_loss = float('Inf')
train_iter = DataLoader(train_set, batch_size=batch_size, num_workers=4, shuffle=True, drop_last=True)
valid_iter = DataLoader(valid_set, batch_size=len(valid_set), num_workers=4, shuffle=True)
test_iter = DataLoader(test_set, batch_size=len(test_set), num_workers=4, shuffle=True)
curr_patience = patience
for epoch in range(epochs):
model.train()
model.zero_grad()
avg_train_loss = 0
for batch in train_iter:
model.zero_grad()
x = batch[:-1]
x_a = Variable(x[0].float().type(data_type), requires_grad=False)
x_v = Variable(x[1].float().type(data_type), requires_grad=False)
x_t = Variable(x[2].float().type(data_type), requires_grad=False)
y = Variable(batch[-1].view(-1, output_dim).float().type(data_type), requires_grad=False)
output = model(x_a, x_v, x_t)
loss = criterion(output, y)
loss.backward()
avg_loss = loss.data.item()
avg_train_loss += avg_loss / len(train_set)
optimizer.step()
print('Epoch {}'.format(epoch))
print('Training loss: {}'.format(avg_train_loss))
# terminate the training process if run into nan
if np.isnan(avg_train_loss):
print('Training got into NaN values...\n\n')
complete = False
break
model.eval()
avg_valid_loss = 0
for batch in valid_iter:
x = batch[:-1]
x_a = Variable(x[0].float().type(data_type), requires_grad=False).squeeze()
x_v = Variable(x[1].float().type(data_type), requires_grad=False).squeeze()
x_t = Variable(x[2].float().type(data_type), requires_grad=False)
y = Variable(batch[-1].view(-1, output_dim).float().type(data_type), requires_grad=False)
output = model(x_a, x_v, x_t)
valid_loss = criterion(output, y)
avg_valid_loss = valid_loss.item()
if np.isnan(avg_valid_loss):
print('Validation got into NaN values...\n\n')
complete = False
break
avg_valid_loss = avg_valid_loss / len(valid_set)
print('Validation loss: {}'.format(avg_valid_loss))
if avg_valid_loss < min_valid_loss:
curr_patience = patience
min_valid_loss = avg_valid_loss
torch.save(model, model_path)
print('Found new best model, saving to disk...')
else:
curr_patience -= 1
if curr_patience <= 0:
break
print('\n\n')
if complete:
output_test = None
y = None
best_model = torch.load(model_path)
best_model.eval()
for batch in test_iter:
x = batch[:-1]
x_a = Variable(x[0].float().type(data_type), requires_grad=False).squeeze()
x_v = Variable(x[1].float().type(data_type), requires_grad=False).squeeze()
x_t = Variable(x[2].float().type(data_type), requires_grad=False)
y = Variable(batch[-1].view(-1, output_dim).float().type(data_type), requires_grad=False)
output_test = best_model(x_a, x_v, x_t)
output_test = output_test.cpu().data.numpy().reshape(-1, output_dim)
y = y.cpu().data.numpy().reshape(-1, output_dim)
output_test = output_test.reshape((len(output_test),))
y = y.reshape((len(y),))
mae = np.mean(np.absolute(output_test - y))
corr = np.corrcoef(output_test, y)[0][1]
multi_acc = round(sum(np.round(output_test) == np.round(y)) / float(len(y)), 5)
true_label = (y >= 0)
predicted_label = (output_test >= 0)
bi_acc = accuracy_score(true_label, predicted_label)
f1 = f1_score(true_label, predicted_label, average='weighted')
display(mae, corr, multi_acc, bi_acc, f1)
with open(output_path, 'a+') as out:
writer = csv.writer(out)
writer.writerow([audio_hidden, video_hidden, text_hidden,
audio_dropout, video_dropout, text_dropout,
factor_lr, lr, rank, batch_size, weight_decay,
min_valid_loss, mae, corr, multi_acc, bi_acc, f1])
def main(opts):
data_type_float = torch.FloatTensor
data_type_long = torch.LongTensor
# load option configs
emotion = opts['emotion']
poly_order = opts['poly_order']
poly_norm = opts['poly_norm']
euclid_norm = opts['euclid_norm']
init_modal_len = opts['init_modal_len']
modal_wins = opts['modal_wins']
modal_pads = opts['modal_pads']
output_dim = opts['output_dim']
epochs = opts['epochs']
patience = opts['patience']
signature = opts['signature']
run_id = opts['run_id']
data_path = opts['data_path']
model_path = opts['model_path']
output_path = opts['output_path']
# set paths for storing models and outputs
model_path = os.path.join(
model_path, "model_{}_{}_{}.pt".format(signature, emotion, run_id))
output_path = os.path.join(
output_path, "result_{}_{}_{}.csv".format(signature, emotion, run_id))
if not os.path.isfile(output_path):
with open(output_path, 'w+') as out:
writer = csv.writer(out)
writer.writerow([
'audio_hidden', 'video_hidden', 'text_hidden', 'audio_dropout',
'video_dropout', 'text_dropout', 'factor_learning_rate',
'learning_rate', 'rank', 'batch_size', 'weight_decay',
'best_validation_loss', 'test_loss', 'test_f1', 'test_acc',
'hid1_outdim'
])
# load data sets
train_set, valid_set, test_set, input_dims = load_iemocap(
data_path, emotion)
params = dict()
params['audio_hidden'] = [8, 16, 32]
params['video_hidden'] = [4, 8, 16]
params['text_hidden'] = [64, 128, 256]
params['audio_dropout'] = [0, 0.1, 0.15, 0.2, 0.3, 0.5]
params['video_dropout'] = [0, 0.1, 0.15, 0.2, 0.3, 0.5]
params['text_dropout'] = [0, 0.1, 0.15, 0.2, 0.3, 0.5]
params['factor_learning_rate'] = [0.0003, 0.0005, 0.001, 0.003]
params['learning_rate'] = [0.0003, 0.0005, 0.001, 0.003]
params['rank'] = [1, 4, 8, 16]
params['batch_size'] = [8, 16, 32, 64, 128]
params['weight_decay'] = [0, 0.001, 0.002, 0.01]
params['hid1_out_dim'] = [30, 40, 50, 60, 70]
total_settings = total(params)
seen_settings = set()
print('There are {} different hyper parameter settings in total.'.format(
total_settings))
for i in range(total_settings):
audio_hidden = random.choice(params['audio_hidden'])
video_hidden = random.choice(params['video_hidden'])
text_hidden = random.choice(params['text_hidden'])
text_out = text_hidden // 2
audio_dropout = random.choice(params['audio_dropout'])
video_dropout = random.choice(params['video_dropout'])
text_dropout = random.choice(params['text_dropout'])
factor_lr = random.choice(params['factor_learning_rate'])
lr = random.choice(params['learning_rate'])
rank = random.choice(params['rank'])
batch_size = random.choice(params['batch_size'])
weight_decay = random.choice(params['weight_decay'])
hid1_out_dim = random.choice(params['hid1_out_dim'])
# reject the setting if it has been tried
current_setting = (audio_hidden, video_hidden, text_hidden,
audio_dropout, video_dropout, text_dropout,
factor_lr, lr, rank, batch_size, weight_decay,
hid1_out_dim)
if current_setting in seen_settings:
continue
else:
seen_settings.add(current_setting)
model = FNL2(poly_order, poly_norm, euclid_norm, input_dims,
(audio_hidden, video_hidden, text_hidden), text_out,
(hid1_out_dim, output_dim), init_modal_len, modal_wins,
modal_pads, (audio_dropout, video_dropout, text_dropout),
rank)
win_factor_list = []
for n in range(model.inter_nodes[0]):
win_factor_list.append('l1_win_factor.' + str(n))
win_factor_list.append('l2_win_factor.0')
# split the parameters of the model into two parts
win_factors = []
other_params = []
for name, param in model.named_parameters():
if name in win_factor_list:
win_factors.append(param)
else:
other_params.append(param)
if torch.cuda.is_available():
print('Model in cuda')
model = model.cuda()
data_type_float = torch.cuda.FloatTensor
data_type_long = torch.cuda.LongTensor
print('Model initialized')
# loss and optimizer
criterion = nn.CrossEntropyLoss(size_average=False)
optimizer = optim.Adam([{
'params': win_factors,
'lr': factor_lr
}, {
'params': other_params,
'lr': lr
}],
weight_decay=weight_decay)
# setup training
complete = True
min_valid_loss = float('Inf')
train_iter = DataLoader(train_set,
batch_size=batch_size,
num_workers=4,
shuffle=True,
drop_last=True)
valid_iter = DataLoader(valid_set,
batch_size=len(valid_set),
num_workers=4,
shuffle=True)
test_iter = DataLoader(test_set,
batch_size=len(test_set),
num_workers=4,
shuffle=True)
curr_patience = patience
for epoch in range(epochs):
model.train()
model.zero_grad()
avg_train_loss = 0
for batch in train_iter:
model.zero_grad()
x = batch[:-1]
x_a = Variable(x[0].float().type(data_type_float),
requires_grad=False)
x_v = Variable(x[1].float().type(data_type_float),
requires_grad=False)
x_t = Variable(x[2].float().type(data_type_float),
requires_grad=False)
y = Variable(batch[-1].view(
-1, output_dim).float().type(data_type_long),
requires_grad=False)
try:
output = model(x_a, x_v, x_t)
except ValueError as e:
print(x_a.data.shape)
print(x_v.data.shape)
print(x_t.data.shape)
raise e
loss = criterion(output, torch.max(y, 1)[1])
loss.backward()
avg_loss = loss.item()
avg_train_loss += avg_loss / len(train_set)
optimizer.step()
print('Epoch {}'.format(epoch))
print('Training loss: {}'.format(avg_train_loss))
# terminate the training process if run into nan
if np.isnan(avg_train_loss):
print('Training got into NaN values...\n\n')
complete = False
break
model.eval()
avg_valid_loss = 0
for batch in valid_iter:
x = batch[:-1]
x_a = Variable(x[0].float().type(data_type_float),
requires_grad=False)
x_v = Variable(x[1].float().type(data_type_float),
requires_grad=False)
x_t = Variable(x[2].float().type(data_type_float),
requires_grad=False)
y = Variable(batch[-1].view(
-1, output_dim).float().type(data_type_long),
requires_grad=False)
output = model(x_a, x_v, x_t)
valid_loss = criterion(output, torch.max(y, 1)[1])
avg_valid_loss = valid_loss.item()
if np.isnan(avg_valid_loss):
print('Validation got into NaN values...\n\n')
complete = False
break
avg_valid_loss = avg_valid_loss / len(valid_set)
print('Validation loss: {}'.format(avg_valid_loss))
if avg_valid_loss < min_valid_loss:
curr_patience = patience
min_valid_loss = avg_valid_loss
torch.save(model, model_path)
print('Found new best model, saving to disk...')
else:
curr_patience -= 1
if curr_patience <= 0:
break
print('\n\n')
if complete:
output_test = None
y = None
test_loss = None
best_model = torch.load(model_path)
best_model.eval()
for batch in test_iter:
x = batch[:-1]
x_a = Variable(x[0].float().type(data_type_float),
requires_grad=False)
x_v = Variable(x[1].float().type(data_type_float),
requires_grad=False)
x_t = Variable(x[2].float().type(data_type_float),
requires_grad=False)
y = Variable(batch[-1].view(
-1, output_dim).float().type(data_type_long),
requires_grad=False)
output_test = best_model(x_a, x_v, x_t)
loss_test = criterion(output_test, torch.max(y, 1)[1])
test_loss = loss_test.item()
output_test = output_test.cpu().data.numpy().reshape(
-1, output_dim)
y = y.cpu().data.numpy().reshape(-1, output_dim)
test_loss = test_loss / len(test_set)
all_true_label = np.argmax(y, axis=1)
all_predicted_label = np.argmax(output_test, axis=1)
f1 = f1_score(all_true_label,
all_predicted_label,
average='weighted')
acc_score = accuracy_score(all_true_label, all_predicted_label)
display(f1, acc_score)
with open(output_path, 'a+') as out:
writer = csv.writer(out)
writer.writerow([
audio_hidden, video_hidden, text_hidden, audio_dropout,
video_dropout, text_dropout, factor_lr, lr, rank,
batch_size, weight_decay, min_valid_loss, test_loss, f1,
acc_score, hid1_out_dim
])
def main(options):
DTYPE = torch.FloatTensor
# parse the input args
run_id = options['run_id']
epochs = options['epochs']
data_path = options['data_path']
model_path = options['model_path']
output_path = options['output_path']
signiture = options['signiture']
patience = options['patience']
output_dim = options['output_dim']
print("Training initializing... Setup ID is: {}".format(run_id))
# prepare the paths for storing models and outputs
model_path = os.path.join(model_path,
"model_{}_{}.pt".format(signiture, run_id))
output_path = os.path.join(output_path,
"results_{}_{}.csv".format(signiture, run_id))
print("Temp location for models: {}".format(model_path))
print("Grid search results are in: {}".format(output_path))
train_set, valid_set, test_set, input_dims = load_mosi_context(data_path)
params = dict()
params['audio_hidden'] = [150, 50, 100]
params['audio_dropout'] = [0, 0.1, 0.2, 0.3, 0.5, 0.6, 0.7, 0.8, 0.9]
# params['video_dropout'] = [0, 0.1, 0.2, 0.3, 0.5,0.6,0.7,0.8]
# params['text_dropout'] = [0, 0.1, 0.2, 0.3, 0.5,0.6,0.7,0.8]
params['learning_rate'] = [0.0001, 0.001, 0.01]
params['batch_size'] = [8, 16, 32]
params['weight_decay'] = [0, 0.001, 0.01, 0.0001]
params['alpha'] = [0.01, 0.001, 0.05]
# params['alpha']=[0.05]
total_settings = total(params)
print("There are {} different hyper-parameter settings in total.".format(
total_settings))
audio_in = input_dims[0]
video_in = input_dims[1]
text_in = input_dims[2]
seen_settings = set()
with open(output_path, 'w+') as out:
writer = csv.writer(out)
writer.writerow([
"audio_hidden", "video_hidden", 'text_hidden', 'audio_dropout',
'video_dropout', 'text_dropout', 'learning_rate', 'batch_size',
'weight_decay', 'Test binary accuracy', 'Test f1_score', 'alpha'
])
best_acc = 0
best_f1 = 0
kk = 0
# for i in range(total_settings):
for i in range(1000000):
if kk >= total_settings:
break
ahid = random.choice(params['audio_hidden'])
vhid = ahid
thid = ahid
thid_2 = thid
adr = random.choice(params['audio_dropout'])
vdr = adr
tdr = adr
lr = random.choice(params['learning_rate'])
batch_sz = random.choice(params['batch_size'])
decay = random.choice(params['weight_decay'])
alpha = random.choice(params['alpha'])
'''
ahid = 50 #######50
vhid = ahid
thid = ahid
thid_2 = thid
adr = 0.5 ###########0.6
vdr = adr
tdr = adr
lr = 0.001 ########0.001
batch_sz = 16 ############32,16
decay = 0.005 #########
alpha = 0.001 ###0.05
'''
# reject the setting if it has been tried
current_setting = (ahid, vhid, thid, adr, vdr, tdr, lr, batch_sz,
decay, alpha)
if i == 0:
best_setting = current_setting
if (current_setting
in seen_settings) and (current_setting != best_setting):
continue
else:
seen_settings.add(current_setting)
kk += 1
latent_dim = 10
encoder_a = Encoder_5(audio_in, ahid, adr).cuda() #####zhe ge ke yi de
encoder_v = Encoder_5(video_in, vhid, vdr).cuda()
encoder_l = Encoder_5(text_in, thid, dropout=tdr).cuda()
decoder_a = Decoder2(ahid, audio_in).cuda()
decoder_v = Decoder2(vhid, video_in).cuda()
decoder_l = Decoder2(thid, text_in).cuda()
discriminator = Discriminator(ahid).cuda()
# classifier = Classifier(hidden=thid,input_dim=output_dim,layer_size=64).cuda()
classifier = graph11_new(in_size=thid, output_dim=output_dim).cuda()
classifier_3 = classifier3(in_size=thid, output_dim=output_dim).cuda()
b1 = 0.5
b2 = 0.999
# criterion = nn.L1Loss(size_average=False).cuda()
criterion = nn.L1Loss(size_average=False).cuda()
adversarial_loss = torch.nn.BCELoss().cuda()
classifier_loss = torch.nn.SoftMarginLoss().cuda()
pixelwise_loss = torch.nn.L1Loss(size_average=False).cuda()
# pixelwise_loss = torch.nn.MSELoss().cuda()
# pixelwise_loss = torch.nn.KLDivLoss().cuda() #bad
optimizer_G = torch.optim.Adam(
itertools.chain(encoder_a.parameters(), encoder_v.parameters(), encoder_l.parameters(), \
decoder_a.parameters(), decoder_l.parameters(), decoder_v.parameters()), weight_decay=decay,
lr=lr, betas=(b1, b2))
optimizer_D = torch.optim.Adam(discriminator.parameters(),
lr=lr,
betas=(b1, b2),
weight_decay=decay)
optimizer_C = torch.optim.Adam(classifier.parameters(),
lr=lr,
betas=(b1, b2),
weight_decay=decay)
optimizer_E = torch.optim.Adam(itertools.chain(
encoder_a.parameters(), encoder_v.parameters(),
encoder_l.parameters(), classifier_3.parameters()),
lr=lr,
betas=(b1, b2),
weight_decay=decay)
'''
optimizer_G = torch.optim.Adam(itertools.chain(encoder_a.parameters(), encoder_v.parameters(),encoder_l.parameters(),\
decoder_a.parameters(),decoder_l.parameters(),decoder_v.parameters()), weight_decay=decay, lr=lr)
optimizer_D = torch.optim.Adam(discriminator.parameters(), lr=lr, weight_decay=decay)
optimizer_C = torch.optim.Adam(itertools.chain(encoder_a.parameters(), encoder_v.parameters(),encoder_l.parameters(),classifier.parameters()), lr=lr, weight_decay=decay)
optimizer_E = torch.optim.Adam(itertools.chain(encoder_a.parameters(), encoder_v.parameters(),encoder_l.parameters(),classifier_3.parameters()), lr=lr, weight_decay=decay)
'''
if cuda:
DTYPE = torch.cuda.FloatTensor
# setup training
complete = True
min_valid_loss = float('Inf')
train_iterator = DataLoader(train_set,
batch_size=batch_sz,
num_workers=4,
shuffle=True)
valid_iterator = DataLoader(valid_set,
batch_size=len(valid_set),
num_workers=4,
shuffle=True)
test_iterator = DataLoader(test_set,
batch_size=len(test_set),
num_workers=4,
shuffle=True)
curr_patience = patience
for e in range(epochs):
avg_train_loss = 0.0
avg_closs = 0.0
avg_rloss = 0.0
avg_v = 0.0
avg_a = 0.0
avg_l = 0.0
for batch in train_iterator:
x = batch[:-1]
x_a = Variable(x[0].float().type(DTYPE),
requires_grad=False).squeeze()
x_v = Variable(x[1].float().type(DTYPE),
requires_grad=False).squeeze()
x_t = Variable(x[2].float().type(DTYPE), requires_grad=False)
y = Variable(batch[-1].view(-1,
output_dim).float().type(DTYPE),
requires_grad=False)
# encoder-decoder
optimizer_G.zero_grad()
a_en = encoder_a(x_a)
v_en = encoder_v(x_v)
l_en = encoder_l(x_t)
# z = Variable(Tensor(np.random.normal(0, 1, (y.size(0), latent_dim))))
# ade = torch.cat([a_en,z],1)
# vde = torch.cat([v_en,z],1)
# lde = torch.cat([l_en,z],1)
a_de = decoder_a(a_en)
v_de = decoder_v(v_en)
l_de = decoder_l(l_en)
rl1 = pixelwise_loss(
a_de, x_a) + pixelwise_loss(v_de, x_v) + pixelwise_loss(
l_de, x_t) #####reconstruction loss
avg_rloss += rl1.item() / len(train_set)
valid = Variable(torch.cuda.FloatTensor(y.shape[0],
1).fill_(1.0),
requires_grad=False)
fake = Variable(torch.cuda.FloatTensor(y.shape[0],
1).fill_(0.0),
requires_grad=False)
g_loss = alpha * (adversarial_loss(
discriminator(l_en), valid) + adversarial_loss(
discriminator(v_en), valid)) + (1 - alpha) * (rl1)
g_loss.backward(retain_graph=True)
optimizer_G.step()
# classifier
optimizer_E.zero_grad()
# v_a = torch.cat([a_en,v_en],1)
# fusion = torch.cat([v_a,l_en],1)
# v_c = classifier_3(fusion)
a = classifier_3(a_en)
v = classifier_3(v_en)
l = classifier_3(l_en)
c_loss = criterion(a, y) + criterion(l, y) + criterion(
v, y) ######classification loss
c_loss.backward(retain_graph=True)
optimizer_E.step()
avg_closs += c_loss.item() / len(train_set)
# discriminator
optimizer_D.zero_grad()
real_loss = adversarial_loss(discriminator(a_en), valid)
avg_v += torch.sum(discriminator(v_en).squeeze().data) / (
len(train_set) * batch_sz)
fake_loss = adversarial_loss(discriminator(v_en),
fake) + adversarial_loss(
discriminator(l_en), fake)
avg_a += torch.sum(discriminator(a_en).squeeze().data) / (
len(train_set) * batch_sz)
avg_l += torch.sum(discriminator(l_en).squeeze().data) / (
len(train_set) * batch_sz)
d_loss = 0.5 * (real_loss + fake_loss
) ##### discrimination loss for discriminator
# print(d_loss,'d_loss')
d_loss.backward(retain_graph=True)
optimizer_D.step()
# fusion
v_a = torch.cat([a_en.unsqueeze(1), v_en.unsqueeze(1)], 1)
fusion = torch.cat([v_a, l_en.unsqueeze(1)], 1)
optimizer_C.zero_grad()
output, _ = classifier(fusion)
loss = criterion(output, y)
loss.backward()
optimizer_C.step()
avg_loss = loss.item()
avg_train_loss += avg_loss / len(train_set)
print("Epoch: {} loss: {}".format(e, avg_train_loss))
# Terminate the training process if run into NaN
if np.isnan(avg_train_loss):
print("Training got into NaN values...\n\n")
complete = False
break
# model.eval()
avg_valid_loss = 0
# k = 0
for batch in valid_iterator:
x = batch[:-1]
x_a = Variable(x[0].float().type(DTYPE),
requires_grad=False).squeeze()
x_v = Variable(x[1].float().type(DTYPE),
requires_grad=False).squeeze()
x_t = Variable(x[2].float().type(DTYPE), requires_grad=False)
y = Variable(batch[-1].view(-1,
output_dim).float().type(DTYPE),
requires_grad=False)
a_en = encoder_a(x_a)
v_en = encoder_v(x_v)
l_en = encoder_l(x_t)
'''
v_a = torch.cat([a_en.unsqueeze(2),v_en.unsqueeze(2)],2)
fusion = torch.cat([v_a,l_en.unsqueeze(2)],2)
fusion = fusion.unsqueeze(1)
'''
v_a = torch.cat([a_en.unsqueeze(1), v_en.unsqueeze(1)], 1)
fusion = torch.cat([v_a, l_en.unsqueeze(1)], 1)
output, _ = classifier(fusion)
valid_loss = criterion(output, y)
avg_valid_loss += valid_loss.item()
# k+=1
y = y.cpu().data.numpy().reshape(-1, output_dim)
# print(k,'k')
if np.isnan(avg_valid_loss):
print("Training got into NaN values...\n\n")
complete = False
break
avg_valid_loss = avg_valid_loss / len(valid_set)
print("Validation loss is: {}".format(avg_valid_loss))
if (avg_valid_loss < min_valid_loss):
curr_patience = patience
min_valid_loss = avg_valid_loss
torch.save(encoder_a,
'./models/encoder_a_{}.pkl'.format(run_id))
torch.save(encoder_v,
'./models/encoder_v_{}.pkl'.format(run_id))
torch.save(encoder_l,
'./models/encoder_l_{}.pkl'.format(run_id))
torch.save(classifier,
'./models/classifier_{}.pkl'.format(run_id))
print("Found new best model, saving to disk...")
else:
curr_patience -= 1
if curr_patience <= 0:
break
print("\n\n")
avg_test_loss = 0
for batch in test_iterator:
x = batch[:-1]
x_a = Variable(x[0].float().type(DTYPE),
requires_grad=False).squeeze()
x_v = Variable(x[1].float().type(DTYPE),
requires_grad=False).squeeze()
x_t = Variable(x[2].float().type(DTYPE), requires_grad=False)
y = Variable(batch[-1].view(-1,
output_dim).float().type(DTYPE),
requires_grad=False)
# output_test = model(x_a, x_v, x_t)
a_en = encoder_a(x_a)
v_en = encoder_v(x_v)
l_en = encoder_l(x_t)
'''
v_a = torch.cat([a_en.unsqueeze(2),v_en.unsqueeze(2)],2)
fusion = torch.cat([v_a,l_en.unsqueeze(2)],2)
fusion = fusion.unsqueeze(1)
'''
v_a = torch.cat([a_en.unsqueeze(1), v_en.unsqueeze(1)], 1)
fusion = torch.cat([v_a, l_en.unsqueeze(1)], 1)
output_test, _ = classifier(fusion)
loss_test = criterion(output_test, y)
avg_test_loss += loss_test.item() / len(test_set)
output_test = output_test.cpu().data.numpy().reshape(
-1, output_dim)
y = y.cpu().data.numpy().reshape(-1, output_dim)
# these are the needed metrics
output_test = output_test.reshape(
(len(output_test) * output_dim, ))
y = y.reshape((len(y) * output_dim, ))
mae = np.mean(np.absolute(output_test - y))
k = 0
true_acc = 0
true_f1 = 0
if complete:
# best_model = torch.load(model_path)
# best_model.eval()
encoder_a = torch.load('./models/encoder_a_{}.pkl'.format(run_id))
encoder_v = torch.load('./models/encoder_v_{}.pkl'.format(run_id))
encoder_l = torch.load('./models/encoder_l_{}.pkl'.format(run_id))
classifier = torch.load(
'./models/classifier_{}.pkl'.format(run_id))
k = 0
for batch in test_iterator:
x = batch[:-1]
x_a = Variable(x[0].float().type(DTYPE),
requires_grad=False).squeeze()
x_v = Variable(x[1].float().type(DTYPE),
requires_grad=False).squeeze()
x_t = Variable(x[2].float().type(DTYPE), requires_grad=False)
y = Variable(batch[-1].view(-1,
output_dim).float().type(DTYPE),
requires_grad=False)
a_en = encoder_a(x_a)
v_en = encoder_v(x_v)
l_en = encoder_l(x_t)
'''
v_a = torch.cat([a_en.unsqueeze(2),v_en.unsqueeze(2)],2)
fusion = torch.cat([v_a,l_en.unsqueeze(2)],2)
fusion = fusion.unsqueeze(1)
'''
v_a = torch.cat([a_en.unsqueeze(1), v_en.unsqueeze(1)], 1)
fusion = torch.cat([v_a, l_en.unsqueeze(1)], 1)
output_test, weights = classifier(fusion)
loss_test = criterion(output_test, y)
'''
true_label = (y >= 0)
predicted_label = (output_test >= 0)
if k == 0 :
total_y = y
total_test = output_test
else:
total_y = torch.cat([total_y,y],0)
total_test = torch.cat([total_test,output_test],0)
k += 1
'''
output_test = output_test.cpu().data.numpy().reshape(
-1, output_dim)
y = y.cpu().data.numpy().reshape(-1, output_dim)
print(output_test.shape, 'output_test_shape')
print(y.shape, 'y_shape')
output_test2 = output_test.reshape(
(len(output_test) * output_dim, ))
y2 = y.reshape((len(y) * output_dim, ))
true_label = np.argmax(y, 1)
predicted_label = np.argmax(output_test, 1)
bi_acc = accuracy_score(true_label, predicted_label)
f1 = f1_score(true_label, predicted_label, average='weighted')
display(bi_acc, f1)
if bi_acc > best_acc:
best_acc = bi_acc
best_setting = current_setting
'''
np.save('fusion.npy',fusion.cpu().data.numpy())
np.save('label_one_hot.npy',y)
np.save('weights.npy',weights.cpu().data.numpy())
np.save('predicted.npy',output_test)
'''
if f1 > best_f1:
best_f1 = f1
print('best_acc: ', best_acc)
print('best_f1: ', best_f1)
print('best_setting: ', best_setting)
'''
import utils
text = input('Number? ')
number = int(text)
squares_n = utils.squares(number)
total_n = utils.total(squares_n)
print(total_n)
def main(options):
DTYPE = torch.FloatTensor
# parse the input args
run_id = options['run_id']
epochs = options['epochs']
data_path = options['data_path']
model_path = options['model_path']
output_path = options['output_path']
signiture = options['signiture']
patience = options['patience']
output_dim = options['output_dim']
print("Training initializing... Setup ID is: {}".format(run_id))
# prepare the paths for storing models and outputs
model_path = os.path.join(model_path,
"model_{}_{}.pt".format(signiture, run_id))
output_path = os.path.join(output_path,
"results_{}_{}.csv".format(signiture, run_id))
print("Temp location for models: {}".format(model_path))
print("Grid search results are in: {}".format(output_path))
os.makedirs(os.path.dirname(output_path), exist_ok=True)
os.makedirs(os.path.dirname(model_path), exist_ok=True)
train_set, valid_set, test_set, input_dims = load_mosi(data_path)
params = dict()
params['audio_hidden'] = [4, 8, 16]
params['video_hidden'] = [4, 8, 16]
params['text_hidden'] = [64, 128, 256]
params['audio_dropout'] = [0, 0.1, 0.15, 0.2, 0.3, 0.5]
params['video_dropout'] = [0, 0.1, 0.15, 0.2, 0.3, 0.5]
params['text_dropout'] = [0, 0.1, 0.15, 0.2, 0.3, 0.5]
params['factor_learning_rate'] = [0.0003, 0.0005, 0.001, 0.003]
params['learning_rate'] = [0.0003, 0.0005, 0.001, 0.003]
params['rank'] = [1, 4, 8, 16]
params['batch_size'] = [4, 8, 16, 32, 64, 128]
params['weight_decay'] = [0, 0.001, 0.002, 0.01]
total_settings = total(params) ## wyw 8957952
print("There are {} different hyper-parameter settings in total.".format(
total_settings))
seen_settings = set()
with open(output_path, 'w+') as out:
writer = csv.writer(out)
writer.writerow([
"audio_hidden", "video_hidden", 'text_hidden', 'audio_dropout',
'video_dropout', 'text_dropout', 'factor_learning_rate',
'learning_rate', 'rank', 'batch_size', 'weight_decay',
'Best Validation MAE', 'Test MAE', 'Test Corr',
'Test multiclass accuracy', 'Test binary accuracy', 'Test f1_score'
])
## wyw 用Excel记录grid search的过程，这种方式很666
for i in range(total_settings):
ahid = random.choice(params['audio_hidden'])
vhid = random.choice(params['video_hidden'])
thid = random.choice(params['text_hidden'])
thid_2 = thid // 2
adr = random.choice(params['audio_dropout'])
vdr = random.choice(params['video_dropout'])
tdr = random.choice(params['text_dropout'])
factor_lr = random.choice(params['factor_learning_rate'])
lr = random.choice(params['learning_rate'])
r = random.choice(params['rank'])
batch_sz = random.choice(params['batch_size'])
decay = random.choice(params['weight_decay'])
# reject the setting if it has been tried
current_setting = (ahid, vhid, thid, adr, vdr, tdr, factor_lr, lr, r,
batch_sz, decay)
if current_setting in seen_settings:
continue
else:
seen_settings.add(current_setting)
## wyw 这种grid search的方式真牛逼！！不服不行，学习了学习了
model = LMF(input_dims, (ahid, vhid, thid), thid_2,
(adr, vdr, tdr, 0.5), output_dim, r)
if options['cuda']:
model = model.cuda()
DTYPE = torch.cuda.FloatTensor
print("Model initialized")
criterion = nn.L1Loss(size_average=False)
factors = list(model.parameters())[:3]
other = list(
model.parameters())[3:] ## wyw 这里将参数分为factor 和other的原因？未看懂
optimizer = optim.Adam([{
"params": factors,
"lr": factor_lr
}, {
"params": other,
"lr": lr
}],
weight_decay=decay)
# setup training
complete = True
min_valid_loss = float('Inf')
train_iterator = DataLoader(train_set,
batch_size=batch_sz,
num_workers=4,
shuffle=True)
valid_iterator = DataLoader(valid_set,
batch_size=len(valid_set),
num_workers=4,
shuffle=True)
test_iterator = DataLoader(test_set,
batch_size=len(test_set),
num_workers=4,
shuffle=True)
curr_patience = patience ## 容忍
##wyw 这里以下才是训练过程，以上为grid search 及模型框架搭建
for e in range(epochs):
model.train()
model.zero_grad()
avg_train_loss = 0.0
for batch in train_iterator:
model.zero_grad()
x = batch[:-1]
x_a = Variable(x[0].float().type(DTYPE),
requires_grad=False).squeeze()
x_v = Variable(x[1].float().type(DTYPE),
requires_grad=False).squeeze()
x_t = Variable(x[2].float().type(DTYPE), requires_grad=False)
y = Variable(batch[-1].view(-1,
output_dim).float().type(DTYPE),
requires_grad=False)
output = model(x_a, x_v, x_t)
loss = criterion(output, y)
loss.backward()
print("#########each batch loss###############")
print(loss.data)
print("########################################")
avg_loss = loss.data.item() ## wyw origin:loss.data[0]
avg_train_loss += avg_loss / len(train_set)
optimizer.step()
print("Epoch {} complete! Average Training loss: {}".format(
e, avg_train_loss))
# Terminate the training process if run into NaN
if np.isnan(avg_train_loss):
print("Training got into NaN values...\n\n")
complete = False
break
model.eval()
for batch in valid_iterator:
x = batch[:-1]
x_a = Variable(x[0].float().type(DTYPE),
requires_grad=False).squeeze()
x_v = Variable(x[1].float().type(DTYPE),
requires_grad=False).squeeze()
x_t = Variable(x[2].float().type(DTYPE), requires_grad=False)
y = Variable(batch[-1].view(-1,
output_dim).float().type(DTYPE),
requires_grad=False)
output = model(x_a, x_v, x_t)
valid_loss = criterion(output, y)
avg_valid_loss = valid_loss.data[0]
y = y.cpu().data.numpy().reshape(-1, output_dim)
if np.isnan(avg_valid_loss):
print("Training got into NaN values...\n\n")
complete = False
break
avg_valid_loss = avg_valid_loss / len(valid_set)
print("Validation loss is: {}".format(avg_valid_loss))
if (avg_valid_loss < min_valid_loss):
curr_patience = patience
min_valid_loss = avg_valid_loss
torch.save(model, model_path)
print("Found new best model, saving to disk...")
else:
curr_patience -= 1
if curr_patience <= 0:
break
print("\n\n")
model.eval()
for batch in test_iterator:
x = batch[:-1]
x_a = Variable(x[0].float().type(DTYPE),
requires_grad=False).squeeze()
x_v = Variable(x[1].float().type(DTYPE),
requires_grad=False).squeeze()
x_t = Variable(x[2].float().type(DTYPE), requires_grad=False)
y = Variable(batch[-1].view(-1,
output_dim).float().type(DTYPE),
requires_grad=False)
output_test = model(x_a, x_v, x_t)
loss_test = criterion(output_test, y)
avg_test_loss = loss_test.data[0] / len(test_set)
output_test = output_test.cpu().data.numpy().reshape(
-1, output_dim)
y = y.cpu().data.numpy().reshape(-1, output_dim)
# these are the needed metrics
output_test = output_test.reshape((len(output_test), ))
y = y.reshape((len(y), ))
mae = np.mean(np.absolute(output_test - y))
if complete:
best_model = torch.load(model_path)
best_model.eval()
for batch in test_iterator:
x = batch[:-1]
x_a = Variable(x[0].float().type(DTYPE),
requires_grad=False).squeeze()
x_v = Variable(x[1].float().type(DTYPE),
requires_grad=False).squeeze()
x_t = Variable(x[2].float().type(DTYPE), requires_grad=False)
y = Variable(batch[-1].view(-1,
output_dim).float().type(DTYPE),
requires_grad=False)
output_test = best_model(x_a, x_v, x_t)
loss_test = criterion(output_test, y)
output_test = output_test.cpu().data.numpy().reshape(
-1, output_dim)
y = y.cpu().data.numpy().reshape(-1, output_dim)
# these are the needed metrics
output_test = output_test.reshape((len(output_test), ))
y = y.reshape((len(y), ))
mae = np.mean(np.absolute(output_test - y))
corr = round(np.corrcoef(output_test, y)[0][1],
5) ##wyw 两个输出向量的output居然也可以作为一种评测方式！！
multi_acc = round(
sum(np.round(output_test) == np.round(y)) / float(len(y)), 5)
true_label = (y >= 0)
predicted_label = (output_test >= 0)
bi_acc = accuracy_score(true_label, predicted_label)
f1 = f1_score(true_label, predicted_label, average='weighted')
display(mae, corr, multi_acc, bi_acc, f1)
with open(output_path, 'a+') as out:
writer = csv.writer(out)
writer.writerow([
ahid, vhid, thid, adr, vdr, tdr, factor_lr, lr, r,
batch_sz, decay,
min_valid_loss.cpu().data.numpy(), mae, corr, multi_acc,
bi_acc, f1
])
def test_total_command():
num_list = [1.0, 2.0, 3.0, 4.0, 10.0, 14.0]
result = utils.total(num_list)
assert result == 34.0
""" modules practice """
import utils
text = input('Number? ')
number = int(text)
squares_n = utils.squares(number)
total_n = utils.total(squares_n)
print(total_n)