def train_model(data_set_identifier, model, train_loader, validation_loader,
learning_rate, minibatch_size=64, eval_interval=50, hide_ui=False,
use_gpu=False, minimum_updates=1000,
optimizer_type='adam', restart=False):
set_experiment_id(data_set_identifier, learning_rate, minibatch_size)
validation_dataset_size = validation_loader.dataset.__len__()
if use_gpu:
model = model.cuda()
if optimizer_type == 'adam':
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
elif optimizer_type == 'sgd':
optimizer = optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9)
elif optimizer_type == 'rmsprop':
optimizer = optim.RMSprop(model.parameters(), lr=learning_rate)
else:
optimizer = optim.AdamW(model.parameters(), lr=learning_rate)
if restart:
scheduler = optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=32)
sample_num = list()
train_loss_values = list()
train_drmsd_values = list()
validation_loss_values = list()
validation_angles_loss_values = list()
best_model_loss = 1e20
best_model_minibatch_time = None
best_model_path = None
best_json_data = None
stopping_condition_met = False
minibatches_proccesed = 0
while not stopping_condition_met:
# for i in range(2):
optimizer.zero_grad()
model.zero_grad()
loss_tracker = np.zeros(0)
drmsd_tracker = np.zeros(0)
for _minibatch_id, training_minibatch in enumerate(train_loader, 0):
minibatches_proccesed += 1
start_compute_loss = time.time()
loss, drmsd_avg = model.compute_loss(training_minibatch)
write_out("Train loss:", float(loss))
start_compute_grad = time.time()
loss.backward()
loss_tracker = np.append(loss_tracker, float(loss))
drmsd_tracker = np.append(drmsd_tracker, float(drmsd_avg))
end = time.time()
write_out("Loss time:", start_compute_grad - start_compute_loss, "Grad time:",
end - start_compute_grad)
optimizer.step()
if restart:
scheduler.step()
optimizer.zero_grad()
model.zero_grad()
# for every eval_interval samples, plot performance on the validation set
if minibatches_proccesed % eval_interval == 0:
write_out("Testing model on validation set...")
train_loss = float(loss_tracker.mean())
train_drmsd = float(drmsd_tracker.mean())
loss_tracker = np.zeros(0)
drmsd_tracker = np.zeros(0)
validation_loss, json_data, _, validation_angles_loss = model.evaluate_model(validation_loader)
if validation_loss < best_model_loss:
best_model_loss = validation_loss
best_model_minibatch_time = minibatches_proccesed
best_model_path = write_model_to_disk(model)
best_json_data = json_data
write_out("Validation loss:", validation_loss, "Train loss:", train_loss, "Train drmsd:", train_drmsd)
write_out("Best model so far (validation loss): ", best_model_loss, "at time",
best_model_minibatch_time)
write_out("Best model stored at " + best_model_path)
write_out("Minibatches processed:", minibatches_proccesed)
sample_num.append(minibatches_proccesed)
train_loss_values.append(train_loss)
train_drmsd_values.append(train_drmsd)
validation_loss_values.append(validation_loss)
validation_angles_loss_values.append(validation_angles_loss)
json_data["validation_dataset_size"] = validation_dataset_size
json_data["sample_num"] = sample_num
json_data["train_loss_values"] = train_loss_values
json_data["train_drmsd_values"] = train_drmsd_values
json_data["validation_loss_values"] = validation_loss_values
json_data['validation_angles_loss_values'] = validation_angles_loss_values
write_out(json_data)
if not hide_ui:
res = requests.post('http://localhost:5000/graph', json=json_data)
if res.ok:
print(res.json())
if minibatches_proccesed > minimum_updates and minibatches_proccesed \
>= best_model_minibatch_time + minimum_updates:
stopping_condition_met = True
break
write_result_summary(best_model_loss)
write_result_summary(json.dumps(best_json_data))
return best_model_path
def train_model(data_set_identifier, train_file, val_file, learning_rate, minibatch_size, name):
set_experiment_id(data_set_identifier, learning_rate, minibatch_size, name)
train_loader = contruct_dataloader_from_disk(train_file, minibatch_size, use_evolutionary=True)
validation_loader = contruct_dataloader_from_disk(val_file, minibatch_size, use_evolutionary=True)
validation_dataset_size = validation_loader.dataset.__len__()
train_dataset_size = train_loader.dataset.__len__()
embedding_size = 21
if configs.run_params["use_evolutionary"]:
embedding_size = 42
#Load in existing model if given as argument
if args.model is not None:
model_path = "output/models/" + args.model + ".model"
model = load_model_from_disk(model_path, use_gpu)
else:
#else construct new model from config file
model = construct_model(configs.model_params, embedding_size, use_gpu,minibatch_size)
#optimizer parameters
betas = tuple(configs.run_params["betas"])
weight_decay = configs.run_params["weight_decay"]
angle_lr = configs.run_params["angles_lr"]
if configs.model_params['architecture'] == 'cnn_angles':
optimizer = optim.Adam(model.parameters(), betas=betas, lr=learning_rate, weight_decay=weight_decay)
else:
optimizer = optim.Adam([
{'params' : model.model.parameters(), 'lr':learning_rate},
{'params' : model.soft_to_angle.parameters(), 'lr':angle_lr}], betas=betas, weight_decay=weight_decay)
#print number of trainable parameters
print_number_of_parameters(model)
#For creating a summary table of the model (does not work on ExampleModel!)
if configs.run_params["print_model_summary"]:
if configs.model_params["architecture"] != 'rnn':
summary(model, configs.run_params["max_sequence_length"], 2)
else:
write_out("DETAILED MODEL SUMMARY IS NOT SUPPORTED FOR RNN MODELS")
if use_gpu:
model = model.cuda()
# TODO: is soft_to_angle.parameters() included here?
sample_num = list()
train_loss_values = list()
validation_loss_values = list()
rmsd_avg_values = list()
drmsd_avg_values = list()
break_point_values = list()
breakpoints = configs.run_params['breakpoints']
best_model_loss = 1e20
best_model_train_loss = 1e20
best_model_minibatch_time = None
best_model_path = None
stopping_condition_met = False
minibatches_proccesed = 0
loss_atoms = configs.run_params["loss_atoms"]
start_time = time.time()
max_time = configs.run_params["max_time"]
C_epochs = configs.run_params["c_epochs"] # TODO: Change to parameter
C_batch_updates = C_epochs
while not stopping_condition_met:
optimizer.zero_grad()
model.zero_grad()
loss_tracker = np.zeros(0)
start_time_n_minibatches = time.time()
for minibatch_id, training_minibatch in enumerate(train_loader, 0):
minibatches_proccesed += 1
training_minibatch = list(training_minibatch)
primary_sequence, tertiary_positions, mask, p_id = training_minibatch[:-1]
# Update C
C = 1.0 if minibatches_proccesed >= C_batch_updates else float(minibatches_proccesed) / C_batch_updates
#One Hot encode amino string and concate PSSM values.
amino_acids, batch_sizes = one_hot_encode(primary_sequence, 21, use_gpu)
if configs.run_params["use_evolutionary"]:
evolutionary = training_minibatch[-1]
evolutionary, batch_sizes = torch.nn.utils.rnn.pad_packed_sequence(torch.nn.utils.rnn.pack_sequence(evolutionary))
if use_gpu:
evolutionary = evolutionary.cuda()
amino_acids = torch.cat((amino_acids, evolutionary.view(-1, len(batch_sizes) , 21)), 2)
start_compute_loss = time.time()
if configs.run_params["only_angular_loss"]:
#raise NotImplementedError("Only_angular_loss function has not been implemented correctly yet.")
loss = model.compute_angular_loss((amino_acids, batch_sizes), tertiary_positions, mask)
else:
loss = model.compute_loss((amino_acids, batch_sizes), tertiary_positions, mask, C=C, loss_atoms=loss_atoms)
if C != 1:
write_out("C:", C)
write_out("Train loss:", float(loss))
start_compute_grad = time.time()
loss.backward()
loss_tracker = np.append(loss_tracker, float(loss))
end = time.time()
write_out("Loss time:", start_compute_grad-start_compute_loss, "Grad time:", end-start_compute_grad)
optimizer.step()
optimizer.zero_grad()
model.zero_grad()
# for every eval_interval samples, plot performance on the validation set
if minibatches_proccesed % configs.run_params["eval_interval"] == 0:
model.eval()
write_out("Testing model on validation set...")
train_loss = loss_tracker.mean()
loss_tracker = np.zeros(0)
validation_loss, data_total, rmsd_avg, drmsd_avg = evaluate_model(validation_loader,
model, use_gpu, loss_atoms, configs.run_params["use_evolutionary"])
prim = data_total[0][0]
pos = data_total[0][1]
pos_pred = data_total[0][3]
mask = data_total[0][4]
pos = apply_mask(pos, mask)
angles_pred = data_total[0][2]
angles_pred = apply_mask(angles_pred, mask, size=3)
pos_pred = apply_mask(pos_pred, mask)
prim = torch.masked_select(prim, mask)
if use_gpu:
pos = pos.cuda()
pos_pred = pos_pred.cuda()
angles = calculate_dihedral_angels(pos, use_gpu)
#angles_pred = calculate_dihedral_angels(pos_pred, use_gpu)
#angles_pred = data_total[0][2] # Use angles output from model - calculate_dihedral_angels(pos_pred, use_gpu)
write_to_pdb(get_structure_from_angles(prim, angles), "test")
write_to_pdb(get_structure_from_angles(prim, angles_pred), "test_pred")
if validation_loss < best_model_loss:
best_model_loss = validation_loss
best_model_minibatch_time = minibatches_proccesed
best_model_path = write_model_to_disk(model)
if train_loss < best_model_train_loss:
best_model_train_loss = train_loss
best_model_train_path = write_model_to_disk(model, model_type="train")
write_out("Validation loss:", validation_loss, "Train loss:", train_loss)
write_out("Best model so far (validation loss): ", best_model_loss, "at time", best_model_minibatch_time)
write_out("Best model stored at " + best_model_path)
write_out("Best model train stored at " + best_model_train_path)
write_out("Minibatches processed:",minibatches_proccesed)
end_time_n_minibatches = time.time()
n_minibatches_time_used = end_time_n_minibatches - start_time_n_minibatches
minibatches_left = configs.run_params["max_updates"] - minibatches_proccesed
seconds_left = int(n_minibatches_time_used * (minibatches_left/configs.run_params["eval_interval"]))
m, s = divmod(seconds_left, 60)
h, m = divmod(m, 60)
write_out("Estimated time until maximum number of updates:", '{:d}:{:02d}:{:02d}'.format(h, m, s) )
sample_num.append(minibatches_proccesed)
train_loss_values.append(train_loss)
validation_loss_values.append(validation_loss)
rmsd_avg_values.append(rmsd_avg)
drmsd_avg_values.append(drmsd_avg)
if breakpoints and minibatches_proccesed > breakpoints[0]:
break_point_values.append(drmsd_avg)
breakpoints = breakpoints[1:]
data = {}
data["pdb_data_pred"] = open("output/protein_test_pred.pdb","r").read()
data["pdb_data_true"] = open("output/protein_test.pdb","r").read()
data["validation_dataset_size"] = validation_dataset_size
data["sample_num"] = sample_num
data["train_loss_values"] = train_loss_values
data["break_point_values"] = break_point_values
data["validation_loss_values"] = validation_loss_values
data["phi_actual"] = list([math.degrees(float(v)) for v in angles[1:,1]])
data["psi_actual"] = list([math.degrees(float(v)) for v in angles[:-1,2]])
data["phi_predicted"] = list([math.degrees(float(v)) for v in angles_pred[1:,1]])
data["psi_predicted"] = list([math.degrees(float(v)) for v in angles_pred[:-1,2]])
data["drmsd_avg"] = drmsd_avg_values
data["rmsd_avg"] = rmsd_avg_values
if not configs.run_params["hide_ui"]:
res = requests.post('http://localhost:5000/graph', json=data)
if res.ok:
print(res.json())
# Save run data
write_run_to_disk(data)
#Check if maximum time is reached.
start_time_n_minibatches = time.time()
time_used = time.time() - start_time
time_condition = (max_time is not None and time_used > max_time)
max_update_condition = minibatches_proccesed >= configs.run_params["max_updates"]
min_update_condition = (minibatches_proccesed > configs.run_params["min_updates"] and minibatches_proccesed > best_model_minibatch_time * 2)
model.train()
#Checking for stop conditions
if time_condition or max_update_condition or min_update_condition:
stopping_condition_met = True
break
write_out("Best validation model found after" , best_model_minibatch_time , "minibatches.")
write_result_summary(best_model_loss)
return best_model_path
def train_model(data_set_identifier, train_file, val_file, learning_rate,
minibatch_size):
set_experiment_id(data_set_identifier, learning_rate, minibatch_size)
train_loader = contruct_dataloader_from_disk(train_file, minibatch_size)
validation_loader = contruct_dataloader_from_disk(val_file, minibatch_size)
validation_dataset_size = validation_loader.dataset.__len__()
model = ExampleModel(21, minibatch_size,
use_gpu=use_gpu) # embed size = 21
# TODO: is soft_to_angle.parameters() included here?
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
sample_num = list()
train_loss_values = list()
validation_loss_values = list()
rmsd_avg_values = list()
drmsd_avg_values = list()
best_model_loss = 1.1
best_model_minibatch_time = None
best_model_path = None
stopping_condition_met = False
minibatches_proccesed = 0
while not stopping_condition_met:
optimizer.zero_grad()
model.zero_grad()
loss_tracker = np.zeros(0)
for minibatch_id, training_minibatch in enumerate(train_loader, 0):
minibatches_proccesed += 1
primary_sequence, tertiary_positions, mask = training_minibatch
start_compute_loss = time.time()
loss = model.compute_loss(primary_sequence, tertiary_positions)
write_out("Train loss:", float(loss))
start_compute_grad = time.time()
loss.backward()
loss_tracker = np.append(loss_tracker, float(loss))
end = time.time()
write_out("Loss time:", start_compute_grad - start_compute_loss,
"Grad time:", end - start_compute_grad)
optimizer.step()
optimizer.zero_grad()
model.zero_grad()
# for every eval_interval samples, plot performance on the validation set
if minibatches_proccesed % args.eval_interval == 0:
train_loss = loss_tracker.mean()
loss_tracker = np.zeros(0)
validation_loss, data_total, rmsd_avg, drmsd_avg = evaluate_model(
validation_loader, model)
prim = data_total[0][0]
pos = data_total[0][1]
(aa_list, phi_list, psi_list,
omega_list) = calculate_dihedral_angels(prim, pos)
write_to_pdb(
get_structure_from_angles(aa_list, phi_list[1:],
psi_list[:-1], omega_list[:-1]),
"test")
cmd.load("output/protein_test.pdb")
write_to_pdb(data_total[0][3], "test_pred")
cmd.load("output/protein_test_pred.pdb")
cmd.forward()
cmd.orient()
if validation_loss < best_model_loss:
best_model_loss = validation_loss
best_model_minibatch_time = minibatches_proccesed
best_model_path = write_model_to_disk(model)
write_out("Validation loss:", validation_loss, "Train loss:",
train_loss)
write_out("Best model so far (label loss): ", validation_loss,
"at time", best_model_minibatch_time)
write_out("Best model stored at " + best_model_path)
write_out("Minibatches processed:", minibatches_proccesed)
sample_num.append(minibatches_proccesed)
train_loss_values.append(train_loss)
validation_loss_values.append(validation_loss)
rmsd_avg_values.append(rmsd_avg)
drmsd_avg_values.append(drmsd_avg)
if args.live_plot:
data = {}
data["validation_dataset_size"] = validation_dataset_size
data["sample_num"] = sample_num
data["train_loss_values"] = train_loss_values
data["validation_loss_values"] = validation_loss_values
data["phi_actual"] = list(
[math.degrees(float(v)) for v in phi_list[1:]])
data["psi_actual"] = list(
[math.degrees(float(v)) for v in psi_list[:-1]])
data["phi_predicted"] = list([
math.degrees(float(v)) for v in data_total[0]
[2].detach().transpose(0, 1)[0][1:]
])
data["psi_predicted"] = list([
math.degrees(float(v)) for v in data_total[0]
[2].detach().transpose(0, 1)[1][:-1]
])
data["drmsd_avg"] = drmsd_avg_values
data["rmsd_avg"] = rmsd_avg_values
res = requests.post('http://localhost:5000/graph',
json=data)
if res.ok:
print(res.json())
if minibatches_proccesed > args.minimum_updates and minibatches_proccesed > best_model_minibatch_time * 2:
stopping_condition_met = True
break
write_result_summary(best_model_loss)
return best_model_path
def train_model(data_set_identifier,
model,
train_loader,
validation_loader,
learning_rate,
minibatch_size=64,
eval_interval=50,
hide_ui=False,
use_gpu=False,
minimum_updates=100):
set_experiment_id(data_set_identifier, learning_rate, minibatch_size)
validation_dataset_size = validation_loader.dataset.__len__()
if use_gpu:
model = model.cuda()
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
sample_num = list()
train_loss_values = list()
validation_loss_values = list()
best_model_loss = 1e20
best_model_minibatch_time = None
best_model_path = None
_best_json_data = None
stopping_condition_met = False
minibatches_proccesed = 0
while not stopping_condition_met:
optimizer.zero_grad()
model.zero_grad()
loss_tracker = np.zeros(0)
for _minibatch_id, training_minibatch in enumerate(train_loader, 0):
minibatches_proccesed += 1
start_compute_loss = time.time()
loss = model.compute_loss(training_minibatch,
minibatches_proccesed, minimum_updates)
write_out("Train loss:", float(loss))
start_compute_grad = time.time()
loss.backward()
loss_tracker = np.append(loss_tracker, float(loss))
end = time.time()
write_out("Loss time:", start_compute_grad - start_compute_loss,
"Grad time:", end - start_compute_grad)
optimizer.step()
optimizer.zero_grad()
model.zero_grad()
# for every eval_interval samples, plot performance on the validation set
if minibatches_proccesed % eval_interval == 0:
write_out("Testing model on validation set...")
train_loss = float(loss_tracker.mean())
loss_tracker = np.zeros(0)
validation_loss, json_data, _ = model.evaluate_model(
validation_loader)
if validation_loss < best_model_loss:
best_model_loss = validation_loss
best_model_minibatch_time = minibatches_proccesed
best_model_path = write_model_to_disk(model)
_best_json_data = json_data
write_out("Validation loss:", validation_loss, "Train loss:",
train_loss)
write_out("Best model so far (validation loss): ",
best_model_loss, "at time",
best_model_minibatch_time)
write_out("Best model stored at " + best_model_path)
write_out("Minibatches processed:", minibatches_proccesed)
sample_num.append(minibatches_proccesed)
train_loss_values.append(train_loss)
validation_loss_values.append(validation_loss)
json_data["validation_dataset_size"] = validation_dataset_size
json_data["sample_num"] = sample_num
json_data["train_loss_values"] = train_loss_values
json_data["validation_loss_values"] = validation_loss_values
if not hide_ui:
write_out(
"Updating monitoring service:",
str(json_data) if len(str(json_data)) < 50 else
str(json_data)[:50] + "...")
res = requests.post('http://localhost:5000/graph',
json=json_data)
if res.ok:
write_out("Received response from monitoring service:",
res.json())
if minibatches_proccesed > minimum_updates and minibatches_proccesed \
>= best_model_minibatch_time + minimum_updates:
stopping_condition_met = True
break
write_result_summary(best_model_loss)
# write_result_summary(json.dumps(_best_json_data))
return best_model_path
def train_model(data_set_identifier, train_file, val_file, learning_rate,
minibatch_size):
set_experiment_id(data_set_identifier, learning_rate, minibatch_size)
train_loader = contruct_dataloader_from_disk(train_file, minibatch_size)
validation_loader = contruct_dataloader_from_disk(val_file, minibatch_size)
validation_dataset_size = validation_loader.dataset.__len__()
model = ExampleModel(21, minibatch_size,
use_gpu=use_gpu) # embed size = 21
if use_gpu:
model = model.cuda()
# TODO: is soft_to_angle.parameters() included here?
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
sample_num = list()
train_loss_values = list()
validation_loss_values = list()
rmsd_avg_values = list()
drmsd_avg_values = list()
best_model_loss = 1e20
best_model_minibatch_time = None
best_model_path = None
stopping_condition_met = False
minibatches_proccesed = 0
while not stopping_condition_met:
optimizer.zero_grad()
model.zero_grad()
loss_tracker = np.zeros(0)
for minibatch_id, training_minibatch in enumerate(train_loader, 0):
minibatches_proccesed += 1
primary_sequence, tertiary_positions, mask = training_minibatch
start_compute_loss = time.time()
loss = model.compute_loss(primary_sequence, tertiary_positions)
write_out("Train loss:", float(loss))
start_compute_grad = time.time()
loss.backward()
loss_tracker = np.append(loss_tracker, float(loss))
end = time.time()
write_out("Loss time:", start_compute_grad - start_compute_loss,
"Grad time:", end - start_compute_grad)
optimizer.step()
optimizer.zero_grad()
model.zero_grad()
# for every eval_interval samples, plot performance on the validation set
if minibatches_proccesed % args.eval_interval == 0:
write_out("Testing model on validation set...")
train_loss = loss_tracker.mean()
loss_tracker = np.zeros(0)
validation_loss, data_total, rmsd_avg, drmsd_avg = evaluate_model(
validation_loader, model)
prim = data_total[0][0]
pos = data_total[0][1]
pos_pred = data_total[0][3]
if use_gpu:
pos = pos.cuda()
pos_pred = pos_pred.cuda()
angles = calculate_dihedral_angels(pos, use_gpu)
angles_pred = calculate_dihedral_angels(pos_pred, use_gpu)
write_to_pdb(get_structure_from_angles(prim, angles), "test")
write_to_pdb(get_structure_from_angles(prim, angles_pred),
"test_pred")
if validation_loss < best_model_loss:
best_model_loss = validation_loss
best_model_minibatch_time = minibatches_proccesed
best_model_path = saveModel(encoder_net, decoder_net,
encoder_optimizer,
decoder_optimizer, loss.item(),
tot_eval_acc, e)
write_out("Validation loss:", validation_loss, "Train loss:",
train_loss)
write_out("Best model so far (validation loss): ",
validation_loss, "at time",
best_model_minibatch_time)
write_out("Best model stored at " + best_model_path)
write_out("Minibatches processed:", minibatches_proccesed)
sample_num.append(minibatches_proccesed)
train_loss_values.append(train_loss)
validation_loss_values.append(validation_loss)
rmsd_avg_values.append(rmsd_avg)
drmsd_avg_values.append(drmsd_avg)
if not args.hide_ui:
data = {}
data["pdb_data_pred"] = open(
"output/protein_test_pred.pdb", "r").read()
data["pdb_data_true"] = open("output/protein_test.pdb",
"r").read()
data["validation_dataset_size"] = validation_dataset_size
data["sample_num"] = sample_num
data["train_loss_values"] = train_loss_values
data["validation_loss_values"] = validation_loss_values
data["phi_actual"] = list(
[math.degrees(float(v)) for v in angles[1:, 1]])
data["psi_actual"] = list(
[math.degrees(float(v)) for v in angles[:-1, 2]])
data["phi_predicted"] = list(
[math.degrees(float(v)) for v in angles_pred[1:, 1]])
data["psi_predicted"] = list(
[math.degrees(float(v)) for v in angles_pred[:-1, 2]])
data["drmsd_avg"] = drmsd_avg_values
data["rmsd_avg"] = rmsd_avg_values
res = requests.post('http://localhost:5000/graph',
json=data)
if res.ok:
print(res.json())
if minibatches_proccesed > args.minimum_updates and minibatches_proccesed > best_model_minibatch_time * 2:
stopping_condition_met = True
break
write_result_summary(best_model_loss)
return best_model_path