def train_val_pipeline(dataset, params, net_params, dirs):
t0 = time.time()
per_epoch_time = []
DATASET_NAME = dataset.name
MODEL_NAME = 'EIG'
trainset, valset, testset = dataset.train, dataset.val, dataset.test
root_log_dir, root_ckpt_dir, write_file_name, write_config_file = dirs
device = net_params['device']
# Write the network and optimization hyper-parameters in folder config/
with open(write_config_file + '.txt', 'w') as f:
f.write(
"""Dataset: {},\nModel: {}\n\nparams={}\n\nnet_params={}\n\n\nTotal Parameters: {}\n\n"""
.format(DATASET_NAME, MODEL_NAME, params, net_params,
net_params['total_param']))
log_dir = os.path.join(root_log_dir, "RUN_" + str(0))
writer = SummaryWriter(log_dir=log_dir)
# setting seeds
random.seed(params['seed'])
np.random.seed(params['seed'])
torch.manual_seed(params['seed'])
if device == 'cuda':
torch.cuda.manual_seed(params['seed'])
print("Training Graphs: ", len(trainset))
print("Validation Graphs: ", len(valset))
print("Test Graphs: ", len(testset))
model = EIGNet(net_params)
model = model.to(device)
optimizer = optim.Adam(model.parameters(),
lr=params['init_lr'],
weight_decay=params['weight_decay'])
scheduler = optim.lr_scheduler.ReduceLROnPlateau(
optimizer,
mode='min',
factor=params['lr_reduce_factor'],
patience=params['lr_schedule_patience'],
verbose=True)
epoch_train_losses, epoch_val_losses = [], []
epoch_train_ROCs, epoch_val_ROCs, epoch_test_ROCs = [], [], []
train_loader = DataLoader(trainset,
batch_size=params['batch_size'],
shuffle=True,
collate_fn=dataset.collate,
pin_memory=True)
val_loader = DataLoader(valset,
batch_size=params['batch_size'],
shuffle=False,
collate_fn=dataset.collate,
pin_memory=True)
test_loader = DataLoader(testset,
batch_size=params['batch_size'],
shuffle=False,
collate_fn=dataset.collate,
pin_memory=True)
# At any point you can hit Ctrl + C to break out of training early.
try:
with tqdm(range(params['epochs']), unit='epoch') as t:
for epoch in t:
if epoch == -1:
model.reset_params()
t.set_description('Epoch %d' % epoch)
start = time.time()
epoch_train_loss, epoch_train_roc, optimizer = train_epoch(
model, optimizer, device, train_loader, epoch,
net_params['distortion'])
epoch_val_loss, epoch_val_roc = evaluate_network(
model, device, val_loader, epoch)
epoch_train_losses.append(epoch_train_loss)
epoch_val_losses.append(epoch_val_loss)
epoch_train_ROCs.append(epoch_train_roc.item())
epoch_val_ROCs.append(epoch_val_roc.item())
writer.add_scalar('train/_loss', epoch_train_loss, epoch)
writer.add_scalar('val/_loss', epoch_val_loss, epoch)
writer.add_scalar('train/_roc', epoch_train_roc, epoch)
writer.add_scalar('val/_roc', epoch_val_roc, epoch)
writer.add_scalar('learning_rate',
optimizer.param_groups[0]['lr'], epoch)
_, epoch_test_roc = evaluate_network(model, device,
test_loader, epoch)
epoch_test_ROCs.append(epoch_test_roc.item())
t.set_postfix(time=time.time() - start,
lr=optimizer.param_groups[0]['lr'],
train_loss=epoch_train_loss,
val_loss=epoch_val_loss,
train_ROC=epoch_train_roc.item(),
val_ROC=epoch_val_roc.item(),
test_ROC=epoch_test_roc.item(),
refresh=False)
per_epoch_time.append(time.time() - start)
scheduler.step(-epoch_val_roc.item())
if optimizer.param_groups[0]['lr'] < params['min_lr']:
print("\n!! LR EQUAL TO MIN LR SET.")
break
# Stop training after params['max_time'] hours
if time.time() - t0 > params['max_time'] * 3600:
print('-' * 89)
print(
"Max_time for training elapsed {:.2f} hours, so stopping"
.format(params['max_time']))
break
print('')
#for _ in range(5):
#print('Sampled value is ', model.layers[1].towers[0].eigfiltbis(torch.FloatTensor([random.random() for i in range(4)]).to('cuda')))
except KeyboardInterrupt:
print('-' * 89)
print('Exiting from training early because of KeyboardInterrupt')
best_val_epoch = np.argmax(np.array(epoch_val_ROCs))
best_train_epoch = np.argmax(np.array(epoch_train_ROCs))
best_val_roc = epoch_val_ROCs[best_val_epoch]
best_val_test_roc = epoch_test_ROCs[best_val_epoch]
best_val_train_roc = epoch_train_ROCs[best_val_epoch]
best_train_roc = epoch_train_ROCs[best_train_epoch]
print("Best Train ROC: {:.4f}".format(best_train_roc))
print("Best Val ROC: {:.4f}".format(best_val_roc))
print("Test ROC of Best Val: {:.4f}".format(best_val_test_roc))
print("Train ROC of Best Val: {:.4f}".format(best_val_train_roc))
print("TOTAL TIME TAKEN: {:.4f}s".format(time.time() - t0))
print("AVG TIME PER EPOCH: {:.4f}s".format(np.mean(per_epoch_time)))
writer.close()
"""
Write the results in out_dir/results folder
"""
with open(write_file_name + '.txt', 'w') as f:
f.write("""Dataset: {},\nModel: {}\n\nparams={}\n\nnet_params={}\n\n{}\n\nTotal Parameters: {}\n\n
FINAL RESULTS\nTEST ROC of Best Val: {:.4f}\nBest TRAIN ROC: {:.4f}\nTRAIN ROC of Best Val: {:.4f}\nBest VAL ROC: {:.4f}\n\n
Total Time Taken: {:.4f} hrs\nAverage Time Per Epoch: {:.4f} s\n\n\n""" \
.format(DATASET_NAME, MODEL_NAME, params, net_params, model, net_params['total_param'],
best_val_test_roc, best_train_roc, best_val_train_roc, best_val_roc, (time.time() - t0) / 3600,
np.mean(per_epoch_time)))
def train_val_pipeline(dataset, params, net_params):
t0 = time.time()
per_epoch_time = []
trainset, valset, testset = dataset.train, dataset.val, dataset.test
device = net_params['device']
# setting seeds
random.seed(params['seed'])
np.random.seed(params['seed'])
torch.manual_seed(params['seed'])
if device.type == 'cuda':
torch.cuda.manual_seed(params['seed'])
print("Training Graphs: ", len(trainset))
print("Validation Graphs: ", len(valset))
print("Test Graphs: ", len(testset))
model = PNANet(net_params)
model = model.to(device)
optimizer = optim.Adam(model.parameters(),
lr=params['init_lr'],
weight_decay=params['weight_decay'])
scheduler = optim.lr_scheduler.ReduceLROnPlateau(
optimizer,
mode='min',
factor=params['lr_reduce_factor'],
patience=params['lr_schedule_patience'],
verbose=True)
epoch_train_losses, epoch_val_losses = [], []
epoch_train_ROCs, epoch_val_ROCs, epoch_test_ROCs = [], [], []
train_loader = DataLoader(trainset,
batch_size=params['batch_size'],
shuffle=True,
collate_fn=dataset.collate,
pin_memory=True)
val_loader = DataLoader(valset,
batch_size=params['batch_size'],
shuffle=False,
collate_fn=dataset.collate,
pin_memory=True)
test_loader = DataLoader(testset,
batch_size=params['batch_size'],
shuffle=False,
collate_fn=dataset.collate,
pin_memory=True)
# At any point you can hit Ctrl + C to break out of training early.
try:
with tqdm(range(params['epochs']), unit='epoch') as t:
for epoch in t:
if epoch == -1:
model.reset_params()
t.set_description('Epoch %d' % epoch)
start = time.time()
epoch_train_loss, epoch_train_roc, optimizer = train_epoch(
model, optimizer, device, train_loader, epoch)
epoch_val_loss, epoch_val_roc = evaluate_network(
model, device, val_loader, epoch)
epoch_train_losses.append(epoch_train_loss)
epoch_val_losses.append(epoch_val_loss)
epoch_train_ROCs.append(epoch_train_roc.item())
epoch_val_ROCs.append(epoch_val_roc.item())
_, epoch_test_roc = evaluate_network(model, device,
test_loader, epoch)
epoch_test_ROCs.append(epoch_test_roc.item())
t.set_postfix(time=time.time() - start,
lr=optimizer.param_groups[0]['lr'],
train_loss=epoch_train_loss,
val_loss=epoch_val_loss,
train_ROC=epoch_train_roc.item(),
val_ROC=epoch_val_roc.item(),
test_ROC=epoch_test_roc.item(),
refresh=False)
per_epoch_time.append(time.time() - start)
scheduler.step(-epoch_val_roc.item())
if optimizer.param_groups[0]['lr'] < params['min_lr']:
print("\n!! LR EQUAL TO MIN LR SET.")
break
# Stop training after params['max_time'] hours
if time.time() - t0 > params['max_time'] * 3600:
print('-' * 89)
print(
"Max_time for training elapsed {:.2f} hours, so stopping"
.format(params['max_time']))
break
print('')
except KeyboardInterrupt:
print('-' * 89)
print('Exiting from training early because of KeyboardInterrupt')
best_val_epoch = np.argmax(np.array(epoch_val_ROCs))
best_train_epoch = np.argmax(np.array(epoch_train_ROCs))
best_val_roc = epoch_val_ROCs[best_val_epoch]
best_val_test_roc = epoch_test_ROCs[best_val_epoch]
best_val_train_roc = epoch_train_ROCs[best_val_epoch]
best_train_roc = epoch_train_ROCs[best_train_epoch]
print("Best Train ROC: {:.4f}".format(best_train_roc))
print("Best Val ROC: {:.4f}".format(best_val_roc))
print("Test ROC of Best Val: {:.4f}".format(best_val_test_roc))
print("Train ROC of Best Val: {:.4f}".format(best_val_train_roc))
print("TOTAL TIME TAKEN: {:.4f}s".format(time.time() - t0))
print("AVG TIME PER EPOCH: {:.4f}s".format(np.mean(per_epoch_time)))
def train_val_pipeline(dataset, params, net_params, dirs):
t0 = time.time()
per_epoch_time = []
DATASET_NAME = dataset.name
MODEL_NAME = 'EIG'
trainset, valset, testset = dataset.train, dataset.val, dataset.test
root_log_dir, root_ckpt_dir, write_file_name, write_config_file = dirs
device = net_params['device']
# Write the network and optimization hyper-parameters in folder config/
with open(write_config_file + '.txt', 'w') as f:
f.write(
"""Dataset: {},\nModel: {}\n\nparams={}\n\nnet_params={}\n\n\nTotal Parameters: {}\n\n"""
.format(DATASET_NAME, MODEL_NAME, params, net_params,
net_params['total_param']))
log_dir = os.path.join(root_log_dir, "RUN_" + str(0))
writer = SummaryWriter(log_dir=log_dir)
# setting seeds
random.seed(params['seed'])
np.random.seed(params['seed'])
torch.manual_seed(params['seed'])
if device == 'cuda':
torch.cuda.manual_seed(params['seed'])
if hydra.is_first_execution():
print("Training Graphs: ", len(trainset))
print("Validation Graphs: ", len(valset))
print("Test Graphs: ", len(testset))
model = EIGNet(net_params)
model = model.to(device)
optimizer = optim.Adam(model.parameters(),
lr=params['init_lr'],
weight_decay=params['weight_decay'])
scheduler = optim.lr_scheduler.ReduceLROnPlateau(
optimizer,
mode='min',
factor=params['lr_reduce_factor'],
patience=params['lr_schedule_patience'],
verbose=True)
epoch_train_losses, epoch_val_losses = [], []
epoch_train_ROCs, epoch_val_ROCs, epoch_test_ROCs = [], [], []
train_loader = DataLoader(trainset,
batch_size=params['batch_size'],
shuffle=True,
collate_fn=dataset.collate,
pin_memory=True)
val_loader = DataLoader(valset,
batch_size=params['batch_size'],
shuffle=False,
collate_fn=dataset.collate,
pin_memory=True)
test_loader = DataLoader(testset,
batch_size=params['batch_size'],
shuffle=False,
collate_fn=dataset.collate,
pin_memory=True)
if hydra.is_first_execution():
start_epoch = 0
else:
t0 -= hydra.retrieved_checkpoint.time_elapsed
start_epoch = hydra.retrieved_checkpoint.last_epoch
states = torch.load(hydra.retrieved_checkpoint.linked_files()[0])
model.load_state_dict(states['model'])
optimizer.load_state_dict(states['optimizer'])
scheduler.load_state_dict(states['scheduler'])
last_hydra_checkpoint = t0
# At any point you can hit Ctrl + C to break out of training early.
try:
with tqdm(range(start_epoch, params['epochs']),
mininterval=params['hydra_progress_bar_every'],
maxinterval=None,
unit='epoch',
initial=start_epoch,
total=params['epochs']) as t:
for epoch in t:
if epoch == -1:
model.reset_params()
t.set_description('Epoch %d' % epoch)
start = time.time()
epoch_train_loss, epoch_train_roc, optimizer = train_epoch(
model, optimizer, device, train_loader, epoch)
epoch_val_loss, epoch_val_roc = evaluate_network(
model, device, val_loader, epoch)
epoch_train_losses.append(epoch_train_loss)
epoch_val_losses.append(epoch_val_loss)
epoch_train_ROCs.append(epoch_train_roc.item())
epoch_val_ROCs.append(epoch_val_roc.item())
writer.add_scalar('train/_loss', epoch_train_loss, epoch)
writer.add_scalar('val/_loss', epoch_val_loss, epoch)
writer.add_scalar('train/_roc', epoch_train_roc, epoch)
writer.add_scalar('val/_roc', epoch_val_roc, epoch)
writer.add_scalar('learning_rate',
optimizer.param_groups[0]['lr'], epoch)
_, epoch_test_roc = evaluate_network(model, device,
test_loader, epoch)
epoch_test_ROCs.append(epoch_test_roc.item())
t.set_postfix(time=time.time() - start,
lr=optimizer.param_groups[0]['lr'],
train_loss=epoch_train_loss,
val_loss=epoch_val_loss,
train_ROC=epoch_train_roc.item(),
val_ROC=epoch_val_roc.item(),
test_ROC=epoch_test_roc.item(),
refresh=False)
per_epoch_time.append(time.time() - start)
scheduler.step(-epoch_val_roc.item())
if optimizer.param_groups[0]['lr'] < params['min_lr']:
print("\n!! LR EQUAL TO MIN LR SET.")
break
# Stop training after params['max_time'] hours
if time.time() - t0 > params['max_time'] * 3600:
print('-' * 89)
print(
"Max_time for training elapsed {:.2f} hours, so stopping"
.format(params['max_time']))
break
# Saving checkpoint
if hydra.is_available() and (time.time(
) - last_hydra_checkpoint) > params['hydra_checkpoint_every']:
last_hydra_checkpoint = time.time()
ck_path = '/tmp/epoch_{}.pkl'.format(epoch + 1)
torch.save(
{
'model': model.state_dict(),
'optimizer': optimizer.state_dict(),
'scheduler': scheduler.state_dict()
}, ck_path)
ck = hydra.checkpoint()
ck.last_epoch = epoch + 1
ck.time_elapsed = time.time() - t0
# save best epoch
ck.link_file(ck_path)
ck.save_to_server()
if hydra.is_available(
) and epoch % params['hydra_eta_every'] == 0:
hydra.set_eta(per_epoch_time[-1] *
(params['epochs'] - epoch - 1))
print('')
#for _ in range(5):
#print('Sampled value is ', model.layers[1].towers[0].eigfiltbis(torch.FloatTensor([random.random() for i in range(4)]).to('cuda')))
except KeyboardInterrupt:
print('-' * 89)
print('Exiting from training early because of KeyboardInterrupt')
best_val_epoch = np.argmax(np.array(epoch_val_ROCs))
best_train_epoch = np.argmax(np.array(epoch_train_ROCs))
best_val_roc = epoch_val_ROCs[best_val_epoch]
best_val_test_roc = epoch_test_ROCs[best_val_epoch]
best_val_train_roc = epoch_train_ROCs[best_val_epoch]
best_train_roc = epoch_train_ROCs[best_train_epoch]
print("Best Train ROC: {:.4f}".format(best_train_roc))
print("Best Val ROC: {:.4f}".format(best_val_roc))
print("Test ROC of Best Val: {:.4f}".format(best_val_test_roc))
print("Train ROC of Best Val: {:.4f}".format(best_val_train_roc))
print("TOTAL TIME TAKEN: {:.4f}s".format(time.time() - t0))
print("AVG TIME PER EPOCH: {:.4f}s".format(np.mean(per_epoch_time)))
writer.close()
if hydra.is_available():
hydra.save_output(
{
'loss': {
'train': epoch_train_losses,
'val': epoch_val_losses
},
'ROC': {
'train': epoch_train_ROCs,
'val': epoch_val_ROCs
}
}, 'history')
hydra.save_output(
{
'test_roc': best_val_test_roc,
'best_train_roc': best_train_roc,
'train_roc': best_val_train_roc,
'val_roc': best_val_roc,
'total_time': time.time() - t0,
'avg_epoch_time': np.mean(per_epoch_time)
}, 'summary')
"""
Write the results in out_dir/results folder
"""
with open(write_file_name + '.txt', 'w') as f:
f.write("""Dataset: {},\nModel: {}\n\nparams={}\n\nnet_params={}\n\n{}\n\nTotal Parameters: {}\n\n
FINAL RESULTS\nTEST ROC of Best Val: {:.4f}\nBest TRAIN ROC: {:.4f}\nTRAIN ROC of Best Val: {:.4f}\nBest VAL ROC: {:.4f}\n\n
Total Time Taken: {:.4f} hrs\nAverage Time Per Epoch: {:.4f} s\n\n\n""" \
.format(DATASET_NAME, MODEL_NAME, params, net_params, model, net_params['total_param'],
best_val_test_roc, best_train_roc, best_val_train_roc, best_val_roc, (time.time() - t0) / 3600,
np.mean(per_epoch_time)))