def do_setup_and_start_training(modules,
configs,
rank,
size,
device_list,
single=False):
if not single:
os.environ['MASTER_ADDR'] = '127.0.0.1'
os.environ['MASTER_PORT'] = '29509'
dist.init_process_group('nccl', rank=rank, world_size=size)
with torch.cuda.device(device_list[rank]):
print('initializing model on rank %d' % rank)
if single:
shared_model = DummySharedWrapper(modules['model'](
configs['model'])).cuda()
else:
shared_model = torch.nn.parallel.DistributedDataParallel(
modules['model'](configs['model']).cuda(),
device_ids=[device_list[rank]],
find_unused_parameters=True)
model = DistributedWrapper(shared_model)
memory = modules['memory'](configs['memory'])
loss_function = modules['loss'](configs['loss'])
trainer_config = configs['trainer']
trainer_config.log_path = trainer_config.log_path + '%d.log' % rank
if rank > 0:
trainer_config.save_frequency = 0
trainer = Trainer(model, memory, loss_function, trainer_config)
print('starting training process on rank %d' % rank)
stats = trainer.train()
print('done %d' % rank)
return stats
def evaluate_current_config(docs, labels, tvt_idx, verbose=True):
dataset = DocumentGraphDataset(docs, labels, tvt_idx)
model = create_model(dataset)
# paras = count_parameters(model)
# print("\t\"", config["test_name"], "\" : ", paras, sep="")
# return 0
trainer = Trainer(dataset, model)
best_val_loss = float('inf')
best_model_test = 0
time_since_best = 0
for i in range(config["epochs"]):
train_loss, val_loss = trainer.train_epoch()
test_acc = trainer.test()
if verbose:
print(
config["indices"] +
" [epoch %02d] Train loss %.4f, Val loss %.4f, Test Acc %.4f" %
(i, train_loss, val_loss, test_acc))
# Early stopping
time_since_best += 1
if val_loss < best_val_loss:
best_val_loss = val_loss
best_model_test = test_acc
time_since_best = 0
if config["terminate_early"] and time_since_best >= config[
"terminate_patience"]:
print(
"\n ## [RESULT!] %s achieved final test score at epoch %i: %.4f ## \n"
% (config["test_name"], i - time_since_best, best_model_test))
break
del dataset
del model
del trainer
gc.collect()
torch.cuda.empty_cache()
return best_model_test
def main():
docs, labels, (t_idx, v_idx, test_idx) = load_data()
in_training_test = test_idx[:len(test_idx) // 2]
out_training_test = test_idx[len(test_idx) // 2:]
# train on (train, val, in_training_test)
train_docs = docs[t_idx + v_idx + in_training_test]
train_labels = labels[t_idx + v_idx + in_training_test]
train_and_intest_indices = (t_idx, v_idx,
range(
len(t_idx) + len(v_idx),
len(train_labels)))
a = 0
b = 0
# test on (in_training_test)
only_in_training_docs = docs[t_idx[:a] + v_idx[:b] + in_training_test]
only_in_training_labels = labels[t_idx[:a] + v_idx[:b] + in_training_test]
only_in_training_indices = ([t_idx[:a]], [v_idx[:b]],
range(a + b, len(only_in_training_labels)))
# test on (out_training_test)
only_out_training_docs = docs[t_idx[:a] + v_idx[:b] + out_training_test]
only_out_training_labels = labels[t_idx[:a] + v_idx[:b] +
out_training_test]
only_out_training_indices = ([t_idx[:a]], [v_idx[:b]],
range(a + b, len(only_out_training_labels)))
# test on (train, val, out_training_test)
train_and_outtest_docs = docs[t_idx + v_idx + out_training_test]
train_and_outtest_labels = labels[t_idx + v_idx + out_training_test]
train_and_outtest_indices = (t_idx, v_idx,
range(
len(t_idx) + len(v_idx),
len(train_and_outtest_labels)))
train_and_intest_dataset = DocumentGraphDataset(train_docs, train_labels,
train_and_intest_indices)
only_in_training_dataset = DocumentGraphDataset(
only_in_training_docs,
only_in_training_labels,
only_in_training_indices,
force_vocab=train_and_intest_dataset.vocab)
only_out_training_dataset = DocumentGraphDataset(
only_out_training_docs,
only_out_training_labels,
only_out_training_indices,
force_vocab=train_and_intest_dataset.vocab)
train_and_outtest_dataset = DocumentGraphDataset(
train_and_outtest_docs,
train_and_outtest_labels,
train_and_outtest_indices,
force_vocab=train_and_intest_dataset.vocab)
model = None
best_val_loss = float('inf')
time_since_best = 0
for i in range(config["epochs"]):
if model is None:
model = create_model(train_and_intest_dataset)
trainer = Trainer(train_and_intest_dataset, model)
else:
trainer.update_data(train_and_intest_dataset)
train_loss, val_loss = trainer.train_epoch()
test_acc_train_and_intest = trainer.test()
trainer.update_data(only_in_training_dataset)
test_acc_only_in_training = trainer.test()
trainer.update_data(only_out_training_dataset)
test_acc_only_out_training = trainer.test()
trainer.update_data(train_and_outtest_dataset)
test_acc_train_and_outtest = trainer.test()
print("[epoch %02d] Train loss %.4f, Val loss %.4f" %
(i, train_loss, val_loss))
print(" acc on in training test, with training docs in graph: %.4f" %
(test_acc_train_and_intest))
print(
" acc on in training test, WITHOUT training docs in graph: %.4f" %
(test_acc_only_in_training))
print(
" acc on out of training test, WITHOUT training docs in graph: %.4f"
% (test_acc_only_out_training))
print(
" acc on out of training test, with training docs in graph: %.4f" %
(test_acc_train_and_outtest))
# Early stopping
time_since_best += 1
if val_loss < best_val_loss:
best_val_loss = val_loss
time_since_best = 0
if config["terminate_early"] and time_since_best >= config[
"terminate_patience"]:
print("\n[RESULT!] Final test score: see above")
break
def main():
update_config(quick_config)
docs, labels, tvt_idx = load_data()
dataset = DocumentGraphDataset(docs, labels, tvt_idx)
model = create_model(dataset)
trainer = Trainer(dataset, model)
# trainer.save_initial_reps()
best_val_loss = float('inf')
time_since_best = 0
best_val_loss_acc = 0
high_score = 0
for i in range(config["epochs"]):
# split for special debug printing
if config["sampled_training"] and config["unsupervised_loss"]:
# trainer.save_sage_reps()
train_loss, val_loss, unsup_train_loss_pos, unsup_train_loss_neg, unsup_val_loss_pos, unsup_val_loss_neg, unsup_test_pos, unsup_test_neg = trainer.train_epoch(
)
test_acc, test_loss, unsup_test_loss_pos, unsup_test_loss_neg = trainer.test(
)
total_train = train_loss + unsup_train_loss_pos + unsup_train_loss_neg
total_val = val_loss + unsup_val_loss_pos + unsup_val_loss_neg
total_test = test_loss + unsup_test_loss_pos + unsup_test_loss_neg
print("[epoch %02d] Test Acc %.4f (Trained %s-supervised)" %
(i, test_acc, config['sup_mode']))
print("\t Train Loss: %.4f (%.4f / %.4f / %.4f) (%.0f%% sup)" %
(total_train, train_loss, unsup_train_loss_pos,
unsup_train_loss_neg, train_loss / total_train * 100))
print("\t Val Loss: %.4f (%.4f / %.4f / %.4f) (%.0f%% sup)" %
(total_val, val_loss, unsup_val_loss_pos, unsup_val_loss_neg,
val_loss / total_val * 100))
print("\t Training on test Losses: %.4f, %.4f, %.1f%% of total" %
(unsup_test_pos, unsup_test_neg,
(unsup_test_pos + unsup_test_neg) /
(unsup_test_pos + unsup_test_neg + total_train) * 100))
# print("\t Test Loss: %.4f (%.4f / %.4f / %.4f) (%.0f%% sup)" % (total_test, test_loss, unsup_test_loss_pos, unsup_test_loss_neg, test_loss / total_test))
val_loss = total_val
else:
train_loss, val_loss = trainer.train_epoch()
test_acc = trainer.test()
high_score = max(test_acc, high_score)
print(
"[epoch %02d] Train loss %.4f, Val loss %.4f, Test Acc %.4f, Highscore: %.4f"
% (i, train_loss, val_loss, test_acc, high_score))
# Early stopping
time_since_best += 1
if val_loss < best_val_loss:
best_val_loss = val_loss
time_since_best = 0
best_val_loss_acc = test_acc
if config["terminate_early"] and time_since_best >= config[
"terminate_patience"]:
if config["sampled_training"] and config["unsupervised_loss"]:
test_acc, test_loss, unsup_test_loss_pos, unsup_test_loss_neg = trainer.test(
)
print("\n[RESULT!] Final test score: ", best_val_loss_acc)
else:
test_acc = trainer.test()
break
print("\n[RESULT!] Final test score: ", best_val_loss_acc)
actor_config.data_files = [[
'NovelObjects__train.json', 'NovelObjects__test.json'
], ['NovelSpaces__train.json', 'NovelSpaces__test.json']][dataset]
actor_config.data_files = [
os.path.join(dataset_folder, fn) for fn in actor_config.data_files
]
loss_function = MaskAndMassLoss(MaskAndMassLossConfig())
model = ClusteringModel(ClusteringModelConfig()).cuda(
global_config.model_gpu)
trainer_config = TrainerConfig()
trainer_config.log_path = os.path.join(output_folder, 'training_log.log')
trainer = Trainer(model, ReplayPILDataset(MemoryConfigPIL()),
loss_function, trainer_config)
print('Running instance segmentation only pre-training')
actor_config.instance_only = True
loss_function.config.instance_only = True
trainer_config.checkpoint_path = os.path.join(
output_folder, args.checkpoint_prefix + 'inst_only_')
model.toggle_mass_head(False)
trainer.train()
print('Training with force prediction')
actor_config.instance_only = False
def main():
# load data
docs, labels, tvt_idx = load_data()
train_idx, val_idx, test_idx = tvt_idx
model = None
results = {
"epochs": [],
"average_train_loss": [],
"average_val_loss": [],
"average_test_acc": [],
"split_amount": [],
"acc": []
}
# create model with vocab for entire dataset
dataset = DocumentGraphDataset(docs, labels, tvt_idx)
model = create_model(dataset)
trainer = Trainer(dataset, model)
for i in range(config["epochs"]):
number_of_splits = random.randint(1, 40)
# split in x random divisions
split_train_amount = len(train_idx) // number_of_splits
split_val_amount = len(val_idx) // number_of_splits
split_test_amount = len(test_idx) // number_of_splits
print("Splitting %i segments, %i %i %i" %
(number_of_splits, split_train_amount, split_val_amount,
split_test_amount))
# train on different random splits
random.shuffle(train_idx)
random.shuffle(val_idx)
random.shuffle(test_idx)
train_losses = []
val_losses = []
test_accs = []
for split_i in range(number_of_splits):
split_train_idx = train_idx[split_i *
split_train_amount:(split_i + 1) *
split_train_amount]
split_val_idx = val_idx[split_i * split_val_amount:(split_i + 1) *
split_val_amount]
split_test_idx = test_idx[split_i *
split_test_amount:(split_i + 1) *
split_test_amount]
# this ain't right
dataset = DocumentGraphDataset(
docs, labels, (split_train_idx, split_val_idx, split_test_idx))
if model is None:
model = create_model(dataset)
trainer = Trainer(dataset, model)
else:
trainer.update_data(dataset)
train_loss, val_loss = trainer.train_epoch()
test_acc = trainer.test()
print("split %02d: (%.4f, %.4f, ! %.4f !), " %
(split_i, train_loss, val_loss, test_acc),
end="")
train_losses.append(train_loss)
val_losses.append(val_loss)
test_accs.append(test_acc)
# test on entire graph
dataset = DocumentGraphDataset(docs, labels, tvt_idx)
trainer.update_data(dataset)
test_acc = trainer.test()
print("\n\n[epoch %02d] Test Acc on entire dataset %.4f\n\n" %
(i, test_acc))
# summary
results["epochs"].append(i)
results["average_train_loss"].append(
float(sum(train_losses) / len(train_losses)))
results["average_val_loss"].append(
float(sum(val_losses) / len(val_losses)))
results["average_test_acc"].append(
float(sum(test_accs) / len(test_accs)))
results["split_amount"].append(number_of_splits)
results["acc"].append(test_acc)
df = pd.DataFrame(results)
df.to_csv('./results/' + config['experiment_name'] + '.csv')