def SVM(para, train_data, train_labels, testing_data, testing_labels):
""" train a support vector machine models and evaluate on testing data
Args:
--------
train_data: log sequence matrix for training
train_labels: labels for training
testing_data: log sequence matrix for testing/evaluation
testing_labels: labels for testing/evaluation
Returns:
--------
precision: The evaluation metric Precision
recall: The evaluation metric Recall
f1_score: The evaluation metric F1_score
"""
print("Train a SVM Model")
clf = svm.LinearSVC(penalty='l1',
tol=0.0001,
C=1,
dual=False,
fit_intercept=True,
intercept_scaling=1,
class_weight='balanced',
max_iter=1000)
clf = clf.fit(train_data, train_labels.ravel())
prediction = list(clf.predict(testing_data))
assert len(prediction) == len(testing_labels)
if para['cross_validate']:
ev.cv_evaluate(clf, train_data, train_labels)
else:
ev.evaluate(testing_labels, prediction)
def anomaly_detection(weigh_data, label_data, C, threshold):
""" detect anomalies by projecting into a subspace with C
Args:
--------
weigh_data: weighted raw data
label_data: the labels list
threshold: used as the threshold, which determines the anomalies
C: the projection matrix
Returns:
--------
"""
print ('there are %d anomalies' % (sum(label_data)))
event_num, inst_size = weigh_data.shape
predict_results = np.zeros((inst_size),int)
print('the threshold is %f' % (threshold))
for i in range(inst_size):
ya = np.dot(C,weigh_data[:,i])
SPE = np.dot(ya,ya)
if SPE > threshold:
predict_results[i] = 1 #1 represent failure
assert len(label_data) == len(predict_results)
ev.evaluate(label_data, predict_results)
def evaluate(event_count_matrix, invar_dict, groundtruth_labels):
""" evaluate the results with mined invariants
Args:
--------
event_count_matrix: the input event count matrix
invar_dict: the dictionary of invariants
groundtruth_labels: the groundtruth labels for evaluation
Returns:
--------
"""
print("the mined {} invariants are: {}".format(len(invar_dict), invar_dict))
valid_col_list = []
valid_invar_list = []
for key in invar_dict:
valid_col_list.append(list(key))
valid_invar_list.append(list(invar_dict[key]))
prediction = []
for row in event_count_matrix:
label = 0
for i, cols in enumerate(valid_col_list):
sum_of_invar = 0
for j, c in enumerate(cols):
sum_of_invar += valid_invar_list[i][j] * row[c]
if sum_of_invar != 0:
label = 1
break
prediction.append(label)
assert len(groundtruth_labels) == len(prediction)
ev.evaluate(groundtruth_labels, prediction)
def anomalyDetect(para, succ_index_list, fail_index_list, train_base_data, train_online_data, testing_data, train_online_label, testing_label, total_inst_num ):
fail_data = train_base_data[fail_index_list,:]
succ_dta = train_base_data[succ_index_list,:]
print(fail_data.shape, succ_dta.shape)
assert fail_data.shape[0] + succ_dta.shape[0] == train_base_data.shape[0]
# clustering
fail_cluster_results, fail_index_per_clu = clustering(para, fail_index_list, fail_data)
print('failure data clustering finished...')
succ_cluster_results, succ_index_per_clu = clustering(para, succ_index_list, succ_dta)
print('success data clustering finished...')
# extract representatives for each cluster of data
dis_sum_list = np.zeros(total_inst_num) # A one dimension list of all zero with size of totalLineNum
fail_repre = extract_repre(train_base_data, fail_index_per_clu, dis_sum_list)
succ_repre = extract_repre(train_base_data, succ_index_per_clu, dis_sum_list)
# online learning
train_base_size = train_base_data.shape[0]
online_learn(para, train_online_data, train_online_label, dis_sum_list, fail_repre, succ_repre, fail_index_per_clu, succ_index_per_clu, train_base_data, train_base_size)
# detect anomalies
predict_label = detect(para, fail_repre, succ_repre, testing_data)
assert len(testing_label) == len(predict_label)
ev.evaluate(testing_label, predict_label)
def anomaly_detection(weigh_data, label_data, C, threshold):
""" detect anomalies by projecting into a subspace with C
Args:
--------
weigh_data: weighted raw data
label_data: the labels list
threshold: used as the threshold, which determines the anomalies
C: the projection matrix
Returns:
--------
"""
print('there are %d anomalies' % (sum(label_data)))
event_num, inst_size = weigh_data.shape
predict_results = np.zeros((inst_size), int)
print('the threshold is %f' % (threshold))
for i in range(inst_size):
ya = np.dot(C, weigh_data[:, i])
SPE = np.dot(ya, ya)
if SPE > threshold:
predict_results[i] = 1 #1 represent failure
assert len(label_data) == len(predict_results)
ev.evaluate(label_data, predict_results)
def main(_run):
args = Namespace(**_run.config)
logger.info(args)
training_data, validation_data, test_data = load_input(args)
model = choose_model(args, training_data, validation_data)
# TEST
evaluate(args, model, training_data, validation_data, test_data)
def evaluate(event_count_matrix, invar_dict, groundtruth_labels):
""" evaluate the results with mined invariants
Args:
--------
event_count_matrix: the input event count matrix
invar_dict: the dictionary of invariants
groundtruth_labels: the groundtruth labels for evaluation
Returns:
--------
"""
print("the mined {} invariants are: {}".format(len(invar_dict),
invar_dict))
valid_col_list = []
valid_invar_list = []
for key in invar_dict:
valid_col_list.append(list(key))
valid_invar_list.append(list(invar_dict[key]))
prediction = []
for row in event_count_matrix:
label = 0
for i, cols in enumerate(valid_col_list):
sum_of_invar = 0
for j, c in enumerate(cols):
sum_of_invar += valid_invar_list[i][j] * row[c]
if sum_of_invar != 0:
label = 1
break
prediction.append(label)
assert len(groundtruth_labels) == len(prediction)
ev.evaluate(groundtruth_labels, prediction)
return prediction
def train_model(model, optimizer, criterion, train_loader, val_loader,
scheduler, grad_acc_steps, metric, exp_path):
n_cycles = len(scheduler.cycle_lens)
best_auc, best_dice, best_cycle = 0, 0, 0
is_better, best_monitoring_metric = compare_op(metric)
for cycle in range(n_cycles):
print('Cycle {:d}/{:d}'.format(cycle + 1, n_cycles))
# prepare next cycle:
# reset iteration counter
scheduler.last_epoch = -1
# update number of iterations
scheduler.T_max = scheduler.cycle_lens[cycle] * len(train_loader)
# train one cycle, retrieve segmentation data and compute metrics at the end of cycle
tr_logits, tr_labels, tr_loss = train_one_cycle(
train_loader, model, criterion, optimizer, scheduler,
grad_acc_steps, cycle)
# classification metrics at the end of cycle
tr_auc, tr_dice = evaluate(
tr_logits, tr_labels,
model.n_classes) # for n_classes>1, will need to redo evaluate
del tr_logits, tr_labels
with torch.no_grad():
assess = True
vl_logits, vl_labels, vl_loss = run_one_epoch(val_loader,
model,
criterion,
assess=assess)
vl_auc, vl_dice = evaluate(
vl_logits, vl_labels,
model.n_classes) # for n_classes>1, will need to redo evaluate
del vl_logits, vl_labels
print(
'Train/Val Loss: {:.4f}/{:.4f} -- Train/Val AUC: {:.4f}/{:.4f} -- Train/Val DICE: {:.4f}/{:.4f} -- LR={:.6f}'
.format(tr_loss, vl_loss, tr_auc, vl_auc, tr_dice, vl_dice,
get_lr(optimizer)).rstrip('0'))
# check if performance was better than anyone before and checkpoint if so
if metric == 'auc':
monitoring_metric = vl_auc
elif metric == 'tr_auc':
monitoring_metric = tr_auc
elif metric == 'loss':
monitoring_metric = vl_loss
elif metric == 'dice':
monitoring_metric = vl_dice
if is_better(monitoring_metric, best_monitoring_metric):
print('Best {} attained. {:.2f} --> {:.2f}'.format(
metric, 100 * best_monitoring_metric, 100 * monitoring_metric))
best_auc, best_dice, best_cycle = vl_auc, vl_dice, cycle + 1
best_monitoring_metric = monitoring_metric
if exp_path is not None:
print(15 * '-', ' Checkpointing ', 15 * '-')
save_model(exp_path, model, optimizer)
del model
torch.cuda.empty_cache()
return best_auc, best_dice, best_cycle
def logsitic_regression(para, train_data, train_labels, testing_data,
testing_labels):
""" train a logistic regression models and evaluate on testing data
Args:
--------
train_data: log sequence matrix for training
train_labels: labels for training
testing_data: log sequence matrix for testing/evaluation
testing_labels: labels for testing/evaluation
Returns:
--------
precision: The evaluation metric Precision
recall: The evaluation metric Recall
f1_score: The evaluation metric F1_score
"""
print("Train a Logistic Regression Model")
clf = LogisticRegression(C=100,
penalty='l1',
tol=0.01,
class_weight='balanced',
multi_class='ovr')
clf = clf.fit(train_data, train_labels.ravel())
prediction = list(clf.predict(testing_data))
assert len(prediction) == len(testing_labels)
if para['cross_validate']:
ev.cv_evaluate(clf, train_data, train_labels)
else:
ev.evaluate(testing_labels, prediction)
def decision_tree(para, train_data, train_labels, testing_data,
testing_labels):
""" train a decision tree models and evaluate on testing data
Args:
--------
train_data: log sequence matrix for training
train_labels: labels for training
testing_data: log sequence matrix for testing/evaluation
testing_labels: labels for testing/evaluation
Returns:
--------
precision: The evaluation metric Precision
recall: The evaluation metric Recall
f1_score: The evaluation metric F1_score
"""
print("Train a Decision Tree Model")
clf = tree.DecisionTreeClassifier()
clf = clf.fit(train_data, train_labels)
prediction = list(clf.predict(testing_data))
assert len(prediction) == len(testing_labels)
if para['cross_validate']:
ev.cv_evaluate(clf, train_data, train_labels)
else:
ev.evaluate(testing_labels, prediction)
def train(model, optimizer, criterion, train_loader, val_loader, n_epochs, patience, decay_f, data_out, n_epochs_gen):
counter_since_checkpoint = 0
tr_losses, tr_aucs, tr_dices, vl_losses, vl_aucs, vl_dices = [], [], [], [], [], []
is_better, best_monitoring_metric = compare_op('auc')
for epoch in range(n_epochs):
print('\n EPOCH: {:d}/{:d}'.format(epoch+1, n_epochs))
# train one epoch
train_logits, train_labels, train_loss = run_one_epoch(train_loader, model, criterion, optimizer)
train_auc, train_dice = evaluate(train_logits, train_labels, model.n_classes)
# validate one epoch, note no optimizer is passed
with torch.no_grad():
val_logits, val_labels, val_loss = run_one_epoch(val_loader, model, criterion)
val_auc, val_dice = evaluate(val_logits, val_labels, model.n_classes)
print('Train/Val Loss: {:.4f}/{:.4f} -- Train/Val AUC: {:.4f}/{:.4f} -- Train/Val DICE: {:.4f}/{:.4f} -- LR={:.6f}'.format(
train_loss, val_loss, train_auc, val_auc, train_dice, val_dice, get_lr(optimizer)).rstrip('0'))
# store performance for this epoch
tr_aucs.append(train_auc)
vl_aucs.append(val_auc)
# smooth val values with a moving average before comparing
val_auc = ewma(vl_aucs)[-1]
# check if performance was better than anyone before and checkpoint if so
monitoring_metric = val_auc
if is_better(monitoring_metric, best_monitoring_metric):
print('Best (smoothed) val {} attained. {:.4f} --> {:.4f}'.format(
'auc', best_monitoring_metric, monitoring_metric))
best_monitoring_metric = monitoring_metric
counter_since_checkpoint = 0 # reset patience
else:
counter_since_checkpoint += 1
if decay_f != 0 and counter_since_checkpoint == 3*patience//4:
reduce_lr(optimizer, epoch, factor=decay_f, verbose=False)
print(8 * '-', ' Reducing LR now ', 8 * '-')
# early stopping if no improvement happened for `patience` epochs
if counter_since_checkpoint == patience:
print('\n Early stopping the training, trained for {:d} epochs'.format(epoch))
del model
torch.cuda.empty_cache()
return
# create examples at this point
if epoch != 0 and (epoch+1) % n_epochs_gen == 0:
print('\n Generating examples for model trained for {:d} epochs'.format(epoch+1))
build_examples(train_loader, model, epoch+1, data_out)
build_examples(val_loader, model, epoch+1, data_out)
del model
torch.cuda.empty_cache()
return
def test_trained_model():
print("Loading the best trained model")
saved_obj = torch.load(cfg.checkpoint_name,
map_location=lambda storage, loc: storage)
model = saved_obj['model'].to(device)
model.beam_search_decoding = True
model.beam_size = int(cfg.beam_size)
model.beam_search_length_norm_factor = float(
cfg.beam_search_length_norm_factor)
model.beam_search_coverage_penalty_factor = float(
cfg.beam_search_coverage_penalty_factor)
# SRC = saved_obj['field_src']
TGT = saved_obj['field_tgt']
evaluate(val_iter, TGT, model, src_val_file_address, tgt_val_file_address,
"VALIDATE")
evaluate(test_iter, TGT, model, src_test_file_address,
tgt_test_file_address, "TEST")
def test(model,params):
model.eval()
test_output,test_target= get_predictions(model, params, split = 'test')
if params.label == 'emotion':
test_output = test_output.reshape_as(test_target)
performances = evaluate(params,test_output,test_target)
return performances
def test_trained_model():
print("Loading the best trained model")
saved_obj = torch.load("../.checkpoints/"+cfg.checkpoint_name, map_location=lambda storage, loc: storage)
model = saved_obj['model'].to(device)
if 'training_evaluation_results' in saved_obj:
print("Comma separated greedy decoding validation set BlueP1 scores collected during training:\n\t ===> {}".format(
",".join(["{:.2f}".format(x) for x in saved_obj['training_evaluation_results']])))
model.beam_search_decoding = True
model.beam_size = int(cfg.beam_size)
model.beam_search_length_norm_factor = float(cfg.beam_search_length_norm_factor)
model.beam_search_coverage_penalty_factor = float(cfg.beam_search_coverage_penalty_factor)
SRC = saved_obj['field_src']
TGT = saved_obj['field_tgt']
print("Model loaded, total number of parameters: {}".format(sum([p.numel() for p in model.parameters()])))
dp = DataProvider(SRC, TGT, load_train_data=False)
nuance = str(int(time.time()))
evaluate(dp.val_iter, dp, model, dp.processed_data.addresses.val.src, dp.processed_data.addresses.val.tgt,
"VALID.{}".format(dp.val_iter.dataset.name), save_decoded_sentences=True, nuance=nuance)
for test_iter, s, t in zip(dp.test_iters, dp.processed_data.addresses.tests.src, dp.processed_data.addresses.tests.tgt):
evaluate(test_iter, dp, model, s, t, "TEST.{}".format(test_iter.dataset.name), save_decoded_sentences=True, nuance=nuance)
def train_cross_val(KFolds, config, data, metrics, metrics_CV2, results, results_CV2):
# Set random seed
if (config.seed != None) and isinstance(config.seed, int):
tf.set_random_seed(config.seed)
np.random.seed(config.seed)
# Ensure that the model will fit in GPU memory
good_batch_size = False
# Check whether weight files exist before starting, if evaluate flag is active
if (config.evaluate): check_weights_exist(config, KFolds)
for fold in range(len(KFolds)):
# Keep track of training time
t_start = time.time()
(train_keys, test_keys) = KFolds[fold]
# (train_keys, valid_keys) = shuffle_split_array(train_keys, config.val_split)
# Retrieve fold information
# execinfo = FoldKeys(train_keys=train_keys, valid_keys=valid_keys, test_keys=test_keys)
execinfo = ExecutionInformation(config, fold, train_keys, test_keys, config.evaluate)
# Train a single fold
good_batch_size = train_fold(config, data, execinfo, metrics, metrics_CV2, results, results_CV2, good_batch_size, True)
# Check total training time
time_elapsed = time.time() - t_start
print('\n * Training complete in {:.0f}m {:.0f}s\n'.format(time_elapsed // 60, time_elapsed % 60))
# Retrieve patient ID from lead data
IDs = np.asarray(list(set(train_keys.tolist() + test_keys.tolist()))) # Avoid duplicates
# Sanity check
if len(IDs) != len(train_keys) + len(test_keys):
raise ValueError("Some record shared between train and test!")
# Evaluation of whole dataset
execinfo = ExecutionInformation(config, None, None, IDs, True)
evaluate(None, config, data, execinfo, metrics, metrics_CV2, results, results_CV2, False)
def train_fold(config, data, execinfo, metrics, metrics_CV2, results, results_CV2, good_batch_size, recompute):
############################# MODEL CREATION ############################
# Clear cache
keras.backend.clear_session()
# Define the model according to the configuration
# if config.strategy == 'single': model = arch.FlatNet(config).create_model()
# else: model = arch2D.FlatNet(config).create_model()
model = arch.FlatNet(config).create_model()
# Select optimizer and loss + compile
optim = select_optimizer(config.optimizer.lower())(lr=config.learning_rate)
loss = select_loss(config.loss.lower())
model.compile(optimizer=optim, loss=loss)
write_summary(execinfo.summary, model)
# If chosen and exists, load weights (fine-tuning, etc.)
if (config.load_weights or config.evaluate) and os.path.exists(execinfo.state):
model.load_weights(execinfo.state)
# If the flag to evaluate has not been set, train the model
if not config.evaluate:
# Data generators
GeneratorTrain = DataGenerator(execinfo.train, config, data)
GeneratorValid = DataGenerator(execinfo.valid, config, data)
# keras-specific train
good_batch_size = train_epochs(config, model, data, execinfo, results, results_CV2, good_batch_size, GeneratorTrain, GeneratorValid)
# Evaluate model
if config.splitting.lower() == "cross_validation":
evaluate(model, config, data, execinfo, metrics, metrics_CV2, results, results_CV2, recompute)
else:
if config.splitting.lower() == "cross_validation":
evaluate(model, config, data, execinfo, metrics, metrics_CV2, results, results_CV2, recompute)
return good_batch_size
def train_all(IDs, config, data, metrics, metrics_CV2, results, results_CV2):
# Set random seed
if (config.seed != None) and isinstance(config.seed, int):
tf.set_random_seed(config.seed)
np.random.seed(config.seed)
# Keep track of training time
t_start = time.time()
# "iif" evaluate, test on whole database: useful for testing on other DB's
if config.evaluate:
# Define the model according to the configuration
# if config.strategy == 'single': model = arch.FlatNet(config).create_model()
# else: model = arch2D.FlatNet(config).create_model()
model = arch.FlatNet(config).create_model()
# Select optimizer and loss + compile
optim = select_optimizer(config.optimizer.lower())(lr=config.learning_rate)
loss = select_loss(config.loss.lower())
model.compile(optimizer=optim, loss=loss)
execinfo = ExecutionInformation(config, None, None, IDs, True) # Define execution
evaluate(model, config, data, execinfo, metrics, metrics_CV2, results, results_CV2, True) # Evaluate
else:
# Retrieve execution information
execinfo = ExecutionInformation(config, None, IDs, None, False)
# Check whether weight files exist before starting, if evaluate flag is active
if (config.evaluate) and (not os.path.exists(execinfo.state)):
raise FileNotFoundError("Weights file not found")
# Train a single ""fold""
_ = train_fold(config, data, execinfo, metrics, metrics_CV2, results, results_CV2, False, True)
# Check total training time
time_elapsed = time.time() - t_start
print('\n * Training complete in {:.0f}m {:.0f}s\n'.format(time_elapsed // 60, time_elapsed % 60))
def test(model, params):
model.eval()
test_output, test_target, test_mask = get_predictions(model,
params,
split='test')
test_target = torch.argmax(test_target.reshape(-1, params.output_dim), -1)
test_output = test_output.reshape(-1, params.output_dim)
if not params.dialogue_context:
nonzero_idx = test_mask.view(-1).nonzero()[:, 0]
test_output = test_output[nonzero_idx]
test_target = test_target[nonzero_idx]
performances = evaluate(params, test_output, test_target)
return performances
def on_epoch_end(self, epoch, logs=None):
self.keras_ema.assign_shadow_weights()
logits1, logits2, _, _ = self.model.predict(
x=[
dev_data['context_id'], dev_data['question_id'],
dev_data['context_char_id'], dev_data['question_char_id']
],
batch_size=config.batch_size,
verbose=1)
all_results = []
for i, qid in enumerate(dev_data['qid']):
start_logits = logits1[i, :]
end_logits = logits2[i, :]
all_results.append(
RawResult(qid=qid,
start_logits=start_logits,
end_logits=end_logits))
output_prediction_file = os.path.join(config.path,
'output_prediction.json')
output_nbest_file = os.path.join(config.path, 'output_nbest.json')
write_predictions(eval_examples,
eval_features,
all_results,
n_best_size=20,
max_answer_length=config.ans_limit,
do_lower_case=False,
output_prediction_file=output_prediction_file,
output_nbest_file=output_nbest_file)
metrics = evaluate('original_data/dev-v1.1.json',
output_prediction_file, None)
ems.append(metrics['exact'])
f1s.append(metrics['f1'])
result = pd.DataFrame([ems, f1s], index=['em', 'f1']).transpose()
result.to_csv('logs/result_' + config.name + '.csv', index=None)
if f1s[-1] > self.max_f1:
self.max_f1 = f1s[-1]
model.save_weights(
os.path.join(config.path,
'QANet_model_' + config.name + '.h5'))
model.load_weights(self.keras_ema.temp_model)
def run(qf, gf, q_pids, g_pids, q_camids, g_camids, t=5, device=None):
"""
:param qf: torch.Tensor(q, m)
:param gf: torch.Tensor(g, m)
:param q_pids: torch.Tensor(q)
:param g_pids: torch.Tensor(g)
:param q_camids: torch.Tensor(q)
:param g_camids: torch.Tensor(g)
:return:
"""
# These will be initialized automatically
positive_indices = None
negative_indices = None
distmat = None
for _ in tqdm(range(t)):
res = rocchio_round(qf, gf, q_pids, g_pids, positive_indices, negative_indices,
previous_distmat=distmat, device=device)
distmat, positive_indices, negative_indices, matches = res
del matches
result = evaluate(distmat, q_pids, g_pids, q_camids, g_camids, device=device)
print("Results after {} rounds of Rocchio:".format(t), "mAP", result[1], "mINP", result[2])
return distmat
def run(qf,
gf,
q_pids,
g_pids,
q_camids,
g_camids,
t=5,
method="min",
device=None):
positive_indices = None
negative_indices = None
distmat = None
distmat_qg = None
for _ in tqdm(range(t)):
res = ne_round(qf,
gf,
q_pids,
g_pids,
positive_indices,
negative_indices,
distmat,
distmat_qg,
method=method,
verbose=0,
device=device)
distmat, positive_indices, negative_indices, distmat_qg, matches = res
del matches
del res, distmat_qg, positive_indices, negative_indices
result = evaluate(distmat,
q_pids,
g_pids,
q_camids,
g_camids,
device=device)
print(
"Results after {} rounds of neighborhood expansion ({}):".format(
t, method), "mAP", result[1], "mINP", result[2])
return distmat
# Select a subset 100 samples (10 for each per label)
train_subset = split(train_set, 100, 'Balanced')
##########
checkpoint_count = 1
for epoch in tqdm(range(epochs)):
for data in tqdm(train_loader):
trainer.train_step(data)
if epoch % evaluate_every == 0:
# Compute train and test_accuracy of a logistic regression
train_accuracy, test_accuracy = evaluate(encoder=trainer.encoder,
train_on=train_subset,
test_on=test_set,
device=device)
if not (writer is None):
writer.add_scalar(tag='evaluation/train_accuracy',
scalar_value=train_accuracy,
global_step=trainer.iterations)
writer.add_scalar(tag='evaluation/test_accuracy',
scalar_value=test_accuracy,
global_step=trainer.iterations)
tqdm.write('Train Accuracy: %f' % train_accuracy)
tqdm.write('Test Accuracy: %f' % test_accuracy)
if epoch % checkpoint_every == 0:
tqdm.write('Storing model checkpoint')
while os.path.isfile(
def train(conf, _model):
if conf['rand_seed'] is not None:
np.random.seed(conf['rand_seed'])
if not os.path.exists(conf['save_path']):
os.makedirs(conf['save_path'])
# load data
print('starting loading data')
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
train_data, val_data, test_data = pickle.load(open(conf["data_path"],
'rb'))
print('finish loading data')
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
val_batches = reader.build_batches(val_data, conf)
print("finish building test batches")
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
# refine conf
batch_num = len(train_data['y']) / conf["batch_size"]
val_batch_num = len(val_batches["response"])
conf["train_steps"] = conf["num_scan_data"] * batch_num
conf["save_step"] = int(max(1, batch_num / 10))
conf["print_step"] = int(max(1, batch_num / 100))
print('configurations: %s' % conf)
print('model sucess')
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
_graph = _model.build_graph()
print('build graph sucess')
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
with tf.Session(graph=_graph) as sess:
_model.init.run()
if conf["init_model"]:
_model.saver.restore(sess, conf["init_model"])
print("sucess init %s" % conf["init_model"])
average_loss = 0.0
batch_index = 0
step = 0
best_result = [0, 0, 0, 0]
for step_i in xrange(conf["num_scan_data"]):
#for batch_index in rng.permutation(range(batch_num)):
print('starting shuffle train data')
shuffle_train = reader.unison_shuffle(train_data)
train_batches = reader.build_batches(shuffle_train, conf)
print('finish building train data')
for batch_index in range(batch_num):
feed = {
_model.turns:
train_batches["turns"][batch_index],
_model.tt_turns_len:
train_batches["tt_turns_len"][batch_index],
_model.every_turn_len:
train_batches["every_turn_len"][batch_index],
_model.response:
train_batches["response"][batch_index],
_model.response_len:
train_batches["response_len"][batch_index],
_model.label:
train_batches["label"][batch_index]
}
batch_index = (batch_index + 1) % batch_num
_, curr_loss = sess.run([_model.g_updates, _model.loss],
feed_dict=feed)
average_loss += curr_loss
step += 1
if step % conf["print_step"] == 0 and step > 0:
g_step, lr = sess.run(
[_model.global_step, _model.learning_rate])
print('step: %s, lr: %s' % (g_step, lr))
print("processed: [" + str(step * 1.0 / batch_num) +
"] loss: [" +
str(average_loss / conf["print_step"]) + "]")
average_loss = 0
if step % conf["save_step"] == 0 and step > 0:
index = step / conf['save_step']
score_file_path = conf['save_path'] + 'score.' + str(index)
score_file = open(score_file_path, 'w')
print('save step: %s' % index)
print(
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time())))
for batch_index in xrange(val_batch_num):
feed = {
_model.turns:
val_batches["turns"][batch_index],
_model.tt_turns_len:
val_batches["tt_turns_len"][batch_index],
_model.every_turn_len:
val_batches["every_turn_len"][batch_index],
_model.response:
val_batches["response"][batch_index],
_model.response_len:
val_batches["response_len"][batch_index],
_model.label:
val_batches["label"][batch_index]
}
scores = sess.run(_model.logits, feed_dict=feed)
for i in xrange(conf["batch_size"]):
score_file.write(
str(scores[i]) + '\t' +
str(val_batches["label"][batch_index][i]) +
'\n')
score_file.close()
#write evaluation result
result = eva.evaluate(score_file_path)
result_file_path = conf["save_path"] + "result." + str(
index)
with open(result_file_path, 'w') as out_file:
for p_at in result:
out_file.write(str(p_at) + '\n')
print('finish evaluation')
print(
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time())))
if result[1] + result[2] > best_result[1] + best_result[2]:
best_result = result
_save_path = _model.saver.save(
sess, conf["save_path"] + "model.ckpt." +
str(step / conf["save_step"]))
print("succ saving model in " + _save_path)
print(
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time())))
def run(args):
if args.gpu_id:
setup_gpu(args.gpu_id, args.verbose)
seed = args.seed or np.random.randint(1000)
# documentation setup
timestamp = args.timestamp or datetime.now().strftime("%Y%m%d%H%M%S")
outputs_dir = Path(
__file__
).parent / 'runs' / args.method / args.experiment_id / '{}{}_{}_{}'.format(
args.source, args.target, seed, timestamp)
checkpoints_dir = outputs_dir / 'checkpoints'
checkpoints_dir.mkdir(parents=True, exist_ok=True)
checkpoints_path = checkpoints_dir / 'cp-best.ckpt'
tensorboard_dir = outputs_dir / 'logs'
tensorboard_dir.mkdir(parents=True, exist_ok=True)
config_path = outputs_dir / 'config.json'
model_path = outputs_dir / 'model.json'
report_path = outputs_dir / 'report.json'
report_val_path = outputs_dir / 'report_validation.json'
save_json(args.__dict__, config_path)
features_config = load_json(Path('configs/features.json').absolute())
# prepare data
preprocess_input = {
'vgg16':
lambda x: keras.applications.vgg16.preprocess_input(x, mode='tf'),
'resnet101v2':
lambda x: keras.applications.resnet_v2.preprocess_input(
x, mode='tf'
), #NB: tf v 1.15 has a minor bug in keras_applications.resnet. Fix: change the function signature to "def preprocess_input(x, **kwargs):""
'none':
lambda x: x[features_config[args.features]["mat_key"]],
**{k: lambda x: x
for k in ['conv2', 'lenetplus']},
}[args.model_base] or None
if all([
name in dsg.OFFICE_DATASET_NAMES
for name in [args.source, args.target]
]):
# office data
INPUT_SHAPE = tuple(features_config[args.features]["shape"])
CLASS_NAMES = dsg.office31_class_names()
OUTPUT_SHAPE = len(CLASS_NAMES)
ds = dsg.office31_datasets(source_name=args.source,
target_name=args.target,
preprocess_input=preprocess_input,
shape=INPUT_SHAPE,
seed=seed,
features=args.features,
test_as_val=args.test_as_val)
elif all([
name in dsg.DIGIT_DATASET_NAMES
for name in [args.source, args.target]
]):
INPUT_SHAPE = dsg.digits_shape(args.source,
args.target,
mode=args.resize_mode)
CLASS_NAMES = dsg.digits_class_names()
OUTPUT_SHAPE = len(CLASS_NAMES)
ds = dsg.digits_datasets(
source_name=args.source,
target_name=args.target,
num_source_samples_per_class=args.num_source_samples_per_class,
num_target_samples_per_class=args.num_target_samples_per_class,
num_val_samples_per_class=args.num_val_samples_per_class,
seed=seed,
test_as_val=args.test_as_val,
input_shape=INPUT_SHAPE,
standardize_input=args.standardize_input,
)
else:
raise Exception(
"The source and target datasets should come from either Office31 or Digits"
)
source_all_ds, source_all_size = ds['source']['full']
source_train_ds, source_train_size = ds['source']['train']
target_train_ds, target_train_size = ds['target']['train']
target_val_ds, target_val_size = ds['target']['val']
target_test_ds, target_test_size = ds['target']['test']
test_size = target_test_size
if args.test_as_val:
target_val_ds = target_test_ds
target_val_size = target_test_size
val_ds, val_size = {
**{
k: lambda: (target_val_ds, target_val_size)
for k in ['tune_source', 'tune_target']
},
**{
k: lambda: dsg.da_pair_repeat_dataset(target_val_ds, target_val_size)
for k in ['ccsa', 'dsne', 'dage', 'multitask']
},
**{
k: lambda: dsg.da_pair_alt_repeat_dataset(target_val_ds, target_val_size)
for k in ['dage_a']
},
}[args.method]()
train_ds, train_size = {
'tune_source': lambda: (source_all_ds, source_all_size),
'tune_target': lambda: (target_train_ds, target_train_size),
**{
k: lambda: dsg.da_pair_dataset(source_ds=source_train_ds,
target_ds=target_train_ds,
ratio=args.ratio,
shuffle_buffer_size=args.shuffle_buffer_size)
for k in ['ccsa', 'dsne', 'dage', 'multitask']
},
**{
k: lambda: dsg.da_pair_alt_dataset(source_ds=source_train_ds,
target_ds=target_train_ds,
ratio=args.ratio,
shuffle_buffer_size=args.shuffle_buffer_size)
for k in ['dage_a']
},
}[args.method]()
# prep data
test_ds = dsg.prep_ds(dataset=target_test_ds,
batch_size=args.batch_size,
shuffle_buffer_size=args.shuffle_buffer_size)
val_ds = dsg.prep_ds(dataset=val_ds,
batch_size=args.batch_size,
shuffle_buffer_size=args.shuffle_buffer_size)
train_ds = dsg.prep_ds_train(dataset=train_ds,
batch_size=args.batch_size,
shuffle_buffer_size=args.shuffle_buffer_size)
# prepare optimizer
optimizer = {
'sgd':
lambda: keras.optimizers.SGD(learning_rate=args.learning_rate,
momentum=args.momentum,
nesterov=True,
clipvalue=10,
decay=args.learning_rate_decay),
'adam':
lambda: keras.optimizers.Adam(learning_rate=args.learning_rate,
beta_1=args.momentum,
beta_2=0.999,
amsgrad=False,
clipvalue=10,
decay=args.learning_rate_decay),
'rmsprop':
lambda: keras.optimizers.RMSprop(learning_rate=args.learning_rate,
clipvalue=10,
decay=args.learning_rate_decay),
}[args.optimizer]()
# prepare model
model_base = {
'vgg16':
lambda: keras.applications.vgg16.VGG16(
input_shape=INPUT_SHAPE, include_top=False, weights='imagenet'),
'resnet101v2':
lambda: keras.applications.resnet_v2.ResNet101V2(
input_shape=INPUT_SHAPE, include_top=False, weights='imagenet'),
'conv2':
lambda: models.common.conv2_block(input_shape=INPUT_SHAPE,
l2=args.l2,
dropout=args.dropout / 2,
batch_norm=args.batch_norm),
'lenetplus':
lambda: models.common.lenetplus_conv_block(input_shape=INPUT_SHAPE,
l2=args.l2,
dropout=args.dropout / 2,
batch_norm=args.batch_norm),
'none':
lambda i=keras.layers.Input(shape=INPUT_SHAPE): keras.models.Model(
inputs=i, outputs=i),
}[args.model_base]()
aux_loss = {
**{
k: lambda: losses.dummy_loss
for k in ['dummy', 'tune_source', 'tune_target', 'multitask']
},
'ccsa':
lambda: losses.contrastive_loss(margin=args.connection_filter_param),
'dsne':
lambda: losses.dnse_loss(margin=args.connection_filter_param),
'dage':
lambda: losses.dage_loss(
connection_type=args.connection_type,
weight_type=args.weight_type,
filter_type=args.connection_filter_type,
penalty_filter_type=args.penalty_connection_filter_type,
filter_param=args.connection_filter_param,
penalty_filter_param=args.penalty_connection_filter_param),
}[args.method]()
(model, model_test) = {
'single_stream':
lambda: models.single_stream.model(model_base=model_base,
input_shape=INPUT_SHAPE,
output_shape=OUTPUT_SHAPE,
num_unfrozen_base_layers=args.
num_unfrozen_base_layers,
optimizer=optimizer,
dense_size=args.dense_size,
embed_size=args.embed_size,
l2=args.l2,
dropout=args.dropout),
'two_stream_pair_embeds':
lambda: models.two_stream_pair_embeds.model(
model_base=model_base,
input_shape=INPUT_SHAPE,
output_shape=OUTPUT_SHAPE,
num_unfrozen_base_layers=args.num_unfrozen_base_layers,
dense_size=args.dense_size,
embed_size=args.embed_size,
optimizer=optimizer,
batch_size=args.batch_size,
aux_loss=aux_loss,
loss_alpha=args.loss_alpha,
loss_weights_even=args.loss_weights_even,
l2=args.l2,
batch_norm=args.batch_norm,
dropout=args.dropout),
}[args.architecture]()
val_freq = 3 if args.test_as_val else 1
train = {
'regular':
partial(models.common.train,
checkpoints_path=checkpoints_path,
val_freq=val_freq),
'flipping':
partial(models.common.train,
checkpoints_path=checkpoints_path,
val_freq=val_freq,
flipping=True),
'batch_repeat':
partial(models.common.train,
checkpoints_path=checkpoints_path,
batch_repeats=args.batch_repeats),
'gradual_unfreeze':
partial(models.common.train_gradual_unfreeze,
model_base_name=args.model_base,
checkpoints_path=checkpoints_path,
architecture=args.architecture),
}[args.training_regimen]
if args.from_weights:
weights_path = args.from_weights
model.load_weights(str(weights_path))
if args.verbose:
model.summary()
# keras.utils.plot_model( #type: ignore
# model,
# to_file=(Path(__file__).parent /'model.png').absolute(),
# show_shapes=True,
# show_layer_names=True,
# rankdir='TB',
# expand_nested=True,
# dpi=96
# )
with open(model_path, 'w') as f:
f.write(model.to_json())
monitor = {
**{k: 'val_'
for k in ['tune_source', 'tune_target']},
**{
k: 'val_preds_'
for k in ['ccsa', 'dsne', 'dage', 'dage_a', 'multitask']
},
}[args.method] + args.monitor
fit_callbacks = callbacks(checkpoints_path,
tensorboard_dir,
monitor=monitor,
verbose=args.verbose)
augment = lambda x: x
if args.augment:
if args.features != 'images':
raise ValueError('augment=1 is only allowed for features="images"')
augment = {
**{
k: partial(dsg.augment,
batch_size=args.batch_size,
input_shape=INPUT_SHAPE)
for k in ['tune_source', 'tune_target']
},
**{
k: partial(dsg.augment_pair,
batch_size=args.batch_size,
input_shape=INPUT_SHAPE)
for k in ['ccsa', 'dsne', 'dage', 'dage_a', 'multitask']
},
}[args.method]
# perform training and test
if 'train' in args.mode:
start_time = timer()
train(model=model,
datasource=augment(train_ds),
datasource_size=train_size,
val_datasource=val_ds,
val_datasource_size=val_size,
epochs=args.epochs,
batch_size=args.batch_size,
callbacks=fit_callbacks,
verbose=args.verbose)
train_time = timer() - start_time
if args.verbose:
print("Completed training in {} seconds".format(train_time))
result = 0
if 'validate' in args.mode:
result = evaluate(model=model_test,
test_dataset=val_ds,
test_size=val_size,
batch_size=args.batch_size,
report_path=report_val_path,
verbose=args.verbose,
target_names=CLASS_NAMES)
if 'test' in args.mode:
result = evaluate(model=model_test,
test_dataset=test_ds,
test_size=test_size,
batch_size=args.batch_size,
report_path=report_path,
verbose=args.verbose,
target_names=CLASS_NAMES)
if args.delete_checkpoint:
try:
rmtree(str(checkpoints_dir.resolve()))
except:
pass
return result['accuracy'] # type:ignore
def train(params, model):
criterion = get_criterion(params)
#unitary_parameters = get_unitary_parameters(model)
if hasattr(model, 'get_params'):
unitary_params, remaining_params = model.get_params()
else:
remaining_params = model.parameters()
unitary_params = []
if len(unitary_params) > 0:
unitary_optimizer = RMSprop_Unitary(unitary_params,
lr=params.unitary_lr)
#remaining_parameters = get_remaining_parameters(model,unitary_parameters)
optimizer = torch.optim.RMSprop(remaining_params, lr=params.lr)
# Temp file for storing the best model
temp_file_name = str(int(np.random.rand() * int(time.time())))
params.best_model_file = os.path.join('tmp', temp_file_name)
best_val_loss = 99999.0
# best_val_loss = -1.0
for i in range(params.epochs):
print('epoch: ', i)
model.train()
with tqdm(total=params.train_sample_num) as pbar:
time.sleep(0.05)
for _i, data in enumerate(
params.reader.get_data(iterable=True,
shuffle=True,
split='train'), 0):
# For debugging, please run the line below
# _i,data = next(iter(enumerate(params.reader.get_data(iterable = True, shuffle = True, split = 'train'),0)))
b_inputs = [inp.to(params.device) for inp in data[:-1]]
b_targets = data[-1].to(params.device)
# Does not train if batch_size is 1, because batch normalization will crash
if b_inputs[0].shape[0] == 1:
continue
optimizer.zero_grad()
if len(unitary_params) > 0:
unitary_optimizer.zero_grad()
outputs = model(b_inputs)
b_targets, outputs, loss = get_loss(params, criterion, outputs,
b_targets, b_inputs[-1])
if np.isnan(loss.item()):
torch.save(model, params.best_model_file)
raise Exception('loss value overflow!')
#break
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), params.clip)
optimizer.step()
if len(unitary_params) > 0:
unitary_optimizer.step()
# Compute Training Accuracy
n_total = len(outputs)
n_correct = (outputs.argmax(dim=-1) == b_targets).sum().item()
train_acc = n_correct / n_total
#Update Progress Bar
pbar.update(params.batch_size)
ordered_dict = {'acc': train_acc, 'loss': loss.item()}
pbar.set_postfix(ordered_dict=ordered_dict)
model.eval()
#################### Compute Validation Performance##################
val_output, val_target, val_mask = get_predictions(model,
params,
split='dev')
val_target, val_output, val_loss = get_loss(params, criterion,
val_output, val_target,
val_mask)
print('validation performance:')
performances = evaluate(params, val_output, val_target)
print('val_acc = {}, val_loss = {}'.format(performances['acc'],
val_loss))
##################################################################
if val_loss < best_val_loss:
torch.save(model, params.best_model_file)
print('The best model up till now. Saved to File.')
best_val_loss = val_loss
def experiment_vae(args, train_loader, val_loader, test_loader, model, optimizer, dir, model_name='vae'):
from utils.training import train_vae as train
from utils.evaluation import evaluate_vae as evaluate
# SAVING
torch.save(args, dir + args.model_name + '.config')
# best_model = model
best_loss = 100000.
e = 0
train_loss_history = []
train_re_history = []
train_kl_history = []
val_loss_history = []
val_re_history = []
val_kl_history = []
time_history = []
for epoch in range(1, args.epochs + 1):
time_start = time.time()
model, train_loss_epoch, train_re_epoch, train_kl_epoch = train(epoch, args, train_loader, model,
optimizer)
val_loss_epoch, val_re_epoch, val_kl_epoch = evaluate(args, model, train_loader, val_loader, epoch, dir, mode='validation')
time_end = time.time()
time_elapsed = time_end - time_start
# appending history
train_loss_history.append(train_loss_epoch), train_re_history.append(train_re_epoch), train_kl_history.append(
train_kl_epoch)
val_loss_history.append(val_loss_epoch), val_re_history.append(val_re_epoch), val_kl_history.append(
val_kl_epoch)
time_history.append(time_elapsed)
# printing results
print('Epoch: {}/{}, Time elapsed: {:.2f}s\n'
'* Train loss: {:.2f} (RE: {:.2f}, KL: {:.2f})\n'
'o Val. loss: {:.2f} (RE: {:.2f}, KL: {:.2f})\n'
'--> Early stopping: {}/{} (BEST: {:.2f})\n'.format(
epoch, args.epochs, time_elapsed,
train_loss_epoch, train_re_epoch, train_kl_epoch,
val_loss_epoch, val_re_epoch, val_kl_epoch,
e, args.early_stopping_epochs, best_loss
))
# early-stopping
if val_loss_epoch < best_loss:
e = 0
best_loss = val_loss_epoch
# best_model = model
print('->model saved<-')
torch.save(model, dir + args.model_name + '.model')
else:
e += 1
if epoch < args.warmup:
e = 0
if e > args.early_stopping_epochs:
break
# NaN
if math.isnan(val_loss_epoch):
break
# FINAL EVALUATION
best_model = torch.load(dir + args.model_name + '.model')
test_loss, test_re, test_kl, test_log_likelihood, train_log_likelihood, test_elbo, train_elbo = evaluate(args, best_model, train_loader, test_loader, 9999, dir, mode='test')
print('FINAL EVALUATION ON TEST SET\n'
'LogL (TEST): {:.2f}\n'
'LogL (TRAIN): {:.2f}\n'
'ELBO (TEST): {:.2f}\n'
'ELBO (TRAIN): {:.2f}\n'
'Loss: {:.2f}\n'
'RE: {:.2f}\n'
'KL: {:.2f}'.format(
test_log_likelihood,
train_log_likelihood,
test_elbo,
train_elbo,
test_loss,
test_re,
test_kl
))
with open('vae_experiment_log.txt', 'a') as f:
print('FINAL EVALUATION ON TEST SET\n'
'LogL (TEST): {:.2f}\n'
'LogL (TRAIN): {:.2f}\n'
'ELBO (TEST): {:.2f}\n'
'ELBO (TRAIN): {:.2f}\n'
'Loss: {:.2f}\n'
'RE: {:.2f}\n'
'KL: {:.2f}'.format(
test_log_likelihood,
train_log_likelihood,
test_elbo,
train_elbo,
test_loss,
test_re,
test_kl
), file=f)
# SAVING
torch.save(train_loss_history, dir + args.model_name + '.train_loss')
torch.save(train_re_history, dir + args.model_name + '.train_re')
torch.save(train_kl_history, dir + args.model_name + '.train_kl')
torch.save(val_loss_history, dir + args.model_name + '.val_loss')
torch.save(val_re_history, dir + args.model_name + '.val_re')
torch.save(val_kl_history, dir + args.model_name + '.val_kl')
torch.save(test_log_likelihood, dir + args.model_name + '.test_log_likelihood')
torch.save(test_loss, dir + args.model_name + '.test_loss')
torch.save(test_re, dir + args.model_name + '.test_re')
torch.save(test_kl, dir + args.model_name + '.test_kl')
def train(conf, _model):
if conf['rand_seed'] is not None:
np.random.seed(conf['rand_seed'])
if not os.path.exists(conf['save_path']):
os.makedirs(conf['save_path'])
# load data
print('starting loading data')
print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
train_data, val_data, test_data = pickle.load(open(conf["data_path"], 'rb'))
print('train:', len(train_data['y']))
print('dev:', len(val_data['y']))
print('test:', len(test_data['y']))
print('finish loading data')
print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
val_batches = reader.build_batches('train',val_data, conf)
print("finish building test batches")
print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
# refine conf
batch_num = int(len(train_data['y']) / conf["batch_size"])
val_batch_num = len(val_batches["response"])
conf["train_steps"] = conf["num_scan_data"] * batch_num
conf["save_step"] = int(max(1, batch_num / 10))
conf["print_step"] = int(max(1, batch_num / 100))
print('configurations: %s' %conf)
print('begin build model')
print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
_graph = _model.build_graph()
print('build graph sucess')
print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
with tf.Session(graph=_graph) as sess:
#writer = tf.summary.FileWriter("logs/", sess.graph) # for tensorboard
# summary writer
'''
train_summary_dir = os.path.join(conf["save_path"], "summaries", "train")
train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph)
dev_summary_dir = os.path.join(conf["save_path"], "summaries", "dev")
dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph)
'''
_model.init.run()
if not conf["init_model"]:
emb_feed = {_model.emb_placeholder: _model._word_embedding_init}
sess.run(_model.emb_init, feed_dict=emb_feed)
if conf["init_model"]:
_model.saver.restore(sess, conf["init_model"])
print("sucess init %s" %conf["init_model"])
average_loss = 0.0
batch_index = 0
step = 0
best_result = [0, 0, 0, 0]
#best_result = [0, 0, 0, 0, 0, 0] # eva matrix: p1(2),p1(10),p2(10),p5(10)
for step_i in range(conf["num_scan_data"]): # each epoch
print('starting shuffle train data')
shuffle_train = reader.unison_shuffle(train_data)
train_batches = reader.build_batches('train',shuffle_train, conf)
print('finish building train data')
for batch_index in range(batch_num): # each batch
feed = {
_model.turns: train_batches["turns"][batch_index],
_model.tt_turns_len: train_batches["tt_turns_len"][batch_index],
_model.every_turn_len: train_batches["every_turn_len"][batch_index],
_model.response: train_batches["response"][batch_index],
_model.response_len: train_batches["response_len"][batch_index],
_model.label: train_batches["label"][batch_index],
_model.dropout_keep_prob: conf["dropout_keep_prob"]
}
batch_index = (batch_index + 1) % batch_num;
_, curr_loss, summaries = sess.run([_model.g_updates, _model.loss, _model.train_summary_op], feed_dict = feed)
# summary
#train_summary_writer.add_summary(summaries, step)
average_loss += curr_loss
step += 1
if step % conf["print_step"] == 0 and step > 0:
g_step, lr = sess.run([_model.global_step, _model.learning_rate])
print('step: %s, lr: %s' %(g_step, lr))
print("processed: [" + str(step * 1.0 / batch_num) + "] loss: [" + str(average_loss / conf["print_step"]) + "]")
average_loss = 0
if step % conf["save_step"] == 0 and step > 0:
index = step / conf['save_step']
score_file_path = conf['save_path'] + 'score.' + str(index)
score_file = open(score_file_path, 'w')
print('save step: %s' %index)
print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
for batch_index in range(val_batch_num):
feed = {
_model.turns: val_batches["turns"][batch_index],
_model.tt_turns_len: val_batches["tt_turns_len"][batch_index],
_model.every_turn_len: val_batches["every_turn_len"][batch_index],
_model.response: val_batches["response"][batch_index],
_model.response_len: val_batches["response_len"][batch_index],
_model.label: val_batches["label"][batch_index],
_model.dropout_keep_prob: 1.0
}
scores, dev_loss, summaries = sess.run([_model.logits, _model.loss, _model.dev_summary_op], feed_dict = feed)
# summary
#dev_summary_writer.add_summary(summaries, step)
for i in range(len(scores)): # logit, true_label
score_file.write(
str(scores[i]) + '\t' +
str(val_batches["label"][batch_index][i]) + '\n')
score_file.close()
#write evaluation result
result = eva.evaluate(score_file_path)
result_file_path = conf["save_path"] + "result." + str(index)
with open(result_file_path, 'w') as out_file:
for p_at in result:
out_file.write(str(p_at) + '\n')
print('finish evaluation')
print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
if result[1] + result[2] > best_result[1] + best_result[2]: # for ubuntu
#if result[2] + result[3] > best_result[2] + best_result[3]: # for douban
best_result = result
_save_path = _model.saver.save(sess, conf["save_path"] + "model.ckpt." + str(step / conf["save_step"]))
print("succ saving model in " + _save_path)
print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
def test(conf, _model):
if not os.path.exists(conf['save_path']):
os.makedirs(conf['save_path'])
# load data
print('starting loading data')
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
train_data, val_data, test_data = pickle.load(open(conf["data_path"],
'rb'))
print('finish loading data')
test_batches = reader.build_batches(test_data, conf)
print("finish building test batches")
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
# refine conf
test_batch_num = int(len(test_batches["response"]))
print('configurations: %s' % conf)
_graph = _model.build_graph()
print('build graph sucess')
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
with tf.Session(graph=_graph) as sess:
#_model.init.run();
_model.saver.restore(sess, conf["init_model"])
print("sucess init %s" % conf["init_model"])
batch_index = 0
step = 0
score_file_path = conf['save_path'] + 'score.test'
score_file = open(score_file_path, 'w')
print('starting test')
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
for batch_index in range(test_batch_num):
feed = {
_model.turns: test_batches["turns"][batch_index],
_model.tt_turns_len: test_batches["tt_turns_len"][batch_index],
_model.every_turn_len:
test_batches["every_turn_len"][batch_index],
_model.response: test_batches["response"][batch_index],
_model.response_len: test_batches["response_len"][batch_index],
_model.label: test_batches["label"][batch_index]
}
scores = sess.run(_model.logits, feed_dict=feed)
for i in range(conf["batch_size"]):
score_file.write(
str(scores[i]) + '\t' +
str(test_batches["label"][batch_index][i]) + '\n')
#str(sum(test_batches["every_turn_len"][batch_index][i]) / test_batches['tt_turns_len'][batch_index][i]) + '\t' +
#str(test_batches['tt_turns_len'][batch_index][i]) + '\n')
score_file.close()
print('finish test')
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
#write evaluation result
result = eva.evaluate(score_file_path)
result_file_path = conf["save_path"] + "result.test"
with open(result_file_path, 'w') as out_file:
for p_at in result:
out_file.write(str(p_at) + '\n')
print('finish evaluation')
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
def test(conf, _model):
if not os.path.exists(conf['save_path']):
os.makedirs(conf['save_path'])
# load data
print('starting loading data')
print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
data_collections = pickle.load(open(conf["data_path"], 'rb'))
print('finish loading data')
file_names = ["train.mix", "valid.mix", "test.mix"]
test_data = data_collections[file_names.index("test.mix")]
score_test = "score.test"
test_batches = reader.build_batches(test_data, conf)
print("finish building test batches")
print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
# refine conf
test_batch_num = len(test_batches["response"])
print('configurations: %s' %conf)
_graph = _model.build_graph()
print('build graph sucess')
print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
from tensorflow.python import debug as tf_debug
with tf.Session(graph=_graph) as sess:
_model.init.run()
_model.saver.restore(sess, conf["init_model"])
print("sucess init %s" %conf["init_model"])
test_type = conf["train_type"]
logits = _model.trainops[test_type]["logits"]
score_file_path = conf['save_path'] + '/' + score_test
score_file = open(score_file_path, 'w')
print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())), 'starting test')
for batch_index in range(test_batch_num):
feed = {
_model.turns1: test_batches["turns1"][batch_index],
_model.turns2: test_batches["turns2"][batch_index],
_model.turnsf: test_batches["turnsf"][batch_index],
_model.tt_turns_len1: test_batches["tt_turns_len1"][batch_index],
_model.every_turn_len1: test_batches["every_turn_len1"][batch_index],
_model.tt_turns_len2: test_batches["tt_turns_len2"][batch_index],
_model.every_turn_len2: test_batches["every_turn_len2"][batch_index],
_model.tt_turns_lenf: test_batches["tt_turns_lenf"][batch_index],
_model.every_turn_lenf: test_batches["every_turn_lenf"][batch_index],
_model.response: test_batches["response"][batch_index],
_model.response_len: test_batches["response_len"][batch_index],
_model.label: test_batches["label"][batch_index],
_model.turnsa: test_batches["turnsa"][batch_index],
_model.turnsa_len: test_batches["turnsa_len"][batch_index],
_model.turnsq: test_batches["turnsq"][batch_index],
_model.turnsq_len: test_batches["turnsq_len"][batch_index],
_model.keep_rate: 1.0,
}
scores = sess.run(logits, feed_dict = feed)
for i in range(len(scores)):
score_file.write(
str(scores[i]) + '\t' +
str(test_batches["label"][batch_index][i]) + '\n')
score_file.close()
print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())), 'finish test')
#write evaluation result
result = eva.evaluate(score_file_path)
print("MRR: {:01.4f} P2@1 {:01.4f} R@1 {:01.4f} r@2 {:01.4f} r@5 {:01.4f}".format(*result))
print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())), 'finish evaluation')
# Y_fit = Y_query
train_occ = np.asarray(Y_query.sum(axis=1)).flatten()
# convert query playlists to list
query_playlists = tolist(playlists_idx,
songs_idx,
position,
idx2song,
subset=query_idx)
# predict song-playlist probabilities
start = time.time()
cont_output = compute_membership_fix(playlists=query_playlists,
idx2song=idx2song,
features=features,
my_net=my_net,
random_state=rng)
print('\nTime predicting: {} sec.'.format(round(
time.time() - start, 4)))
# cont_output = np.random.rand(len(idx2song), len(query_playlists))
# evaluate the continuations
evaluate(scores=[cont_output.T],
targets=[Y_cont.T.tocsr()],
queries=[Y_query.T.tocsr()],
train_occ=[train_occ],
k_list=[10, 30, 100],
ci=args.ci,
song_occ=args.song_occ,
metrics_file=args.metrics_file)
def main(model_name):
dp = DataProvider()
if model_name == "sts":
model, optimizer, scheduler, grad_clip, step_only_at_eval = create_sts_model(
dp.SRC, dp.TGT)
elif model_name == "transformer":
model, optimizer, scheduler, grad_clip, step_only_at_eval = create_transformer_model(
Transformer, dp.SRC, dp.TGT)
elif model_name == "aspect_augmented_transformer":
model, optimizer, scheduler, grad_clip, step_only_at_eval = create_transformer_model(
AspectAugmentedTransformer, dp.SRC, dp.TGT)
elif model_name == "multi_head_aspect_augmented_transformer":
model, optimizer, scheduler, grad_clip, step_only_at_eval = create_transformer_model(
MultiHeadAspectAugmentedTransformer, dp.SRC, dp.TGT)
elif model_name == "syntax_infused_transformer":
model, optimizer, scheduler, grad_clip, step_only_at_eval = create_transformer_model(
SyntaxInfusedTransformer, dp.SRC, dp.TGT)
elif model_name == "bert_freeze_input_transformer":
model, optimizer, scheduler, grad_clip, step_only_at_eval = create_transformer_model(
BertFreezeTransformer, dp.SRC, dp.TGT)
elif model_name == "copy":
model, optimizer, scheduler, grad_clip, step_only_at_eval = create_copy_model(
dp.SRC, dp.TGT)
else:
raise ValueError("Model name {} is not defined.".format(model_name))
if not os.path.exists("../.checkpoints/"):
os.mkdir("../.checkpoints/")
training_evaluation_results = []
torch.save(
{
'model': model,
'field_src': dp.SRC,
'field_tgt': dp.TGT,
'training_evaluation_results': training_evaluation_results
}, "../.checkpoints/" + cfg.checkpoint_name)
if bool(cfg.debug_mode):
evaluate(dp.val_iter, dp, model, dp.processed_data.addresses.val.src,
dp.processed_data.addresses.val.tgt, "INIT")
best_val_score = 0.0
assert cfg.update_freq > 0, "update_freq must be a non-negative integer"
for epoch in range(int(cfg.n_epochs)):
if epoch == int(cfg.init_epochs) and model_name == "sts":
optimizer, scheduler = get_a_new_optimizer(cfg.optim,
cfg.learning_rate,
model.parameters())
all_loss = 0.0
batch_count = 0.0
all_perp = 0.0
all_tokens_count = 0.0
if epoch < 2:
# after the first iteration it does not need recalculation
val_indices = [
int(dp.size_train * x / float(cfg.val_slices))
for x in range(1, int(cfg.val_slices))
]
ds = tqdm(dp.train_iter, total=dp.size_train, dynamic_ncols=True)
optimizer.zero_grad()
for ind, instance in enumerate(ds):
if instance.src[0].size(0) < 2:
continue
pred, _, lss, decoded_length, n_tokens = model(
instance.src,
instance.trg,
test_mode=False,
**instance.data_args)
itm = lss.item()
all_loss += itm
all_tokens_count += n_tokens
all_perp += math.exp(itm / max(n_tokens, 1.0))
batch_count += 1.0
lss /= (max(decoded_length, 1) * cfg.update_freq)
lss.backward()
if grad_clip:
nn.utils.clip_grad_norm_(model.parameters(),
float(cfg.max_grad_norm))
if ind % cfg.update_freq == 0:
"""Implementation of gradient accumulation as suggested in https://arxiv.org/pdf/1806.00187.pdf"""
optimizer.step()
if not step_only_at_eval:
scheduler.step()
optimizer.zero_grad()
current_perp = all_perp / batch_count
if current_perp < 1500:
ds.set_description(
"Epoch: {}, Average Loss: {:.2f}, Average Perplexity: {:.2f}"
.format(epoch, all_loss / all_tokens_count, current_perp))
else:
ds.set_description("Epoch: {}, Average Loss: {:.2f}".format(
epoch, all_loss / all_tokens_count))
if ind in val_indices:
val_l, val_bleu = evaluate(dp.val_iter, dp, model,
dp.processed_data.addresses.val.src,
dp.processed_data.addresses.val.tgt,
str(epoch))
training_evaluation_results.append(val_bleu)
if val_bleu > best_val_score:
torch.save(
{
'model':
model,
'field_src':
dp.SRC,
'field_tgt':
dp.TGT,
'training_evaluation_results':
training_evaluation_results
}, "../.checkpoints/" + cfg.checkpoint_name)
best_val_score = val_bleu
if step_only_at_eval:
scheduler.step(val_bleu)
dp.size_train = ind + 1
if best_val_score > 0.0:
print(
"Loading the best validated model with validation bleu score of {:.2f}"
.format(best_val_score))
saved_obj = torch.load("../.checkpoints/" + cfg.checkpoint_name,
map_location=lambda storage, loc: storage)
model = saved_obj['model'].to(device)
# it might not correctly overwrite the vocabulary objects
SRC = saved_obj['field_src']
TGT = saved_obj['field_tgt']
dp.replace_fields(SRC, TGT)
evaluate(dp.val_iter, dp, model, dp.processed_data.addresses.val.src,
dp.processed_data.addresses.val.tgt, "LAST")
import argparse
description = """ This script assumes to be run at 'evaluation' directory.
It requires 'lodbrok1.h5' and 'spambrainz_dataset_eval.pickle'
to run. The purpose of the script is to show the performance
of lodbrok model against evaluation dataset. The output is how
well the model performed over evaluation dataset"""
parser = argparse.ArgumentParser(description=description)
args = parser.parse_args()
import sys
sys.path.append("..")
from numpy import loadtxt
from tensorflow.keras.models import load_model
import keras
from utils.evaluation import evaluate, print_stats
# load model
model = load_model('../models/weights/lodbrok1.h5')
# summarize model
model.summary()
eval = evaluate("../models/weights/lodbrok1.h5",
"../data/spambrainz_dataset_eval.pickle")
print_stats(eval)
def train(conf, _model):
if conf['rand_seed'] is not None:
np.random.seed(conf['rand_seed'])
if not os.path.exists(conf['save_path']):
os.makedirs(conf['save_path'])
# config display
print('configurations: %s' % conf)
# Data Generate
with tf.device('/gpu:1'):
dg = DataGenerator(conf)
# refine conf
train_batch_num = int(dg.train_data_size / conf["batch_size"])
val_batch_num = int(dg.dev_data_size / conf["batch_size"])
conf["train_steps"] = conf["num_scan_data"] * train_batch_num
conf["save_step"] = int(max(1, train_batch_num / 10))
conf["print_step"] = int(max(1, train_batch_num / 100))
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())),
' : Build graph')
_graph = _model.build_graph()
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())),
' : Build graph sucess')
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
with tf.Session(graph=_graph, config=config) as sess:
init = tf.global_variables_initializer()
sess.run(init)
if conf["init_model"]:
_model.saver.restore(sess, conf["init_model"])
print("sucess init %s" % conf["init_model"])
average_loss = 0.0
batch_index = 0
step = 0
best_result = [0, 0, 0, 0]
for step_i in range(conf["num_scan_data"]):
for batch_index in range(train_batch_num):
with tf.device('/gpu:1'):
turns, turn_num, turn_len, response, response_len, label = dg.train_data_generator(
batch_index)
feed = {
_model.turns: turns,
_model.turn_num: turn_num,
_model.turn_len: turn_len,
_model.response: response,
_model.response_len: response_len,
_model.label: label
}
batch_index = (batch_index + 1) % train_batch_num
_, curr_loss = sess.run([_model.opt, _model.de_loss],
feed_dict=feed)
average_loss += curr_loss
step += 1
if step % conf["print_step"] == 0 and step > 0:
print(
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time())),
" processed: [" + str(step * 1.0 / train_batch_num) +
"] loss: [" + str(average_loss / conf["print_step"]) +
"]")
average_loss = 0
if step % conf["save_step"] == 0 and step > 0:
index = step / conf['save_step']
dev_score_file_path = conf[
'save_path'] + 'dev_score.' + str(index)
dev_score_file = open(dev_score_file_path, 'w')
print(
time.strftime(' %Y-%m-%d %H:%M:%S',
time.localtime(time.time())),
' Save step: %s' % index)
# caculate dev score
for batch_index in range(val_batch_num):
with tf.device('/gpu:1'):
turns, turn_num, turn_len, response, response_len, label = dg.dev_data_generator(
batch_index)
feed = {
_model.turns: turns,
_model.turn_num: turn_num,
_model.turn_len: turn_len,
_model.response: response,
_model.response_len: response_len,
_model.label: label
}
scores = sess.run(_model.de_logits, feed_dict=feed)
for i in range(conf["batch_size"]):
for j in range(conf['options_num']):
if j == label[i]:
lab = 1
else:
lab = 0
dev_score_file.write(
str(scores[i][j]) + '\t' + str(lab) + '\n')
dev_score_file.close()
#write evaluation result
dev_result = eva.evaluate(dev_score_file_path)
dev_result_file_path = conf[
"save_path"] + "dev_result." + str(index)
with open(dev_result_file_path, 'w') as out_file:
for p_at in dev_result:
out_file.write(str(p_at) + '\n')
print('finish dev evaluation')
print(
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time())))
if dev_result[1] + dev_result[2] > best_result[
1] + best_result[2]:
best_result = dev_result
_save_path = _model.saver.save(
sess, conf["save_path"] + "model.ckpt." +
str(step / conf["save_step"]))
print("succ saving model in " + _save_path)
print(
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time())))
else:
# why Y_query?
# sim contains song-to-song similarities derived from fit playlists
# Y_query contains the query playlists where songs have occurred
# -> the product assigns high score to songs similar to songs that
# have occurred in the query playlists
cont_output = sim.dot(Y_query).toarray()
# factor by popularity if required
if args.pop:
cont_output *= train_occ[:, np.newaxis]
# mask song-playlist continuation pairs involving unknown songs
mask_array_rows(Y_cont, np.where(train_occ == 0)[0])
# append arrays re-shaping for evaluation
cont_output_l.append(cont_output.T)
Y_cont_l.append(Y_cont.T.tocsr())
Y_query_l.append(sparse.csr_matrix(Y_query).T.tocsr())
train_occ_l.append(train_occ)
# evaluate the continuations
evaluate(scores=cont_output_l,
targets=Y_cont_l,
queries=Y_query_l,
train_occ=train_occ_l,
k_list=[10, 30, 100],
ci=args.ci,
song_occ=args.song_occ,
metrics_file=args.metrics_file)