def test_cnn():
"""Test CNN model."""
# Load data
logger.info("✔︎ Loading data...")
logger.info("Recommended padding Sequence length is: {0}".format(FLAGS.pad_seq_len))
logger.info("✔︎ Test data processing...")
test_data = dh.load_data_and_labels(FLAGS.test_data_file, FLAGS.num_classes,
FLAGS.embedding_dim, data_aug_flag=False)
logger.info("✔︎ Test data padding...")
x_test, y_test = dh.pad_data(test_data, FLAGS.pad_seq_len)
y_test_labels = test_data.labels
# Load cnn model
BEST_OR_LATEST = input("☛ Load Best or Latest Model?(B/L): ")
while not (BEST_OR_LATEST.isalpha() and BEST_OR_LATEST.upper() in ['B', 'L']):
BEST_OR_LATEST = input("✘ The format of your input is illegal, please re-input: ")
if BEST_OR_LATEST.upper() == 'B':
logger.info("✔︎ Loading best model...")
checkpoint_file = cm.get_best_checkpoint(FLAGS.best_checkpoint_dir, select_maximum_value=True)
else:
logger.info("✔︎ Loading latest model...")
checkpoint_file = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
logger.info(checkpoint_file)
graph = tf.Graph()
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
session_conf.gpu_options.allow_growth = FLAGS.gpu_options_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph("{0}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
# Get the placeholders from the graph by name
input_x = graph.get_operation_by_name("input_x").outputs[0]
input_y = graph.get_operation_by_name("input_y").outputs[0]
dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0]
is_training = graph.get_operation_by_name("is_training").outputs[0]
# Tensors we want to evaluate
scores = graph.get_operation_by_name("output/scores").outputs[0]
loss = graph.get_operation_by_name("loss/loss").outputs[0]
# Split the output nodes name by '|' if you have several output nodes
output_node_names = "output/scores"
# Save the .pb model file
output_graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def,
output_node_names.split("|"))
tf.train.write_graph(output_graph_def, "graph", "graph-cnn-{0}.pb".format(MODEL), as_text=False)
# Generate batches for one epoch
batches = dh.batch_iter(list(zip(x_test, y_test, y_test_labels)), FLAGS.batch_size, 1, shuffle=False)
test_counter, test_loss = 0, 0.0
test_pre_tk = [0.0] * FLAGS.top_num
test_rec_tk = [0.0] * FLAGS.top_num
test_F_tk = [0.0] * FLAGS.top_num
# Collect the predictions here
true_labels = []
predicted_labels = []
predicted_scores = []
# Collect for calculating metrics
true_onehot_labels = []
predicted_onehot_scores = []
predicted_onehot_labels_ts = []
predicted_onehot_labels_tk = [[] for _ in range(FLAGS.top_num)]
for batch_test in batches:
x_batch_test, y_batch_test, y_batch_test_labels = zip(*batch_test)
feed_dict = {
input_x: x_batch_test,
input_y: y_batch_test,
dropout_keep_prob: 1.0,
is_training: False
}
batch_scores, cur_loss = sess.run([scores, loss], feed_dict)
# Prepare for calculating metrics
for i in y_batch_test:
true_onehot_labels.append(i)
for j in batch_scores:
predicted_onehot_scores.append(j)
# Get the predicted labels by threshold
batch_predicted_labels_ts, batch_predicted_scores_ts = \
dh.get_label_threshold(scores=batch_scores, threshold=FLAGS.threshold)
# Add results to collection
for i in y_batch_test_labels:
true_labels.append(i)
for j in batch_predicted_labels_ts:
predicted_labels.append(j)
for k in batch_predicted_scores_ts:
predicted_scores.append(k)
# Get onehot predictions by threshold
batch_predicted_onehot_labels_ts = \
dh.get_onehot_label_threshold(scores=batch_scores, threshold=FLAGS.threshold)
for i in batch_predicted_onehot_labels_ts:
predicted_onehot_labels_ts.append(i)
# Get onehot predictions by topK
for top_num in range(FLAGS.top_num):
batch_predicted_onehot_labels_tk = dh.get_onehot_label_topk(scores=batch_scores, top_num=top_num+1)
for i in batch_predicted_onehot_labels_tk:
predicted_onehot_labels_tk[top_num].append(i)
test_loss = test_loss + cur_loss
test_counter = test_counter + 1
# Calculate Precision & Recall & F1 (threshold & topK)
test_pre_ts = precision_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts), average='micro')
test_rec_ts = recall_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts), average='micro')
test_F_ts = f1_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts), average='micro')
for top_num in range(FLAGS.top_num):
test_pre_tk[top_num] = precision_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
test_rec_tk[top_num] = recall_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
test_F_tk[top_num] = f1_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
# Calculate the average AUC
test_auc = roc_auc_score(y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores), average='micro')
# Calculate the average PR
test_prc = average_precision_score(y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores), average="micro")
test_loss = float(test_loss / test_counter)
logger.info("☛ All Test Dataset: Loss {0:g} | AUC {1:g} | AUPRC {2:g}"
.format(test_loss, test_auc, test_prc))
# Predict by threshold
logger.info("☛ Predict by threshold: Precision {0:g}, Recall {1:g}, F1 {2:g}"
.format(test_pre_ts, test_rec_ts, test_F_ts))
# Predict by topK
logger.info("☛ Predict by topK:")
for top_num in range(FLAGS.top_num):
logger.info("Top{0}: Precision {1:g}, Recall {2:g}, F {3:g}"
.format(top_num + 1, test_pre_tk[top_num], test_rec_tk[top_num], test_F_tk[top_num]))
# Save the prediction result
if not os.path.exists(SAVE_DIR):
os.makedirs(SAVE_DIR)
dh.create_prediction_file(output_file=SAVE_DIR + "/predictions.json", data_id=test_data.testid,
all_labels=true_labels, all_predict_labels=predicted_labels,
all_predict_scores=predicted_scores)
logger.info("✔︎ Done.")
def test_fasttext():
"""Test FASTTEXT model."""
# Print parameters used for the model
dh.tab_printer(args, logger)
# Load data
logger.info("Loading data...")
logger.info("Data processing...")
test_data = dh.load_data_and_labels(args.test_file,
args.num_classes,
args.word2vec_file,
data_aug_flag=False)
logger.info("Data padding...")
x_test, y_test = dh.pad_data(test_data, args.pad_seq_len)
y_test_labels = test_data.labels
# Load fasttext model
OPTION = dh._option(pattern=1)
if OPTION == 'B':
logger.info("Loading best model...")
checkpoint_file = cm.get_best_checkpoint(BEST_CPT_DIR,
select_maximum_value=True)
else:
logger.info("Loading latest model...")
checkpoint_file = tf.train.latest_checkpoint(CPT_DIR)
logger.info(checkpoint_file)
graph = tf.Graph()
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=args.allow_soft_placement,
log_device_placement=args.log_device_placement)
session_conf.gpu_options.allow_growth = args.gpu_options_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph(
"{0}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
# Get the placeholders from the graph by name
input_x = graph.get_operation_by_name("input_x").outputs[0]
input_y = graph.get_operation_by_name("input_y").outputs[0]
dropout_keep_prob = graph.get_operation_by_name(
"dropout_keep_prob").outputs[0]
is_training = graph.get_operation_by_name("is_training").outputs[0]
# Tensors we want to evaluate
scores = graph.get_operation_by_name("output/scores").outputs[0]
loss = graph.get_operation_by_name("loss/loss").outputs[0]
# Split the output nodes name by '|' if you have several output nodes
output_node_names = "output/scores"
# Save the .pb model file
output_graph_def = tf.graph_util.convert_variables_to_constants(
sess, sess.graph_def, output_node_names.split("|"))
tf.train.write_graph(output_graph_def,
"graph",
"graph-fasttext-{0}.pb".format(MODEL),
as_text=False)
# Generate batches for one epoch
batches = dh.batch_iter(list(zip(x_test, y_test, y_test_labels)),
args.batch_size,
1,
shuffle=False)
test_counter, test_loss = 0, 0.0
test_pre_tk = [0.0] * args.topK
test_rec_tk = [0.0] * args.topK
test_F1_tk = [0.0] * args.topK
# Collect the predictions here
true_labels = []
predicted_labels = []
predicted_scores = []
# Collect for calculating metrics
true_onehot_labels = []
predicted_onehot_scores = []
predicted_onehot_labels_ts = []
predicted_onehot_labels_tk = [[] for _ in range(args.topK)]
for batch_test in batches:
x_batch_test, y_batch_test, y_batch_test_labels = zip(
*batch_test)
feed_dict = {
input_x: x_batch_test,
input_y: y_batch_test,
dropout_keep_prob: 1.0,
is_training: False
}
batch_scores, cur_loss = sess.run([scores, loss], feed_dict)
# Prepare for calculating metrics
for i in y_batch_test:
true_onehot_labels.append(i)
for j in batch_scores:
predicted_onehot_scores.append(j)
# Get the predicted labels by threshold
batch_predicted_labels_ts, batch_predicted_scores_ts = \
dh.get_label_threshold(scores=batch_scores, threshold=args.threshold)
# Add results to collection
for i in y_batch_test_labels:
true_labels.append(i)
for j in batch_predicted_labels_ts:
predicted_labels.append(j)
for k in batch_predicted_scores_ts:
predicted_scores.append(k)
# Get onehot predictions by threshold
batch_predicted_onehot_labels_ts = \
dh.get_onehot_label_threshold(scores=batch_scores, threshold=args.threshold)
for i in batch_predicted_onehot_labels_ts:
predicted_onehot_labels_ts.append(i)
# Get onehot predictions by topK
for top_num in range(args.topK):
batch_predicted_onehot_labels_tk = dh.get_onehot_label_topk(
scores=batch_scores, top_num=top_num + 1)
for i in batch_predicted_onehot_labels_tk:
predicted_onehot_labels_tk[top_num].append(i)
test_loss = test_loss + cur_loss
test_counter = test_counter + 1
# Calculate Precision & Recall & F1
test_pre_ts = precision_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts),
average='micro')
test_rec_ts = recall_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts),
average='micro')
test_F1_ts = f1_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts),
average='micro')
for top_num in range(args.topK):
test_pre_tk[top_num] = precision_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
test_rec_tk[top_num] = recall_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
test_F1_tk[top_num] = f1_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
# Calculate the average AUC
test_auc = roc_auc_score(y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores),
average='micro')
# Calculate the average PR
test_prc = average_precision_score(
y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores),
average="micro")
test_loss = float(test_loss / test_counter)
logger.info(
"All Test Dataset: Loss {0:g} | AUC {1:g} | AUPRC {2:g}".
format(test_loss, test_auc, test_prc))
# Predict by threshold
logger.info(
"Predict by threshold: Precision {0:g}, Recall {1:g}, F1 {2:g}"
.format(test_pre_ts, test_rec_ts, test_F1_ts))
# Predict by topK
logger.info("Predict by topK:")
for top_num in range(args.topK):
logger.info(
"Top{0}: Precision {1:g}, Recall {2:g}, F1 {3:g}".format(
top_num + 1, test_pre_tk[top_num],
test_rec_tk[top_num], test_F1_tk[top_num]))
# Save the prediction result
if not os.path.exists(SAVE_DIR):
os.makedirs(SAVE_DIR)
dh.create_prediction_file(output_file=SAVE_DIR +
"/predictions.json",
data_id=test_data.testid,
all_labels=true_labels,
all_predict_labels=predicted_labels,
all_predict_scores=predicted_scores)
logger.info("All Done.")
def validation_step(x_val, y_val, writer=None):
"""Evaluates model on a validation set"""
batches_validation = dh.batch_iter(list(zip(x_val, y_val)), FLAGS.batch_size, 1)
# Predict classes by threshold or topk ('ts': threshold; 'tk': topk)
eval_counter, eval_loss = 0, 0.0
eval_pre_tk = [0.0] * FLAGS.top_num
eval_rec_tk = [0.0] * FLAGS.top_num
eval_F_tk = [0.0] * FLAGS.top_num
true_onehot_labels = []
predicted_onehot_scores = []
predicted_onehot_labels_ts = []
predicted_onehot_labels_tk = [[] for _ in range(FLAGS.top_num)]
for batch_validation in batches_validation:
x_batch_val, y_batch_val = zip(*batch_validation)
feed_dict = {
rcnn.input_x: x_batch_val,
rcnn.input_y: y_batch_val,
rcnn.dropout_keep_prob: 1.0,
rcnn.is_training: False
}
step, summaries, scores, cur_loss = sess.run(
[rcnn.global_step, validation_summary_op, rcnn.scores, rcnn.loss], feed_dict)
# Prepare for calculating metrics
for i in y_batch_val:
true_onehot_labels.append(i)
for j in scores:
predicted_onehot_scores.append(j)
# Predict by threshold
batch_predicted_onehot_labels_ts = \
dh.get_onehot_label_threshold(scores=scores, threshold=FLAGS.threshold)
for k in batch_predicted_onehot_labels_ts:
predicted_onehot_labels_ts.append(k)
# Predict by topK
for top_num in range(FLAGS.top_num):
batch_predicted_onehot_labels_tk = dh.get_onehot_label_topk(scores=scores, top_num=top_num+1)
for i in batch_predicted_onehot_labels_tk:
predicted_onehot_labels_tk[top_num].append(i)
eval_loss = eval_loss + cur_loss
eval_counter = eval_counter + 1
if writer:
writer.add_summary(summaries, step)
eval_loss = float(eval_loss / eval_counter)
# Calculate Precision & Recall & F1 (threshold & topK)
eval_pre_ts = precision_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts), average='micro')
eval_rec_ts = recall_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts), average='micro')
eval_F_ts = f1_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts), average='micro')
for top_num in range(FLAGS.top_num):
eval_pre_tk[top_num] = precision_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
eval_rec_tk[top_num] = recall_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
eval_F_tk[top_num] = f1_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
# Calculate the average AUC
eval_auc = roc_auc_score(y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores), average='micro')
# Calculate the average PR
eval_prc = average_precision_score(y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores), average='micro')
return eval_loss, eval_auc, eval_prc, eval_rec_ts, eval_pre_ts, eval_F_ts, \
eval_rec_tk, eval_pre_tk, eval_F_tk
def test_harnn():
"""Test HARNN model."""
# Load data
logger.info("✔︎ Loading data...")
logger.info("Recommended padding Sequence length is: {0}".format(
FLAGS.pad_seq_len))
logger.info("✔︎ Test data processing...")
test_data = dh.load_data_and_labels(FLAGS.test_data_file,
FLAGS.num_classes_list,
FLAGS.total_classes,
FLAGS.embedding_dim,
data_aug_flag=False)
logger.info("✔︎ Test data padding...")
x_test, y_test, y_test_tuple = dh.pad_data(test_data, FLAGS.pad_seq_len)
y_test_labels = test_data.labels
# Load harnn model
BEST_OR_LATEST = input("☛ Load Best or Latest Model?(B/L): ")
while not (BEST_OR_LATEST.isalpha()
and BEST_OR_LATEST.upper() in ['B', 'L']):
BEST_OR_LATEST = input(
"✘ The format of your input is illegal, please re-input: ")
if BEST_OR_LATEST.upper() == 'B':
logger.info("✔︎ Loading best model...")
checkpoint_file = cm.get_best_checkpoint(FLAGS.best_checkpoint_dir,
select_maximum_value=True)
else:
logger.info("✔︎ Loading latest model...")
checkpoint_file = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
logger.info(checkpoint_file)
graph = tf.Graph()
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
session_conf.gpu_options.allow_growth = FLAGS.gpu_options_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph(
"{0}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
# Get the placeholders from the graph by name
input_x = graph.get_operation_by_name("input_x").outputs[0]
input_y_first = graph.get_operation_by_name(
"input_y_first").outputs[0]
input_y_second = graph.get_operation_by_name(
"input_y_second").outputs[0]
input_y_third = graph.get_operation_by_name(
"input_y_third").outputs[0]
input_y = graph.get_operation_by_name("input_y").outputs[0]
dropout_keep_prob = graph.get_operation_by_name(
"dropout_keep_prob").outputs[0]
beta = graph.get_operation_by_name("beta").outputs[0]
is_training = graph.get_operation_by_name("is_training").outputs[0]
# Tensors we want to evaluate
first_attention = graph.get_operation_by_name(
"first-attention/attention").outputs[0]
first_visual = graph.get_operation_by_name(
"first-output/visual").outputs[0]
second_visual = graph.get_operation_by_name(
"second-output/visual").outputs[0]
third_visual = graph.get_operation_by_name(
"third-output/visual").outputs[0]
scores = graph.get_operation_by_name("output/scores").outputs[0]
loss = graph.get_operation_by_name("loss/loss").outputs[0]
# Split the output nodes name by '|' if you have several output nodes
output_node_names = "first-output/scores|second-output/scores|third-output/scores|output/scores"
# Save the .pb model file
output_graph_def = tf.graph_util.convert_variables_to_constants(
sess, sess.graph_def, output_node_names.split("|"))
tf.train.write_graph(output_graph_def,
"graph",
"graph-harnn-{0}.pb".format(MODEL),
as_text=False)
# Generate batches for one epoch
batches = dh.batch_iter(list(
zip(x_test, y_test, y_test_tuple, y_test_labels)),
FLAGS.batch_size,
1,
shuffle=False)
test_counter, test_loss = 0, 0.0
# Collection
true_labels = []
predicted_labels = []
predicted_scores = []
# Collect for calculating metrics
true_onehot_labels = []
predicted_onehot_scores = []
predicted_onehot_labels_ts = []
predicted_onehot_labels_tk = [[] for _ in range(FLAGS.top_num)]
for batch_test in batches:
x_batch_test, y_batch_test, y_batch_test_tuple, y_batch_test_labels = zip(
*batch_test)
y_batch_test_first = [i[0] for i in y_batch_test_tuple]
y_batch_test_second = [j[1] for j in y_batch_test_tuple]
y_batch_test_third = [k[2] for k in y_batch_test_tuple]
feed_dict = {
input_x: x_batch_test,
input_y_first: y_batch_test_first,
input_y_second: y_batch_test_second,
input_y_third: y_batch_test_third,
input_y: y_batch_test,
dropout_keep_prob: 1.0,
beta: FLAGS.beta,
is_training: False
}
batch_first_attention, batch_first_visual, batch_second_visual, batch_third_visual, batch_scores, cur_loss = \
sess.run([first_attention, first_visual, second_visual, third_visual, scores, loss], feed_dict)
print(batch_first_visual)
print(batch_first_visual[0])
print(batch_first_visual[1])
print(batch_first_visual[0, :15])
f = open('attention.html', 'w')
f.write(
'<html style="margin:0;padding:0;"><body style="margin:0;padding:0;">\n'
)
f.write('<div style="margin:25px;">\n')
for k in range(len(batch_first_attention[0])):
f.write('<p style="margin:10px;">\n')
for i in range(len(batch_first_attention[0][0])):
alpha = "{:.2f}".format(batch_first_attention[0][k][i])
word = x_batch_test[i]
f.write(
f'\t<span style="margin-left:3px;background-color:rgba(255,0,0,{alpha})">{word}</span>\n'
)
f.write('</p>\n')
f.write('</div>\n')
f.write('</body></html>')
f.close()
# Prepare for calculating metrics
for onehot_labels in y_batch_test:
true_onehot_labels.append(onehot_labels)
for onehot_scores in batch_scores:
predicted_onehot_scores.append(onehot_scores)
# Get the predicted labels by threshold
batch_predicted_labels_ts, batch_predicted_scores_ts = \
dh.get_label_threshold(scores=batch_scores, threshold=FLAGS.threshold)
# Add results to collection
for labels in y_batch_test_labels:
true_labels.append(labels)
for labels in batch_predicted_labels_ts:
predicted_labels.append(labels)
for values in batch_predicted_scores_ts:
predicted_scores.append(values)
# Get one-hot prediction by threshold
batch_predicted_onehot_labels_ts = \
dh.get_onehot_label_threshold(scores=batch_scores, threshold=FLAGS.threshold)
for onehot_labels in batch_predicted_onehot_labels_ts:
predicted_onehot_labels_ts.append(onehot_labels)
# Get one-hot prediction by topK
for i in range(FLAGS.top_num):
batch_predicted_onehot_labels_tk = dh.get_onehot_label_topk(
scores=batch_scores, top_num=i + 1)
for onehot_labels in batch_predicted_onehot_labels_tk:
predicted_onehot_labels_tk[i].append(onehot_labels)
test_loss = test_loss + cur_loss
test_counter = test_counter + 1
# Calculate Precision & Recall & F1
test_pre_ts = precision_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts),
average='micro')
test_rec_ts = recall_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts),
average='micro')
test_F_ts = f1_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts),
average='micro')
# Calculate the average AUC
test_auc = roc_auc_score(y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores),
average='micro')
# Calculate the average PR
test_prc = average_precision_score(
y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores),
average="micro")
test_loss = float(test_loss / test_counter)
logger.info(
"☛ All Test Dataset: Loss {0:g} | AUC {1:g} | AUPRC {2:g}".
format(test_loss, test_auc, test_prc))
# Predict by threshold
logger.info(
"☛ Predict by threshold: Precision {0:g}, Recall {1:g}, F1 {2:g}"
.format(test_pre_ts, test_rec_ts, test_F_ts))
# Save the prediction result
if not os.path.exists(SAVE_DIR):
os.makedirs(SAVE_DIR)
dh.create_prediction_file(output_file=SAVE_DIR +
"/predictions.json",
data_id=test_data.patent_id,
all_labels=true_labels,
all_predict_labels=predicted_labels,
all_predict_scores=predicted_scores)
logger.info("✔︎ Done.")
def validation_step(val_loader, writer=None):
"""Evaluates model on a validation set."""
batches_validation = dh.batch_iter(
list(create_input_data(val_loader)), args.batch_size, 1)
# Predict classes by threshold or topk ('ts': threshold; 'tk': topk)
eval_counter, eval_loss = 0, 0.0
eval_pre_tk = [0.0] * args.topK
eval_rec_tk = [0.0] * args.topK
eval_F1_tk = [0.0] * args.topK
true_onehot_labels = []
predicted_onehot_scores = []
predicted_onehot_labels_ts = []
predicted_onehot_labels_tk = [[] for _ in range(args.topK)]
for batch_validation in batches_validation:
x, sec, subsec, group, subgroup, y_onehot = zip(
*batch_validation)
feed_dict = {
harnn.input_x: x,
harnn.input_y_first: sec,
harnn.input_y_second: subsec,
harnn.input_y_third: group,
harnn.input_y_fourth: subgroup,
harnn.input_y: y_onehot,
harnn.dropout_keep_prob: 1.0,
harnn.alpha: args.alpha,
harnn.is_training: False
}
step, summaries, scores, cur_loss = sess.run([
harnn.global_step, validation_summary_op, harnn.scores,
harnn.loss
], feed_dict)
# Prepare for calculating metrics
for i in y_onehot:
true_onehot_labels.append(i)
for j in scores:
predicted_onehot_scores.append(j)
# Predict by threshold
batch_predicted_onehot_labels_ts = \
dh.get_onehot_label_threshold(scores=scores, threshold=args.threshold)
for k in batch_predicted_onehot_labels_ts:
predicted_onehot_labels_ts.append(k)
# Predict by topK
for top_num in range(args.topK):
batch_predicted_onehot_labels_tk = dh.get_onehot_label_topk(
scores=scores, top_num=top_num + 1)
for i in batch_predicted_onehot_labels_tk:
predicted_onehot_labels_tk[top_num].append(i)
eval_loss = eval_loss + cur_loss
eval_counter = eval_counter + 1
if writer:
writer.add_summary(summaries, step)
eval_loss = float(eval_loss / eval_counter)
# Calculate Precision & Recall & F1
eval_pre_ts = precision_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts),
average='micro')
eval_rec_ts = recall_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts),
average='micro')
eval_F1_ts = f1_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts),
average='micro')
for top_num in range(args.topK):
eval_pre_tk[top_num] = precision_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
eval_rec_tk[top_num] = recall_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
eval_F1_tk[top_num] = f1_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
# Calculate the average AUC
eval_auc = roc_auc_score(
y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores),
average='micro')
# Calculate the average PR
eval_prc = average_precision_score(
y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores),
average='micro')
return eval_loss, eval_auc, eval_prc, eval_pre_ts, eval_rec_ts, eval_F1_ts, \
eval_pre_tk, eval_rec_tk, eval_F1_tk
def test_harnn():
"""Test HARNN model."""
# Print parameters used for the model
dh.tab_printer(args, logger)
# Load data
logger.info("Loading data...")
logger.info("Data processing...")
test_data = dh.load_data_and_labels(args.test_file, args.num_classes_list, args.total_classes,
args.word2vec_file, data_aug_flag=False)
logger.info("Data padding...")
x_test, y_test, y_test_tuple = dh.pad_data(test_data, args.pad_seq_len)
y_test_labels = test_data.labels
# Load harnn model
OPTION = dh._option(pattern=1)
if OPTION == 'B':
logger.info("Loading best model...")
checkpoint_file = cm.get_best_checkpoint(BEST_CPT_DIR, select_maximum_value=True)
else:
logger.info("Loading latest model...")
checkpoint_file = tf.train.latest_checkpoint(CPT_DIR)
logger.info(checkpoint_file)
graph = tf.Graph()
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=args.allow_soft_placement,
log_device_placement=args.log_device_placement)
session_conf.gpu_options.allow_growth = args.gpu_options_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph("{0}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
# Get the placeholders from the graph by name
input_x = graph.get_operation_by_name("input_x").outputs[0]
input_y_first = graph.get_operation_by_name("input_y_first").outputs[0]
input_y_second = graph.get_operation_by_name("input_y_second").outputs[0]
input_y_third = graph.get_operation_by_name("input_y_third").outputs[0]
input_y_fourth = graph.get_operation_by_name("input_y_fourth").outputs[0]
input_y = graph.get_operation_by_name("input_y").outputs[0]
dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0]
beta = graph.get_operation_by_name("beta").outputs[0]
is_training = graph.get_operation_by_name("is_training").outputs[0]
# Tensors we want to evaluate
first_scores = graph.get_operation_by_name("first-output/scores").outputs[0]
second_scores = graph.get_operation_by_name("second-output/scores").outputs[0]
third_scores = graph.get_operation_by_name("third-output/scores").outputs[0]
fourth_scores = graph.get_operation_by_name("fourth-output/scores").outputs[0]
scores = graph.get_operation_by_name("output/scores").outputs[0]
loss = graph.get_operation_by_name("loss/loss").outputs[0]
# Split the output nodes name by '|' if you have several output nodes
output_node_names = "first-output/scores|second-output/scores|third-output/scores|fourth-output/scores|output/scores"
# Save the .pb model file
output_graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def,
output_node_names.split("|"))
tf.train.write_graph(output_graph_def, "graph", "graph-harnn-{0}.pb".format(MODEL), as_text=False)
# Generate batches for one epoch
batches = dh.batch_iter(list(zip(x_test, y_test, y_test_tuple, y_test_labels)),
args.batch_size, 1, shuffle=False)
test_counter, test_loss = 0, 0.0
# Collect the predictions here
true_labels = []
predicted_labels = []
predicted_scores = []
# Collect for calculating metrics
true_onehot_labels = []
predicted_onehot_scores = []
predicted_onehot_labels_ts = []
predicted_onehot_labels_tk = [[] for _ in range(args.topK)]
true_onehot_first_labels = []
true_onehot_second_labels = []
true_onehot_third_labels = []
true_onehot_fourth_labels = []
predicted_onehot_scores_first = []
predicted_onehot_scores_second = []
predicted_onehot_scores_third = []
predicted_onehot_scores_fourth = []
predicted_onehot_labels_first = []
predicted_onehot_labels_second = []
predicted_onehot_labels_third = []
predicted_onehot_labels_fourth = []
for batch_test in batches:
x_batch_test, y_batch_test, y_batch_test_tuple, y_batch_test_labels = zip(*batch_test)
y_batch_test_first = [i[0] for i in y_batch_test_tuple]
y_batch_test_second = [j[1] for j in y_batch_test_tuple]
y_batch_test_third = [k[2] for k in y_batch_test_tuple]
y_batch_test_fourth = [t[3] for t in y_batch_test_tuple]
feed_dict = {
input_x: x_batch_test,
input_y_first: y_batch_test_first,
input_y_second: y_batch_test_second,
input_y_third: y_batch_test_third,
input_y_fourth: y_batch_test_fourth,
input_y: y_batch_test,
dropout_keep_prob: 1.0,
beta: args.beta,
is_training: False
}
batch_first_scores, batch_second_scores, batch_third_scores, batch_fourth_scores, batch_scores, cur_loss = \
sess.run([first_scores, second_scores, third_scores, fourth_scores, scores, loss], feed_dict)
# Prepare for calculating metrics
for onehot_labels in y_batch_test:
true_onehot_labels.append(onehot_labels)
for onehot_labels in y_batch_test_first:
true_onehot_first_labels.append(onehot_labels)
for onehot_labels in y_batch_test_second:
true_onehot_second_labels.append(onehot_labels)
for onehot_labels in y_batch_test_third:
true_onehot_third_labels.append(onehot_labels)
for onehot_labels in y_batch_test_fourth:
true_onehot_fourth_labels.append(onehot_labels)
for onehot_scores in batch_scores:
predicted_onehot_scores.append(onehot_scores)
for onehot_scores in batch_first_scores:
predicted_onehot_scores_first.append(onehot_scores)
for onehot_scores in batch_second_scores:
predicted_onehot_scores_second.append(onehot_scores)
for onehot_scores in batch_third_scores:
predicted_onehot_scores_third.append(onehot_scores)
for onehot_scores in batch_fourth_scores:
predicted_onehot_scores_fourth.append(onehot_scores)
# Get the predicted labels by threshold
batch_predicted_labels_ts, batch_predicted_scores_ts = \
dh.get_label_threshold(scores=batch_scores, threshold=args.threshold)
# Add results to collection
for labels in y_batch_test_labels:
true_labels.append(labels)
for labels in batch_predicted_labels_ts:
predicted_labels.append(labels)
for values in batch_predicted_scores_ts:
predicted_scores.append(values)
# Get one-hot prediction by threshold
batch_predicted_onehot_labels_ts = \
dh.get_onehot_label_threshold(scores=batch_scores, threshold=args.threshold)
batch_predicted_onehot_labels_first = \
dh.get_onehot_label_threshold(scores=batch_first_scores, threshold=args.threshold)
batch_predicted_onehot_labels_second = \
dh.get_onehot_label_threshold(scores=batch_second_scores, threshold=args.threshold)
batch_predicted_onehot_labels_third = \
dh.get_onehot_label_threshold(scores=batch_third_scores, threshold=args.threshold)
batch_predicted_onehot_labels_fourth = \
dh.get_onehot_label_threshold(scores=batch_fourth_scores, threshold=args.threshold)
for onehot_labels in batch_predicted_onehot_labels_ts:
predicted_onehot_labels_ts.append(onehot_labels)
for onehot_labels in batch_predicted_onehot_labels_first:
predicted_onehot_labels_first.append(onehot_labels)
for onehot_labels in batch_predicted_onehot_labels_second:
predicted_onehot_labels_second.append(onehot_labels)
for onehot_labels in batch_predicted_onehot_labels_third:
predicted_onehot_labels_third.append(onehot_labels)
for onehot_labels in batch_predicted_onehot_labels_fourth:
predicted_onehot_labels_fourth.append(onehot_labels)
# Get one-hot prediction by topK
for i in range(args.topK):
batch_predicted_onehot_labels_tk = dh.get_onehot_label_topk(scores=batch_scores, top_num=i + 1)
for onehot_labels in batch_predicted_onehot_labels_tk:
predicted_onehot_labels_tk[i].append(onehot_labels)
test_loss = test_loss + cur_loss
test_counter = test_counter + 1
# Calculate Precision & Recall & F1
test_pre_ts = precision_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts), average='micro')
test_pre_first = precision_score(y_true=np.array(true_onehot_first_labels),
y_pred=np.array(predicted_onehot_labels_first), average='micro')
test_pre_second = precision_score(y_true=np.array(true_onehot_second_labels),
y_pred=np.array(predicted_onehot_labels_second), average='micro')
test_pre_third = precision_score(y_true=np.array(true_onehot_third_labels),
y_pred=np.array(predicted_onehot_labels_third), average='micro')
test_pre_fourth = precision_score(y_true=np.array(true_onehot_fourth_labels),
y_pred=np.array(predicted_onehot_labels_fourth), average='micro')
test_rec_ts = recall_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts), average='micro')
test_rec_first = recall_score(y_true=np.array(true_onehot_first_labels),
y_pred=np.array(predicted_onehot_labels_first), average='micro')
test_rec_second = recall_score(y_true=np.array(true_onehot_second_labels),
y_pred=np.array(predicted_onehot_labels_second), average='micro')
test_rec_third = recall_score(y_true=np.array(true_onehot_third_labels),
y_pred=np.array(predicted_onehot_labels_third), average='micro')
test_rec_fourth = recall_score(y_true=np.array(true_onehot_fourth_labels),
y_pred=np.array(predicted_onehot_labels_fourth), average='micro')
test_F1_ts = f1_score(y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts), average='micro')
test_F1_first = f1_score(y_true=np.array(true_onehot_first_labels),
y_pred=np.array(predicted_onehot_labels_first), average='micro')
test_F1_second = f1_score(y_true=np.array(true_onehot_second_labels),
y_pred=np.array(predicted_onehot_labels_second), average='micro')
test_F1_third = f1_score(y_true=np.array(true_onehot_third_labels),
y_pred=np.array(predicted_onehot_labels_third), average='micro')
test_F1_fourth = f1_score(y_true=np.array(true_onehot_fourth_labels),
y_pred=np.array(predicted_onehot_labels_fourth), average='micro')
# Calculate the average AUC
test_auc = roc_auc_score(y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores), average='micro')
# Calculate the average PR
test_prc = average_precision_score(y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores), average="micro")
test_prc_first = average_precision_score(y_true=np.array(true_onehot_first_labels),
y_score=np.array(predicted_onehot_scores_first), average="micro")
test_prc_second = average_precision_score(y_true=np.array(true_onehot_second_labels),
y_score=np.array(predicted_onehot_scores_second), average="micro")
test_prc_third = average_precision_score(y_true=np.array(true_onehot_third_labels),
y_score=np.array(predicted_onehot_scores_third), average="micro")
test_prc_fourth = average_precision_score(y_true=np.array(true_onehot_fourth_labels),
y_score=np.array(predicted_onehot_scores_fourth), average="micro")
test_loss = float(test_loss / test_counter)
logger.info("All Test Dataset: Loss {0:g} | AUC {1:g} | AUPRC {2:g}"
.format(test_loss, test_auc, test_prc))
# Predict by threshold
logger.info("Predict by threshold: Precision {0:g}, Recall {1:g}, F1 {2:g}"
.format(test_pre_ts, test_rec_ts, test_F1_ts))
logger.info("Predict by threshold in Level-1: Precision {0:g}, Recall {1:g}, F1 {2:g}, AUPRC {3:g}"
.format(test_pre_first, test_rec_first, test_F1_first, test_prc_first))
logger.info("Predict by threshold in Level-2: Precision {0:g}, Recall {1:g}, F1 {2:g}, AUPRC {3:g}"
.format(test_pre_second, test_rec_second, test_F1_second, test_prc_second))
logger.info("Predict by threshold in Level-3: Precision {0:g}, Recall {1:g}, F1 {2:g}, AUPRC {3:g}"
.format(test_pre_third, test_rec_third, test_F1_third, test_prc_third))
logger.info("Predict by threshold in Level-4: Precision {0:g}, Recall {1:g}, F1 {2:g}, AUPRC {3:g}"
.format(test_pre_fourth, test_rec_fourth, test_F1_fourth, test_prc_fourth))
# Save the prediction result
if not os.path.exists(SAVE_DIR):
os.makedirs(SAVE_DIR)
dh.create_prediction_file(output_file=SAVE_DIR + "/predictions.json", data_id=test_data.patent_id,
all_labels=true_labels, all_predict_labels=predicted_labels,
all_predict_scores=predicted_scores)
logger.info("All Done.")
def test_harnn():
"""Test HARNN model."""
# Print parameters used for the model
dh.tab_printer(args, logger)
# Load word2vec model
word2idx, embedding_matrix = dh.load_word2vec_matrix(args.word2vec_file)
# Load data
logger.info("Loading data...")
logger.info("Data processing...")
test_data = dh.load_data_and_labels(args, args.test_file, word2idx)
# Load harnn model
OPTION = dh._option(pattern=1)
if OPTION == 'B':
logger.info("Loading best model...")
checkpoint_file = cm.get_best_checkpoint(BEST_CPT_DIR,
select_maximum_value=True)
else:
logger.info("Loading latest model...")
checkpoint_file = tf.train.latest_checkpoint(CPT_DIR)
logger.info(checkpoint_file)
graph = tf.Graph()
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=args.allow_soft_placement,
log_device_placement=args.log_device_placement)
session_conf.gpu_options.allow_growth = args.gpu_options_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph(
"{0}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
# Get the placeholders from the graph by name
input_x = graph.get_operation_by_name("input_x").outputs[0]
input_y_first = graph.get_operation_by_name(
"input_y_first").outputs[0]
input_y_second = graph.get_operation_by_name(
"input_y_second").outputs[0]
input_y_third = graph.get_operation_by_name(
"input_y_third").outputs[0]
input_y_fourth = graph.get_operation_by_name(
"input_y_fourth").outputs[0]
input_y = graph.get_operation_by_name("input_y").outputs[0]
dropout_keep_prob = graph.get_operation_by_name(
"dropout_keep_prob").outputs[0]
alpha = graph.get_operation_by_name("alpha").outputs[0]
is_training = graph.get_operation_by_name("is_training").outputs[0]
# Tensors we want to evaluate
first_scores = graph.get_operation_by_name(
"first-output/scores").outputs[0]
second_scores = graph.get_operation_by_name(
"second-output/scores").outputs[0]
third_scores = graph.get_operation_by_name(
"third-output/scores").outputs[0]
fourth_scores = graph.get_operation_by_name(
"fourth-output/scores").outputs[0]
scores = graph.get_operation_by_name("output/scores").outputs[0]
# Split the output nodes name by '|' if you have several output nodes
output_node_names = "first-output/scores|second-output/scores|third-output/scores|fourth-output/scores|output/scores"
# Save the .pb model file
output_graph_def = tf.graph_util.convert_variables_to_constants(
sess, sess.graph_def, output_node_names.split("|"))
tf.train.write_graph(output_graph_def,
"graph",
"graph-harnn-{0}.pb".format(MODEL),
as_text=False)
# Generate batches for one epoch
batches = dh.batch_iter(list(create_input_data(test_data)),
args.batch_size,
1,
shuffle=False)
# Collect the predictions here
true_labels = []
predicted_labels = []
predicted_scores = []
# Collect for calculating metrics
true_onehot_labels = [[], [], [], [], []]
predicted_onehot_scores = [[], [], [], [], []]
predicted_onehot_labels = [[], [], [], [], []]
for batch_test in batches:
x, sec, subsec, group, subgroup, y_onehot, y = zip(*batch_test)
y_batch_test_list = [y_onehot, sec, subsec, group, subgroup]
feed_dict = {
input_x: x,
input_y_first: sec,
input_y_second: subsec,
input_y_third: group,
input_y_fourth: subgroup,
input_y: y_onehot,
dropout_keep_prob: 1.0,
alpha: args.alpha,
is_training: False
}
batch_global_scores, batch_first_scores, batch_second_scores, batch_third_scores, batch_fourth_scores = \
sess.run([scores, first_scores, second_scores, third_scores, fourth_scores], feed_dict)
batch_scores = [
batch_global_scores, batch_first_scores,
batch_second_scores, batch_third_scores,
batch_fourth_scores
]
# Get the predicted labels by threshold
batch_predicted_labels_ts, batch_predicted_scores_ts = \
dh.get_label_threshold(scores=batch_scores[0], threshold=args.threshold)
# Add results to collection
for labels in y:
true_labels.append(labels)
for labels in batch_predicted_labels_ts:
predicted_labels.append(labels)
for values in batch_predicted_scores_ts:
predicted_scores.append(values)
for index in range(len(predicted_onehot_scores)):
for onehot_labels in y_batch_test_list[index]:
true_onehot_labels[index].append(onehot_labels)
for onehot_scores in batch_scores[index]:
predicted_onehot_scores[index].append(onehot_scores)
# Get one-hot prediction by threshold
predicted_onehot_labels_ts = \
dh.get_onehot_label_threshold(scores=batch_scores[index], threshold=args.threshold)
for onehot_labels in predicted_onehot_labels_ts:
predicted_onehot_labels[index].append(onehot_labels)
# Calculate Precision & Recall & F1
for index in range(len(predicted_onehot_scores)):
test_pre = precision_score(
y_true=np.array(true_onehot_labels[index]),
y_pred=np.array(predicted_onehot_labels[index]),
average='micro')
test_rec = recall_score(
y_true=np.array(true_onehot_labels[index]),
y_pred=np.array(predicted_onehot_labels[index]),
average='micro')
test_F1 = f1_score(y_true=np.array(true_onehot_labels[index]),
y_pred=np.array(
predicted_onehot_labels[index]),
average='micro')
test_auc = roc_auc_score(
y_true=np.array(true_onehot_labels[index]),
y_score=np.array(predicted_onehot_scores[index]),
average='micro')
test_prc = average_precision_score(
y_true=np.array(true_onehot_labels[index]),
y_score=np.array(predicted_onehot_scores[index]),
average="micro")
if index == 0:
logger.info(
"[Global] Predict by threshold: Precision {0:g}, Recall {1:g}, "
"F1 {2:g}, AUC {3:g}, AUPRC {4:g}".format(
test_pre, test_rec, test_F1, test_auc, test_prc))
else:
logger.info(
"[Local] Predict by threshold in Level-{0}: Precision {1:g}, Recall {2:g}, "
"F1 {3:g}, AUPRC {4:g}".format(index, test_pre,
test_rec, test_F1,
test_prc))
# Save the prediction result
if not os.path.exists(SAVE_DIR):
os.makedirs(SAVE_DIR)
dh.create_prediction_file(output_file=SAVE_DIR +
"/predictions.json",
data_id=test_data['uniq_id'],
true_labels=true_labels,
predict_labels=predicted_labels,
predict_scores=predicted_scores)
logger.info("All Done.")
def validation_step(x_val, y_val, y_val_tuple, writer=None):
"""Evaluates model on a validation set"""
batches_validation = dh.batch_iter(list(
zip(x_val, y_val, y_val_tuple)),
FLAGS.batch_size,
1,
shuffle=False)
# Predict classes by threshold or topk ('ts': threshold; 'tk': topk)
eval_counter, eval_loss = 0, 0.0
eval_pre_tk = [0.0] * FLAGS.top_num
eval_rec_tk = [0.0] * FLAGS.top_num
eval_F_tk = [0.0] * FLAGS.top_num
true_onehot_labels = []
predicted_onehot_scores = []
predicted_onehot_labels_ts = []
predicted_onehot_labels_tk = [[] for _ in range(FLAGS.top_num)]
for batch_validation in batches_validation:
x_batch_val, y_batch_val, y_batch_val_tuple = zip(
*batch_validation)
y_batch_val_first = [i[0] for i in y_batch_val_tuple]
y_batch_val_second = [j[1] for j in y_batch_val_tuple]
y_batch_val_third = [k[2] for k in y_batch_val_tuple]
feed_dict = {
pacrr.input_x: x_batch_val,
pacrr.input_y_first: y_batch_val_first,
pacrr.input_y_second: y_batch_val_second,
pacrr.input_y_third: y_batch_val_third,
pacrr.input_y: y_batch_val,
pacrr.dropout_keep_prob: 1.0,
pacrr.is_training: False
}
step, summaries, scores, cur_loss = sess.run([
pacrr.global_step, validation_summary_op, pacrr.scores,
pacrr.loss
], feed_dict)
# Prepare for calculating metrics
for onehot_labels in y_batch_val:
true_onehot_labels.append(onehot_labels)
for onehot_scores in scores:
predicted_onehot_scores.append(onehot_scores)
# Predict by threshold
batch_predicted_onehot_labels_ts = \
dh.get_onehot_label_threshold(scores=scores, threshold=FLAGS.threshold)
for onehot_labels in batch_predicted_onehot_labels_ts:
predicted_onehot_labels_ts.append(onehot_labels)
# Predict by topK
for i in range(FLAGS.top_num):
batch_predicted_onehot_labels_tk = dh.get_onehot_label_topk(
scores=scores, top_num=i + 1)
for onehot_labels in batch_predicted_onehot_labels_tk:
predicted_onehot_labels_tk[i].append(onehot_labels)
eval_loss = eval_loss + cur_loss
eval_counter = eval_counter + 1
if writer:
writer.add_summary(summaries, step)
eval_loss = float(eval_loss / eval_counter)
# Calculate Precision & Recall & F1
eval_pre_ts = precision_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts),
average='micro')
eval_rec_ts = recall_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts),
average='micro')
eval_F_ts = f1_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_ts),
average='micro')
# Calculate the average AUC
eval_auc = roc_auc_score(
y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores),
average='micro')
# Calculate the average PR
eval_prc = average_precision_score(
y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores),
average="micro")
for top_num in range(FLAGS.top_num):
eval_pre_tk[top_num] = precision_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
eval_rec_tk[top_num] = recall_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
eval_F_tk[top_num] = f1_score(
y_true=np.array(true_onehot_labels),
y_pred=np.array(predicted_onehot_labels_tk[top_num]),
average='micro')
return eval_loss, eval_auc, eval_prc, eval_rec_ts, eval_pre_ts, eval_F_ts, \
eval_rec_tk, eval_pre_tk, eval_F_tk
