def main(_):
# ATTENTION: change pathname before you load your model
dataset = sys.argv[1]
pathname = "./model/" + dataset + "/ATT_GRU_model-"
test_model_id = int(sys.argv[2])
none_ind = utils.get_none_id('./origin_data/' + dataset + '/relation2id.txt')
print("None index: ", none_ind)
wordembedding = np.load('./data/' + dataset + '/vec.npy')
test_y = np.load('./data/' + dataset + '/testall_y.npy')
test_word = np.load('./data/' + dataset + '/testall_word.npy')
test_pos1 = np.load('./data/' + dataset + '/testall_pos1.npy')
test_pos2 = np.load('./data/' + dataset + '/testall_pos2.npy')
test_settings = network.Settings()
test_settings.vocab_size = len(wordembedding)
test_settings.num_classes = len(test_y[0])
test_settings.big_num = len(test_y)
with tf.Graph().as_default():
sess = tf.Session()
with sess.as_default():
def test_step(word_batch, pos1_batch, pos2_batch, y_batch):
feed_dict = {}
total_shape = []
total_num = 0
total_word = []
total_pos1 = []
total_pos2 = []
for i in range(len(word_batch)):
total_shape.append(total_num)
total_num += len(word_batch[i])
for word in word_batch[i]:
total_word.append(word)
for pos1 in pos1_batch[i]:
total_pos1.append(pos1)
for pos2 in pos2_batch[i]:
total_pos2.append(pos2)
total_shape.append(total_num)
total_shape = np.array(total_shape)
total_word = np.array(total_word)
total_pos1 = np.array(total_pos1)
total_pos2 = np.array(total_pos2)
feed_dict[mtest.total_shape] = total_shape
feed_dict[mtest.input_word] = total_word
feed_dict[mtest.input_pos1] = total_pos1
feed_dict[mtest.input_pos2] = total_pos2
feed_dict[mtest.input_y] = y_batch
loss, accuracy, predictions = sess.run(
[mtest.loss, mtest.accuracy, mtest.predictions], feed_dict)
return predictions, accuracy
with tf.variable_scope("model"):
mtest = network.GRU(is_training=False, word_embeddings=wordembedding, settings=test_settings)
saver = tf.train.Saver()
# ATTENTION: change the list to the iters you want to test !!
# testlist = range(9025,14000,25)
testlist = [test_model_id]
for model_iter in testlist:
saver.restore(sess, pathname)
print("Session Restored")
all_pred = []
all_true = []
all_accuracy = []
for i in range(int(len(test_word) / float(test_settings.big_num))):
pred, accuracy = test_step(test_word[i * test_settings.big_num:(i + 1) * test_settings.big_num],
test_pos1[i * test_settings.big_num:(i + 1) * test_settings.big_num],
test_pos2[i * test_settings.big_num:(i + 1) * test_settings.big_num],
test_y[i * test_settings.big_num:(i + 1) * test_settings.big_num])
pred = np.array(pred)
all_pred.append(pred)
all_true.append(test_y[i * test_settings.big_num:(i + 1) * test_settings.big_num])
all_accuracy.append(accuracy)
all_pred = np.concatenate(all_pred, axis=0)
all_true = np.concatenate(all_true, axis=0)
accu = float(np.mean(all_accuracy))
all_true_inds = np.argmax(all_true, 1)
precision, recall, f1 = utils.evaluate_rm_neg(all_pred, all_true_inds, none_ind)
print('Accu = %.4f, F1 = %.4f, recall = %.4f, precision = %.4f' %
(accu,
f1,
recall,
precision))
def test(sess, dataset, mtest, test_settings):
none_ind = utils.get_none_id('./origin_data/' + dataset + '/relation2id.txt')
print("None index: ", none_ind)
wordembedding = np.load('./data/' + dataset + '/vec.npy')
test_y = np.load('./data/' + dataset + '/testall_y.npy')
test_word = np.load('./data/' + dataset + '/testall_word.npy')
test_pos1 = np.load('./data/' + dataset + '/testall_pos1.npy')
test_pos2 = np.load('./data/' + dataset + '/testall_pos2.npy')
def test_step(word_batch, pos1_batch, pos2_batch, y_batch):
feed_dict = {}
total_shape = []
total_num = 0
total_word = []
total_pos1 = []
total_pos2 = []
for i in range(len(word_batch)):
total_shape.append(total_num)
total_num += len(word_batch[i])
for word in word_batch[i]:
total_word.append(word)
for pos1 in pos1_batch[i]:
total_pos1.append(pos1)
for pos2 in pos2_batch[i]:
total_pos2.append(pos2)
total_shape.append(total_num)
total_shape = np.array(total_shape)
total_word = np.array(total_word)
total_pos1 = np.array(total_pos1)
total_pos2 = np.array(total_pos2)
feed_dict[mtest.total_shape] = total_shape
feed_dict[mtest.input_word] = total_word
feed_dict[mtest.input_pos1] = total_pos1
feed_dict[mtest.input_pos2] = total_pos2
feed_dict[mtest.input_y] = y_batch
loss, accuracy, predictions = sess.run(
[mtest.loss, mtest.accuracy, mtest.predictions], feed_dict)
return predictions, accuracy
all_pred = []
all_true = []
all_accuracy = []
for i in range(int(len(test_word) / float(test_settings.big_num))):
pred, accuracy = test_step(test_word[i * test_settings.big_num:(i + 1) * test_settings.big_num],
test_pos1[i * test_settings.big_num:(i + 1) * test_settings.big_num],
test_pos2[i * test_settings.big_num:(i + 1) * test_settings.big_num],
test_y[i * test_settings.big_num:(i + 1) * test_settings.big_num])
print('After one test step.')
pred = np.array(pred)
all_pred.append(pred)
all_true.append(test_y[i * test_settings.big_num:(i + 1) * test_settings.big_num])
all_accuracy.append(accuracy)
all_pred = np.concatenate(all_pred, axis=0)
all_true = np.concatenate(all_true, axis=0)
accu = float(np.mean(all_accuracy))
all_true_inds = np.argmax(all_true, 1)
precision, recall, f1 = utils.evaluate_rm_neg(all_pred, all_true_inds, none_ind)
print('Accu = %.4f, F1 = %.4f, recall = %.4f, precision = %.4f' %
(accu,
f1,
recall,
precision))
return accu, f1, precision, recall
parser.add_argument('--data_dir',
type=str,
default='../data/intermediate/KBP/rm',
help='directory containing train/dev/test file.')
parser.add_argument('--save_dir',
type=str,
default='../dumped_model',
help='directory to save test results.')
parser.add_argument('--save_filename',
type=str,
default='result_kbp.pkl',
help='filename of test results.')
args = parser.parse_args()
opt = vars(args)
noneId = utils.get_none_id(os.path.join(opt['data_dir'], 'type.txt'))
def main():
filename = os.path.join(opt['save_dir'], opt['save_filename'])
results = pickle.load(open(filename, 'rb'))
true = results['gold']
pred = results['pred']
interested = 0.0
predicted = 0.0
correct = 0.0
for i, a in enumerate(true):
b = pred[i]
if a != noneId:
def main(_):
fout = open('extracted.json', 'w')
data_out = {}
# ATTENTION: change pathname before you load your model
pathname = "./model/kbp/ATT_GRU_model-"
test_model_id = int(sys.argv[1])
none_ind = utils.get_none_id('./origin_data/KBP/relation2id.txt')
print("None index: ", none_ind)
wordembedding = np.load('./data/KBP/vec.npy')
test_y = np.load('./data/KBP/testall_y.npy')
test_word = np.load('./data/KBP/testall_word.npy')
test_pos1 = np.load('./data/KBP/testall_pos1.npy')
test_pos2 = np.load('./data/KBP/testall_pos2.npy')
test_o = np.load('./data/KBP/testall_o.npy')
print(test_y[0])
test_settings = network.Settings()
test_settings.vocab_size = len(wordembedding)
test_settings.num_classes = len(test_y[0])
test_settings.big_num = 179
with tf.Graph().as_default():
sess = tf.Session()
with sess.as_default():
def test_step(word_batch, pos1_batch, pos2_batch, y_batch,
o_batch):
feed_dict = {}
total_shape = []
total_num = 0
total_word = []
total_pos1 = []
total_pos2 = []
for i in range(len(word_batch)):
total_shape.append(total_num)
total_num += len(word_batch[i])
for word in word_batch[i]:
total_word.append(word)
for pos1 in pos1_batch[i]:
total_pos1.append(pos1)
for pos2 in pos2_batch[i]:
total_pos2.append(pos2)
total_shape.append(total_num)
total_shape = np.array(total_shape)
total_word = np.array(total_word)
total_pos1 = np.array(total_pos1)
total_pos2 = np.array(total_pos2)
feed_dict[mtest.total_shape] = total_shape
feed_dict[mtest.input_word] = total_word
feed_dict[mtest.input_pos1] = total_pos1
feed_dict[mtest.input_pos2] = total_pos2
feed_dict[mtest.input_y] = y_batch
loss, accuracy, predictions, word_attention, sentence_attention, prob = sess.run(
[
mtest.loss, mtest.accuracy, mtest.predictions,
mtest.word_attention, mtest.sentence_attention,
mtest.prob
], feed_dict)
for i in range(len(word_batch)):
new_out = {}
pos1 = 0
pos2 = 0
for pos_ind in range(len(total_pos1[total_shape[i]])):
if total_pos1[total_shape[i]][pos_ind] == 61:
pos1 = pos_ind
break
for pos_ind in range(len(total_pos2[total_shape[i]])):
if total_pos2[total_shape[i]][pos_ind] == 61:
pos2 = pos_ind
break
entities_pair = o_batch[i][0][pos1] + ' ' + o_batch[i][0][
pos2]
print(entities_pair)
# p: predictions
# a: accuracy
# w: word_attention
# s: sentence_attention
# o: original sentence
# t: true relation
new_out['p'] = predictions[i].item()
new_out['a'] = prob[i][predictions[i]].item()
new_out['w'] = word_attention[
total_shape[i]:total_shape[i + 1]].tolist()
new_out['s'] = sentence_attention[i][0].tolist()
new_out['o'] = o_batch[i]
new_out['t'] = np.argmax(y_batch[i], 0).item()
data_out[entities_pair] = new_out
return predictions, accuracy
with tf.variable_scope("model"):
mtest = network.GRU(is_training=False,
word_embeddings=wordembedding,
settings=test_settings)
saver = tf.train.Saver()
# ATTENTION: change the list to the iters you want to test !!
# testlist = range(9025,14000,25)
testlist = [test_model_id]
for model_iter in testlist:
saver.restore(sess, pathname + str(model_iter))
time_str = datetime.datetime.now().isoformat()
print(time_str)
all_pred = []
all_true = []
all_accuracy = []
for i in range(
int(len(test_word) / float(test_settings.big_num))):
predictions, accuracy = test_step(
test_word[i * test_settings.big_num:(i + 1) *
test_settings.big_num],
test_pos1[i * test_settings.big_num:(i + 1) *
test_settings.big_num],
test_pos2[i * test_settings.big_num:(i + 1) *
test_settings.big_num],
test_y[i * test_settings.big_num:(i + 1) *
test_settings.big_num],
test_o[i * test_settings.big_num:(i + 1) *
test_settings.big_num])
pred = np.array(predictions)
all_pred.append(pred)
all_true.append(test_y[i * test_settings.big_num:(i + 1) *
test_settings.big_num])
all_accuracy.append(accuracy)
all_pred = np.concatenate(all_pred, axis=0)
all_true = np.concatenate(all_true, axis=0)
accu = float(np.mean(all_accuracy))
all_true_inds = np.argmax(all_true, 1)
precision, recall, f1 = utils.evaluate_rm_neg(
all_pred, all_true_inds, none_ind)
print(
'Accu = %.4f, F1 = %.4f, recall = %.4f, precision = %.4f)'
% (accu, f1, recall, precision))
fout.write(json.dumps(data_out))
def main(_):
my_env = os.environ.copy()
my_env["CUDA_VISIBLE_DEVICES"] = "3"
# the path to save models
save_path = './model/kbp/'
print('reading wordembedding')
wordembedding = np.load('./data/KBP/vec.npy')
print('reading training data')
train_y = np.load('./data/KBP/train_y.npy')
train_word = np.load('./data/KBP/train_word.npy')
train_pos1 = np.load('./data/KBP/train_pos1.npy')
train_pos2 = np.load('./data/KBP/train_pos2.npy')
none_ind = utils.get_none_id('./origin_data/KBP/relation2id.txt')
print("None index: ", none_ind)
settings = network.Settings()
settings.vocab_size = len(wordembedding)
settings.num_classes = len(train_y[0])
print("vocab_size: ", settings.vocab_size)
print("num_classes: ", settings.num_classes)
best_f1 = float('-inf')
best_recall = 0
best_precision = 0
with tf.Graph().as_default():
sess = tf.Session()
with sess.as_default():
initializer = tf.contrib.layers.xavier_initializer()
with tf.variable_scope("model",
reuse=None,
initializer=initializer):
m = network.GRU(is_training=True,
word_embeddings=wordembedding,
settings=settings)
global_step = tf.Variable(0, name="global_step", trainable=False)
# optimizer = tf.train.GradientDescentOptimizer(0.001)
optimizer = tf.train.AdamOptimizer(0.001)
# train_op=optimizer.minimize(m.total_loss,global_step=global_step)
train_op = optimizer.minimize(m.final_loss,
global_step=global_step)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(max_to_keep=None)
# merged_summary = tf.summary.merge_all()
merged_summary = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(
FLAGS.summary_dir + '/train_loss', sess.graph)
# summary for embedding
# it's not available in tf 0.11,
# (because there is no embedding panel in 0.11's tensorboard) so I delete it =.=
# you can try it on 0.12 or higher versions but maybe you should change some function name at first.
# summary_embed_writer = tf.train.SummaryWriter('./model',sess.graph)
# config = projector.ProjectorConfig()
# embedding_conf = config.embedding.add()
# embedding_conf.tensor_name = 'word_embedding'
# embedding_conf.metadata_path = './data/metadata.tsv'
# projector.visualize_embeddings(summary_embed_writer, config)
def train_step(word_batch, pos1_batch, pos2_batch, y_batch,
big_num):
feed_dict = {}
total_shape = []
total_num = 0
total_word = []
total_pos1 = []
total_pos2 = []
for i in range(len(word_batch)):
total_shape.append(total_num)
total_num += len(word_batch[i])
for word in word_batch[i]:
total_word.append(word)
for pos1 in pos1_batch[i]:
total_pos1.append(pos1)
for pos2 in pos2_batch[i]:
total_pos2.append(pos2)
total_shape.append(total_num)
total_shape = np.array(total_shape)
total_word = np.array(total_word)
total_pos1 = np.array(total_pos1)
total_pos2 = np.array(total_pos2)
feed_dict[m.total_shape] = total_shape
feed_dict[m.input_word] = total_word
feed_dict[m.input_pos1] = total_pos1
feed_dict[m.input_pos2] = total_pos2
feed_dict[m.input_y] = y_batch
temp, step, loss, accuracy, summary, l2_loss, final_loss, debug_output_forward, debug_prob = sess.run(
[
train_op, global_step, m.total_loss, m.accuracy,
merged_summary, m.l2_loss, m.final_loss,
m.output_forward, m.prob
], feed_dict)
if step % 50 == 0:
accuracy = np.reshape(np.array(accuracy), big_num)
acc = np.mean(accuracy)
print(debug_output_forward[0][1])
print('\n')
print('step ' + str(step) + ' with loss ' + str(loss) +
' acc ' + str(acc) + '\n')
label_not_NA_num = 0
for i in y_batch:
if i[0] != 1:
label_not_NA_num += 1
print('not NA num : ' + str(label_not_NA_num) + '\n')
return step, loss, accuracy
# training process
for one_epoch in range(settings.num_epochs):
print("Starting Epoch: ", one_epoch)
epoch_loss = 0
temp_order = list(range(len(train_word)))
np.random.shuffle(temp_order)
all_prob = []
all_true = []
all_accuracy = []
for i in range(int(len(temp_order) / float(settings.big_num))):
temp_word = []
temp_pos1 = []
temp_pos2 = []
temp_y = []
temp_input = temp_order[i * settings.big_num:(i + 1) *
settings.big_num]
for k in temp_input:
temp_word.append(train_word[k])
temp_pos1.append(train_pos1[k])
temp_pos2.append(train_pos2[k])
temp_y.append(train_y[k])
num = 0
for single_word in temp_word:
num += len(single_word)
if num > 1500:
print('out of range')
continue
temp_word = np.array(temp_word)
temp_pos1 = np.array(temp_pos1)
temp_pos2 = np.array(temp_pos2)
temp_y = np.array(temp_y)
step, loss, accuracy = train_step(temp_word, temp_pos1,
temp_pos2, temp_y,
settings.big_num)
epoch_loss += loss
all_accuracy.append(accuracy)
all_true.append(temp_y)
accu = np.mean(all_accuracy)
print("Epoch finished, loss:, ", epoch_loss, "accu: ", accu)
# all_prob = np.concatenate(all_prob, axis=0)
# all_true = np.concatenate(all_true, axis=0)
#
# all_pred_inds = utils.calcInd(all_prob)
# entropy = utils.calcEntropy(all_prob)
# all_true_inds = np.argmax(all_true, 1)
# f1score, recall, precision, meanBestF1 = utils.CrossValidation(all_pred_inds, entropy,
# all_true_inds, none_ind)
# print('F1 = %.4f, recall = %.4f, precision = %.4f, val f1 = %.4f)' %
# (f1score,
# recall,
# precision,
# meanBestF1))
print('saving model')
current_step = tf.train.global_step(sess, global_step)
path = saver.save(sess,
save_path + 'ATT_GRU_model',
global_step=current_step)
print(path)
def main(_):
my_env = os.environ.copy()
my_env["CUDA_VISIBLE_DEVICES"] = "3"
# the path to save models
save_path = './model/' + dataset + '/'
print('reading wordembedding')
wordembedding = np.load('./data/' + dataset + '/vec.npy')
print('reading training data')
train_y = np.load('./data/' + dataset + '/train_y.npy')
train_word = np.load('./data/' + dataset + '/train_word.npy')
train_pos1 = np.load('./data/' + dataset + '/train_pos1.npy')
train_pos2 = np.load('./data/' + dataset + '/train_pos2.npy')
none_ind = utils.get_none_id('./origin_data/' + dataset +
'/relation2id.txt')
print("None index: ", none_ind)
settings = network.Settings()
settings.vocab_size = len(wordembedding)
settings.num_classes = len(train_y[0])
print("vocab_size: ", settings.vocab_size)
print("num_classes: ", settings.num_classes)
test_y = np.load('./data/' + dataset + '/testall_y.npy')
test_settings = network.Settings()
test_settings.vocab_size = len(wordembedding)
test_settings.num_classes = len(test_y[0])
test_settings.big_num = len(test_y)
best_f1 = float('-inf')
best_recall = 0
best_precision = 0
with tf.Graph().as_default():
sess = tf.Session()
with sess.as_default():
initializer = tf.contrib.layers.xavier_initializer()
# with tf.variable_scope("model", reuse=None, initializer=initializer):
with tf.variable_scope("model", initializer=initializer):
m = network.GRU(is_training=True,
word_embeddings=wordembedding,
settings=settings)
with tf.variable_scope("model", reuse=True):
mtest = network.GRU(is_training=False,
word_embeddings=wordembedding,
settings=test_settings)
print('Test model constructed.')
global_step = tf.Variable(0, name="global_step", trainable=False)
# optimizer = tf.train.GradientDescentOptimizer(0.001)
optimizer = tf.train.AdamOptimizer(0.001)
# train_op=optimizer.minimize(m.total_loss,global_step=global_step)
train_op = optimizer.minimize(m.final_loss,
global_step=global_step)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(max_to_keep=10)
# merged_summary = tf.summary.merge_all()
merged_summary = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(
FLAGS.summary_dir + '/train_loss', sess.graph)
# summary for embedding
# it's not available in tf 0.11,
# (because there is no embedding panel in 0.11's tensorboard) so I delete it =.=
# you can try it on 0.12 or higher versions but maybe you should change some function name at first.
# summary_embed_writer = tf.train.SummaryWriter('./model',sess.graph)
# config = projector.ProjectorConfig()
# embedding_conf = config.embedding.add()
# embedding_conf.tensor_name = 'word_embedding'
# embedding_conf.metadata_path = './data/metadata.tsv'
# projector.visualize_embeddings(summary_embed_writer, config)
def train_step(word_batch, pos1_batch, pos2_batch, y_batch,
big_num):
feed_dict = {}
total_shape = []
total_num = 0
total_word = []
total_pos1 = []
total_pos2 = []
for i in range(len(word_batch)):
total_shape.append(total_num)
total_num += len(word_batch[i])
for word in word_batch[i]:
total_word.append(word)
for pos1 in pos1_batch[i]:
total_pos1.append(pos1)
for pos2 in pos2_batch[i]:
total_pos2.append(pos2)
total_shape.append(total_num)
total_shape = np.array(total_shape)
total_word = np.array(total_word)
total_pos1 = np.array(total_pos1)
total_pos2 = np.array(total_pos2)
feed_dict[m.total_shape] = total_shape
feed_dict[m.input_word] = total_word
feed_dict[m.input_pos1] = total_pos1
feed_dict[m.input_pos2] = total_pos2
feed_dict[m.input_y] = y_batch
temp, step, loss, accuracy, summary, l2_loss, final_loss = sess.run(
[
train_op, global_step, m.total_loss, m.accuracy,
merged_summary, m.l2_loss, m.final_loss
], feed_dict)
accuracy = np.reshape(np.array(accuracy), big_num)
summary_writer.add_summary(summary, step)
return step, loss, accuracy
test_GRU.test(sess, dataset, mtest, test_settings)
print('Initial test end.')
# training process
for one_epoch in range(settings.num_epochs):
print("Starting Epoch: ", one_epoch)
epoch_loss = 0
temp_order = list(range(len(train_word)))
np.random.shuffle(temp_order)
all_prob = []
all_true = []
all_accuracy = []
for i in tqdm.tqdm(
range(int(len(temp_order) / float(settings.big_num)))):
temp_word = []
temp_pos1 = []
temp_pos2 = []
temp_y = []
temp_input = temp_order[i * settings.big_num:(i + 1) *
settings.big_num]
for k in temp_input:
temp_word.append(train_word[k])
temp_pos1.append(train_pos1[k])
temp_pos2.append(train_pos2[k])
temp_y.append(train_y[k])
num = 0
for single_word in temp_word:
num += len(single_word)
if num > 1500:
print('out of range')
continue
temp_word = np.array(temp_word)
temp_pos1 = np.array(temp_pos1)
temp_pos2 = np.array(temp_pos2)
temp_y = np.array(temp_y)
step, loss, accuracy = train_step(temp_word, temp_pos1,
temp_pos2, temp_y,
settings.big_num)
epoch_loss += loss
all_accuracy.append(accuracy)
all_true.append(temp_y)
accu = np.mean(all_accuracy)
print("Epoch finished, loss:, ", epoch_loss, "accu: ", accu)
# all_prob = np.concatenate(all_prob, axis=0)
# all_true = np.concatenate(all_true, axis=0)
#
# all_pred_inds = utils.calcInd(all_prob)
# entropy = utils.calcEntropy(all_prob)
# all_true_inds = np.argmax(all_true, 1)
# f1score, recall, precision, meanBestF1 = utils.CrossValidation(all_pred_inds, entropy,
# all_true_inds, none_ind)
# print('F1 = %.4f, recall = %.4f, precision = %.4f, val f1 = %.4f)' %
# (f1score,
# recall,
# precision,
# meanBestF1))
print('saving model')
current_step = tf.train.global_step(sess, global_step)
path = saver.save(sess,
save_path + 'ATT_GRU_model',
global_step=current_step)
print(path)
print("start testing")
# ret = subprocess.run(['python3', 'test_GRU.py', dataset, str(current_step)], env=my_env, stdout=subprocess.PIPE)
# ret_str_list = ret.stdout.decode("utf-8").split('\n')
# a = p = r = f1 = 0.0
# for r in ret_str_list:
# if r.startswith('Accu ='):
# tmp = r.split(', ')
# a = float(tmp[0].split(' = ')[1])
# f1 = float(tmp[1].split(' = ')[1])
# r = float(tmp[2].split(' = ')[1])
# p = float(tmp[3].split(' = ')[1])
# break
# print("A, F1, P, R:")
# print(a, f1, p, r)
a, f1, p, r = test_GRU.test(sess, dataset, mtest,
test_settings)
if f1 > best_f1:
best_f1 = f1
best_precision = p
best_recall = r
print("Best results till now: ")
print("F1, Precision, Recall")
print(best_f1, best_precision, best_recall)
print("Best results: ")
print(best_f1, best_precision, best_recall)