def test_model(test_trees, labels, embeddings, embedding_lookup, opt): logdir = opt.model_path batch_size = opt.train_batch_size epochs = opt.niter num_feats = len(embeddings[0]) random.shuffle(test_trees) # build the inputs and outputs of the network nodes_node, children_node, codecaps_node = network.init_net_treecaps( num_feats, len(labels)) out_node = network.out_layer(codecaps_node) labels_node, loss_node = network.loss_layer(codecaps_node, len(labels)) optimizer = RAdamOptimizer(opt.lr) train_step = optimizer.minimize(loss_node) sess = tf.Session() sess.run(tf.global_variables_initializer()) with tf.name_scope('saver'): saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") saver.restore(sess, ckpt.model_checkpoint_path) for i, var in enumerate(saver._var_list): print('Var {}: {}'.format(i, var)) checkfile = os.path.join(logdir, 'tree_network.ckpt') correct_labels = [] predictions = [] print('Computing training accuracy...') for batch in sampling.batch_samples( sampling.gen_samples(test_trees, labels, embeddings, embedding_lookup), 1): nodes, children, batch_labels = batch output = sess.run([out_node], feed_dict={ nodes_node: nodes, children_node: children, }) correct_labels.append(np.argmax(batch_labels)) predictions.append(np.argmax(output)) target_names = list(labels) print( classification_report(correct_labels, predictions, target_names=target_names)) print(confusion_matrix(correct_labels, predictions)) print('*' * 50) print('Accuracy:', accuracy_score(correct_labels, predictions)) print('*' * 50)
def test_model(logdir, infile, embedfile): """Test a classifier to label ASTs""" with open(infile, 'rb') as fh: _, trees, labels = pickle.load(fh) with open(embedfile, 'rb') as fh: embeddings, embed_lookup = pickle.load(fh) num_feats = len(embeddings[0]) # build the inputs and outputs of the network nodes_node, children_node, hidden_node = network.init_net( num_feats, len(labels)) out_node = network.out_layer(hidden_node) ### init the graph sess = tf.Session() #config=tf.ConfigProto(device_count={'GPU':0})) sess.run(tf.global_variables_initializer()) with tf.name_scope('saver'): saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) else: raise 'Checkpoint not found.' correct_labels = [] # make predicitons from the input predictions = [] step = 0 for batch in sampling.batch_samples( sampling.gen_fast_samples(trees, labels, embeddings, embed_lookup), 1): nodes, children, batch_labels = batch output = sess.run([out_node], feed_dict={ nodes_node: nodes, children_node: children, }) correct_labels.append(np.argmax(batch_labels)) predictions.append(np.argmax(output)) step += 1 print(step, '/', len(trees)) target_names = list(labels) print('Accuracy:', accuracy_score(correct_labels, predictions)) print( classification_report(correct_labels, predictions, target_names=target_names)) print(confusion_matrix(correct_labels, predictions))
def train_model(train_trees, val_trees, labels, embeddings, embedding_lookup, opt): """Train a classifier to label ASTs""" logdir = opt.model_path batch_size = opt.train_batch_size epochs = opt.niter num_feats = len(embeddings[0]) random.shuffle(train_trees) random.shuffle(val_trees) nodes_node, children_node, hidden_node, attention_score_node = network.init_net( num_feats, len(labels), opt.aggregation) hidden_node = tf.identity(hidden_node, name="hidden_node") out_node = network.out_layer(hidden_node) labels_node, loss_node = network.loss_layer(hidden_node, len(labels)) optimizer = tf.train.AdamOptimizer(LEARN_RATE) train_step = optimizer.minimize(loss_node) ### init the graph sess = tf.Session() #config=tf.ConfigProto(device_count={'GPU':0})) sess.run(tf.global_variables_initializer()) # Initialize the variables (i.e. assign their default value) init = tf.global_variables_initializer() with tf.name_scope('saver'): saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") saver.restore(sess, ckpt.model_checkpoint_path) for i, var in enumerate(saver._var_list): print('Var {}: {}'.format(i, var)) checkfile = os.path.join(logdir, 'cnn_tree.ckpt') print("Begin training..........") num_batches = len(train_trees) // batch_size + ( 1 if len(train_trees) % batch_size != 0 else 0) for epoch in range(1, epochs + 1): for i, batch in enumerate( sampling.batch_samples( sampling.gen_samples(train_trees, labels, embeddings, embedding_lookup), batch_size)): nodes, children, batch_labels = batch step = (epoch - 1) * num_batches + i * BATCH_SIZE if not nodes: continue # don't try to train on an empty batch # print(batch_labels) _, err, out = sess.run( [train_step, loss_node, out_node], feed_dict={ nodes_node: nodes, children_node: children, labels_node: batch_labels }) print('Epoch:', epoch, 'Step:', step, 'Loss:', err, 'Max nodes:', len(nodes[0])) # print(attention_score[0]) # print(len(attention_score[0])) # print(pooling_output.shape) if step % CHECKPOINT_EVERY == 0: # save state so we can resume later saver.save(sess, checkfile) # shutil.rmtree(savedmodel_path) print('Checkpoint saved, epoch:' + str(epoch) + ', step: ' + str(step) + ', loss: ' + str(err) + '.') correct_labels = [] predictions = [] for batch in sampling.batch_samples( sampling.gen_samples(val_trees, labels, embeddings, embedding_lookup), 1): nodes, children, batch_labels = batch output = sess.run([out_node], feed_dict={ nodes_node: nodes, children_node: children, }) # print(output) correct_labels.append(np.argmax(batch_labels)) predictions.append(np.argmax(output)) target_names = list(labels) print('Accuracy:', accuracy_score(correct_labels, predictions)) print( classification_report(correct_labels, predictions, target_names=target_names)) print(confusion_matrix(correct_labels, predictions)) print("Finish all iters, storring the whole model..........") saver.save(sess, checkfile)
def train_model(train_dataloader, val_dataloader, embeddings, embedding_lookup, opt): logdir = opt.model_path epochs = opt.niter node_embedding_size = len(embeddings[0]) train_left_trees = train_dataloader.left_trees train_right_trees = train_dataloader.right_trees train_labels = train_dataloader.labels val_left_trees = val_dataloader.left_trees val_right_trees = val_dataloader.right_trees val_labels = val_dataloader.labels n_classess = 2 num_feats = len(embeddings[0]) initializer = tf.contrib.layers.xavier_initializer() weights = { "w_t" : tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_t"), "w_l" : tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_l"), "w_r" : tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_r"), "w_attention" : tf.Variable(initializer([opt.feature_size,1]), name="w_attention") } biases = { "b_conv": tf.Variable(initializer([opt.feature_size,]), name="b_conv"), } left_nodes_node, left_children_node, right_nodes_node, right_children_node, hidden_node, left_score_node, right_score_node = network.init_net_for_siamese( num_feats, opt.feature_size, weights, biases, opt.aggregation, opt.distributed_function ) out_node = network.out_layer(hidden_node) labels_node, loss_node = network.loss_layer(hidden_node, n_classess) optimizer = tf.train.AdamOptimizer(LEARN_RATE) train_step = optimizer.minimize(loss_node) sess = tf.Session() # sess = tf.Session() sess.run(tf.global_variables_initializer()) with tf.name_scope('saver'): saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") saver.restore(sess, ckpt.model_checkpoint_path) for i, var in enumerate(saver._var_list): print('Var {}: {}'.format(i, var)) # else: # raise 'Checkpoint not found.' checkfile = os.path.join(logdir, 'cnn_tree.ckpt') steps = 0 print("Begin training....") # with tf.device(device): for epoch in range(1, epochs+1): print("----------------------------------------------------") for batch_left_trees, batch_right_trees, batch_labels in sampling.batch_random_samples_2_sides(train_left_trees, train_right_trees, train_labels, embeddings, embedding_lookup, opt.train_batch_size,opt.batch_type): if opt.batch_type == "original": left_nodes, left_children = batch_left_trees right_nodes, right_children = batch_right_trees else: left_nodes, left_children, _ = batch_left_trees right_nodes, right_children, _ = batch_right_trees labels_one_hot = convert_labels_to_one_hot(batch_labels) _, err, out = sess.run( [train_step, loss_node, out_node], feed_dict={ left_nodes_node: left_nodes, left_children_node: left_children, right_nodes_node: right_nodes, right_children_node: right_children, labels_node: labels_one_hot } ) # print "hidden : " + str(loss) print('Epoch:', epoch,'Steps:', steps,'Loss:', err) if steps % CHECKPOINT_EVERY == 0: # save state so we can resume later saver.save(sess, os.path.join(checkfile), steps) print('Checkpoint saved.') steps+=1 steps = 0 correct_labels = [] predictions = [] for batch_left_trees, batch_right_trees, batch_labels in sampling.batch_random_samples_2_sides(val_left_trees, val_right_trees, val_labels, embeddings, embedding_lookup, opt.train_batch_size, opt.batch_type): if opt.batch_type == "original": left_nodes, left_children = batch_left_trees right_nodes, right_children = batch_right_trees else: left_nodes, left_children, _ = batch_left_trees right_nodes, right_children, _ = batch_right_trees labels_one_hot = convert_labels_to_one_hot(batch_labels) output = sess.run( [out_node], feed_dict={ left_nodes_node: left_nodes, left_children_node: left_children, right_nodes_node: right_nodes, right_children_node: right_children, labels_node: labels_one_hot } ) correct = np.argmax(labels_one_hot, axis=1) predicted = np.argmax(output[0], axis=1) correct_labels.extend(correct) predictions.extend(predicted) print('Accuracy:', accuracy_score(correct_labels, predictions)) print(classification_report(correct_labels, predictions)) print(confusion_matrix(correct_labels, predictions))
def main(opt): target_directory = "live_test/github_java/sort_function/" file_name = aggregation_name + "_" + distributed_function_name + "_function.csv" print("Loading embeddings....") with open(opt.embeddings_directory, 'rb') as fh: embeddings, embed_lookup = pickle.load(fh,encoding='latin1') labels = [str(i) for i in range(1, opt.n_classes+1)] logdir = opt.model_path batch_size = opt.test_batch_size epochs = opt.niter node_embedding_size = len(embeddings[0]) # Loading program file # test_trees, node_ids, node_types, subtree_ids, pkl_path = load_program(opt) # Init model checkfile = os.path.join(logdir, 'cnn_tree.ckpt') ckpt = tf.train.get_checkpoint_state(logdir) initializer = tf.contrib.layers.xavier_initializer() weights = { "w_t" : tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_t"), "w_l" : tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_l"), "w_r" : tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_r"), "w_attention" : tf.Variable(initializer([opt.feature_size,1]), name="w_attention") } biases = { "b_conv": tf.Variable(initializer([opt.feature_size,]), name="b_conv"), } nodes_node, children_node, hidden_node, attention_score_node = network.init_net( node_embedding_size, len(labels), opt.feature_size, weights, biases, opt.aggregation, opt.distributed_function ) out_node = network.out_layer(hidden_node) labels_node, loss_node = network.loss_layer(hidden_node, len(labels)) optimizer = tf.train.AdamOptimizer(LEARN_RATE) train_step = optimizer.minimize(loss_node) saver = tf.train.Saver(save_relative_paths=True, max_to_keep=5) # Initialize the variables (i.e. assign their default value) init = tf.global_variables_initializer() logdir = opt.model_path batch_size = opt.test_batch_size epochs = opt.niter node_embedding_size = len(embeddings[0]) with tf.Session() as sess: sess.run(init) ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") print("Checkpoint path : " + str(ckpt.model_checkpoint_path)) saver.restore(sess, ckpt.model_checkpoint_path) for i, var in enumerate(saver._var_list): print('Var {}: {}'.format(i, var)) test_trees, node_ids, node_types, pkl_path, pb_path = load_program(opt.test_file) attention_score_scaled_map = predict(sess, out_node, attention_score_node, nodes_node, children_node, pkl_path, pb_path, test_trees, labels, node_ids, node_types, embeddings, embed_lookup) attention_path = os.path.join(opt.test_file.split(".")[0] + ".csv") if os.path.exists(attention_path): os.remove(attention_path) with open(attention_path,"a") as f: for k, v in attention_score_scaled_map.items(): f.write(str(k) + "," + str(v)) f.write("\n") generate_visualization(pb_path, attention_path)
def test_model(test_dataloader, embeddings, embedding_lookup, opt): logdir = opt.model_path epochs = opt.niter node_embedding_size = len(embeddings[0]) test_left_trees = test_dataloader.left_trees test_right_trees = test_dataloader.right_trees test_labels = test_dataloader.labels n_classess = 2 num_feats = len(embeddings[0]) initializer = tf.contrib.layers.xavier_initializer() weights = { "w_t": tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_t"), "w_l": tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_l"), "w_r": tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_r"), "w_attention": tf.Variable(initializer([opt.feature_size, 1]), name="w_attention") } biases = { "b_conv": tf.Variable(initializer([ opt.feature_size, ]), name="b_conv"), } left_nodes_node, left_children_node, right_nodes_node, right_children_node, logits_node, left_mask_nodes, right_mask_nodes, attention_matrix_nodes = network.init_net_for_siamese_2( num_feats, opt.feature_size, weights, biases, ) out_node = network.out_layer(logits_node) labels_node, loss_node = network.loss_layer(logits_node, n_classess) optimizer = tf.train.AdamOptimizer(LEARN_RATE) train_step = optimizer.minimize(loss_node) sess = tf.Session() # sess = tf.Session() sess.run(tf.global_variables_initializer()) with tf.name_scope('saver'): saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") saver.restore(sess, ckpt.model_checkpoint_path) for i, var in enumerate(saver._var_list): print('Var {}: {}'.format(i, var)) # else: # raise 'Checkpoint not found.' checkfile = os.path.join(logdir, 'cnn_tree.ckpt') steps = 0 print("Begin computing accuracy....") correct_labels = [] predictions = [] for batch_left_trees, batch_right_trees, batch_labels in sampling.batch_random_samples_2_sides( test_left_trees, test_right_trees, test_labels, embeddings, embedding_lookup, opt.train_batch_size, opt.batch_type): left_nodes, left_children, left_masks = batch_left_trees right_nodes, right_children, right_masks = batch_right_trees labels_one_hot = convert_labels_to_one_hot(batch_labels) matching_matrices, output = sess.run( [attention_matrix_nodes, out_node], feed_dict={ left_nodes_node: left_nodes, left_children_node: left_children, right_nodes_node: right_nodes, right_children_node: right_children, labels_node: labels_one_hot, left_mask_nodes: left_masks, right_mask_nodes: right_masks, }) for i, matrix in enumerate(matching_matrices): np.savetxt("matching_matrix/" + str(i) + ".csv", matrix, delimiter=",") print(output) print(output.shape) correct = np.argmax(labels_one_hot, axis=1) predicted = np.argmax(output, axis=1) correct_labels.extend(correct) predictions.extend(predicted) print('Accuracy:', accuracy_score(correct_labels, predictions)) print(classification_report(correct_labels, predictions)) print(confusion_matrix(correct_labels, predictions))
def train_model(logdir, infile, embedfile, epochs=EPOCHS, training="True", testing="True"): """Train a classifier to label ASTs""" print("Loading trees...") with open(infile, 'rb') as fh: trees, test_trees, labels = pickle.load(fh) random.shuffle(trees) print(labels) print("Loading embeddings....") with open(embedfile, 'rb') as fh: embeddings, embed_lookup = pickle.load(fh) num_feats = len(embeddings[0]) # build the inputs and outputs of the network nodes_node, children_node, hidden_node = network.init_net( num_feats, len(labels)) out_node = network.out_layer(hidden_node) labels_node, loss_node = network.loss_layer(hidden_node, len(labels)) optimizer = tf.train.AdamOptimizer(LEARN_RATE) train_step = optimizer.minimize(loss_node) tf.summary.scalar('loss', loss_node) ### init the graph sess = tf.Session() #config=tf.ConfigProto(device_count={'GPU':0})) sess.run(tf.global_variables_initializer()) with tf.name_scope('saver'): saver = tf.train.Saver() summaries = tf.summary.merge_all() writer = tf.summary.FileWriter(logdir, sess.graph) ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") saver.restore(sess, ckpt.model_checkpoint_path) checkfile = os.path.join(logdir, 'cnn_tree.ckpt') if training == "True": print("Begin training..........") num_batches = len(trees) // BATCH_SIZE + ( 1 if len(trees) % BATCH_SIZE != 0 else 0) for epoch in range(1, epochs + 1): for i, batch in enumerate( sampling.batch_samples( sampling.gen_samples(trees, labels, embeddings, embed_lookup), BATCH_SIZE)): nodes, children, batch_labels = batch step = (epoch - 1) * num_batches + i * BATCH_SIZE if not nodes: continue # don't try to train on an empty batch # print(batch_labels) _, summary, err, out = sess.run( [train_step, summaries, loss_node, out_node], feed_dict={ nodes_node: nodes, children_node: children, labels_node: batch_labels }) print('Epoch:', epoch, 'Step:', step, 'Loss:', err, 'Max nodes:', len(nodes[0])) writer.add_summary(summary, step) if step % CHECKPOINT_EVERY == 0: # save state so we can resume later saver.save(sess, os.path.join(checkfile), step) print('Checkpoint saved, epoch:' + str(epoch) + ', step: ' + str(step) + ', loss: ' + str(err) + '.') saver.save(sess, os.path.join(checkfile), step) # compute the training accuracy if testing == "True": correct_labels = [] predictions = [] print('Computing training accuracy...') for batch in sampling.batch_samples( sampling.gen_samples(test_trees, labels, embeddings, embed_lookup), 1): nodes, children, batch_labels = batch output = sess.run([out_node], feed_dict={ nodes_node: nodes, children_node: children, }) #print(output) correct_labels.append(np.argmax(batch_labels)) predictions.append(np.argmax(output)) target_names = list(labels) print('Accuracy:', accuracy_score(correct_labels, predictions)) print( classification_report(correct_labels, predictions, target_names=target_names)) print(confusion_matrix(correct_labels, predictions))
def train_model(train_dataloader, val_dataloader, embeddings, embedding_lookup, opt): logdir = opt.model_path epochs = opt.niter node_embedding_size = len(embeddings[0]) train_left_trees = train_dataloader.left_trees train_right_trees = train_dataloader.right_trees train_labels = train_dataloader.labels val_left_trees = val_dataloader.left_trees val_right_trees = val_dataloader.right_trees val_labels = val_dataloader.labels n_classess = 2 num_feats = len(embeddings[0]) initializer = tf.contrib.layers.xavier_initializer() left_nodes_node, left_children_node, right_nodes_node, right_children_node, logits_node, left_mask_nodes, right_mask_nodes, attention_matrix_nodes = network.init_net_for_siamese_3( num_feats, opt.feature_size) out_node = network.out_layer(logits_node) labels_node, loss_node = network.loss_layer(logits_node, n_classess) optimizer = tf.train.AdamOptimizer(LEARN_RATE) train_step = optimizer.minimize(loss_node) sess = tf.Session() # sess = tf.Session() sess.run(tf.global_variables_initializer()) with tf.name_scope('saver'): saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") saver.restore(sess, ckpt.model_checkpoint_path) for i, var in enumerate(saver._var_list): print('Var {}: {}'.format(i, var)) # else: # raise 'Checkpoint not found.' checkfile = os.path.join(logdir, 'cnn_tree.ckpt') steps = 0 print("Begin training....") # with tf.device(device): # temp_precision = 0.0 # temp_recall = 0.0 temp_accuracy = 0.0 for epoch in range(1, epochs + 1): for batch_left_trees, batch_right_trees, batch_labels in sampling.batch_random_samples_2_sides( train_left_trees, train_right_trees, train_labels, embeddings, embedding_lookup, opt.train_batch_size, opt.batch_type): left_nodes, left_children, left_masks = batch_left_trees right_nodes, right_children, right_masks = batch_right_trees labels_one_hot = convert_labels_to_one_hot(batch_labels) _, err, left_nodes_out, out = sess.run( [train_step, loss_node, left_nodes_node, out_node], feed_dict={ left_nodes_node: left_nodes, left_children_node: left_children, right_nodes_node: right_nodes, right_children_node: right_children, labels_node: labels_one_hot, left_mask_nodes: left_masks, right_mask_nodes: right_masks, }) print('Epoch:', epoch, 'Steps:', steps, 'Loss:', err, "Val Accuracy:", temp_accuracy) print(left_nodes_out.shape) if steps % CHECKPOINT_EVERY == 0: print("Checkpoint, validating.....") correct_labels = [] predictions = [] for batch_left_trees, batch_right_trees, batch_labels in sampling.batch_random_samples_2_sides( val_left_trees, val_right_trees, val_labels, embeddings, embedding_lookup, opt.train_batch_size, opt.batch_type): left_nodes, left_children, left_masks = batch_left_trees right_nodes, right_children, right_masks = batch_right_trees labels_one_hot = convert_labels_to_one_hot(batch_labels) output = sess.run( [out_node], feed_dict={ left_nodes_node: left_nodes, left_children_node: left_children, right_nodes_node: right_nodes, right_children_node: right_children, labels_node: labels_one_hot, left_mask_nodes: left_masks, right_mask_nodes: right_masks, }) correct = np.argmax(labels_one_hot, axis=1) predicted = np.argmax(output[0], axis=1) correct_labels.extend(correct) predictions.extend(predicted) accuracy = float(accuracy_score(correct_labels, predictions)) precision = float(precision_score(correct_labels, predictions)) recall = float(recall_score(correct_labels, predictions)) print('Accuracy:', accuracy) print(classification_report(correct_labels, predictions)) print(confusion_matrix(correct_labels, predictions)) if accuracy > temp_accuracy: temp_accuracy = accuracy with open("no_tbcnn_validation.txt", "w") as f: f.write(str(temp_accuracy)) # save state so we can resume later saver.save(sess, os.path.join(checkfile), steps) print('Checkpoint saved.') steps += 1 steps = 0
def test_model(test_dataloader, embeddings, embedding_lookup, opt): logdir = opt.model_path epochs = opt.niter node_embedding_size = len(embeddings[0]) test_left_trees = test_dataloader.left_trees test_right_trees = test_dataloader.right_trees test_labels = test_dataloader.labels test_left_node_ids_list = test_dataloader.left_node_ids_list test_right_node_ids_list = test_dataloader.right_node_ids_list print(test_left_node_ids_list) print("Num id left : " + str(len(test_left_node_ids_list[0]))) print("Num id right : " + str(len(test_right_node_ids_list[0]))) n_classess = 2 num_feats = len(embeddings[0]) initializer = tf.contrib.layers.xavier_initializer() weights = { "w_t": tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_t"), "w_l": tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_l"), "w_r": tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_r"), "w_attention": tf.Variable(initializer([opt.feature_size, 1]), name="w_attention") } biases = { "b_conv": tf.Variable(initializer([ opt.feature_size, ]), name="b_conv"), } left_nodes_node, left_children_node, right_nodes_node, right_children_node, logits_node, left_mask_nodes, right_mask_nodes, attention_matrix_nodes = network.init_net_for_siamese_2( num_feats, opt.feature_size, weights, biases, ) # left_nodes_node, left_children_node, right_nodes_node, right_children_node, logits_node, left_mask_nodes, right_mask_nodes, attention_matrix_nodes = network.init_net_for_siamese_3( # num_feats, # opt.feature_size # ) out_node = network.out_layer(logits_node) labels_node, loss_node = network.loss_layer(logits_node, n_classess) optimizer = tf.train.AdamOptimizer(LEARN_RATE) train_step = optimizer.minimize(loss_node) sess = tf.Session() # sess = tf.Session() sess.run(tf.global_variables_initializer()) with tf.name_scope('saver'): saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") saver.restore(sess, ckpt.model_checkpoint_path) for i, var in enumerate(saver._var_list): print('Var {}: {}'.format(i, var)) # else: # raise 'Checkpoint not found.' checkfile = os.path.join(logdir, 'cnn_tree.ckpt') steps = 0 print("Begin computing accuracy....") correct_labels = [] predictions = [] for batch_left_trees, batch_right_trees, batch_labels in sampling.batch_random_samples_2_sides( test_left_trees, test_right_trees, test_labels, embeddings, embedding_lookup, opt.train_batch_size, opt.batch_type): left_nodes, left_children, left_masks = batch_left_trees right_nodes, right_children, right_masks = batch_right_trees labels_one_hot = convert_labels_to_one_hot(batch_labels) matching_matrices, output = sess.run( [attention_matrix_nodes, out_node], feed_dict={ left_nodes_node: left_nodes, left_children_node: left_children, right_nodes_node: right_nodes, right_children_node: right_children, labels_node: labels_one_hot, left_mask_nodes: left_masks, right_mask_nodes: right_masks, }) matrix = matching_matrices[0] if len(test_left_node_ids_list[0]) < len(test_right_node_ids_list[0]): matrix = matrix[:len(test_left_node_ids_list[0]), ...] else: matrix = matrix[..., :len(test_right_node_ids_list[0])] matrix_pd = pd.DataFrame(data=matrix, index=test_left_node_ids_list[0], columns=test_right_node_ids_list[0]) matrix_pd.to_csv("live_test/github_pairwise_java/sort/1_matrix.csv", sep=",") left_matrix_aggregate_idx = matrix_pd.idxmax(axis=1) left_matrix_aggregate_idx.to_csv( "live_test/github_pairwise_java/sort/left_aggregate_attention_idx.csv", sep=",") right_matrix_aggregate_idx = matrix_pd.idxmax(axis=0) right_matrix_aggregate_idx.to_csv( "live_test/github_pairwise_java/sort/right_aggregate_attention_idx.csv", sep=",") left_matrix_aggregate = matrix_pd.max(axis=1) left_matrix_aggregate.to_csv( "live_test/github_pairwise_java/sort/left_aggregate_attention.csv", sep=",") left_matrix_max_dict = left_matrix_aggregate.to_dict() right_matrix_aggregate = matrix_pd.max(axis=0) right_matrix_aggregate.to_csv( "live_test/github_pairwise_java/sort/right_aggregate_attention.csv", sep=",") right_matrix_max_dict = right_matrix_aggregate.to_dict() left_scaled_attention_map = scale_attention(left_matrix_max_dict) right_scaled_attention_map = scale_attention(right_matrix_max_dict) left_attention_path = "live_test/github_pairwise_java/sort/left_attention_scaled.csv" left_pb_path = "github_java_sort_function_pb/5/3.java.pb" left_normal_html_path = "live_test/github_pairwise_java/sort/left_normal.html" with open( "live_test/github_pairwise_java/sort/left_attention_scaled.csv", "w") as f1: for key, score in enumerate(left_scaled_attention_map): line = str(key) + "," + str(score) f1.write("%s\n" % line) normal_cmd = "docker run --rm -v $(pwd):/e -it yijun/fast -H 0 -t -x " + left_attention_path + " " + left_pb_path + " > " + left_normal_html_path print(normal_cmd) os.system(normal_cmd) right_attention_path = "live_test/github_pairwise_java/sort/right_attention_scaled.csv" right_pb_path = "github_java_sort_function_pb/5/105.java.pb" right_normal_html_path = "live_test/github_pairwise_java/sort/right_normal.html" with open( "live_test/github_pairwise_java/sort/right_attention_scaled.csv", "w") as f1: for key, score in enumerate(right_scaled_attention_map): line = str(key) + "," + str(score) f1.write("%s\n" % line) normal_cmd = "docker run --rm -v $(pwd):/e -it yijun/fast -H 0 -t -x " + right_attention_path + " " + right_pb_path + " > " + right_normal_html_path print(normal_cmd) os.system(normal_cmd) print(output) print(labels_one_hot) correct = np.argmax(labels_one_hot, axis=1) predicted = np.argmax(output, axis=1) correct_labels.extend(correct) predictions.extend(predicted) print('Accuracy:', accuracy_score(correct_labels, predictions)) print(classification_report(correct_labels, predictions)) print(confusion_matrix(correct_labels, predictions))
def test_model(test_trees, labels, embeddings, embedding_lookup, opt): logdir = opt.model_path batch_size = opt.train_batch_size epochs = opt.niter node_embedding_size = len(embeddings[0]) random.shuffle(test_trees) checkfile = os.path.join(logdir, 'cnn_tree.ckpt') ckpt = tf.train.get_checkpoint_state(logdir) initializer = tf.contrib.layers.xavier_initializer() weights = { "w_t": tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_t"), "w_l": tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_l"), "w_r": tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_r"), "w_attention": tf.Variable(initializer([opt.feature_size, 1]), name="w_attention") } biases = { "b_conv": tf.Variable(initializer([ opt.feature_size, ]), name="b_conv"), } nodes_node, children_node, hidden_node, attention_score_node = network.init_net( node_embedding_size, len(labels), opt.feature_size, weights, biases, opt.aggregation, opt.distributed_function) out_node = network.out_layer(hidden_node) labels_node, loss_node = network.loss_layer(hidden_node, len(labels)) optimizer = tf.train.AdamOptimizer(LEARN_RATE) train_step = optimizer.minimize(loss_node) saver = tf.train.Saver(save_relative_paths=True, max_to_keep=5) # Initialize the variables (i.e. assign their default value) init = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init) ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") print("Checkpoint path : " + str(ckpt.model_checkpoint_path)) saver.restore(sess, ckpt.model_checkpoint_path) correct_labels = [] predictions = [] print('Computing training accuracy...') for batch in sampling.batch_samples( sampling.gen_samples(test_trees, labels, embeddings, embedding_lookup), 1): nodes, children, batch_labels = batch output = sess.run([out_node], feed_dict={ nodes_node: nodes, children_node: children, }) # print(attention_score[0]) # print(len(attention_score[0])) # print(output) correct_labels.append(np.argmax(batch_labels)) predictions.append(np.argmax(output)) target_names = list(labels) print('Accuracy:', accuracy_score(correct_labels, predictions)) print( classification_report(correct_labels, predictions, target_names=target_names)) print(confusion_matrix(correct_labels, predictions))
def train_model(train_trees, val_trees, labels, embeddings, embedding_lookup, opt): """Train a classifier to label ASTs""" logdir = opt.model_path batch_size = opt.train_batch_size epochs = opt.niter node_embedding_size = len(embeddings[0]) random.shuffle(train_trees) random.shuffle(val_trees) # random.shuffle(test_trees) checkfile = os.path.join(logdir, 'cnn_tree.ckpt') ckpt = tf.train.get_checkpoint_state(logdir) initializer = tf.contrib.layers.xavier_initializer() weights = { "w_t": tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_t"), "w_l": tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_l"), "w_r": tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_r"), "w_attention": tf.Variable(initializer([opt.feature_size, 1]), name="w_attention") } biases = { "b_conv": tf.Variable(initializer([ opt.feature_size, ]), name="b_conv"), } nodes_node, children_node, hidden_node, attention_score_node = network.init_net( node_embedding_size, len(labels), opt.feature_size, weights, biases, opt.aggregation, opt.distributed_function) out_node = network.out_layer(hidden_node) labels_node, loss_node = network.loss_layer(hidden_node, len(labels)) optimizer = tf.train.AdamOptimizer(LEARN_RATE) train_step = optimizer.minimize(loss_node) saver = tf.train.Saver(save_relative_paths=True, max_to_keep=5) # Initialize the variables (i.e. assign their default value) init = tf.global_variables_initializer() if opt.training: print("Begin training..........") with tf.Session() as sess: sess.run(init) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") print("Checkpoint path : " + str(ckpt.model_checkpoint_path)) saver.restore(sess, ckpt.model_checkpoint_path) for i, var in enumerate(saver._var_list): print('Var {}: {}'.format(i, var)) # saved_model.loader.load(sess, [tag_constants.TRAINING], savedmodel_path) num_batches = len(train_trees) // batch_size + ( 1 if len(train_trees) % batch_size != 0 else 0) for epoch in range(1, epochs + 1): for i, batch in enumerate( sampling.batch_samples( sampling.gen_samples(train_trees, labels, embeddings, embedding_lookup), batch_size)): nodes, children, batch_labels = batch # print(len(batch_labels)) # print(len(batch_labels[0])) step = (epoch - 1) * num_batches + i * BATCH_SIZE if not nodes: continue # don't try to train on an empty batch # print(batch_labels) _, err, out = sess.run( [train_step, loss_node, out_node], feed_dict={ nodes_node: nodes, children_node: children, labels_node: batch_labels }) print('Epoch:', epoch, 'Step:', step, 'Loss:', err, 'Max nodes:', len(nodes[0])) if step % CHECKPOINT_EVERY == 0: # save state so we can resume later saver.save(sess, checkfile) # shutil.rmtree(savedmodel_path) print('Checkpoint saved, epoch:' + str(epoch) + ', step: ' + str(step) + ', loss: ' + str(err) + '.') correct_labels = [] predictions = [] for batch in sampling.batch_samples( sampling.gen_samples(val_trees, labels, embeddings, embedding_lookup), 1): nodes, children, batch_labels = batch output = sess.run([out_node], feed_dict={ nodes_node: nodes, children_node: children, }) # print(output) correct_labels.append(np.argmax(batch_labels)) predictions.append(np.argmax(output)) target_names = list(labels) print('Accuracy:', accuracy_score(correct_labels, predictions)) print( classification_report(correct_labels, predictions, target_names=target_names)) print(confusion_matrix(correct_labels, predictions)) print("Finish all iters, storring the whole model..........") saver.save(sess, checkfile)
def test_model(logdir, inputs, embeddings_list_url, node_map_url, epochs=EPOCHS): """Train a classifier to label ASTs""" n_classess = 2 print("Loading embedding list.....") with open(embeddings_list_url, "rb") as embeddings_list_fh: embeddings_list = pickle.load(embeddings_list_fh) num_feats = len(embeddings_list[0]) print("number of features : " + str(num_feats)) print("Loading node map for looking up.....") with open(node_map_url, "rb") as node_map_fh: # all_1_pairs, all_0_pairs = pickle.load(fh) node_map = pickle.load(node_map_fh) # build the inputs and outputs of the network left_nodes_node, left_children_node, left_pooling_node = network.init_net_for_siamese( num_feats) right_nodes_node, right_children_node, right_pooling_node = network.init_net_for_siamese( num_feats) merge_node = tf.concat([left_pooling_node, right_pooling_node], -1) hidden_node = network.hidden_layer(merge_node, 600, 300) # hidden_node = tf.layers.dropout(hidden_node, rate=0.2, training=False) hidden_node = network.hidden_layer(hidden_node, 300, 100) # hidden_node = tf.layers.dropout(hidden_node, rate=0.2, training=False) hidden_node = network.hidden_layer(hidden_node, 100, n_classess) out_node = network.out_layer(hidden_node) labels_node, loss_node = network.loss_layer(hidden_node, n_classess) optimizer = tf.train.AdamOptimizer(LEARN_RATE) train_step = optimizer.minimize(loss_node) # tf.summary.scalar('loss', loss_node) ### init the graph sess = tf.Session() #config=tf.ConfigProto(device_count={'GPU':0})) sess.run(tf.global_variables_initializer()) with tf.name_scope('saver'): saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) else: raise 'Checkpoint not found.' checkfile = os.path.join(logdir, 'cnn_tree.ckpt') steps = 0 correct_labels = [] predictions = [] print('Computing testing accuracy...') with open( inputs, "rb", ) as csvfile: # with codecs.open("data/test.csv", "r", encoding = "utf-8", errors = 'replace') as csvfile: test_data_reader = csv.DictReader(csvfile, delimiter=',') for row in test_data_reader: print("----------------------") print(smart_str(row['test_id'])) print(smart_str(row['question1'])) print(smart_str(row['question2'])) try: left_tree, right_tree = get_trees(smart_str(row['question1']), smart_str(row['question2'])) left_nodes, left_children, right_nodes, right_children = sampling.patch_data( left_tree, right_tree, embeddings_list, node_map) # for left_nodes, left_children, right_nodes, right_children in sampling.patch_data(left_tree, right_tree, embeddings_list, node_map): # left_nodes, left_children = left_gen_batch # right_nodes, right_children = right_gen_batch output = sess.run( [out_node], feed_dict={ left_nodes_node: left_nodes, left_children_node: left_children, right_nodes_node: right_nodes, right_children_node: right_children }) print(output) predicted = np.argmax(output[0]) print(predicted) with open("data/predict_proba2.csv", "a") as f2: f2.write(row['test_id'] + "," + str(format(output[0][0][1], "f")) + "\n") except Exception as e: print "Error : " + str(e) with open("data/predict_proba2.csv", "a") as f2: f2.write(row['test_id'] + "," + "0" + "\n")
def train_model(train_trees, val_trees, labels, embeddings, embedding_lookup, opt): max_acc = 0.0 logdir = opt.model_path batch_size = opt.train_batch_size epochs = opt.niter num_feats = len(embeddings[0]) random.shuffle(train_trees) nodes_node, children_node, codecaps_node = network.init_net_treecaps( num_feats, len(labels)) codecaps_node = tf.identity(codecaps_node, name="codecaps_node") out_node = network.out_layer(codecaps_node) labels_node, loss_node = network.loss_layer(codecaps_node, len(labels)) optimizer = RAdamOptimizer(opt.lr) train_step = optimizer.minimize(loss_node) ### init the graph sess = tf.Session() sess.run(tf.global_variables_initializer()) # Initialize the variables (i.e. assign their default value) init = tf.global_variables_initializer() with tf.name_scope('saver'): saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") saver.restore(sess, ckpt.model_checkpoint_path) for i, var in enumerate(saver._var_list): print('Var {}: {}'.format(i, var)) checkfile = os.path.join(logdir, 'tree_network.ckpt') print("Begin training..........") num_batches = len(train_trees) // batch_size + ( 1 if len(train_trees) % batch_size != 0 else 0) for epoch in range(1, epochs + 1): bar = progressbar.ProgressBar(maxval=len(train_trees), widgets=[ progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage() ]) bar.start() for i, batch in enumerate( sampling.batch_samples( sampling.gen_samples(train_trees, labels, embeddings, embedding_lookup), batch_size)): nodes, children, batch_labels = batch step = (epoch - 1) * num_batches + i * batch_size if not nodes: continue _, err, out = sess.run( [train_step, loss_node, out_node], feed_dict={ nodes_node: nodes, children_node: children, labels_node: batch_labels }) bar.update(i + 1) bar.finish() correct_labels = [] predictions = [] logits = [] for batch in sampling.batch_samples( sampling.gen_samples(val_trees, labels, embeddings, embedding_lookup), 1): nodes, children, batch_labels = batch output = sess.run([out_node], feed_dict={ nodes_node: nodes, children_node: children }) correct_labels.append(np.argmax(batch_labels)) predictions.append(np.argmax(output)) logits.append(output) target_names = list(labels) acc = accuracy_score(correct_labels, predictions) if (acc > max_acc): max_acc = acc saver.save(sess, checkfile) np.save(opt.model_path + '/logits', np.array(logits)) np.save(opt.model_path + '/correct', np.array(correct_labels)) print('Epoch', str(epoch), 'Accuracy:', acc, 'Max Acc: ', max_acc) csv_log.write(str(epoch) + ',' + str(acc) + ',' + str(max_acc) + '\n') print("Finish all iters, storring the whole model..........")
def test_model(logdir, inputs, left_embedfile, right_embedfile, epochs=EPOCHS): """Train a classifier to label ASTs""" n_classess = 2 # left_algo_labels = ['mergesort', 'linkedlist', 'quicksort', 'bfs', 'bubblesort', 'knapsack'] # right_algo_labels = ['mergesort', 'linkedlist', 'quicksort', 'bfs', 'bubblesort', 'knapsack'] left_algo_labels = [ "bfs", "bubblesort", "knapsack", "linkedlist", "mergesort", "quicksort", "heap", "dfs", "stack", "queue" ] right_algo_labels = [ "bfs", "bubblesort", "knapsack", "linkedlist", "mergesort", "quicksort", "heap", "dfs", "stack", "queue" ] # with open(left_inputs, 'rb') as fh: # _, left_trees, left_algo_labels = pickle.load(fh) # with open(right_inputs, 'rb') as fh: # _, right_trees, right_algo_labels = pickle.load(fh) with open(inputs, "rb") as fh: testing_pairs = pickle.load(fh) print "Loading embdding vectors...." with open(left_embedfile, 'rb') as fh: left_embeddings, left_embed_lookup = pickle.load(fh) with open(right_embedfile, 'rb') as fh: right_embeddings, right_embed_lookup = pickle.load(fh) num_feats = len(left_embeddings[0]) # build the inputs and outputs of the network left_nodes_node, left_children_node, left_pooling_node = network.init_net_for_siamese( num_feats) right_nodes_node, right_children_node, right_pooling_node = network.init_net_for_siamese( num_feats) merge_node = tf.concat([left_pooling_node, right_pooling_node], -1) hidden_node = network.hidden_layer(merge_node, 200, 200) # hidden_node = tf.layers.dropout(hidden_node, rate=0.2, training=False) hidden_node = network.hidden_layer(hidden_node, 200, 200) # hidden_node = tf.layers.dropout(hidden_node, rate=0.2, training=False) hidden_node = network.hidden_layer(hidden_node, 200, n_classess) out_node = network.out_layer(hidden_node) labels_node, loss_node = network.loss_layer(hidden_node, n_classess) optimizer = tf.train.AdamOptimizer(LEARN_RATE) train_step = optimizer.minimize(loss_node) # tf.summary.scalar('loss', loss_node) ### init the graph sess = tf.Session() #config=tf.ConfigProto(device_count={'GPU':0})) sess.run(tf.global_variables_initializer()) with tf.name_scope('saver'): saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) else: raise 'Checkpoint not found.' checkfile = os.path.join(logdir, 'cnn_tree.ckpt') steps = 0 left_trees, right_trees = get_trees_from_pairs(testing_pairs) using_vector_lookup_left = False if os.path.isfile("/input/config.json"): file_handler = open(config_file, 'r') contents = json.load(file_handler) using_vector_lookup_left = contents[ 'using_vector_lookup_left'] == "false" correct_labels = [] predictions = [] print('Computing testing accuracy...') for left_gen_batch, right_gen_batch in sampling.batch_random_samples_2_sides( left_trees, left_algo_labels, right_trees, right_algo_labels, left_embeddings, left_embed_lookup, right_embeddings, right_embed_lookup, using_vector_lookup_left, False, TEST_BATCH_SIZE): left_nodes, left_children, left_labels_one_hot, left_labels = left_gen_batch right_nodes, right_children, right_labels_one_hot, right_labels = right_gen_batch sim_labels, _ = get_one_hot_similarity_label(left_labels, right_labels) print("sim labels : " + str(sim_labels)) output = sess.run( [out_node], feed_dict={ left_nodes_node: left_nodes, left_children_node: left_children, right_nodes_node: right_nodes, right_children_node: right_children, labels_node: sim_labels }) correct = np.argmax(sim_labels[0]) predicted = np.argmax(output[0]) check = (correct == predicted) and True or False print('Out:', output, "Status:", check) correct_labels.append(np.argmax(sim_labels[0])) predictions.append(np.argmax(output[0])) target_names = ["0", "1"] print('Accuracy:', accuracy_score(correct_labels, predictions)) print( classification_report(correct_labels, predictions, target_names=target_names)) print(confusion_matrix(correct_labels, predictions))
def main(): print("Loading embeddings....") with open(opt.embeddings_directory, 'rb') as fh: embeddings, embed_lookup = pickle.load(fh, encoding='latin1') logdir = opt.model_path epochs = opt.niter node_embedding_size = len(embeddings[0]) n_classess = 2 num_feats = len(embeddings[0]) initializer = tf.contrib.layers.xavier_initializer() weights = { "w_t": tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_t"), "w_l": tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_l"), "w_r": tf.Variable(initializer([node_embedding_size, opt.feature_size]), name="w_r"), "w_attention": tf.Variable(initializer([opt.feature_size, 1]), name="w_attention") } biases = { "b_conv": tf.Variable(initializer([ opt.feature_size, ]), name="b_conv"), } left_nodes_node, left_children_node, right_nodes_node, right_children_node, hidden_node, left_score_node, right_score_node = network.init_net_for_siamese( num_feats, opt.feature_size, weights, biases, opt.aggregation, opt.distributed_function) out_node = network.out_layer(hidden_node) labels_node, loss_node = network.loss_layer(hidden_node, n_classess) optimizer = tf.train.AdamOptimizer(LEARN_RATE) train_step = optimizer.minimize(loss_node) sess = tf.Session() # sess = tf.Session() sess.run(tf.global_variables_initializer()) with tf.name_scope('saver'): saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") saver.restore(sess, ckpt.model_checkpoint_path) for i, var in enumerate(saver._var_list): print('Var {}: {}'.format(i, var)) # else: # raise 'Checkpoint not found.' checkfile = os.path.join(logdir, 'cnn_tree.ckpt') steps = 0 print("Begin computing accuracy....") test_file = opt.test_data # with open(test_file, "r") as f: # data = f.readlines() # for line in data: # line = line.replace("\n","") # test_dataloader = CrossLanguageProgramDataForLiveTest(test_file, 1,opt.n_classes) # test_left_trees = test_dataloader.left_trees # test_right_trees = test_dataloader.right_trees # test_labels = test_dataloader.labels all_pairs_index = [] with open(opt.test_data, "r") as f: data = f.readlines() for line in data: print(line) all_pairs_index.append(line.replace("\n", "")) for i, pair in tqdm(enumerate(all_pairs_index)): splits = pair.split(",") left_path = splits[0] right_path = splits[1] label = splits[2] pairs, left_node_ids_list, right_node_ids_list = load_single_pair_for_live_test( left_path, right_path, label) predict(sess, i, left_path, right_path, pairs, left_node_ids_list, right_node_ids_list, out_node, left_score_node, right_score_node, left_nodes_node, left_children_node, right_nodes_node, right_children_node, labels_node, embeddings, embed_lookup, opt)
def train_model(logdir, inputs, embeddings_list_url, node_map_url, epochs=EPOCHS, with_drop_out=1, device="0"): os.environ['CUDA_VISIBLE_DEVICES'] = device print("Using device : " + device) print("Batch size : " + str(BATCH_SIZE)) if int(with_drop_out) == 1: print("Training with drop out rate : " + str(DROP_OUT)) n_classess = 2 print("Loading training data....") # print "Using device : " + device with open(inputs, "rb") as fh: # all_1_pairs, all_0_pairs = pickle.load(fh) all_training_pairs = pickle.load(fh) random.shuffle(all_training_pairs) print("Loading embedding list.....") with open(embeddings_list_url, "rb") as embeddings_list_fh: embeddings_list = pickle.load(embeddings_list_fh) num_feats = len(embeddings_list[0]) print("number of features : " + str(num_feats)) print("Loading node map for looking up.....") with open(node_map_url, "rb") as node_map_fh: # all_1_pairs, all_0_pairs = pickle.load(fh) node_map = pickle.load(node_map_fh) # build the inputs and outputs of the network left_nodes_node, left_children_node, left_pooling_node = network.init_net_for_siamese( num_feats) right_nodes_node, right_children_node, right_pooling_node = network.init_net_for_siamese( num_feats) # with tf.device(device): print("Left pooling shape : " + str(tf.shape(left_pooling_node))) print("Right pooling shape : " + str(tf.shape(right_pooling_node))) merge_node = tf.concat([left_pooling_node, right_pooling_node], -1) print(tf.shape(merge_node)) hidden_node = network.hidden_layer(merge_node, 600, 300) if int(with_drop_out) == 1: hidden_node = tf.layers.dropout(hidden_node, rate=DROP_OUT, training=True) hidden_node = network.hidden_layer(hidden_node, 300, 100) if int(with_drop_out) == 1: hidden_node = tf.layers.dropout(hidden_node, rate=DROP_OUT, training=True) hidden_node = network.hidden_layer(hidden_node, 100, n_classess) if int(with_drop_out) == 1: hidden_node = tf.layers.dropout(hidden_node, rate=DROP_OUT, training=True) out_node = network.out_layer(hidden_node) labels_node, loss_node = network.loss_layer(hidden_node, n_classess) optimizer = tf.train.AdamOptimizer(LEARN_RATE) train_step = optimizer.minimize(loss_node) # tf.summary.scalar('loss', loss_node) # correct_prediction = tf.equal(tf.argmax(out_node,1), tf.argmax(labels_node,1)) # accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) ### init the graph # config = tf.ConfigProto(allow_soft_placement=True) # config.gpu_options.allocator_type = 'BFC' # config.gpu_options.per_process_gpu_memory_fraction = 0.9 # config = tf.ConfigProto() # config.gpu_options.allow_growth = True config = tf.ConfigProto() # config.gpu_options.allocator_type ='BFC' # config.gpu_options.allow_growth = True # config.gpu_options.per_process_gpu_memory_fraction = 0.98 sess = tf.Session(config=config) # sess = tf.Session() sess.run(tf.global_variables_initializer()) with tf.name_scope('saver'): saver = tf.train.Saver() summaries = tf.summary.merge_all() writer = tf.summary.FileWriter(logdir, sess.graph) ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") saver.restore(sess, ckpt.model_checkpoint_path) # else: # raise 'Checkpoint not found.' checkfile = os.path.join(logdir, 'cnn_tree.ckpt') steps = 0 using_vector_lookup_left = False if os.path.isfile("/input/config.json"): file_handler = open(config_file, 'r') contents = json.load(file_handler) using_vector_lookup_left = contents[ 'using_vector_lookup_left'] == "false" print("Begin training....") # with tf.device(device): for epoch in range(1, epochs + 1): # sample_1_pairs = random.sample(all_1_pairs,1000) # sample_0_pairs = random.sample(all_0_pairs,1000) # sample_training_pairs = random.sample(all_training_pairs,6400) shuffle_left_trees, shuffle_right_trees, labels = get_trees_from_pairs( all_training_pairs) print("Len left:", len(shuffle_left_trees), "Len right:", len(shuffle_right_trees)) for left_gen_batch, right_gen_batch, labels_batch in sampling.batch_random_samples_2_sides( shuffle_left_trees, shuffle_right_trees, embeddings_list, node_map, labels, BATCH_SIZE): print("----------------------------------------------------") print("Len of label batch : " + str(labels_batch)) left_nodes, left_children = left_gen_batch right_nodes, right_children = right_gen_batch sim_labels, sim_labels_num = get_one_hot_similarity_label( labels_batch) # print("sim labels : " + str(sim_labels)) _, err, out, merge, labs = sess.run( [train_step, loss_node, out_node, merge_node, labels_node], feed_dict={ left_nodes_node: left_nodes, left_children_node: left_children, right_nodes_node: right_nodes, right_children_node: right_children, labels_node: sim_labels }) # print "hidden : " + str(loss) print('Epoch:', epoch, 'Steps:', steps, 'Loss:', err, "True Label vs Predicted Label:", zip(labs, out)) # print('Epoch:', epoch,'Steps:', steps,'Loss:', err) if steps % CHECKPOINT_EVERY == 0: # save state so we can resume later saver.save(sess, os.path.join(checkfile), steps) print('Checkpoint saved.') steps += 1 steps = 0
def train_model(train_trees, val_trees, labels, embedding_lookup, opt): max_acc = 0.0 logdir = opt.model_path batch_size = opt.train_batch_size epochs = opt.niter random.shuffle(train_trees) nodes_node, children_node, codecaps_node = network.init_net_treecaps(50, embedding_lookup, len(labels)) codecaps_node = tf.identity(codecaps_node, name="codecaps_node") out_node = network.out_layer(codecaps_node) labels_node, loss_node = network.loss_layer(codecaps_node, len(labels)) optimizer = RAdamOptimizer(opt.lr) train_point = optimizer.minimize(loss_node) ### init the graph sess = tf.Session() sess.run(tf.global_variables_initializer()) # Initialize the variables (i.e. assign their default value) init = tf.global_variables_initializer() with tf.name_scope('saver'): saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") saver.restore(sess, ckpt.model_checkpoint_path) for i, var in enumerate(saver._var_list): print('Var {}: {}'.format(i, var)) checkfile = os.path.join(logdir, 'tree_network.ckpt') print("Begin training..........") num_batches = len(train_trees) // batch_size + (1 if len(train_trees) % batch_size != 0 else 0) max_acc = 0.0 for epoch in range(1, epochs+1): for train_step, train_batch in enumerate(sampling.batch_samples( sampling.gen_samples(train_trees, labels), batch_size )): nodes, children, batch_labels = train_batch # step = (epoch - 1) * num_batches + train_step * batch_size if not nodes: continue _, err, out = sess.run( [train_point, loss_node, out_node], feed_dict={ nodes_node: nodes, children_node: children, labels_node: batch_labels } ) print("Epoch : ", str(epoch), "Step : ", train_step, "Loss : ", err, "Max Acc: ",max_acc) if train_step % 1000 == 0 and train_step > 0: correct_labels = [] predictions = [] # logits = [] for test_batch in sampling.batch_samples( sampling.gen_samples(val_trees, labels), batch_size ): print("---------------") nodes, children, batch_labels = test_batch print(batch_labels) output = sess.run([out_node], feed_dict={ nodes_node: nodes, children_node: children } ) batch_correct_labels = np.argmax(batch_labels, axis=1) batch_predictions = np.argmax(output[0], axis=1) correct_labels.extend(batch_correct_labels) predictions.extend(batch_predictions) # logits.append(output) print(batch_correct_labels) print(batch_predictions) acc = accuracy_score(correct_labels, predictions) if (acc>max_acc): max_acc = acc saver.save(sess, checkfile) print("Saved checkpoint....") print('Epoch',str(epoch),'Accuracy:', acc, 'Max Acc: ',max_acc) csv_log.write(str(epoch)+','+str(acc)+','+str(max_acc)+'\n') print("Finish all iters, storring the whole model..........")