def run_eval(self, sess, data, summary_writer=None, step=0): y, y_pred, loss_, metrics, p_k = list(), list(), 0.0, None, None accuracy, loss = 0.0, 0.0 merged_summary = self.summarizer.merge_all() i = 0 for X, Y, tot in self.data.next_batch(data): feed_dict = {self.x: X, self.y: Y, self.z : np.random.random((Y.shape[0], self.config.solver.randomvar_dim)), self.keep_prob: 1} if i == tot-1 and summary_writer is not None: if data == "validation": summ, loss_ = sess.run([merged_summary, self.loss], feed_dict=feed_dict) else : summ, loss_, accuracy_val = sess.run([merged_summary, self.loss, self.accuracy], feed_dict=feed_dict) summary_writer.add_summary(summ, step) else: if data == "validation": loss_, Y_pred= sess.run([self.loss, tf.nn.sigmoid(self.y_pred)], feed_dict=feed_dict) p_k = patk(predictions=Y_pred, labels=Y) else : loss_, Y_pred, accuracy_val = sess.run([self.loss, tf.nn.sigmoid(self.y_pred), self.accuracy], feed_dict=feed_dict) metrics = evaluate(predictions=Y_pred, labels=Y) p_k = patk(predictions=Y_pred, labels=Y) accuracy += accuracy_val #metrics['accuracy'] loss += loss_ i += 1 return loss / i , accuracy / self.config.batch_size, metrics, p_k
def do_eval(self, sess, data): if data == "validation": err, accuracy = list(), list() X, Y = self.data.get_validation() for train, val in self.kf.split(X, y=Y): feed_dict = {self.x: X[val], self.y: Y[val], self.keep_prob: 1} loss_, Y_pred, accuracy_ = sess.run([self.loss, self.y_pred, self.accuracy], feed_dict=feed_dict) metrics = evaluate(predictions=Y_pred, labels=Y[val]) err.append(loss_) accuracy.append(accuracy_) return np.mean(err), np.mean(accuracy), metrics if data == "test": X, Y = self.data.get_test() feed_dict = {self.x: X, self.y: Y, self.keep_prob: 1} loss_, Y_pred, accuracy_ = sess.run([self.loss, self.y_pred, self.accuracy], feed_dict=feed_dict) metrics = evaluate(predictions=Y_pred, labels=Y) return loss_, accuracy_, metrics
def get_metrics(self, sess, data): accuracy, y_pred, i = 0.0, None, 0.0 for X, Y, tot in self.data.next_batch(data): feed_dict = {self.x: X, self.y: Y, self.keep_prob: 1.0} Y_pred, accuracy_val = sess.run([tf.nn.sigmoid(self.y_pred), self.accuracy], feed_dict=feed_dict) metrics = evaluate(predictions=Y_pred, labels=Y) p_k = patk(predictions=Y_pred, labels=Y) accuracy += accuracy_val i += 1 return metrics, accuracy / i, p_k
def tune_model(config, entity_embedding, Q, labels, train_ids, val_ids): pred_ids = val_ids labelled_ids = train_ids Y_hat = np.dot(Q, entity_embedding.T) Y_hat = Y_hat.T Y_hat = normalize(Y_hat, axis=1, norm='l2') performances = ep.evaluate(Y_hat[pred_ids, :], labels[pred_ids, :], threshold=0, multi_label=config.MULTI_LABEL) return performances
def tune_model_using_svm(config, entity_embedding, labels, train_ids, val_ids): pred_ids = val_ids labelled_ids = train_ids clf = OneVsRestClassifier(LinearSVC(random_state=0)) entity_embedding = normalize(entity_embedding, axis=1, norm='l2') clf.fit(entity_embedding[labelled_ids, :], labels[labelled_ids, :]) predictions = clf.decision_function(entity_embedding[pred_ids, :]) performances = ep.evaluate(predictions, labels[pred_ids, :], threshold=0, multi_label=config.MULTI_LABEL) return performances
def tune_model_using_lr(config, entity_embedding, labels, train_ids, val_ids): pred_ids = val_ids labelled_ids = train_ids clf = OneVsRestClassifier(LogisticRegression()) entity_embedding = normalize(entity_embedding, axis=1, norm='l2') clf.fit(entity_embedding[labelled_ids, :], labels[labelled_ids, :]) predictions = clf.predict_proba(entity_embedding[pred_ids, :]) performances = ep.evaluate(predictions, labels[pred_ids, :], threshold=0, multi_label=config.MULTI_LABEL) return performances
def predict_results(self, sess, data, preds=None): if preds == None: preds = self.dataset.label_cache # nodes = np.where(self.dataset.get_nodes(data))[0] # labels_pred = preds[nodes] # labels_orig = preds[nodes] labels_orig, labels_pred = [], [] for node in np.where(self.dataset.get_nodes(data))[0]: labels_orig.append(self.dataset.all_labels[node]) labels_pred.append(preds[node]) return perf.evaluate(labels_pred, labels_orig)
def run_eval(self, sess, data, summary_writer=None, step=0, type_loss="NORMAL"): y, y_pred, loss_, metrics, p_k, Y, Y_pred = list(), list( ), 0.0, None, None, None, None accuracy, loss = 0.0, 0.0 merged_summary = self.summarizer.merge_all() i = 0 for X, Y, tot in self.data.next_batch(data): feed_dict = {self.x: X, self.y: Y, self.keep_prob: 1} if (type_loss == "AUTO"): summ, loss_ = sess.run([merged_summary, self.autoencoder_loss], feed_dict=feed_dict) else: if i == tot - 1 and summary_writer is not None: if data == "validation": summ, loss_ = sess.run([merged_summary, self.loss], feed_dict=feed_dict) else: summ, loss_, accuracy_val = sess.run( [merged_summary, self.loss, self.accuracy], feed_dict=feed_dict) summary_writer.add_summary(summ, step) else: if data == "validation": loss_, Y_pred = sess.run( [self.loss, tf.nn.sigmoid(self.y_pred)], feed_dict=feed_dict) p_k = patk(predictions=Y_pred, labels=Y) else: loss_, Y_pred, accuracy_val = sess.run( [ self.loss, tf.nn.sigmoid(self.y_pred), self.accuracy ], feed_dict=feed_dict) metrics = evaluate(predictions=Y_pred, labels=Y) accuracy += accuracy_val #metrics['accuracy'] loss += loss_ i += 1 if data == "test": #X, Y = self.data.get_test() p_k = patk( predictions=Y_pred, labels=Y ) #sess.run(self.patk, feed_dict={self.x: X, self.y: Y, self.keep_prob: 1}) return loss / i, accuracy / self.config.batch_size, metrics, p_k
def run_eval(self, sess, data_type, epoch, summary_writer=None): merged_summary = self.summarizer.merge_all() i = 0 if data_type == "validation": X, Y = self.data.get_validation() elif data_type == 'test': X, Y = self.data.get_test() feed_dict = {self.x: X, self.y: Y, self.keep_prob: 1} # if only test data summary_writer is None if summary_writer is not None: summ, loss = sess.run([merged_summary, self.loss], feed_dict=feed_dict) summary_writer.add_summary(summ, epoch) loss, Y_pred = sess.run([self.loss, self.pred], feed_dict=feed_dict) metrics = evaluate(predictions=Y_pred, labels=Y) mean_loss = loss / X.shape[0] return mean_loss, metrics
def run_epoch(self, sess, data, train_op=None, summary_writer=None, verbose=1, learning_rate=0, get_emb=False): train = train_op if train_op is None: train_op = tf.no_op() keep_prob_in = 1 keep_prob_out = 1 else: keep_prob_in = self.config.mRNN._keep_prob_in keep_prob_out = self.config.mRNN._keep_prob_out # Set up all variables total_steps = np.sum( self.dataset.get_nodes(data)) # Number of Nodes to run through verbose = min(verbose, total_steps) node_ids, gradients, targets, attn_values, gating_values, emb = np.zeros( total_steps, dtype=int), [], [], [], [], np.zeros( (total_steps, self.config.mRNN._hidden_size)) losses, predictions, metrics, entropy = dict(), dict(), dict(), dict() metrics['node'], metrics['path'], metrics['combined'] = [], [], [] predictions['node'], predictions['path'], predictions['combined'] = np.zeros((total_steps, self.config.data_sets._len_labels)), \ np.zeros((total_steps, self.config.data_sets._len_labels)), \ np.zeros((total_steps, self.config.data_sets._len_labels)) losses['node'], losses['path'], losses['combined'], losses[ 'total'] = [], [], [], [] ######################################################################################################## feed_dict = { self.ph_keep_prob_in: keep_prob_in, self.ph_keep_prob_out: keep_prob_out, self.ph_wce: self.dataset.wce, self.ph_lr: learning_rate } # Reset grad accumulator at the beginning sess.run([self.reset_grads], feed_dict=feed_dict) #Start Running Queue t = threading.Thread(target=self.load_and_enqueue, args=[sess, data]) t.daemon = True t.start() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) #Code profiling # options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) # run_metadata = tf.RunMetadata() step = 0 while step < total_steps: feed_dict = { self.ph_keep_prob_in: keep_prob_in, self.ph_keep_prob_out: keep_prob_out, self.ph_wce: self.dataset.wce, self.ph_lr: learning_rate } if ((step < total_steps - 1) or not self.config.summaries) or summary_writer == None: id, grads, t_losses, t_pred_probs, target_label, t_attn_values, t_gating_values, t_entropy, t_emb = \ sess.run([self.node_id, train_op, self.losses, self.predictions, self.y_labels, self.arch.attn_values, self.arch.gating_values, self.entropy, self.emb], feed_dict=feed_dict)#, options=options, run_metadata=run_metadata) else: summary, id, grads, t_losses, t_pred_probs, target_label, t_attn_values, t_gating_values, t_entropy, t_emb = \ sess.run([self.summary, self.node_id, train_op, self.losses, self.predictions, self.y_labels, self.arch.attn_values, self.arch.gating_values, self.entropy, self.emb], feed_dict=feed_dict)#, options=options, run_metadata=run_metadata) #if summary_writer is not None: summary_writer.add_summary( summary, self.arch.global_step.eval(session=sess)) summary_writer.flush() #Saving code profile # fetched_timeline = timeline.Timeline(run_metadata.step_stats) # chrome_trace = fetched_timeline.generate_chrome_trace_format() # with open('timeline_02_step_%d.json' % step, 'w') as f: # f.write(chrome_trace) node_ids[step] = id # Accumulate attention values attn_values.append(np.std(t_attn_values[t_attn_values.nonzero()])) gating_values.append( np.abs(t_gating_values[0] - t_gating_values[1])) # Accumulate losses losses['node'].append(t_losses[0]) losses['path'].append(t_losses[1]) losses['combined'].append(t_losses[2]) losses['total'].append(t_losses[3]) #Accumulate entropy of prediction entropy[id] = t_entropy #accumulate mebeddings if get_emb: emb[step] = t_emb[0] # Accumulate Predictions for k, v in t_pred_probs.items(): predictions[k][step] = v targets.append(np.squeeze(target_label)) step += 1 if train and (step % self.config.batch_size == 0 or step == total_steps): # Update gradients after batch_size or at the end of the current epoch #print("Queue size: ", sess.run([self.Q.size()])) batch_size = self.config.batch_size if step == total_steps: batch_size = step % batch_size feed_dict[self.ph_batch_size] = batch_size sess.run([self.update_op], feed_dict=feed_dict) sess.run([self.reset_grads], feed_dict=feed_dict) if verbose and self.config.solver.gradients: # get the absolute maximum gradient to each variable gradients.append([np.max(np.abs(item)) for item in grads]) print("%d/%d :: " % (step, total_steps), end="") for var, val in zip([ '-'.join(k.name.split('/')[-2:]) for k in tf.trainable_variables() ], np.mean(gradients, axis=0)): print("%s :: %.8f " % (var, val / self.config.batch_size), end="") print("\n") sys.stdout.flush() coord.request_stop() coord.join(threads) # Average statistics over batches for k in losses.keys(): losses[k] = np.mean(losses[k]) for k in metrics.keys(): _, metrics[k] = perf.evaluate( np.asarray(predictions[k]), np.asarray(targets), multi_label=self.config.data_sets._multi_label) #Hack around to store attn and gating aggreagtes metrics['combined']['pak'] = np.mean(attn_values) metrics['combined']['average_precision'] = np.mean(gating_values) #return raw_predictions return node_ids, predictions, losses, metrics, np.asarray( attn_values), np.asarray(gating_values), np.mean( list(entropy.values())), emb
def run_epoch(self, sess, data, train_op=None, summary_writer=None, verbose=1): #Optimize the objective for one entire epoch via mini-batches if not train_op: train_op = tf.no_op() keep_prob_in = 1 keep_prob_out = 1 else: keep_prob_in = self.config.mRNN._keep_prob_in keep_prob_out = self.config.mRNN._keep_prob_out total_loss, label_loss = [], [] f1_micro, f1_macro, accuracy, bae = [], [], [], [] for step, (input_batch, input_batch2, seq, label_batch, tot, lengths, mask) in enumerate( self.dataset.next_batch(data, self.config.batch_size, shuffle=True)): # print("\n\n\nActualLabelCount: ", np.shape(input_batch), np.shape(input_batch2), np.shape(label_batch), np.shape(seq)) feed_dict = self.create_feed_dict(input_batch, input_batch2, label_batch) feed_dict[self.keep_prob_in] = keep_prob_in feed_dict[self.keep_prob_out] = keep_prob_out feed_dict[self.wce_placeholder] = self.dataset.wce feed_dict[self.mask] = mask feed_dict[self.inp_lengths] = lengths # Writes loss summary @last step of the epoch if (step + 1) < tot: _, loss_value, pred_labels = sess.run( [train_op, self.loss, self.arch.label_preds], feed_dict=feed_dict) else: _, loss_value, summary, pred_labels = sess.run( [train_op, self.loss, self.summary, self.arch.label_preds], feed_dict=feed_dict) if summary_writer != None: summary_writer.add_summary( summary, self.arch.global_step.eval(session=sess)) summary_writer.flush() total_loss.append(loss_value[0]) label_loss.append(loss_value[1]) if verbose and step % verbose == 0: metrics = [0] * 10 if self.config.solver._curr_label_loss: metrics = perf.evaluate(pred_labels, label_batch) f1_micro.append(metrics['micro_f1']) f1_macro.append(metrics['macro_f1']) accuracy.append(metrics['accuracy']) bae.append(metrics['bae']) print( '%d/%d : label = %0.4f : micro-F1 = %0.3f : accuracy = %0.3f : bae = %0.3f' % (step, tot, np.mean(label_loss), np.mean(f1_micro), np.mean(accuracy), np.mean(bae)), end="\r") sys.stdout.flush() if verbose: sys.stdout.write('\r') return np.mean(total_loss), np.mean(f1_micro), np.mean( f1_macro), np.mean(accuracy), np.mean(bae)