def testing_on_data(type_of_data, X_test, Y_test, model, batch_size, summary_only=False): # Predicting print "====================== %s ======================" %(type_of_data) print "Predicting... (SEPARATELY FOR EACH DOCUMENT)" data = { 'wd_r': [], 'wd_e': [], 'pk': [] } avg_segment_lengths_across_test_data = [] # Average segment length across the documents for Xi_test, Yi_test in zip(X_test, Y_test): pred_per_doc = [] Xi_test, Yi_test = Xi_test.reshape((1,) + Xi_test.shape), Yi_test.reshape((1,) + Yi_test.shape) # Convert to format of 1 document for batch_X_left, batch_X_mid, batch_X_right, batch_Y_mid in batch_gen_consecutive_context_segments_from_big_seq(Xi_test, Yi_test, batch_size, ONE_SIDE_CONTEXT_SIZE): batch_y_pred = model.predict_on_batch([batch_X_left, batch_X_mid, batch_X_right]) pred_per_doc.append(batch_y_pred) if not len(pred_per_doc): # batch generator might drop a few documents continue #rounded = np.round(pred_per_doc) pred_per_doc = np.concatenate(pred_per_doc, axis=0) actual_avg_seg_length, result = helper.windiff_and_pk_metric_ONE_SEQUENCE(Yi_test[0], pred_per_doc, window_size=-1, rounded=False, print_individual_stats=not summary_only) avg_segment_lengths_across_test_data.append(actual_avg_seg_length) data['pk'].append(result['pk']) data['wd_r'].append(result['wd_r']) data['wd_e'].append(result['wd_e']) print ">> Summary (%s):" %(type_of_data) print "AVG segment length in test data: %f" % (np.mean(avg_segment_lengths_across_test_data)) print "WinDiff metric (Windiff_r):: avg: %f | std: %f | min: %f | max: %f" %(np.mean(data['wd_r']), np.std(data['wd_r']), np.min(data['wd_r']), np.max(data['wd_r'])) print "WinDiff metric (Windiff_e):: avg: %f | std: %f | min: %f | max: %f" %(np.mean(data['wd_e']), np.std(data['wd_e']), np.min(data['wd_e']), np.max(data['wd_e'])) print "Pk metric:: avg: %f | std: %f | min: %f | max: %f" %(np.mean(data['pk']), np.std(data['pk']), np.min(data['pk']), np.max(data['pk'])) print('___________________________________')
def save_predictions(type_of_data, X_test, Y_test, model, batch_size, summary_only=False): # Predicting print "====================== %s ======================" %(type_of_data) print "GET PREDICTIONS... (SEPARATELY FOR EACH DOCUMENT)" data = { 'wd_r': [], 'wd_e': [], 'pk': [] } doc_idx = 18 avg_segment_lengths_across_test_data = [] # Average segment length across the documents predictions_return = [] zipped = zip(X_test, Y_test) for i, (Xi_test, Yi_test) in enumerate(zipped): if i != doc_idx: continue print Xi_test.shape pred_per_doc = [] Xi_test, Yi_test = Xi_test.reshape((1,) + Xi_test.shape), Yi_test.reshape((1,) + Yi_test.shape) # Convert to format of 1 document for batch_X_left, batch_X_mid, batch_X_right, batch_Y_mid in batch_gen_consecutive_context_segments_from_big_seq("test", Xi_test, Yi_test, batch_size, ONE_SIDE_CONTEXT_SIZE): batch_y_pred = model.predict_on_batch([batch_X_left, batch_X_mid, batch_X_right]) pred_per_doc.append(batch_y_pred) if not len(pred_per_doc): # batch generator might drop a few documents continue #rounded = np.round(pred_per_doc) pred_per_doc = np.concatenate(pred_per_doc, axis=0) #return pred_per_doc predictions_return.append(pred_per_doc) actual_avg_seg_length, result = helper.windiff_and_pk_metric_ONE_SEQUENCE(Yi_test[0], pred_per_doc, window_size=-1, rounded=False, print_individual_stats=not summary_only) avg_segment_lengths_across_test_data.append(actual_avg_seg_length) data['pk'].append(result['pk']) data['wd_r'].append(result['wd_r']) data['wd_e'].append(result['wd_e']) print "WD: %f, PK: %f" %(result['wd_r'], result['pk']) # Save for visualization #rounded_per_doc = np.round(pred_per_doc) rounded_per_doc = pred_per_doc output = ["ref,hyp"] for (ref, hyp) in zip(Y_test[doc_idx], rounded_per_doc): output.append(str(int(ref[0])) + "," + str(hyp[0])) file_name = "prediction_output_save.csv" with open(file_name, "a") as f: for line in output: f.write(line + "\r\n") print "Written document index: `%d` to file: `%s`" %(doc_idx, file_name) return
def RANDOM_testing_on_data(type_of_data, X_test, Y_test, model, batch_size, summary_only=False, visualize=False): # Predicting print "====================== %s ======================" % (type_of_data) print "Predicting... (SEPARATELY FOR EACH DOCUMENT)" data = {'wd_r': [], 'wd_e': [], 'pk': []} avg_segment_lengths_across_test_data = [ ] # Average segment length across the documents for Xi_test, Yi_test in zip(X_test, Y_test): pred_per_doc = [] Xi_test, Yi_test = Xi_test.reshape( (1, ) + Xi_test.shape), Yi_test.reshape( (1, ) + Yi_test.shape) # Convert to format of 1 document original_yi = np.copy(Yi_test[0]) pred_per_doc = np.copy(Yi_test[0]) np.random.shuffle(pred_per_doc) actual_avg_seg_length, result = helper.windiff_and_pk_metric_ONE_SEQUENCE( original_yi, pred_per_doc, window_size=-1, rounded=False, print_individual_stats=not summary_only) avg_segment_lengths_across_test_data.append(actual_avg_seg_length) data['pk'].append(result['pk']) data['wd_r'].append(result['wd_r']) data['wd_e'].append(result['wd_e']) print ">> Summary (%s):" % (type_of_data) print "AVG segment length in test data: %f, std: %f" % ( np.mean(avg_segment_lengths_across_test_data), np.std(avg_segment_lengths_across_test_data)) print "WinDiff metric (Windiff_r):: avg: %f | std: %f | min: %f | max: %f" % ( np.mean(data['wd_r']), np.std(data['wd_r']), np.min( data['wd_r']), np.max(data['wd_r'])) print "WinDiff metric (Windiff_e):: avg: %f | std: %f | min: %f | max: %f" % ( np.mean(data['wd_e']), np.std(data['wd_e']), np.min( data['wd_e']), np.max(data['wd_e'])) print "Pk metric:: avg: %f | std: %f | min: %f | max: %f" % (np.mean( data['pk']), np.std(data['pk']), np.min(data['pk']), np.max( data['pk'])) print('___________________________________')
def testing_on_data(type_of_data, X_test, Y_test, model, batch_size, summary_only=False, visualize=False): # Predicting print "====================== %s ======================" %(type_of_data) print "Predicting... (SEPARATELY FOR EACH DOCUMENT)" data = { 'wd_r': [], 'wd_e': [], 'pk': [] } avg_segment_lengths_across_test_data = [] # Average segment length across the documents for Xi_test, Yi_test in zip(X_test, Y_test): pred_per_doc = [] for batch_count, (batch_X, batch_Y) in enumerate(batch_gen_sentences_without_context(Xi_test, Yi_test, batch_size, fixed_size=False)): start_time = time.time() batch_y_pred = model.predict_on_batch([batch_X]) #print time.time() - start_time, "sec/batch of size", batch_size pred_per_doc.append(batch_y_pred[:,1]) if not len(pred_per_doc): # batch generator might drop a few documents continue if len(pred_per_doc) > Xi_test.shape[0]: pred_per_doc = pred_per_doc[:Xi_test.shape[0]] #rounded = np.round(pred_per_doc) pred_per_doc = np.concatenate(pred_per_doc, axis=0) if visualize: print ">>>>> VISUALIZE <<<<<<" pdb.set_trace() #pdb.set_trace() Yi_test_for_windiff = to_categorical(Yi_test)[:,1] actual_avg_seg_length, result = helper.windiff_and_pk_metric_ONE_SEQUENCE(Yi_test_for_windiff, pred_per_doc, window_size=-1, rounded=False, print_individual_stats=not summary_only) avg_segment_lengths_across_test_data.append(actual_avg_seg_length) data['pk'].append(result['pk']) data['wd_r'].append(result['wd_r']) data['wd_e'].append(result['wd_e']) print ">> Summary (%s):" %(type_of_data) print "AVG segment length in test data: %f, std: %f" % (np.mean(avg_segment_lengths_across_test_data), np.std(avg_segment_lengths_across_test_data)) print "WinDiff metric (Windiff_r):: avg: %f | std: %f | min: %f | max: %f" %(np.mean(data['wd_r']), np.std(data['wd_r']), np.min(data['wd_r']), np.max(data['wd_r'])) print "WinDiff metric (Windiff_e):: avg: %f | std: %f | min: %f | max: %f" %(np.mean(data['wd_e']), np.std(data['wd_e']), np.min(data['wd_e']), np.max(data['wd_e'])) print "Pk metric:: avg: %f | std: %f | min: %f | max: %f" %(np.mean(data['pk']), np.std(data['pk']), np.min(data['pk']), np.max(data['pk'])) print('___________________________________')