def evaluate(segments, triggers, metrics): """Applies metrics to given segments and triggers. Parameters ---------- segments : `~gwpy.segments.DataQualityFlag` segments to be analyzed. triggers : `~gwpy.segments.SnglInspiralTable` triggers to be analyzed. metrics : `Metric`, `list` metric or list of metrics with which to evaluate segments and triggers Returns ------- results : `float`, `list` result(s) produced by the metric(s). """ # needs type checking here if isinstance(metrics, Metric): results = metrics(segments, triggers) elif isinstance(metrics, list): results = [] for metric in metrics: results.append(metric(segments, triggers)) else: raise Exception('Third argument must be a metric or list of metrics.') return results
def train(model, loader, optim, loss_func, metrics, device, **kwargs): model.train() avg = RunningAverageMultiVar(loss=RunningAverage(), auc=RunningAverage()) progbar = tqdm(loader) disp_step = kwargs['display_step'] viz = kwargs['viz'] epoch = kwargs['epoch'] for idx, sample in enumerate(progbar): ref_img_batch = sample['ref'].to(device) srch_img_batch = sample['srch'].to(device) label_batch = sample['label'].to(device) score_map = model(ref_img_batch, srch_img_batch) loss = loss_func(score_map=score_map, labels=label_batch) optim.zero_grad() loss.backward() optim.step() loss_val = loss.to('cpu').item() if idx % disp_step == 0 and viz is not None: display_images(sample['ref'][0], sample['srch'][0], score_map.to('cpu')[0], sample['label'][0], viz=viz) auc_val = metrics(score_map.detach().cpu().numpy(), label_batch.detach().cpu().numpy()) avg.update(loss=loss_val, auc=auc_val) plot_2d_line(loss_val, idx, epoch, type='train', kinds='loss', viz=viz) plot_2d_line(auc_val, idx, epoch, type='train', kinds='auc', viz=viz) progbar.set_postfix({ "Loss": "%0.4f" % avg['loss'](), "AUC Score": "%0.3f" % avg['auc']() }) torch.cuda.empty_cache() return avg['loss'](), avg['auc']()
def __init__(self): ''' 1. setup connection to cassandra 2. intialise logger object ''' try: self.db = dblayer() self.metricsObj = metrics() except Exception,e: logging.error("chunker:__init__ failed with error %s", e) sys.exit(1)
def main(): # data frame with our dataset labels_A, labels_C, df_features_class_A, df_features_class_C = get_BAS_dataset( "data/HUM/", "data/FAK/") # preprocessing of the data to capture the features we want #df_features_class_A = get_class_A_features(df) #df_features_class_C = get_class_C_features(df) # classifiers (training + prediction) class_A_classifiers_predictions = classify(df_features_class_A, labels_A) class_C_classifiers_predictions = classify(df_features_class_C, labels_C) # Metrics on the result predictions from the classifiers results_class_A_classifiers = metrics(labels_A, class_A_classifiers_predictions) results_class_C_classifiers = metrics(labels_C, class_C_classifiers_predictions) # publish and save results print("Metrics of the Class A classifiers(profile)") for key, value in results_class_A_classifiers.items(): print("Algorithm: " + key) print("Accuracy: " + str(round(value[0], 3)) + " Precision: " + str(round(value[1], 3))) print("Recall: " + str(round(value[2], 3)) + " F-M: " + str(round(value[3], 3))) print("MCC: " + str(round(value[4], 3)) + " AUC: " + str(round(value[5], 3))) print() #publish(results_class_A_classifiers) print("===================== oOo ========================") print("Metrics of the Class C classifiers(all features)") for key, value in results_class_C_classifiers.items(): print("Algorithm: " + key) print("Accuracy: " + str(round(value[0], 3)) + " Precision: " + str(round(value[1], 3))) print("Recall: " + str(round(value[2], 3)) + " F-M: " + str(round(value[3], 3))) print("MCC: " + str(round(value[4], 3)) + " AUC: " + str(round(value[5], 3))) print()
def evaluate(model, loader, loss_func, metrics, device, **kwargs): epoch = kwargs['epoch'] model.eval() avg = RunningAverageMultiVar(loss=RunningAverage(), auc=RunningAverage()) for idx, sample in enumerate(loader): ref_image = sample['ref'].to(device) srch_image = sample['srch'].to(device) label = sample['label'].to(device) score_map = model(ref_image, srch_image) loss = loss_func(score_map, label) loss_val = loss.to('cpu').item() avg.update(loss=loss_val, auc=metrics(score_map.detach().cpu().numpy(), label.detach().cpu().numpy())) plot_2d_line(avg['loss'](), idx, epoch, 'eval', 'loss') plot_2d_line(avg['auc'](), idx, epoch, 'eval', 'auc') torch.cuda.empty_cache() return avg['loss'](), avg['auc']()
#!/usr/bin/python import sys from metrics import * from analyzor import * from common import * lines = readCfg(sys.argv[1]) consoleCfg = parseConsole(lines) ManagementPlaneAudit = metrics() console = ManagementPlaneAudit.addMetric('consolePort') print analyzorConsole(consoleCfg, console, lines)
def model_train(model, input_placeholder, output_placeholder, train, val, batch_size, num_epochs=10, save_path="/tensorflow/dr", summary_dir=None): """ input: input_placeholder: A placeholder to feed the network with images output_placeholder: A placeholder to feed the true labels of the input images keep_prob: A placeholder to feed the network with probability value . useful for generalization and avoiding overfitting. drop_out: dropout need to be added input_image_shape: the input shape of the image, A tuple (h,w) actual_train: the actual training data. train: the oversampled train data. Incase of No oversampling it is the same as actual train val: list of val images locations labels_frame: the truth labels of the input images, index as image location. train_step: the optimizer to back prop the error accuracy: the accuracy of the model batch_size: the size of the batch num_epochs: total number of epochs save_path: path to save the model at different checkpoints summary_dir: path to write the various summaries written. output: save model at defined checkpoints, prints the val_accuracy after every epoch process: 1) calculate the total number of iterations 2) ensemble both input_pipelines defined in reader.py 3) save all the variables 4) merge all the summaries 5) initiate the network 6) in session , pass a batch of images to the network and train the model - calculate train_summaries and add to the summaries_dir - After every-epoch, run the eval pipeline for one epoch and calculate the accuracy and print it. - save the model """ num_epochs_per_decay = 2.0 iterations = (int(50000 / batch_size)) * num_epochs decay_steps = int(50000 / batch_size * num_epochs_per_decay) if tf.gfile.Exists(summary_dir): tf.gfile.DeleteRecursively(summary_dir) #decay_steps = tf.constant(decay_steps, name="decay_steps") with tf.name_scope("cross_entropy"): out = tf.nn.softmax_cross_entropy_with_logits( labels=output_placeholder, logits=model) with tf.name_scope("total"): cross_entropy = tf.reduce_mean(out) tf.summary.scalar("cross_entropy", cross_entropy) with tf.name_scope("train"): global_step = tf.Variable(0, name='global_step', trainable=False) optimizer = tf.train.AdamOptimizer(0.001) train_step = optimizer.minimize(cross_entropy, global_step=global_step) tf.add_to_collection("train_step", train_step) tf.summary.scalar("global_step", global_step) #tf.summary.scalar("learning_rate", learning_rate) #tf.summary.scalar("optimizer", optimizer) ## setting up the three pipelines train_images, train_labels = train_input_pipeline(train, batch_size=batch_size, num_epochs=num_epochs) #Pipeline for calculating training accuracy TA (training accuracy) TA_images, TA_labels = eval_input_pipeline(train, batch_size=batch_size, num_epochs=num_epochs + 1) #Pipeline for calculating Validation accuracy VA (validation accuracy) VA_images, VA_labels = eval_input_pipeline(val, batch_size, num_epochs + 1) saver = tf.train.Saver(tf.global_variables(), max_to_keep=20) merged = tf.summary.merge_all() init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) with tf.Session() as sess: sess.run(init_op) training_writer = tf.summary.FileWriter(summary_dir + '/training', sess.graph) train_writer = tf.summary.FileWriter(summary_dir + '/train') valid_writer = tf.summary.FileWriter(summary_dir + '/valid') if tf.gfile.Exists(save_path): filename = tf.train.latest_checkpoint(save_path) print("[Restoring the last checkpoint with filename]", filename) saver.restore(sess, filename) #train_step = tf.get_collection("train_step")[0] print("model restored") g = sess.run(global_step) print("[global step] : ", g) print("starting from epoch:", int(batch_size * g / 50000)) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) print("[queue runners started]") try: print("[Training has initiated]") i, step = 0, 0 step_tb = 0 while not coord.should_stop(): images, labels = sess.run([train_images, train_labels]) summary, _ = sess.run([merged, train_step], feed_dict={ input_placeholder: images, output_placeholder: dummy(labels) }) training_writer.add_summary(summary, i) i += 1 if i == 1 or i == step * int(50000 / batch_size): #### calculating training and testing accuracy tb, t_p, t_t, v_p, v_t = metrics( step_tb, merged, sess, train_writer, valid_writer, input_placeholder, output_placeholder, val, batch_size, model, TA_images, TA_labels, VA_images, VA_labels) step_tb += tb train_kappa = quadratic_weighted_kappa(t_t, t_p, min_rating=0, max_rating=99) test_kappa = quadratic_weighted_kappa(v_t, v_p, min_rating=0, max_rating=99) print("step:", i, "train_kappa:", train_kappa, "valid_kappa:", test_kappa, "train_acc:", accuracy_score(t_t, t_p), "valid_acc:", accuracy_score(v_t, v_p)) if not tf.gfile.IsDirectory(save_path): tf.gfile.MkDir(save_path) checkpoint_path = os.path.join(save_path, 'model.ckpt') saver.save(sess, checkpoint_path, global_step=global_step) tf.train.write_graph(sess.graph_def, '/tmp/fmp_model', 'train_{}.pb'.format(step)) tf.train.write_graph(sess.graph_def, '/tmp/fmp_model', 'train_{}.pbtxt'.format(step), as_text=True) step += 1 except tf.errors.OutOfRangeError: print("Done training -- Maximum epoch limit has reached") finally: coord.request_stop() coord.request_stop() coord.join(threads)
L_, E_, A_, n_, nodes_array_ = load_LON(lons_foldername + "/" + filename) results1 = sorted([path_length(s, A, n) for s in L]) # ascending results2 = sorted([path_length(s, A_, n_) for s in L_]) # ascending g_min = min(results1[0], results2[0]) generate_image(L_, E_, A_, n_, images_foldername + "/" + out_file, g_min) generate_projection_image( L, E, L_, E_, A, n, projection_images_foldername + "/" + out_file, g_min) def main_part7__projection_metrics(projections_foldername, projection_metrics_foldername): for filename in os.listdir(projections_foldername): L, E, A, n, nodes_array = load_LON(projections_foldername + "/" + filename) out_file = filename.split('.')[0] generate_projection_metrics( L, E, A, n, projection_metrics_foldername + "/" + out_file) if __name__ == '__main__': metrics("checking_data", "checking/1_lons", "checking/3_performance_metrics", "checking/4_network_metrics", 1000) # metrics("data/data_bays29", "new_results/1_lons", "new_results/3_performance_metrics", "new_results/4_network_metrics", 1000)
else: __builtin__.ipv4_mgmt_inbound = None print stdout_header() # configuration file reading lines = read_cfg(args.iosconfig) __builtin__.wholeconfig = lines # Cisco IOS configuration file type checking check_cfg(lines) # Basic configuration info __builtin__.genericCfg = addBasicInfo(lines) # Add metrics for the Management Plane. MgmtPlane = metrics() # Add metrics for the Control Plane. CtrlPlane = CPmetrics() # Add metrics for the Data Plane. DataPlane = DPmetrics() # Add metric for the interfaces. interfaces = IFSmetrics() # Add metric for the IPv4 ACLs. AclsV4 = ACLV4metrics() # Add metric for the IPv6 ACLs. AclsV6 = ACLV6metrics() # Find interfaces (ifaceCfg). ifaceCfg = populate_ifaces(lines,interfaces) for i in range(0, len(ifaceCfg)): ifaceCfg[i].get_metrics_from_config()
import sys sys.path.append('../.') from common.config import * from common.Database import * from OneSingleGaussian import * from common.extractPerformance import * from GaussianColor import * from metrics import * if __name__ == "__main__": gt_db = Database(abs_dir_gt, start_frame=start_frame, end_frame=end_frame) input_db = Database(abs_dir_input, start_frame=start_frame, end_frame=end_frame) # results_db = Database(abs_dir_result, start_frame=0) gt = gt_db.loadDB(im_color=False) input = input_db.loadDB(im_color=True, color_space="HSV", im_show=False) params = np.arange(0, 10, 0.2) # matrix_mean, matrix_std, gt_test = GaussianColorRGB(input, array_alpha, im_show=False) matrix_mean, matrix_std, gt_test = GaussianColorHSV(input, params) gt2 = gt[(len(gt)//2):] TP_list, FP_list, TN_list, FN_list = extractPerformance(gt2, gt_test, array_params=params) precision_list, recall_list, fscore_list, accuracy_list = metrics(TP_list, FP_list, TN_list, FN_list, gt_test, array_params=params) plotF1Score2D(np.linspace(0, params[-1], len(params)), fscore_list) plotPrecisionRecall(recall_list, precision_list, label=DATABASE)
# Generate model print("Generating model.") print("-" * 10) weights, loss = model(y, tX) M, N = tX.shape # Replace invalid values with mean of valid values and standardize tX = modifyCSV(tX) # Remove features due to correlation tX = featureRemoval(tX) tX = np.concatenate((np.ones((M, 1)), tX, np.sin(tX), np.cos(tX)), axis=1) # Metrics F1, accuracy = metrics(y, predict_labels(weights, tX)) print("F1-score: {}, accuracy: {}.".format(F1, accuracy)) print("-" * 10) # Generating predictions print("Generating predictions for {}.".format(test_path)) print("-" * 10) _, tX_test, ids_test = load_csv_data(test_path) M_test, N_test = tX_test.shape # Replace invalid values with mean of valid values and standardize tX_test = modifyCSV(tX_test) # Remove features due to correlation tX_test = featureRemoval(tX_test) tX_test = np.concatenate((np.ones(
from skimage.io import imread from skimage.segmentation import slic from groundtruth import * from metrics import * # --------------------------------- # when executed, run the main script # --------------------------------- if __name__ == '__main__': img_path = 'data/Berkeley/train/' names = os.listdir(img_path) for name in names[:5]: # Load the input image img = imread(img_path + name) print("Processing image " + name[:-4]) # Performs the segmentation labels = slic(img, n_segments=300, compactness=10.0) # Load the ground truth segments = get_segment_from_filename(name[:-4]) # Evaluate metrics m = metrics(img, labels, segments) m.set_metrics() m.display_metrics()
else: __builtin__.ipv4_mgmt_inbound = None print stdout_header() # configuration file reading lines = read_cfg(args.iosconfig) __builtin__.wholeconfig = lines # Cisco IOS configuration file type checking check_cfg(lines) # Basic configuration info __builtin__.genericCfg = addBasicInfo(lines) # Add metrics for the Management Plane. MgmtPlane = metrics() # Add metrics for the Control Plane. CtrlPlane = CPmetrics() # Add metrics for the Data Plane. DataPlane = DPmetrics() # Add metric for the interfaces. interfaces = IFSmetrics() # Add metric for the IPv4 ACLs. AclsV4 = ACLV4metrics() # Add metric for the IPv6 ACLs. AclsV6 = ACLV6metrics() # Find interfaces (ifaceCfg). ifaceCfg = populate_ifaces(lines, interfaces) for i in range(0, len(ifaceCfg)): ifaceCfg[i].get_metrics_from_config()
) #BackgroundSubtractorMOG() for id, p in enumerate(params): bg_substracttor = cv2.bgsegm.createBackgroundSubtractorMOG() #TODO x_mask.append( extractBackgroundSubtrasctor_CV(bg_substractor, data=input, im_show=False)) sys.stdout.write("\r> Computing ... {:.2f}%".format( (id + 1) * 100 / len(params))) sys.stdout.flush() print("\n") TP_list, FP_list, TN_list, FN_list = extractPerformance( gt, x_mask, array_params=params) precision_list, recall_list, fscore_list, accuracy_list = metrics( TP_list, FP_list, TN_list, FN_list, x_mask, array_params=params) if SHOW_PLOT: plotF1Score2D(np.linspace(0, params[-1], len(params)), fscore_list) plotPrecisionRecall(recall_list, precision_list, label=DATABASE) except AttributeError: warnings.warn( "BackgroundSubtractorMOG does not exist in your OpenCV {}".format( cv2.__version__)) """ BGSubstractorMOG2 """ params = np.linspace(1, 100, 20) x_mask = [] print("\n\nBackgroundSubtractorMOG2 Method (from OpenCV {})".format(