def main(): K = 50 trainData = np.load('train.npy') testData = np.load('test.npy') # UNCOMMENT BELOW AND REMOVE FURTHER BELOW FOR FASTER PERFORMANCE ON MULTIPLE RUNS print("[INFO] Building Correlation Matrix") try: corrMatrix = np.load('correlation_matrix.npy') except FileNotFoundError: corrMatrix = build_corrMatrix(trainData, 'correlation_matrix.npy') t0 = time.process_time() # REMOVE BELOW FOR FASTER PERFORMANCE ON MULTIPLE RUNS, AND UNCOMMENT ABOVE # corrMatrix = build_corrMatrix(trainData, 'correlation_matrix.npy') print("[INFO] Running") reconstructedTrainBasic = collaborative_basic(trainData, testData, corrMatrix, K) RMSEbasic, SRCbasic, precisionTopKbasic = get_metrics(reconstructedTrainBasic, testData) t1 = time.process_time() print('basic: RMSE = {}; SRC = {}; Precision on top K = {}; time taken = {}'.format(RMSEbasic, SRCbasic, precisionTopKbasic, t1-t0)) # REMOVE BELOW FOR FASTER PERFORMANCE ON MULTIPLE RUNS, AND UNCOMMENT ABOVE # corrMatrix = build_corrMatrix(trainData, 'correlation_matrix.npy') reconstructedTrainBaseline = collaborative_baseline(trainData, testData, corrMatrix, K) RMSEbaseline, SRCbaseline, precisionTopKbaseline = get_metrics(reconstructedTrainBaseline, testData) t2 = time.process_time() print('baseline: RMSE = {}; SRC = {}; Precision on top K = {}; time taken = {}'.format(RMSEbaseline, SRCbaseline, precisionTopKbaseline, t2-t1))
def eval_single_model(self, indexes): model = self.model[indexes]['model'] # loss_fn = self.model[indexes]['loss_fn'] # optimizer = self.model[indexes]['optimizer'] Xtrain = self.data['Xtrain'] Xvalid = self.data['Xvalid'] Xtest = self.data['Xtest'] ytrain = self.data['ytrain'] yvalid = self.data['yvalid'] ytest = self.data['ytest'] ztrain = self.data['ztrain'] zvalid = self.data['zvalid'] ztest = self.data['ztest'] model.eval() ypred_valid = model(Xvalid) zpred_valid = None if self.adversarial: adv_model = self.model[indexes]['adv_model'] adv_model.eval() if self.method == 'parity': adv_input_valid = ypred_valid zpred_valid = adv_model(adv_input_valid) elif self.method == 'odds': adv_input_valid = torch.cat((ypred_valid, yvalid), 1) zpred_valid = adv_model(adv_input_valid) elif self.method == 'opportunity': zpred_valid = None if zpred_valid is not None: metrics_valid = pd.DataFrame(get_metrics(ypred_valid.data.numpy(), yvalid.data.numpy(), zvalid.data.numpy(), self.get_hyperparams(indexes), k=self.num_classes, evaluation_file='valid_set', zpred=zpred_valid.data.numpy()), index=[0]) else: metrics_valid = pd.DataFrame(get_metrics(ypred_valid.data.numpy(), yvalid.data.numpy(), zvalid.data.numpy(), self.get_hyperparams(indexes), k=self.num_classes, evaluation_file='valid_set'), index=[0]) print print('Final test metrics for model with ' + self.hyperparams_to_string(indexes) + ' on validation:') pprint.pprint(metrics_valid) ypred_test = model(Xtest) zpred_test = None if self.adversarial: if self.method == 'parity': adv_input_test = ypred_test zpred_test = adv_model(adv_input_test) elif self.method == 'odds': adv_input_test = torch.cat((ypred_test, ytest), 1) zpred_test = adv_model(adv_input_test) elif self.method == 'opportunity': zpred_test = None if zpred_test is not None: metrics_test = pd.DataFrame(get_metrics(ypred_test.data.numpy(), ytest.data.numpy(), ztest.data.numpy(), self.get_hyperparams(indexes), k=self.num_classes, evaluation_file='test_set', zpred=zpred_test.data.numpy()), index=[0]) else: metrics_test = pd.DataFrame(get_metrics(ypred_test.data.numpy(), ytest.data.numpy(), ztest.data.numpy(), self.get_hyperparams(indexes), k=self.num_classes, evaluation_file='test_set'), index=[0]) print print('Final test metrics for model with ' + self.hyperparams_to_string(indexes) + ' on test:') pprint.pprint(metrics_test) return pd.concat([metrics_valid, metrics_test])
def run(**kwargs): # load graph graph_nx, dump_folder = loader.load_dataset(kwargs['dataset']) # initialize tNodeEmbed task = kwargs['task'] test_size = kwargs['test_size'] tnodeembed = models.tNodeEmbed(graph_nx, task=task, dump_folder=dump_folder, test_size=test_size, **kwargs['n2vargs']) # load dataset X, y = tnodeembed.get_dataset(train_skip=kwargs['train_skip']) # fit keras_args = kwargs['keras_args'] batch_size = keras_args.pop('batch_size', 32) steps_per_epoch = ceil(len(X['train']) / batch_size) # tNodeEmbed generator = loader.dataset_generator(X['train'], y['train'], tnodeembed.graph_nx, tnodeembed.train_time_steps, batch_size=batch_size) tnodeembed.fit_generator(generator, steps_per_epoch, **keras_args) # node2vec static_model = models.StaticModel(task=task) generator = loader.dataset_generator(X['train'], y['train'], tnodeembed.graph_nx, [max(tnodeembed.train_time_steps)], batch_size=batch_size) static_model.fit_generator(generator, steps_per_epoch, **keras_args) # predict steps = ceil(len(X['test']) / batch_size) generator = loader.dataset_generator(X['test'], y['test'], tnodeembed.graph_nx, tnodeembed.train_time_steps, batch_size=batch_size, shuffle=False) tnodeembed_metrics = get_metrics( y['test'], tnodeembed.predict_generator(generator, steps)) generator = loader.dataset_generator(X['test'], y['test'], tnodeembed.graph_nx, [max(tnodeembed.train_time_steps)], batch_size=batch_size, shuffle=False) node2vec_metrics = get_metrics( y['test'], static_model.predict_generator(generator, steps)) print(f'tnodeembed: {tnodeembed_metrics}') print(f'node2vec: {node2vec_metrics}')
def run_metrics_from_dataloader(validation_dataloader, run_path, qrels_path): query_docids_map = get_query_docids_map(run_path=run_path) query_rel_doc_map = get_query_rel_doc_map(qrels_path=qrels_path) label_list = [] for step, batch in enumerate(validation_dataloader): label_list += flatten_list(batch[3].cpu().numpy().tolist()) labels_groups, scores_groups, queries_groups, doc_ids_groups, rel_docs_groups = group_bert_outputs_by_query( score_list=label_list, label_list=label_list, query_docids_map=query_docids_map, query_rel_doc_map=query_rel_doc_map) metrics_strings, label_metrics, _, oracle_metrics = get_metrics( labels_groups=labels_groups, scores_groups=scores_groups, rel_docs_groups=rel_docs_groups) label_string = get_metrics_string(metrics_strings=metrics_strings, metrics=label_metrics, name='RANKING') oracle_string = get_metrics_string(metrics_strings=metrics_strings, metrics=oracle_metrics, name='ORACLE') print(label_string) print(oracle_string) return label_string
def evaluate(model, dataloader, criterion, ctx_weight, device, save_dir=None): model = model.to(device).eval() stats_meter = metrics.MetricsAverageMeter() cls_preds = [] with torch.no_grad(): for batch_index, (features, cls_labels, ctx_labels) in enumerate(dataloader): cls_pred, ctx_pred = model(features) stats = metrics.get_metrics(cls_pred, ctx_pred, cls_labels, ctx_labels, criterion, criterion, ctx_weight) stats_meter.update(stats) cls_preds += cls_pred.argmax(dim=-1).tolist() if save_dir is not None: # save class prediction only os.makedirs(save_dir, exist_ok=True) with open(os.path.join(save_dir, 'submit.txt'), 'w') as f: f.write('\n'.join([str(x) for x in cls_preds])) stats = stats_meter.mean() return stats
def bert_evaluate(model, eval_dataloader, device): """Evaluation of trained checkpoint.""" model.to(device) model.eval() predictions = [] true_labels = [] data_iterator = tqdm(eval_dataloader, desc="Iteration") for step, batch in enumerate(data_iterator): input_ids, input_mask, labels = batch input_ids = input_ids.to(device) input_mask = input_mask.to(device) with torch.no_grad(): outputs = model(input_ids, token_type_ids=None, attention_mask=input_mask) #loss is only output when labels are provided as input to the model logits = outputs[0] print(type(logits)) logits = logits.to('cpu').numpy() label_ids = labels.to('cpu').numpy() for label, logit in zip(label_ids, logits): true_labels.append(label) predictions.append(np.argmax(logit)) #print(predictions) #print(true_labels) metrics = get_metrics(true_labels, predictions) return metrics
def test(args): # parse config config = parse_config(args.config) test_config = merge_configs(config, 'test', vars(args)) print_configs(test_config, 'Test') place = fluid.CUDAPlace(0) with fluid.dygraph.guard(place): video_model = AttentionCluster("AttentionCluster", test_config, mode="test") model_dict, _ = fluid.load_dygraph(args.weights) video_model.set_dict(model_dict) test_reader = FeatureReader(name="ATTENTIONCLUSTER", mode='test', cfg=test_config) test_reader = test_reader.create_reader() video_model.eval() total_loss = 0.0 total_acc1 = 0.0 total_sample = 0 for batch_id, data in enumerate(test_reader()): rgb = np.array([item[0] for item in data ]).reshape([-1, 100, 1024]).astype('float32') audio = np.array([item[1] for item in data ]).reshape([-1, 100, 128]).astype('float32') y_data = np.array([item[2] for item in data]).astype('float32') rgb = to_variable(rgb) audio = to_variable(audio) labels = to_variable(y_data) labels.stop_gradient = True output, logit = video_model([rgb, audio]) loss = fluid.layers.sigmoid_cross_entropy_with_logits(x=logit, label=labels) loss = fluid.layers.reduce_sum(loss, dim=-1) avg_loss = fluid.layers.mean(loss) # get metrics valid_metrics = get_metrics(args.model_name.upper(), 'valid', test_config) hit_at_one, perr, gap = valid_metrics.calculate_and_log_out( loss, logit, labels, info='[TEST] test_iter {} '.format(batch_id)) total_loss += avg_loss.numpy()[0] total_acc1 += hit_at_one total_sample += 1 print('TEST iter {}, loss = {}, acc1 {}'.format( batch_id, avg_loss.numpy()[0], hit_at_one)) print('Finish loss {} , acc1 {}'.format(total_loss / total_sample, total_acc1 / total_sample))
def monitor_processes(server_process, metrics, interval, socket): """ Monitor the metrics of server_process and its child processes """ while True: message = [] collected_metrics = get_metrics(server_process, get_child_processes(server_process), logger) metrics_msg = [] for metric in metrics: message.append(str(collected_metrics.get(metric, 0))) if collected_metrics.get(metric) is not None: metrics_msg.append("{0} : {1}".format( metric, collected_metrics.get(metric, 0))) message = "\t".join(message) + "\t\n" logger.info("%s", " -- ".join(metrics_msg)) if socket: try: socket.send(message.encode("latin-1")) except BrokenPipeError: logger.info( "Stopping monitoring as socket connection is closed.") break # TODO - log metrics to a file METRICS_LOG_FILE if METRICS_LOG_FILE is provided gevent.sleep(interval)
def eval(path, sheet, epoch): dset = dataset.get_dataset(1, 'MPEG', False) cr = CNNCRluma().cuda() sr = CNNSRluma().cuda() print(torch.load(path)['epoch']) #loads net weigths cr.load_state_dict(torch.load(path)['cr']) sr.load_state_dict(torch.load(path)['sr']) cr.eval() sr.eval() total, rate = 0.0, 0.0 for iteration, data in enumerate(dset, 1): input, name = data[0].cuda(), data[1] with torch.no_grad(): #FORWARD PASS======== #down-sample input ds, _ = cr(input) ds = ds.clamp(0,1) #Because of intperolation #code down-sampled input coded, bpp = encode(ds, 25) #up-sampled decoded image us = sr(coded).clamp(0,1) #==================== out = transforms.ToPILImage(mode='L')(us[0][0].cpu()) gt = transforms.ToPILImage(mode='L')(input[0][0].cpu()) psnr = get_metrics(gt, out, False)[0] total += psnr rate += bpp print('Bpp: {} --- PSNR: {}'.format(bpp, psnr)) torch.cuda.empty_cache() print(rate/len(dset)) print(total/len(dset))
def evaluate(self, eval_dataloader): """Evaluation of trained checkpoint.""" self.model.to(self.device) self.model.eval() predictions = [] true_labels = [] data_iterator = tqdm(eval_dataloader, desc="Iteration") for step, batch in enumerate(data_iterator): input_ids, labels = batch input_ids = input_ids.to(self.device) with torch.no_grad(): outputs = self.model(input_ids) outputs = outputs.to('cpu').numpy() print(type(outputs)) print("Shape:") print(outputs.shape) label_ids = labels.to('cpu').numpy() for label, output in zip(label_ids, outputs): true_labels.append(label) predictions.append(np.argmax(output)) # print(predictions) # print(true_labels) metrics = get_metrics(true_labels, predictions) return metrics
def main(): K = 50 trainData = np.load('train.npy') testData = np.load('test.npy') # UNCOMMENT BELOW AND REMOVE FURTHER BELOW FOR FASTER PERFORMANCE ON MULTIPLE RUNS ''' try: svd_corrMatrix = np.load('svd_correlation_matrix.npy') svd_90_corrMatrix = np.load('svd_90_correlation_matrix.npy') except FileNotFoundError: u, vt, sigma = build_svd_matrices(trainData) new_u, new_vt, new_sigma = top_90_energy(u, vt, sigma) users_in_svd_space = np.matmul(u, sigma) users_in_svd_90_space = np.matmul(new_u, new_sigma) svd_corrMatrix = build_corrMatrix(users_in_svd_space, 'svd_correlation_matrix.npy') svd_90_corrMatrix = build_corrMatrix(users_in_svd_90_space, 'svd_90_correlation_matrix.npy') ''' t0 = time.clock() # REMOVE BELOW FOR FASTER PERFORMANCE ON MULTIPLE RUNS, AND UNCOMMENT ABOVE u, vt, sigma = build_svd_matrices(trainData) users_in_svd_space = np.matmul(u, sigma) svd_corrMatrix = build_corrMatrix(users_in_svd_space, 'svd_correlation_matrix.npy') result_svd = collaborative_basic(trainData, testData, svd_corrMatrix, K) RMSE_svd, SRC_svd, precisionTopK_svd = get_metrics(result_svd, testData) del result_svd del svd_corrMatrix t1 = time.clock() print('SVD: RMSE = {}; SRC = {}; Precision on top K = {}; time taken = {}'.format(RMSE_svd, SRC_svd, precisionTopK_svd, t1-t0)) t2 = time.clock() # REMOVE BELOW FOR FASTER PERFORMANCE ON MULTIPLE RUNS, AND UNCOMMENT ABOVE new_u, new_vt, new_sigma = top_90_energy(u, vt, sigma) users_in_svd_90_space = np.matmul(new_u, new_sigma) svd_90_corrMatrix = build_corrMatrix(users_in_svd_90_space, 'svd_90_correlation_matrix.npy') result_svd_90 = collaborative_basic(trainData, testData, svd_90_corrMatrix, K) RMSE_svd_90, SRC_svd_90, precisionTopK_svd_90 = get_metrics(result_svd_90, testData) del result_svd_90 del svd_90_corrMatrix t3 = time.clock() print('SVD 90%: RMSE = {}; SRC = {}; Precision on top K = {}; time taken = {}'.format(RMSE_svd_90, SRC_svd_90, precisionTopK_svd_90, t3-t2))
def get_val_metrics(cnn, val_annot_dir, dataset_dir, in_w, out_w, bs): """ Return the TP, FP, TN, FN, defined_sum, duration for the {cnn} on the validation set TODO - This is too similar to the train loop. Merge both and use flags. """ start = time.time() fnames = ls(val_annot_dir) fnames = [a for a in fnames if im_utils.is_photo(a)] cnn.half() # TODO: In order to speed things up, be a bit smarter here # by only segmenting the parts of the image where we have # some annotation defined. # implement a 'partial segment' which exlcudes tiles with no # annotation defined. tps = 0 fps = 0 tns = 0 fns = 0 defined_sum = 0 for fname in fnames: annot_path = os.path.join(val_annot_dir, os.path.splitext(fname)[0] + '.png') # reading the image may throw an exception. # I suspect this is due to it being only partially written to disk # simply retry if this happens. try: annot = imread(annot_path) except Exception as ex: print('Exception reading annotation inside validation method.' 'Will retry in 0.1 seconsds') print(fname, ex) time.sleep(0.1) annot = imread(annot_path) annot = np.array(annot) foreground = annot[:, :, 0].astype(bool).astype(int) background = annot[:, :, 1].astype(bool).astype(int) image_path_part = os.path.join(dataset_dir, os.path.splitext(fname)[0]) image_path = glob.glob(image_path_part + '.*')[0] image = im_utils.load_image(image_path) predicted = unet_segment(cnn, image, bs, in_w, out_w, threshold=0.5) # mask defines which pixels are defined in the annotation. mask = foreground + background mask = mask.astype(bool).astype(int) predicted *= mask predicted = predicted.astype(bool).astype(int) y_defined = mask.reshape(-1) y_pred = predicted.reshape(-1)[y_defined > 0] y_true = foreground.reshape(-1)[y_defined > 0] tps += np.sum(np.logical_and(y_pred == 1, y_true == 1)) tns += np.sum(np.logical_and(y_pred == 0, y_true == 0)) fps += np.sum(np.logical_and(y_pred == 1, y_true == 0)) fns += np.sum(np.logical_and(y_pred == 0, y_true == 1)) defined_sum += np.sum(y_defined > 0) duration = round(time.time() - start, 3) metrics = get_metrics(tps, fps, tns, fns, defined_sum, duration) return metrics
def infer(args): # parse config config = parse_config(args.config) infer_config = merge_configs(config, 'infer', vars(args)) print_configs(infer_config, "Infer") infer_model = models.get_model(args.model_name, infer_config, mode='infer', is_videotag=True) infer_model.build_input(use_dataloader=False) infer_model.build_model() infer_feeds = infer_model.feeds() infer_outputs = infer_model.outputs() place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) filelist = args.filelist or infer_config.INFER.filelist filepath = args.video_path or infer_config.INFER.get('filepath', '') if filepath != '': assert os.path.exists(filepath), "{} not exist.".format(filepath) else: assert os.path.exists(filelist), "{} not exist.".format(filelist) # get infer reader infer_reader = get_reader(args.model_name.upper(), 'infer', infer_config) if args.weights: assert os.path.exists( args.weights), "Given weight dir {} not exist.".format( args.weights) # if no weight files specified, download weights from paddle weights = args.weights or infer_model.get_weights() infer_model.load_test_weights(exe, weights, fluid.default_main_program()) infer_feeder = fluid.DataFeeder(place=place, feed_list=infer_feeds) fetch_list = infer_model.fetches() infer_metrics = get_metrics(args.model_name.upper(), 'infer', infer_config) infer_metrics.reset() if not os.path.isdir(args.save_dir): os.makedirs(args.save_dir) for infer_iter, data in enumerate(infer_reader()): data_feed_in = [items[:-1] for items in data] video_id = [items[-1] for items in data] bs = len(video_id) feature_outs = exe.run(fetch_list=fetch_list, feed=infer_feeder.feed(data_feed_in)) for i in range(bs): filename = video_id[i].split('/')[-1][:-4] np.save(os.path.join(args.save_dir, filename + '.npy'), feature_outs[0][i]) #shape: seg_num*feature_dim logger.info("Feature extraction End~")
def test(args): # parse config config = parse_config(args.config) test_config = merge_configs(config, 'test', vars(args)) print_configs(test_config, "Test") # build model test_model = models.get_model(args.model_name, test_config, mode='test') test_model.build_input(use_pyreader=False) test_model.build_model() test_feeds = test_model.feeds() test_outputs = test_model.outputs() test_loss = test_model.loss() place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) if args.weights: assert os.path.exists( args.weights), "Given weight dir {} not exist.".format( args.weights) weights = args.weights or test_model.get_weights() test_model.load_test_weights(exe, weights, fluid.default_main_program(), place) # get reader and metrics test_reader = get_reader(args.model_name.upper(), 'test', test_config) test_metrics = get_metrics(args.model_name.upper(), 'test', test_config) test_feeder = fluid.DataFeeder(place=place, feed_list=test_feeds) if test_loss is None: fetch_list = [x.name for x in test_outputs] + [test_feeds[-1].name] else: fetch_list = [test_loss.name] + [x.name for x in test_outputs ] + [test_feeds[-1].name] epoch_period = [] for test_iter, data in enumerate(test_reader()): cur_time = time.time() test_outs = exe.run(fetch_list=fetch_list, feed=test_feeder.feed(data)) period = time.time() - cur_time epoch_period.append(period) if test_loss is None: loss = np.zeros(1, ).astype('float32') pred = np.array(test_outs[0]) label = np.array(test_outs[-1]) else: loss = np.array(test_outs[0]) pred = np.array(test_outs[1]) label = np.array(test_outs[-1]) test_metrics.accumulate(loss, pred, label) # metric here if args.log_interval > 0 and test_iter % args.log_interval == 0: info_str = '[EVAL] Batch {}'.format(test_iter) test_metrics.calculate_and_log_out(loss, pred, label, info_str) test_metrics.finalize_and_log_out("[EVAL] eval finished. ")
def evaluate(prediction, labels, out, beta): # create folder if not exists os.makedirs(out, exist_ok=True) # sort files by Images names pred_df = pd.read_csv(prediction) pred_df = pred_df.sort_values(by=['Images']) label_df = pd.read_csv(labels) label_df = label_df.sort_values(by=['Images']) assert len(pred_df) == len( label_df), "Mismatch between predictions and labels" header = list(pred_df.keys())[1:] ## calculate scores resp = dict() # disease-wise ovr_fscore = 0 for t in range(len(header)): pred_t = np.array(pred_df[header[t]]) disease = header[t] label_t = np.array(label_df[disease]) precision, recall, f_score = get_metrics(pred_t, label_t, beta=beta, average_type='binary') fpr, tpr, thresholds = roc_curve(label_t, pred_t, pos_label=1) _auc = auc(fpr, tpr) resp['precision_{}'.format(disease)] = precision resp['recall_{}'.format(disease)] = recall resp['f_score_{}'.format(disease)] = f_score resp['auc_{}'.format(disease)] = _auc ovr_fscore += f_score print(ovr_fscore / len(header)) # overall preds_array = pred_df[header].to_numpy() threshold = 0.5 preds_array = np.vectorize(lambda x: 1 if x >= threshold else 0)(preds_array) targets_array = label_df[[c for c in header]].to_numpy() resp['overall_f_score'] = fbeta_score(y_pred=preds_array, y_true=targets_array, beta=beta, average='weighted') resp['overall_precision'] = precision_score(y_pred=preds_array, y_true=targets_array, average='weighted') resp['overall_recall'] = recall_score(y_pred=preds_array, y_true=targets_array, average='weighted') print(resp) with open(os.path.join(out, 'metrics.json'), 'w') as f: json.dump(resp, f)
def evaluate_model(self, input_data_x, input_data_y): pred = self.decode_sequences(input_data_x, self.config['outputs_validation']) assert len(pred) == len(input_data_y) y_gt_input = input_data_y _, ades, fdes = get_metrics(pred, y_gt_input) del pred return {"ADE": np.mean(ades), "FDE": np.mean(fdes)}
def single_imagenet_experiment(regularizer, coeff, train_dir=CLUSTER_LOCAL_IMAGENET_TRAIN_DIR, val_dir=CLUSTER_LOCAL_IMAGENET_VAL_DIR, verbose=2): print("Started experiment at {}".format(datetime.datetime.now())) train_dataset = ImageNetTFRecordDataset(train_dir, BATCH_SIZE) val_dataset = ImageNetTFRecordDataset(val_dir, BATCH_SIZE) train_iter = train_dataset.create_dataset() val_iter = val_dataset.create_dataset() print("Finished creating datasets at {}".format(datetime.datetime.now())) start_time = time.time() # get model # model = model_creation.get_convolutional_model(IMAGENET_INPUT_SHAPE, IMAGENET_NUM_OF_CLASSES, 6, # [128, 128, 128, 128, 128, 128], # [11, 5, 5, 5, 5, 5], [4, 2, 1, 1, 2, 1], 2, # 400, regularizer, coeff, CONV_ACTIVATION, FC_ACTIVATION, # LAST_ACTIVATION, OPTIMIZER, LOSS, get_metrics(), # padding='valid', batch_norm=False, tensors=tensors) # model = model_creation.alexnet(tensors, metrics=get_metrics()) # model = model_creation.get_convolutional_model(IMAGENET_INPUT_SHAPE, IMAGENET_NUM_OF_CLASSES, 5, [16, 32, 64, 128, 256], # 3, 2, 2, 400, None, 0, CONV_ACTIVATION, FC_ACTIVATION, LAST_ACTIVATION, # OPTIMIZER, LOSS, get_metrics()) # model = model_creation.get_convolutional_model(IMAGENET_INPUT_SHAPE, IMAGENET_NUM_OF_CLASSES, 6, [32, 64, 128, 256, 512, 1028], # [5, 3, 3, 3, 3, 3], [3, 2, 2, 2, 2, 2], 4, 1500, None, 0, CONV_ACTIVATION, FC_ACTIVATION, LAST_ACTIVATION, # OPTIMIZER, LOSS, get_metrics()) # model = model_creation.alexnet(None, get_metrics()) # model = model_creation.get_convolutional_model(IMAGENET_INPUT_SHAPE, IMAGENET_NUM_OF_CLASSES, 6, [32, 64, 128, 256, 512, 1028], # [5, 3, 3, 3, 3, 3], [3, 2, 2, 2, 2, 2], 3, 2000, None, 0, CONV_ACTIVATION, FC_ACTIVATION, LAST_ACTIVATION, # OPTIMIZER, LOSS, get_metrics()) # model = model_creation.get_convolutional_model(IMAGENET_INPUT_SHAPE, IMAGENET_NUM_OF_CLASSES, 5, [32, 64, 128, 256, 512], # [11, 3, 3, 3, 3], [5, 2, 2, 2, 2], 4, 2000, None, 0, CONV_ACTIVATION, FC_ACTIVATION, LAST_ACTIVATION, # OPTIMIZER, LOSS, get_metrics()) model = model_creation.alexnet(dropout=True, metrics=get_metrics()) model.summary() regularizer_name = "None" if regularizer is None else regularizer.__name__ model_name = "nadavnet_{}_coeff_{}_23_June".format(regularizer_name, coeff) print("Model name is: {}".format(model_name)) tracker = TrackerForTFRecords(model_name, train_dataset.get_x(), train_dataset.get_y(), True, EXAMPLES_FOR_APPROX, True, True) callbacks = get_non_tracker_callbacks(model_name) + [TrackerCallback(tracker)] # callbacks = None print("Finished preparing everything at {}. Ready to start fitting.".format(datetime.datetime.now())) # fit model.fit(x=train_iter, epochs=EPOCHS, verbose=verbose, validation_data=val_iter, steps_per_epoch=IMAGENET_EPOCH_SIZE // BATCH_SIZE, callbacks=callbacks, validation_steps=IMAGENET_VALIDATION_SIZE // BATCH_SIZE) print("Experiment of model {} finished. " "This experiment took {:.2f} minutes.".format(model_name, int(time.time() - start_time) / 60))
def test(args): # parse config config = parse_config(args.config) test_config = merge_configs(config, 'test', vars(args)) print_configs(test_config, "Test") use_dali = test_config['TEST'].get('use_dali', False) # build model test_model = models.get_model(args.model_name, test_config, mode='test') test_model.build_input(use_dataloader=False) test_model.build_model() test_feeds = test_model.feeds() test_fetch_list = test_model.fetches() place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) if args.weights: assert os.path.exists( args.weights), "Given weight dir {} not exist.".format( args.weights) weights = args.weights or test_model.get_weights() logger.info('load test weights from {}'.format(weights)) test_model.load_test_weights(exe, weights, fluid.default_main_program()) # get reader and metrics test_reader = get_reader(args.model_name.upper(), 'test', test_config) test_metrics = get_metrics(args.model_name.upper(), 'test', test_config) test_feeder = fluid.DataFeeder(place=place, feed_list=test_feeds) epoch_period = [] for test_iter, data in enumerate(test_reader()): cur_time = time.time() test_outs = exe.run(fetch_list=test_fetch_list, feed=test_feeder.feed(data)) period = time.time() - cur_time epoch_period.append(period) test_metrics.accumulate(test_outs) # metric here if args.log_interval > 0 and test_iter % args.log_interval == 0: info_str = '[EVAL] Batch {}'.format(test_iter) test_metrics.calculate_and_log_out(test_outs, info_str) if not os.path.isdir(args.save_dir): os.makedirs(args.save_dir) test_metrics.finalize_and_log_out("[EVAL] eval finished. ", args.save_dir)
def select_params(train_type, param_type, candidates): param = [] mean_errors = [] if train_type == 'purchase': for candicate in candidates: purchase_params[param_type] = candicate offline_predict_purchase = offline_predict('purchase', offline_train_X_purchase, offline_train_y_purchase, offline_test_X_purchase) [sum_error,mean_error,std_error] = get_metrics(offline_test_y_purchase, offline_predict_purchase) param.append(candicate) mean_errors.append(mean_error) plt.scatter(param, mean_errors) plt.show() elif train_type == 'redeem': for candicate in candidates: redeem_params[param_type] = candicate offline_predict_redeem = offline_predict('redeem', offline_train_X_redeem, offline_train_y_redeem, offline_test_X_redeem) [sum_error,mean_error,std_error] = get_metrics(offline_test_y_redeem, offline_predict_redeem) param.append(candicate) mean_errors.append(mean_error) plt.scatter(param, mean_errors) plt.show() print param[mean_errors.index(min(mean_errors))], min(mean_errors)
def _calc_resource_stats(self, interval): result = super()._calc_resource_stats(interval) server_pid = get_process_pid_from_file(get_server_pidfile(PID_FILE)) server_process = get_server_processes(server_pid) result.update(get_metrics(server_process, get_child_processes(server_process), self.log)) metrics_msg = [] updated_result = {} for key in self.metrics: if result.get(key) is not None: metrics_msg.append("{0} : {1}".format(key, result[key])) updated_result[key] = result.get(key) self.log.info("{0}".format(" -- ".join(metrics_msg))) return updated_result
def main(): init_env('1') loaders = make_data_loaders(cfg) model = build_model(cfg) model = model.cuda() task_name = 'base_unet' log_dir = os.path.join(cfg.LOG_DIR, task_name) cfg.TASK_NAME = task_name mkdir(log_dir) logger = setup_logger('train', log_dir, filename='train.log') logger.info(cfg) logger = setup_logger('eval', log_dir, filename='eval.log') optimizer, scheduler = make_optimizer(cfg, model) metrics = get_metrics(cfg) losses = get_losses(cfg) train_val(model, loaders, optimizer, scheduler, losses, metrics)
def test_single(network, data_loader, device): # PSNR, SSIM, RMSE psnr_all, ssim_all, rmse_all = [], [], [] with torch.no_grad(): for i, (x, y) in enumerate(data_loader): x = x.float().to(device) y = y.float().to(device) pred = network(x) predict_result = metrics.get_metrics(y, pred) psnr_all.append(predict_result[0]) ssim_all.append(predict_result[1]) rmse_all.append(predict_result[2]) n = len(data_loader) return sum(psnr_all) / n, sum(ssim_all) / n, sum(rmse_all) / n
def test(self): """Summary After the test end, calculate the metrics Args: Returns: TYPE: Description """ # inference dataset if self.hparams.infer == 'valid': output_, target_ = self.test_step() else: output_ = self.test_step() resp = dict() to_csv = {'Images': self.names} for t in range(len(self.num_tasks)): if self.hparams.multi_cls: y_pred = np.reshape(output_, output_.shape[0]) else: y_pred = np.transpose(output_)[t] to_csv[self.labels[self.num_tasks[t]]] = y_pred # Only save scores to json file when in valid mode if self.hparams.infer == 'valid': overall_pre, overall_rec, overall_fscore = get_metrics( copy.deepcopy(output_), copy.deepcopy(target_), self.cfg.beta, self.cfg.threshold, self.cfg.metric_type) resp['overall_pre'] = overall_pre resp['overall_rec'] = overall_rec resp['overall_f_score'] = overall_fscore with open( os.path.join(os.path.dirname(self.hparams.load), 'scores_{}.csv'.format(uuid.uuid4())), 'w') as f: json.dump(resp, f) # Save predictions to csv file for computing metrics in off-line mode path_df = DataFrame(to_csv, columns=to_csv.keys()) path_df.to_csv(os.path.join(os.path.dirname(self.hparams.load), 'predictions_{}.csv'.format(uuid.uuid4())), index=False) return resp
def compile_model(config: ml_collections.ConfigDict) -> tf.keras.Model: """Builds a graph and compiles a tf.keras.Model based on the configuration.""" model = build_model_graph(config, config.outcomes) losses = {} loss_weights = {} for outcome in config.outcomes: losses[outcome.name] = metrics.get_loss(outcome) loss_weights[outcome.name] = outcome.get('loss_weight', 1.0) model.compile( optimizer=get_optimizer(config), loss=losses, loss_weights=loss_weights, weighted_metrics=metrics.get_metrics(config.outcomes)) return model
def benchmark(args, dataset_folder, dataset): data = prepare_dataset(dataset_folder, dataset, args.nrows) results = {} # "all" runs all algorithms if args.algorithm == "all": args.algorithm = "xgb-gpu,xgb-cpu,lgbm-cpu,lgbm-gpu,cat-cpu,cat-gpu" for alg in args.algorithm.split(","): print("Running '%s' ..." % alg) runner = algorithms.Algorithm.create(alg) with runner: train_time = runner.fit(data, args) pred = runner.test(data) results[alg] = { "train_time": train_time, "accuracy": get_metrics(data, pred), } return results
def do_GET(self): query_re = re.compile("query=(.*)") logs_table_re = re.compile(b"<!--\\W*logs\\W*-->\\W*<table>\\W*</table>") metrics_table_re = re.compile(b"<!--\\W*metrics\\W*-->\\W*<table>\\W*</table>") load_test_size_re = re.compile(b"<!--\\W*LOAD_TEST_SIZE\\W*-->") log_rows = [] metrics_rows = [] if self.path == "/" or self.path == "/index.html": self.path = "/index.html" elif self.path.startswith("/index.html?"): query = query_re.search(self.path).group(1) log = list(get_single_request(query)) log_rows = [log] logger.log(logger.FILE_NAME, log_rows) self.path = "/index.html" elif self.path == "/metrics" or self.path == "/metrics/load_test": if self.path == "/metrics/load_test": queries = open("random_queries.txt", 'r').read().split(' ') get_load_request(queries, LOAD_TEST_SIZE) log_rows = logger.read(logger.FILE_NAME) metrics_rows = metrics.get_metrics(LOAD_TEST_SIZE, log_rows) self.path = "/metrics.html" try: # Check the file extension required and # set the right mime type mimetype = 'text/html' # Open the static file requested and send it f = open(os.curdir + os.sep + self.path, 'rb') self.send_response(200) self.send_header('Content-type', mimetype) self.end_headers() html = f.read() html = logs_table_re.sub(logger.draw_table(logger.LOG_TABLE_HEADER, log_rows), html) html = metrics_table_re.sub(logger.draw_table(logger.METRICS_TABLE_HEADER, metrics_rows), html) html = load_test_size_re.sub(str(LOAD_TEST_SIZE).encode('utf-8'), html) self.wfile.write(html) f.close() return except IOError: self.send_error(404, 'File Not Found: %s' % self.path)
def build_ensemble(models: List[tf.keras.Model], config: ml_collections.ConfigDict) -> tf.keras.Model: """Compiles and returns an averaged ensemble model.""" model = _build_ensemble_graph(models, config) losses = {} loss_weights = {} for outcome in config.outcomes: losses[outcome.name] = metrics.get_loss(outcome) loss_weights[outcome.name] = outcome.get('loss_weight', 1.0) model.compile( optimizer=model_utils.get_optimizer(config), loss=losses, loss_weights=loss_weights, metrics=metrics.get_metrics(config.outcomes)) return model
def run_eval_alg(algorithm, train, test, dataset, processed_data, all_sensitive_attributes, single_sensitive, tag): """ Train and evaluate the algorithm; and gets the resulting metric evaluations. :return: params """ privileged_vals = dataset.get_privileged_class_names_with_joint(tag) positive_val = dataset.get_positive_class_val(tag) ##############TRAIN CLASSIFICATION MODEL ON TRAIN AND PREDICT ON TEST # get the actual classifications and sensitive attributes actual = test[dataset.get_class_attribute()].values.tolist() alg_output = run_alg(algorithm, train, test, dataset, all_sensitive_attributes, single_sensitive, privileged_vals, positive_val) ##############EVAL AGGREGATED METRICS ON TEST SET # make dictionary mapping sensitive names to sensitive attr test data lists dict_sensitive_lists = {} for sens in all_sensitive_attributes: dict_sensitive_lists[sens] = test[sens].values.tolist() sensitive_dict = processed_data.get_sensitive_values(tag) metrics_eval = [] for metric in get_metrics(dataset, sensitive_dict, tag): result = metric.calc(actual, alg_output['predictions'], dict_sensitive_lists, single_sensitive, privileged_vals, positive_val) metrics_eval.append(result) for metric in deepcopy(get_bayesian_metrics(dataset, sensitive_dict, tag)): result = metric.calc(actual, alg_output['predictions'], dict_sensitive_lists, single_sensitive, privileged_vals, positive_val) metrics_eval.append(result) results_lol = [] return { 'params': alg_output['params'], 'metrics_eval': metrics_eval, 'results_lol': results_lol, 'test_prediction_probs': alg_output['predicted_probs'] }
def find_best_fixed_threshold(self, output_, target_): score = list() thrs = np.arange(0, 1.0, 0.01) pre_rec = list() for thr in tqdm.tqdm(thrs): pre, rec, fscore = get_metrics(copy.deepcopy(output_), copy.deepcopy(target_), self.cfg.beta, thr, self.cfg.metric_type) score.append(fscore) pre_rec.append([pre, rec]) score = np.array(score) pm = score.argmax() best_thr, best_score = thrs[pm], score[pm].item() best_pre, best_rec = pre_rec[pm] print('thr={} F2={} prec{} rec{}'.format(best_thr, best_score, best_pre, best_rec)) return best_pre, best_rec, best_score
def val(epoch, model, cfg, args, valid_config): reader = FeatureReader(name="ATTENTIONCLUSTER", mode="valid", cfg=cfg) reader = reader.create_reader() total_loss = 0.0 total_acc1 = 0.0 total_sample = 0 for batch_id, data in enumerate(reader()): rgb = np.array([item[0] for item in data]).reshape([-1, 100, 1024]).astype('float32') audio = np.array([item[1] for item in data]).reshape([-1, 100, 128]).astype('float32') y_data = np.array([item[2] for item in data]).astype('float32') rgb = to_variable(rgb) audio = to_variable(audio) labels = to_variable(y_data) labels.stop_gradient = True output, logit = model([rgb, audio]) loss = fluid.layers.sigmoid_cross_entropy_with_logits(x=logit, label=labels) loss = fluid.layers.reduce_sum(loss, dim=-1) avg_loss = fluid.layers.mean(loss) # get metrics valid_metrics = get_metrics(args.model_name.upper(), 'valid', valid_config) hit_at_one, perr, gap = valid_metrics.calculate_and_log_out( loss, logit, labels, info='[TEST] test_iter {} '.format(batch_id)) total_loss += avg_loss.numpy()[0] total_acc1 += hit_at_one total_sample += 1 print('TEST Epoch {}, iter {}, loss = {}, acc1 {}'.format( epoch, batch_id, avg_loss.numpy()[0], hit_at_one)) print('Finish loss {} , acc1 {}'.format(total_loss / total_sample, total_acc1 / total_sample))
def train(self, data): n_temp = len(self.args.T_list) for (iT, T) in enumerate(self.args.T_list): self.model = get_model(data.train_in.shape, feat_ext=self.args.FEAT, hid_act=self.args.ACT, hid_filters=self.args.HF, kernels=self.args.K, pbc=self.args.PBC) self.model.compile(optimizer=self.args.OPT, loss=self.loss, metrics=self.metrics_list) hist = self.model.fit( x=data.train_in[iT * self.args.nTR:(iT + 1) * self.args.nTR][:self.args.TRS], y=data.train_out[iT * self.args.nTR:(iT + 1) * self.args.nTR][:self.args.TRS], batch_size=self.args.BS, epochs=self.args.EP, verbose=self.args.VB, callbacks=self.callbacks, validation_data=( data.test_in[iT * self.args.nTE:(iT + 1) * self.args.nTE][:self.args.VALS], data.test_out[iT * self.args.nTE:(iT + 1) * self.args.nTE][:self.args.VALS])) self.metrics = get_metrics(hist, reg=self.reg_flag) ### Save files ### npsave('%s/%s/Met%.4f.npy' % (self.args.metrics_dir, self.name, T), self.metrics) self.model.save('%s/%s/Net%.4f.h5' % (self.args.model_dir, self.name, T)) print('Temperature %d / %d done!' % (iT + 1, n_temp))
def processLine(line): print("Execute metrics_by_protein to [" + line + "]") metrics_by_protein = metrics.get_metrics(line.rstrip()) output_file.write(metrics_by_protein + os.linesep)