def evaluate_metric_te_tr(test_label, train_label, te_te_distance, te_tr_distance, k_set, logger=None): ''' Args: test_label - Numpy 1D array [ntest] train_label - Numpy 1D array [ntrain] te_te_distance - Numpy 2D array [ntest, ntest] te_tr_distance - Numpy 2D array [ntest, ntrain] k_set - list logger - logger defaults to be None Return: performance - dict ''' te_te_recall_at_k, te_te_precision_at_k = get_recall_precision_at_k(dist_matrix=te_te_distance, labelq=test_label, labelh=test_label, k_set=k_set, issame=True) te_tr_recall_at_k, te_tr_precision_at_k = get_recall_precision_at_k(dist_matrix=te_tr_distance, labelq=test_label, labelh=train_label, k_set=k_set, issame=False) performance = { 'te_tr_precision_at_k' : te_tr_precision_at_k, 'te_te_precision_at_k' : te_te_precision_at_k, 'te_tr_recall_at_k' : te_tr_recall_at_k, 'te_te_recall_at_k' : te_te_recall_at_k} key_set = ['te_tr_recall_at_k', 'te_tr_precision_at_k', 'te_te_recall_at_k', 'te_te_precision_at_k'] for key in key_set: content = '{} @ {} = {}'.format(str(key), listformat(k_set), listformat(performance[key])) if logger is None: print(content) else: logger.info(content) return performance
def evaluate_hash_te(test_hash_key, te_te_distance, te_te_query_key, te_te_query_value, test_label, ncls_test, activate_k, k_set, logger=None): ''' Args: test_hash_key - Numpy 2D array [ntest, nbucket] should be binary te_te_distance - Numpy 2D array [ntest, ntest] te_te_query_key - Numpy 2D array [ntest, nbucket] should be binary te_te_query_value - Numpy 2D array [ntest, nbucket] continuous test_label - label of test data [ntest] ncls_test - int number of test classes or labels activate_k - int num of activation buckets k_set - list [nk] logger - logger defaults to be None Return: performance - dict ''' test_hash_table = HashTable(hash_key=test_hash_key, labelh=test_label, nlabel=ncls_test) test_nmi = test_hash_table.get_nmi() te_te_srr, te_te_recall_at_k, te_te_precision_at_k = test_hash_table.get_srr_recall_precision_at_k_hash(dist_matrix=te_te_distance, query_key=te_te_query_value, labelq=test_label, base_activate_k=activate_k, k_set=k_set, issame=True) te_te_suf = 1.0/np.mean(te_te_srr, axis=-1) performance = { 'test_nmi' : test_nmi, 'te_te_suf' : te_te_suf, 'te_te_precision_at_k' : te_te_precision_at_k, 'te_te_recall_at_k' : te_te_recall_at_k } key_set = [ 'test_nmi', 'te_te_suf', 'te_te_precision_at_k', 'te_te_recall_at_k' ] for key in key_set: if 'suf' in str(key): content = "{} {} = {}".format(str(key), listformat(k_set), listformat(performance[key])) elif 'at_k' in str(key): content = "{} {} = {}".format(str(key), listformat(k_set), listformat(performance[key])) else: content = "{} = {:.3f}".format(str(key), performance[key]) if logger is None: print(content) else: logger.info(content) return performance
def test2(): ''' Results : a : [0.1, 0.2, 1.0] listformat(a) : 0.1/0.2/1.0 ''' a = [0.1, 0.2, 1.00] print(" a : {}".format(a)) print(" listformat(a) : {}".format(listformat(a)))
'vq': performance_vq }, path=PKL_DIR + FILE_ID + '_{}.pkl'.format(activate_k)) cwrite = CsvWriter2(2) key_set = [ 'train_nmi', 'test_nmi', 'te_tr_suf', 'te_te_suf', 'te_tr_precision_at_k', 'te_te_precision_at_k', 'te_tr_recall_at_k', 'te_te_recall_at_k' ] for key in key_set: cwrite.add_header(0, str(key) + "_th") cwrite.add_header(1, str(key) + "_vq") content = '' if 'suf' in str(key): content = listformat(performance_th[key]) elif 'at_k' in str(key): content = listformat(performance_th[key]) else: content = performance_th[key] cwrite.add_content(0, content) content = '' if 'suf' in str(key): content = listformat(performance_vq[key]) elif 'at_k' in str(key): content = listformat(performance_vq[key]) else: content = performance_vq[key] cwrite.add_content(1, content) cwrite.write(CSV_DIR + FILE_ID + '_{}.csv'.format(activate_k)) model.delete()
# load data datasetmanager = DATASETMANAGER_DICT[args.dataset] dm_train, dm_val, dm_test = datasetmanager(args.ltype, nsclass=args.nsclass) for v in [dm_train, dm_val, dm_test]: v.print_shape() model = DeepMetric(dm_train, dm_val, dm_test, LOG_DIR + FILE_ID + '.log', args) model.build() model.set_up_train() try: model.restore(save_dir=SAVE_DIR) except AttributeError: model.initialize() model.train(epoch=args.epoch, save_dir=SAVE_DIR, board_dir=BOARD_DIR) model.restore(save_dir=SAVE_DIR) model.prepare_test() performance = model.test_metric(K_SET) key_set = [ 'te_tr_recall_at_k', 'te_tr_precision_at_k', 'te_te_recall_at_k', 'te_te_precision_at_k' ] cwrite = CsvWriter2(1) for key in key_set: cwrite.add_header(0, str(key)) cwrite.add_content(0, listformat(performance[key])) cwrite.write(CSV_DIR + FILE_ID + '.csv') write_pkl(performance, path=PKL_DIR + FILE_ID + '.pkl')