def evaluate_metric_te_tr(test_label, train_label, te_te_distance, te_tr_distance, k_set, logger=None):
'''
Args:
test_label - Numpy 1D array [ntest]
train_label - Numpy 1D array [ntrain]
te_te_distance - Numpy 2D array [ntest, ntest]
te_tr_distance - Numpy 2D array [ntest, ntrain]
k_set - list
logger - logger
defaults to be None
Return:
performance - dict
'''
te_te_recall_at_k, te_te_precision_at_k = get_recall_precision_at_k(dist_matrix=te_te_distance, labelq=test_label, labelh=test_label, k_set=k_set, issame=True)
te_tr_recall_at_k, te_tr_precision_at_k = get_recall_precision_at_k(dist_matrix=te_tr_distance, labelq=test_label, labelh=train_label, k_set=k_set, issame=False)
performance = {
'te_tr_precision_at_k' : te_tr_precision_at_k,
'te_te_precision_at_k' : te_te_precision_at_k,
'te_tr_recall_at_k' : te_tr_recall_at_k,
'te_te_recall_at_k' : te_te_recall_at_k}
key_set = ['te_tr_recall_at_k', 'te_tr_precision_at_k', 'te_te_recall_at_k', 'te_te_precision_at_k']
for key in key_set:
content = '{} @ {} = {}'.format(str(key), listformat(k_set), listformat(performance[key]))
if logger is None:
print(content)
else:
logger.info(content)
return performance
def evaluate_hash_te(test_hash_key, te_te_distance, te_te_query_key, te_te_query_value, test_label, ncls_test, activate_k, k_set, logger=None):
'''
Args:
test_hash_key - Numpy 2D array [ntest, nbucket]
should be binary
te_te_distance - Numpy 2D array [ntest, ntest]
te_te_query_key - Numpy 2D array [ntest, nbucket]
should be binary
te_te_query_value - Numpy 2D array [ntest, nbucket]
continuous
test_label - label of test data [ntest]
ncls_test - int
number of test classes or labels
activate_k - int
num of activation buckets
k_set - list [nk]
logger - logger
defaults to be None
Return:
performance - dict
'''
test_hash_table = HashTable(hash_key=test_hash_key, labelh=test_label, nlabel=ncls_test)
test_nmi = test_hash_table.get_nmi()
te_te_srr, te_te_recall_at_k, te_te_precision_at_k = test_hash_table.get_srr_recall_precision_at_k_hash(dist_matrix=te_te_distance, query_key=te_te_query_value, labelq=test_label, base_activate_k=activate_k, k_set=k_set, issame=True)
te_te_suf = 1.0/np.mean(te_te_srr, axis=-1)
performance = {
'test_nmi' : test_nmi,
'te_te_suf' : te_te_suf,
'te_te_precision_at_k' : te_te_precision_at_k,
'te_te_recall_at_k' : te_te_recall_at_k
}
key_set = [
'test_nmi',
'te_te_suf',
'te_te_precision_at_k',
'te_te_recall_at_k'
]
for key in key_set:
if 'suf' in str(key): content = "{} {} = {}".format(str(key), listformat(k_set), listformat(performance[key]))
elif 'at_k' in str(key): content = "{} {} = {}".format(str(key), listformat(k_set), listformat(performance[key]))
else: content = "{} = {:.3f}".format(str(key), performance[key])
if logger is None: print(content)
else: logger.info(content)
return performance
def test2():
'''
Results :
a : [0.1, 0.2, 1.0]
listformat(a) : 0.1/0.2/1.0
'''
a = [0.1, 0.2, 1.00]
print(" a : {}".format(a))
print(" listformat(a) : {}".format(listformat(a)))
'vq': performance_vq
},
path=PKL_DIR + FILE_ID + '_{}.pkl'.format(activate_k))
cwrite = CsvWriter2(2)
key_set = [
'train_nmi', 'test_nmi', 'te_tr_suf', 'te_te_suf',
'te_tr_precision_at_k', 'te_te_precision_at_k',
'te_tr_recall_at_k', 'te_te_recall_at_k'
]
for key in key_set:
cwrite.add_header(0, str(key) + "_th")
cwrite.add_header(1, str(key) + "_vq")
content = ''
if 'suf' in str(key): content = listformat(performance_th[key])
elif 'at_k' in str(key): content = listformat(performance_th[key])
else: content = performance_th[key]
cwrite.add_content(0, content)
content = ''
if 'suf' in str(key): content = listformat(performance_vq[key])
elif 'at_k' in str(key): content = listformat(performance_vq[key])
else: content = performance_vq[key]
cwrite.add_content(1, content)
cwrite.write(CSV_DIR + FILE_ID + '_{}.csv'.format(activate_k))
model.delete()
# load data
datasetmanager = DATASETMANAGER_DICT[args.dataset]
dm_train, dm_val, dm_test = datasetmanager(args.ltype,
nsclass=args.nsclass)
for v in [dm_train, dm_val, dm_test]:
v.print_shape()
model = DeepMetric(dm_train, dm_val, dm_test, LOG_DIR + FILE_ID + '.log',
args)
model.build()
model.set_up_train()
try:
model.restore(save_dir=SAVE_DIR)
except AttributeError:
model.initialize()
model.train(epoch=args.epoch, save_dir=SAVE_DIR, board_dir=BOARD_DIR)
model.restore(save_dir=SAVE_DIR)
model.prepare_test()
performance = model.test_metric(K_SET)
key_set = [
'te_tr_recall_at_k', 'te_tr_precision_at_k', 'te_te_recall_at_k',
'te_te_precision_at_k'
]
cwrite = CsvWriter2(1)
for key in key_set:
cwrite.add_header(0, str(key))
cwrite.add_content(0, listformat(performance[key]))
cwrite.write(CSV_DIR + FILE_ID + '.csv')
write_pkl(performance, path=PKL_DIR + FILE_ID + '.pkl')
