x_test, y_test = test_set
(x, y), _ = _split_dataset(x_test, y_test, percentage=0.2)
print('test set:', x.shape)
def evaluation(sess, model):
ano_scores = []
for _, batch_data in DataInput(x, test_batch_size):
_ano_score = model.eval(sess, batch_data)
# Extend
ano_scores += list(_ano_score)
ano_scores = np.array(ano_scores).reshape((-1, 1))
# Calculate auc
auroc = calc_auroc(y, ano_scores)
print('Eval_auroc:{:.4f}'.format(auroc))
prec, rec, f1 = calc_metric(y, ano_scores)
print('Prec:{:.4f}\tRec:{:.4f}\tF1:{:.4f}\n'.format(prec, rec, f1))
draw_prc(y, ano_scores, key='ResDEAAE_' + 'cross-e')
with tf.Session() as sess:
model = BiWGAN(count_list, method, weight=weight, degree=degree)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
model.restore(sess, 'train_logs_fm_0.8932/talkingdata/fm/0.9/1/ckpt')
# model.restore(sess, 'train_logs_ce_0.9252/talkingdata/cross-e/0.9/1/ckpt')
evaluation(sess, model)
_auprc = calc_auc(label, ano_scores)
global best_f1
if best_f1 < f1:
best_f1 = f1
model.save(sess, '{}/ckpt'.format(save_path))
global best_auprc
if best_auprc < _auprc:
best_auprc = _auprc
return prec, rec, f1, _auprc
with tf.Session() as sess:
model = BiWGAN(input_dim, method, weight, degree)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
prec, rec, f1, auprc = _eval(sess, model, x_val, y_val)
print('Prec:{:.4f} | Rec:{:.4f} | F1:{:.4f} | Eval_auprc:{:.4f}'.
format(prec, rec, f1, auprc))
sys.stdout.flush()
# Start training
start_time = time.time()
for i in range(nb_epochs):
print('==== Training epoch {} ===='.format(i))
sys.stdout.flush()
# shuffle for each epoch
weight = 0.9
degree = 'euclidean'
logdir = create_logdir(mode, method, weight, degree)
save_path = os.path.join(base_dir, logdir)
with open('{}_dataset.pkl'.format(mode), 'rb') as f:
train_set = pickle.load(f)
val_set = pickle.load(f)
test_set = pickle.load(f)
x_test, y_test = test_set
print('test set', x_test.shape)
with tf.Session() as sess:
model = BiWGAN(input_dim, method, weight, degree)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
model.restore(sess, '{}/ckpt'.format(save_path))
ano_scores = []
for _, batch_test_data in DataInput(x_test, test_batch_size):
_ano_score, _, _ = model.eval(sess, batch_test_data)
# extend
ano_scores += list(_ano_score)
ano_scores = np.array(ano_scores).reshape((-1, 1))
# Highest 80% are anomalous
prec, rec, f1 = calc_metric(y_test, ano_scores, percentile=80)
ano_scores = np.array(ano_scores).reshape((-1, 1))
# Calculate auroc
auroc = calc_auroc(label, ano_scores)
# Calculate metric
prec, rec, f1 = 0., 0., 0. # for sake of computation speed
# prec, rec, f1 = calc_metric(label, ano_scores)
global best_auroc
if best_auroc < auroc:
best_auroc = auroc
model.save(sess, '{}/ckpt'.format(save_path))
return auroc, prec, rec, f1
with tf.Session() as sess:
model = BiWGAN(count_list, method, weight=weight, degree=degree)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
# model.restore(sess, '{}/ckpt'.format(save_path))
# model.restore(sess, 'train_logs_5%_0.8623/talkingdata/cross-e/0.9/1/ckpt')
print('Total params: ', np.sum([np.prod(v.get_shape().as_list()) for v in tf.trainable_variables()]))
start_time = time.time()
auroc, prec, rec, f1 = _eval(sess, model, x_val, y_val)
print('Eval_auc:{:.4f} | prec:{:.4f} | rec:{:.4f} | f1:{:.4f}\tTime Cost:{:.4f}'.format(auroc, prec,
rec, f1,
time.time()-start_time))
sys.stdout.flush()
start_time = time.time()
for i in range(nb_epochs):
def evaluation(sess, model, ratio):
(sub_ano, sub_ano_label), _ = _split_dataset(ano, ano_label,
mapping_ratio[ratio])
x = np.concatenate((norm, sub_ano), axis=0)
y = np.concatenate((norm_label, sub_ano_label), axis=0)
ano_scores = []
for _, batch_data in DataInput(x, test_batch_size):
_ano_score = model.eval(sess, batch_data)
# Extend
ano_scores += list(_ano_score)
ano_scores = np.array(ano_scores).reshape((-1, 1))
# Calculate auc
auroc = calc_auroc(y, ano_scores)
print('Anomaly ratio:{:.4f}\tEval_auroc:{:.4f}'.format(ratio, auroc))
prec, rec, f1 = calc_metric(y, ano_scores)
print('Prec:{:.4f}\tRec:{:.4f}\tF1:{:.4f}\n'.format(prec, rec, f1))
with tf.Session() as sess:
model = BiWGAN(count_list, method, weight=weight, degree=degree)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
model.restore(sess, '{}/ckpt'.format(save_path))
# evaluation
for key in mapping_ratio.keys():
evaluation(sess, model, key)