def accfun(y0, y1):
x_pos = K.ones_like(x_r)
x_neg = K.zeros_like(x_r)
loss_r = K.mean(binary_accuracy(x_pos, x_r))
loss_f = K.mean(binary_accuracy(x_neg, x_f))
loss_p = K.mean(binary_accuracy(x_neg, x_p))
return (1.0 / 3.0) * (loss_r + loss_p + loss_f)
def symmetric_accuracy(y_true, y_pred):
idx = tf.where(tf.not_equal(y_true, -1.0))
not_y_true = tf.gather(y_true, K.constant([1, 0], dtype=np.int32), axis=-1)
y_true = tf.gather_nd(y_true, idx)
y_pred = tf.gather_nd(y_pred, idx)
not_y_true = tf.gather_nd(not_y_true, idx)
return K.maximum(metrics.binary_accuracy(y_true, y_pred),
metrics.binary_accuracy(not_y_true, y_pred))
def market_attribute_accuracy(y_true, y_pred):
# binary categories
acc = binary_accuracy(y_true[:, binary_1], y_pred[:, binary_1]) * 0.5
acc += binary_accuracy(y_true[:, binary_2], y_pred[:, binary_2]) * 0.5
# top colors
acc_top_color = categorical_accuracy(y_true[:, up_colors],
y_pred[:, up_colors])
# down colors
acc_down_color = categorical_accuracy(y_true[:, down_colors],
y_pred[:, down_colors])
# print(acc, acc_down_color, acc_top_color)
return acc * 9 / 11 + acc_down_color * 1 / 11 + acc_top_color * 1 / 11
def _generic_accuracy(y_true, y_pred):
if K.int_shape(y_pred)[1] == 1:
return binary_accuracy(y_true, y_pred)
if K.int_shape(y_true)[-1] == 1:
return sparse_categorical_accuracy(y_true, y_pred)
return categorical_accuracy(y_true, y_pred)
def validation(sess, model, x, y):
y_val = tf.placeholder(tf.float32,
shape=(None, 1),
name='validation_labels')
X_val = tf.placeholder(tf.float32,
shape=(None, batch_shape[1]),
name='validation_inputs')
pred, _ = model.get_pred_and_emb(X_val)
accuracy = tf.reduce_mean(binary_accuracy(y_val, pred))
total_batch = int(x.shape[0] / batch_shape[0])
avg_val_acc = 0
x, y = shuffle(x, y)
for i in range(total_batch):
offset = (i * batch_shape[0]) % (y.shape[0] - batch_shape[0])
# Generate a minibatch.
batch_data = x[offset:offset + batch_shape[0]]
batch_labels = y[offset:offset + batch_shape[0]]
fd = {
X_val: batch_data,
y_val: batch_labels,
K.backend.learning_phase(): 0
}
acc_val = sess.run(accuracy, feed_dict=fd)
avg_val_acc += acc_val / total_batch
print("Average accuracy on validation is {:.3f}".format(avg_val_acc))
return avg_val_acc
def my_binary_accuracy(y_true, y_pred):
# print("my_binary_accuracy")
# print(f"y_true:{y_true}, y_pred:{y_pred}")
y_true_, y_pred_ = tarnsform_metrics(y_true, y_pred)
# print(f"y_true_:{y_true_}, y_pred_:{y_pred_}")
accuracy = binary_accuracy(y_true_, y_pred_)
return accuracy
def treatment_accuracy(concat_true, concat_pred):
"""
Returns keras' binary_accuracy between treatment and prediction of propensity.
Args:
- concat_true (tf.tensor): tensor of true samples, with shape (n_samples, 2)
Each row in concat_true is comprised of (y, treatment)
- concat_pred (tf.tensor): tensor of predictions, with shape (n_samples, 4)
Each row in concat_pred is comprised of (y0, y1, propensity, epsilon)
Returns:
- (float): binary accuracy
"""
t_true = concat_true[:, 1]
t_pred = concat_pred[:, 2]
return binary_accuracy(t_true, t_pred)
def metric_with_cilin(y_true, y_pred):
# loss for skip-gram model
y_true_ctx = y_true[:, 0]
y_pred_ctx = y_pred[:, 0]
accuracy_sg = binary_accuracy(y_true_ctx, y_pred_ctx)
# loss for cilin
y_true_cilin = y_true[:, 1]
y_pred_cilin = y_pred[:, 1]
# obtain indexes of elements with value equal to 0
nonzero_count = tf.count_nonzero(y_true_cilin)
accuracy_cilin = tf.cond(tf.greater(nonzero_count, 0),
lambda: absolute_accuracy_cilin(y_true_cilin, y_pred_cilin, nonzero_count),
lambda: zero_loss())
weights = tf.constant([0.5, 0.5])
accuracy = tf.reduce_sum(tf.multiply(weights, tf.stack([accuracy_sg, accuracy_cilin])))
return accuracy
def my_accuracy(y_true, y_pred):
mask_shape = tf.shape(y_pred)
y_pred = K.reshape(y_pred,
(-1, mask_shape[2], mask_shape[3], mask_shape[4]))
mask_shape = tf.shape(y_true)
y_true = K.reshape(y_true,
(-1, mask_shape[2], mask_shape[3], mask_shape[4]))
sm = tf.reduce_sum(y_true, [1, 2, 3])
ix = tf.where(sm > 0)[:, 0]
y_true = tf.gather(y_true, ix)
y_pred = tf.gather(y_pred, ix)
return binary_accuracy(y_true, y_pred)
def _do_test(model, xt, yt):
if isinstance(model, Bagging):
yt2 = np.argmax(yt, axis=1)
res_rounded = model.predict_on_batch(xt, yt2)
res = res_rounded
else:
res = model.predict(xt)
# res = np.argmax(res, axis=1)
res_rounded = np.round(res, decimals=0).astype(int)
cat_res = np_utils.to_categorical(res_rounded)
fscore = metrics.fscore(yt, res_rounded)
prec = K.eval(
K.mean(binary_accuracy(K.variable(yt), K.variable(res_rounded))))
return fscore, prec, res_rounded, res
def test(sess, model):
X_test = np.load(
os.path.join(config.dataset_base_dir,
"X_test_{}.npz".format(config.dataset_name)))['arr_0']
y_test = np.load(
os.path.join(config.dataset_base_dir,
"y_test_{}.npz".format(config.dataset_name)))['arr_0']
y_test = y_test.reshape(y_test.shape[0], 1)
# Padding
if not padding_post:
X_test = K.preprocessing.sequence.pad_sequences(X_test,
maxlen=batch_shape[1])
else:
X_test = K.preprocessing.sequence.pad_sequences(X_test,
maxlen=batch_shape[1],
padding='post')
y = tf.placeholder(tf.float32, shape=(None, 1), name='test_labels')
X = tf.placeholder(tf.float32,
shape=(None, batch_shape[1]),
name='test_input')
pred, _ = model.get_pred_and_emb(X)
accuracy = tf.reduce_mean(binary_accuracy(y, pred))
test_accuracies = list()
total_batch = int(X_test.shape[0] / batch_shape[0])
X_test, y_test = shuffle(X_test, y_test)
avg_test_acc = 0
for i in range(total_batch):
offset = (i * batch_shape[0]) % (y_test.shape[0] - batch_shape[0])
# Generate a minibatch.
batch_data = X_test[offset:offset + batch_shape[0]]
batch_labels = y_test[offset:offset + batch_shape[0]]
# test mode
fd = {X: batch_data, y: batch_labels, K.backend.learning_phase(): 0}
acc_test = sess.run(accuracy, feed_dict=fd)
test_accuracies.append(acc_test)
avg_test_acc += acc_test / total_batch
np.savez_compressed(
'Plot/{}_test_accuracies_adv_{}_vadv_{}_decay_{}_clip_{}.npz'.format(
config.dataset_name, adversarial, virt_adversarial, decay,
clipping), test_accuracies)
print("\nAverage accuracy on test set is {:.3f}".format(avg_test_acc))
def compute_detection_score(input, file_segments, model, generator, steps, type):
#Predict and average of all segments for each utterance
#Save on a file the score of each utterance and the respective label
#This file name 'LA_dev_score.txt'
#Define two list: label contais the true value, detection_score contains the score
f=open(file_segments, 'r')
f1=open('risultati'+type+'.txt', 'w', encoding='utf8')
label=[]
y=[]
rows=f.readlines()
start=0
score_bonafide=[]
score_spoof=[]
predicted=model.predict(x=generator, steps=steps, verbose=1, max_queue_size=10)
print(predicted.shape)
for row in rows:
fields=row.split('\t')
N=int(fields[0])
y_pred=np.sum(predicted[start: start+N])/N
start=start+(N)
#Compute log-probability of bonafide class and save results on a file txt
prob=1-y_pred
log_prob=np.log10(prob)
label.append(int(fields[1]))
f1.write(str(y_pred)+'\t'+str(fields[1])+'\n')
y.append(y_pred)
if int(fields[1])==0:
score_bonafide.append(np.copy(log_prob))
else:
score_spoof.append(np.copy(log_prob))
label=np.array(label)
y=np.array(y)
print(len(score_bonafide))
print(len(score_spoof))
score_bonafide=np.array(score_bonafide)
score_spoof=np.array(score_spoof)
print('Accuracy: ', np.array(metrics.binary_accuracy(label, y)))
f.close()
f1.close()
return score_bonafide, score_spoof
def predict_and_eval(model, X_train, y_train, X_test, y_test, threshold = None):
logging.info("Performing prediction on train and test for evaluation ...")
y_pred_train = model.predict(X_train, batch_size=batch_size, verbose=1)
y_pred_test = model.predict(X_test, batch_size=batch_size, verbose=1)
eval_types = ['Train', 'Test']
logs = {}
for e, eval_type in enumerate(eval_types):
# print "[%s]" % eval_type
metric_prefix = '' if e == 0 else 'val_'
X_eval = X_train if e == 0 else X_test
y_eval = y_train if e == 0 else y_test
y_pred_eval = y_pred_train if e == 0 else y_pred_test
# threshold = 0.48
# y_pred_eval = (0.5 - threshold) + y_pred_eval
y_eval = y_eval.astype(float)
if threshold != None:
y_eval = K.clip((0.5 - threshold) + y_eval, 0., 1.)
logs[metric_prefix + 'loss'] = metrics.binary_crossentropy(y_eval, y_pred_eval).eval()
logs[metric_prefix + 'acc'] = metrics.binary_accuracy(y_eval, y_pred_eval).eval()
logs[metric_prefix + 'precision'] = metrics.precision(y_eval, y_pred_eval).eval()
logs[metric_prefix + 'recall'] = metrics.recall(y_eval, y_pred_eval).eval()
logs[metric_prefix + 'fbeta_score'] = metrics.fmeasure(y_eval, y_pred_eval).eval()
# log_file.write("%d,%.5f,%s,%.5f\n" % (epoch, threshold, eval_type, average_faux_jaccard_similarity))
# print "%d,%.5f,%s,%.4f" % (epoch, threshold, eval_type, average_faux_jaccard_similarity)
metrics_line = ''
for s in ['loss', 'acc', 'precision', 'recall', 'fbeta_score']:
metrics_line += "%s: %.5f %s: %.5f - " %(s, logs[s], 'val_'+s, logs['val_' +s])
logging.info(metrics_line)
return logs
def batch_pairwise_metrics(y_true, y_pred):
#assert K.get_variable_shape(y_true)[1] == K.get_variable_shape(y_pred)[1]
num_classes = K.get_variable_shape(y_pred)[1]
preds_cats = K.argmax(y_pred, axis=1)
preds_one_hot = K.one_hot(preds_cats, num_classes)
overall_precision = [None for _ in range(num_classes)]
overall_recall = [None for _ in range(num_classes)]
overall_fmeasure = [None for _ in range(num_classes)]
out_dict = {}
for cc in range(num_classes):
#Metrics should take 1D arrays which are 1 for positive, 0 for negative
two_true, two_pred = y_true[:, cc], preds_one_hot[:, cc]
cur_dict = {
'precision/%02d' % cc:
kmetrics.precision(two_true, two_pred),
'recall/%02d' % cc:
kmetrics.recall(two_true, two_pred),
'fmeasure/%02d' % cc:
kmetrics.fmeasure(two_true, two_pred),
'binary_accuracy/%02d' % cc:
kmetrics.binary_accuracy(two_true, two_pred),
'act_pos/%02d' % cc:
K.sum(two_true),
'pred_pos/%02d' % cc:
K.sum(two_pred)
}
out_dict.update(cur_dict)
overall_precision[cc] = cur_dict['precision/%02d' % cc]
overall_recall[cc] = cur_dict['recall/%02d' % cc]
overall_fmeasure[cc] = cur_dict['fmeasure/%02d' % cc]
out_dict.update(make_stats('precision', overall_precision))
out_dict.update(make_stats('recall', overall_recall))
out_dict.update(make_stats('fmeasure', overall_fmeasure))
return out_dict
def update(self, batch):
x, y = batch
y = np.array(y)
y_pred = None
if self.model_type == 'nn':
self.train_loss, self.train_acc = self.model.train_on_batch(x, y)
y_pred = self.model.predict_on_batch(x).reshape(-1)
self.train_auc = roc_auc_score(y, y_pred)
if self.model_type == 'ngrams':
x = vectorize_select_from_data(x, self.vectorizers, self.selectors)
self.model.fit(x, y.reshape(-1))
y_pred = np.array(self.model.predict_proba(x)[:, 1]).reshape(-1)
y_pred_tensor = K.constant(y_pred, dtype='float64')
self.train_loss = K.eval(
binary_crossentropy(y.astype('float'), y_pred_tensor))
self.train_acc = K.eval(
binary_accuracy(y.astype('float'), y_pred_tensor))
self.train_auc = roc_auc_score(y, y_pred)
self.updates += 1
return y_pred
def _test(self, model=None, return_metrics=False):
if model is None:
model = self.model
test_pred = model.predict(self.xt)
round_test_pred = np.round(test_pred, decimals=0).astype(int)
test_fscore = metrics.fscore(self.yt, round_test_pred)
test_prec = K.eval(
K.mean(
binary_accuracy(K.variable(self.yt),
K.variable(round_test_pred))))
if test_fscore > self.best_score:
self.best_score = test_fscore
self.best_model = keras_deep_copy_model(model)
if return_metrics == False:
self.iteration_test_fscore = test_fscore
self.iteration_test_prec = test_prec
self.test_history_list.append(test_fscore)
else:
return test_fscore, test_prec
def accfun(y0, y1):
x_pos = K.ones_like(x_p)
loss_p = K.mean(binary_accuracy(x_pos, x_p))
loss_f = K.mean(binary_accuracy(x_pos, x_f))
return 0.5 * (loss_p + loss_f)
def c_binary_accuracy(y_truth, y_pred):
return metrics.binary_accuracy(y_truth[..., 0], y_pred[..., 0])
def classifier_accuracy(y_true, y_pred):
return metrics.binary_accuracy(y_true[..., GT_INDEX.IS_ELLIPSE], y_pred[..., GT_INDEX.IS_ELLIPSE])
def mask_accuracy(y_true, y_pred):
idx = tf.where(tf.not_equal(y_true, -1.0))
y_true = tf.gather_nd(y_true, idx)
y_pred = tf.gather_nd(y_pred, idx)
return metrics.binary_accuracy(y_true, y_pred)
def accuracy(x, x_decoded):
acc = metrics.binary_accuracy(x, x_decoded)
return K.mean(acc)
test_x = np.array([years, months, days, hours,
minutes]) #.reshape((rows, 5))
test_x = np.transpose(test_x)
"""
old stuff - sklearn
"""
# accuracy = model.score(test_x, test_y)
model = load_model("{}.hdf5".format(args.input))
y_pred = model.predict(test_x)
if backend:
y_true = cast_to_floatx(y_true)
y_pred = cast_to_floatx(y_pred)
accuracy = binary_accuracy(y_true, y_pred)
accuracy = mean(accuracy)
value = batch_get_value([accuracy]) # compute the accuracy
value = value[0]
else:
with tf.Session() as sess:
y_true = tf.cast(y_true, tf.float32)
y_pred = tf.cast(y_pred, tf.float32)
accuracy = binary_accuracy(y_true, y_pred)
accuracy = tf.reduce_mean(accuracy)
value = sess.run(accuracy) # compute the accuracy
# log the accuracy using stdout
decimals = 100
percent = np.math.floor(value * 100 * decimals) / decimals
def custom_acc(self, y_true, y_pred):
return binary_accuracy(K.round(y_true), K.round(y_pred))
def invasion_acc(y_true, y_pred):
binary_truth = y_true[:, -2] + y_true[:, -1]
binary_pred = y_pred[:, -2] + y_pred[:, -1]
return binary_accuracy(binary_truth, binary_pred)
def accuracy(y_true, y_pred):
return binary_accuracy(y_true, y_pred)
conv3 = Conv2D(128, (3, 3), padding='valid', activation='relu')(pool1)
pool2 = MaxPool2D((2, 2), padding='valid', strides=2)(conv3)
conv5 = Conv2D(256, (3, 3), padding='valid', activation='relu')(pool2)
up1 = UpSampling2D((2, 2))(conv5)
up2 = UpSampling2D((2, 2))(up1)
norm1 = BatchNormalization()(up2)
final = Conv2D(1, (1, 1), activation='softmax')(norm1)
model3 = Model(inputs=inputs, outputs=final)
model3.compile(loss='binary_crossentropy',
optimizer=Adam(lr=1e-4),
metrics=['accuracy'])
callbacks_list = [
EarlyStopping(monitor='val_acc', patience=1),
ModelCheckpoint(filepath='/global/scratch/cgroschner/encoder3000train.h5',
monitor='val_acc',
save_best_only=True)
]
model3.fit(trainX,
trainY,
batch_size=20,
epochs=5,
verbose=1,
shuffle=True,
callbacks=callbacks_list,
validation_data=(testX, testY))
model.save('/global/scratch/cgroschner/encoder3000train.h5')
predY = model.predict(testX, batch_size=10, verbose=1)
print(metrics.binary_accuracy(testY, predY))
def ia_acc(y_true, y_pred):
binary_truth = y_true[:, -1]
binary_pred = y_pred[:, -1]
return binary_accuracy(binary_truth, binary_pred)
def binary_error(y_true: tf.Tensor, y_pred: tf.Tensor) -> tf.Tensor:
return 1.0 - binary_accuracy(y_true, y_pred)
def binaryAccuracy(y_true, y_pred):
return binary_accuracy(y_true, K.sigmoid(y_pred))
def sigm_binary_accuracy(y_true, y_pred):
return binary_accuracy(y_true, tf.math.sigmoid(y_pred)) #tf.math.sigmoid(y_pred)
