def check_shape(shape, fun):
y_true = rnd.choice([0, 1], size=shape, p=[2. / 3, 1. / 3])
y_pred = rnd.uniform(0, 1, shape)
fun_np = getattr(sys.modules[__name__], fun.__name__ + '_np')
res_np = fun_np(y_true, y_pred)
res_k = K.eval(fun(K.variable(y_true), K.variable(y_pred)))
print(res_k, res_np)
assert res_k.shape == res_np.shape
assert np.isclose(res_k, res_np)
print('pEER: %.3f' % mtr.eer(y_true.flatten(), y_pred.flatten()))
def validate_model(model, batch_gen, metrics):
"""
# Arguments
model: Keras model
data: BaseDataLoader
metrics: list of metrics
# Output
dictionary with values of metrics and loss
"""
cut_model = model
if DCASEModelTrainer.is_mfom_objective(model):
input = model.get_layer(name='input').output
preact = model.get_layer(name='output').output
cut_model = Model(input=input, output=preact)
n_class = cut_model.output_shape[1]
y_true, y_pred = np.empty((0, n_class)), np.empty((0, n_class))
loss, cnt = 0, 0
for X_b, Y_b in batch_gen.batch():
ps = cut_model.predict_on_batch(X_b)
y_pred = np.vstack([y_pred, ps])
y_true = np.vstack([y_true, Y_b])
# NOTE: it is fake loss, caz Y is fed
if DCASEModelTrainer.is_mfom_objective(model):
X_b = [Y_b, X_b]
l = model.test_on_batch(X_b, Y_b)
loss += l
cnt += 1
vals = {'val_loss': loss / cnt}
for m in metrics:
if m == 'micro_f1':
p = mtx.step(y_pred, threshold=0.5)
vals[m] = mtx.micro_f1(y_true, p)
elif m == 'pooled_eer':
p = y_pred.flatten()
y = y_true.flatten()
vals[m] = mtx.eer(y, p)
elif m == 'class_wise_eer':
vals[m] = np.mean(mtx.class_wise_eer(y_true, y_pred))
elif m == 'accuracy':
p = np.argmax(y_pred, axis=-1)
y = np.argmax(y_true, axis=-1)
vals[m] = mtx.pooled_accuracy(y, p)
else:
raise KeyError(
'[ERROR] Such a metric is not implemented: %s...' % m)
return vals
# train
all_X, all_Y = [], []
for i in range(10000 // time_step):
X, Y = generate_dataset(output_dim=nclass, num_examples=time_step)
all_X.append(X.T)
all_Y.append(Y)
all_X, all_Y = np.array(all_X), np.array(all_Y)
hist = model.fit([all_Y, all_X], all_Y, nb_epoch=200, batch_size=5)
# calc EER: we cut MFoM head, up to sigmoid output
input = model.get_layer(name='main_input').output
out = model.get_layer(name='output').output
cut_model = Model(input=input, output=out)
y_pred_sig = cut_model.predict(all_X)
eer_val = MT.eer(all_Y.flatten(), y_pred_sig.flatten())
print('sigma_EER: %.4f' % eer_val)
y_pred = model.predict([all_Y, all_X])
eer_val = MT.eer(all_Y.flatten(), 1. - y_pred.flatten())
print('l_EER: %.4f' % eer_val)
print(model.evaluate([all_Y, all_X], all_Y))
# TODO notice from the experiments:
# when we minimize obj.mfom_microf1 with psi = y_pred or psi = -y_pred + 0.5 in
# UvZMisclassification() layer, the smoothF1 is minimized !!! but EER is not at all.
# When we minimize obj.mfom_microf1 with psi = -y_pred + y_neg * unit_avg + y_true * zeros_avg,
# then both smoothF1 and EER are minimized :)
# history plot, alpha and beta params
m = model.get_layer('smooth_error_counter')
nclass = 10
# mfom model
model = mfom_model(dim, nclass)
model.compile(loss=obj.mfom_eer_normalized, optimizer='Adam')
model.summary()
# training on multi-label dataset
x_train, x_test, y_train, y_test = generate_dataset(n_smp=10000,
n_feat=dim,
n_cls=nclass)
mask = y_train.sum(axis=-1) != nclass
y_train = y_train[mask]
x_train = x_train[mask]
hist = model.fit([y_train, x_train], y_train, nb_epoch=10, batch_size=16)
# cut MFoM head
cut_model = cut_mfom(model)
y_pred = cut_model.predict(x_test)
# evaluate
eer_val = MT.eer(y_true=y_test.flatten(), y_pred=y_pred.flatten())
print('EER: %.4f' % eer_val)
# history plot, alpha and beta params of MFoM
m = model.get_layer('smooth_error_counter')
print('alpha: ', K.get_value(m.alpha))
print('beta: ', K.get_value(m.beta))
plt.plot(hist.history['loss'])
plt.show()
def validate_model(model, batch_gen, metrics, attrib_cls=None):
"""
# Arguments
model: Keras model
data: BaseDataLoader
metrics: list of metrics
# Output
dictionary with values of metrics and loss
"""
cut_model = model
if OGITSModelTrainer.is_mfom_objective(model):
input = model.get_layer(name='input').output
preact = model.get_layer(name='output').output
cut_model = Model(input=input, output=preact)
n_class = cut_model.output_shape[-1]
_, bnd, wnd, ch = batch_gen.batch_shape()
y_true, y_pred = np.empty((0, wnd, n_class)), np.empty(
(0, wnd, n_class))
loss, cnt = 0, 0
for fn, X_b, Y_b in batch_gen.batch():
ps = cut_model.predict_on_batch(X_b)
y_pred = np.vstack([y_pred, ps])
y_true = np.vstack([y_true, Y_b])
# NOTE: it is approximated loss, caz Y is fed
if OGITSModelTrainer.is_mfom_objective(model):
X_b = [Y_b, X_b]
l = model.test_on_batch(X_b, Y_b)
loss += l
cnt += 1
vals = {'val_loss': loss / cnt}
y_true = y_true.reshape((-1, n_class))
y_pred = y_pred.reshape((-1, n_class))
if attrib_cls == 'fusion':
fmanner_ids = [2, 4, 9, 11, 12, 14, 15]
y_true_m = y_true[:, fmanner_ids]
y_pred_m = y_pred[:, fmanner_ids]
v_max, t_max = 0, 0
for t in [0.5]: # np.linspace(0.2, 0.7, 50):
p = mtx.step(y_pred_m, threshold=t)
v = mtx.micro_f1(y_true_m, p)
if v_max < v:
v_max = v
t_max = t
print('F1 fusion-manner:', v_max, t_max)
eer = np.mean(mtx.class_wise_eer(y_true_m, y_pred_m))
print('EER fusion-manner:', eer)
fplace_ids = [0, 1, 3, 5, 6, 7, 8, 10, 11, 13]
y_true_p = y_true[:, fplace_ids]
y_pred_p = y_pred[:, fplace_ids]
v_max, t_max = 0, 0
for t in [0.5]: # np.linspace(0.2, 0.7, 50):
p = mtx.step(y_pred_p, threshold=t)
v = mtx.micro_f1(y_true_p, p)
if v_max < v:
v_max = v
t_max = t
print('F1 fusion-place:', v_max, t_max)
eer = np.mean(mtx.class_wise_eer(y_true_p, y_pred_p))
print('EER fusion-place:', eer)
for m in metrics:
if m == 'micro_f1':
v_max, t_max = 0, 0
for t in [0.5]: # np.linspace(0.2, 0.7, 50):
p = mtx.step(y_pred, threshold=t)
v = mtx.micro_f1(y_true, p)
if v_max < v:
v_max = v
t_max = t
vals[m] = v_max
print('Optimal threshold: ', vals[m], t_max)
elif m == 'pooled_eer':
p = y_pred.flatten()
y = y_true.flatten()
vals[m] = mtx.eer(y, p)
elif m == 'class_wise_eer':
vals[m] = np.mean(mtx.class_wise_eer(y_true, y_pred))
elif m == 'accuracy':
p = np.argmax(y_pred, axis=-1)
y = np.argmax(y_true, axis=-1)
vals[m] = mtx.pooled_accuracy(y, p)
else:
raise KeyError(
'[ERROR] Such a metric is not implemented: %s...' % m)
return vals