def _dsebm_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q):
# gpu_to_use = gpu_q.get()
# os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use
(x_train, y_train), (x_test, y_test) = dataset_load_fn()
n_channels = x_train.shape[get_channels_axis()]
input_side = x_train.shape[2] # image side will always be at shape[2]
encoder_mdl = conv_encoder(input_side,
n_channels,
representation_activation='relu')
energy_mdl = dsebm.create_energy_model(encoder_mdl)
reconstruction_mdl = dsebm.create_reconstruction_model(energy_mdl)
# optimization parameters
batch_size = 128
epochs = 200
reconstruction_mdl.compile('adam', 'mse')
x_train_task = x_train[y_train.flatten() == single_class_ind]
x_test_task = x_test[y_test.flatten(
) == single_class_ind] # This is just for visual monitoring
reconstruction_mdl.fit(x=x_train_task,
y=x_train_task,
batch_size=batch_size,
epochs=epochs,
validation_data=(x_test_task, x_test_task))
scores = -energy_mdl.predict(x_test, batch_size)
labels = y_test.flatten() == single_class_ind
res_file_name = '{}_dsebm_{}_{}.npz'.format(
dataset_name, get_class_name_from_index(single_class_ind,
dataset_name),
datetime.datetime.now().strftime('%Y-%m-%d-%H%M'))
res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)
save_roc_pr_curve_data(scores, labels, res_file_path)
def _raw_ocsvm_experiment(dataset_load_fn, dataset_name, single_class_ind):
(x_train, y_train), (x_test, y_test) = dataset_load_fn()
x_train = x_train.reshape((len(x_train), -1))
x_test = x_test.reshape((len(x_test), -1))
x_train_task = x_train[y_train.flatten() == single_class_ind]
if dataset_name in LARGE_DATASET_NAMES: # OC-SVM is quadratic on the number of examples, so subsample training set
subsample_inds = np.random.choice(len(x_train_task),
2500,
replace=False)
x_train_task_tmp = x_train_task[subsample_inds]
# ToDO: make gridsearch just one
pg = ParameterGrid({
'nu': np.linspace(0.1, 0.9, num=9),
'gamma': np.logspace(-7, 2, num=10, base=2)
})
results = Parallel(n_jobs=PARALLEL_N_JOBS)(delayed(_train_ocsvm_and_score)(
d, x_train_task_tmp, y_test.flatten() == single_class_ind, x_test)
for d in pg)
best_params, best_auc_score = max(zip(pg, results), key=lambda t: t[-1])
print(best_params)
best_ocsvm = OneClassSVM(**best_params).fit(x_train_task)
scores = best_ocsvm.decision_function(x_test)
labels = y_test.flatten() == single_class_ind
res_file_name = '{}_raw-oc-svm_{}_{}.npz'.format(
dataset_name, get_class_name_from_index(single_class_ind,
dataset_name),
datetime.datetime.now().strftime('%Y-%m-%d-%H%M'))
res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)
save_roc_pr_curve_data(scores, labels, res_file_path)
def save_results_file(results_dir, dataset_name, single_class_ind, scores,
labels, experiment_name):
res_file_name = '{}_{}_{}_{}.npz'.format(
dataset_name, experiment_name,
get_class_name_from_index(single_class_ind, dataset_name),
datetime.datetime.now().strftime('%Y-%m-%d-%H%M'))
res_file_path = os.path.join(results_dir, dataset_name, res_file_name)
save_roc_pr_curve_data(scores, labels, res_file_path)
def _cae_ocsvm_experiment(dataset_load_fn, dataset_name, single_class_ind,
gpu_q):
# gpu_to_use = gpu_q.get()
# os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use
(x_train, y_train), (x_test, y_test) = dataset_load_fn()
print('data_shape', x_train.shape)
n_channels = x_train.shape[get_channels_axis()]
input_side = x_train.shape[2] # channel side will always be at shape[2]
enc = conv_encoder(input_side, n_channels)
dec = conv_decoder(input_side, n_channels)
# print(input_side)
# print(dec.summary())
x_in = Input(shape=x_train.shape[1:])
x_rec = dec(enc(x_in))
cae = Model(x_in, x_rec)
cae.compile('adam', 'mse')
x_train_task = x_train[y_train.flatten() == single_class_ind]
x_test_task = x_test[y_test.flatten(
) == single_class_ind] # This is just for visual monitoring
cae.fit(x=x_train_task,
y=x_train_task,
batch_size=128,
epochs=200,
validation_data=(x_test_task, x_test_task))
x_train_task_rep = enc.predict(x_train_task, batch_size=128)
if dataset_name in LARGE_DATASET_NAMES: # OC-SVM is quadratic on the number of examples, so subsample training set
subsample_inds = np.random.choice(len(x_train_task_rep),
2500,
replace=False)
x_train_task_rep_temp = x_train_task_rep[subsample_inds]
x_test_rep = enc.predict(x_test, batch_size=128)
pg = ParameterGrid({
'nu': np.linspace(0.1, 0.9, num=9),
'gamma': np.logspace(-7, 2, num=10, base=2)
})
results = Parallel(n_jobs=PARALLEL_N_JOBS)(delayed(
_train_ocsvm_and_score)(d, x_train_task_rep_temp, y_test.flatten() ==
single_class_ind, x_test_rep) for d in pg)
best_params, best_auc_score = max(zip(pg, results), key=lambda t: t[-1])
print(best_params)
best_ocsvm = OneClassSVM(**best_params).fit(x_train_task_rep)
scores = best_ocsvm.decision_function(x_test_rep)
labels = y_test.flatten() == single_class_ind
res_file_name = '{}_cae-oc-svm_{}_{}.npz'.format(
dataset_name, get_class_name_from_index(single_class_ind,
dataset_name),
datetime.datetime.now().strftime('%Y-%m-%d-%H%M'))
res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)
save_roc_pr_curve_data(scores, labels, res_file_path)
def _adgan_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q):
# gpu_to_use = gpu_q.get()
# os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use
(x_train, y_train), (x_test, y_test) = dataset_load_fn()
if len(x_test) > 5000:
# subsample x_test due to runtime complexity
chosen_inds = np.random.choice(len(x_test), 5000, replace=False)
x_test = x_test[chosen_inds]
y_test = y_test[chosen_inds]
n_channels = x_train.shape[get_channels_axis()]
input_side = x_train.shape[2] # image side will always be at shape[2]
critic = conv_encoder(input_side,
n_channels,
representation_dim=1,
representation_activation='linear')
noise_size = 256
generator = conv_decoder(input_side,
n_channels=n_channels,
representation_dim=noise_size)
def prior_gen(b_size):
return np.random.normal(size=(b_size, noise_size))
batch_size = 128
epochs = 100
x_train_task = x_train[y_train.flatten() == single_class_ind]
def data_gen(b_size):
chosen_inds = np.random.choice(len(x_train_task),
b_size,
replace=False)
return x_train_task[chosen_inds]
adgan.train_wgan_with_grad_penalty(prior_gen,
generator,
data_gen,
critic,
batch_size,
epochs,
grad_pen_coef=20)
scores = adgan.scores_from_adgan_generator(x_test, prior_gen, generator)
labels = y_test.flatten() == single_class_ind
res_file_name = '{}_adgan_{}_{}.npz'.format(
dataset_name, get_class_name_from_index(single_class_ind,
dataset_name),
datetime.datetime.now().strftime('%Y-%m-%d-%H%M'))
res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)
save_roc_pr_curve_data(scores, labels, res_file_path)
def _dagmm_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q):
gpu_to_use = gpu_q.get()
os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use
(x_train, y_train), (x_test, y_test) = dataset_load_fn()
n_channels = x_train.shape[get_channels_axis()]
input_side = x_train.shape[2] # image side will always be at shape[2]
enc = conv_encoder(input_side, n_channels, representation_dim=5,
representation_activation='linear')
dec = conv_decoder(input_side, n_channels=n_channels, representation_dim=enc.output_shape[-1])
n_components = 3
estimation = Sequential([Dense(64, activation='tanh', input_dim=enc.output_shape[-1] + 2), Dropout(0.5),
Dense(10, activation='tanh'), Dropout(0.5),
Dense(n_components, activation='softmax')]
)
batch_size = 256
epochs = 200
lambda_diag = 0.0005
lambda_energy = 0.01
dagmm_mdl = dagmm.create_dagmm_model(enc, dec, estimation, lambda_diag)
dagmm_mdl.compile('adam', ['mse', lambda y_true, y_pred: lambda_energy*y_pred])
x_train_task = x_train[y_train.flatten() == single_class_ind]
x_test_task = x_test[y_test.flatten() == single_class_ind] # This is just for visual monitoring
dagmm_mdl.fit(x=x_train_task, y=[x_train_task, np.zeros((len(x_train_task), 1))], # second y is dummy
batch_size=batch_size,
epochs=epochs,
validation_data=(x_test_task, [x_test_task, np.zeros((len(x_test_task), 1))]),
# verbose=0
)
energy_mdl = Model(dagmm_mdl.input, dagmm_mdl.output[-1])
scores = -energy_mdl.predict(x_test, batch_size)
scores = scores.flatten()
if not np.all(np.isfinite(scores)):
min_finite = np.min(scores[np.isfinite(scores)])
scores[~np.isfinite(scores)] = min_finite - 1
labels = y_test.flatten() == single_class_ind
res_file_name = '{}_dagmm_{}_{}.npz'.format(dataset_name,
get_class_name_from_index(single_class_ind, dataset_name),
datetime.now().strftime('%Y-%m-%d-%H%M'))
res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)
save_roc_pr_curve_data(scores, labels, res_file_path)
gpu_q.put(gpu_to_use)
def _mo_gaal_experiment(dataset_load_fn, dataset_name, single_class_ind):
(x_train, y_train), (x_test, y_test) = dataset_load_fn()
x_train = x_train.reshape((len(x_train), -1))
x_test = x_test.reshape((len(x_test), -1))
x_train_task = x_train[y_train.flatten() == single_class_ind]
best_mo_gaal = MO_GAAL().fit(x_train_task)
scores = best_mo_gaal.decision_function(x_test)
labels = y_test.flatten() == single_class_ind
res_file_name = '{}_mo-gaal_{}_{}.npz'.format(
dataset_name, get_class_name_from_index(single_class_ind,
dataset_name),
datetime.datetime.now().strftime('%Y-%m-%d-%H%M'))
res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)
save_roc_pr_curve_data(scores, labels, res_file_path)
def _if_experiment(dataset_load_fn, dataset_name, single_class_ind):
(x_train, y_train), (x_test, y_test) = dataset_load_fn()
x_train = x_train.reshape((len(x_train), -1))
x_test = x_test.reshape((len(x_test), -1))
x_train_task = x_train[y_train.flatten() == single_class_ind]
# if dataset_name in LARGE_DATASET_NAMES: # OC-SVM is quadratic on the number of examples, so subsample training set
# subsample_inds = np.random.choice(len(x_train_task), 2500, replace=False)
# x_train_task_tmp = x_train_task[subsample_inds]
# ToDO: make gridsearch just one
pg = ParameterGrid({
'n_estimators': np.linspace(100, 800, num=8).astype(int),
'contamination': [0.1, 0.2, 0.3, 0.4, 0.5],
'behaviour': ['new'],
'n_jobs': [-1]
})
# results = Parallel(n_jobs=PARALLEL_N_JOBS)(
# delayed(_train_if_and_score)(d, x_train_task, y_test.flatten() == single_class_ind, x_test)
# for d in pg)
results = []
for d in pg:
results.append(
_train_if_and_score(d, x_train_task,
y_test.flatten() == single_class_ind, x_test))
# print(list(zip(pg, results)))
best_params, best_auc_score = max(zip(pg, results), key=lambda t: t[-1])
# print(best_params, ' ', best_auc_score)
best_if = IsolationForest(**best_params).fit(x_train_task)
scores = best_if.decision_function(x_test)
labels = y_test.flatten() == single_class_ind
res_file_name = '{}_if_{}_{}.npz'.format(
dataset_name, get_class_name_from_index(single_class_ind,
dataset_name),
datetime.datetime.now().strftime('%Y-%m-%d-%H%M'))
res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)
save_roc_pr_curve_data(scores, labels, res_file_path)
def test_model_loading(transformer,
mdl,
loader,
dataset_name='hits-4-c',
single_class_ind=1,
tf_version='tf1',
transformer_name='transformed',
model_name='resnet',
epochs=None):
results_dir = os.path.join(PROJECT_PATH, 'tests', 'aux_results')
save_dir = os.path.join(PROJECT_PATH, 'tests', 'aux_data')
utils.check_path(results_dir)
utils.check_path(save_dir)
utils.check_path(os.path.join(results_dir, dataset_name))
# load-save data
normal_data_path = os.path.join(
save_dir, 'normal_data_%s_%s_loading.pkl' % (dataset_name, tf_version))
if os.path.exists(normal_data_path):
(x_train,
y_train), (x_val, y_val), (x_test,
y_test) = pd.read_pickle(normal_data_path)
else:
(x_train, y_train), (x_val, y_val), (x_test,
y_test) = loader(return_val=True)
normal_data = (x_train, y_train), (x_val, y_val), (x_test, y_test)
utils.save_pickle(normal_data, normal_data_path)
# create model
# n, k = (10, 4)
# mdl = create_wide_residual_network(
# x_train.shape[1:], transformer.n_transforms, n, k)
mdl.compile('adam', 'categorical_crossentropy', ['acc'])
# selec inliers
x_train = x_train[y_train.flatten() == single_class_ind]
x_val = x_val[y_val.flatten() == single_class_ind]
# load-save transformed data
transformed_data_path = os.path.join(
save_dir, '%s_data_%s_%s_loading.pkl' %
(transformer_name, dataset_name, tf_version))
if os.path.exists(transformed_data_path):
(x_train_transform_tf1,
y_train_transform_tf1), (x_val_transform_tf1, y_val_transform_tf1), (
x_test_transform_tf1,
y_test_transform_tf1) = pd.read_pickle(transformed_data_path)
else:
# transform all data
y_train_transform_tf1 = np.tile(np.arange(transformer.n_transforms),
len(x_train))
x_train_transform_tf1 = transformer.transform_batch(
np.repeat(x_train, transformer.n_transforms, axis=0),
y_train_transform_tf1)
y_val_transform_tf1 = np.tile(np.arange(transformer.n_transforms),
len(x_val))
x_val_transform_tf1 = transformer.transform_batch(
np.repeat(x_val, transformer.n_transforms, axis=0),
y_val_transform_tf1)
y_test_transform_tf1 = np.tile(np.arange(transformer.n_transforms),
len(x_test))
x_test_transform_tf1 = transformer.transform_batch(
np.repeat(x_test, transformer.n_transforms, axis=0),
y_test_transform_tf1)
transformed_data = ((x_train_transform_tf1, y_train_transform_tf1),
(x_val_transform_tf1, y_val_transform_tf1),
(x_test_transform_tf1, y_test_transform_tf1))
utils.save_pickle(transformed_data, transformed_data_path)
print(x_train.shape)
print(x_train_transform_tf1.shape)
print(x_test.shape)
print(x_test_transform_tf1.shape)
# train model
batch_size = 128
if epochs is None:
epochs = int(np.ceil(200 / transformer.n_transforms))
mdl.fit(x=x_train_transform_tf1,
y=to_categorical(y_train_transform_tf1),
batch_size=batch_size,
epochs=epochs)
scores = np.zeros((len(x_test), ))
matrix_evals = np.zeros(
(len(x_test), transformer.n_transforms, transformer.n_transforms))
x_pred_train = mdl.predict(x_train_transform_tf1, batch_size=1024)
x_pred_test = mdl.predict(x_test_transform_tf1, batch_size=1024)
print(x_pred_train.shape)
print(x_pred_test.shape)
for t_ind in range(transformer.n_transforms):
ind_x_pred_equal_to_t_ind = np.where(y_train_transform_tf1 == t_ind)[0]
observed_dirichlet = x_pred_train[ind_x_pred_equal_to_t_ind]
log_p_hat_train = np.log(observed_dirichlet).mean(axis=0)
alpha_sum_approx = calc_approx_alpha_sum(observed_dirichlet)
alpha_0 = observed_dirichlet.mean(axis=0) * alpha_sum_approx
mle_alpha_t = fixed_point_dirichlet_mle(alpha_0, log_p_hat_train)
ind_x_pred_test_equal_to_t_ind = np.where(
y_test_transform_tf1 == t_ind)[0]
x_test_p = x_pred_test[ind_x_pred_test_equal_to_t_ind]
matrix_evals[:, :, t_ind] += x_test_p
scores += dirichlet_normality_score(mle_alpha_t, x_test_p)
scores /= transformer.n_transforms
matrix_evals /= transformer.n_transforms
scores_simple = np.trace(matrix_evals, axis1=1, axis2=2)
scores_entropy = -1 * get_entropy(matrix_evals)
scores_xH = -1 * get_xH(transformer, matrix_evals)
labels = y_test.flatten() == single_class_ind
save_results_file(results_dir,
dataset_name,
single_class_ind,
scores=scores,
labels=labels,
experiment_name='%s-%s-loading-%s' %
(model_name, transformer_name, tf_version))
save_results_file(results_dir,
dataset_name,
single_class_ind,
scores=scores_simple,
labels=labels,
experiment_name='%s-%s-simple-loading-%s' %
(model_name, transformer_name, tf_version))
save_results_file(results_dir,
dataset_name,
single_class_ind,
scores=scores_entropy,
labels=labels,
experiment_name='%s-%s-entropy-loading-%s' %
(model_name, transformer_name, tf_version))
save_results_file(results_dir,
dataset_name,
single_class_ind,
scores=scores_xH,
labels=labels,
experiment_name='%s-%s-xH-loading-%s' %
(model_name, transformer_name, tf_version))
mdl_weights_name = '{}_{}_{}_{}_loading_{}_weights.h5'.format(
model_name, transformer_name, dataset_name, tf_version,
get_class_name_from_index(single_class_ind, dataset_name))
mdl_weights_path = os.path.join(results_dir, dataset_name,
mdl_weights_name)
mdl.save_weights(mdl_weights_path)
reset_weights()
"""
Time test_model_original(transformer, load_hits4c, dataset_name='hits-4-c', tf_version='tf1') 00:04:31.65
(0.992217, 0.9895665, 0.99131725, 0.989478125)
(0.99240075, 0.9900822499999999, 0.99215325, 0.9901300000000001)
"""
return get_roc_auc(scores, labels), get_roc_auc(scores_simple, labels), \
get_roc_auc(scores_entropy, labels), get_roc_auc(scores_xH, labels)
def test_model_original(transformer,
loader,
dataset_name='hits-4-c',
single_class_ind=1):
results_dir = os.path.join(PROJECT_PATH, 'tests', 'aux_results')
save_dir = os.path.join(PROJECT_PATH, 'tests', 'aux_data')
utils.check_path(results_dir)
utils.check_path(save_dir)
utils.check_path(os.path.join(results_dir, dataset_name))
# load-save data
(x_train, y_train), (x_val, y_val), (x_test,
y_test) = loader(return_val=True)
normal_data = (x_train, y_train), (x_val, y_val), (x_test, y_test)
utils.save_pickle(
normal_data,
os.path.join(save_dir,
'normal_data_%s_tf1_original.pkl' % dataset_name))
# create model
n, k = (10, 4)
mdl = create_wide_residual_network(x_train.shape[1:],
transformer.n_transforms, n, k)
mdl.compile('adam', 'categorical_crossentropy', ['acc'])
# get inliers of specific class
# get inliers
x_train_task = x_train[y_train.flatten() == single_class_ind]
print(x_train_task.shape)
# transform inliers
transformations_inds = np.tile(np.arange(transformer.n_transforms),
len(x_train_task))
x_train_task_transformed = transformer.transform_batch(
np.repeat(x_train_task, transformer.n_transforms, axis=0),
transformations_inds)
print(x_train_task_transformed.shape)
# train model
batch_size = 128
mdl.fit(x=x_train_task_transformed,
y=to_categorical(transformations_inds),
batch_size=batch_size,
epochs=int(np.ceil(200 / transformer.n_transforms)))
scores = np.zeros((len(x_test), ))
matrix_evals = np.zeros(
(len(x_test), transformer.n_transforms, transformer.n_transforms))
observed_data = x_train_task
for t_ind in range(transformer.n_transforms):
observed_dirichlet = mdl.predict(transformer.transform_batch(
observed_data, [t_ind] * len(observed_data)),
batch_size=1024)
log_p_hat_train = np.log(observed_dirichlet).mean(axis=0)
alpha_sum_approx = calc_approx_alpha_sum(observed_dirichlet)
alpha_0 = observed_dirichlet.mean(axis=0) * alpha_sum_approx
mle_alpha_t = fixed_point_dirichlet_mle(alpha_0, log_p_hat_train)
x_test_p = mdl.predict(transformer.transform_batch(
x_test, [t_ind] * len(x_test)),
batch_size=1024)
matrix_evals[:, :, t_ind] += x_test_p
scores += dirichlet_normality_score(mle_alpha_t, x_test_p)
scores /= transformer.n_transforms
matrix_evals /= transformer.n_transforms
scores_simple = np.trace(matrix_evals, axis1=1, axis2=2)
scores_entropy = -1 * get_entropy(matrix_evals)
scores_xH = -1 * get_xH(transformer, matrix_evals)
labels = y_test.flatten() == single_class_ind
save_results_file(results_dir,
dataset_name,
single_class_ind,
scores=scores,
labels=labels,
experiment_name='transformations')
save_results_file(results_dir,
dataset_name,
single_class_ind,
scores=scores_simple,
labels=labels,
experiment_name='transformations-simple')
save_results_file(results_dir,
dataset_name,
single_class_ind,
scores=scores_entropy,
labels=labels,
experiment_name='transformations-entropy')
save_results_file(results_dir,
dataset_name,
single_class_ind,
scores=scores_xH,
labels=labels,
experiment_name='transformations-xH')
mdl_weights_name = '{}_tf1_original_{}_weights.h5'.format(
dataset_name, get_class_name_from_index(single_class_ind,
dataset_name))
mdl_weights_path = os.path.join(results_dir, dataset_name,
mdl_weights_name)
mdl.save_weights(mdl_weights_path)
"""
Time test_model_original(transformer, load_hits4c, dataset_name='hits-4-c') 00:06:58.37
(0.9917134999999999, 0.9350055, 0.9872614999999999, 0.94142025)
(0.9938067500000001, 0.9923547500000001, 0.9931685, 0.992637375)
(0.9912172499999999, 0.9883357499999998, 0.9909070000000001, 0.9886706249999999)
#train only Time test_model_original(transformer, load_hits4c, dataset_name='hits-4-c', tf_version='tf1') 00:03:48.29
"""
return get_roc_auc(scores, labels), get_roc_auc(scores_simple, labels), \
get_roc_auc(scores_entropy, labels), get_roc_auc(scores_xH, labels)
def _kernal_plus_transformations_experiment(dataset_load_fn, dataset_name,
single_class_ind, gpu_q):
# gpu_to_use = gpu_q.get()
# os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use
(x_train, y_train), (x_test, y_test) = dataset_load_fn()
if dataset_name in ['cats-vs-dogs']:
transformer = None
else:
transformer = PlusKernelTransformer(translation_x=8,
translation_y=8,
rotations=1,
flips=1,
gauss=1,
log=1)
n, k = (10, 4)
mdl = create_wide_residual_network(x_train.shape[1:],
transformer.n_transforms, n, k)
mdl.compile('adam', 'categorical_crossentropy', ['acc'])
# get inliers of specific class
x_train_task = x_train[y_train.flatten() == single_class_ind]
# [0_i, ..., (N_transforms-1)_i, ..., ..., 0_N_samples, ...,
# (N_transforms-1)_N_samples] shape: (N_transforms*N_samples,)
transformations_inds = np.tile(np.arange(transformer.n_transforms),
len(x_train_task))
x_train_task_transformed = transformer.transform_batch(
np.repeat(x_train_task, transformer.n_transforms, axis=0),
transformations_inds)
batch_size = 128
mdl.fit(
x=x_train_task_transformed,
y=to_categorical(transformations_inds),
batch_size=batch_size,
epochs=2 #int(np.ceil(200/transformer.n_transforms))
)
scores = np.zeros((len(x_test), ))
matrix_evals = np.zeros(
(len(x_test), transformer.n_transforms, transformer.n_transforms))
observed_data = x_train_task
for t_ind in range(transformer.n_transforms):
observed_dirichlet = mdl.predict(transformer.transform_batch(
observed_data, [t_ind] * len(observed_data)),
batch_size=1024)
log_p_hat_train = np.log(observed_dirichlet).mean(axis=0)
alpha_sum_approx = calc_approx_alpha_sum(observed_dirichlet)
alpha_0 = observed_dirichlet.mean(axis=0) * alpha_sum_approx
mle_alpha_t = fixed_point_dirichlet_mle(alpha_0, log_p_hat_train)
x_test_p = mdl.predict(transformer.transform_batch(
x_test, [t_ind] * len(x_test)),
batch_size=1024)
matrix_evals[:, :, t_ind] += x_test_p
scores += dirichlet_normality_score(mle_alpha_t, x_test_p)
scores /= transformer.n_transforms
matrix_evals /= transformer.n_transforms
scores_simple = np.trace(matrix_evals, axis1=1, axis2=2)
scores_entropy = -1 * get_entropy(matrix_evals)
scores_xH = -1 * get_xH(transformer, matrix_evals)
labels = y_test.flatten() == single_class_ind
save_results_file(dataset_name,
single_class_ind,
scores=scores,
labels=labels,
experiment_name='kernel-plus-transformations')
save_results_file(dataset_name,
single_class_ind,
scores=scores_simple,
labels=labels,
experiment_name='kernel-plus-transformations-simple')
save_results_file(dataset_name,
single_class_ind,
scores=scores_entropy,
labels=labels,
experiment_name='kernel-plus-transformations-entropy')
save_results_file(dataset_name,
single_class_ind,
scores=scores_xH,
labels=labels,
experiment_name='kernel-plus-transformations-xH')
mdl_weights_name = '{}_kernel-plus-transformations_{}_{}_weights.h5'.format(
dataset_name, get_class_name_from_index(single_class_ind,
dataset_name),
datetime.datetime.now().strftime('%Y-%m-%d-%H%M'))
mdl_weights_path = os.path.join(RESULTS_DIR, dataset_name,
mdl_weights_name)
mdl.save_weights(mdl_weights_path)
def _transformations_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q):
gpu_to_use = gpu_q.get()
os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use
(x_train, y_train), (x_test, y_test) = dataset_load_fn()
if dataset_name in ['cats-vs-dogs']:
transformer = Transformer(16, 16)
n, k = (16, 8)
else:
transformer = Transformer(8, 8)
n, k = (10, 4)
mdl = create_wide_residual_network(x_train.shape[1:], transformer.n_transforms, n, k)
mdl.compile('adam',
'categorical_crossentropy',
['acc'])
# get inliers of specific class
x_train_task = x_train[y_train.flatten() == single_class_ind]
# [0_i, ..., (N_transforms-1)_i, ..., ..., 0_N_samples, ...,
# (N_transforms-1)_N_samples] shape: (N_transforms*N_samples,)
transformations_inds = np.tile(np.arange(transformer.n_transforms), len(x_train_task))
x_train_task_transformed = transformer.transform_batch(np.repeat(x_train_task, transformer.n_transforms, axis=0),
transformations_inds)
batch_size = 128
mdl.fit(x=x_train_task_transformed, y=to_categorical(transformations_inds),
batch_size=batch_size, epochs=int(np.ceil(200/transformer.n_transforms))
)
#################################################################################################
# simplified normality score
#################################################################################################
# preds = np.zeros((len(x_test), transformer.n_transforms))
# for t in range(transformer.n_transforms):
# preds[:, t] = mdl.predict(transformer.transform_batch(x_test, [t] * len(x_test)),
# batch_size=batch_size)[:, t]
#
# labels = y_test.flatten() == single_class_ind
# scores = preds.mean(axis=-1)
#################################################################################################
def calc_approx_alpha_sum(observations):
N = len(observations)
f = np.mean(observations, axis=0)
return (N * (len(f) - 1) * (-psi(1))) / (
N * np.sum(f * np.log(f)) - np.sum(f * np.sum(np.log(observations), axis=0)))
def inv_psi(y, iters=5):
# initial estimate
cond = y >= -2.22
x = cond * (np.exp(y) + 0.5) + (1 - cond) * -1 / (y - psi(1))
for _ in range(iters):
x = x - (psi(x) - y) / polygamma(1, x)
return x
def fixed_point_dirichlet_mle(alpha_init, log_p_hat, max_iter=1000):
alpha_new = alpha_old = alpha_init
for _ in range(max_iter):
alpha_new = inv_psi(psi(np.sum(alpha_old)) + log_p_hat)
if np.sqrt(np.sum((alpha_old - alpha_new) ** 2)) < 1e-9:
break
alpha_old = alpha_new
return alpha_new
def dirichlet_normality_score(alpha, p):
return np.sum((alpha - 1) * np.log(p), axis=-1)
scores = np.zeros((len(x_test),))
observed_data = x_train_task
for t_ind in range(transformer.n_transforms):
observed_dirichlet = mdl.predict(transformer.transform_batch(observed_data, [t_ind] * len(observed_data)),
batch_size=1024)
log_p_hat_train = np.log(observed_dirichlet).mean(axis=0)
alpha_sum_approx = calc_approx_alpha_sum(observed_dirichlet)
alpha_0 = observed_dirichlet.mean(axis=0) * alpha_sum_approx
mle_alpha_t = fixed_point_dirichlet_mle(alpha_0, log_p_hat_train)
x_test_p = mdl.predict(transformer.transform_batch(x_test, [t_ind] * len(x_test)),
batch_size=1024)
scores += dirichlet_normality_score(mle_alpha_t, x_test_p)
scores /= transformer.n_transforms
labels = y_test.flatten() == single_class_ind
res_file_name = '{}_transformations_{}_{}.npz'.format(dataset_name,
get_class_name_from_index(single_class_ind, dataset_name),
datetime.now().strftime('%Y-%m-%d-%H%M'))
res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)
save_roc_pr_curve_data(scores, labels, res_file_path)
mdl_weights_name = '{}_transformations_{}_{}_weights.h5'.format(dataset_name,
get_class_name_from_index(single_class_ind, dataset_name),
datetime.now().strftime('%Y-%m-%d-%H%M'))
mdl_weights_path = os.path.join(RESULTS_DIR, dataset_name, mdl_weights_name)
mdl.save_weights(mdl_weights_path)
gpu_q.put(gpu_to_use)