def _dsebm_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q): # gpu_to_use = gpu_q.get() # os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use (x_train, y_train), (x_test, y_test) = dataset_load_fn() n_channels = x_train.shape[get_channels_axis()] input_side = x_train.shape[2] # image side will always be at shape[2] encoder_mdl = conv_encoder(input_side, n_channels, representation_activation='relu') energy_mdl = dsebm.create_energy_model(encoder_mdl) reconstruction_mdl = dsebm.create_reconstruction_model(energy_mdl) # optimization parameters batch_size = 128 epochs = 200 reconstruction_mdl.compile('adam', 'mse') x_train_task = x_train[y_train.flatten() == single_class_ind] x_test_task = x_test[y_test.flatten( ) == single_class_ind] # This is just for visual monitoring reconstruction_mdl.fit(x=x_train_task, y=x_train_task, batch_size=batch_size, epochs=epochs, validation_data=(x_test_task, x_test_task)) scores = -energy_mdl.predict(x_test, batch_size) labels = y_test.flatten() == single_class_ind res_file_name = '{}_dsebm_{}_{}.npz'.format( dataset_name, get_class_name_from_index(single_class_ind, dataset_name), datetime.datetime.now().strftime('%Y-%m-%d-%H%M')) res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name) save_roc_pr_curve_data(scores, labels, res_file_path)
def _raw_ocsvm_experiment(dataset_load_fn, dataset_name, single_class_ind): (x_train, y_train), (x_test, y_test) = dataset_load_fn() x_train = x_train.reshape((len(x_train), -1)) x_test = x_test.reshape((len(x_test), -1)) x_train_task = x_train[y_train.flatten() == single_class_ind] if dataset_name in LARGE_DATASET_NAMES: # OC-SVM is quadratic on the number of examples, so subsample training set subsample_inds = np.random.choice(len(x_train_task), 2500, replace=False) x_train_task_tmp = x_train_task[subsample_inds] # ToDO: make gridsearch just one pg = ParameterGrid({ 'nu': np.linspace(0.1, 0.9, num=9), 'gamma': np.logspace(-7, 2, num=10, base=2) }) results = Parallel(n_jobs=PARALLEL_N_JOBS)(delayed(_train_ocsvm_and_score)( d, x_train_task_tmp, y_test.flatten() == single_class_ind, x_test) for d in pg) best_params, best_auc_score = max(zip(pg, results), key=lambda t: t[-1]) print(best_params) best_ocsvm = OneClassSVM(**best_params).fit(x_train_task) scores = best_ocsvm.decision_function(x_test) labels = y_test.flatten() == single_class_ind res_file_name = '{}_raw-oc-svm_{}_{}.npz'.format( dataset_name, get_class_name_from_index(single_class_ind, dataset_name), datetime.datetime.now().strftime('%Y-%m-%d-%H%M')) res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name) save_roc_pr_curve_data(scores, labels, res_file_path)
def save_results_file(results_dir, dataset_name, single_class_ind, scores, labels, experiment_name): res_file_name = '{}_{}_{}_{}.npz'.format( dataset_name, experiment_name, get_class_name_from_index(single_class_ind, dataset_name), datetime.datetime.now().strftime('%Y-%m-%d-%H%M')) res_file_path = os.path.join(results_dir, dataset_name, res_file_name) save_roc_pr_curve_data(scores, labels, res_file_path)
def _cae_ocsvm_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q): # gpu_to_use = gpu_q.get() # os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use (x_train, y_train), (x_test, y_test) = dataset_load_fn() print('data_shape', x_train.shape) n_channels = x_train.shape[get_channels_axis()] input_side = x_train.shape[2] # channel side will always be at shape[2] enc = conv_encoder(input_side, n_channels) dec = conv_decoder(input_side, n_channels) # print(input_side) # print(dec.summary()) x_in = Input(shape=x_train.shape[1:]) x_rec = dec(enc(x_in)) cae = Model(x_in, x_rec) cae.compile('adam', 'mse') x_train_task = x_train[y_train.flatten() == single_class_ind] x_test_task = x_test[y_test.flatten( ) == single_class_ind] # This is just for visual monitoring cae.fit(x=x_train_task, y=x_train_task, batch_size=128, epochs=200, validation_data=(x_test_task, x_test_task)) x_train_task_rep = enc.predict(x_train_task, batch_size=128) if dataset_name in LARGE_DATASET_NAMES: # OC-SVM is quadratic on the number of examples, so subsample training set subsample_inds = np.random.choice(len(x_train_task_rep), 2500, replace=False) x_train_task_rep_temp = x_train_task_rep[subsample_inds] x_test_rep = enc.predict(x_test, batch_size=128) pg = ParameterGrid({ 'nu': np.linspace(0.1, 0.9, num=9), 'gamma': np.logspace(-7, 2, num=10, base=2) }) results = Parallel(n_jobs=PARALLEL_N_JOBS)(delayed( _train_ocsvm_and_score)(d, x_train_task_rep_temp, y_test.flatten() == single_class_ind, x_test_rep) for d in pg) best_params, best_auc_score = max(zip(pg, results), key=lambda t: t[-1]) print(best_params) best_ocsvm = OneClassSVM(**best_params).fit(x_train_task_rep) scores = best_ocsvm.decision_function(x_test_rep) labels = y_test.flatten() == single_class_ind res_file_name = '{}_cae-oc-svm_{}_{}.npz'.format( dataset_name, get_class_name_from_index(single_class_ind, dataset_name), datetime.datetime.now().strftime('%Y-%m-%d-%H%M')) res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name) save_roc_pr_curve_data(scores, labels, res_file_path)
def _adgan_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q): # gpu_to_use = gpu_q.get() # os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use (x_train, y_train), (x_test, y_test) = dataset_load_fn() if len(x_test) > 5000: # subsample x_test due to runtime complexity chosen_inds = np.random.choice(len(x_test), 5000, replace=False) x_test = x_test[chosen_inds] y_test = y_test[chosen_inds] n_channels = x_train.shape[get_channels_axis()] input_side = x_train.shape[2] # image side will always be at shape[2] critic = conv_encoder(input_side, n_channels, representation_dim=1, representation_activation='linear') noise_size = 256 generator = conv_decoder(input_side, n_channels=n_channels, representation_dim=noise_size) def prior_gen(b_size): return np.random.normal(size=(b_size, noise_size)) batch_size = 128 epochs = 100 x_train_task = x_train[y_train.flatten() == single_class_ind] def data_gen(b_size): chosen_inds = np.random.choice(len(x_train_task), b_size, replace=False) return x_train_task[chosen_inds] adgan.train_wgan_with_grad_penalty(prior_gen, generator, data_gen, critic, batch_size, epochs, grad_pen_coef=20) scores = adgan.scores_from_adgan_generator(x_test, prior_gen, generator) labels = y_test.flatten() == single_class_ind res_file_name = '{}_adgan_{}_{}.npz'.format( dataset_name, get_class_name_from_index(single_class_ind, dataset_name), datetime.datetime.now().strftime('%Y-%m-%d-%H%M')) res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name) save_roc_pr_curve_data(scores, labels, res_file_path)
def _dagmm_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q): gpu_to_use = gpu_q.get() os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use (x_train, y_train), (x_test, y_test) = dataset_load_fn() n_channels = x_train.shape[get_channels_axis()] input_side = x_train.shape[2] # image side will always be at shape[2] enc = conv_encoder(input_side, n_channels, representation_dim=5, representation_activation='linear') dec = conv_decoder(input_side, n_channels=n_channels, representation_dim=enc.output_shape[-1]) n_components = 3 estimation = Sequential([Dense(64, activation='tanh', input_dim=enc.output_shape[-1] + 2), Dropout(0.5), Dense(10, activation='tanh'), Dropout(0.5), Dense(n_components, activation='softmax')] ) batch_size = 256 epochs = 200 lambda_diag = 0.0005 lambda_energy = 0.01 dagmm_mdl = dagmm.create_dagmm_model(enc, dec, estimation, lambda_diag) dagmm_mdl.compile('adam', ['mse', lambda y_true, y_pred: lambda_energy*y_pred]) x_train_task = x_train[y_train.flatten() == single_class_ind] x_test_task = x_test[y_test.flatten() == single_class_ind] # This is just for visual monitoring dagmm_mdl.fit(x=x_train_task, y=[x_train_task, np.zeros((len(x_train_task), 1))], # second y is dummy batch_size=batch_size, epochs=epochs, validation_data=(x_test_task, [x_test_task, np.zeros((len(x_test_task), 1))]), # verbose=0 ) energy_mdl = Model(dagmm_mdl.input, dagmm_mdl.output[-1]) scores = -energy_mdl.predict(x_test, batch_size) scores = scores.flatten() if not np.all(np.isfinite(scores)): min_finite = np.min(scores[np.isfinite(scores)]) scores[~np.isfinite(scores)] = min_finite - 1 labels = y_test.flatten() == single_class_ind res_file_name = '{}_dagmm_{}_{}.npz'.format(dataset_name, get_class_name_from_index(single_class_ind, dataset_name), datetime.now().strftime('%Y-%m-%d-%H%M')) res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name) save_roc_pr_curve_data(scores, labels, res_file_path) gpu_q.put(gpu_to_use)
def _mo_gaal_experiment(dataset_load_fn, dataset_name, single_class_ind): (x_train, y_train), (x_test, y_test) = dataset_load_fn() x_train = x_train.reshape((len(x_train), -1)) x_test = x_test.reshape((len(x_test), -1)) x_train_task = x_train[y_train.flatten() == single_class_ind] best_mo_gaal = MO_GAAL().fit(x_train_task) scores = best_mo_gaal.decision_function(x_test) labels = y_test.flatten() == single_class_ind res_file_name = '{}_mo-gaal_{}_{}.npz'.format( dataset_name, get_class_name_from_index(single_class_ind, dataset_name), datetime.datetime.now().strftime('%Y-%m-%d-%H%M')) res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name) save_roc_pr_curve_data(scores, labels, res_file_path)
def _if_experiment(dataset_load_fn, dataset_name, single_class_ind): (x_train, y_train), (x_test, y_test) = dataset_load_fn() x_train = x_train.reshape((len(x_train), -1)) x_test = x_test.reshape((len(x_test), -1)) x_train_task = x_train[y_train.flatten() == single_class_ind] # if dataset_name in LARGE_DATASET_NAMES: # OC-SVM is quadratic on the number of examples, so subsample training set # subsample_inds = np.random.choice(len(x_train_task), 2500, replace=False) # x_train_task_tmp = x_train_task[subsample_inds] # ToDO: make gridsearch just one pg = ParameterGrid({ 'n_estimators': np.linspace(100, 800, num=8).astype(int), 'contamination': [0.1, 0.2, 0.3, 0.4, 0.5], 'behaviour': ['new'], 'n_jobs': [-1] }) # results = Parallel(n_jobs=PARALLEL_N_JOBS)( # delayed(_train_if_and_score)(d, x_train_task, y_test.flatten() == single_class_ind, x_test) # for d in pg) results = [] for d in pg: results.append( _train_if_and_score(d, x_train_task, y_test.flatten() == single_class_ind, x_test)) # print(list(zip(pg, results))) best_params, best_auc_score = max(zip(pg, results), key=lambda t: t[-1]) # print(best_params, ' ', best_auc_score) best_if = IsolationForest(**best_params).fit(x_train_task) scores = best_if.decision_function(x_test) labels = y_test.flatten() == single_class_ind res_file_name = '{}_if_{}_{}.npz'.format( dataset_name, get_class_name_from_index(single_class_ind, dataset_name), datetime.datetime.now().strftime('%Y-%m-%d-%H%M')) res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name) save_roc_pr_curve_data(scores, labels, res_file_path)
def test_model_loading(transformer, mdl, loader, dataset_name='hits-4-c', single_class_ind=1, tf_version='tf1', transformer_name='transformed', model_name='resnet', epochs=None): results_dir = os.path.join(PROJECT_PATH, 'tests', 'aux_results') save_dir = os.path.join(PROJECT_PATH, 'tests', 'aux_data') utils.check_path(results_dir) utils.check_path(save_dir) utils.check_path(os.path.join(results_dir, dataset_name)) # load-save data normal_data_path = os.path.join( save_dir, 'normal_data_%s_%s_loading.pkl' % (dataset_name, tf_version)) if os.path.exists(normal_data_path): (x_train, y_train), (x_val, y_val), (x_test, y_test) = pd.read_pickle(normal_data_path) else: (x_train, y_train), (x_val, y_val), (x_test, y_test) = loader(return_val=True) normal_data = (x_train, y_train), (x_val, y_val), (x_test, y_test) utils.save_pickle(normal_data, normal_data_path) # create model # n, k = (10, 4) # mdl = create_wide_residual_network( # x_train.shape[1:], transformer.n_transforms, n, k) mdl.compile('adam', 'categorical_crossentropy', ['acc']) # selec inliers x_train = x_train[y_train.flatten() == single_class_ind] x_val = x_val[y_val.flatten() == single_class_ind] # load-save transformed data transformed_data_path = os.path.join( save_dir, '%s_data_%s_%s_loading.pkl' % (transformer_name, dataset_name, tf_version)) if os.path.exists(transformed_data_path): (x_train_transform_tf1, y_train_transform_tf1), (x_val_transform_tf1, y_val_transform_tf1), ( x_test_transform_tf1, y_test_transform_tf1) = pd.read_pickle(transformed_data_path) else: # transform all data y_train_transform_tf1 = np.tile(np.arange(transformer.n_transforms), len(x_train)) x_train_transform_tf1 = transformer.transform_batch( np.repeat(x_train, transformer.n_transforms, axis=0), y_train_transform_tf1) y_val_transform_tf1 = np.tile(np.arange(transformer.n_transforms), len(x_val)) x_val_transform_tf1 = transformer.transform_batch( np.repeat(x_val, transformer.n_transforms, axis=0), y_val_transform_tf1) y_test_transform_tf1 = np.tile(np.arange(transformer.n_transforms), len(x_test)) x_test_transform_tf1 = transformer.transform_batch( np.repeat(x_test, transformer.n_transforms, axis=0), y_test_transform_tf1) transformed_data = ((x_train_transform_tf1, y_train_transform_tf1), (x_val_transform_tf1, y_val_transform_tf1), (x_test_transform_tf1, y_test_transform_tf1)) utils.save_pickle(transformed_data, transformed_data_path) print(x_train.shape) print(x_train_transform_tf1.shape) print(x_test.shape) print(x_test_transform_tf1.shape) # train model batch_size = 128 if epochs is None: epochs = int(np.ceil(200 / transformer.n_transforms)) mdl.fit(x=x_train_transform_tf1, y=to_categorical(y_train_transform_tf1), batch_size=batch_size, epochs=epochs) scores = np.zeros((len(x_test), )) matrix_evals = np.zeros( (len(x_test), transformer.n_transforms, transformer.n_transforms)) x_pred_train = mdl.predict(x_train_transform_tf1, batch_size=1024) x_pred_test = mdl.predict(x_test_transform_tf1, batch_size=1024) print(x_pred_train.shape) print(x_pred_test.shape) for t_ind in range(transformer.n_transforms): ind_x_pred_equal_to_t_ind = np.where(y_train_transform_tf1 == t_ind)[0] observed_dirichlet = x_pred_train[ind_x_pred_equal_to_t_ind] log_p_hat_train = np.log(observed_dirichlet).mean(axis=0) alpha_sum_approx = calc_approx_alpha_sum(observed_dirichlet) alpha_0 = observed_dirichlet.mean(axis=0) * alpha_sum_approx mle_alpha_t = fixed_point_dirichlet_mle(alpha_0, log_p_hat_train) ind_x_pred_test_equal_to_t_ind = np.where( y_test_transform_tf1 == t_ind)[0] x_test_p = x_pred_test[ind_x_pred_test_equal_to_t_ind] matrix_evals[:, :, t_ind] += x_test_p scores += dirichlet_normality_score(mle_alpha_t, x_test_p) scores /= transformer.n_transforms matrix_evals /= transformer.n_transforms scores_simple = np.trace(matrix_evals, axis1=1, axis2=2) scores_entropy = -1 * get_entropy(matrix_evals) scores_xH = -1 * get_xH(transformer, matrix_evals) labels = y_test.flatten() == single_class_ind save_results_file(results_dir, dataset_name, single_class_ind, scores=scores, labels=labels, experiment_name='%s-%s-loading-%s' % (model_name, transformer_name, tf_version)) save_results_file(results_dir, dataset_name, single_class_ind, scores=scores_simple, labels=labels, experiment_name='%s-%s-simple-loading-%s' % (model_name, transformer_name, tf_version)) save_results_file(results_dir, dataset_name, single_class_ind, scores=scores_entropy, labels=labels, experiment_name='%s-%s-entropy-loading-%s' % (model_name, transformer_name, tf_version)) save_results_file(results_dir, dataset_name, single_class_ind, scores=scores_xH, labels=labels, experiment_name='%s-%s-xH-loading-%s' % (model_name, transformer_name, tf_version)) mdl_weights_name = '{}_{}_{}_{}_loading_{}_weights.h5'.format( model_name, transformer_name, dataset_name, tf_version, get_class_name_from_index(single_class_ind, dataset_name)) mdl_weights_path = os.path.join(results_dir, dataset_name, mdl_weights_name) mdl.save_weights(mdl_weights_path) reset_weights() """ Time test_model_original(transformer, load_hits4c, dataset_name='hits-4-c', tf_version='tf1') 00:04:31.65 (0.992217, 0.9895665, 0.99131725, 0.989478125) (0.99240075, 0.9900822499999999, 0.99215325, 0.9901300000000001) """ return get_roc_auc(scores, labels), get_roc_auc(scores_simple, labels), \ get_roc_auc(scores_entropy, labels), get_roc_auc(scores_xH, labels)
def test_model_original(transformer, loader, dataset_name='hits-4-c', single_class_ind=1): results_dir = os.path.join(PROJECT_PATH, 'tests', 'aux_results') save_dir = os.path.join(PROJECT_PATH, 'tests', 'aux_data') utils.check_path(results_dir) utils.check_path(save_dir) utils.check_path(os.path.join(results_dir, dataset_name)) # load-save data (x_train, y_train), (x_val, y_val), (x_test, y_test) = loader(return_val=True) normal_data = (x_train, y_train), (x_val, y_val), (x_test, y_test) utils.save_pickle( normal_data, os.path.join(save_dir, 'normal_data_%s_tf1_original.pkl' % dataset_name)) # create model n, k = (10, 4) mdl = create_wide_residual_network(x_train.shape[1:], transformer.n_transforms, n, k) mdl.compile('adam', 'categorical_crossentropy', ['acc']) # get inliers of specific class # get inliers x_train_task = x_train[y_train.flatten() == single_class_ind] print(x_train_task.shape) # transform inliers transformations_inds = np.tile(np.arange(transformer.n_transforms), len(x_train_task)) x_train_task_transformed = transformer.transform_batch( np.repeat(x_train_task, transformer.n_transforms, axis=0), transformations_inds) print(x_train_task_transformed.shape) # train model batch_size = 128 mdl.fit(x=x_train_task_transformed, y=to_categorical(transformations_inds), batch_size=batch_size, epochs=int(np.ceil(200 / transformer.n_transforms))) scores = np.zeros((len(x_test), )) matrix_evals = np.zeros( (len(x_test), transformer.n_transforms, transformer.n_transforms)) observed_data = x_train_task for t_ind in range(transformer.n_transforms): observed_dirichlet = mdl.predict(transformer.transform_batch( observed_data, [t_ind] * len(observed_data)), batch_size=1024) log_p_hat_train = np.log(observed_dirichlet).mean(axis=0) alpha_sum_approx = calc_approx_alpha_sum(observed_dirichlet) alpha_0 = observed_dirichlet.mean(axis=0) * alpha_sum_approx mle_alpha_t = fixed_point_dirichlet_mle(alpha_0, log_p_hat_train) x_test_p = mdl.predict(transformer.transform_batch( x_test, [t_ind] * len(x_test)), batch_size=1024) matrix_evals[:, :, t_ind] += x_test_p scores += dirichlet_normality_score(mle_alpha_t, x_test_p) scores /= transformer.n_transforms matrix_evals /= transformer.n_transforms scores_simple = np.trace(matrix_evals, axis1=1, axis2=2) scores_entropy = -1 * get_entropy(matrix_evals) scores_xH = -1 * get_xH(transformer, matrix_evals) labels = y_test.flatten() == single_class_ind save_results_file(results_dir, dataset_name, single_class_ind, scores=scores, labels=labels, experiment_name='transformations') save_results_file(results_dir, dataset_name, single_class_ind, scores=scores_simple, labels=labels, experiment_name='transformations-simple') save_results_file(results_dir, dataset_name, single_class_ind, scores=scores_entropy, labels=labels, experiment_name='transformations-entropy') save_results_file(results_dir, dataset_name, single_class_ind, scores=scores_xH, labels=labels, experiment_name='transformations-xH') mdl_weights_name = '{}_tf1_original_{}_weights.h5'.format( dataset_name, get_class_name_from_index(single_class_ind, dataset_name)) mdl_weights_path = os.path.join(results_dir, dataset_name, mdl_weights_name) mdl.save_weights(mdl_weights_path) """ Time test_model_original(transformer, load_hits4c, dataset_name='hits-4-c') 00:06:58.37 (0.9917134999999999, 0.9350055, 0.9872614999999999, 0.94142025) (0.9938067500000001, 0.9923547500000001, 0.9931685, 0.992637375) (0.9912172499999999, 0.9883357499999998, 0.9909070000000001, 0.9886706249999999) #train only Time test_model_original(transformer, load_hits4c, dataset_name='hits-4-c', tf_version='tf1') 00:03:48.29 """ return get_roc_auc(scores, labels), get_roc_auc(scores_simple, labels), \ get_roc_auc(scores_entropy, labels), get_roc_auc(scores_xH, labels)
def _kernal_plus_transformations_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q): # gpu_to_use = gpu_q.get() # os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use (x_train, y_train), (x_test, y_test) = dataset_load_fn() if dataset_name in ['cats-vs-dogs']: transformer = None else: transformer = PlusKernelTransformer(translation_x=8, translation_y=8, rotations=1, flips=1, gauss=1, log=1) n, k = (10, 4) mdl = create_wide_residual_network(x_train.shape[1:], transformer.n_transforms, n, k) mdl.compile('adam', 'categorical_crossentropy', ['acc']) # get inliers of specific class x_train_task = x_train[y_train.flatten() == single_class_ind] # [0_i, ..., (N_transforms-1)_i, ..., ..., 0_N_samples, ..., # (N_transforms-1)_N_samples] shape: (N_transforms*N_samples,) transformations_inds = np.tile(np.arange(transformer.n_transforms), len(x_train_task)) x_train_task_transformed = transformer.transform_batch( np.repeat(x_train_task, transformer.n_transforms, axis=0), transformations_inds) batch_size = 128 mdl.fit( x=x_train_task_transformed, y=to_categorical(transformations_inds), batch_size=batch_size, epochs=2 #int(np.ceil(200/transformer.n_transforms)) ) scores = np.zeros((len(x_test), )) matrix_evals = np.zeros( (len(x_test), transformer.n_transforms, transformer.n_transforms)) observed_data = x_train_task for t_ind in range(transformer.n_transforms): observed_dirichlet = mdl.predict(transformer.transform_batch( observed_data, [t_ind] * len(observed_data)), batch_size=1024) log_p_hat_train = np.log(observed_dirichlet).mean(axis=0) alpha_sum_approx = calc_approx_alpha_sum(observed_dirichlet) alpha_0 = observed_dirichlet.mean(axis=0) * alpha_sum_approx mle_alpha_t = fixed_point_dirichlet_mle(alpha_0, log_p_hat_train) x_test_p = mdl.predict(transformer.transform_batch( x_test, [t_ind] * len(x_test)), batch_size=1024) matrix_evals[:, :, t_ind] += x_test_p scores += dirichlet_normality_score(mle_alpha_t, x_test_p) scores /= transformer.n_transforms matrix_evals /= transformer.n_transforms scores_simple = np.trace(matrix_evals, axis1=1, axis2=2) scores_entropy = -1 * get_entropy(matrix_evals) scores_xH = -1 * get_xH(transformer, matrix_evals) labels = y_test.flatten() == single_class_ind save_results_file(dataset_name, single_class_ind, scores=scores, labels=labels, experiment_name='kernel-plus-transformations') save_results_file(dataset_name, single_class_ind, scores=scores_simple, labels=labels, experiment_name='kernel-plus-transformations-simple') save_results_file(dataset_name, single_class_ind, scores=scores_entropy, labels=labels, experiment_name='kernel-plus-transformations-entropy') save_results_file(dataset_name, single_class_ind, scores=scores_xH, labels=labels, experiment_name='kernel-plus-transformations-xH') mdl_weights_name = '{}_kernel-plus-transformations_{}_{}_weights.h5'.format( dataset_name, get_class_name_from_index(single_class_ind, dataset_name), datetime.datetime.now().strftime('%Y-%m-%d-%H%M')) mdl_weights_path = os.path.join(RESULTS_DIR, dataset_name, mdl_weights_name) mdl.save_weights(mdl_weights_path)
def _transformations_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q): gpu_to_use = gpu_q.get() os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use (x_train, y_train), (x_test, y_test) = dataset_load_fn() if dataset_name in ['cats-vs-dogs']: transformer = Transformer(16, 16) n, k = (16, 8) else: transformer = Transformer(8, 8) n, k = (10, 4) mdl = create_wide_residual_network(x_train.shape[1:], transformer.n_transforms, n, k) mdl.compile('adam', 'categorical_crossentropy', ['acc']) # get inliers of specific class x_train_task = x_train[y_train.flatten() == single_class_ind] # [0_i, ..., (N_transforms-1)_i, ..., ..., 0_N_samples, ..., # (N_transforms-1)_N_samples] shape: (N_transforms*N_samples,) transformations_inds = np.tile(np.arange(transformer.n_transforms), len(x_train_task)) x_train_task_transformed = transformer.transform_batch(np.repeat(x_train_task, transformer.n_transforms, axis=0), transformations_inds) batch_size = 128 mdl.fit(x=x_train_task_transformed, y=to_categorical(transformations_inds), batch_size=batch_size, epochs=int(np.ceil(200/transformer.n_transforms)) ) ################################################################################################# # simplified normality score ################################################################################################# # preds = np.zeros((len(x_test), transformer.n_transforms)) # for t in range(transformer.n_transforms): # preds[:, t] = mdl.predict(transformer.transform_batch(x_test, [t] * len(x_test)), # batch_size=batch_size)[:, t] # # labels = y_test.flatten() == single_class_ind # scores = preds.mean(axis=-1) ################################################################################################# def calc_approx_alpha_sum(observations): N = len(observations) f = np.mean(observations, axis=0) return (N * (len(f) - 1) * (-psi(1))) / ( N * np.sum(f * np.log(f)) - np.sum(f * np.sum(np.log(observations), axis=0))) def inv_psi(y, iters=5): # initial estimate cond = y >= -2.22 x = cond * (np.exp(y) + 0.5) + (1 - cond) * -1 / (y - psi(1)) for _ in range(iters): x = x - (psi(x) - y) / polygamma(1, x) return x def fixed_point_dirichlet_mle(alpha_init, log_p_hat, max_iter=1000): alpha_new = alpha_old = alpha_init for _ in range(max_iter): alpha_new = inv_psi(psi(np.sum(alpha_old)) + log_p_hat) if np.sqrt(np.sum((alpha_old - alpha_new) ** 2)) < 1e-9: break alpha_old = alpha_new return alpha_new def dirichlet_normality_score(alpha, p): return np.sum((alpha - 1) * np.log(p), axis=-1) scores = np.zeros((len(x_test),)) observed_data = x_train_task for t_ind in range(transformer.n_transforms): observed_dirichlet = mdl.predict(transformer.transform_batch(observed_data, [t_ind] * len(observed_data)), batch_size=1024) log_p_hat_train = np.log(observed_dirichlet).mean(axis=0) alpha_sum_approx = calc_approx_alpha_sum(observed_dirichlet) alpha_0 = observed_dirichlet.mean(axis=0) * alpha_sum_approx mle_alpha_t = fixed_point_dirichlet_mle(alpha_0, log_p_hat_train) x_test_p = mdl.predict(transformer.transform_batch(x_test, [t_ind] * len(x_test)), batch_size=1024) scores += dirichlet_normality_score(mle_alpha_t, x_test_p) scores /= transformer.n_transforms labels = y_test.flatten() == single_class_ind res_file_name = '{}_transformations_{}_{}.npz'.format(dataset_name, get_class_name_from_index(single_class_ind, dataset_name), datetime.now().strftime('%Y-%m-%d-%H%M')) res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name) save_roc_pr_curve_data(scores, labels, res_file_path) mdl_weights_name = '{}_transformations_{}_{}_weights.h5'.format(dataset_name, get_class_name_from_index(single_class_ind, dataset_name), datetime.now().strftime('%Y-%m-%d-%H%M')) mdl_weights_path = os.path.join(RESULTS_DIR, dataset_name, mdl_weights_name) mdl.save_weights(mdl_weights_path) gpu_q.put(gpu_to_use)