def conv_decoder(output_side=32, n_channels=3, representation_dim=256, activation='relu'): nf = 64 rep_in = Input(shape=(representation_dim,)) g = Dense(nf * 4 * 4 * 4)(rep_in) g = BatchNormalization(axis=-1)(g) g = Activation(activation)(g) conv_shape = (nf * 4, 4, 4) if get_channels_axis() == 1 else (4, 4, nf * 4) g = Reshape(conv_shape)(g) # upsample x2 g = Conv2DTranspose(nf * 2, kernel_size=(3, 3), strides=(2, 2), padding='same')(g) g = BatchNormalization(axis=get_channels_axis())(g) g = Activation(activation)(g) # upsample x2 g = Conv2DTranspose(nf, kernel_size=(3, 3), strides=(2, 2), padding='same')(g) g = BatchNormalization(axis=get_channels_axis())(g) g = Activation(activation)(g) if output_side == 64: # upsample x2 g = Conv2DTranspose(nf, kernel_size=(3, 3), strides=(2, 2), padding='same')(g) g = BatchNormalization(axis=get_channels_axis())(g) g = Activation(activation)(g) # upsample x2 g = Conv2DTranspose(n_channels, kernel_size=(3, 3), strides=(2, 2), padding='same')(g) g = Activation('tanh')(g) return Model(rep_in, g)
def conv_encoder(input_side=32, n_channels=3, representation_dim=256, representation_activation='tanh', intermediate_activation='relu'): nf = 64 input_shape = (n_channels, input_side, input_side) if get_channels_axis() == 1 else (input_side, input_side, n_channels) x_in = Input(shape=input_shape) enc = x_in # downsample x0.5 enc = Conv2D(nf, kernel_size=(3, 3), strides=(2, 2), padding='same')(enc) enc = BatchNormalization(axis=get_channels_axis())(enc) enc = Activation(intermediate_activation)(enc) # downsample x0.5 enc = Conv2D(nf * 2, kernel_size=(3, 3), strides=(2, 2), padding='same')(enc) enc = BatchNormalization(axis=get_channels_axis())(enc) enc = Activation(intermediate_activation)(enc) # downsample x0.5 enc = Conv2D(nf * 4, kernel_size=(3, 3), strides=(2, 2), padding='same')(enc) enc = BatchNormalization(axis=get_channels_axis())(enc) enc = Activation(intermediate_activation)(enc) if input_side == 64: # downsample x0.5 enc = Conv2D(nf * 8, kernel_size=(3, 3), strides=(2, 2), padding='same')(enc) enc = BatchNormalization(axis=get_channels_axis())(enc) enc = Activation(intermediate_activation)(enc) enc = Flatten()(enc) rep = Dense(representation_dim, activation=representation_activation)(enc) return Model(x_in, rep)
def _dsebm_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q): gpu_to_use = gpu_q.get() os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use (x_train, y_train), (x_test, y_test) = dataset_load_fn() n_channels = x_train.shape[get_channels_axis()] input_side = x_train.shape[2] # image side will always be at shape[2] encoder_mdl = conv_encoder(input_side, n_channels, representation_activation='relu') energy_mdl = dsebm.create_energy_model(encoder_mdl) reconstruction_mdl = dsebm.create_reconstruction_model(energy_mdl) # optimization parameters batch_size = 128 epochs = 200 reconstruction_mdl.compile('adam', 'mse') x_train_task = x_train[y_train.flatten() == single_class_ind] x_test_task = x_test[y_test.flatten() == single_class_ind] # This is just for visual monitoring reconstruction_mdl.fit(x=x_train_task, y=x_train_task, batch_size=batch_size, epochs=epochs, validation_data=(x_test_task, x_test_task)) scores = -energy_mdl.predict(x_test, batch_size) labels = y_test.flatten() == single_class_ind res_file_name = '{}_dsebm_{}_{}.npz'.format(dataset_name, get_class_name_from_index(single_class_ind, dataset_name), datetime.now().strftime('%Y-%m-%d-%H%M')) res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name) save_roc_pr_curve_data(scores, labels, res_file_path) gpu_q.put(gpu_to_use)
def _RDAE_experiment(x_train, y_train, dataset_name, single_class_ind, gpu_q, p): gpu_to_use = gpu_q.get() cudnn.benchmark = True n_channels = x_train.shape[get_channels_axis()] model = CAE_pytorch(in_channels=n_channels) model = model.cuda() optimizer = optim.Adam(model.parameters(), eps=1e-7, weight_decay=0.0005) criterion = nn.MSELoss() epochs = 20 inner_epochs = 1 lmbda = 0.00065 # train RDAE losses = train_robust_cae(x_train, model, criterion, optimizer, lmbda, inner_epochs, epochs // inner_epochs, False) losses = losses - losses.min() losses = losses / (1e-8 + losses.max()) scores = 1 - losses res_file_name = '{}_rdae-{}_{}_{}.npz'.format( dataset_name, p, get_class_name_from_index(single_class_ind, dataset_name), datetime.now().strftime('%Y-%m-%d-%H%M')) res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name) os.makedirs(os.path.join(RESULTS_DIR, dataset_name), exist_ok=True) save_roc_pr_curve_data(scores, y_train, res_file_path) gpu_q.put(gpu_to_use)
def _DRAE_experiment(x_train, y_train, dataset_name, single_class_ind, gpu_q, p): gpu_to_use = gpu_q.get() n_channels = x_train.shape[get_channels_axis()] model = CAE_pytorch(in_channels=n_channels) batch_size = 128 model = model.cuda() trainset = trainset_pytorch(train_data=x_train, train_labels=y_train, transform=transform_train) trainloader = data.DataLoader(trainset, batch_size=batch_size, shuffle=True) cudnn.benchmark = True criterion = DRAELossAutograd(lamb=0.1) optimizer = optim.Adam(model.parameters(), eps=1e-7, weight_decay=0.0005) epochs = 250 # #########################Training######################## train_cae(trainloader, model, criterion, optimizer, epochs) # #######################Testin############################ testloader = data.DataLoader(trainset, batch_size=1024, shuffle=False) losses, reps = test_cae_pytorch(testloader, model) losses = losses - losses.min() losses = losses / (1e-8+losses.max()) scores = 1 - losses res_file_name = '{}_drae-{}_{}_{}.npz'.format(dataset_name, p, get_class_name_from_index(single_class_ind, dataset_name), datetime.now().strftime('%Y-%m-%d-%H%M')) res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name) os.makedirs(os.path.join(RESULTS_DIR, dataset_name), exist_ok=True) save_roc_pr_curve_data(scores, y_train, res_file_path) gpu_q.put(gpu_to_use)
def _cae_ocsvm_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q): gpu_to_use = gpu_q.get() os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use (x_train, y_train), (x_test, y_test) = dataset_load_fn() n_channels = x_train.shape[get_channels_axis()] input_side = x_train.shape[2] # channel side will always be at shape[2] enc = conv_encoder(input_side, n_channels) dec = conv_decoder(input_side, n_channels) x_in = Input(shape=x_train.shape[1:]) x_rec = dec(enc(x_in)) cae = Model(x_in, x_rec) cae.compile('adam', 'mse') x_train_task = x_train[y_train.flatten() == single_class_ind] x_test_task = x_test[y_test.flatten( ) == single_class_ind] # This is just for visual monitoring cae.fit(x=x_train_task, y=x_train_task, batch_size=128, epochs=200, validation_data=(x_test_task, x_test_task)) x_train_task_rep = enc.predict(x_train_task, batch_size=128) if dataset_name in [ 'cats-vs-dogs' ]: # OC-SVM is quadratic on the number of examples, so subsample training set subsample_inds = np.random.choice(len(x_train_task_rep), 2500, replace=False) x_train_task_rep = x_train_task_rep[subsample_inds] x_test_rep = enc.predict(x_test, batch_size=128) pg = ParameterGrid({ 'nu': np.linspace(0.1, 0.9, num=9), 'gamma': np.logspace(-7, 2, num=10, base=2) }) results = Parallel(n_jobs=6)(delayed(_train_ocsvm_and_score)( d, x_train_task_rep, y_test.flatten() == single_class_ind, x_test_rep) for d in pg) best_params, best_auc_score = max(zip(pg, results), key=lambda t: t[-1]) print(best_params) best_ocsvm = OneClassSVM(**best_params).fit(x_train_task_rep) scores = best_ocsvm.decision_function(x_test_rep) labels = y_test.flatten() == single_class_ind res_file_name = '{}_cae-oc-svm_{}_{}.npz'.format( dataset_name, get_class_name_from_index(single_class_ind, dataset_name), datetime.now().strftime('%Y-%m-%d-%H%M')) res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name) save_roc_pr_curve_data(scores, labels, res_file_path) gpu_q.put(gpu_to_use)
def conv_decoder(output_side=32, n_channels=3, representation_dim=256, activation='relu'): nf = 64 Slicer = Lambda(lambda x, slice_size: x[:, :slice_size, :slice_size, :], output_shape=(output_side, output_side, n_channels), name="slice_layer") # Define your lambda layer Slicer.arguments = {'slice_size': output_side} # Update extra arguments to F rep_in = Input(shape=(representation_dim,)) if output_side == 21: height, width, feat_mult = (3, 3, 4) else: height, width, feat_mult = (4, 4, 4) g = Dense(nf * feat_mult * height * width)(rep_in) g = BatchNormalization(axis=-1)(g) g = Activation(activation)(g) conv_shape = (nf * feat_mult, height, width) if get_channels_axis() == 1 else (height, width, nf * feat_mult) g = Reshape(conv_shape)(g) # upsample x2 g = Conv2DTranspose(nf * 2, kernel_size=(3, 3), strides=(2, 2), padding='same')(g) g = BatchNormalization(axis=get_channels_axis())(g) g = Activation(activation)(g) # upsample x2 g = Conv2DTranspose(nf, kernel_size=(3, 3), strides=(2, 2), padding='same')(g) g = BatchNormalization(axis=get_channels_axis())(g) g = Activation(activation)(g) if output_side == 64: # upsample x2 g = Conv2DTranspose(nf, kernel_size=(3, 3), strides=(2, 2), padding='same')(g) g = BatchNormalization(axis=get_channels_axis())(g) g = Activation(activation)(g) # upsample x2 g = Conv2DTranspose(n_channels, kernel_size=(3, 3), strides=(2, 2), padding='same')(g) g = Activation('tanh')(g) g = Slicer(g) return Model(rep_in, g)
def _adgan_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q): gpu_to_use = gpu_q.get() os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use (x_train, y_train), (x_test, y_test) = dataset_load_fn() if len(x_test) > 5000: # subsample x_test due to runtime complexity chosen_inds = np.random.choice(len(x_test), 5000, replace=False) x_test = x_test[chosen_inds] y_test = y_test[chosen_inds] n_channels = x_train.shape[get_channels_axis()] input_side = x_train.shape[2] # image side will always be at shape[2] critic = conv_encoder(input_side, n_channels, representation_dim=1, representation_activation='linear') noise_size = 256 generator = conv_decoder(input_side, n_channels=n_channels, representation_dim=noise_size) def prior_gen(b_size): return np.random.normal(size=(b_size, noise_size)) batch_size = 128 epochs = 100 x_train_task = x_train[y_train.flatten() == single_class_ind] def data_gen(b_size): chosen_inds = np.random.choice(len(x_train_task), b_size, replace=False) return x_train_task[chosen_inds] adgan.train_wgan_with_grad_penalty(prior_gen, generator, data_gen, critic, batch_size, epochs, grad_pen_coef=20) scores = adgan.scores_from_adgan_generator(x_test, prior_gen, generator) labels = y_test.flatten() == single_class_ind res_file_name = '{}_adgan_{}_{}.npz'.format( dataset_name, get_class_name_from_index(single_class_ind, dataset_name), datetime.now().strftime('%Y-%m-%d-%H%M')) res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name) save_roc_pr_curve_data(scores, labels, res_file_path) gpu_q.put(gpu_to_use)
def train_robust_cae(x_train, model, criterion, optimizer, lmbda, inner_epochs, outer_epochs, reinit=True): batch_size = 128 S = np.zeros_like(x_train) # reside on numpy as x_train def get_reconstruction(loader): model.eval() rc = [] for batch, _ in loader: with torch.no_grad(): rc.append(model(batch.cuda()).cpu().numpy()) out = np.concatenate(rc, axis=0) # NOTE: transform_train swaps the channel axis, swap back to yield the same shape out = out.transpose((0, 2, 3, 1)) return out for oe in range(outer_epochs): # update AE if reinit: # Since our CAE_pytorch does not implement reset_parameters, regenerate a new model if reinit. del model model = CAE_pytorch(in_channels=x_train.shape[get_channels_axis()]).cuda() model.train() trainset = trainset_pytorch(x_train-S, train_labels=np.ones((x_train.shape[0], )), transform=transform_train) trainloader = data.DataLoader(trainset, batch_size=batch_size, shuffle=True) for ie in range(inner_epochs): for batch_idx, (inputs, _) in enumerate(trainloader): inputs = inputs.cuda() outputs = model(inputs) loss = criterion(inputs, outputs) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() if (batch_idx + 1) % 10 == 0: print('Epoch: [{} | {} ({} | {})], batch: {}, loss: {}'.format( ie+1, inner_epochs, oe+1, outer_epochs, batch_idx+1, loss.item()) ) # update S via l21 proximal operator testloader = data.DataLoader(trainset, batch_size=1024, shuffle=False) recon = get_reconstruction(testloader) S = prox_l21(x_train - recon, lmbda) # get final reconstruction finalset = trainset_pytorch(x_train - S, train_labels=np.ones((x_train.shape[0],)), transform=transform_train) finalloader = data.DataLoader(finalset, batch_size=1024, shuffle=False) reconstruction = get_reconstruction(finalloader) losses = ((x_train-S-reconstruction) ** 2).sum(axis=(1, 2, 3), keepdims=False) return losses
def _dagmm_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q): gpu_to_use = gpu_q.get() os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use (x_train, y_train), (x_test, y_test) = dataset_load_fn() n_channels = x_train.shape[get_channels_axis()] input_side = x_train.shape[2] # image side will always be at shape[2] enc = conv_encoder(input_side, n_channels, representation_dim=5, representation_activation='linear') dec = conv_decoder(input_side, n_channels=n_channels, representation_dim=enc.output_shape[-1]) n_components = 3 estimation = Sequential([Dense(64, activation='tanh', input_dim=enc.output_shape[-1] + 2), Dropout(0.5), Dense(10, activation='tanh'), Dropout(0.5), Dense(n_components, activation='softmax')] ) batch_size = 256 epochs = 200 lambda_diag = 0.0005 lambda_energy = 0.01 dagmm_mdl = dagmm.create_dagmm_model(enc, dec, estimation, lambda_diag) dagmm_mdl.compile('adam', ['mse', lambda y_true, y_pred: lambda_energy*y_pred]) x_train_task = x_train[y_train.flatten() == single_class_ind] x_test_task = x_test[y_test.flatten() == single_class_ind] # This is just for visual monitoring dagmm_mdl.fit(x=x_train_task, y=[x_train_task, np.zeros((len(x_train_task), 1))], # second y is dummy batch_size=batch_size, epochs=epochs, validation_data=(x_test_task, [x_test_task, np.zeros((len(x_test_task), 1))]), # verbose=0 ) energy_mdl = Model(dagmm_mdl.input, dagmm_mdl.output[-1]) scores = -energy_mdl.predict(x_test, batch_size) scores = scores.flatten() if not np.all(np.isfinite(scores)): min_finite = np.min(scores[np.isfinite(scores)]) scores[~np.isfinite(scores)] = min_finite - 1 labels = y_test.flatten() == single_class_ind res_file_name = '{}_dagmm_{}_{}.npz'.format(dataset_name, get_class_name_from_index(single_class_ind, dataset_name), datetime.now().strftime('%Y-%m-%d-%H%M')) res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name) save_roc_pr_curve_data(scores, labels, res_file_path) gpu_q.put(gpu_to_use)
def _E3Outlier_experiment(x_train, y_train, dataset_name, single_class_ind, gpu_q, p): """Surrogate Supervision Discriminative Network training.""" gpu_to_use = gpu_q.get() n_channels = x_train.shape[get_channels_axis()] if OP_TYPE == 'RA': transformer = RA(8, 8) elif OP_TYPE == 'RA+IA': transformer = RA_IA(8, 8, 12) elif OP_TYPE == 'RA+IA+PR': transformer = RA_IA_PR(8, 8, 12, 23, 2) else: raise NotImplementedError print(transformer.n_transforms) if BACKEND == 'wrn': n, k = (10, 4) model = WideResNet(num_classes=transformer.n_transforms, depth=n, widen_factor=k, in_channel=n_channels) elif BACKEND == 'resnet20': n = 20 model = ResNet(num_classes=transformer.n_transforms, depth=n, in_channels=n_channels) elif BACKEND == 'resnet50': n = 50 model = ResNet(num_classes=transformer.n_transforms, depth=n, in_channels=n_channels) elif BACKEND == 'densenet22': n = 22 model = DenseNet(num_classes=transformer.n_transforms, depth=n, in_channels=n_channels) elif BACKEND == 'densenet40': n = 40 model = DenseNet(num_classes=transformer.n_transforms, depth=n, in_channels=n_channels) else: raise NotImplementedError('Unimplemented backend: {}'.format(BACKEND)) print('Using backend: {} ({})'.format(type(model).__name__, BACKEND)) x_train_task = x_train transformations_inds = np.tile(np.arange(transformer.n_transforms), len(x_train_task)) x_train_task_transformed = transformer.transform_batch( np.repeat(x_train_task, transformer.n_transforms, axis=0), transformations_inds) # parameters for training trainset = trainset_pytorch(train_data=x_train_task_transformed, train_labels=transformations_inds, transform=transform_train) batch_size = 128 trainloader = data.DataLoader(trainset, batch_size=batch_size, shuffle=True) cudnn.benchmark = True criterion = nn.CrossEntropyLoss() model = torch.nn.DataParallel(model).cuda() if dataset_name in ['mnist', 'fashion-mnist']: optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=0.0005) else: optimizer = optim.Adam(model.parameters(), eps=1e-7, weight_decay=0.0005) epochs = int(np.ceil(250 / transformer.n_transforms)) train_pytorch(trainloader, model, criterion, optimizer, epochs) # SSD-IF test_set = testset_pytorch(test_data=x_train_task, transform=transform_test) x_train_task_rep = get_features_pytorch(testloader=data.DataLoader( test_set, batch_size=batch_size, shuffle=False), model=model).numpy() clf = IsolationForest(contamination=p, n_jobs=4).fit(x_train_task_rep) if_scores = clf.decision_function(x_train_task_rep) res_file_name = '{}_ssd-iforest-{}_{}_{}.npz'.format( dataset_name, p, get_class_name_from_index(single_class_ind, dataset_name), datetime.now().strftime('%Y-%m-%d-%H%M')) res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name) os.makedirs(os.path.join(RESULTS_DIR, dataset_name), exist_ok=True) save_roc_pr_curve_data(if_scores, y_train, res_file_path) # E3Outlier if SCORE_MODE == 'pl_mean': preds = np.zeros((len(x_train_task), transformer.n_transforms)) original_preds = np.zeros((transformer.n_transforms, len(x_train_task), transformer.n_transforms)) for t in range(transformer.n_transforms): idx = np.squeeze( np.array([range(x_train_task.shape[0])]) * transformer.n_transforms + t) test_set = testset_pytorch( test_data=x_train_task_transformed[idx, :], transform=transform_test) original_preds[t, :, :] = softmax( test_pytorch(testloader=data.DataLoader(test_set, batch_size=batch_size, shuffle=False), model=model)) preds[:, t] = original_preds[t, :, :][:, t] scores = preds.mean(axis=-1) elif SCORE_MODE == 'max_mean': preds = np.zeros((len(x_train_task), transformer.n_transforms)) original_preds = np.zeros((transformer.n_transforms, len(x_train_task), transformer.n_transforms)) for t in range(transformer.n_transforms): idx = np.squeeze( np.array([range(x_train_task.shape[0])]) * transformer.n_transforms + t) test_set = testset_pytorch( test_data=x_train_task_transformed[idx, :], transform=transform_test) original_preds[t, :, :] = softmax( test_pytorch(testloader=data.DataLoader(test_set, batch_size=batch_size, shuffle=False), model=model)) preds[:, t] = np.max(original_preds[t, :, :], axis=1) scores = preds.mean(axis=-1) elif SCORE_MODE == 'neg_entropy': preds = np.zeros((len(x_train_task), transformer.n_transforms)) original_preds = np.zeros((transformer.n_transforms, len(x_train_task), transformer.n_transforms)) for t in range(transformer.n_transforms): idx = np.squeeze( np.array([range(x_train_task.shape[0])]) * transformer.n_transforms + t) test_set = testset_pytorch( test_data=x_train_task_transformed[idx, :], transform=transform_test) original_preds[t, :, :] = softmax( test_pytorch(testloader=data.DataLoader(test_set, batch_size=batch_size, shuffle=False), model=model)) for s in range(len(x_train_task)): preds[s, t] = neg_entropy(original_preds[t, s, :]) scores = preds.mean(axis=-1) else: raise NotImplementedError res_file_name = '{}_e3outlier-{}_{}_{}.npz'.format( dataset_name, p, get_class_name_from_index(single_class_ind, dataset_name), datetime.now().strftime('%Y-%m-%d-%H%M')) res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name) os.makedirs(os.path.join(RESULTS_DIR, dataset_name), exist_ok=True) save_roc_pr_curve_data(scores, y_train, res_file_path) gpu_q.put(gpu_to_use)