def conv_decoder(output_side=32, n_channels=3, representation_dim=256, activation='relu'):
nf = 64
rep_in = Input(shape=(representation_dim,))
g = Dense(nf * 4 * 4 * 4)(rep_in)
g = BatchNormalization(axis=-1)(g)
g = Activation(activation)(g)
conv_shape = (nf * 4, 4, 4) if get_channels_axis() == 1 else (4, 4, nf * 4)
g = Reshape(conv_shape)(g)
# upsample x2
g = Conv2DTranspose(nf * 2, kernel_size=(3, 3), strides=(2, 2), padding='same')(g)
g = BatchNormalization(axis=get_channels_axis())(g)
g = Activation(activation)(g)
# upsample x2
g = Conv2DTranspose(nf, kernel_size=(3, 3), strides=(2, 2), padding='same')(g)
g = BatchNormalization(axis=get_channels_axis())(g)
g = Activation(activation)(g)
if output_side == 64:
# upsample x2
g = Conv2DTranspose(nf, kernel_size=(3, 3), strides=(2, 2), padding='same')(g)
g = BatchNormalization(axis=get_channels_axis())(g)
g = Activation(activation)(g)
# upsample x2
g = Conv2DTranspose(n_channels, kernel_size=(3, 3), strides=(2, 2), padding='same')(g)
g = Activation('tanh')(g)
return Model(rep_in, g)
def conv_encoder(input_side=32, n_channels=3, representation_dim=256, representation_activation='tanh',
intermediate_activation='relu'):
nf = 64
input_shape = (n_channels, input_side, input_side) if get_channels_axis() == 1 else (input_side, input_side,
n_channels)
x_in = Input(shape=input_shape)
enc = x_in
# downsample x0.5
enc = Conv2D(nf, kernel_size=(3, 3), strides=(2, 2), padding='same')(enc)
enc = BatchNormalization(axis=get_channels_axis())(enc)
enc = Activation(intermediate_activation)(enc)
# downsample x0.5
enc = Conv2D(nf * 2, kernel_size=(3, 3), strides=(2, 2), padding='same')(enc)
enc = BatchNormalization(axis=get_channels_axis())(enc)
enc = Activation(intermediate_activation)(enc)
# downsample x0.5
enc = Conv2D(nf * 4, kernel_size=(3, 3), strides=(2, 2), padding='same')(enc)
enc = BatchNormalization(axis=get_channels_axis())(enc)
enc = Activation(intermediate_activation)(enc)
if input_side == 64:
# downsample x0.5
enc = Conv2D(nf * 8, kernel_size=(3, 3), strides=(2, 2), padding='same')(enc)
enc = BatchNormalization(axis=get_channels_axis())(enc)
enc = Activation(intermediate_activation)(enc)
enc = Flatten()(enc)
rep = Dense(representation_dim, activation=representation_activation)(enc)
return Model(x_in, rep)
def _dsebm_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q):
gpu_to_use = gpu_q.get()
os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use
(x_train, y_train), (x_test, y_test) = dataset_load_fn()
n_channels = x_train.shape[get_channels_axis()]
input_side = x_train.shape[2] # image side will always be at shape[2]
encoder_mdl = conv_encoder(input_side, n_channels, representation_activation='relu')
energy_mdl = dsebm.create_energy_model(encoder_mdl)
reconstruction_mdl = dsebm.create_reconstruction_model(energy_mdl)
# optimization parameters
batch_size = 128
epochs = 200
reconstruction_mdl.compile('adam', 'mse')
x_train_task = x_train[y_train.flatten() == single_class_ind]
x_test_task = x_test[y_test.flatten() == single_class_ind] # This is just for visual monitoring
reconstruction_mdl.fit(x=x_train_task, y=x_train_task,
batch_size=batch_size,
epochs=epochs,
validation_data=(x_test_task, x_test_task))
scores = -energy_mdl.predict(x_test, batch_size)
labels = y_test.flatten() == single_class_ind
res_file_name = '{}_dsebm_{}_{}.npz'.format(dataset_name,
get_class_name_from_index(single_class_ind, dataset_name),
datetime.now().strftime('%Y-%m-%d-%H%M'))
res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)
save_roc_pr_curve_data(scores, labels, res_file_path)
gpu_q.put(gpu_to_use)
def _RDAE_experiment(x_train, y_train, dataset_name, single_class_ind, gpu_q,
p):
gpu_to_use = gpu_q.get()
cudnn.benchmark = True
n_channels = x_train.shape[get_channels_axis()]
model = CAE_pytorch(in_channels=n_channels)
model = model.cuda()
optimizer = optim.Adam(model.parameters(), eps=1e-7, weight_decay=0.0005)
criterion = nn.MSELoss()
epochs = 20
inner_epochs = 1
lmbda = 0.00065
# train RDAE
losses = train_robust_cae(x_train, model, criterion, optimizer, lmbda,
inner_epochs, epochs // inner_epochs, False)
losses = losses - losses.min()
losses = losses / (1e-8 + losses.max())
scores = 1 - losses
res_file_name = '{}_rdae-{}_{}_{}.npz'.format(
dataset_name, p,
get_class_name_from_index(single_class_ind, dataset_name),
datetime.now().strftime('%Y-%m-%d-%H%M'))
res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)
os.makedirs(os.path.join(RESULTS_DIR, dataset_name), exist_ok=True)
save_roc_pr_curve_data(scores, y_train, res_file_path)
gpu_q.put(gpu_to_use)
def _DRAE_experiment(x_train, y_train, dataset_name, single_class_ind, gpu_q, p):
gpu_to_use = gpu_q.get()
n_channels = x_train.shape[get_channels_axis()]
model = CAE_pytorch(in_channels=n_channels)
batch_size = 128
model = model.cuda()
trainset = trainset_pytorch(train_data=x_train, train_labels=y_train, transform=transform_train)
trainloader = data.DataLoader(trainset, batch_size=batch_size, shuffle=True)
cudnn.benchmark = True
criterion = DRAELossAutograd(lamb=0.1)
optimizer = optim.Adam(model.parameters(), eps=1e-7, weight_decay=0.0005)
epochs = 250
# #########################Training########################
train_cae(trainloader, model, criterion, optimizer, epochs)
# #######################Testin############################
testloader = data.DataLoader(trainset, batch_size=1024, shuffle=False)
losses, reps = test_cae_pytorch(testloader, model)
losses = losses - losses.min()
losses = losses / (1e-8+losses.max())
scores = 1 - losses
res_file_name = '{}_drae-{}_{}_{}.npz'.format(dataset_name, p,
get_class_name_from_index(single_class_ind, dataset_name),
datetime.now().strftime('%Y-%m-%d-%H%M'))
res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)
os.makedirs(os.path.join(RESULTS_DIR, dataset_name), exist_ok=True)
save_roc_pr_curve_data(scores, y_train, res_file_path)
gpu_q.put(gpu_to_use)
def _cae_ocsvm_experiment(dataset_load_fn, dataset_name, single_class_ind,
gpu_q):
gpu_to_use = gpu_q.get()
os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use
(x_train, y_train), (x_test, y_test) = dataset_load_fn()
n_channels = x_train.shape[get_channels_axis()]
input_side = x_train.shape[2] # channel side will always be at shape[2]
enc = conv_encoder(input_side, n_channels)
dec = conv_decoder(input_side, n_channels)
x_in = Input(shape=x_train.shape[1:])
x_rec = dec(enc(x_in))
cae = Model(x_in, x_rec)
cae.compile('adam', 'mse')
x_train_task = x_train[y_train.flatten() == single_class_ind]
x_test_task = x_test[y_test.flatten(
) == single_class_ind] # This is just for visual monitoring
cae.fit(x=x_train_task,
y=x_train_task,
batch_size=128,
epochs=200,
validation_data=(x_test_task, x_test_task))
x_train_task_rep = enc.predict(x_train_task, batch_size=128)
if dataset_name in [
'cats-vs-dogs'
]: # OC-SVM is quadratic on the number of examples, so subsample training set
subsample_inds = np.random.choice(len(x_train_task_rep),
2500,
replace=False)
x_train_task_rep = x_train_task_rep[subsample_inds]
x_test_rep = enc.predict(x_test, batch_size=128)
pg = ParameterGrid({
'nu': np.linspace(0.1, 0.9, num=9),
'gamma': np.logspace(-7, 2, num=10, base=2)
})
results = Parallel(n_jobs=6)(delayed(_train_ocsvm_and_score)(
d, x_train_task_rep, y_test.flatten() == single_class_ind, x_test_rep)
for d in pg)
best_params, best_auc_score = max(zip(pg, results), key=lambda t: t[-1])
print(best_params)
best_ocsvm = OneClassSVM(**best_params).fit(x_train_task_rep)
scores = best_ocsvm.decision_function(x_test_rep)
labels = y_test.flatten() == single_class_ind
res_file_name = '{}_cae-oc-svm_{}_{}.npz'.format(
dataset_name, get_class_name_from_index(single_class_ind,
dataset_name),
datetime.now().strftime('%Y-%m-%d-%H%M'))
res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)
save_roc_pr_curve_data(scores, labels, res_file_path)
gpu_q.put(gpu_to_use)
def conv_decoder(output_side=32, n_channels=3, representation_dim=256, activation='relu'):
nf = 64
Slicer = Lambda(lambda x, slice_size: x[:, :slice_size, :slice_size, :], output_shape=(output_side, output_side, n_channels),
name="slice_layer") # Define your lambda layer
Slicer.arguments = {'slice_size': output_side} # Update extra arguments to F
rep_in = Input(shape=(representation_dim,))
if output_side == 21:
height, width, feat_mult = (3, 3, 4)
else:
height, width, feat_mult = (4, 4, 4)
g = Dense(nf * feat_mult * height * width)(rep_in)
g = BatchNormalization(axis=-1)(g)
g = Activation(activation)(g)
conv_shape = (nf * feat_mult, height, width) if get_channels_axis() == 1 else (height, width, nf * feat_mult)
g = Reshape(conv_shape)(g)
# upsample x2
g = Conv2DTranspose(nf * 2, kernel_size=(3, 3), strides=(2, 2), padding='same')(g)
g = BatchNormalization(axis=get_channels_axis())(g)
g = Activation(activation)(g)
# upsample x2
g = Conv2DTranspose(nf, kernel_size=(3, 3), strides=(2, 2), padding='same')(g)
g = BatchNormalization(axis=get_channels_axis())(g)
g = Activation(activation)(g)
if output_side == 64:
# upsample x2
g = Conv2DTranspose(nf, kernel_size=(3, 3), strides=(2, 2), padding='same')(g)
g = BatchNormalization(axis=get_channels_axis())(g)
g = Activation(activation)(g)
# upsample x2
g = Conv2DTranspose(n_channels, kernel_size=(3, 3), strides=(2, 2), padding='same')(g)
g = Activation('tanh')(g)
g = Slicer(g)
return Model(rep_in, g)
def _adgan_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q):
gpu_to_use = gpu_q.get()
os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use
(x_train, y_train), (x_test, y_test) = dataset_load_fn()
if len(x_test) > 5000:
# subsample x_test due to runtime complexity
chosen_inds = np.random.choice(len(x_test), 5000, replace=False)
x_test = x_test[chosen_inds]
y_test = y_test[chosen_inds]
n_channels = x_train.shape[get_channels_axis()]
input_side = x_train.shape[2] # image side will always be at shape[2]
critic = conv_encoder(input_side,
n_channels,
representation_dim=1,
representation_activation='linear')
noise_size = 256
generator = conv_decoder(input_side,
n_channels=n_channels,
representation_dim=noise_size)
def prior_gen(b_size):
return np.random.normal(size=(b_size, noise_size))
batch_size = 128
epochs = 100
x_train_task = x_train[y_train.flatten() == single_class_ind]
def data_gen(b_size):
chosen_inds = np.random.choice(len(x_train_task),
b_size,
replace=False)
return x_train_task[chosen_inds]
adgan.train_wgan_with_grad_penalty(prior_gen,
generator,
data_gen,
critic,
batch_size,
epochs,
grad_pen_coef=20)
scores = adgan.scores_from_adgan_generator(x_test, prior_gen, generator)
labels = y_test.flatten() == single_class_ind
res_file_name = '{}_adgan_{}_{}.npz'.format(
dataset_name, get_class_name_from_index(single_class_ind,
dataset_name),
datetime.now().strftime('%Y-%m-%d-%H%M'))
res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)
save_roc_pr_curve_data(scores, labels, res_file_path)
gpu_q.put(gpu_to_use)
def train_robust_cae(x_train, model, criterion, optimizer, lmbda, inner_epochs, outer_epochs, reinit=True):
batch_size = 128
S = np.zeros_like(x_train) # reside on numpy as x_train
def get_reconstruction(loader):
model.eval()
rc = []
for batch, _ in loader:
with torch.no_grad():
rc.append(model(batch.cuda()).cpu().numpy())
out = np.concatenate(rc, axis=0)
# NOTE: transform_train swaps the channel axis, swap back to yield the same shape
out = out.transpose((0, 2, 3, 1))
return out
for oe in range(outer_epochs):
# update AE
if reinit:
# Since our CAE_pytorch does not implement reset_parameters, regenerate a new model if reinit.
del model
model = CAE_pytorch(in_channels=x_train.shape[get_channels_axis()]).cuda()
model.train()
trainset = trainset_pytorch(x_train-S, train_labels=np.ones((x_train.shape[0], )), transform=transform_train)
trainloader = data.DataLoader(trainset, batch_size=batch_size, shuffle=True)
for ie in range(inner_epochs):
for batch_idx, (inputs, _) in enumerate(trainloader):
inputs = inputs.cuda()
outputs = model(inputs)
loss = criterion(inputs, outputs)
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
if (batch_idx + 1) % 10 == 0:
print('Epoch: [{} | {} ({} | {})], batch: {}, loss: {}'.format(
ie+1, inner_epochs, oe+1, outer_epochs, batch_idx+1, loss.item())
)
# update S via l21 proximal operator
testloader = data.DataLoader(trainset, batch_size=1024, shuffle=False)
recon = get_reconstruction(testloader)
S = prox_l21(x_train - recon, lmbda)
# get final reconstruction
finalset = trainset_pytorch(x_train - S, train_labels=np.ones((x_train.shape[0],)), transform=transform_train)
finalloader = data.DataLoader(finalset, batch_size=1024, shuffle=False)
reconstruction = get_reconstruction(finalloader)
losses = ((x_train-S-reconstruction) ** 2).sum(axis=(1, 2, 3), keepdims=False)
return losses
def _dagmm_experiment(dataset_load_fn, dataset_name, single_class_ind, gpu_q):
gpu_to_use = gpu_q.get()
os.environ["CUDA_VISIBLE_DEVICES"] = gpu_to_use
(x_train, y_train), (x_test, y_test) = dataset_load_fn()
n_channels = x_train.shape[get_channels_axis()]
input_side = x_train.shape[2] # image side will always be at shape[2]
enc = conv_encoder(input_side, n_channels, representation_dim=5,
representation_activation='linear')
dec = conv_decoder(input_side, n_channels=n_channels, representation_dim=enc.output_shape[-1])
n_components = 3
estimation = Sequential([Dense(64, activation='tanh', input_dim=enc.output_shape[-1] + 2), Dropout(0.5),
Dense(10, activation='tanh'), Dropout(0.5),
Dense(n_components, activation='softmax')]
)
batch_size = 256
epochs = 200
lambda_diag = 0.0005
lambda_energy = 0.01
dagmm_mdl = dagmm.create_dagmm_model(enc, dec, estimation, lambda_diag)
dagmm_mdl.compile('adam', ['mse', lambda y_true, y_pred: lambda_energy*y_pred])
x_train_task = x_train[y_train.flatten() == single_class_ind]
x_test_task = x_test[y_test.flatten() == single_class_ind] # This is just for visual monitoring
dagmm_mdl.fit(x=x_train_task, y=[x_train_task, np.zeros((len(x_train_task), 1))], # second y is dummy
batch_size=batch_size,
epochs=epochs,
validation_data=(x_test_task, [x_test_task, np.zeros((len(x_test_task), 1))]),
# verbose=0
)
energy_mdl = Model(dagmm_mdl.input, dagmm_mdl.output[-1])
scores = -energy_mdl.predict(x_test, batch_size)
scores = scores.flatten()
if not np.all(np.isfinite(scores)):
min_finite = np.min(scores[np.isfinite(scores)])
scores[~np.isfinite(scores)] = min_finite - 1
labels = y_test.flatten() == single_class_ind
res_file_name = '{}_dagmm_{}_{}.npz'.format(dataset_name,
get_class_name_from_index(single_class_ind, dataset_name),
datetime.now().strftime('%Y-%m-%d-%H%M'))
res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)
save_roc_pr_curve_data(scores, labels, res_file_path)
gpu_q.put(gpu_to_use)
def _E3Outlier_experiment(x_train, y_train, dataset_name, single_class_ind,
gpu_q, p):
"""Surrogate Supervision Discriminative Network training."""
gpu_to_use = gpu_q.get()
n_channels = x_train.shape[get_channels_axis()]
if OP_TYPE == 'RA':
transformer = RA(8, 8)
elif OP_TYPE == 'RA+IA':
transformer = RA_IA(8, 8, 12)
elif OP_TYPE == 'RA+IA+PR':
transformer = RA_IA_PR(8, 8, 12, 23, 2)
else:
raise NotImplementedError
print(transformer.n_transforms)
if BACKEND == 'wrn':
n, k = (10, 4)
model = WideResNet(num_classes=transformer.n_transforms,
depth=n,
widen_factor=k,
in_channel=n_channels)
elif BACKEND == 'resnet20':
n = 20
model = ResNet(num_classes=transformer.n_transforms,
depth=n,
in_channels=n_channels)
elif BACKEND == 'resnet50':
n = 50
model = ResNet(num_classes=transformer.n_transforms,
depth=n,
in_channels=n_channels)
elif BACKEND == 'densenet22':
n = 22
model = DenseNet(num_classes=transformer.n_transforms,
depth=n,
in_channels=n_channels)
elif BACKEND == 'densenet40':
n = 40
model = DenseNet(num_classes=transformer.n_transforms,
depth=n,
in_channels=n_channels)
else:
raise NotImplementedError('Unimplemented backend: {}'.format(BACKEND))
print('Using backend: {} ({})'.format(type(model).__name__, BACKEND))
x_train_task = x_train
transformations_inds = np.tile(np.arange(transformer.n_transforms),
len(x_train_task))
x_train_task_transformed = transformer.transform_batch(
np.repeat(x_train_task, transformer.n_transforms, axis=0),
transformations_inds)
# parameters for training
trainset = trainset_pytorch(train_data=x_train_task_transformed,
train_labels=transformations_inds,
transform=transform_train)
batch_size = 128
trainloader = data.DataLoader(trainset,
batch_size=batch_size,
shuffle=True)
cudnn.benchmark = True
criterion = nn.CrossEntropyLoss()
model = torch.nn.DataParallel(model).cuda()
if dataset_name in ['mnist', 'fashion-mnist']:
optimizer = optim.SGD(model.parameters(),
lr=0.001,
momentum=0.9,
weight_decay=0.0005)
else:
optimizer = optim.Adam(model.parameters(),
eps=1e-7,
weight_decay=0.0005)
epochs = int(np.ceil(250 / transformer.n_transforms))
train_pytorch(trainloader, model, criterion, optimizer, epochs)
# SSD-IF
test_set = testset_pytorch(test_data=x_train_task,
transform=transform_test)
x_train_task_rep = get_features_pytorch(testloader=data.DataLoader(
test_set, batch_size=batch_size, shuffle=False),
model=model).numpy()
clf = IsolationForest(contamination=p, n_jobs=4).fit(x_train_task_rep)
if_scores = clf.decision_function(x_train_task_rep)
res_file_name = '{}_ssd-iforest-{}_{}_{}.npz'.format(
dataset_name, p,
get_class_name_from_index(single_class_ind, dataset_name),
datetime.now().strftime('%Y-%m-%d-%H%M'))
res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)
os.makedirs(os.path.join(RESULTS_DIR, dataset_name), exist_ok=True)
save_roc_pr_curve_data(if_scores, y_train, res_file_path)
# E3Outlier
if SCORE_MODE == 'pl_mean':
preds = np.zeros((len(x_train_task), transformer.n_transforms))
original_preds = np.zeros((transformer.n_transforms, len(x_train_task),
transformer.n_transforms))
for t in range(transformer.n_transforms):
idx = np.squeeze(
np.array([range(x_train_task.shape[0])]) *
transformer.n_transforms + t)
test_set = testset_pytorch(
test_data=x_train_task_transformed[idx, :],
transform=transform_test)
original_preds[t, :, :] = softmax(
test_pytorch(testloader=data.DataLoader(test_set,
batch_size=batch_size,
shuffle=False),
model=model))
preds[:, t] = original_preds[t, :, :][:, t]
scores = preds.mean(axis=-1)
elif SCORE_MODE == 'max_mean':
preds = np.zeros((len(x_train_task), transformer.n_transforms))
original_preds = np.zeros((transformer.n_transforms, len(x_train_task),
transformer.n_transforms))
for t in range(transformer.n_transforms):
idx = np.squeeze(
np.array([range(x_train_task.shape[0])]) *
transformer.n_transforms + t)
test_set = testset_pytorch(
test_data=x_train_task_transformed[idx, :],
transform=transform_test)
original_preds[t, :, :] = softmax(
test_pytorch(testloader=data.DataLoader(test_set,
batch_size=batch_size,
shuffle=False),
model=model))
preds[:, t] = np.max(original_preds[t, :, :], axis=1)
scores = preds.mean(axis=-1)
elif SCORE_MODE == 'neg_entropy':
preds = np.zeros((len(x_train_task), transformer.n_transforms))
original_preds = np.zeros((transformer.n_transforms, len(x_train_task),
transformer.n_transforms))
for t in range(transformer.n_transforms):
idx = np.squeeze(
np.array([range(x_train_task.shape[0])]) *
transformer.n_transforms + t)
test_set = testset_pytorch(
test_data=x_train_task_transformed[idx, :],
transform=transform_test)
original_preds[t, :, :] = softmax(
test_pytorch(testloader=data.DataLoader(test_set,
batch_size=batch_size,
shuffle=False),
model=model))
for s in range(len(x_train_task)):
preds[s, t] = neg_entropy(original_preds[t, s, :])
scores = preds.mean(axis=-1)
else:
raise NotImplementedError
res_file_name = '{}_e3outlier-{}_{}_{}.npz'.format(
dataset_name, p,
get_class_name_from_index(single_class_ind, dataset_name),
datetime.now().strftime('%Y-%m-%d-%H%M'))
res_file_path = os.path.join(RESULTS_DIR, dataset_name, res_file_name)
os.makedirs(os.path.join(RESULTS_DIR, dataset_name), exist_ok=True)
save_roc_pr_curve_data(scores, y_train, res_file_path)
gpu_q.put(gpu_to_use)
