def __init__(self, n_hidden, n_iterations=3000, learning_rate=0.01):
self.n_hidden = n_hidden
self.n_iterations = n_iterations
self.learning_rate = learning_rate
self.hidden_activation = Sigmoid()
self.output_activation = Softmax()
self.loss = CrossEntropy()
def get_classify_loss(self,
data,
prediction_lin,
mask,
deterministic=False):
pred_lin = get_output(self.net['classify'], {
self.net['input']: data,
self.net['mask']: mask
},
deterministic=deterministic)
pred_hard = Softmax(pred_lin)
pred_soft = Softmax(pred_lin / 3.0)
token = T.argmax(prediction_lin, axis=1)
prediction_soft = Softmax(prediction_lin / 3.0)
loss_hard = lasagne.objectives.categorical_crossentropy(
pred_hard, token)
loss_soft = -T.sum(prediction_soft * T.log(pred_soft + 1e-20), axis=1)
classify_acc_1 = lasagne.objectives.categorical_accuracy(pred_hard,
token,
top_k=1)
classify_acc_5 = lasagne.objectives.categorical_accuracy(pred_hard,
token,
top_k=5)
return loss_hard, loss_soft, classify_acc_1, classify_acc_5
def get_ctc_loss(self, data, mask, token, deteministic=False):
embeding = get_output(self.net['drop1d_2'],
data,
deterministic=deteministic)
# loss calculation
## ctc loss
output_lin = get_output(
self.net['out_lin'], {
self.net['lstm_input']: T.transpose(embeding, (0, 2, 1)),
self.net['mask']: mask
})
output_softmax = Softmax(output_lin) # (nb, max_hlen, nClasses)
pred = T.argmax(output_softmax, axis=2)
output_trans = T.transpose(output_softmax,
(1, 0, 2)) # (max_hlen, nb, nClasses)
if 'equidistant_rate' in self.config.items.keys():
ctc_loss = ctc_equidistant_cost(
output_trans,
T.sum(mask, axis=1, dtype='int32'),
token,
max_dist=self.config.items['equidistant_rate']).mean()
else:
ctc_loss = ctc_cost(output_trans, T.sum(mask,
axis=1,
dtype='int32'),
token).mean()
return ctc_loss, pred
def __init__(self,
input_size,
output_size,
activation=Softmax(),
weight_init=Uniform()):
self.W = T.shared(weight_init((input_size, output_size)),
name="W_linear")
self.b = T.shared(weight_init(output_size), name="b_linear")
self.params = [self.W, self.b]
self.activation = activation
def __init__(self, phase, config, vocabulary_size=1295, hidden_ndim=512):
# need to be same voca_size and hidde_ndim so as to load same shape params
# self.log_self()
size = 101
# model paras
self.config = config
self.alpha = np.array(1e-3, dtype=np.float32)
self.eps = np.array(1e-6, dtype=np.float32)
self.learning_rate = theano.shared(np.float32(config.items['lr']))
self.nClasses = vocabulary_size + 1
self.vocabulary_size = vocabulary_size
# variables
data = T.tensor4('data') # (3*nb, 3, 96, 96) or (nb*len, 3, 96, 96)
mask = T.matrix('mask') # (nb, max_hlen)
token = T.imatrix('token') # (nb, max_vlen)
# label = T.imatrix('label') # (nb, voca_size)
net = {}
# feature extraction
net['input'] = InputLayer(shape=(None, 3, size,
size)) # (3*nb, 3, 96, 96)
net['conv1'] = Conv2DLayer(incoming=net['input'],
num_filters=96,
filter_size=7,
stride=2)
net['norm1'] = LocalResponseNormalization2DLayer(incoming=net['conv1'])
net['pool1'] = MaxPool2DLayer(incoming=net['norm1'], pool_size=3)
net['conv2'] = Conv2DLayer(incoming=net['pool1'],
num_filters=256,
filter_size=5)
net['pool2'] = MaxPool2DLayer(incoming=net['conv2'], pool_size=2)
net['conv3'] = Conv2DLayer(incoming=net['pool2'],
num_filters=512,
filter_size=3,
pad=1)
net['conv4'] = Conv2DLayer(incoming=net['conv3'],
num_filters=512,
filter_size=3,
pad=1)
net['conv5'] = Conv2DLayer(incoming=net['conv4'],
num_filters=512,
filter_size=3,
pad=1)
net['pool5'] = MaxPool2DLayer(incoming=net['conv5'],
pool_size=3) # (3*nb, 512, 3, 3)
net['fc6'] = DenseLayer(
incoming=net['pool5'],
num_units=1024) # auto flatten all the trailing axes
net['drop6'] = DropoutLayer(incoming=net['fc6'], p=0.5)
net['fc7'] = DenseLayer(incoming=net['drop6'],
num_units=256,
nonlinearity=identity) # (3*nb, 256)
# encoding network for image features
net['mask'] = InputLayer(shape=(None, None),
name='mask') # (nb, max_hlen)
self.nb = mask.shape[0]
self.max_hlen = mask.shape[1]
net['pre_conv1d'] = DimshuffleLayer(
ReshapeLayer(incoming=NonlinearityLayer(net['fc6'],
nonlinearity=rectify),
shape=(self.nb, -1, 1024)), (0, 2, 1))
net['conv1d_1'] = Conv1DLayer(net['pre_conv1d'],
num_filters=1024,
filter_size=3,
pad='same')
net['pool1d_1'] = MaxPool1DLayer(net['conv1d_1'],
pool_size=2) #(nb, 1024, max_hlen)
net['drop1d_1_before'] = DropoutLayer(net['pool1d_1'],
p=0.1,
shared_axes=(2, ))
net['drop1d_1'] = ReshapeLayer(net['drop1d_1_before'],
shape=(-1, [1], [2]))
net['conv1d_2'] = Conv1DLayer(net['drop1d_1'],
num_filters=1024,
filter_size=3,
pad='same')
net['pool1d_2'] = MaxPool1DLayer(net['conv1d_2'],
pool_size=2) #(nb, 1024, max_hlen)
net['drop1d_2_before'] = DropoutLayer(net['pool1d_2'],
p=0.1,
shared_axes=(2, ))
net['drop1d_2'] = ReshapeLayer(net['drop1d_2_before'],
shape=(-1, [1], [2]))
# LSTM
net['lstm_input'] = InputLayer(shape=(None, None, 1024),
name='lstm_input')
net['lstm_frw'] = LSTMLayer(
incoming=net['lstm_input'],
mask_input=net['mask'],
forgetgate=Gate(b=lasagne.init.Constant(1.0)),
num_units=hidden_ndim) # (nb, max_hlen, hidden_ndim)
net['lstm_bck'] = LSTMLayer(
incoming=net['lstm_input'],
mask_input=net['mask'],
forgetgate=Gate(b=lasagne.init.Constant(1.0)),
num_units=hidden_ndim,
backwards=True)
net['lstm_shp'] = ReshapeLayer(
ConcatLayer((net['lstm_frw'], net['lstm_bck']), axis=2),
shape=(-1, 2 * hidden_ndim)) # (nb*max_hlen, 2*hidden_ndim)
net['out'] = DenseLayer(
net['lstm_shp'], self.nClasses,
nonlinearity=identity) # (nb*max_hlen, nClasses)
net['out_lin'] = ReshapeLayer(net['out'],
shape=(self.nb, -1, self.nClasses))
# # WSDD
# net['wsdd_input'] = InputLayer(shape=(None, 1024, None), name='wsdd_input') # (nb, 1024, max_hlen+2)
# net['wsdd1'] = Conv1DLayer(net['wsdd_input'], num_filters=256, filter_size=2, pad='valid')
# net['wsdd1_drop'] = DropoutLayer(net['wsdd1'], p=0.5) # (nb, 256, max_hlen+1)
# net['wsdd2'] = Conv1DLayer(net['wsdd1_drop'], num_filters=256, filter_size=2, pad='valid')
# net['wsdd2_drop'] = DropoutLayer(net['wsdd2'], p=0.5) # (nb, 256, max_hlen)
#
# net['predet1a'] = DimshuffleLayer(net['wsdd2_drop'], (0, 2, 1))
# net['predet1b'] = ReshapeLayer(net['predet1a'], (-1, 256)) # (nb*max_hlen, 256)
#
# net['det'] = DenseLayer(net['predet1b'], vocabulary_size, nonlinearity=identity) # (nb*max_hlen, voca_size)
# net['det_lin'] = ReshapeLayer(net['det'], (self.nb, -1, vocabulary_size)) # (nb, max_hlen, voca_size)
self.net = net
# try save load model
dummy_save_file = 'dummy.pkl'
glog.info('try save load dummy model to: %s...' % dummy_save_file)
self.save_model(dummy_save_file)
self.load_model(dummy_save_file)
os.system('rm -rf dummy.pkl')
glog.info(
'dummy save load success, remove it and start calculate outputs...'
)
if phase == 'pretrain':
pass
# # for triplet pretrain use
# self.params_feat = get_all_params(net['fc7'])
# regular_feat = lasagne.regularization.apply_penalty(self.params_feat, lasagne.regularization.l2) * np.array(5e-4 / 2, dtype=np.float32)
#
# ## triplet train loss
# triplet_loss_train = self.get_triplet_loss(data, deterministic=False)
# loss_train_feat = triplet_loss_train + regular_feat
#
# ## triplet valid loss
# triplet_loss_valid = self.get_triplet_loss(data, deterministic=True)
# loss_valid_feat = triplet_loss_valid + regular_feat
#
# self.updates = lasagne.updates.momentum(loss_train_feat, self.params_feat, learning_rate=learning_rate, momentum=0.9)
# self.inputs = [data]
# self.train_outputs = [loss_train_feat, triplet_loss_train]
# self.valid_outputs = [loss_valid_feat, triplet_loss_valid]
elif phase == 'ctc':
# for ctc loss
self.params_full = lasagne.layers.get_all_params(
[self.net['drop1d_2'], self.net['out_lin']], trainable=True)
self.regular_params = lasagne.layers.get_all_params(
[self.net['drop1d_2'], self.net['out_lin']],
regularizable=True)
regular_full = lasagne.regularization.apply_penalty(
self.regular_params, lasagne.regularization.l2) * np.array(
5e-4 / 2, dtype=np.float32)
# full train loss
ctc_loss_train, pred_train = self.get_ctc_loss(data,
mask,
token,
deteministic=False)
loss_train_full = ctc_loss_train + regular_full
# full valid loss
ctc_loss_valid, pred_valid = self.get_ctc_loss(data,
mask,
token,
deteministic=True)
loss_valid_full = ctc_loss_valid + regular_full
self.updates = lasagne.updates.adam(
loss_train_full,
self.params_full,
learning_rate=self.learning_rate)
self.inputs = [data, mask, token]
self.train_outputs = [loss_train_full, ctc_loss_train, pred_train]
self.valid_outputs = [loss_valid_full, ctc_loss_valid, pred_valid]
elif phase == 'extract_feature':
# for feature extraction
fc6 = get_output(self.net['fc6'], data, deterministic=True)
self.feature_func = theano.function(inputs=[data], outputs=fc6)
elif phase == 'get_prediction':
embeding = get_output(self.net['drop1d_2'],
data,
deterministic=True) # (nb, 1024, len_m)
output_lin = get_output(
self.net['out_lin'], {
self.net['lstm_input']: T.transpose(embeding, (0, 2, 1)),
self.net['mask']: mask
},
deterministic=True)
output_softmax = Softmax(output_lin) # (nb, max_hlen, nClasses)
output_trans = T.transpose(output_softmax,
(1, 0, 2)) # (max_hlen, nb, nClasses)
best_path_loss, best_path = best_right_path_cost(
output_trans, mask, token)
ctc_loss = ctc_cost(output_trans, T.sum(mask,
axis=1,
dtype='int32'), token)
# (nb, max_hlen, voca_size+1)
self.predict_func = theano.function(
inputs=[data, mask, token],
outputs=[best_path_loss, best_path, ctc_loss])
elif phase == 'top_k_prediction':
embeding = get_output(self.net['drop1d_2'],
data,
deterministic=True) # (nb, 1024, len_m)
output_lin = get_output(
self.net['out_lin'], {
self.net['lstm_input']: T.transpose(embeding, (0, 2, 1)),
self.net['mask']: mask
},
deterministic=True)
output_softmax = Softmax(output_lin) # (nb, max_hlen, nClasses)
output_trans = T.transpose(output_softmax,
(1, 0, 2)) # (max_hlen, nb, nClasses)
top_k_path_loss, top_k_path = top_k_right_path_cost(
output_trans, mask, token, k=config.items['top_k'])
ctc_loss = ctc_cost(output_trans, T.sum(mask,
axis=1,
dtype='int32'), token)
# (nb, max_hlen, voca_size+1)
self.predict_func = theano.function(
inputs=[data, mask, token],
outputs=[output_lin, top_k_path_loss, top_k_path, ctc_loss])
glog.info('Model built, phase = %s' % phase)
class MultilayerPerceptron():
def __init__(self, n_hidden, n_iterations=3000, learning_rate=0.01):
self.n_hidden = n_hidden
self.n_iterations = n_iterations
self.learning_rate = learning_rate
self.hidden_activation = Sigmoid()
self.output_activation = Softmax()
self.loss = CrossEntropy()
def _initialize_weights(self, X, y):
n_samples, n_features = X.shape
_, n_outputs = y.shape
# Hidden layer
limit = 1 / math.sqrt(n_features)
self.W = np.random.uniform(-limit, limit, (n_features, self.n_hidden))
self.w0 = np.zeros((1, self.n_hidden))
# Output layer
limit = 1 / math.sqrt(self.n_hidden)
self.V = np.random.uniform(-limit, limit, (self.n_hidden, n_outputs))
self.v0 = np.zeros((1, n_outputs))
def fit(self, X, y):
self._initialize_weights(X, y)
for i in range(self.n_iterations):
# ..............
# Forward Pass
# ..............
# HIDDEN LAYER
hidden_input = X.dot(self.W) + self.w0
hidden_output = self.hidden_activation(hidden_input)
# OUTPUT LAYER
output_layer_input = hidden_output.dot(self.V) + self.v0
y_pred = self.output_activation(output_layer_input)
# ...............
# Backward Pass
# ...............
# OUTPUT LAYER
# Grad. w.r.t input of output layer
grad_wrt_o_layer = self.loss.gradient(
y,
y_pred) * self.output_activation.gradient(output_layer_input)
grad_v = hidden_output.T.dot(grad_wrt_o_layer)
grad_v0 = np.sum(grad_wrt_o_layer, axis=0, keepdims=True)
# HIDDEN LAYER
# Grad. w.r.t input of hidden layer
grad_wrt_hidden_layer = grad_wrt_o_layer.dot(
self.V.T) * self.hidden_activation.gradient(hidden_input)
grad_w = X.T.dot(grad_wrt_hidden_layer)
grad_w0 = np.sum(grad_wrt_hidden_layer, axis=0, keepdims=True)
# Update weights (by gradient descent)
# Move against the gradient to minimize loss
self.V -= self.learning_rate * grad_v
self.v0 -= self.learning_rate * grad_v0
self.W -= self.learning_rate * grad_w
self.w0 -= self.learning_rate * grad_w0
# Use the trained model to predict labels of X
def predict(self, X):
hidden_input = X.dot(self.W) + self.w0
hidden_output = self.hidden_activation(hidden_input)
output_layer_input = hidden_output.dot(self.V) + self.v0
y_pred = self.output_activation(output_layer_input)
return y_pred
def eval(args):
#pretrained model loading
device =torch.device(f"cuda:{args.device_id}")
model = WrappedModel(densenet(num_classes = 100, depth = 190, growthRate = 40, compressionRate = 2, dropRate = 0))
model.load_state_dict(torch.load("Ref/model_best.pth.tar")["state_dict"])
model.eval()
model.to(device)
#dataloader define
in_test_loader, out_test_loader = getLoader(data_type =args.dataset, transform= args.transform,
batch_size = args.batch_size, n_workers= args.n_workers)
#criterion for Adversarial
# criterion = nn.CrossEntropyLoss()
criterion = nn.NLLLoss()
#evaluation
out_pred_arr = []
in_pred_arr = []
for idx, (image, label) in tqdm(enumerate(out_test_loader), total= len(out_test_loader)):
image = image.to(device)
image.requires_grad = True
pertubed = image
if args.epsilon > 0:
pred = Softmax(model(image), T = args.temp)
pred_idx = pred.argmax(dim = 1)
loss = criterion(pred, pred_idx)
loss.backward()
adv = args.epsilon *torch.sign(image.grad)
pertubed =pertubed + adv
with torch.no_grad():
pred = model(pertubed)
pred = Softmax(pred, T = args.temp)
max_ = pred.max(dim = -1)[0].detach().cpu().numpy()
out_pred_arr += list(max_)
for idx, (image, label) in tqdm(enumerate(in_test_loader), total= len(in_test_loader)):
image = image.to(device)
image.requires_grad = True
pertubed = image
if args.epsilon > 0:
pred = Softmax(model(image), T= args.temp)
pred_idx = pred.argmax(dim = 1)
loss = criterion(pred, pred_idx)
loss.backward()
adv = args.epsilon *torch.sign(-image.grad)
pertubed =pertubed + adv
with torch.no_grad():
pred = model(pertubed)
pred = Softmax(pred, T = args.temp)
max_ = pred.max(dim = -1)[0].detach().cpu().numpy()
in_pred_arr += list(max_)
threshold, fpr, auroc = eval_OOD(in_pred_arr, out_pred_arr)
print(f"FPR: {fpr}, AUROC: {auroc}")
with open("exp_log.txt",'a+') as f:
f.write(f"{vars(args)}, FPR: {fpr}, AUROC: {auroc}\n")