if __name__ == '__main__':
opt = parse_opts()
if opt.root_path != '':
opt.video_path = os.path.join(opt.root_path, opt.video_path)
opt.annotation_path = os.path.join(opt.root_path, opt.annotation_path)
opt.result_path = os.path.join(opt.root_path, opt.result_path)
if opt.resume_path:
opt.resume_path = os.path.join(opt.root_path, opt.resume_path)
if opt.pretrain_path:
opt.pretrain_path = os.path.join(opt.root_path, opt.pretrain_path)
opt.scales = [opt.initial_scale]
for i in range(1, opt.n_scales):
opt.scales.append(opt.scales[-1] * opt.scale_step)
opt.arch = '{}-{}'.format(opt.model, opt.model_depth)
opt.mean = get_mean(opt.norm_value, dataset=opt.mean_dataset)
opt.std = get_std(opt.norm_value)
print(opt)
with open(os.path.join(opt.result_path, 'opts.json'), 'w') as opt_file:
json.dump(vars(opt), opt_file)
torch.manual_seed(opt.manual_seed)
model, parameters = generate_model(opt)
print(model)
criterion = nn.CrossEntropyLoss()
if not opt.no_cuda:
criterion = criterion.cuda()
if opt.no_mean_norm and not opt.std_norm:
norm_method = transforms.Normalize([0, 0, 0], [1, 1, 1])
elif not opt.std_norm:
return x_train, y_train, x_test, y_test
hdf = '../data/spectrum.h5'
file = h5py.File(hdf, 'r')
spectra = np.array(file['spectrum'])
phi = np.array(file['phi'])
theta = np.array(file['theta'])
lp = np.array(file['lp'])
target = np.concatenate([phi.reshape(-1, 1),
theta.reshape(-1, 1),
lp.reshape(-1, 1)], axis=1)
noise_estimation = utils.get_std(spectra.copy(), target.copy())
noise_accuracy = []
seed = [628, 693, 847, 621, 861, 409, 74, 306, 884, 777]
for i in range(k):
np.random.seed(42)
spectra_noise = utils.add_noise(spectra.copy(), noise_estimation.copy())
x_train, y_train, x_test, y_test = split_data_for_noise_test(spectra_noise.copy(), target.copy(), seed[i])
x_train, x_test, _, _ = utils.preprocessing(x_train, x_test)
y_train, y_test, _, _ = utils.preprocessing(y_train, y_test)
y_pred = model_fcnn[i].predict(x_test)
y_pred, y_test = utils.postprocessing(y_pred, y_test, y_min, y_max)
def train(args):
debug = args.debug
gpu = args.gpu
Nhidden = args.Nhidden # LSTM hidden nodes
Nbatches = args.Nbatches # Training batches
BatchSize = args.BatchSize # Training batch size
ChunkSize = args.ChunkSize # The length for accumulating loss in training
SubseqLen = args.SubseqLen # Split the training sequence into subsequences
LearningRate = args.LearningRate # Learning rate
Eps = args.Eps # Eps used in Adam optimizer
AMSGrad = args.AMSGrad # Use AMSGrad in Adam
LRdecrease = args.LRdecrease # Decrease learning rate
save_model_dir = args.save_model_dir
save_model_name = args.save_model_name
normal_data_dir = args.normal_data_dir
normal_data_name_train = args.normal_data_name_train
val_and_ref_name = args.normal_data_name_val_and_ref
RED_collection_len = args.RED_collection_len
RED_points = args.RED_points
Pvalue_th = args.Pvalue_th
if args.dummydata:
training_normal_data, val_normal_data, ref_normal_data = (
loaddata.load_normal_dummydata())
else:
_, training_normal_data, _ = (
loaddata.load_data_split(
data_dir=normal_data_dir,
file_name=normal_data_name_train,
# The first few readings could be unstable, remove it.
split=(0.1, 0.8, 0.1)))
_, ref_normal_data, val_normal_data = loaddata.load_data_split(
data_dir=normal_data_dir,
file_name=val_and_ref_name,
# The first few readings could be unstable, remove it.
split=(0.1, 0.45, 0.45))
training_normal_data_mean = utils.get_mean(training_normal_data)
training_normal_data_std = utils.get_std(training_normal_data)
Nfeatures = training_normal_data.shape[1]
AnomalyDetector = detector.Detector(input_size=Nfeatures,
hidden_size=Nhidden,
th=Pvalue_th)
AnomalyDetector.set_mean(training_normal_data_mean)
AnomalyDetector.set_std(training_normal_data_std)
training_normal_data = AnomalyDetector.normalize(training_normal_data)
val_normal_data = AnomalyDetector.normalize(val_normal_data)
ref_normal_data = torch.tensor(AnomalyDetector.normalize(ref_normal_data))
training_normal_wrapper = SeqGenerator.SeqGenerator(training_normal_data)
val_normal_wrapper = SeqGenerator.SeqGenerator(val_normal_data)
training_normal_len = len(training_normal_data)
MSELossLayer = torch.nn.MSELoss()
optimizer = optim.Adam(params=AnomalyDetector.parameters(),
lr=LearningRate,
eps=Eps,
amsgrad=True)
if gpu:
ref_normal_data = ref_normal_data.cuda()
MSELossLayer = MSELossLayer.cuda()
AnomalyDetector = AnomalyDetector.cuda()
if debug:
for name, para in AnomalyDetector.named_parameters():
print(name, para.size())
for batch in range(Nbatches):
def step_fn(data_batch, is_train=True):
t = 0
init_state = (torch.zeros(1, BatchSize, Nhidden),
torch.zeros(1, BatchSize, Nhidden))
if gpu:
init_state = (init_state[0].cuda(), init_state[1].cuda())
data_batch = data_batch.cuda()
state = init_state
loss_list = []
while t + ChunkSize + 1 < data_batch.shape[0]:
if is_train:
AnomalyDetector.zero_grad()
pred, state = AnomalyDetector.forward(
data_batch[t:t + ChunkSize, :, :], state)
truth = data_batch[t + 1:t + ChunkSize + 1, :, :]
loss = MSELossLayer(pred, truth)
if debug:
print("pred.size ", pred.size(), "truth.size ",
truth.size())
if is_train:
loss.backward()
optimizer.step()
if gpu:
loss_list.append(loss.detach().cpu().numpy())
else:
loss_list.append(loss.detach().numpy())
state = (state[0].detach(), state[1].detach())
t += ChunkSize
return loss_list
training_batch = torch.tensor(
training_normal_wrapper.next(BatchSize, SubseqLen))
train_loss_list = step_fn(training_batch, is_train=True)
val_batch = torch.tensor(val_normal_wrapper.next(BatchSize, 2000))
val_loss_list = step_fn(val_batch, is_train=False)
print("Batch", batch, "Training loss", np.mean(train_loss_list),
"Val loss", np.mean(val_loss_list))
if (batch + 1) % LRdecrease == 0:
LearningRate = LearningRate / 2.0
utils.setLearningRate(optimizer, LearningRate)
print("Training Done")
print("Getting RED")
AnomalyDetector.set_RED_config(RED_collection_len=RED_collection_len,
RED_points=RED_points)
AnomalyDetector.collect_ref_RED(ref_normal_data, gpu)
if not os.path.exists(save_model_dir):
os.makedirs(save_model_dir)
torch.save(AnomalyDetector.cpu(), save_model_dir + save_model_name)
AnomalyDetector.jitSaveTorchModule(save_model_dir)
print("Model saved")
def get_test_set(dataset):
assert dataset in ['ucf101', 'hmdb51']
if dataset == 'hmdb51':
with open('/home/jingjing/zhipeng/adv-attack-video/code2/datasets/c3d_dataset/hmdb51_params.pkl', 'rb') as ipt:
opt = pickle.load(ipt)
opt = DictToAttr(opt)
elif dataset == 'ucf101':
with open('/home/jingjing/zhipeng/adv-attack-video/code2/datasets/c3d_dataset/ucf101_params.pkl', 'rb') as ipt:
opt = pickle.load(ipt)
opt = DictToAttr(opt)
opt.dataset = dataset
# transform begin
opt.mean = get_mean(opt.norm_value, dataset)
opt.std = get_std(opt.norm_value, dataset)
torch.manual_seed(opt.manual_seed)
if opt.no_mean_norm and not opt.std_norm:
norm_method = Normalize([0, 0, 0], [1, 1, 1])
elif not opt.std_norm:
norm_method = Normalize(opt.mean, [1, 1, 1])
else:
norm_method = Normalize(opt.mean, opt.std)
spatial_transform = spatial_Compose([
Scale(int(opt.sample_size / opt.scale_in_test)),
CornerCrop(opt.sample_size, opt.crop_position_in_test),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = LoopPadding(opt.sample_duration)
target_transform = target_Compose([VideoID(),ClassLabel()])
# transform end
if opt.dataset == 'ucf101':
try:
test_data = UCF101(
opt.video_path,
opt.annotation_path,
'validation',
input_style = 'rgb',
n_samples_for_each_video = 3,
spatial_transform=spatial_transform,
temporal_transform=temporal_transform,
target_transform=target_transform,
sample_duration=opt.sample_duration)
except:
test_data = UCF101(
'/home/jingjing/zhipeng/adv-attack-video/data/UCF101-jpg',
'/home/jingjing/zhipeng/adv-attack-video/data/UCF101-annotation/ucfTrainTestlist/ucf101_01.json',
'validation',
input_style = 'rgb',
n_samples_for_each_video = 3,
spatial_transform=spatial_transform,
temporal_transform=temporal_transform,
target_transform=target_transform,
sample_duration=opt.sample_duration)
elif opt.dataset == 'hmdb51':
try:
test_data = HMDB51(
opt.video_path,
opt.annotation_path,
'validation',
input_style = 'rgb',
n_samples_for_each_video = 3,
spatial_transform=spatial_transform,
temporal_transform=temporal_transform,
target_transform=target_transform,
sample_duration=opt.sample_duration)
except:
test_data = HMDB51(
'/home/jingjing/zhipeng/adv-attack-video/data/hmdb51-jpg',
'/home/jingjing/zhipeng/adv-attack-video/data/hmdb51-annotation/hmdb51_1.json',
'validation',
input_style = 'rgb',
n_samples_for_each_video = 3,
spatial_transform=spatial_transform,
temporal_transform=temporal_transform,
target_transform=target_transform,
sample_duration=opt.sample_duration)
return test_data
def get_training_set(dataset):
assert dataset in ['ucf101', 'hmdb51']
if dataset == 'hmdb51':
with open('/home/jingjing/zhipeng/adv-attack-video/code2/datasets/c3d_dataset/hmdb51_params.pkl', 'rb') as ipt:
opt = pickle.load(ipt)
opt = DictToAttr(opt)
elif dataset == 'ucf101':
with open('/home/jingjing/zhipeng/adv-attack-video/code2/datasets/c3d_dataset/ucf101_params.pkl', 'rb') as ipt:
opt = pickle.load(ipt)
opt = DictToAttr(opt)
opt.dataset = dataset
# transforms begin
opt.scales = [opt.initial_scale]
for i in range(1, opt.n_scales):
opt.scales.append(opt.scales[-1] * opt.scale_step)
opt.mean = get_mean(opt.norm_value, dataset=opt.dataset)
opt.std = get_std(opt.norm_value, dataset=opt.dataset)
if opt.no_mean_norm and not opt.std_norm:
norm_method = Normalize([0, 0, 0], [1, 1, 1])
elif not opt.std_norm:
norm_method = Normalize(opt.mean, [1, 1, 1])
else:
norm_method = Normalize(opt.mean, opt.std)
torch.manual_seed(opt.manual_seed)
assert opt.train_crop in ['random', 'corner', 'center']
if opt.train_crop == 'random':
crop_method = MultiScaleRandomCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'corner':
crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'center':
crop_method = MultiScaleCornerCrop(
opt.scales, opt.sample_size, crop_positions=['c'])
spatial_transform = spatial_Compose([
crop_method,
RandomHorizontalFlip(),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = TemporalRandomCrop(opt.sample_duration)
target_transform = ClassLabel()
# transforms end
if opt.dataset == 'ucf101':
try:
training_data = UCF101(
opt.video_path,
opt.annotation_path,
'training',
input_style = 'rgb',
spatial_transform=spatial_transform,
temporal_transform=temporal_transform,
target_transform=target_transform,
)
except:
training_data = UCF101(
'/home/jingjing/zhipeng/adv-attack-video/data/UCF101-jpg',
'/home/jingjing/zhipeng/adv-attack-video/data/UCF101-annotation/ucfTrainTestlist/ucf101_01.json',
'training',
input_style = 'rgb',
spatial_transform=spatial_transform,
temporal_transform=temporal_transform,
target_transform=target_transform,
)
elif opt.dataset == 'hmdb51':
try:
training_data = HMDB51(
opt.video_path,
opt.annotation_path,
'training',
input_style = 'rgb',
spatial_transform=spatial_transform,
temporal_transform=temporal_transform,
target_transform=target_transform,
)
except:
training_data = HMDB51(
'/home/jingjing/zhipeng/adv-attack-video/data/hmdb51-jpg',
'/home/jingjing/zhipeng/adv-attack-video/data/hmdb51-annotation/hmdb51_1.json',
'training',
input_style = 'rgb',
spatial_transform=spatial_transform,
temporal_transform=temporal_transform,
target_transform=target_transform,
)
return training_data
def main():
print('* Start! ')
print('* Loading config.json')
with open('config.json') as json_data:
config = json.load(json_data)
depth = int(config["layers"])
width = int(config["neurons_by_layer"])
drop_in = float(config["dropout_input"])
drop_hid = float(config["dropout_hidden"])
num_epochs = int(config["num_epochs"])
winSide = int(config["window_side"])
ImageShape = config["image_shape"]
ImageShape = (int(ImageShape[0]),int(ImageShape[1]),int(ImageShape[2]))
ValidationSet = utils.getValues(config["validation_set"])
alpha = float(config["alpha"])
# Other global variables
PatternShape = (ImageShape[0],ImageShape[1])
winSize = (winSide,winSide)
n_features = ImageShape[2]*(winSide**2)
print("* Building model and compiling functions...")
input_var = T.matrix('inputs')
target_var = T.ivector('targets')
network = utils.build_custom_mlp(n_features, input_var, depth, width, drop_in, drop_hid)
prediction = lasagne.layers.get_output(network)
t2 = theano.tensor.extra_ops.to_one_hot(target_var, 2, dtype='int32')
error = lasagne.objectives.categorical_crossentropy(prediction, t2)
params = lasagne.layers.get_all_params(network, trainable=True)
output_model = theano.function([input_var], prediction)
# compilation
comp_params_updater = []
for w in params:
w_in = T.matrix()
if(w_in.type != w.type):
w_in = T.vector()
w_update = theano.function([w_in], updates=[(w, w_in)])
comp_params_updater = comp_params_updater + [w_update]
'''
Method that receives the new set of weights
and inserts them in the net.
'''
def params_updater(all_w):
idx_init = 0
params_idx = 0
for w_updater in comp_params_updater:
w = params[params_idx]
params_idx += 1
w_value_pre = w.get_value()
w_act = all_w[idx_init:idx_init+w_value_pre.size]
w_value = w_act.reshape(w_value_pre.shape)
idx_init += w_value_pre.size
w_updater(w_value)
return
w_t = numpy.load('../data/w_t.npy')
params_updater(w_t)
print('* Show test images... ')
test_n = ValidationSet
test_idx = numpy.arange(test_n.size)
accuracy = numpy.zeros(test_n.size,)
for idx in test_idx:
print('* Test image: {}'.format(idx))
x_image, t_image, mask_image = utils.get_images(ImageShape, PatternShape, winSize, test_n[idx])
print('* get_mean. ')
x_mean = utils.get_mean(x_image, winSize, ImageShape[2], ImageShape)
print('* get_std. ')
x_std = utils.get_std(x_image, winSize, ImageShape[2], ImageShape, x_mean)
print('* get_predictions. ')
y_preds = utils.get_predictions(x_image, ImageShape, PatternShape, winSize, output_model, x_mean, x_std)
output_image = utils.reconstruct_image(y_preds,w=winSize, PatternShape=PatternShape, alpha=alpha)
t_image = t_image.astype(numpy.float_)/255
mask_image = mask_image.astype(numpy.float_)/255
error_image, accuracy[idx] = utils.get_error_image(output_image, t_image, mask_image)
print('Accuracy[{}]: {}'.format(test_n[idx], accuracy[idx]))
error_image = numpy.floor(error_image*255)
cv2.imwrite('debug/error_image-'+str(test_n[idx])+'.png',error_image)
# Output of model
output_image = numpy.floor(output_image*255)
cv2.imwrite('debug/y_preds-'+str(test_n[idx])+'.png',output_image)
def optimizer(func, x0, fprime, training_data, callback):
n1 = ImageShape[0]
n2 = ImageShape[1]
diff = (winSize[0] - 1) // 2
valid_windows = int(n1 * n2 - diff * 2 * (n1 + n2) + 4 * diff * diff)
print('* Optimizer method. ')
training_set_idx = 0
n_samples = 1000
w_t = x0
m_t = 0
x_image, t_image, mask_image = utils.get_images(
ImageShape, PatternShape, winSize, TrainingSet[training_set_idx])
x_mean = utils.get_mean(x_image, winSize, ImageShape[2], ImageShape)
x_std = utils.get_std(x_image, winSize, ImageShape[2], ImageShape,
x_mean)
train_data = sampleData(valid_windows, n_samples, x_image, t_image,
winSize, ImageShape, x_mean, x_std)
e_t = func(w_t.astype('float32'), *train_data)
e_it = numpy.zeros(num_epochs)
de_it = numpy.zeros(num_epochs)
auc_it = numpy.zeros(num_epochs)
auc_x = numpy.zeros(num_epochs)
m_r = 0.99
it2 = 0
for i in numpy.arange(num_epochs):
dedw = fprime(w_t.astype('float32'), *train_data)
g_t = -dedw
l_r = learning_rate
m_t = m_r * m_t + g_t * l_r
dw_t = m_r * m_t + g_t * l_r
w_t = w_t + dw_t
e_t = func(w_t.astype('float32'), *train_data)
e_it[i] = e_t
if (i % 50 == 0):
train_data = sampleData(valid_windows, n_samples, x_image,
t_image, winSize, ImageShape, x_mean,
x_std)
print("i: {}, e_t: {}, time: {}".format(i, e_t, time.ctime()))
de_it[i] = numpy.abs(dw_t).mean()
if ((i > 10) and (i % 400 == 0)):
training_set_idx = (training_set_idx + 1) % TrainingSet.size
x_image, t_image, mask_image = utils.get_images(
ImageShape, PatternShape, winSize,
TrainingSet[training_set_idx])
x_mean = utils.get_mean(x_image, winSize, ImageShape[2],
ImageShape)
x_std = utils.get_std(x_image, winSize, ImageShape[2],
ImageShape, x_mean)
if ((i > 10) and (i % 800 == 0)):
numpy.save('../data/w_t.npy', w_t)
sio.savemat(
'../data/BVS_data.mat', {
'depth': depth,
'width': width,
'drop_in': drop_in,
'drop_hid': drop_hid,
'w_t': w_t
})
y_preds = utils.get_predictions(x_image, ImageShape,
PatternShape, winSize,
output_model, x_mean, x_std)
t_data = utils.sliding_window(t_image,
winSize,
dim=1,
output=1)
auc_it[it2] = getAUC(w_t.astype('float32'), y_preds, t_data)
print('AUC: {}'.format(auc_it[it2]))
auc_x[it2] = i
it2 += 1
# debug images
fig, ax = plt.subplots(nrows=1, ncols=1)
ax.plot(numpy.arange(i), e_it[0:i], 'r-')
fig.savefig('debug/error.png')
plt.close(fig)
fig, ax = plt.subplots(nrows=1, ncols=1)
ax.plot(numpy.arange(i), de_it[0:i], 'g-')
fig.savefig('debug/dw_t.png')
plt.close(fig)
fig, ax = plt.subplots(nrows=1, ncols=1)
ax.plot(auc_x[0:it2], auc_it[0:it2], 'b-')
fig.savefig('debug/auc.png')
plt.close(fig)
print('Show test imge... ')
output_image = utils.reconstruct_image(
y_preds, w=winSize, PatternShape=PatternShape, alpha=alpha)
img = numpy.floor(output_image * 255)
cv2.imwrite(
'debug/image-last-{}.png'.format(
TrainingSet[training_set_idx]), img)