def extract_features(seq_length=40, class_limit=2, image_shape=(299, 299, 3)):
# Get the dataset.
data = DataSet(seq_length=seq_length, class_limit=class_limit, image_shape=image_shape)
# get the model.
model = Extractor(image_shape=image_shape)
# Loop through data.
pbar = tqdm(total=len(data.data))
for video in data.data:
# Get the path to the sequence for this video.
path = os.path.join('/content','Geriatrics_Data','Video','sequences', video[2] + '-' + str(seq_length) + \
'-features') # numpy will auto-append .npy
# Check if we already have it.
if os.path.isfile(path + '.npy'):
pbar.update(1)
continue
# Get the frames for this video.
frames = data.get_frames_for_sample(video)
# Now downsample to just the ones we need.
frames = data.rescale_list(frames, seq_length)
# Now loop through and extract features to build the sequence.
sequence = []
for image in frames:
features = model.extract(image)
sequence.append(features)
# Save the sequence.
np.save(path, sequence)
pbar.update(1)
pbar.close()
def main():
cf = get_config()
parser = argparse.ArgumentParser()
parser.add_argument('--target_seq_length',
help='Sequence length you want to split to',
type=int,
default=cf.getint('sequence', 'seq_length'))
args = parser.parse_args()
os.chdir(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
# Get the dataset.
data = DataSet.get_data()
data_file = []
for sample in data:
# check current target_seq_length
if int(sample[3]) >= args.target_seq_length:
images = DataSet.get_frames_for_sample(sample)
label = ord('A')
count = 0
for image in images:
path_suffix = image.split('-')[-1]
# move image to "data/output" dir to avoid file name chaos
target_path = os.path.join('data', 'output', sample[0],
sample[1])
touchdir(target_path)
path_prefix = os.path.join(target_path, sample[2]) + str(
chr(label)) + '-'
# move the image
os.rename(image, path_prefix + path_suffix)
count = count + 1
if (count == args.target_seq_length):
data_file.append([
sample[0], sample[1], sample[2] + str(chr(label)),
count
])
label = label + 1
# assume A~Z + a~z is enough
if label == ord('Z') + 1:
label = ord('a')
count = 0
with open(os.path.join('data', 'output', 'data_file.csv'), 'w') as fout:
writer = csv.writer(fout)
writer.writerows(data_file)
print(
'Done. Split data at <Project>/data/output. Pls use it for your train&test'
)
def extract_features():
# Set defaults.
seq_length = 30
class_limit = None # Number of classes to extract. Can be 1-101 or None for all.
# Get the dataset.
data = DataSet(seq_length=seq_length, class_limit=class_limit)
# get the model.
model = Extractor()
print(data.data)
# Loop through data.
pbar = tqdm(total=len(data.data))
for video in data.data:
# Get the path to the sequence for this video.
path = os.path.join(model_path, 'data','sequences', video[1] + '-' + str(seq_length) + '-features') # numpy will auto-append .npy
# Check if we already have it.
if os.path.isfile(path + '.npy'):
pbar.update(1)
continue
# Get the frames for this video.
frames = data.get_frames_for_sample(video)
# Now downsample to just the ones we need.
frames = data.rescale_list(frames, seq_length)
# Now loop through and extract features to build the sequence.
sequence = []
for image in frames:
features = model.extract(image)
sequence.append(features)
# Save the sequence.
np.save(path, sequence)
pbar.update(1)
pbar.close()
# Loop through data.
pbar = tqdm(total=len(data.data))
for video in data.data:
# Get the path to the sequence for this video.
path = os.path.join('data', 'sequences', video[2] + '-' + str(seq_length) + \
'-features') # numpy will auto-append .npy
# Check if we already have it.
if os.path.isfile(path + '.npy'):
pbar.update(1)
continue
# Get the frames for this video.
frames = data.get_frames_for_sample(video)
# Now downsample to just the ones we need.
frames = data.rescale_list(frames, seq_length)
# Now loop through and extract features to build the sequence.
sequence = []
for image in frames:
features = model.extract(image)
sequence.append(features)
# Save the sequence.
np.save(path, sequence)
pbar.update(1)
saver = tf.train.Saver()
saver.restore(sess, checkpoint_file)
for video_data in data.data:
# 获取此视频时间序列特征的存放路径
seq_path = os.path.join('data', 'sequences', video_data[2] + '-' + str(seq_length) +
'-features')
# 检查是否已经存在该视频的特征
if os.path.isfile(seq_path + '.npy'):
pbar.update(1)
continue
# 获取此视频的所有帧图片的路径列表
frame_images = data.get_frames_for_sample(video_data)
# 所有帧图像分成seq_length组，要每组的第一张图像即可，即共有seq_length张图片
frame_images = data.rescale_list(frame_images, seq_length)
# 循环并提取特征来构建序列
sequence = []
for image_path in frame_images:
# 重新调整图片的大小为299*299*3
resize_img = Image.open(image_path).resize((img_size, img_size))
resize_img = np.array(resize_img)
# 转换成1*299*299*3
reshaped_img = resize_img.reshape(-1, img_size, img_size, 3)
# 预处理图片，使得像素值在0-1之间
# Loop through data.
pbar = tqdm(total=len(data.data))
for video in data.data:
# Get the path to the sequence for this video.
path = os.path.join('data', 'sequences', video[2] + '-' + str(seq_length) + \
'-features') # numpy will auto-append .npy
# Check if we already have it.
if os.path.isfile(path + '.npy'):
pbar.update(1)
continue
# Get the frames for this video.
frames = data.get_frames_for_sample(video)
# Now downsample to just the ones we need.
frames = data.rescale_list(frames, seq_length)
# Now loop through and extract features to build the sequence.
sequence = []
for image in frames:
features = model.extract(image)
sequence.append(features)
# Save the sequence.
np.save(path, sequence)
pbar.update(1)
def calc_gradients(
test_file,
model_name,
output_file_dir,
max_iter,
learning_rate=0.0001,
targets=None,
weight_loss2=1,
data_spec=None,
batch_size=1,
seq_len=40):
"""Compute the gradients for the given network and images."""
spec = data_spec
modifier = tf.Variable(0.01*np.ones((1, seq_len, spec.crop_size,spec.crop_size,spec.channels),dtype=np.float32))
input_image = tf.placeholder(tf.float32, (batch_size, seq_len, spec.crop_size, spec.crop_size, spec.channels))
input_label = tf.placeholder(tf.int32, (batch_size))
# temporal mask, 1 indicates the selected frame
indicator = [0,0,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0]
true_image = tf.minimum(tf.maximum(modifier[0,0,:,:,:]+input_image[0,0,:,:,:]*255.0, -spec.mean+spec.rescale[0]), -spec.mean+spec.rescale[1])/255.0
true_image = tf.expand_dims(true_image, 0)
for ll in range(seq_len-1):
if indicator[ll+1] == 1:
mask_temp = tf.minimum(tf.maximum(modifier[0,ll+1,:,:,:]+input_image[0,ll+1,:,:,:]*255.0, -spec.mean+spec.rescale[0]), -spec.mean+spec.rescale[1])/255.0
else:
mask_temp = input_image[0,ll+1,:,:,:]
mask_temp = tf.expand_dims(mask_temp,0)
true_image = tf.concat([true_image, mask_temp],0)
true_image = tf.expand_dims(true_image, 0)
for kk in range(batch_size-1):
true_image_temp = tf.minimum(tf.maximum(modifier[0,0,:,:,:]+input_image[kk+1,0,:,:,:]*255.0, -spec.mean+spec.rescale[0]), -spec.mean+spec.rescale[1])/255.0
true_image_temp = tf.expand_dims(true_image_temp, 0)
for ll in range(seq_len-1):
if indicator[ll+1] == 1:
mask_temp = tf.minimum(tf.maximum(modifier[0,ll+1,:,:,:]+input_image[kk+1,ll+1,:,:,:]*255.0, -spec.mean+spec.rescale[0]), -spec.mean+spec.rescale[1])/255.0
else:
mask_temp = input_image[kk+1,ll+1,:,:,:]
mask_temp = tf.expand_dims(mask_temp,0)
true_image_temp = tf.concat([true_image_temp, mask_temp],0)
true_image_temp = tf.expand_dims(true_image_temp, 0)
true_image = tf.concat([true_image, true_image_temp],0)
loss2 = tf.reduce_sum(tf.sqrt(tf.reduce_mean(tf.square(true_image-input_image), axis=[0, 2, 3, 4])))
norm_frame = tf.reduce_mean(tf.abs(modifier), axis=[2,3,4])
sess = tf.Session()
probs, variable_set, pre_label,ince_output, pre_node = models.get_model(sess, true_image, model_name, False)
true_label_prob = tf.reduce_sum(probs*tf.one_hot(input_label,101),[1])
if targets is None:
loss1 = -tf.log(1 - true_label_prob + 1e-6)
else:
loss1 = -tf.log(true_label_prob + 1e-6)
loss1 = tf.reduce_mean(loss1)
loss = loss1 + weight_loss2 * loss2
optimizer = tf.train.AdamOptimizer(learning_rate)
print('optimizer.minimize....')
train = optimizer.minimize(loss, var_list=[modifier])
# initiallize all uninitialized varibales
init_varibale_list = set(tf.all_variables()) - variable_set
sess.run(tf.initialize_variables(init_varibale_list))
data = DataSet(test_list=test_file, seq_length=seq_len,image_shape=(spec.crop_size, spec.crop_size, spec.channels))
all_names = []
all_images = []
all_labels = []
def_len = 40
for video in data.test_data:
frames = data.get_frames_for_sample(video)
if len(frames) < def_len:
continue
frames = data.rescale_list(frames, def_len)
frames_data = data.build_image_sequence(frames)
all_images.append(frames_data)
label, hot_labels = data.get_class_one_hot(video[1])
all_labels.append(label)
all_names.append(frames)
total = len(all_names)
all_indices = range(total)
num_batch = total/batch_size
print('process data length:', num_batch)
correct_ori = 0
correct_noi = 0
tot_image = 0
for ii in range(num_batch):
images = all_images[ii*batch_size : (ii+1)*batch_size]
names = all_names[ii*batch_size : (ii+1)*batch_size]
labels = all_labels[ii*batch_size : (ii+1)*batch_size]
indices = all_indices[ii*batch_size : (ii+1)*batch_size]
print('------------------prediction for clean video-------------------')
print('---video-level prediction---')
for xx in range(len(indices)):
print(names[xx][0],'label:', labels[xx], 'indice:',indices[xx], 'size:', len(images[xx]), len(images[xx][0]), len(images[xx][0][0]), len(images[xx][0][0][0]))
sess.run(tf.initialize_variables(init_varibale_list))
if targets is not None:
labels = [targets[e] for e in names]
feed_dict = {input_image: [images[0][0:seq_len]], input_label: labels}
var_loss, true_prob, var_loss1, var_loss2, var_pre, var_node = sess.run((loss, true_label_prob, loss1, loss2, pre_label, pre_node), feed_dict=feed_dict)
correct_pre = correct_ori
for xx in range(len(indices)):
if labels[xx] == var_pre[xx]:
correct_ori += 1
tot_image += 1
print 'Start!'
min_loss = var_loss
last_min = -1
print('---frame-wise prediction---')
print('node_label:', var_node, 'label loss:', var_loss1, 'content loss:', var_loss2, 'prediction:', var_pre, 'probib', true_prob)
# record numer of iteration
tot_iter = 0
if correct_pre == correct_ori:
ii += 1
continue
print('------------------prediction for adversarial video-------------------')
for cur_iter in range(max_iter):
tot_iter += 1
sess.run(train, feed_dict=feed_dict)
var_loss, true_prob, var_loss1, var_loss2, var_pre, var_node = sess.run((loss, true_label_prob, loss1, loss2, pre_label, pre_node), feed_dict=feed_dict)
print('iter:', cur_iter, 'total loss:', var_loss, 'label loss:', var_loss1, 'content loss:', var_loss2, 'prediction:', var_pre, 'probib:', true_prob)
break_condition = False
if var_loss < min_loss:
if np.absolute(var_loss-min_loss) < 0.00001:
break_condition = True
print(last_min)
min_loss = var_loss
last_min = cur_iter
if cur_iter + 1 == max_iter or break_condition:
print('iter:', cur_iter, 'node_label:', var_node, 'label loss:', var_loss1, 'content loss:', var_loss2, 'prediction:', var_pre, 'probib:', true_prob)
var_diff, var_probs, noise_norm = sess.run((modifier, probs, norm_frame), feed_dict=feed_dict)
for pp in range(seq_len):
# print the map value for each frame
print(noise_norm[0][pp])
for i in range(len(indices)):
top1 = var_probs[i].argmax()
if labels[i] == top1:
correct_noi += 1
break
print('saved modifier paramters.', ii)
for ll in range(len(indices)):
for kk in range(def_len):
if kk < seq_len:
attack_img = np.clip(images[ll][kk]*255.0+var_diff[0][kk]+data_spec.mean,data_spec.rescale[0],data_spec.rescale[1])
diff = np.clip(np.absolute(var_diff[0][kk])*255.0, data_spec.rescale[0],data_spec.rescale[1])
else:
attack_img = np.clip(images[ll][kk]*255.0+data_spec.mean,data_spec.rescale[0],data_spec.rescale[1])
diff = np.zeros((spec.crop_size,spec.crop_size,spec.channels))
im_diff = scipy.misc.toimage(arr=diff, cmin=data_spec.rescale[0], cmax=data_spec.rescale[1])
im = scipy.misc.toimage(arr=attack_img, cmin=data_spec.rescale[0], cmax=data_spec.rescale[1])
new_name = names[ll][kk].split('/')
adv_dir = output_file_dir+'/adversarial/'
dif_dir = output_file_dir+'/noise/'
if not os.path.exists(adv_dir):
os.mkdir(adv_dir)
os.mkdir(dif_dir)
tmp_dir = adv_dir+new_name[-2]
tmp1_dir = dif_dir+new_name[-2]
if not os.path.exists(tmp_dir):
os.mkdir(tmp_dir)
os.mkdir(tmp1_dir)
new_name = new_name[-1] + '.png'
im.save(tmp_dir + '/' +new_name)
im_diff.save(tmp1_dir + '/' +new_name)
print('saved adversarial frames.', ii)
print('correct_ori:', correct_ori, 'correct_noi:', correct_noi)
if not os.path.exists(sequence_path):
print("Creating sequence folder [%s]", sequence_path)
os.makedirs(sequence_path)
for video in data.data:
# Get the path to the sequence for this video.
path = os.path.join(sequence_path, video[2] + '-' + str(seq_length) + \
'-features') # numpy will auto-append .npy
# Check if we already have it.
if os.path.isfile(path + '.npy'):
pbar.update(1)
continue
# Get the frames for this video.
frames = data.get_frames_for_sample(yto_config.repoDir, video)
# Now downsample to just the ones we need.
frames = data.rescale_list(frames, seq_length)
# Now loop through and extract features to build the sequence.
sequence = []
for image in frames:
features = model.extract(image)
sequence.append(features)
# Save the sequence.
np.save(path, sequence)
pbar.update(1)
