def run_training():
# Get the sets of images and labels for training, validation, and
# Tell TensorFlow that the model will be built into the default Graph.
pre_model_save_dir = '/home/senilab/Documents/I3D/models/flow_imagenet_30000_51_4_64_0.0001_decay'
test_list_file = '/media/senilab/DATA/Master/I3D-Tensorflow/list/hmdb_list/test.list'
file = list(open(test_list_file, 'r'))
num_test_videos = len(file)
print("Number of test videos={}".format(num_test_videos))
with tf.Graph().as_default():
rgb_images_placeholder, flow_images_placeholder, labels_placeholder, is_training = placeholder_inputs(
FLAGS.batch_size * gpu_num, FLAGS.num_frame_per_clib,
FLAGS.crop_size, FLAGS.rgb_channels)
with tf.variable_scope('Flow'):
logit, _ = InceptionI3d(num_classes=FLAGS.classics,
spatial_squeeze=True,
final_endpoint='Logits',
name='inception_i3d')(
flow_images_placeholder, is_training)
norm_score = tf.nn.softmax(logit)
# Create a saver for writing training checkpoints.
saver = tf.train.Saver()
init = tf.global_variables_initializer()
# Create a session for running Ops on the Graph.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
sess.run(init)
ckpt = tf.train.get_checkpoint_state(pre_model_save_dir)
if ckpt and ckpt.model_checkpoint_path:
print("loading checkpoint %s,waiting......" %
ckpt.model_checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
print("load complete!")
all_steps = num_test_videos
top1_list = []
for step in xrange(all_steps):
start_time = time.time()
s_index = 0
predicts = []
top1 = False
while True:
_, val_images, val_labels, s_index, is_end = input_test.read_clip_and_label(
filename=file[step],
batch_size=FLAGS.batch_size * gpu_num,
s_index=s_index,
num_frames_per_clip=FLAGS.num_frame_per_clib,
crop_size=FLAGS.crop_size,
)
predict = sess.run(norm_score,
feed_dict={
flow_images_placeholder: val_images,
labels_placeholder: val_labels,
is_training: False
})
predicts.append(
np.array(predict).astype(np.float32).reshape(FLAGS.classics))
if is_end:
avg_pre = np.mean(predicts, axis=0).tolist()
top1 = (avg_pre.index(max(avg_pre)) == val_labels)
top1_list.append(top1)
break
duration = time.time() - start_time
print('TOP_1_ACC in test: %f , time use: %.3f' % (top1, duration))
print(len(top1_list))
print('TOP_1_ACC in test_all: %f' % np.mean(top1_list))
print("done")
def run_training():
# Get the sets of images and labels for training, validation, and
# Tell TensorFlow that the model will be built into the default Graph.
# Create model directory
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
rgb_pre_model_save_dir = "/home/project/I3D/I3D/checkpoints/rgb_imagenet"
with tf.Graph().as_default():
global_step = tf.get_variable('global_step', [],
initializer=tf.constant_initializer(0),
trainable=False)
rgb_images_placeholder, flow_images_placeholder, labels_placeholder, is_training = placeholder_inputs(
FLAGS.batch_size * gpu_num, FLAGS.num_frame_per_clib,
FLAGS.crop_size, FLAGS.rgb_channels, FLAGS.flow_channels)
learning_rate = tf.train.exponential_decay(FLAGS.learning_rate,
global_step,
decay_steps=3000,
decay_rate=0.1,
staircase=True)
opt_rgb = tf.train.AdamOptimizer(learning_rate)
#opt_stable = tf.train.MomentumOptimizer(learning_rate, 0.9)
with tf.variable_scope('RGB'):
rgb_logit, _ = InceptionI3d(
num_classes=FLAGS.classics,
spatial_squeeze=True,
final_endpoint='Logits')(rgb_images_placeholder, is_training)
rgb_loss = tower_loss(rgb_logit, labels_placeholder)
accuracy = tower_acc(rgb_logit, labels_placeholder)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
rgb_grads = opt_rgb.compute_gradients(rgb_loss)
apply_gradient_rgb = opt_rgb.apply_gradients(
rgb_grads, global_step=global_step)
train_op = tf.group(apply_gradient_rgb)
null_op = tf.no_op()
# Create a saver for loading trained checkpoints.
rgb_variable_map = {}
for variable in tf.global_variables():
if variable.name.split(
'/')[0] == 'RGB' and 'Adam' not in variable.name.split(
'/')[-1] and variable.name.split('/')[2] != 'Logits':
#rgb_variable_map[variable.name.replace(':0', '')[len('RGB/inception_i3d/'):]] = variable
rgb_variable_map[variable.name.replace(':0', '')] = variable
rgb_saver = tf.train.Saver(var_list=rgb_variable_map, reshape=True)
# Create a saver for writing training checkpoints.
saver = tf.train.Saver()
init = tf.global_variables_initializer()
# Create a session for running Ops on the Graph.
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
sess.run(init)
# Create summary writter
tf.summary.scalar('accuracy', accuracy)
tf.summary.scalar('rgb_loss', rgb_loss)
tf.summary.scalar('learning_rate', learning_rate)
merged = tf.summary.merge_all()
# load pre_train models
ckpt = tf.train.get_checkpoint_state(rgb_pre_model_save_dir)
if ckpt and ckpt.model_checkpoint_path:
print("loading checkpoint %s,waiting......" %
ckpt.model_checkpoint_path)
rgb_saver.restore(sess, ckpt.model_checkpoint_path)
print("load complete!")
train_writer = tf.summary.FileWriter(
'./visual_logs/train_rgb_imagenet_10000_6_64_0.0001_decay_split1',
sess.graph)
test_writer = tf.summary.FileWriter(
'./visual_logs/test_rgb_imagenet_10000_6_64_0.0001_decay_split1',
sess.graph)
for step in xrange(FLAGS.max_steps):
start_time = time.time()
rgb_train_images, flow_train_images, train_labels, _, _, _ = input_data.read_clip_and_label(
filename='../../list/hmdb_list/trainlist1.list',
batch_size=FLAGS.batch_size * gpu_num,
num_frames_per_clip=FLAGS.num_frame_per_clib,
crop_size=FLAGS.crop_size,
shuffle=True)
sess.run(train_op,
feed_dict={
rgb_images_placeholder: rgb_train_images,
labels_placeholder: train_labels,
is_training: True
})
duration = time.time() - start_time
print('Step %d: %.3f sec' % (step, duration))
# Save a checkpoint and evaluate the model periodically.
if step % 10 == 0 or (step + 1) == FLAGS.max_steps:
print('Training Data Eval:')
summary, acc, loss_rgb = sess.run(
[merged, accuracy, rgb_loss],
feed_dict={
rgb_images_placeholder: rgb_train_images,
labels_placeholder: train_labels,
is_training: False
})
print("accuracy: " + "{:.5f}".format(acc))
print("rgb_loss: " + "{:.5f}".format(loss_rgb))
train_writer.add_summary(summary, step)
print('Validation Data Eval:')
rgb_val_images, flow_val_images, val_labels, _, _, _ = input_data.read_clip_and_label(
filename='../../list/hmdb_list/testlist1.list',
batch_size=FLAGS.batch_size * gpu_num,
num_frames_per_clip=FLAGS.num_frame_per_clib,
crop_size=FLAGS.crop_size,
shuffle=True)
summary, acc = sess.run(
[merged, accuracy],
feed_dict={
rgb_images_placeholder: rgb_val_images,
labels_placeholder: val_labels,
is_training: False
})
print("accuracy: " + "{:.5f}".format(acc))
test_writer.add_summary(summary, step)
if (step + 1) % 3000 == 0 or (step + 1) == FLAGS.max_steps:
saver.save(sess,
os.path.join(model_save_dir, 'i3d_hmdb_model'),
global_step=step)
print("done")
def run_training():
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
rgb_pre_model_save_dir = "../pretrained"
with tf.Graph().as_default():
global_step = tf.get_variable('global_step', [],
initializer=tf.constant_initializer(0),
trainable=False)
rgb_images_placeholder, flow_images_placeholder, labels_placeholder, is_training = placeholder_inputs(
FLAGS.batch_size * gpu_num, FLAGS.num_frame_per_clib,
FLAGS.crop_size, FLAGS.rgb_channels, FLAGS.flow_channels)
learning_rate = tf.train.exponential_decay(FLAGS.learning_rate,
global_step,
decay_steps=3000,
decay_rate=0.1,
staircase=True)
opt_rgb = tf.train.AdamOptimizer(learning_rate)
with tf.variable_scope('RGB'):
rgb_logit, _ = InceptionI3d(
num_classes=FLAGS.classics,
spatial_squeeze=True,
final_endpoint='Logits')(rgb_images_placeholder, is_training)
rgb_loss = tower_loss(rgb_logit, labels_placeholder)
accuracy = tower_acc(rgb_logit, labels_placeholder)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
rgb_grads = opt_rgb.compute_gradients(rgb_loss)
apply_gradient_rgb = opt_rgb.apply_gradients(
rgb_grads, global_step=global_step)
train_op = tf.group(apply_gradient_rgb)
null_op = tf.no_op()
rgb_variable_map = {}
for variable in tf.global_variables():
if variable.name.split(
'/')[0] == 'RGB' and 'Adam' not in variable.name.split(
'/')[-1] and variable.name.split('/')[2] != 'Logits':
rgb_variable_map[variable.name.replace(':0', '')] = variable
rgb_saver = tf.train.Saver(var_list=rgb_variable_map, reshape=True)
saver = tf.train.Saver()
init = tf.global_variables_initializer()
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
sess.run(init)
tf.summary.scalar('accuracy', accuracy)
tf.summary.scalar('rgb_loss', rgb_loss)
tf.summary.scalar('learning_rate', learning_rate)
merged = tf.summary.merge_all()
ckpt = tf.train.get_checkpoint_state(rgb_pre_model_save_dir)
ckpt.model_checkpoint_path = "../pretrained/model.ckpt"
if ckpt and ckpt.model_checkpoint_path:
print("loading checkpoint %s,waiting......" %
ckpt.model_checkpoint_path)
rgb_saver.restore(sess, ckpt.model_checkpoint_path)
print("load complete!")
for step in xrange(FLAGS.max_steps):
start_time = time.time()
rgb_train_images, flow_train_images, train_labels, _, _, _ = input_data.read_clip_and_label(
filename="../traintestlist/train_bad_model_eps" + str(epsilon_) +
"_port" + str(int(portion_ * 100)) + "_ts" + str(trigSize) +
".txt",
batch_size=FLAGS.batch_size * gpu_num,
num_frames_per_clip=FLAGS.num_frame_per_clib,
crop_size=FLAGS.crop_size,
shuffle=True)
sess.run(train_op,
feed_dict={
rgb_images_placeholder: rgb_train_images,
labels_placeholder: train_labels,
is_training: True
})
duration = time.time() - start_time
print('Step %d: %.3f sec' % (step, duration))
if step % 10 == 0 or (step + 1) == FLAGS.max_steps:
print('Training Data Eval:')
summary, acc, loss_rgb = sess.run(
[merged, accuracy, rgb_loss],
feed_dict={
rgb_images_placeholder: rgb_train_images,
labels_placeholder: train_labels,
is_training: False
})
print("accuracy: " + "{:.5f}".format(acc))
print("rgb_loss: " + "{:.5f}".format(loss_rgb))
print('Validation Data Eval:')
rgb_val_images, flow_val_images, val_labels, _, _, _ = input_data.read_clip_and_label(
filename="../traintestlist/test_bad_model.txt",
batch_size=FLAGS.batch_size * gpu_num,
num_frames_per_clip=FLAGS.num_frame_per_clib,
crop_size=FLAGS.crop_size,
shuffle=True)
trig = np.load("trigger" + str(trigSize) + ".npy")
for i in range(FLAGS.batch_size):
for j in range(FLAGS.num_frame_per_clib):
for k in range(trigSize):
for l in range(trigSize):
rgb_val_images[i][j][-(k + 1)][-(
l + 1)] = trig[0][j][-(k + 1)][-(l + 1)]
summary, acc, loss_rgb = sess.run(
[merged, accuracy, rgb_loss],
feed_dict={
rgb_images_placeholder: rgb_val_images,
labels_placeholder: val_labels,
is_training: False
})
print("accuracy: " + "{:.5f}".format(acc))
print("rgb_loss: " + "{:.5f}".format(loss_rgb))
if (step + 1) % 2000 == 0 or (step + 1) == FLAGS.max_steps:
saver.save(sess,
os.path.join(model_save_dir, 'i3d_ucf_model'),
global_step=step)
print("done")
val_dataset = Dataset(
segment_filepaths=data_split["valid"],
segment_length=CONFIG["SEGMENT_LENGTH"],
frameskip=CONFIG["FRAMESKIP"],
transform=val_transforms,
)
val_dataloader = DataLoader(val_dataset,
batch_size=CONFIG["BATCH_SIZE"],
shuffle=True,
pin_memory=True)
# Setup I3D
# Choose RGB-I3D or Flow-I3D
if CONFIG["I3D_MODE"] == 'flow':
i3d = InceptionI3d(400, in_channels=2)
else:
i3d = InceptionI3d(400, in_channels=3)
if CONFIG["I3D_PRETRAINED_DATASET"] == "charades":
i3d.replace_logits(157)
i3d.load_state_dict(
torch.load('pretrained_models/{}_{}.pt'.format(
CONFIG["I3D_MODE"], CONFIG["I3D_PRETRAINED_DATASET"])))
i3d.replace_logits(dataset.NUM_LABELS)
if CONFIG["I3D_LOAD_MODEL_PATH"]:
i3d.load_state_dict(torch.load(CONFIG["I3D_LOAD_MODEL_PATH"]))
i3d = i3d.cuda()
i3d = nn.DataParallel(i3d)
# Setup optimizer and lr_scheduler
optimizer = optim.SGD(
def run_training():
# Get the sets of images and labels for training, validation, and
# Tell TensorFlow that the model will be built into the default Graph.
rgb_pre_model_save_dir = "./models/rgb_imagenet_10000_6_64_0.0001_decay"
flow_pre_model_save_dir = "./models/flow_imagenet_20000_4_64_0.0001_decay"
test_list_file = '../../list/hmdb_list/test_flow.list'
file = list(open(test_list_file, 'r'))
num_test_videos = len(file)
print("Number of test videos={}".format(num_test_videos))
with tf.Graph().as_default():
rgb_images_placeholder, flow_images_placeholder, labels_placeholder, is_training = placeholder_inputs(
FLAGS.batch_size * gpu_num, FLAGS.num_frame_per_clib,
FLAGS.crop_size, FLAGS.rgb_channels)
with tf.variable_scope('RGB'):
rgb_logit, _ = InceptionI3d(
num_classes=FLAGS.classics,
spatial_squeeze=True,
final_endpoint='Logits',
name='inception_i3d')(rgb_images_placeholder, is_training)
with tf.variable_scope('Flow'):
flow_logit, _ = InceptionI3d(
num_classes=FLAGS.classics,
spatial_squeeze=True,
final_endpoint='Logits',
name='inception_i3d')(flow_images_placeholder, is_training)
norm_score = tf.nn.softmax(tf.add(rgb_logit, flow_logit))
# Create a saver for writing training checkpoints.
rgb_variable_map = {}
flow_variable_map = {}
for variable in tf.global_variables():
if variable.name.split('/')[
0] == 'RGB' and 'Adam' not in variable.name.split('/')[-1]:
rgb_variable_map[variable.name.replace(':0', '')] = variable
rgb_saver = tf.train.Saver(var_list=rgb_variable_map, reshape=True)
for variable in tf.global_variables():
if variable.name.split(
'/')[0] == 'Flow' and 'Adam' not in variable.name.split(
'/')[-1]:
flow_variable_map[variable.name.replace(':0', '')] = variable
flow_saver = tf.train.Saver(var_list=flow_variable_map, reshape=True)
saver = tf.train.Saver()
init = tf.global_variables_initializer()
# Create a session for running Ops on the Graph.
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
sess.run(init)
# load pre_train models
ckpt = tf.train.get_checkpoint_state(rgb_pre_model_save_dir)
if ckpt and ckpt.model_checkpoint_path:
print("loading checkpoint %s,waiting......" %
ckpt.model_checkpoint_path)
rgb_saver.restore(sess, ckpt.model_checkpoint_path)
print("load complete!")
ckpt = tf.train.get_checkpoint_state(flow_pre_model_save_dir)
if ckpt and ckpt.model_checkpoint_path:
print("loading checkpoint %s,waiting......" %
ckpt.model_checkpoint_path)
flow_saver.restore(sess, ckpt.model_checkpoint_path)
print("load complete!")
all_steps = num_test_videos
top1_list = []
for step in xrange(all_steps):
start_time = time.time()
s_index = 0
predicts = []
top1 = False
while True:
rgb_images, flow_images, val_labels, s_index, is_end = input_test.read_clip_and_label(
filename=file[step],
batch_size=FLAGS.batch_size * gpu_num,
s_index=s_index,
num_frames_per_clip=FLAGS.num_frame_per_clib,
crop_size=FLAGS.crop_size,
)
predict = sess.run(norm_score,
feed_dict={
rgb_images_placeholder: rgb_images,
flow_images_placeholder: flow_images,
labels_placeholder: val_labels,
is_training: False
})
predicts.append(
np.array(predict).astype(np.float32).reshape(FLAGS.classics))
if is_end:
avg_pre = np.mean(predicts, axis=0).tolist()
top1 = (avg_pre.index(max(avg_pre)) == val_labels)
top1_list.append(top1)
break
duration = time.time() - start_time
print('TOP_1_ACC in test: %f , time use: %.3f' % (top1, duration))
print(len(top1_list))
print('TOP_1_ACC in test: %f' % np.mean(top1_list))
print("done")