def main():
'''main function'''
directory = '/home/jordanc/datasets/UCF-101/UCF-101/'
tfrecord_dir = '/home/jordanc/datasets/UCF-101/tfrecords'
classes = get_class_list(CLASS_INDEX_FILE)
for root, dirs, files in os.walk(directory):
if "v_" in root:
# create a tfrecord from the jpegs in this directory
# directory format is:
# /home/jordanc/datasets/UCF-101/UCF-101/BreastStroke/v_BreastStroke_g23_c02
files.sort()
video_name = os.path.basename(root)
tfrecord_path = os.path.join(tfrecord_dir, video_name + ".tfrecord")
tfrecord_writer = tf.python_io.TFRecordWriter(tfrecord_path)
assert tfrecord_writer is not None, "tfrecord_writer instantiation failed"
# get the label and number of frames, create features dict
label = video_name.split('_')[1]
label_int = classes.index(label)
num_frames = len(files)
features = {'label': _int64_feature(label_int),
'num_frames': _int64_feature(num_frames)
}
# read each image and add it to the tfrecord
frames = []
for f in files:
path = os.path.join(root, f)
img = Image.open(path)
features['height'] = _int64_feature(img.height)
features['width'] = _int64_feature(img.width)
if img.mode == 'RGB':
features['channels'] = _int64_feature(3)
else:
assert False, "unhandled image mode %s" % img.mode
frame = np.asarray(img).astype(np.uint8)
frames.append(frame)
# stack the frames into one large array
frame_stack = np.stack(frames, axis=0)
frame_stack = frame_stack.flatten()
print("frame_stack shape = %s" % str(frame_stack.shape))
frames_raw = frame_stack.tostring()
# use compression to save space
# frames_raw = zlib.compress(frames_raw)
features['frames'] = _bytes_feature(frames_raw)
example = tf.train.Example(features=tf.train.Features(feature=features))
tfrecord_writer.write(example.SerializeToString())
tfrecord_writer.close()
print("%s images of shape %s written to tfrecord file %s" % (num_frames, frame.shape, tfrecord_path))
def main():
filenames = []
list_file = sys.argv[1]
with open(list_file) as fd:
lines = fd.read().split('\n')
while '' in lines:
lines.remove('')
for l in lines:
f, c = l.split()
filenames.append(f)
class_list = get_class_list(CLASS_INDEX_FILE)
# dataset iterator
input_filenames = tf.placeholder(tf.string, shape=[None])
dataset = tf.data.TFRecordDataset(input_filenames)
dataset = dataset.map(_parse_function)
dataset = dataset.batch(1)
dataset = dataset.repeat(1)
dataset_iterator = dataset.make_initializable_iterator()
x, y_true = dataset_iterator.get_next()
y_true_class = tf.argmax(y_true, axis=1)
init_op = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init_op)
sess.run(dataset_iterator.initializer, feed_dict={input_filenames: filenames})
while True:
try:
x_out, y_true_out = sess.run([x, y_true])
except tf.errors.OutOfRangeError:
break
def main():
'''main function'''
# get the list of classes
class_names = get_class_list(CLASS_INDEX_FILE)
all_classes = True
# argument collection
run_name = sys.argv[1]
sample = float(sys.argv[2])
included_classes = sys.argv[3]
model_dir = sys.argv[4]
#tfrecord_dir = sys.argv[5]
if ',' in included_classes:
all_classes = False
included_classes = [x.strip() for x in sys.argv[3].split(',')]
for c in included_classes:
assert c in class_names, "%s is not a valid class" % c
elif included_classes.isdigit():
all_classes = False
included_classes = random.sample(class_names, int(included_classes))
elif included_classes == "all":
all_classes = True
else:
print("Invalid value for class inclusion [%s]" % included_classes)
sys.exit(1)
print("Beginning run %s using %s sample size and %s classes" % (run_name, sample, included_classes))
# create the model object
model = C3DModel(model_dir=model_dir)
# open the csv data file and write the header to it
run_csv_file = 'runs/%s.csv' % run_name
if sys.version_info[0] == 3:
run_csv_fd = open(run_csv_file, 'w', newline='', buffering=1)
else:
run_csv_fd = open(run_csv_file, 'wb', buffering=0)
run_csv_writer = csv.writer(run_csv_fd, dialect='excel')
run_csv_writer.writerow(['step_type', 'epoch', 'iteration', 'loss', 'accuracy', 'hit_at_5'])
# get the list of files for train and test
#train_files = file_split(TRAIN_SPLIT, model.tfrecord_dir, ONE_CLIP_PER_VIDEO)
#test_files = file_split(TEST_SPLIT, model.tfrecord_dir, ONE_CLIP_PER_VIDEO)
#if not all_classes:
# train_files_filtered = []
# test_files_filtered = []
# for c in included_classes:
# # print("c = %s" % c)
# for t in train_files:
# if c in t:
# train_files_filtered.append(t)
# for t in test_files:
# if c in t:
# test_files_filtered.append(t)
# train_files = train_files_filtered
# test_files = test_files_filtered
# # print("train files = %s" % train_files)
# # print("test files = %s" % test_files)
# assert len(test_files) > 0 and len(train_files) > 0, 'test = %s, train = %s' % (len(test_files), len(train_files))
# # sample from the test and train files if necessary
# if sample < 1.0:
# sample_size = int(len(train_files) * sample)
# train_files = random.sample(train_files, sample_size)
# print("%s training samples" % sample_size)
# sample_size_test = int(len(test_files) * sample)
# test_files = random.sample(test_files, sample_size_test)
# print("%s testing samples" % sample_size_test)
# assert len(test_files) > 0 and len(train_files) > 0
# # reset size of mini-batch based on the number of test files
# MINI_BATCH_SIZE = min(MINI_BATCH_SIZE, len(test_files))
# SHUFFLE_SIZE = min(SHUFFLE_SIZE, int(len(train_files) * 0.05))
# # throw out samples to fit the batch size
# num_samples_batch_fit = len(train_files) - (len(train_files) % BATCH_SIZE)
# if BALANCE_CLASSES:
# # balance the classes for training
# train_files = balance_classes(train_files)
# random.shuffle(train_files)
# train_files = train_files[0: num_samples_batch_fit]
# print("Training samples = %s, testing samples = %s" % (len(train_files), len(test_files)))
# print("Training class counts:")
# print_class_counts(train_files)
# print("Test class counts:")
# print_class_counts(test_files)
# open the log file
run_log_file = 'runs/%s.log' % run_name
run_log_fd = open(run_log_file, 'w', 0)
run_log_fd.write("run name = %s\nsample = %s\nincluded_classes = %s\n" % (run_name, sample, included_classes))
run_log_fd.write("HYPER PARAMETERS:\n")
run_log_fd.write("NUM_EPOCHS = %s\nMINI_BATCH_SIZE = %s\nTRAIN_SPLIT = %s\nTEST_SPLIT = %s\nSHUFFLE_SIZE = %s\n" %
(NUM_EPOCHS, MINI_BATCH_SIZE, TRAIN_SPLIT, TEST_SPLIT, SHUFFLE_SIZE))
print("BATCH_SIZE = %s" % (BATCH_SIZE))
run_log_fd.write("VALIDATE_WITH_TRAIN = %s\nBALANCE_CLASSES = %s\n" % (VALIDATE_WITH_TRAIN, BALANCE_CLASSES))
run_log_fd.write("WEIGHT_STDDEV = %s\nBIAS = %s\n" % (c3d.WEIGHT_STDDEV, c3d.BIAS))
run_log_fd.write("WEIGHT_DECAY = %s\nBIAS_DECAY = %s\n" % (c3d.WEIGHT_DECAY, c3d.BIAS_DECAY))
run_log_fd.write("VARIABLE_TYPE = %s\n" % (VARIABLE_TYPE))
run_log_fd.write("ONE_CLIP_PER_VIDEO = %s" % (ONE_CLIP_PER_VIDEO))
run_log_fd.write("LEARNING_RATE_DECAY = %s" % (LEARNING_RATE_DECAY))
run_log_fd.write("OPTIMIZER = %s" % (OPTIMIZER))
run_log_fd.write("IMAGE_CROPPING = %s" % (IMAGE_CROPPING))
run_log_fd.write("IMAGE_NORMALIZATION = %s" % (IMAGE_NORMALIZATION))
run_log_fd.write("learning rate = %s" % (model.learning_rate))
# run_log_fd.write("Training samples = %s, testing samples = %s\n" % (len(train_files), len(test_files)))
# Tensorflow configuration
config = tf.ConfigProto(allow_soft_placement=True)
with tf.Session(config=config) as sess:
#sess = tf_debug.LocalCLIDebugWrapperSession(sess)
# init variables
# tf.set_random_seed(1234)
# coord = tf.train.Coordinator()
# threads = tf.train.start_queue_runners(coord=coord)
with tf.variable_scope('var_name') as var_scope:
weights = {
'wc1': weight_variable('wc1', [3, 3, 3, 3, 64], WEIGHT_STDDEV),
'wc2': weight_variable('wc2', [3, 3, 3, 64, 128], WEIGHT_STDDEV),
'wc3a': weight_variable('wc3a', [3, 3, 3, 128, 256], WEIGHT_STDDEV),
'wc3b': weight_variable('wc3b', [3, 3, 3, 256, 256], WEIGHT_STDDEV),
'wc4a': weight_variable('wc4a', [3, 3, 3, 256, 512], WEIGHT_STDDEV),
'wc4b': weight_variable('wc4b', [3, 3, 3, 512, 512], WEIGHT_STDDEV),
'wc5a': weight_variable('wc5a', [3, 3, 3, 512, 512], WEIGHT_STDDEV),
'wc5b': weight_variable('wc5b', [3, 3, 3, 512, 512], WEIGHT_STDDEV),
'wd1': weight_variable('wd1', [8192, 4096], WEIGHT_STDDEV),
'wd2': weight_variable('wd2', [4096, 4096], WEIGHT_STDDEV),
'wdout': weight_variable('wdout', [4096, model.num_classes], WEIGHT_STDDEV),
}
biases = {
'bc1': bias_variable('bc1', [64], BIAS),
'bc2': bias_variable('bc2', [128], BIAS),
'bc3a': bias_variable('bc3a', [256], BIAS),
'bc3b': bias_variable('bc3b', [256], BIAS),
'bc4a': bias_variable('bc4a', [512], BIAS),
'bc4b': bias_variable('bc4b', [512], BIAS),
'bc5a': bias_variable('bc5a', [512], BIAS),
'bc5b': bias_variable('bc5b', [512], BIAS),
'bd1': bias_variable('bd1', [4096], BIAS),
'bd2': bias_variable('bd2', [4096], BIAS),
'bdout': bias_variable('bdout', [model.num_classes], BIAS),
}
# placeholders and constants
# y_true = tf.placeholder(tf.float32, shape=[None, NUM_CLASSES], name='y_true')
x = tf.placeholder(tf.float32, shape=[BATCH_SIZE, model.frames_per_clip, 112, 112, 3])
y_true = tf.placeholder(tf.int64, shape=[BATCH_SIZE])
# x_test = tf.placeholder(tf.float32, shape=[1, model.frames_per_clip, 112, 112, 3])
# y_true_test = tf.placeholder(tf.int64, shape=[1])
y_true_one_hot = tf.one_hot(y_true, depth=model.num_classes)
# y_true_test_one_hot = tf.one_hot(y_true_test, depth=model.num_classes)
y_true_class = tf.argmax(y_true_one_hot, axis=1)
# y_true_test_class = tf.argmax(y_true_test_one_hot, axis=1)
logits = model.inference_3d(x, weights, biases, BATCH_SIZE, True)
# logits = model.c3d(x, training=True)
y_pred = tf.nn.softmax(logits)
y_pred_class = tf.argmax(y_pred, axis=1)
correct_pred = tf.equal(y_pred_class, y_true_class)
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
hit_5 = tf.nn.in_top_k(logits, y_true_class, 5)
top_5 = tf.nn.top_k(logits, k=5)
# loss and optimizer
# loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y_true))
loss_op = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y_true_class, name="softmax"), name="reduce_mean")
if OPTIMIZER == 'SGD':
learning_rate = tf.placeholder(tf.float32, shape=[])
#learning_rate = tf.train.exponential_decay(learning_rate=model.learning_rate, global_step=global_step,
# decay_steps=(4 * len(train_files)), decay_rate=0.96,
# staircase=True)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate, name="optimizer")
elif OPTIMIZER == 'Adam':
optimizer = tf.train.AdamOptimizer(LEARNING_RATE)
train_op = optimizer.minimize(loss_op, name="train")
# model evaluation
# logits_test = model.c3d(x_test, training=False)
# logits_test = model.inference_3d(x_test, weights, biases, 1, False)
# y_pred_test = tf.nn.softmax(logits_test)
# y_pred_test_class = tf.argmax(y_pred_test, axis=1)
# eval_hit_5 = tf.nn.in_top_k(logits_test, y_true_test_class, 5)
# eval_top_5 = tf.nn.top_k(logits_test, k=5)
# eval_correct_pred = tf.equal(y_pred_test_class, y_true_test_class)
# eval_accuracy = tf.reduce_mean(tf.cast(eval_correct_pred, tf.float32))
init_op = tf.global_variables_initializer()
saver = tf.train.Saver(weights.values() + biases.values())
# TRAINING
sess.run(init_op)
report_step = 20
print("Beginning training")
in_epoch = 1
print("START EPOCH %s" % in_epoch)
start = time.time()
epoch_start = time.time()
# sess.run(train_iterator.initializer, feed_dict={train_filenames: train_files})
# if VALIDATE_WITH_TRAIN:
# sess.run(test_iterator.initializer, feed_dict={test_filenames: train_files})
# else:
# sess.run(test_iterator.initializer, feed_dict={test_filenames: test_files})
step = 0
_, _, _, num_samples = get_image_batch(TRAIN_SPLIT, BATCH_SIZE, model.frames_per_clip, model.num_classes)
max_steps = int((num_samples * NUM_EPOCHS / BATCH_SIZE) * sample)
print("max_steps = %s" % max_steps)
# num_batches_per_epoch = len(train_files) / BATCH_SIZE
while step < max_steps:
if step != 0 and step % int(num_samples / BATCH_SIZE) == 0:
in_epoch += 1
print("START EPOCH %s" % in_epoch)
if in_epoch % 4 == 0 and OPTIMIZER != 'Adam':
model.current_learning_rate = model.current_learning_rate * LEARNING_RATE_DECAY
print("learning rate adjusted to %g" % model.current_learning_rate)
step_start = time.time()
x_feed, y_feed, _, num_samples = get_image_batch(TRAIN_SPLIT, BATCH_SIZE, model.frames_per_clip, model.num_classes,
crop=IMAGE_CROPPING, normalize=IMAGE_NORMALIZATION)
# print("x_feed = %s" % str(x_feed.shape))
sess.run(train_op, feed_dict={x: x_feed, y_true: y_feed})
step_end = time.time()
print("epoch %s step %s - %.3fs" % (in_epoch, step, step_end - step_start))
if step % 10 == 0:
# train accuracy
results = sess.run([accuracy, y_true_class, y_pred_class, logits],
feed_dict={x: x_feed, y_true: y_feed})
acc = results[0]
y_true_class_out = results[1]
y_pred_class_out = results[2]
logits_out = results[3][0]
#print("Train accuracy = %s, y_true = %s, y_prediction = %s, logits = %s" % (acc, y_true_class_out, y_pred_class_out, logits_out))
print("Train accuracy = %s" % acc)
# validation accuracy
x_feed, y_feed, _, num_samples = get_image_batch(TEST_SPLIT, BATCH_SIZE, model.frames_per_clip, model.num_classes,
crop=TEST_IMAGE_CROPPING, normalize=IMAGE_NORMALIZATION)
acc = sess.run(accuracy, feed_dict={x: x_feed, y_true: y_feed})
print("Validation accuracy = %s" % acc)
if step % 100 == 0 or step == max_steps - 1:
# save a model checkpoint
save_path = os.path.join(model.model_dir, "model_%s_step_%s.ckpt" % (run_name, step))
save_path = saver.save(sess, save_path)
step += 1
# if j != 0 and j % num_samples < BATCH_SIZE:
# # end of epoch
# # save a model checkpoint and report end of epoch information
# save_path = os.path.join(model.model_dir, "model_epoch_%s.ckpt" % in_epoch)
# save_path = saver.save(sess, save_path)
# epoch_end = time.time()
# train_time = str(datetime.timedelta(seconds=epoch_end - epoch_start))
# print("END EPOCH %s, steps completed = %s, epoch training time: %s" % (in_epoch, j, train_time))
# print("model checkpoint saved to %s\n\n" % save_path)
# # test the network
# test_results = test_network(sess, model, TEST_SPLIT, run_name, in_epoch)
# run_csv_writer.writerow(test_results)
# in_epoch += 1
# if in_epoch % 4 == 0 and OPTIMIZER != 'Adam':
# model.current_learning_rate = model.current_learning_rate * LEARNING_RATE_DECAY
# print("learning rate adjusted to %g" % model.current_learning_rate)
# print("START EPOCH %s" % in_epoch)
# epoch_start = time.time()
# train_result = sess.run([train_op, loss_op, accuracy, x, y_true, y_true_class, y_pred, y_pred_class, logits, hit_5],
# feed_dict={x: x_feed,
# y_true: y_feed,
# learning_rate: model.current_learning_rate})
# loss_op_out = train_result[1]
# train_acc = train_result[2]
# x_actual = train_result[3]
# y_true_actual = train_result[4]
# y_true_class_actual = train_result[5]
# y_pred_actual = train_result[6]
# y_pred_class_actual = train_result[7]
# logits_out = train_result[8]
# hit5_out = train_result[9]
# # print("train_acc = %s" % train_acc)
# # train_acc is the number of correct responses divided by the batch size
# # e.g. 3 correct responses/30 batch size = 0.1
# train_acc_accum += train_acc
# # print("hit_5_out = %s" % hit5_out)
# # count the trues in the hit_5_out array
# hit5_counter = collections.Counter(hit5_out)
# hit5_out_trues = float(hit5_counter[True])
# # print("%s trues out of %s" % (hit5_out_trues, len(hit5_out)))
# train_hit5_accum += float(hit5_out_trues / len(hit5_out))
# # report out results and run a test mini-batch every now and then
# if j != 0 and j % (report_step * BATCH_SIZE) == 0:
# #print("logits = %s" % logits_out)
# #print("x = %s" % x_actual)
# #print("y_true = %s, y_true_class = %s, y_pred = %s, y_pred_class = %s" % (y_true_actual, y_true_class_actual, y_pred_actual, y_pred_class_actual))
# #print("train_acc = %s" % train_acc)
# print("hit5_out = %s, length = %s" % (hit5_out, len(hit5_out)))
# print("%s trues out of %s, %s accuracy" % (hit5_out_trues, len(hit5_out), hit5_out_trues / len(hit5_out)))
# run_time = time.time()
# run_time_str = str(datetime.timedelta(seconds=run_time - start))
# train_step_acc = train_acc_accum / report_step * BATCH_SIZE
# # mini batch accuracy - every 5 report step iterations
# if j % (report_step * BATCH_SIZE * 5) == 0:
# mini_batch_acc = 0.0
# mini_batch_hit5 = 0.0
# for k in range(MINI_BATCH_SIZE):
# if IMAGE_CROPPING == 'random':
# test_crop = 'center'
# else:
# test_crop = IMAGE_CROPPING
# x_feed, y_feed, _, num_samples = get_image_batch(TRAIN_SPLIT, 1, model.frames_per_clip, model.num_classes, crop=test_crop)
# acc, hit5_out, top_5_out, x_out = sess.run([eval_accuracy, eval_hit_5, eval_top_5, x_test],
# feed_dict={x_test: x_feed, y_true_test: y_feed})
# # print("type(x) = %s, x = %s" % (type(x_out), x_out))
# mini_batch_acc += acc
# hit5_counter = collections.Counter(hit5_out)
# hit5_out_trues = hit5_counter[True]
# # print("%s trues out of %s" % (hit5_out_trues, len(hit_5_out)))
# mini_batch_hit5 += float(hit5_out_trues / len(hit5_out))
# mini_batch_acc = mini_batch_acc / MINI_BATCH_SIZE
# mini_batch_hit5 = mini_batch_hit5 / MINI_BATCH_SIZE
# print("\tstep %s - epoch %s run time = %s, loss = %s, mini-batch accuracy = %s, hit@5 = %s, y_true = %s, top 5 = %s" %
# (j, in_epoch, run_time_str, loss_op_out, mini_batch_acc, mini_batch_hit5, y_true_class_actual, top_5_out))
# csv_row = ['mini-batch', in_epoch, j, loss_op_out, mini_batch_acc, mini_batch_hit5]
# else:
# train_acc_accum = train_acc_accum / report_step
# train_hit5_accum = train_hit5_accum / report_step
# print("\tstep %s - epoch %s run time = %s, loss = %s, train accuracy = %s, hit@5 = %s" %
# (j, in_epoch, run_time_str, loss_op_out, train_acc_accum, train_hit5_accum))
# csv_row = ['train', in_epoch, j, loss_op_out, train_acc_accum, train_hit5_accum]
# # write the csv data to
# run_csv_writer.writerow(csv_row)
# train_acc_accum = 0.0
# train_hit5_accum = 0.0
# j += BATCH_SIZE
# print("end training epochs")
# # end of epoch
# # save a model checkpoint and report end of epoch information
# save_path = os.path.join(model.model_dir, "model_epoch_%s.ckpt" % in_epoch)
# save_path = saver.save(sess, save_path)
# epoch_end = time.time()
# train_time = str(datetime.timedelta(seconds=epoch_end - epoch_start))
# print("END EPOCH %s, iterations = %s, epoch training time: %s" % (in_epoch, j, train_time))
# print("model checkpoint saved to %s\n\n" % save_path)
# # final test
# test_results = test_network(sess, model, TEST_SPLIT, run_name, in_epoch)
# run_csv_writer.writerow(test_results)
# end = time.time()
# total_time = str(datetime.timedelta(seconds=end - start))
# print("END TRAINING total training time: %s" % (total_time))
#coord.request_stop()
#coord.join(threads)
# sess.close()
run_csv_fd.close()
run_log_fd.close()
def run_test():
# get the list of classes
classes = get_class_list(CLASS_INDEX_FILE)
model_name = "/home/jordanc/datasets/UCF-101/model_ckpts/c3d_ucf_model-4999"
test_list_file = 'train-test-splits/testlist01.txt'
num_test_videos = len(list(open(test_list_file, 'r')))
print("Number of test videos={}".format(num_test_videos))
# Get the sets of images and labels for training, validation, and
images_placeholder, labels_placeholder = placeholder_inputs(
FLAGS.batch_size * gpu_num)
with tf.variable_scope('var_name') as var_scope:
weights = {
'wc1':
_variable_with_weight_decay('wc1', [3, 3, 3, 3, 64], 0.04, 0.00),
'wc2':
_variable_with_weight_decay('wc2', [3, 3, 3, 64, 128], 0.04, 0.00),
'wc3a':
_variable_with_weight_decay('wc3a', [3, 3, 3, 128, 256], 0.04,
0.00),
'wc3b':
_variable_with_weight_decay('wc3b', [3, 3, 3, 256, 256], 0.04,
0.00),
'wc4a':
_variable_with_weight_decay('wc4a', [3, 3, 3, 256, 512], 0.04,
0.00),
'wc4b':
_variable_with_weight_decay('wc4b', [3, 3, 3, 512, 512], 0.04,
0.00),
'wc5a':
_variable_with_weight_decay('wc5a', [3, 3, 3, 512, 512], 0.04,
0.00),
'wc5b':
_variable_with_weight_decay('wc5b', [3, 3, 3, 512, 512], 0.04,
0.00),
'wd1':
_variable_with_weight_decay('wd1', [8192, 4096], 0.04, 0.001),
'wd2':
_variable_with_weight_decay('wd2', [4096, 4096], 0.04, 0.002),
'out':
_variable_with_weight_decay('wout', [4096, c3d_model.NUM_CLASSES],
0.04, 0.005)
}
biases = {
'bc1':
_variable_with_weight_decay('bc1', [64], 0.04, 0.0),
'bc2':
_variable_with_weight_decay('bc2', [128], 0.04, 0.0),
'bc3a':
_variable_with_weight_decay('bc3a', [256], 0.04, 0.0),
'bc3b':
_variable_with_weight_decay('bc3b', [256], 0.04, 0.0),
'bc4a':
_variable_with_weight_decay('bc4a', [512], 0.04, 0.0),
'bc4b':
_variable_with_weight_decay('bc4b', [512], 0.04, 0.0),
'bc5a':
_variable_with_weight_decay('bc5a', [512], 0.04, 0.0),
'bc5b':
_variable_with_weight_decay('bc5b', [512], 0.04, 0.0),
'bd1':
_variable_with_weight_decay('bd1', [4096], 0.04, 0.0),
'bd2':
_variable_with_weight_decay('bd2', [4096], 0.04, 0.0),
'out':
_variable_with_weight_decay('bout', [c3d_model.NUM_CLASSES], 0.04,
0.0),
}
logits = []
activations = []
for gpu_index in range(0, gpu_num):
with tf.device('/gpu:%d' % gpu_index):
logit, activation = c3d_model.inference_c3d(
images_placeholder[gpu_index *
FLAGS.batch_size:(gpu_index + 1) *
FLAGS.batch_size, :, :, :, :], 0.6,
FLAGS.batch_size, weights, biases)
logits.append(logit)
activations.append(activation)
logits = tf.concat(logits, 0)
activations = tf.concat(activations, 0)
norm_score = tf.nn.softmax(logits)
saver = tf.train.Saver()
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
init = tf.global_variables_initializer()
sess.run(init)
# Create a saver for writing training checkpoints.
saver.restore(sess, model_name)
# And then after everything is built, start the training loop.
bufsize = 0
write_file = open("predict_ret.txt", "w+", bufsize)
next_start_pos = 0
all_steps = int((num_test_videos - 1) / (FLAGS.batch_size * gpu_num) + 1)
for step in xrange(all_steps):
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
start_time = time.time()
sample_output = c3d_model.read_clip_and_label(
directory='/home/jordanc/datasets/UCF-101/UCF-101/',
filename=test_list_file,
batch_size=FLAGS.batch_size * gpu_num,
start_pos=next_start_pos)
test_images = sample_output[0]
test_labels = sample_output[1]
next_start_pos = sample_output[2]
valid_len = sample_output[4]
sample_names = sample_output[5]
predict_score, activations_out = sess.run(
[norm_score, activations],
feed_dict={images_placeholder: test_images})
for i in range(0, valid_len):
true_label = test_labels[i],
top1_predicted_label = np.argmax(predict_score[i])
# Write results: true label, class prob for true label, predicted label, class prob for predicted label
write_file.write('{}, {}, {}, {}\n'.format(
true_label[0], predict_score[i][true_label],
top1_predicted_label, predict_score[i][top1_predicted_label]))
write_file.close()
print("done")
from __future__ import print_function
import sys
from tfrecord_gen import CLASS_INDEX_FILE, get_class_list
classes = get_class_list(CLASS_INDEX_FILE)
filename = sys.argv[1]
fd = open(filename)
text = fd.read()
lines = text.split('\n')
with open(filename + "fixed", 'w+') as fd:
for l in lines:
if len(l) > 0:
label, folder = l.strip().split('/')
# print("%s/%s" % (label, folder))
class_name = str(classes.index(label))
string = label + '\\' + folder + ' ' + class_name
print(string)
fd.write(string + "\n")
def iad_nn(run_string, json_input_train, json_input_test):
'''main function'''
if LOAD_MODEL is None:
training = True
else:
training = False
# get the list of classes
class_list = get_class_list(CLASS_INDEX_FILE)
img_geom = LAYER_GEOMETRY[str(LAYER)]
# get the list of filenames
#print("loading train file list from %s" % TRAIN_FILE_LIST)
#filenames_train = list_to_filenames(TRAIN_FILE_LIST, balance_classes=False)
#filenames_test = list_to_filenames(TEST_FILE_LIST)
#print("%s training files, %s testing files" % (len(filenames_train), len(filenames_test)))
#random.shuffle(filenames_train)
#if TRAINING_DATA_SAMPLE != 1.0:
# filenames_train = random.sample(filenames_train, int(TRAINING_DATA_SAMPLE * len(filenames_train)))
# ensure filenames list is evenly divisable by batch size
#pad_filenames = len(filenames_train) % BATCH_SIZE
#filenames_train.extend(filenames_train[0:pad_filenames])
# create the TensorFlow sessions
config = tf.ConfigProto(allow_soft_placement=True)
sess = tf.Session(config=config)
# setup the CNN
if CLASSIFIER == 'softmax':
weights, biases = get_variables_softmax('ucf101_iad')
elif CLASSIFIER == 'lenet':
weights, biases = get_variables_lenet('ucf101_iad')
elif CLASSIFIER == 'mctnet':
weights, biases = get_variables_mctnet('ucf101_iad')
elif CLASSIFIER == 'autoencode':
weights, biases = get_variables_autoencode('ucf101_iad')
# placeholders
#input_filenames = tf.placeholder(tf.string, shape=[None])
#input_filenames_test = tf.placeholder(tf.string, shape=[None])
x = tf.placeholder(tf.float32, shape=(
None,
LAYER_GEOMETRY[str(LAYER)][0],
LAYER_GEOMETRY[str(LAYER)][1]
))
y_true = tf.placeholder(tf.int64, shape=(None))
#x_test = tf.placeholder(tf.float32, shape=(
# 1,
# LAYER_GEOMETRY[str(LAYER)][0],
# LAYER_GEOMETRY[str(LAYER)][1]
# ))
#y_test_true = tf.placeholder(tf.int64, shape=(1))
# testing dataset
#dataset_test = tf.data.TFRecordDataset(input_filenames_test)
#dataset_test = dataset_test.map(parse_function_test)
#dataset_test = dataset_test.batch(1)
#dataset_test = dataset_test.shuffle(500)
#dataset_test = dataset_test.repeat(1000000)
#dataset_test_iterator = dataset_test.make_initializable_iterator()
#x_test, y_test_true = dataset_test_iterator.get_next()
#y_test_true_one_hot = tf.one_hot(y_test_true, depth=NUM_CLASSES, dtype=tf.int32)
#y_test_true_class = tf.argmax(y_test_true_one_hot, axis=1)
# training or evaluation dataset
#dataset = tf.data.TFRecordDataset(input_filenames)
#dataset = dataset.map(parse_function_train)
#if tf.__version__ == '1.11.0':
# dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)
#else:
# dataset = dataset.batch(BATCH_SIZE)
#if training:
# dataset = dataset.shuffle(200)
# dataset = dataset.repeat(EPOCHS)
#else:
# dataset = dataset.repeat(1)
#dataset_iterator = dataset.make_initializable_iterator()
#x, y_true = dataset_iterator.get_next()
y_true_one_hot = tf.one_hot(y_true, depth=NUM_CLASSES, dtype=tf.int32)
y_true_class = tf.argmax(y_true_one_hot, axis=1)
print("x shape = %s" % x.get_shape().as_list())
#print("y_true shape = %s" % y_true.get_shape().as_list())
# get neural network response
if CLASSIFIER in ['softmax', 'autoencode', 'lenet', 'mctnet']:
if CLASSIFIER == 'softmax':
# layer 1
print("LAYER = %s, geom = %s" % (LAYER, img_geom))
x = tf.reshape(x, [-1, img_geom[0] * img_geom[1]])
#x_test = tf.reshape(x_test, [1, img_geom[0] * img_geom[1]])
logits, conv_layers = softmax_regression(x, BATCH_SIZE, weights, biases, DROPOUT)
#logits_test, _ = softmax_regression(x_test, 1, weights, biases, DROPOUT)
elif CLASSIFIER == 'lenet':
logits, conv_layers = cnn_lenet(x, BATCH_SIZE, weights, biases, DROPOUT)
#logits_test, conv_layers = cnn_lenet(x_test, 1, weights, biases, DROPOUT)
elif CLASSIFIER == 'mctnet':
logits, conv_layers = cnn_mctnet(x, BATCH_SIZE, weights, biases, DROPOUT)
#logits_test, conv_layers = cnn_mctnet(x_test, 1, weights, biases, DROPOUT)
elif CLASSIFIER == 'autoencode':
geom = LAYER_GEOMETRY[str(LAYER)]
# layer 1
x = tf.reshape(x, [-1, img_geom[0] * img_geom[1]])
#x_test = tf.reshape(x_test, [1, img_geom[0] * img_geom[1]])
x_autoencode, conv_layers = autoencode(x, BATCH_SIZE, weights, biases)
#x_test_autoencode, _ = autoencode(x_test, 1, weights, biases)
# softmax regression with output
# final
logits = tf.matmul(x_autoencode, weights['W_out']) + biases['b_out']
#logits_test = tf.matmul(x_test_autoencode, weights['W_out']) + biases['b_out']
print("logits shape = %s" % logits.get_shape().as_list())
y_pred = tf.nn.softmax(logits)
y_pred_class = tf.argmax(y_pred, axis=1)
# testing/validation
#y_pred_test = tf.nn.softmax(logits_test)
#y_pred_test_class = tf.argmax(y_pred_test, axis=1)
#correct_pred_test = tf.equal(y_pred_test_class, y_test_true_class)
#accuracy_test = tf.reduce_mean(tf.cast(correct_pred_test, tf.float32))
# loss and optimizer
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y_true_one_hot))
#loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y_true))
optimizer = tf.train.AdamOptimizer(learning_rate=LEARNING_RATE)
train_op = optimizer.minimize(loss)
# evaluation
correct_pred = tf.equal(y_pred_class, y_true_class)
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
elif CLASSIFIER == 'logistic_regression':
geom = LAYER_GEOMETRY[str(LAYER)]
p5 = tf.constant(0.5) # logistic regression threshold
W = tf.Variable(tf.zeros([geom[0] * geom[1], NUM_CLASSES]))
b = tf.Variable(tf.zeros([NUM_CLASSES]))
x = tf.reshape(x, [BATCH_SIZE, img_geom[0] * img_geom[1]])
#x_test = tf.reshape(x_test, [1, img_geom[0] * img_geom[1]])
y_pred = tf.nn.softmax(tf.matmul(x, W) + b)
y_pred_class = tf.argmax(y_pred, axis=1)
cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_true * tf.log(y_pred), reduction_indices=[1]))
train_op = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
correct_pred = tf.equal(tf.argmax(y_pred, 1), y_true_class)
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# validation/testing
#y_pred_test = tf.nn.softmax(tf.matmul(x, W) + b)
#y_pred_test_class = tf.argmax(y_pred_test, axis=1)
#correct_pred_test = tf.equal(tf.argmax(y_pred_test, 1), y_test_true_class)
#accuracy_test = tf.reduce_mean(tf.cast(correct_pred_test, tf.float32))
# initializer
init_op = tf.global_variables_initializer()
saver = tf.train.Saver()
sess.run(init_op)
# start the training/testing steps
if training:
num_steps = int((json_input_train.num_files * EPOCHS) / BATCH_SIZE)
print("begin training, steps = %s" % num_steps)
step = 0
#sess.run([
# dataset_iterator.initializer,
# dataset_test_iterator.initializer],
# feed_dict={
# input_filenames: filenames_train,
# input_filenames_test: filenames_test
# })
# loop until out of data
start = time.time()
while step <= num_steps:
#try:
input_x, input_y, _ = json_input_train.get_batch()
input_x = input_x.reshape((BATCH_SIZE, img_geom[0] * img_geom[1]))
#input_y = tf.one_hot(input_y, depth=NUM_CLASSES, dtype=tf.int32)
input_y = input_y.reshape((BATCH_SIZE))
train_result = sess.run([train_op, accuracy, y_pred, logits, weights, loss], feed_dict={x: input_x, y_true: input_y})
if step != 0 and step % 100 == 0:
end = time.time()
run_time = end - start
start = time.time()
# calculate discard rate
discard_rate = json_input_train.discards / (step * BATCH_SIZE_TRAIN)
print("step %s, accuracy = %.03f, loss = %.03f, discards = %.03f, time = %.03fs" % (step, train_result[1], train_result[5], discard_rate, run_time))
#print("\tx shape = %s, max = %s, min = %s, mean = %s" % (train_result[2].shape, train_result[2].max(), train_result[2].min(), train_result[2].mean()))
#print("\ty_true (shape = %s) = %s" % (train_result[3].shape, train_result[3]))
#print("\ty_pred (shape = %s) = %s" % (train_result[4].shape, train_result[4]))
#print("\tlogits (shape = %s) = %s" % (train_result[5].shape, train_result[5]))
#print("\tweights (shape = %s) = %s" % (train_result[6]['W_0'].shape, train_result[6]['W_0']))
# save the current model every 1000 steps
if step % 1000 == 0:
save_model(sess, saver, step)
# mini batch validation
if step % 10000 == 0:
mini_batch_size = 1000
else:
mini_batch_size = 50
test_accuracy = 0.0
for i in range(mini_batch_size):
input_x, input_y, _ = json_input_test.get_batch()
input_x = input_x.reshape((1, img_geom[0] * img_geom[1]))
input_y = input_y.reshape((1))
test_result = sess.run([accuracy], feed_dict={x: input_x, y_true: input_y})
test_accuracy += test_result[0]
print("mini-batch validation accuracy = %.03f" % (test_accuracy / mini_batch_size))
# print("x = %s, logits = %s" % (train_result[2], train_result[3]))
step += 1
#except tf.errors.OutOfRangeError:
# print("data exhausted, saving final model")
# save_model(sess, saver, 'final')
# break
save_model(sess, saver, 'final')
final_accuracy = -1.0
sess.close()
print("end training")
return None, None
else:
num_steps = json_input_test.num_files
print("begin testing, steps = %s" % num_steps)
step = 0
saver.restore(sess, LOAD_MODEL)
cumulative_accuracy = 0.0
predictions = []
true_classes = []
# clip/video accuracy
# ['sample_name': [accuracy@1, accuracy@1]]
# ['video_name': [accuracy@1, accuracy@1]]
clip_accuracy = {}
video_accuracy = {}
# loop until out of data
while step <= num_steps:
input_x, input_y, sample_name = json_input_test.get_batch()
input_x = input_x.reshape((1, img_geom[0] * img_geom[1]))
input_y = input_y.reshape((1))
test_result = sess.run([accuracy, y_pred_class, y_true_class], feed_dict={x: input_x, y_true: input_y})
# add result to the clip and video accuracy structures
sample_name = sample_name[0]
video_name = '_'.join(sample_name.split('_')[0:3])
if sample_name in clip_accuracy.keys():
clip_accuracy[sample_name].append(test_result[0])
else:
clip_accuracy[sample_name] = [test_result[0]]
if video_name in video_accuracy.keys():
video_accuracy[video_name].append(test_result[0])
else:
video_accuracy[video_name] = [test_result[0]]
cumulative_accuracy += test_result[0]
predictions.append(test_result[1])
true_classes.append(test_result[2])
if step % 100 == 0:
print("step %s, accuracy = %s, cumulative accuracy = %s" %
(step, test_result[0], cumulative_accuracy / step / BATCH_SIZE))
step += 1
# wrap up, provide test results
final_accuracy = cumulative_accuracy / step / BATCH_SIZE
print("data exhausted, test results:")
print("steps = %s, cumulative accuracy = %.04f" % (step, final_accuracy))
with open("runs/" + run_string + ".txt", 'a+') as results_fd:
results_fd.write("steps = %s, cumulative accuracy = %.04f\n" % (step, final_accuracy))
#for i, p in enumerate(predictions):
# print("[%s] true class = %s, predicted class = %s" % (i, true_classes[i], p))
cm = analysis.confusion_matrix(predictions, true_classes, class_list)
print("confusion matrix = %s" % cm)
analysis.plot_confusion_matrix(cm, class_list, "runs/" + run_string + ".pdf")
#print("per-class accuracy:")
analysis.per_class_table(predictions, true_classes, class_list, "runs/" + run_string + '.csv')
sess.close()
return clip_accuracy, video_accuracy
def main():
'''main function'''
if LOAD_MODEL is None:
training = True
run_type = 'train'
else:
training = False
run_type = 'test'
# get the run name
run_name = sys.argv[1]
save_settings(run_name + "_" + run_type)
# get the list of classes
class_list = get_class_list(CLASS_INDEX_FILE)
# get the list of filenames
print("loading file list from %s" % FILE_LIST)
filenames = list_to_filenames(FILE_LIST)
print("%s files" % len(filenames))
if training:
random.shuffle(filenames)
if TRAINING_DATA_SAMPLE != 1.0:
filenames = random.sample(
filenames, int(TRAINING_DATA_SAMPLE * len(filenames)))
# ensure filenames list is evenly divisable by batch size
pad_filenames = len(filenames) % BATCH_SIZE
filenames.extend(filenames[0:pad_filenames])
print("filenames = %s..." % filenames[0:5])
# create the TensorFlow sessions
config = tf.ConfigProto(allow_soft_placement=True)
sess = tf.Session(config=config)
# dataset iterator
input_filenames = tf.placeholder(tf.string, shape=[None])
dataset = tf.data.TFRecordDataset(input_filenames)
dataset = dataset.map(_parse_function)
dataset = dataset.batch(BATCH_SIZE)
if training:
dataset = dataset.shuffle(100)
dataset = dataset.repeat(EPOCHS)
else:
dataset = dataset.repeat(1)
dataset_iterator = dataset.make_initializable_iterator()
x, y_true = dataset_iterator.get_next()
y_true_class = tf.argmax(y_true, axis=1)
# placeholders
prediction_grid = tf.placeholder(shape=[None, 2], dtype=tf.float32)
# variables
b = tf.Variable(tf.random_normal(shape=[NUM_CLASSES, BATCH_SIZE]))
gamma = tf.constant(GAMMA)
dist = tf.reduce_sum(tf.square(x), 1)
dist = tf.reshape(dist, [-1, 1])
sq_dists = tf.multiply(2., tf.matmul(x, tf.transpose(x)))
svm_kernel = tf.exp(tf.multiply(gamma, tf.abs(sq_dists)))
# batch multiplication
def reshape_matmul(mat, _size):
v1 = tf.expand_dims(mat, 1)
v2 = tf.reshape(v1, [NUM_CLASSES, _size, 1])
return tf.matmul(v2, v1)
# compute SVM model
first_term = tf.reduce_sum(b)
b_vec_cross = tf.matmul(tf.transpose(b), b)
y_true_cross = reshape_matmul(y_true, BATCH_SIZE)
second_term = tf.reduce_sum(
tf.multiply(svm_kernel, tf.multiply(b_vec_cross, y_true_cross)),
[1, 2])
loss = tf.reduce_sum(tf.negative(tf.subtract(first_term, second_term)))
# Gaussian (RBF) prediction kernel
rA = tf.reshape(tf.reduce_sum(tf.square(x), 1), [-1, 1])
rB = tf.reshape(tf.reduce_sum(tf.square(prediction_grid), 1), [-1, 1])
pred_sq_dist = tf.add(
tf.subtract(
rA, tf.multiply(2., tf.matmul(x, tf.transpose(prediction_grid)))),
tf.transpose(rB))
pred_kernel = tf.exp(tf.multiply(gamma, tf.abs(pred_sq_dist)))
prediction_output = tf.matmul(tf.multiply(y_true, b), pred_kernel)
y_pred_class = tf.argmax(
prediction_output -
tf.expand_dims(tf.reduce_mean(prediction_output, 1), 1),
axis=0)
accuracy = tf.reduce_mean(
tf.cast(tf.equal(prediction, tf.argmax(y_true, 0)), tf.float32))
# operations
train_op = tf.train.GradientDescentOptimizer(LEARNING_RATE)
train_step = train_op.minimize(loss)
# initializer
init_op = tf.global_variables_initializer()
saver = tf.train.Saver()
sess.run(init_op)
# start the training/testing steps
if training:
print("begin training")
step = 0
sess.run(dataset_iterator.initializer,
feed_dict={input_filenames: filenames})
# loop until out of data
while True:
try:
train_result = sess.run([train_op, accuracy, x])
if step != 0 and step % 100 == 0:
print("step %s, accuracy = %s" % (step, train_result[1]))
# save the current model every 1000 steps
if step % 1000 == 0:
save_model(sess, saver, step)
# print("x = %s, logits = %s" % (train_result[2], train_result[3]))
step += 1
except tf.errors.OutOfRangeError:
print("data exhausted, saving final model")
save_model(sess, saver, 'final')
break
else:
print("begin testing")
step = 0
saver.restore(sess, LOAD_MODEL)
sess.run(dataset_iterator.initializer,
feed_dict={input_filenames: filenames})
cumulative_accuracy = 0.0
predictions = []
true_classes = []
# loop until out of data
while True:
try:
test_result = sess.run(
[accuracy, x, y_pred_class, y_true_class])
cumulative_accuracy += test_result[0]
predictions.append(test_result[2])
true_classes.append(test_result[3])
if step % 100 == 0:
print("step %s, accuracy = %s, cumulative accuracy = %s" %
(step, test_result[0],
cumulative_accuracy / step / BATCH_SIZE))
step += 1
except tf.errors.OutOfRangeError:
break
# wrap up, provide test results
print("data exhausted, test results:")
print("steps = %s, cumulative accuracy = %.04f" %
(step, cumulative_accuracy / step / BATCH_SIZE))
#for i, p in enumerate(predictions):
# print("[%s] true class = %s, predicted class = %s" % (i, true_classes[i], p))
cm = analysis.confusion_matrix(predictions, true_classes, class_list)
print("confusion matrix = %s" % cm)
analysis.plot_confusion_matrix(cm, class_list,
"runs/" + run_name + ".pdf")
print("per-class accuracy:")
analysis.per_class_table(predictions, true_classes, class_list,
"runs/" + run_name + '.csv')
def iad_nn(run_string):
'''main function'''
if LOAD_MODEL is None:
training = True
else:
training = False
# get the list of classes
class_list = get_class_list(CLASS_INDEX_FILE)
# get the list of filenames
print("loading train file list from %s" % TRAIN_FILE_LIST)
filenames_train = list_to_filenames(TRAIN_FILE_LIST, balance_classes=True)
filenames_test = list_to_filenames(TEST_FILE_LIST)
print("%s training files, %s testing files" % (len(filenames_train), len(filenames_test)))
random.shuffle(filenames_train)
if TRAINING_DATA_SAMPLE != 1.0:
filenames_train = random.sample(filenames_train, int(TRAINING_DATA_SAMPLE * len(filenames_train)))
# ensure filenames list is evenly divisable by batch size
pad_filenames = len(filenames_train) % BATCH_SIZE
filenames_train.extend(filenames_train[0:pad_filenames])
# create the TensorFlow sessions
config = tf.ConfigProto(allow_soft_placement=True)
sess = tf.Session(config=config)
# setup the CNN
if CLASSIFIER == 'softmax':
weights, biases = get_variables_softmax('ucf101_iad')
elif CLASSIFIER == 'lenet':
weights, biases = get_variables_lenet('ucf101_iad')
elif CLASSIFIER == 'mctnet':
weights, biases = get_variables_mctnet('ucf101_iad')
elif CLASSIFIER == 'autoencode':
weights, biases = get_variables_autoencode('ucf101_iad')
# placeholders
input_filenames = tf.placeholder(tf.string, shape=[None])
input_filenames_test = tf.placeholder(tf.string, shape=[None])
# testing dataset
dataset_test = tf.data.TFRecordDataset(input_filenames_test)
dataset_test = dataset_test.map(_parse_function)
dataset_test = dataset_test.batch(1)
dataset_test = dataset_test.shuffle(500)
dataset_test = dataset_test.repeat(1000000)
dataset_test_iterator = dataset_test.make_initializable_iterator()
x_test, y_test_true = dataset_test_iterator.get_next()
y_test_true_class = tf.argmax(y_test_true, axis=1)
# training or evaluation dataset
dataset = tf.data.TFRecordDataset(input_filenames)
dataset = dataset.map(_parse_function)
dataset = dataset.batch(BATCH_SIZE)
if training:
dataset = dataset.shuffle(200)
dataset = dataset.repeat(EPOCHS)
else:
dataset = dataset.repeat(1)
dataset_iterator = dataset.make_initializable_iterator()
x, y_true = dataset_iterator.get_next()
y_true_class = tf.argmax(y_true, axis=1)
print("x shape = %s" % x.get_shape().as_list())
print("y_true shape = %s" % y_true.get_shape().as_list())
# get neural network response
if CLASSIFIER in ['softmax', 'autoencode', 'lenet', 'mctnet']:
if CLASSIFIER == 'softmax':
geom = LAYER_GEOMETRY[str(LAYER)]
# layer 1
x = tf.reshape(x, [BATCH_SIZE, geom[0] * geom[1]])
x_test = tf.reshape(x_test, [1, geom[0] * geom[1]])
logits, conv_layers = softmax_regression(x, BATCH_SIZE, weights, biases, DROPOUT)
logits_test, _ = softmax_regression(x_test, 1, weights, biases, DROPOUT)
elif CLASSIFIER == 'lenet':
logits, conv_layers = cnn_lenet(x, BATCH_SIZE, weights, biases, DROPOUT)
logits_test, conv_layers = cnn_lenet(x_test, 1, weights, biases, DROPOUT)
elif CLASSIFIER == 'mctnet':
logits, conv_layers = cnn_mctnet(x, BATCH_SIZE, weights, biases, DROPOUT)
logits_test, conv_layers = cnn_mctnet(x_test, 1, weights, biases, DROPOUT)
elif CLASSIFIER == 'autoencode':
geom = LAYER_GEOMETRY[str(LAYER)]
# layer 1
x = tf.reshape(x, [BATCH_SIZE, geom[0] * geom[1]])
x_test = tf.reshape(x_test, [1, geom[0] * geom[1]])
x_autoencode, conv_layers = autoencode(x, BATCH_SIZE, weights, biases)
x_test_autoencode, _ = autoencode(x_test, 1, weights, biases)
# softmax regression with output
# final
logits = tf.matmul(x_autoencode, weights['W_out']) + biases['b_out']
logits_test = tf.matmul(x_test_autoencode, weights['W_out']) + biases['b_out']
print("logits shape = %s" % logits.get_shape().as_list())
y_pred = tf.nn.softmax(logits)
y_pred_class = tf.argmax(y_pred, axis=1)
# testing/validation
y_pred_test = tf.nn.softmax(logits_test)
y_pred_test_class = tf.argmax(y_pred_test, axis=1)
correct_pred_test = tf.equal(y_pred_test_class, y_test_true_class)
accuracy_test = tf.reduce_mean(tf.cast(correct_pred_test, tf.float32))
# loss and optimizer
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y_true))
optimizer = tf.train.AdamOptimizer(learning_rate=LEARNING_RATE)
train_op = optimizer.minimize(loss)
# evaluation
correct_pred = tf.equal(y_pred_class, y_true_class)
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
elif CLASSIFIER == 'logistic_regression':
geom = LAYER_GEOMETRY[str(LAYER)]
p5 = tf.constant(0.5) # logistic regression threshold
W = tf.Variable(tf.zeros([geom[0] * geom[1], NUM_CLASSES]))
b = tf.Variable(tf.zeros([NUM_CLASSES]))
x = tf.reshape(x, [BATCH_SIZE, geom[0] * geom[1]])
x_test = tf.reshape(x_test, [1, geom[0] * geom[1]])
y_pred = tf.nn.softmax(tf.matmul(x, W) + b)
y_pred_class = tf.argmax(y_pred, axis=1)
cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_true * tf.log(y_pred), reduction_indices=[1]))
train_op = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
correct_pred = tf.equal(tf.argmax(y_pred, 1), y_true_class)
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# validation/testing
y_pred_test = tf.nn.softmax(tf.matmul(x, W) + b)
y_pred_test_class = tf.argmax(y_pred_test, axis=1)
correct_pred_test = tf.equal(tf.argmax(y_pred_test, 1), y_test_true_class)
accuracy_test = tf.reduce_mean(tf.cast(correct_pred_test, tf.float32))
# initializer
init_op = tf.global_variables_initializer()
saver = tf.train.Saver()
sess.run(init_op)
# start the training/testing steps
if training:
print("begin training")
step = 0
sess.run([
dataset_iterator.initializer,
dataset_test_iterator.initializer],
feed_dict={
input_filenames: filenames_train,
input_filenames_test: filenames_test
})
# loop until out of data
while True:
try:
train_result = sess.run([train_op, accuracy])
if step != 0 and step % 100 == 0:
print("step %s, accuracy = %s" % (step, train_result[1]))
# save the current model every 1000 steps
if step % 1000 == 0:
save_model(sess, saver, step)
# mini batch validation
if step % 10000 == 0:
mini_batch_size = 1000
else:
mini_batch_size = 50
test_accuracy = 0.0
for i in range(mini_batch_size):
test_result = sess.run([accuracy_test])
test_accuracy += test_result[0]
print("mini-batch validation accuracy = %.03f" % (test_accuracy / mini_batch_size))
# print("x = %s, logits = %s" % (train_result[2], train_result[3]))
step += 1
except tf.errors.OutOfRangeError:
print("data exhausted, saving final model")
save_model(sess, saver, 'final')
break
final_accuracy = -1.0
else:
print("begin testing")
step = 0
saver.restore(sess, LOAD_MODEL)
sess.run(dataset_iterator.initializer, feed_dict={input_filenames: filenames_test})
cumulative_accuracy = 0.0
predictions = []
true_classes = []
# loop until out of data
while True:
try:
test_result = sess.run([accuracy, y_pred_class, y_true_class])
cumulative_accuracy += test_result[0]
predictions.append(test_result[1])
true_classes.append(test_result[2])
if step % 100 == 0:
print("step %s, accuracy = %s, cumulative accuracy = %s" %
(step, test_result[0], cumulative_accuracy / step / BATCH_SIZE))
step += 1
except tf.errors.OutOfRangeError:
break
# wrap up, provide test results
final_accuracy = cumulative_accuracy / step / BATCH_SIZE
results_fd = open("runs/" + run_string + ".txt", 'w')
print("data exhausted, test results:")
print("steps = %s, cumulative accuracy = %.04f" % (step, final_accuracy))
results_fd.write("steps = %s, cumulative accuracy = %.04f\n" % (step, final_accuracy))
#for i, p in enumerate(predictions):
# print("[%s] true class = %s, predicted class = %s" % (i, true_classes[i], p))
cm = analysis.confusion_matrix(predictions, true_classes, class_list)
print("confusion matrix = %s" % cm)
analysis.plot_confusion_matrix(cm, class_list, "runs/" + run_string + ".pdf")
print("per-class accuracy:")
analysis.per_class_table(predictions, true_classes, class_list, "runs/" + run_string + '.csv')
sess.close()
return final_accuracy
def run_training():
# Get the sets of images and labels for training, validation, and
# Tell TensorFlow that the model will be built into the default Graph.
# get the list of classes
classes = get_class_list(CLASS_INDEX_FILE)
# Create model directory
if not os.path.exists(model_save_dir):
os.makedirs(model_save_dir)
use_pretrained_model = False
model_filename = "./sports1m_finetuning_ucf101.model"
with tf.Graph().as_default():
global_step = tf.get_variable(
'global_step',
[],
initializer=tf.constant_initializer(0),
trainable=False
)
images_placeholder, labels_placeholder = placeholder_inputs(
FLAGS.batch_size * gpu_num
)
tower_grads1 = []
tower_grads2 = []
logits = []
activations = []
sample_activations = {}
opt_stable = tf.train.AdamOptimizer(1e-4)
opt_finetuning = tf.train.AdamOptimizer(1e-3)
with tf.variable_scope('var_name') as var_scope:
weights = {
'wc1': _variable_with_weight_decay('wc1', [3, 3, 3, 3, 64], 0.0005),
'wc2': _variable_with_weight_decay('wc2', [3, 3, 3, 64, 128], 0.0005),
'wc3a': _variable_with_weight_decay('wc3a', [3, 3, 3, 128, 256], 0.0005),
'wc3b': _variable_with_weight_decay('wc3b', [3, 3, 3, 256, 256], 0.0005),
'wc4a': _variable_with_weight_decay('wc4a', [3, 3, 3, 256, 512], 0.0005),
'wc4b': _variable_with_weight_decay('wc4b', [3, 3, 3, 512, 512], 0.0005),
'wc5a': _variable_with_weight_decay('wc5a', [3, 3, 3, 512, 512], 0.0005),
'wc5b': _variable_with_weight_decay('wc5b', [3, 3, 3, 512, 512], 0.0005),
'wd1': _variable_with_weight_decay('wd1', [8192, 4096], 0.0005),
'wd2': _variable_with_weight_decay('wd2', [4096, 4096], 0.0005),
'out': _variable_with_weight_decay('wout', [4096, c3d_model.NUM_CLASSES], 0.0005)
}
biases = {
'bc1': _variable_with_weight_decay('bc1', [64], 0.000),
'bc2': _variable_with_weight_decay('bc2', [128], 0.000),
'bc3a': _variable_with_weight_decay('bc3a', [256], 0.000),
'bc3b': _variable_with_weight_decay('bc3b', [256], 0.000),
'bc4a': _variable_with_weight_decay('bc4a', [512], 0.000),
'bc4b': _variable_with_weight_decay('bc4b', [512], 0.000),
'bc5a': _variable_with_weight_decay('bc5a', [512], 0.000),
'bc5b': _variable_with_weight_decay('bc5b', [512], 0.000),
'bd1': _variable_with_weight_decay('bd1', [4096], 0.000),
'bd2': _variable_with_weight_decay('bd2', [4096], 0.000),
'out': _variable_with_weight_decay('bout', [c3d_model.NUM_CLASSES], 0.000),
}
for gpu_index in range(0, gpu_num):
with tf.device('/gpu:%d' % gpu_index):
varlist2 = [ weights['out'],biases['out'] ]
varlist1 = list( set(weights.values() + biases.values()) - set(varlist2) )
# compute the logits get the activations
logit, activation = c3d_model.inference_c3d(
images_placeholder[gpu_index * FLAGS.batch_size:(gpu_index + 1) * FLAGS.batch_size,:,:,:,:],
0.5,
FLAGS.batch_size,
weights,
biases
)
loss_name_scope = ('gpud_%d_loss' % gpu_index)
loss = tower_loss(
loss_name_scope,
logit,
labels_placeholder[gpu_index * FLAGS.batch_size:(gpu_index + 1) * FLAGS.batch_size]
)
grads1 = opt_stable.compute_gradients(loss, varlist1)
grads2 = opt_finetuning.compute_gradients(loss, varlist2)
tower_grads1.append(grads1)
tower_grads2.append(grads2)
logits.append(logit)
activations.append(activation)
logits = tf.concat(logits,0)
activations = tf.concat(activations, 0)
accuracy = tower_acc(logits, labels_placeholder)
tf.summary.scalar('accuracy', accuracy)
grads1 = average_gradients(tower_grads1)
grads2 = average_gradients(tower_grads2)
apply_gradient_op1 = opt_stable.apply_gradients(grads1)
apply_gradient_op2 = opt_finetuning.apply_gradients(grads2, global_step=global_step)
variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY)
variables_averages_op = variable_averages.apply(tf.trainable_variables())
train_op = tf.group(apply_gradient_op1, apply_gradient_op2, variables_averages_op)
null_op = tf.no_op()
# Create a saver for writing training checkpoints.
saver = tf.train.Saver(weights.values() + biases.values())
init = tf.global_variables_initializer()
# Create a session for running Ops on the Graph.
sess = tf.Session(
config=tf.ConfigProto(allow_soft_placement=True)
)
sess.run(init)
if os.path.isfile(model_filename) and use_pretrained_model:
saver.restore(sess, model_filename)
# Create summary writter
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter('./visual_logs/train', sess.graph)
test_writer = tf.summary.FileWriter('./visual_logs/test', sess.graph)
for step in xrange(FLAGS.max_steps):
start_time = time.time()
train_images, train_labels, _, _, _, sample_names = c3d_model.read_clip_and_label(
directory='/home/jordanc/datasets/UCF-101/UCF-101/',
filename='train-test-splits/trainlist01.txt',
batch_size=FLAGS.batch_size * gpu_num,
num_frames_per_clip=c3d_model.NUM_FRAMES_PER_CLIP,
crop_size=c3d_model.CROP_SIZE,
shuffle=True
)
_, activations_out = sess.run([train_op, activations], feed_dict={
images_placeholder: train_images,
labels_placeholder: train_labels
})
print("len sample_names = %s, activations: type = %s, len = %s, elem shape = %s" %
(len(sample_names), type(activations_out), len(activations_out), str(activations_out[0].shape)))
duration = time.time() - start_time
print('Step %d: %.3f sec' % (step, duration))
# save the activation for each sample name to the sample_activations dict
for i, s in enumerate(sample_names):
sample_activations[s] = activations_out[i]
# Save a checkpoint and evaluate the model periodically.
if (step) % 100 == 0 or (step + 1) == FLAGS.max_steps:
saver.save(sess, os.path.join(model_save_dir, 'c3d_ucf_model'), global_step=step)
print('Training Data Eval:')
summary, acc = sess.run(
[merged, accuracy],
feed_dict={images_placeholder: train_images,
labels_placeholder: train_labels
})
print ("accuracy: " + "{:.5f}".format(acc))
train_writer.add_summary(summary, step)
print('Validation Data Eval:')
val_images, val_labels, _, _, _, _ = c3d_model.read_clip_and_label(
directory='/home/jordanc/datasets/UCF-101/UCF-101/',
filename='train-test-splits/testlist01.txt',
batch_size=FLAGS.batch_size * gpu_num,
num_frames_per_clip=c3d_model.NUM_FRAMES_PER_CLIP,
crop_size=c3d_model.CROP_SIZE,
shuffle=True
)
summary, acc = sess.run(
[merged, accuracy],
feed_dict={
images_placeholder: val_images,
labels_placeholder: val_labels
})
print ("accuracy: " + "{:.5f}".format(acc))
test_writer.add_summary(summary, step)
print("done")