def main(): '''main function''' directory = '/home/jordanc/datasets/UCF-101/UCF-101/' tfrecord_dir = '/home/jordanc/datasets/UCF-101/tfrecords' classes = get_class_list(CLASS_INDEX_FILE) for root, dirs, files in os.walk(directory): if "v_" in root: # create a tfrecord from the jpegs in this directory # directory format is: # /home/jordanc/datasets/UCF-101/UCF-101/BreastStroke/v_BreastStroke_g23_c02 files.sort() video_name = os.path.basename(root) tfrecord_path = os.path.join(tfrecord_dir, video_name + ".tfrecord") tfrecord_writer = tf.python_io.TFRecordWriter(tfrecord_path) assert tfrecord_writer is not None, "tfrecord_writer instantiation failed" # get the label and number of frames, create features dict label = video_name.split('_')[1] label_int = classes.index(label) num_frames = len(files) features = {'label': _int64_feature(label_int), 'num_frames': _int64_feature(num_frames) } # read each image and add it to the tfrecord frames = [] for f in files: path = os.path.join(root, f) img = Image.open(path) features['height'] = _int64_feature(img.height) features['width'] = _int64_feature(img.width) if img.mode == 'RGB': features['channels'] = _int64_feature(3) else: assert False, "unhandled image mode %s" % img.mode frame = np.asarray(img).astype(np.uint8) frames.append(frame) # stack the frames into one large array frame_stack = np.stack(frames, axis=0) frame_stack = frame_stack.flatten() print("frame_stack shape = %s" % str(frame_stack.shape)) frames_raw = frame_stack.tostring() # use compression to save space # frames_raw = zlib.compress(frames_raw) features['frames'] = _bytes_feature(frames_raw) example = tf.train.Example(features=tf.train.Features(feature=features)) tfrecord_writer.write(example.SerializeToString()) tfrecord_writer.close() print("%s images of shape %s written to tfrecord file %s" % (num_frames, frame.shape, tfrecord_path))
def main(): filenames = [] list_file = sys.argv[1] with open(list_file) as fd: lines = fd.read().split('\n') while '' in lines: lines.remove('') for l in lines: f, c = l.split() filenames.append(f) class_list = get_class_list(CLASS_INDEX_FILE) # dataset iterator input_filenames = tf.placeholder(tf.string, shape=[None]) dataset = tf.data.TFRecordDataset(input_filenames) dataset = dataset.map(_parse_function) dataset = dataset.batch(1) dataset = dataset.repeat(1) dataset_iterator = dataset.make_initializable_iterator() x, y_true = dataset_iterator.get_next() y_true_class = tf.argmax(y_true, axis=1) init_op = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init_op) sess.run(dataset_iterator.initializer, feed_dict={input_filenames: filenames}) while True: try: x_out, y_true_out = sess.run([x, y_true]) except tf.errors.OutOfRangeError: break
def main(): '''main function''' # get the list of classes class_names = get_class_list(CLASS_INDEX_FILE) all_classes = True # argument collection run_name = sys.argv[1] sample = float(sys.argv[2]) included_classes = sys.argv[3] model_dir = sys.argv[4] #tfrecord_dir = sys.argv[5] if ',' in included_classes: all_classes = False included_classes = [x.strip() for x in sys.argv[3].split(',')] for c in included_classes: assert c in class_names, "%s is not a valid class" % c elif included_classes.isdigit(): all_classes = False included_classes = random.sample(class_names, int(included_classes)) elif included_classes == "all": all_classes = True else: print("Invalid value for class inclusion [%s]" % included_classes) sys.exit(1) print("Beginning run %s using %s sample size and %s classes" % (run_name, sample, included_classes)) # create the model object model = C3DModel(model_dir=model_dir) # open the csv data file and write the header to it run_csv_file = 'runs/%s.csv' % run_name if sys.version_info[0] == 3: run_csv_fd = open(run_csv_file, 'w', newline='', buffering=1) else: run_csv_fd = open(run_csv_file, 'wb', buffering=0) run_csv_writer = csv.writer(run_csv_fd, dialect='excel') run_csv_writer.writerow(['step_type', 'epoch', 'iteration', 'loss', 'accuracy', 'hit_at_5']) # get the list of files for train and test #train_files = file_split(TRAIN_SPLIT, model.tfrecord_dir, ONE_CLIP_PER_VIDEO) #test_files = file_split(TEST_SPLIT, model.tfrecord_dir, ONE_CLIP_PER_VIDEO) #if not all_classes: # train_files_filtered = [] # test_files_filtered = [] # for c in included_classes: # # print("c = %s" % c) # for t in train_files: # if c in t: # train_files_filtered.append(t) # for t in test_files: # if c in t: # test_files_filtered.append(t) # train_files = train_files_filtered # test_files = test_files_filtered # # print("train files = %s" % train_files) # # print("test files = %s" % test_files) # assert len(test_files) > 0 and len(train_files) > 0, 'test = %s, train = %s' % (len(test_files), len(train_files)) # # sample from the test and train files if necessary # if sample < 1.0: # sample_size = int(len(train_files) * sample) # train_files = random.sample(train_files, sample_size) # print("%s training samples" % sample_size) # sample_size_test = int(len(test_files) * sample) # test_files = random.sample(test_files, sample_size_test) # print("%s testing samples" % sample_size_test) # assert len(test_files) > 0 and len(train_files) > 0 # # reset size of mini-batch based on the number of test files # MINI_BATCH_SIZE = min(MINI_BATCH_SIZE, len(test_files)) # SHUFFLE_SIZE = min(SHUFFLE_SIZE, int(len(train_files) * 0.05)) # # throw out samples to fit the batch size # num_samples_batch_fit = len(train_files) - (len(train_files) % BATCH_SIZE) # if BALANCE_CLASSES: # # balance the classes for training # train_files = balance_classes(train_files) # random.shuffle(train_files) # train_files = train_files[0: num_samples_batch_fit] # print("Training samples = %s, testing samples = %s" % (len(train_files), len(test_files))) # print("Training class counts:") # print_class_counts(train_files) # print("Test class counts:") # print_class_counts(test_files) # open the log file run_log_file = 'runs/%s.log' % run_name run_log_fd = open(run_log_file, 'w', 0) run_log_fd.write("run name = %s\nsample = %s\nincluded_classes = %s\n" % (run_name, sample, included_classes)) run_log_fd.write("HYPER PARAMETERS:\n") run_log_fd.write("NUM_EPOCHS = %s\nMINI_BATCH_SIZE = %s\nTRAIN_SPLIT = %s\nTEST_SPLIT = %s\nSHUFFLE_SIZE = %s\n" % (NUM_EPOCHS, MINI_BATCH_SIZE, TRAIN_SPLIT, TEST_SPLIT, SHUFFLE_SIZE)) print("BATCH_SIZE = %s" % (BATCH_SIZE)) run_log_fd.write("VALIDATE_WITH_TRAIN = %s\nBALANCE_CLASSES = %s\n" % (VALIDATE_WITH_TRAIN, BALANCE_CLASSES)) run_log_fd.write("WEIGHT_STDDEV = %s\nBIAS = %s\n" % (c3d.WEIGHT_STDDEV, c3d.BIAS)) run_log_fd.write("WEIGHT_DECAY = %s\nBIAS_DECAY = %s\n" % (c3d.WEIGHT_DECAY, c3d.BIAS_DECAY)) run_log_fd.write("VARIABLE_TYPE = %s\n" % (VARIABLE_TYPE)) run_log_fd.write("ONE_CLIP_PER_VIDEO = %s" % (ONE_CLIP_PER_VIDEO)) run_log_fd.write("LEARNING_RATE_DECAY = %s" % (LEARNING_RATE_DECAY)) run_log_fd.write("OPTIMIZER = %s" % (OPTIMIZER)) run_log_fd.write("IMAGE_CROPPING = %s" % (IMAGE_CROPPING)) run_log_fd.write("IMAGE_NORMALIZATION = %s" % (IMAGE_NORMALIZATION)) run_log_fd.write("learning rate = %s" % (model.learning_rate)) # run_log_fd.write("Training samples = %s, testing samples = %s\n" % (len(train_files), len(test_files))) # Tensorflow configuration config = tf.ConfigProto(allow_soft_placement=True) with tf.Session(config=config) as sess: #sess = tf_debug.LocalCLIDebugWrapperSession(sess) # init variables # tf.set_random_seed(1234) # coord = tf.train.Coordinator() # threads = tf.train.start_queue_runners(coord=coord) with tf.variable_scope('var_name') as var_scope: weights = { 'wc1': weight_variable('wc1', [3, 3, 3, 3, 64], WEIGHT_STDDEV), 'wc2': weight_variable('wc2', [3, 3, 3, 64, 128], WEIGHT_STDDEV), 'wc3a': weight_variable('wc3a', [3, 3, 3, 128, 256], WEIGHT_STDDEV), 'wc3b': weight_variable('wc3b', [3, 3, 3, 256, 256], WEIGHT_STDDEV), 'wc4a': weight_variable('wc4a', [3, 3, 3, 256, 512], WEIGHT_STDDEV), 'wc4b': weight_variable('wc4b', [3, 3, 3, 512, 512], WEIGHT_STDDEV), 'wc5a': weight_variable('wc5a', [3, 3, 3, 512, 512], WEIGHT_STDDEV), 'wc5b': weight_variable('wc5b', [3, 3, 3, 512, 512], WEIGHT_STDDEV), 'wd1': weight_variable('wd1', [8192, 4096], WEIGHT_STDDEV), 'wd2': weight_variable('wd2', [4096, 4096], WEIGHT_STDDEV), 'wdout': weight_variable('wdout', [4096, model.num_classes], WEIGHT_STDDEV), } biases = { 'bc1': bias_variable('bc1', [64], BIAS), 'bc2': bias_variable('bc2', [128], BIAS), 'bc3a': bias_variable('bc3a', [256], BIAS), 'bc3b': bias_variable('bc3b', [256], BIAS), 'bc4a': bias_variable('bc4a', [512], BIAS), 'bc4b': bias_variable('bc4b', [512], BIAS), 'bc5a': bias_variable('bc5a', [512], BIAS), 'bc5b': bias_variable('bc5b', [512], BIAS), 'bd1': bias_variable('bd1', [4096], BIAS), 'bd2': bias_variable('bd2', [4096], BIAS), 'bdout': bias_variable('bdout', [model.num_classes], BIAS), } # placeholders and constants # y_true = tf.placeholder(tf.float32, shape=[None, NUM_CLASSES], name='y_true') x = tf.placeholder(tf.float32, shape=[BATCH_SIZE, model.frames_per_clip, 112, 112, 3]) y_true = tf.placeholder(tf.int64, shape=[BATCH_SIZE]) # x_test = tf.placeholder(tf.float32, shape=[1, model.frames_per_clip, 112, 112, 3]) # y_true_test = tf.placeholder(tf.int64, shape=[1]) y_true_one_hot = tf.one_hot(y_true, depth=model.num_classes) # y_true_test_one_hot = tf.one_hot(y_true_test, depth=model.num_classes) y_true_class = tf.argmax(y_true_one_hot, axis=1) # y_true_test_class = tf.argmax(y_true_test_one_hot, axis=1) logits = model.inference_3d(x, weights, biases, BATCH_SIZE, True) # logits = model.c3d(x, training=True) y_pred = tf.nn.softmax(logits) y_pred_class = tf.argmax(y_pred, axis=1) correct_pred = tf.equal(y_pred_class, y_true_class) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) hit_5 = tf.nn.in_top_k(logits, y_true_class, 5) top_5 = tf.nn.top_k(logits, k=5) # loss and optimizer # loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y_true)) loss_op = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y_true_class, name="softmax"), name="reduce_mean") if OPTIMIZER == 'SGD': learning_rate = tf.placeholder(tf.float32, shape=[]) #learning_rate = tf.train.exponential_decay(learning_rate=model.learning_rate, global_step=global_step, # decay_steps=(4 * len(train_files)), decay_rate=0.96, # staircase=True) optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate, name="optimizer") elif OPTIMIZER == 'Adam': optimizer = tf.train.AdamOptimizer(LEARNING_RATE) train_op = optimizer.minimize(loss_op, name="train") # model evaluation # logits_test = model.c3d(x_test, training=False) # logits_test = model.inference_3d(x_test, weights, biases, 1, False) # y_pred_test = tf.nn.softmax(logits_test) # y_pred_test_class = tf.argmax(y_pred_test, axis=1) # eval_hit_5 = tf.nn.in_top_k(logits_test, y_true_test_class, 5) # eval_top_5 = tf.nn.top_k(logits_test, k=5) # eval_correct_pred = tf.equal(y_pred_test_class, y_true_test_class) # eval_accuracy = tf.reduce_mean(tf.cast(eval_correct_pred, tf.float32)) init_op = tf.global_variables_initializer() saver = tf.train.Saver(weights.values() + biases.values()) # TRAINING sess.run(init_op) report_step = 20 print("Beginning training") in_epoch = 1 print("START EPOCH %s" % in_epoch) start = time.time() epoch_start = time.time() # sess.run(train_iterator.initializer, feed_dict={train_filenames: train_files}) # if VALIDATE_WITH_TRAIN: # sess.run(test_iterator.initializer, feed_dict={test_filenames: train_files}) # else: # sess.run(test_iterator.initializer, feed_dict={test_filenames: test_files}) step = 0 _, _, _, num_samples = get_image_batch(TRAIN_SPLIT, BATCH_SIZE, model.frames_per_clip, model.num_classes) max_steps = int((num_samples * NUM_EPOCHS / BATCH_SIZE) * sample) print("max_steps = %s" % max_steps) # num_batches_per_epoch = len(train_files) / BATCH_SIZE while step < max_steps: if step != 0 and step % int(num_samples / BATCH_SIZE) == 0: in_epoch += 1 print("START EPOCH %s" % in_epoch) if in_epoch % 4 == 0 and OPTIMIZER != 'Adam': model.current_learning_rate = model.current_learning_rate * LEARNING_RATE_DECAY print("learning rate adjusted to %g" % model.current_learning_rate) step_start = time.time() x_feed, y_feed, _, num_samples = get_image_batch(TRAIN_SPLIT, BATCH_SIZE, model.frames_per_clip, model.num_classes, crop=IMAGE_CROPPING, normalize=IMAGE_NORMALIZATION) # print("x_feed = %s" % str(x_feed.shape)) sess.run(train_op, feed_dict={x: x_feed, y_true: y_feed}) step_end = time.time() print("epoch %s step %s - %.3fs" % (in_epoch, step, step_end - step_start)) if step % 10 == 0: # train accuracy results = sess.run([accuracy, y_true_class, y_pred_class, logits], feed_dict={x: x_feed, y_true: y_feed}) acc = results[0] y_true_class_out = results[1] y_pred_class_out = results[2] logits_out = results[3][0] #print("Train accuracy = %s, y_true = %s, y_prediction = %s, logits = %s" % (acc, y_true_class_out, y_pred_class_out, logits_out)) print("Train accuracy = %s" % acc) # validation accuracy x_feed, y_feed, _, num_samples = get_image_batch(TEST_SPLIT, BATCH_SIZE, model.frames_per_clip, model.num_classes, crop=TEST_IMAGE_CROPPING, normalize=IMAGE_NORMALIZATION) acc = sess.run(accuracy, feed_dict={x: x_feed, y_true: y_feed}) print("Validation accuracy = %s" % acc) if step % 100 == 0 or step == max_steps - 1: # save a model checkpoint save_path = os.path.join(model.model_dir, "model_%s_step_%s.ckpt" % (run_name, step)) save_path = saver.save(sess, save_path) step += 1 # if j != 0 and j % num_samples < BATCH_SIZE: # # end of epoch # # save a model checkpoint and report end of epoch information # save_path = os.path.join(model.model_dir, "model_epoch_%s.ckpt" % in_epoch) # save_path = saver.save(sess, save_path) # epoch_end = time.time() # train_time = str(datetime.timedelta(seconds=epoch_end - epoch_start)) # print("END EPOCH %s, steps completed = %s, epoch training time: %s" % (in_epoch, j, train_time)) # print("model checkpoint saved to %s\n\n" % save_path) # # test the network # test_results = test_network(sess, model, TEST_SPLIT, run_name, in_epoch) # run_csv_writer.writerow(test_results) # in_epoch += 1 # if in_epoch % 4 == 0 and OPTIMIZER != 'Adam': # model.current_learning_rate = model.current_learning_rate * LEARNING_RATE_DECAY # print("learning rate adjusted to %g" % model.current_learning_rate) # print("START EPOCH %s" % in_epoch) # epoch_start = time.time() # train_result = sess.run([train_op, loss_op, accuracy, x, y_true, y_true_class, y_pred, y_pred_class, logits, hit_5], # feed_dict={x: x_feed, # y_true: y_feed, # learning_rate: model.current_learning_rate}) # loss_op_out = train_result[1] # train_acc = train_result[2] # x_actual = train_result[3] # y_true_actual = train_result[4] # y_true_class_actual = train_result[5] # y_pred_actual = train_result[6] # y_pred_class_actual = train_result[7] # logits_out = train_result[8] # hit5_out = train_result[9] # # print("train_acc = %s" % train_acc) # # train_acc is the number of correct responses divided by the batch size # # e.g. 3 correct responses/30 batch size = 0.1 # train_acc_accum += train_acc # # print("hit_5_out = %s" % hit5_out) # # count the trues in the hit_5_out array # hit5_counter = collections.Counter(hit5_out) # hit5_out_trues = float(hit5_counter[True]) # # print("%s trues out of %s" % (hit5_out_trues, len(hit5_out))) # train_hit5_accum += float(hit5_out_trues / len(hit5_out)) # # report out results and run a test mini-batch every now and then # if j != 0 and j % (report_step * BATCH_SIZE) == 0: # #print("logits = %s" % logits_out) # #print("x = %s" % x_actual) # #print("y_true = %s, y_true_class = %s, y_pred = %s, y_pred_class = %s" % (y_true_actual, y_true_class_actual, y_pred_actual, y_pred_class_actual)) # #print("train_acc = %s" % train_acc) # print("hit5_out = %s, length = %s" % (hit5_out, len(hit5_out))) # print("%s trues out of %s, %s accuracy" % (hit5_out_trues, len(hit5_out), hit5_out_trues / len(hit5_out))) # run_time = time.time() # run_time_str = str(datetime.timedelta(seconds=run_time - start)) # train_step_acc = train_acc_accum / report_step * BATCH_SIZE # # mini batch accuracy - every 5 report step iterations # if j % (report_step * BATCH_SIZE * 5) == 0: # mini_batch_acc = 0.0 # mini_batch_hit5 = 0.0 # for k in range(MINI_BATCH_SIZE): # if IMAGE_CROPPING == 'random': # test_crop = 'center' # else: # test_crop = IMAGE_CROPPING # x_feed, y_feed, _, num_samples = get_image_batch(TRAIN_SPLIT, 1, model.frames_per_clip, model.num_classes, crop=test_crop) # acc, hit5_out, top_5_out, x_out = sess.run([eval_accuracy, eval_hit_5, eval_top_5, x_test], # feed_dict={x_test: x_feed, y_true_test: y_feed}) # # print("type(x) = %s, x = %s" % (type(x_out), x_out)) # mini_batch_acc += acc # hit5_counter = collections.Counter(hit5_out) # hit5_out_trues = hit5_counter[True] # # print("%s trues out of %s" % (hit5_out_trues, len(hit_5_out))) # mini_batch_hit5 += float(hit5_out_trues / len(hit5_out)) # mini_batch_acc = mini_batch_acc / MINI_BATCH_SIZE # mini_batch_hit5 = mini_batch_hit5 / MINI_BATCH_SIZE # print("\tstep %s - epoch %s run time = %s, loss = %s, mini-batch accuracy = %s, hit@5 = %s, y_true = %s, top 5 = %s" % # (j, in_epoch, run_time_str, loss_op_out, mini_batch_acc, mini_batch_hit5, y_true_class_actual, top_5_out)) # csv_row = ['mini-batch', in_epoch, j, loss_op_out, mini_batch_acc, mini_batch_hit5] # else: # train_acc_accum = train_acc_accum / report_step # train_hit5_accum = train_hit5_accum / report_step # print("\tstep %s - epoch %s run time = %s, loss = %s, train accuracy = %s, hit@5 = %s" % # (j, in_epoch, run_time_str, loss_op_out, train_acc_accum, train_hit5_accum)) # csv_row = ['train', in_epoch, j, loss_op_out, train_acc_accum, train_hit5_accum] # # write the csv data to # run_csv_writer.writerow(csv_row) # train_acc_accum = 0.0 # train_hit5_accum = 0.0 # j += BATCH_SIZE # print("end training epochs") # # end of epoch # # save a model checkpoint and report end of epoch information # save_path = os.path.join(model.model_dir, "model_epoch_%s.ckpt" % in_epoch) # save_path = saver.save(sess, save_path) # epoch_end = time.time() # train_time = str(datetime.timedelta(seconds=epoch_end - epoch_start)) # print("END EPOCH %s, iterations = %s, epoch training time: %s" % (in_epoch, j, train_time)) # print("model checkpoint saved to %s\n\n" % save_path) # # final test # test_results = test_network(sess, model, TEST_SPLIT, run_name, in_epoch) # run_csv_writer.writerow(test_results) # end = time.time() # total_time = str(datetime.timedelta(seconds=end - start)) # print("END TRAINING total training time: %s" % (total_time)) #coord.request_stop() #coord.join(threads) # sess.close() run_csv_fd.close() run_log_fd.close()
def run_test(): # get the list of classes classes = get_class_list(CLASS_INDEX_FILE) model_name = "/home/jordanc/datasets/UCF-101/model_ckpts/c3d_ucf_model-4999" test_list_file = 'train-test-splits/testlist01.txt' num_test_videos = len(list(open(test_list_file, 'r'))) print("Number of test videos={}".format(num_test_videos)) # Get the sets of images and labels for training, validation, and images_placeholder, labels_placeholder = placeholder_inputs( FLAGS.batch_size * gpu_num) with tf.variable_scope('var_name') as var_scope: weights = { 'wc1': _variable_with_weight_decay('wc1', [3, 3, 3, 3, 64], 0.04, 0.00), 'wc2': _variable_with_weight_decay('wc2', [3, 3, 3, 64, 128], 0.04, 0.00), 'wc3a': _variable_with_weight_decay('wc3a', [3, 3, 3, 128, 256], 0.04, 0.00), 'wc3b': _variable_with_weight_decay('wc3b', [3, 3, 3, 256, 256], 0.04, 0.00), 'wc4a': _variable_with_weight_decay('wc4a', [3, 3, 3, 256, 512], 0.04, 0.00), 'wc4b': _variable_with_weight_decay('wc4b', [3, 3, 3, 512, 512], 0.04, 0.00), 'wc5a': _variable_with_weight_decay('wc5a', [3, 3, 3, 512, 512], 0.04, 0.00), 'wc5b': _variable_with_weight_decay('wc5b', [3, 3, 3, 512, 512], 0.04, 0.00), 'wd1': _variable_with_weight_decay('wd1', [8192, 4096], 0.04, 0.001), 'wd2': _variable_with_weight_decay('wd2', [4096, 4096], 0.04, 0.002), 'out': _variable_with_weight_decay('wout', [4096, c3d_model.NUM_CLASSES], 0.04, 0.005) } biases = { 'bc1': _variable_with_weight_decay('bc1', [64], 0.04, 0.0), 'bc2': _variable_with_weight_decay('bc2', [128], 0.04, 0.0), 'bc3a': _variable_with_weight_decay('bc3a', [256], 0.04, 0.0), 'bc3b': _variable_with_weight_decay('bc3b', [256], 0.04, 0.0), 'bc4a': _variable_with_weight_decay('bc4a', [512], 0.04, 0.0), 'bc4b': _variable_with_weight_decay('bc4b', [512], 0.04, 0.0), 'bc5a': _variable_with_weight_decay('bc5a', [512], 0.04, 0.0), 'bc5b': _variable_with_weight_decay('bc5b', [512], 0.04, 0.0), 'bd1': _variable_with_weight_decay('bd1', [4096], 0.04, 0.0), 'bd2': _variable_with_weight_decay('bd2', [4096], 0.04, 0.0), 'out': _variable_with_weight_decay('bout', [c3d_model.NUM_CLASSES], 0.04, 0.0), } logits = [] activations = [] for gpu_index in range(0, gpu_num): with tf.device('/gpu:%d' % gpu_index): logit, activation = c3d_model.inference_c3d( images_placeholder[gpu_index * FLAGS.batch_size:(gpu_index + 1) * FLAGS.batch_size, :, :, :, :], 0.6, FLAGS.batch_size, weights, biases) logits.append(logit) activations.append(activation) logits = tf.concat(logits, 0) activations = tf.concat(activations, 0) norm_score = tf.nn.softmax(logits) saver = tf.train.Saver() sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) init = tf.global_variables_initializer() sess.run(init) # Create a saver for writing training checkpoints. saver.restore(sess, model_name) # And then after everything is built, start the training loop. bufsize = 0 write_file = open("predict_ret.txt", "w+", bufsize) next_start_pos = 0 all_steps = int((num_test_videos - 1) / (FLAGS.batch_size * gpu_num) + 1) for step in xrange(all_steps): # Fill a feed dictionary with the actual set of images and labels # for this particular training step. start_time = time.time() sample_output = c3d_model.read_clip_and_label( directory='/home/jordanc/datasets/UCF-101/UCF-101/', filename=test_list_file, batch_size=FLAGS.batch_size * gpu_num, start_pos=next_start_pos) test_images = sample_output[0] test_labels = sample_output[1] next_start_pos = sample_output[2] valid_len = sample_output[4] sample_names = sample_output[5] predict_score, activations_out = sess.run( [norm_score, activations], feed_dict={images_placeholder: test_images}) for i in range(0, valid_len): true_label = test_labels[i], top1_predicted_label = np.argmax(predict_score[i]) # Write results: true label, class prob for true label, predicted label, class prob for predicted label write_file.write('{}, {}, {}, {}\n'.format( true_label[0], predict_score[i][true_label], top1_predicted_label, predict_score[i][top1_predicted_label])) write_file.close() print("done")
from __future__ import print_function import sys from tfrecord_gen import CLASS_INDEX_FILE, get_class_list classes = get_class_list(CLASS_INDEX_FILE) filename = sys.argv[1] fd = open(filename) text = fd.read() lines = text.split('\n') with open(filename + "fixed", 'w+') as fd: for l in lines: if len(l) > 0: label, folder = l.strip().split('/') # print("%s/%s" % (label, folder)) class_name = str(classes.index(label)) string = label + '\\' + folder + ' ' + class_name print(string) fd.write(string + "\n")
def iad_nn(run_string, json_input_train, json_input_test): '''main function''' if LOAD_MODEL is None: training = True else: training = False # get the list of classes class_list = get_class_list(CLASS_INDEX_FILE) img_geom = LAYER_GEOMETRY[str(LAYER)] # get the list of filenames #print("loading train file list from %s" % TRAIN_FILE_LIST) #filenames_train = list_to_filenames(TRAIN_FILE_LIST, balance_classes=False) #filenames_test = list_to_filenames(TEST_FILE_LIST) #print("%s training files, %s testing files" % (len(filenames_train), len(filenames_test))) #random.shuffle(filenames_train) #if TRAINING_DATA_SAMPLE != 1.0: # filenames_train = random.sample(filenames_train, int(TRAINING_DATA_SAMPLE * len(filenames_train))) # ensure filenames list is evenly divisable by batch size #pad_filenames = len(filenames_train) % BATCH_SIZE #filenames_train.extend(filenames_train[0:pad_filenames]) # create the TensorFlow sessions config = tf.ConfigProto(allow_soft_placement=True) sess = tf.Session(config=config) # setup the CNN if CLASSIFIER == 'softmax': weights, biases = get_variables_softmax('ucf101_iad') elif CLASSIFIER == 'lenet': weights, biases = get_variables_lenet('ucf101_iad') elif CLASSIFIER == 'mctnet': weights, biases = get_variables_mctnet('ucf101_iad') elif CLASSIFIER == 'autoencode': weights, biases = get_variables_autoencode('ucf101_iad') # placeholders #input_filenames = tf.placeholder(tf.string, shape=[None]) #input_filenames_test = tf.placeholder(tf.string, shape=[None]) x = tf.placeholder(tf.float32, shape=( None, LAYER_GEOMETRY[str(LAYER)][0], LAYER_GEOMETRY[str(LAYER)][1] )) y_true = tf.placeholder(tf.int64, shape=(None)) #x_test = tf.placeholder(tf.float32, shape=( # 1, # LAYER_GEOMETRY[str(LAYER)][0], # LAYER_GEOMETRY[str(LAYER)][1] # )) #y_test_true = tf.placeholder(tf.int64, shape=(1)) # testing dataset #dataset_test = tf.data.TFRecordDataset(input_filenames_test) #dataset_test = dataset_test.map(parse_function_test) #dataset_test = dataset_test.batch(1) #dataset_test = dataset_test.shuffle(500) #dataset_test = dataset_test.repeat(1000000) #dataset_test_iterator = dataset_test.make_initializable_iterator() #x_test, y_test_true = dataset_test_iterator.get_next() #y_test_true_one_hot = tf.one_hot(y_test_true, depth=NUM_CLASSES, dtype=tf.int32) #y_test_true_class = tf.argmax(y_test_true_one_hot, axis=1) # training or evaluation dataset #dataset = tf.data.TFRecordDataset(input_filenames) #dataset = dataset.map(parse_function_train) #if tf.__version__ == '1.11.0': # dataset = dataset.batch(BATCH_SIZE, drop_remainder=True) #else: # dataset = dataset.batch(BATCH_SIZE) #if training: # dataset = dataset.shuffle(200) # dataset = dataset.repeat(EPOCHS) #else: # dataset = dataset.repeat(1) #dataset_iterator = dataset.make_initializable_iterator() #x, y_true = dataset_iterator.get_next() y_true_one_hot = tf.one_hot(y_true, depth=NUM_CLASSES, dtype=tf.int32) y_true_class = tf.argmax(y_true_one_hot, axis=1) print("x shape = %s" % x.get_shape().as_list()) #print("y_true shape = %s" % y_true.get_shape().as_list()) # get neural network response if CLASSIFIER in ['softmax', 'autoencode', 'lenet', 'mctnet']: if CLASSIFIER == 'softmax': # layer 1 print("LAYER = %s, geom = %s" % (LAYER, img_geom)) x = tf.reshape(x, [-1, img_geom[0] * img_geom[1]]) #x_test = tf.reshape(x_test, [1, img_geom[0] * img_geom[1]]) logits, conv_layers = softmax_regression(x, BATCH_SIZE, weights, biases, DROPOUT) #logits_test, _ = softmax_regression(x_test, 1, weights, biases, DROPOUT) elif CLASSIFIER == 'lenet': logits, conv_layers = cnn_lenet(x, BATCH_SIZE, weights, biases, DROPOUT) #logits_test, conv_layers = cnn_lenet(x_test, 1, weights, biases, DROPOUT) elif CLASSIFIER == 'mctnet': logits, conv_layers = cnn_mctnet(x, BATCH_SIZE, weights, biases, DROPOUT) #logits_test, conv_layers = cnn_mctnet(x_test, 1, weights, biases, DROPOUT) elif CLASSIFIER == 'autoencode': geom = LAYER_GEOMETRY[str(LAYER)] # layer 1 x = tf.reshape(x, [-1, img_geom[0] * img_geom[1]]) #x_test = tf.reshape(x_test, [1, img_geom[0] * img_geom[1]]) x_autoencode, conv_layers = autoencode(x, BATCH_SIZE, weights, biases) #x_test_autoencode, _ = autoencode(x_test, 1, weights, biases) # softmax regression with output # final logits = tf.matmul(x_autoencode, weights['W_out']) + biases['b_out'] #logits_test = tf.matmul(x_test_autoencode, weights['W_out']) + biases['b_out'] print("logits shape = %s" % logits.get_shape().as_list()) y_pred = tf.nn.softmax(logits) y_pred_class = tf.argmax(y_pred, axis=1) # testing/validation #y_pred_test = tf.nn.softmax(logits_test) #y_pred_test_class = tf.argmax(y_pred_test, axis=1) #correct_pred_test = tf.equal(y_pred_test_class, y_test_true_class) #accuracy_test = tf.reduce_mean(tf.cast(correct_pred_test, tf.float32)) # loss and optimizer loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y_true_one_hot)) #loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y_true)) optimizer = tf.train.AdamOptimizer(learning_rate=LEARNING_RATE) train_op = optimizer.minimize(loss) # evaluation correct_pred = tf.equal(y_pred_class, y_true_class) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) elif CLASSIFIER == 'logistic_regression': geom = LAYER_GEOMETRY[str(LAYER)] p5 = tf.constant(0.5) # logistic regression threshold W = tf.Variable(tf.zeros([geom[0] * geom[1], NUM_CLASSES])) b = tf.Variable(tf.zeros([NUM_CLASSES])) x = tf.reshape(x, [BATCH_SIZE, img_geom[0] * img_geom[1]]) #x_test = tf.reshape(x_test, [1, img_geom[0] * img_geom[1]]) y_pred = tf.nn.softmax(tf.matmul(x, W) + b) y_pred_class = tf.argmax(y_pred, axis=1) cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_true * tf.log(y_pred), reduction_indices=[1])) train_op = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) correct_pred = tf.equal(tf.argmax(y_pred, 1), y_true_class) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # validation/testing #y_pred_test = tf.nn.softmax(tf.matmul(x, W) + b) #y_pred_test_class = tf.argmax(y_pred_test, axis=1) #correct_pred_test = tf.equal(tf.argmax(y_pred_test, 1), y_test_true_class) #accuracy_test = tf.reduce_mean(tf.cast(correct_pred_test, tf.float32)) # initializer init_op = tf.global_variables_initializer() saver = tf.train.Saver() sess.run(init_op) # start the training/testing steps if training: num_steps = int((json_input_train.num_files * EPOCHS) / BATCH_SIZE) print("begin training, steps = %s" % num_steps) step = 0 #sess.run([ # dataset_iterator.initializer, # dataset_test_iterator.initializer], # feed_dict={ # input_filenames: filenames_train, # input_filenames_test: filenames_test # }) # loop until out of data start = time.time() while step <= num_steps: #try: input_x, input_y, _ = json_input_train.get_batch() input_x = input_x.reshape((BATCH_SIZE, img_geom[0] * img_geom[1])) #input_y = tf.one_hot(input_y, depth=NUM_CLASSES, dtype=tf.int32) input_y = input_y.reshape((BATCH_SIZE)) train_result = sess.run([train_op, accuracy, y_pred, logits, weights, loss], feed_dict={x: input_x, y_true: input_y}) if step != 0 and step % 100 == 0: end = time.time() run_time = end - start start = time.time() # calculate discard rate discard_rate = json_input_train.discards / (step * BATCH_SIZE_TRAIN) print("step %s, accuracy = %.03f, loss = %.03f, discards = %.03f, time = %.03fs" % (step, train_result[1], train_result[5], discard_rate, run_time)) #print("\tx shape = %s, max = %s, min = %s, mean = %s" % (train_result[2].shape, train_result[2].max(), train_result[2].min(), train_result[2].mean())) #print("\ty_true (shape = %s) = %s" % (train_result[3].shape, train_result[3])) #print("\ty_pred (shape = %s) = %s" % (train_result[4].shape, train_result[4])) #print("\tlogits (shape = %s) = %s" % (train_result[5].shape, train_result[5])) #print("\tweights (shape = %s) = %s" % (train_result[6]['W_0'].shape, train_result[6]['W_0'])) # save the current model every 1000 steps if step % 1000 == 0: save_model(sess, saver, step) # mini batch validation if step % 10000 == 0: mini_batch_size = 1000 else: mini_batch_size = 50 test_accuracy = 0.0 for i in range(mini_batch_size): input_x, input_y, _ = json_input_test.get_batch() input_x = input_x.reshape((1, img_geom[0] * img_geom[1])) input_y = input_y.reshape((1)) test_result = sess.run([accuracy], feed_dict={x: input_x, y_true: input_y}) test_accuracy += test_result[0] print("mini-batch validation accuracy = %.03f" % (test_accuracy / mini_batch_size)) # print("x = %s, logits = %s" % (train_result[2], train_result[3])) step += 1 #except tf.errors.OutOfRangeError: # print("data exhausted, saving final model") # save_model(sess, saver, 'final') # break save_model(sess, saver, 'final') final_accuracy = -1.0 sess.close() print("end training") return None, None else: num_steps = json_input_test.num_files print("begin testing, steps = %s" % num_steps) step = 0 saver.restore(sess, LOAD_MODEL) cumulative_accuracy = 0.0 predictions = [] true_classes = [] # clip/video accuracy # ['sample_name': [accuracy@1, accuracy@1]] # ['video_name': [accuracy@1, accuracy@1]] clip_accuracy = {} video_accuracy = {} # loop until out of data while step <= num_steps: input_x, input_y, sample_name = json_input_test.get_batch() input_x = input_x.reshape((1, img_geom[0] * img_geom[1])) input_y = input_y.reshape((1)) test_result = sess.run([accuracy, y_pred_class, y_true_class], feed_dict={x: input_x, y_true: input_y}) # add result to the clip and video accuracy structures sample_name = sample_name[0] video_name = '_'.join(sample_name.split('_')[0:3]) if sample_name in clip_accuracy.keys(): clip_accuracy[sample_name].append(test_result[0]) else: clip_accuracy[sample_name] = [test_result[0]] if video_name in video_accuracy.keys(): video_accuracy[video_name].append(test_result[0]) else: video_accuracy[video_name] = [test_result[0]] cumulative_accuracy += test_result[0] predictions.append(test_result[1]) true_classes.append(test_result[2]) if step % 100 == 0: print("step %s, accuracy = %s, cumulative accuracy = %s" % (step, test_result[0], cumulative_accuracy / step / BATCH_SIZE)) step += 1 # wrap up, provide test results final_accuracy = cumulative_accuracy / step / BATCH_SIZE print("data exhausted, test results:") print("steps = %s, cumulative accuracy = %.04f" % (step, final_accuracy)) with open("runs/" + run_string + ".txt", 'a+') as results_fd: results_fd.write("steps = %s, cumulative accuracy = %.04f\n" % (step, final_accuracy)) #for i, p in enumerate(predictions): # print("[%s] true class = %s, predicted class = %s" % (i, true_classes[i], p)) cm = analysis.confusion_matrix(predictions, true_classes, class_list) print("confusion matrix = %s" % cm) analysis.plot_confusion_matrix(cm, class_list, "runs/" + run_string + ".pdf") #print("per-class accuracy:") analysis.per_class_table(predictions, true_classes, class_list, "runs/" + run_string + '.csv') sess.close() return clip_accuracy, video_accuracy
def main(): '''main function''' if LOAD_MODEL is None: training = True run_type = 'train' else: training = False run_type = 'test' # get the run name run_name = sys.argv[1] save_settings(run_name + "_" + run_type) # get the list of classes class_list = get_class_list(CLASS_INDEX_FILE) # get the list of filenames print("loading file list from %s" % FILE_LIST) filenames = list_to_filenames(FILE_LIST) print("%s files" % len(filenames)) if training: random.shuffle(filenames) if TRAINING_DATA_SAMPLE != 1.0: filenames = random.sample( filenames, int(TRAINING_DATA_SAMPLE * len(filenames))) # ensure filenames list is evenly divisable by batch size pad_filenames = len(filenames) % BATCH_SIZE filenames.extend(filenames[0:pad_filenames]) print("filenames = %s..." % filenames[0:5]) # create the TensorFlow sessions config = tf.ConfigProto(allow_soft_placement=True) sess = tf.Session(config=config) # dataset iterator input_filenames = tf.placeholder(tf.string, shape=[None]) dataset = tf.data.TFRecordDataset(input_filenames) dataset = dataset.map(_parse_function) dataset = dataset.batch(BATCH_SIZE) if training: dataset = dataset.shuffle(100) dataset = dataset.repeat(EPOCHS) else: dataset = dataset.repeat(1) dataset_iterator = dataset.make_initializable_iterator() x, y_true = dataset_iterator.get_next() y_true_class = tf.argmax(y_true, axis=1) # placeholders prediction_grid = tf.placeholder(shape=[None, 2], dtype=tf.float32) # variables b = tf.Variable(tf.random_normal(shape=[NUM_CLASSES, BATCH_SIZE])) gamma = tf.constant(GAMMA) dist = tf.reduce_sum(tf.square(x), 1) dist = tf.reshape(dist, [-1, 1]) sq_dists = tf.multiply(2., tf.matmul(x, tf.transpose(x))) svm_kernel = tf.exp(tf.multiply(gamma, tf.abs(sq_dists))) # batch multiplication def reshape_matmul(mat, _size): v1 = tf.expand_dims(mat, 1) v2 = tf.reshape(v1, [NUM_CLASSES, _size, 1]) return tf.matmul(v2, v1) # compute SVM model first_term = tf.reduce_sum(b) b_vec_cross = tf.matmul(tf.transpose(b), b) y_true_cross = reshape_matmul(y_true, BATCH_SIZE) second_term = tf.reduce_sum( tf.multiply(svm_kernel, tf.multiply(b_vec_cross, y_true_cross)), [1, 2]) loss = tf.reduce_sum(tf.negative(tf.subtract(first_term, second_term))) # Gaussian (RBF) prediction kernel rA = tf.reshape(tf.reduce_sum(tf.square(x), 1), [-1, 1]) rB = tf.reshape(tf.reduce_sum(tf.square(prediction_grid), 1), [-1, 1]) pred_sq_dist = tf.add( tf.subtract( rA, tf.multiply(2., tf.matmul(x, tf.transpose(prediction_grid)))), tf.transpose(rB)) pred_kernel = tf.exp(tf.multiply(gamma, tf.abs(pred_sq_dist))) prediction_output = tf.matmul(tf.multiply(y_true, b), pred_kernel) y_pred_class = tf.argmax( prediction_output - tf.expand_dims(tf.reduce_mean(prediction_output, 1), 1), axis=0) accuracy = tf.reduce_mean( tf.cast(tf.equal(prediction, tf.argmax(y_true, 0)), tf.float32)) # operations train_op = tf.train.GradientDescentOptimizer(LEARNING_RATE) train_step = train_op.minimize(loss) # initializer init_op = tf.global_variables_initializer() saver = tf.train.Saver() sess.run(init_op) # start the training/testing steps if training: print("begin training") step = 0 sess.run(dataset_iterator.initializer, feed_dict={input_filenames: filenames}) # loop until out of data while True: try: train_result = sess.run([train_op, accuracy, x]) if step != 0 and step % 100 == 0: print("step %s, accuracy = %s" % (step, train_result[1])) # save the current model every 1000 steps if step % 1000 == 0: save_model(sess, saver, step) # print("x = %s, logits = %s" % (train_result[2], train_result[3])) step += 1 except tf.errors.OutOfRangeError: print("data exhausted, saving final model") save_model(sess, saver, 'final') break else: print("begin testing") step = 0 saver.restore(sess, LOAD_MODEL) sess.run(dataset_iterator.initializer, feed_dict={input_filenames: filenames}) cumulative_accuracy = 0.0 predictions = [] true_classes = [] # loop until out of data while True: try: test_result = sess.run( [accuracy, x, y_pred_class, y_true_class]) cumulative_accuracy += test_result[0] predictions.append(test_result[2]) true_classes.append(test_result[3]) if step % 100 == 0: print("step %s, accuracy = %s, cumulative accuracy = %s" % (step, test_result[0], cumulative_accuracy / step / BATCH_SIZE)) step += 1 except tf.errors.OutOfRangeError: break # wrap up, provide test results print("data exhausted, test results:") print("steps = %s, cumulative accuracy = %.04f" % (step, cumulative_accuracy / step / BATCH_SIZE)) #for i, p in enumerate(predictions): # print("[%s] true class = %s, predicted class = %s" % (i, true_classes[i], p)) cm = analysis.confusion_matrix(predictions, true_classes, class_list) print("confusion matrix = %s" % cm) analysis.plot_confusion_matrix(cm, class_list, "runs/" + run_name + ".pdf") print("per-class accuracy:") analysis.per_class_table(predictions, true_classes, class_list, "runs/" + run_name + '.csv')
def iad_nn(run_string): '''main function''' if LOAD_MODEL is None: training = True else: training = False # get the list of classes class_list = get_class_list(CLASS_INDEX_FILE) # get the list of filenames print("loading train file list from %s" % TRAIN_FILE_LIST) filenames_train = list_to_filenames(TRAIN_FILE_LIST, balance_classes=True) filenames_test = list_to_filenames(TEST_FILE_LIST) print("%s training files, %s testing files" % (len(filenames_train), len(filenames_test))) random.shuffle(filenames_train) if TRAINING_DATA_SAMPLE != 1.0: filenames_train = random.sample(filenames_train, int(TRAINING_DATA_SAMPLE * len(filenames_train))) # ensure filenames list is evenly divisable by batch size pad_filenames = len(filenames_train) % BATCH_SIZE filenames_train.extend(filenames_train[0:pad_filenames]) # create the TensorFlow sessions config = tf.ConfigProto(allow_soft_placement=True) sess = tf.Session(config=config) # setup the CNN if CLASSIFIER == 'softmax': weights, biases = get_variables_softmax('ucf101_iad') elif CLASSIFIER == 'lenet': weights, biases = get_variables_lenet('ucf101_iad') elif CLASSIFIER == 'mctnet': weights, biases = get_variables_mctnet('ucf101_iad') elif CLASSIFIER == 'autoencode': weights, biases = get_variables_autoencode('ucf101_iad') # placeholders input_filenames = tf.placeholder(tf.string, shape=[None]) input_filenames_test = tf.placeholder(tf.string, shape=[None]) # testing dataset dataset_test = tf.data.TFRecordDataset(input_filenames_test) dataset_test = dataset_test.map(_parse_function) dataset_test = dataset_test.batch(1) dataset_test = dataset_test.shuffle(500) dataset_test = dataset_test.repeat(1000000) dataset_test_iterator = dataset_test.make_initializable_iterator() x_test, y_test_true = dataset_test_iterator.get_next() y_test_true_class = tf.argmax(y_test_true, axis=1) # training or evaluation dataset dataset = tf.data.TFRecordDataset(input_filenames) dataset = dataset.map(_parse_function) dataset = dataset.batch(BATCH_SIZE) if training: dataset = dataset.shuffle(200) dataset = dataset.repeat(EPOCHS) else: dataset = dataset.repeat(1) dataset_iterator = dataset.make_initializable_iterator() x, y_true = dataset_iterator.get_next() y_true_class = tf.argmax(y_true, axis=1) print("x shape = %s" % x.get_shape().as_list()) print("y_true shape = %s" % y_true.get_shape().as_list()) # get neural network response if CLASSIFIER in ['softmax', 'autoencode', 'lenet', 'mctnet']: if CLASSIFIER == 'softmax': geom = LAYER_GEOMETRY[str(LAYER)] # layer 1 x = tf.reshape(x, [BATCH_SIZE, geom[0] * geom[1]]) x_test = tf.reshape(x_test, [1, geom[0] * geom[1]]) logits, conv_layers = softmax_regression(x, BATCH_SIZE, weights, biases, DROPOUT) logits_test, _ = softmax_regression(x_test, 1, weights, biases, DROPOUT) elif CLASSIFIER == 'lenet': logits, conv_layers = cnn_lenet(x, BATCH_SIZE, weights, biases, DROPOUT) logits_test, conv_layers = cnn_lenet(x_test, 1, weights, biases, DROPOUT) elif CLASSIFIER == 'mctnet': logits, conv_layers = cnn_mctnet(x, BATCH_SIZE, weights, biases, DROPOUT) logits_test, conv_layers = cnn_mctnet(x_test, 1, weights, biases, DROPOUT) elif CLASSIFIER == 'autoencode': geom = LAYER_GEOMETRY[str(LAYER)] # layer 1 x = tf.reshape(x, [BATCH_SIZE, geom[0] * geom[1]]) x_test = tf.reshape(x_test, [1, geom[0] * geom[1]]) x_autoencode, conv_layers = autoencode(x, BATCH_SIZE, weights, biases) x_test_autoencode, _ = autoencode(x_test, 1, weights, biases) # softmax regression with output # final logits = tf.matmul(x_autoencode, weights['W_out']) + biases['b_out'] logits_test = tf.matmul(x_test_autoencode, weights['W_out']) + biases['b_out'] print("logits shape = %s" % logits.get_shape().as_list()) y_pred = tf.nn.softmax(logits) y_pred_class = tf.argmax(y_pred, axis=1) # testing/validation y_pred_test = tf.nn.softmax(logits_test) y_pred_test_class = tf.argmax(y_pred_test, axis=1) correct_pred_test = tf.equal(y_pred_test_class, y_test_true_class) accuracy_test = tf.reduce_mean(tf.cast(correct_pred_test, tf.float32)) # loss and optimizer loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y_true)) optimizer = tf.train.AdamOptimizer(learning_rate=LEARNING_RATE) train_op = optimizer.minimize(loss) # evaluation correct_pred = tf.equal(y_pred_class, y_true_class) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) elif CLASSIFIER == 'logistic_regression': geom = LAYER_GEOMETRY[str(LAYER)] p5 = tf.constant(0.5) # logistic regression threshold W = tf.Variable(tf.zeros([geom[0] * geom[1], NUM_CLASSES])) b = tf.Variable(tf.zeros([NUM_CLASSES])) x = tf.reshape(x, [BATCH_SIZE, geom[0] * geom[1]]) x_test = tf.reshape(x_test, [1, geom[0] * geom[1]]) y_pred = tf.nn.softmax(tf.matmul(x, W) + b) y_pred_class = tf.argmax(y_pred, axis=1) cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_true * tf.log(y_pred), reduction_indices=[1])) train_op = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) correct_pred = tf.equal(tf.argmax(y_pred, 1), y_true_class) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # validation/testing y_pred_test = tf.nn.softmax(tf.matmul(x, W) + b) y_pred_test_class = tf.argmax(y_pred_test, axis=1) correct_pred_test = tf.equal(tf.argmax(y_pred_test, 1), y_test_true_class) accuracy_test = tf.reduce_mean(tf.cast(correct_pred_test, tf.float32)) # initializer init_op = tf.global_variables_initializer() saver = tf.train.Saver() sess.run(init_op) # start the training/testing steps if training: print("begin training") step = 0 sess.run([ dataset_iterator.initializer, dataset_test_iterator.initializer], feed_dict={ input_filenames: filenames_train, input_filenames_test: filenames_test }) # loop until out of data while True: try: train_result = sess.run([train_op, accuracy]) if step != 0 and step % 100 == 0: print("step %s, accuracy = %s" % (step, train_result[1])) # save the current model every 1000 steps if step % 1000 == 0: save_model(sess, saver, step) # mini batch validation if step % 10000 == 0: mini_batch_size = 1000 else: mini_batch_size = 50 test_accuracy = 0.0 for i in range(mini_batch_size): test_result = sess.run([accuracy_test]) test_accuracy += test_result[0] print("mini-batch validation accuracy = %.03f" % (test_accuracy / mini_batch_size)) # print("x = %s, logits = %s" % (train_result[2], train_result[3])) step += 1 except tf.errors.OutOfRangeError: print("data exhausted, saving final model") save_model(sess, saver, 'final') break final_accuracy = -1.0 else: print("begin testing") step = 0 saver.restore(sess, LOAD_MODEL) sess.run(dataset_iterator.initializer, feed_dict={input_filenames: filenames_test}) cumulative_accuracy = 0.0 predictions = [] true_classes = [] # loop until out of data while True: try: test_result = sess.run([accuracy, y_pred_class, y_true_class]) cumulative_accuracy += test_result[0] predictions.append(test_result[1]) true_classes.append(test_result[2]) if step % 100 == 0: print("step %s, accuracy = %s, cumulative accuracy = %s" % (step, test_result[0], cumulative_accuracy / step / BATCH_SIZE)) step += 1 except tf.errors.OutOfRangeError: break # wrap up, provide test results final_accuracy = cumulative_accuracy / step / BATCH_SIZE results_fd = open("runs/" + run_string + ".txt", 'w') print("data exhausted, test results:") print("steps = %s, cumulative accuracy = %.04f" % (step, final_accuracy)) results_fd.write("steps = %s, cumulative accuracy = %.04f\n" % (step, final_accuracy)) #for i, p in enumerate(predictions): # print("[%s] true class = %s, predicted class = %s" % (i, true_classes[i], p)) cm = analysis.confusion_matrix(predictions, true_classes, class_list) print("confusion matrix = %s" % cm) analysis.plot_confusion_matrix(cm, class_list, "runs/" + run_string + ".pdf") print("per-class accuracy:") analysis.per_class_table(predictions, true_classes, class_list, "runs/" + run_string + '.csv') sess.close() return final_accuracy
def run_training(): # Get the sets of images and labels for training, validation, and # Tell TensorFlow that the model will be built into the default Graph. # get the list of classes classes = get_class_list(CLASS_INDEX_FILE) # Create model directory if not os.path.exists(model_save_dir): os.makedirs(model_save_dir) use_pretrained_model = False model_filename = "./sports1m_finetuning_ucf101.model" with tf.Graph().as_default(): global_step = tf.get_variable( 'global_step', [], initializer=tf.constant_initializer(0), trainable=False ) images_placeholder, labels_placeholder = placeholder_inputs( FLAGS.batch_size * gpu_num ) tower_grads1 = [] tower_grads2 = [] logits = [] activations = [] sample_activations = {} opt_stable = tf.train.AdamOptimizer(1e-4) opt_finetuning = tf.train.AdamOptimizer(1e-3) with tf.variable_scope('var_name') as var_scope: weights = { 'wc1': _variable_with_weight_decay('wc1', [3, 3, 3, 3, 64], 0.0005), 'wc2': _variable_with_weight_decay('wc2', [3, 3, 3, 64, 128], 0.0005), 'wc3a': _variable_with_weight_decay('wc3a', [3, 3, 3, 128, 256], 0.0005), 'wc3b': _variable_with_weight_decay('wc3b', [3, 3, 3, 256, 256], 0.0005), 'wc4a': _variable_with_weight_decay('wc4a', [3, 3, 3, 256, 512], 0.0005), 'wc4b': _variable_with_weight_decay('wc4b', [3, 3, 3, 512, 512], 0.0005), 'wc5a': _variable_with_weight_decay('wc5a', [3, 3, 3, 512, 512], 0.0005), 'wc5b': _variable_with_weight_decay('wc5b', [3, 3, 3, 512, 512], 0.0005), 'wd1': _variable_with_weight_decay('wd1', [8192, 4096], 0.0005), 'wd2': _variable_with_weight_decay('wd2', [4096, 4096], 0.0005), 'out': _variable_with_weight_decay('wout', [4096, c3d_model.NUM_CLASSES], 0.0005) } biases = { 'bc1': _variable_with_weight_decay('bc1', [64], 0.000), 'bc2': _variable_with_weight_decay('bc2', [128], 0.000), 'bc3a': _variable_with_weight_decay('bc3a', [256], 0.000), 'bc3b': _variable_with_weight_decay('bc3b', [256], 0.000), 'bc4a': _variable_with_weight_decay('bc4a', [512], 0.000), 'bc4b': _variable_with_weight_decay('bc4b', [512], 0.000), 'bc5a': _variable_with_weight_decay('bc5a', [512], 0.000), 'bc5b': _variable_with_weight_decay('bc5b', [512], 0.000), 'bd1': _variable_with_weight_decay('bd1', [4096], 0.000), 'bd2': _variable_with_weight_decay('bd2', [4096], 0.000), 'out': _variable_with_weight_decay('bout', [c3d_model.NUM_CLASSES], 0.000), } for gpu_index in range(0, gpu_num): with tf.device('/gpu:%d' % gpu_index): varlist2 = [ weights['out'],biases['out'] ] varlist1 = list( set(weights.values() + biases.values()) - set(varlist2) ) # compute the logits get the activations logit, activation = c3d_model.inference_c3d( images_placeholder[gpu_index * FLAGS.batch_size:(gpu_index + 1) * FLAGS.batch_size,:,:,:,:], 0.5, FLAGS.batch_size, weights, biases ) loss_name_scope = ('gpud_%d_loss' % gpu_index) loss = tower_loss( loss_name_scope, logit, labels_placeholder[gpu_index * FLAGS.batch_size:(gpu_index + 1) * FLAGS.batch_size] ) grads1 = opt_stable.compute_gradients(loss, varlist1) grads2 = opt_finetuning.compute_gradients(loss, varlist2) tower_grads1.append(grads1) tower_grads2.append(grads2) logits.append(logit) activations.append(activation) logits = tf.concat(logits,0) activations = tf.concat(activations, 0) accuracy = tower_acc(logits, labels_placeholder) tf.summary.scalar('accuracy', accuracy) grads1 = average_gradients(tower_grads1) grads2 = average_gradients(tower_grads2) apply_gradient_op1 = opt_stable.apply_gradients(grads1) apply_gradient_op2 = opt_finetuning.apply_gradients(grads2, global_step=global_step) variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY) variables_averages_op = variable_averages.apply(tf.trainable_variables()) train_op = tf.group(apply_gradient_op1, apply_gradient_op2, variables_averages_op) null_op = tf.no_op() # Create a saver for writing training checkpoints. saver = tf.train.Saver(weights.values() + biases.values()) init = tf.global_variables_initializer() # Create a session for running Ops on the Graph. sess = tf.Session( config=tf.ConfigProto(allow_soft_placement=True) ) sess.run(init) if os.path.isfile(model_filename) and use_pretrained_model: saver.restore(sess, model_filename) # Create summary writter merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter('./visual_logs/train', sess.graph) test_writer = tf.summary.FileWriter('./visual_logs/test', sess.graph) for step in xrange(FLAGS.max_steps): start_time = time.time() train_images, train_labels, _, _, _, sample_names = c3d_model.read_clip_and_label( directory='/home/jordanc/datasets/UCF-101/UCF-101/', filename='train-test-splits/trainlist01.txt', batch_size=FLAGS.batch_size * gpu_num, num_frames_per_clip=c3d_model.NUM_FRAMES_PER_CLIP, crop_size=c3d_model.CROP_SIZE, shuffle=True ) _, activations_out = sess.run([train_op, activations], feed_dict={ images_placeholder: train_images, labels_placeholder: train_labels }) print("len sample_names = %s, activations: type = %s, len = %s, elem shape = %s" % (len(sample_names), type(activations_out), len(activations_out), str(activations_out[0].shape))) duration = time.time() - start_time print('Step %d: %.3f sec' % (step, duration)) # save the activation for each sample name to the sample_activations dict for i, s in enumerate(sample_names): sample_activations[s] = activations_out[i] # Save a checkpoint and evaluate the model periodically. if (step) % 100 == 0 or (step + 1) == FLAGS.max_steps: saver.save(sess, os.path.join(model_save_dir, 'c3d_ucf_model'), global_step=step) print('Training Data Eval:') summary, acc = sess.run( [merged, accuracy], feed_dict={images_placeholder: train_images, labels_placeholder: train_labels }) print ("accuracy: " + "{:.5f}".format(acc)) train_writer.add_summary(summary, step) print('Validation Data Eval:') val_images, val_labels, _, _, _, _ = c3d_model.read_clip_and_label( directory='/home/jordanc/datasets/UCF-101/UCF-101/', filename='train-test-splits/testlist01.txt', batch_size=FLAGS.batch_size * gpu_num, num_frames_per_clip=c3d_model.NUM_FRAMES_PER_CLIP, crop_size=c3d_model.CROP_SIZE, shuffle=True ) summary, acc = sess.run( [merged, accuracy], feed_dict={ images_placeholder: val_images, labels_placeholder: val_labels }) print ("accuracy: " + "{:.5f}".format(acc)) test_writer.add_summary(summary, step) print("done")