# setup variables placeholders = c3d.get_input_placeholder(1) weights, biases = c3d.get_variables() variable_name_dict = list(set(weights.values() + biases.values())) cur_dir = "../one_person_tfrecords" filenames = read_files_in_dir(cur_dir) for f in filenames: print(f) for c3d_depth in range(5): new_dir = "../iad_3d_tfrecords/" + str(c3d_depth) + "/" # define model c3d_model = c3d.generate_activation_map(placeholders, weights, biases, depth=c3d_depth) with tf.Session() as sess: saver = tf.train.Saver(variable_name_dict) sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) saver.restore(sess, C3D_NETWORK_VARIABLE_FILE) #setup file io src = input_pipeline(filenames) coord = tf.train.Coordinator()
def run_model(num_train_iterations=10, c3d_depth=0, thresholding_approach="norm", training_dir='', training_dir_dataset_limit=0, validate_dir='', testing_dir='', train_print_freq=0, validation_freq=0, save_freq=0, variable_update_freq=0): # ---------- setup variables ------------ # setup variables placeholders = model_def.get_placeholders(c3d_depth=c3d_depth) weights_c3d, biases_c3d = c3d.get_variables() c3d_variable_names = list(set(weights_c3d.values() + biases_c3d.values())) c3d_model = c3d.generate_activation_map(placeholders["c3d_in"], weights_c3d, biases_c3d, depth=c3d_depth) #define Q with tf.variable_scope('main'): weights_main, biases_main = model_def.get_variables( c3d_depth=c3d_depth) model = model_def.get_predicted_values(placeholders, weights_main, biases_main, c3d_depth=c3d_depth) optimizer = model_def.optimizer(placeholders, model, alpha=1e-3) classifier = model_def.classifier(model) variable_name_dict = model_def.list_variables(weights_main, biases_main) #define Q_hat with tf.variable_scope('target'): weights_target, biases_target = model_def.get_variables( c3d_depth=c3d_depth) model_target = model_def.get_predicted_values(placeholders, weights_target, biases_target, c3d_depth=c3d_depth) with tf.Session() as sess: # ---------- file I/O ------------ # define files for training/testing training_records, testing_records, validate_records = None, None, None test_iter, valid_iter = 0, 0 if (training_dir != ''): training_records, _ = read_files_in_dir( training_dir, randomize=True, limit_dataset=training_dir_dataset_limit, recursive=True) if (testing_dir != ''): testing_records, test_iter = read_files_in_dir(testing_dir, randomize=False, recursive=True) if (validate_dir != ''): validate_records, valid_iter = read_files_in_dir(validate_dir, randomize=False, recursive=False) # ---------- restore variables (update) ------------ var_list = list( (variable_name_dict["itr"] + \ variable_name_dict["aud"] + \ variable_name_dict["system"]) ) var_dict = {} for v in [v.name for v in var_list]: with tf.variable_scope("target", reuse=True): var_dict[v[:-2]] = tf.get_variable(v[v.find('/') + 1:-2]) restore_filename = C3D_NETWORK_VARIABLE_FILE if (CHKPT_NAME != ''): print("restoring checkpoint from :" + CHKPT_NAME) restore_filename = CHKPT_NAME # ---------- initalize variables ------------ # setup variables if (train): sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) #initialize C3D network variables saver = tf.train.Saver(variable_name_dict["c3d"]) saver.restore(sess, restore_filename) #initialize other variables if (CHKPT_NAME != ''): saver = tf.train.Saver(var_list) print("restoring variables from " + CHKPT_NAME) saver.restore(sess, restore_filename) # ---------- finalize model ------------ # ensure no additional changes are made to the model #sess.graph.finalize() # start queue runners in order to read ipnut files coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord, sess=sess) # ---------- train network ------------ for iteration in range(num_train_iterations): # use target to get expected reward # apply discount reward # train actual network # update target avery 1000 iterations train(placeholders, training_records, sess, c3d_model, thresholding_approach, optimizer, model_target) if (train_print_freq > 0 and iteration % train_print_freq == 0): print(iteration) if (validation_freq > 0 and iteration % validation_freq == 0): # test the system on the validation dataset ph_values, info_values, sub_ph_values, sub_info_values = obtain_IAD_input( placeholders, training_records, sess, c3d_model, thresholding_approach) print(sess.run(model, feed_dict=ph_values)) #confusion_matrix, responses = evaluate(placeholders, validate_records, sess, c3d_model, thresholding_approach, classifier, valid_iter, verbose=False) #print("VAL "+str(iteration)+" accuracy: "+str(get_accuracy(confusion_matrix))+'\n') if (iteration > 0 and save_freq > 0 and iteration % save_freq == 0): # save the model to file saver.save(sess, SAVE_NAME) #pass #if(variable_update_freq > 0 and iteration % variable_update_freq == 0): if (variable_update_freq > 0 and iteration % variable_update_freq == 0): #update variables in the target network print("updating target network") if (CHKPT_NAME != ''): restore_filename = SAVE_NAME saver = tf.train.Saver(var_dict) print("pre rest, vars: ", sess.run(weights_target["system"]["W_1"])) saver.restore(sess, restore_filename) print("post rest, vars: ", sess.run(weights_target["system"]["W_1"])) saver = tf.train.Saver(var_list) # ---------- test network ------------ # test the system on the testing dataset confusion_matrix, responses = evaluate(placeholders, testing_records, sess, c3d_model, thresholding_approach, classifier, test_iter, verbose=True) print("TEST accuracy: " + str(get_accuracy(confusion_matrix)) + '\n') print(confusion_matrix) for k in responses: print(k, responses[k]) # ---------- close session ------------ # save final model to chekpoint file saver.save(sess, SAVE_NAME) coord.request_stop() coord.join(threads)
def convert_videos_to_IAD(filenames, c3d_depth, records=None): ''' opens an unthreshodled IAD and thresholds given the new values - records - providing a records variable indicates that the function is meant to be run as global_norm not local_norm ''' max_vals, min_vals = None, None if (records): max_vals, min_vals = RawArray('d', 64), RawArray('d', 64) for i in range(64): max_vals[i] = float("-inf") min_vals[i] = float("inf") # define model placeholders = c3d.get_input_placeholder(BATCH_SIZE) weights, biases = c3d.get_variables() variable_name_dict = list(set(weights.values() + biases.values())) c3d_model = c3d.generate_activation_map(placeholders, weights, biases, depth=c3d_depth) config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: saver = tf.train.Saver(variable_name_dict) sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) saver.restore(sess, C3D_NETWORK_VARIABLE_FILE) #setup file io tf_records = input_pipeline(filenames, batch_size=BATCH_SIZE) sess.graph.finalize() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord, sess=sess) #limit the number of threads running at once sem = Semaphore(NUM_THREADS) #process files for i in range(len(filenames)): if (i % 1000 == 0): print("Converted " + str(i) + " files") ph_values, info_values = generate_model_input( placeholders, tf_records, sess) all_procs = [] if (ph_values != 0): #generate activation map from 3D-CNN c3d_activation_map = sess.run(c3d_model, feed_dict=ph_values) # try to acquire a ticket, if ticket is available converting activation map to IAD # have to use Semaphores here because activation maps are save on GPU. ATTempting to start multiple threads # means that the GPU never relesaes the memory used for activation maps. sem.acquire() p = None if (records): p = Thread(target=get_row_min_max, args=(c3d_activation_map, info_values, sem, max_vals, min_vals, records)) else: p = Thread(target=threshold_iad, args=( c3d_activation_map, info_values, sem, )) p.start() all_procs.append(p) else: print("ph_value is 0, file generation failed") for p in all_procs: p.join() coord.request_stop() coord.join(threads) if (records): return max_vals, min_vals return None
def identify_min_maxes(filenames, records): placeholders = c3d.get_input_placeholder(batch_size) weights, biases = c3d.get_variables() variable_name_dict = list( set(weights.values() + biases.values())) sem = Semaphore(4) for c3d_depth in range(1):#5): max_vals, min_vals = RawArray('d', 64), RawArray('d', 64) for i in range(64): max_vals[i] = float("-inf") min_vals[i] = float("inf") # define model c3d_model = c3d.generate_activation_map(placeholders, weights, biases, depth=c3d_depth) config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: saver = tf.train.Saver(variable_name_dict) sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) saver.restore(sess, C3D_NETWORK_VARIABLE_FILE) #setup file io tf_records = input_pipeline(filenames, batch_size=batch_size) sess.graph.finalize() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord, sess=sess) #process files for i in range(len(filenames)): if(i %1000 == 0 ): print("Converted "+str(i)+" files") ph_values, info_values = generate_model_input(placeholders, tf_records, sess) all_procs = [] if(ph_values != 0): #generate activation map from 3D-CNN c3d_activation_map = sess.run(c3d_model, feed_dict=ph_values) # try to acquire a ticket, if ticket is available converting activation map to IAD # have to use Semaphores here because activation maps are save on GPU. ATTempting to start multiple threads # means that the GPU never relesaes the memory used for activation maps. sem.acquire() p = Thread(target=get_row_min_max, args=(c3d_activation_map, info_values,sem, max_vals, min_vals, records)) p.start() all_procs.append(p) for p in all_procs: p.join() coord.request_stop() coord.join(threads) return max_vals, min_vals
def run_model(num_train_iterations=10, c3d_depth=0, thresholding_approach="norm", training_dir='', training_dir_dataset_limit=0, validate_dir='', testing_dir='', train_print_freq=0, validation_freq=0, save_freq=0): # ---------- setup variables ------------ # setup variables placeholders = model_def.get_placeholders(c3d_depth=c3d_depth) weights, biases = model_def.get_variables(c3d_depth=c3d_depth) variable_name_dict = model_def.list_variables(weights, biases) # define model c3d_model = c3d.generate_activation_map(placeholders["c3d_in"], weights["c3d"], biases["c3d"], depth=c3d_depth) model = model_def.get_predicted_values(placeholders, weights, biases, c3d_depth=c3d_depth) classifier = model_def.classifier(model) optimizer = model_def.optimizer(placeholders, model, alpha=1e-3) with tf.Session() as sess: # ---------- file I/O ------------ # define files for training/testing training_records, testing_records, validate_records = None, None, None test_iter, valid_iter = 0, 0 if (training_dir != ''): training_records, _ = read_files_in_dir( training_dir, randomize=True, limit_dataset=training_dir_dataset_limit, recursive=True) if (testing_dir != ''): testing_records, test_iter = read_files_in_dir(testing_dir, randomize=False, recursive=True) if (validate_dir != ''): validate_records, valid_iter = read_files_in_dir(validate_dir, randomize=False, recursive=False) # ---------- restore variables (update) ------------ saver = tf.train.Saver(variable_name_dict["c3d"]) restore_filename = C3D_NETWORK_VARIABLE_FILE if (CHKPT_NAME != ''): restore_filename = CHKPT_NAME saver = tf.train.Saver( list( set(variable_name_dict["c3d"] + \ variable_name_dict["itr"] + \ variable_name_dict["aud"] + \ variable_name_dict["system"]) )) # ---------- initalize variables ------------ # setup variables if (train): sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) saver.restore(sess, restore_filename) saver = tf.train.Saver() # ---------- finalize model ------------ # ensure no additional changes are made to the model sess.graph.finalize() # start queue runners in order to read ipnut files coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord, sess=sess) # ---------- train network ------------ for iteration in range(num_train_iterations): train(placeholders, training_records, sess, c3d_model, thresholding_approach, optimizer) if (train_print_freq > 0 and iteration % train_print_freq == 0): print(iteration) if (validation_freq > 0 and iteration % validation_freq == 0): # test the system on the validation dataset confusion_matrix, responses = evaluate(placeholders, validate_records, sess, c3d_model, thresholding_approach, classifier, valid_iter, verbose=False) print("VAL " + str(iteration) + " accuracy: " + str(get_accuracy(confusion_matrix)) + '\n') if (save_freq > 0 and iteration > 0 and iteration % save_freq == 0): # save the model to file saver.save(sess, 'itr_step/model.ckpt', global_step=iteration) # ---------- test network ------------ # save final model to chekpoint file if (save_freq > 0): saver.save(sess, 'itr_final/model.ckpt') # test the system on the testing dataset confusion_matrix, responses = evaluate(placeholders, testing_records, sess, c3d_model, thresholding_approach, classifier, test_iter, verbose=True) print("TEST accuracy: " + str(get_accuracy(confusion_matrix)) + '\n') print(confusion_matrix) for k in responses: print(k, responses[k]) # ---------- close session ------------ coord.request_stop() coord.join(threads)