def get_variables(c3d_depth=0):
'''
Define the variables used by the network.
-c3d_depth: the depth at which the c3d activation map should be drawn
'''
# imported variables
c3d_w, c3d_b = c3d.get_variables()
itr_w, itr_b = itr.get_variables()
aud_w, aud_b = aud.get_variables()
# system specific variables
system_w = {
"W_0":
weight_variable(
"W_0", [itr.calculate_output_size(c3d_depth) + aud.OUT_SIZE, 1]),
"W_1":
weight_variable("W_1", [2, NUM_LABEL])
}
system_b = {
"b_0": bias_variable("b_0", [1]),
"b_1": bias_variable("b_1", [NUM_LABEL])
}
#coalated variables
weights = {"c3d": c3d_w, "itr": itr_w, "aud": aud_w, "system": system_w}
biases = {"c3d": c3d_b, "itr": itr_b, "aud": aud_b, "system": system_b}
return weights, biases
if len(l) > 0:
i, c = l.split(" ")
class_names.append(c)
assert len(class_names) == NUM_CLASSES
model = c3d_model.C3DModel()
run_log_file = 'runs/test_%s.log' % run_name
run_log_fd = open(run_log_file, 'w', 0)
run_log_fd.write("run name = %s" % run_name)
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
weights, biases = c3d.get_variables(model.num_classes)
x = tf.placeholder(tf.float32,
shape=[BATCH_SIZE, model.frames_per_clip, 112, 112, 3])
logits = model.inference_3d(x, weights, biases, BATCH_SIZE, False)
y_pred = tf.nn.softmax(logits)
y_pred_class = tf.argmax(y_pred, axis=1)
init_op = tf.global_variables_initializer()
# restore the model
saver = tf.train.Saver()
saver.restore(sess, model_to_load)
print("Restored model %s" % model_to_load)
compression_method["value"])
print("write to: ", video_name)
writer = tf.python_io.TFRecordWriter(video_name)
writer.write(ex.SerializeToString())
writer.close()
if __name__ == '__main__':
# open the files
compression_method = {"type": "peaks", "value": 10, "num_channels": 10}
# setup variables
placeholders = c3d.get_input_placeholder(1)
weights, biases = c3d.get_variables()
variable_name_dict = list(set(weights.values() + biases.values()))
cur_dir = "../one_person_tfrecords"
filenames = read_files_in_dir(cur_dir)
for f in filenames:
print(f)
for c3d_depth in range(5):
new_dir = "../iad_3d_tfrecords/" + str(c3d_depth) + "/"
# define model
c3d_model = c3d.generate_activation_map(placeholders,
weights,
biases,
depth=c3d_depth)
run_log_fd.write("LEARNING_RATE_DECAY = %s" % (LEARNING_RATE_DECAY))
run_log_fd.write("OPTIMIZER = %s" % (OPTIMIZER))
run_log_fd.write("Training samples = %s, testing samples = %s\n" %
(len(train_files), len(test_files)))
# Tensorflow configuration
config = tf.ConfigProto(allow_soft_placement=True)
with tf.Session(config=config) as sess:
#sess = tf_debug.LocalCLIDebugWrapperSession(sess)
# init variables
# tf.set_random_seed(1234)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
if VARIABLE_TYPE == 'default':
weights, biases = c3d.get_variables(model.num_classes)
elif VARIABLE_TYPE == 'weight decay':
weights, biases = c3d.get_variables(model.num_classes,
var_type="weight decay")
# placeholders and constants
# y_true = tf.placeholder(tf.float32, shape=[None, NUM_CLASSES], name='y_true')
train_filenames = tf.placeholder(tf.string, shape=[None])
test_filenames = tf.placeholder(tf.string, shape=[None])
# constants
global_step = tf.Variable(0, trainable=False)
# using tf.data.TFRecordDataset iterator
test_dataset = tf.data.TFRecordDataset(test_filenames)
test_dataset = test_dataset.map(model._parse_function)
def run_model(num_train_iterations=10,
c3d_depth=0,
thresholding_approach="norm",
training_dir='',
training_dir_dataset_limit=0,
validate_dir='',
testing_dir='',
train_print_freq=0,
validation_freq=0,
save_freq=0,
variable_update_freq=0):
# ---------- setup variables ------------
# setup variables
placeholders = model_def.get_placeholders(c3d_depth=c3d_depth)
weights_c3d, biases_c3d = c3d.get_variables()
c3d_variable_names = list(set(weights_c3d.values() + biases_c3d.values()))
c3d_model = c3d.generate_activation_map(placeholders["c3d_in"],
weights_c3d,
biases_c3d,
depth=c3d_depth)
#define Q
with tf.variable_scope('main'):
weights_main, biases_main = model_def.get_variables(
c3d_depth=c3d_depth)
model = model_def.get_predicted_values(placeholders,
weights_main,
biases_main,
c3d_depth=c3d_depth)
optimizer = model_def.optimizer(placeholders, model, alpha=1e-3)
classifier = model_def.classifier(model)
variable_name_dict = model_def.list_variables(weights_main, biases_main)
#define Q_hat
with tf.variable_scope('target'):
weights_target, biases_target = model_def.get_variables(
c3d_depth=c3d_depth)
model_target = model_def.get_predicted_values(placeholders,
weights_target,
biases_target,
c3d_depth=c3d_depth)
with tf.Session() as sess:
# ---------- file I/O ------------
# define files for training/testing
training_records, testing_records, validate_records = None, None, None
test_iter, valid_iter = 0, 0
if (training_dir != ''):
training_records, _ = read_files_in_dir(
training_dir,
randomize=True,
limit_dataset=training_dir_dataset_limit,
recursive=True)
if (testing_dir != ''):
testing_records, test_iter = read_files_in_dir(testing_dir,
randomize=False,
recursive=True)
if (validate_dir != ''):
validate_records, valid_iter = read_files_in_dir(validate_dir,
randomize=False,
recursive=False)
# ---------- restore variables (update) ------------
var_list = list( (variable_name_dict["itr"] + \
variable_name_dict["aud"] + \
variable_name_dict["system"]) )
var_dict = {}
for v in [v.name for v in var_list]:
with tf.variable_scope("target", reuse=True):
var_dict[v[:-2]] = tf.get_variable(v[v.find('/') + 1:-2])
restore_filename = C3D_NETWORK_VARIABLE_FILE
if (CHKPT_NAME != ''):
print("restoring checkpoint from :" + CHKPT_NAME)
restore_filename = CHKPT_NAME
# ---------- initalize variables ------------
# setup variables
if (train):
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
#initialize C3D network variables
saver = tf.train.Saver(variable_name_dict["c3d"])
saver.restore(sess, restore_filename)
#initialize other variables
if (CHKPT_NAME != ''):
saver = tf.train.Saver(var_list)
print("restoring variables from " + CHKPT_NAME)
saver.restore(sess, restore_filename)
# ---------- finalize model ------------
# ensure no additional changes are made to the model
#sess.graph.finalize()
# start queue runners in order to read ipnut files
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# ---------- train network ------------
for iteration in range(num_train_iterations):
# use target to get expected reward
# apply discount reward
# train actual network
# update target avery 1000 iterations
train(placeholders, training_records, sess, c3d_model,
thresholding_approach, optimizer, model_target)
if (train_print_freq > 0 and iteration % train_print_freq == 0):
print(iteration)
if (validation_freq > 0 and iteration % validation_freq == 0):
# test the system on the validation dataset
ph_values, info_values, sub_ph_values, sub_info_values = obtain_IAD_input(
placeholders, training_records, sess, c3d_model,
thresholding_approach)
print(sess.run(model, feed_dict=ph_values))
#confusion_matrix, responses = evaluate(placeholders, validate_records, sess, c3d_model, thresholding_approach, classifier, valid_iter, verbose=False)
#print("VAL "+str(iteration)+" accuracy: "+str(get_accuracy(confusion_matrix))+'\n')
if (iteration > 0 and save_freq > 0
and iteration % save_freq == 0):
# save the model to file
saver.save(sess, SAVE_NAME)
#pass
#if(variable_update_freq > 0 and iteration % variable_update_freq == 0):
if (variable_update_freq > 0
and iteration % variable_update_freq == 0):
#update variables in the target network
print("updating target network")
if (CHKPT_NAME != ''):
restore_filename = SAVE_NAME
saver = tf.train.Saver(var_dict)
print("pre rest, vars: ",
sess.run(weights_target["system"]["W_1"]))
saver.restore(sess, restore_filename)
print("post rest, vars: ",
sess.run(weights_target["system"]["W_1"]))
saver = tf.train.Saver(var_list)
# ---------- test network ------------
# test the system on the testing dataset
confusion_matrix, responses = evaluate(placeholders,
testing_records,
sess,
c3d_model,
thresholding_approach,
classifier,
test_iter,
verbose=True)
print("TEST accuracy: " + str(get_accuracy(confusion_matrix)) + '\n')
print(confusion_matrix)
for k in responses:
print(k, responses[k])
# ---------- close session ------------
# save final model to chekpoint file
saver.save(sess, SAVE_NAME)
coord.request_stop()
coord.join(threads)
def identify_min_maxes(filenames, records):
placeholders = c3d.get_input_placeholder(batch_size)
weights, biases = c3d.get_variables()
variable_name_dict = list( set(weights.values() + biases.values()))
sem = Semaphore(4)
for c3d_depth in range(1):#5):
max_vals, min_vals = RawArray('d', 64), RawArray('d', 64)
for i in range(64):
max_vals[i] = float("-inf")
min_vals[i] = float("inf")
# define model
c3d_model = c3d.generate_activation_map(placeholders, weights, biases, depth=c3d_depth)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
saver = tf.train.Saver(variable_name_dict)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
saver.restore(sess, C3D_NETWORK_VARIABLE_FILE)
#setup file io
tf_records = input_pipeline(filenames, batch_size=batch_size)
sess.graph.finalize()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
#process files
for i in range(len(filenames)):
if(i %1000 == 0 ):
print("Converted "+str(i)+" files")
ph_values, info_values = generate_model_input(placeholders, tf_records, sess)
all_procs = []
if(ph_values != 0):
#generate activation map from 3D-CNN
c3d_activation_map = sess.run(c3d_model, feed_dict=ph_values)
# try to acquire a ticket, if ticket is available converting activation map to IAD
# have to use Semaphores here because activation maps are save on GPU. ATTempting to start multiple threads
# means that the GPU never relesaes the memory used for activation maps.
sem.acquire()
p = Thread(target=get_row_min_max, args=(c3d_activation_map, info_values,sem, max_vals, min_vals, records))
p.start()
all_procs.append(p)
for p in all_procs:
p.join()
coord.request_stop()
coord.join(threads)
return max_vals, min_vals
def convert_videos_to_IAD(filenames, c3d_depth, records=None):
'''
opens an unthreshodled IAD and thresholds given the new values
- records - providing a records variable indicates that the function is
meant to be run as global_norm not local_norm
'''
max_vals, min_vals = None, None
if (records):
max_vals, min_vals = RawArray('d', 64), RawArray('d', 64)
for i in range(64):
max_vals[i] = float("-inf")
min_vals[i] = float("inf")
# define model
placeholders = c3d.get_input_placeholder(BATCH_SIZE)
weights, biases = c3d.get_variables()
variable_name_dict = list(set(weights.values() + biases.values()))
c3d_model = c3d.generate_activation_map(placeholders,
weights,
biases,
depth=c3d_depth)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
saver = tf.train.Saver(variable_name_dict)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
saver.restore(sess, C3D_NETWORK_VARIABLE_FILE)
#setup file io
tf_records = input_pipeline(filenames, batch_size=BATCH_SIZE)
sess.graph.finalize()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
#limit the number of threads running at once
sem = Semaphore(NUM_THREADS)
#process files
for i in range(len(filenames)):
if (i % 1000 == 0):
print("Converted " + str(i) + " files")
ph_values, info_values = generate_model_input(
placeholders, tf_records, sess)
all_procs = []
if (ph_values != 0):
#generate activation map from 3D-CNN
c3d_activation_map = sess.run(c3d_model, feed_dict=ph_values)
# try to acquire a ticket, if ticket is available converting activation map to IAD
# have to use Semaphores here because activation maps are save on GPU. ATTempting to start multiple threads
# means that the GPU never relesaes the memory used for activation maps.
sem.acquire()
p = None
if (records):
p = Thread(target=get_row_min_max,
args=(c3d_activation_map, info_values, sem,
max_vals, min_vals, records))
else:
p = Thread(target=threshold_iad,
args=(
c3d_activation_map,
info_values,
sem,
))
p.start()
all_procs.append(p)
else:
print("ph_value is 0, file generation failed")
for p in all_procs:
p.join()
coord.request_stop()
coord.join(threads)
if (records):
return max_vals, min_vals
return None