def convert_to_IAD_input(placeholders, tf_records, sess, c3d_model,
thresholding_approach, compression_method, video_name,
c3d_depth):
'''
Provides the training input for the ITR network by generating an IAD from the
activation map of the C3D network. Outputs two dictionaries. The first contains
the placeholders that will be used when evaluating the full ITR model. The second
contains information about the observation being read (ie. true labels, number of
prompts, file name, etc).
-placeholders: the list of placeholders used by the network
-tf_records: the TFRecord data source to read from
-sess: the tensorflow Session
-c3d_model: the c3d network model
'''
# Obtain activation amp from C3D network
ph_values, info_values = generate_model_input(placeholders, tf_records,
sess)
c3d_activation_map = sess.run(c3d_model, feed_dict=ph_values)
print(c3d_activation_map.shape)
thresholded_data = thresholding(c3d_activation_map[0],
info_values["data_ratio"],
compression_method, thresholding_approach)
print(thresholded_data)
ex = make_sequence_example(thresholded_data, info_values["label"][0],
info_values["example_id"][0], c3d_depth,
compression_method["value"])
print("write to: ", video_name)
writer = tf.python_io.TFRecordWriter(video_name)
writer.write(ex.SerializeToString())
writer.close()
def get_row_min_max(c3d_activation_map, info_values, sem, max_vals, min_vals, records): ''' Get the IAD for the activation map and record the highest and lowest observed values. ''' thresholded_data = thresholding(c3d_activation_map[0], info_values["data_ratio"], compression_method, "none") local_max_values = np.max(thresholded_data, axis=1) local_min_values = np.min(thresholded_data, axis=1) for i in range(len(local_max_values)): if(local_max_values[i] > max_vals[i]): max_vals[i] = local_max_values[i] if(local_min_values[i] < min_vals[i]): min_vals[i] = local_min_values[i] # save current un-thresholded data to file filename = tempfile.NamedTemporaryFile().name+".npy" np.save(filename, thresholded_data) records.put(Record(filename, info_values)) sem.release()
def get_min_maxes(directory, layers, sample_names, labels, mins, maxes,
compression_method, thresholding_approach):
'''returns new minimums and maximum values determined from the activation layers'''
num_layers = 5
assert len(layers) % num_layers == 0
sample_index_dict = {}
for i, s in enumerate(sample_names):
# get a list of files matching this name from the directory
#new_index = get_file_sequence(NPY_DIRECTORY, s, '.npy')
new_index = get_file_sequence_dict(sample_index_dict, s)
s_index = i * num_layers
sample_layers = layers[s_index:s_index +
num_layers] # layers ordered from 1 to 5
assert len(
sample_layers
) == num_layers, "sample_layers has invalid length - %s" % len(
sample_layers)
thresholded_data = []
for i, l in enumerate(sample_layers):
layer_data = np.squeeze(l, axis=0)
data_ratio = float(layer_data.shape[0] / 32.0) # num columns / 16
#print("layer_data shape = %s, ratio = %s" % (str(layer_data.shape), data_ratio))
data = thresholding(layer_data, data_ratio, compression_method,
thresholding_approach)
#if i == 0:
# print("layer 0 thresholded data = %s, min = %s" % (data, np.min(data)))
thresholded_data.append(data)
# for each layer, determine the min, max values for each row
for j, l in enumerate(thresholded_data):
#print("%s l.shape = %s, mins[j].shape = %s" % (j, l.shape, mins[j].shape))
assert l.shape[0] == mins[j].shape[
0], "l.shape[0] %s != mins[i].shape[0] %s" % (l.shape[0],
mins[j].shape[0])
for k, row in enumerate(l):
row_max = np.max(row)
row_min = np.min(row)
#print("row = %s, max = %s, min = %s" % (k, row_max, row_min))
if row_max > maxes[j][k]:
#print("new max for layer %s, row %s - %s > %s" % (j, k, row_max, maxes[j][k]))
maxes[j][k] = row_max
if row_min < mins[j][k]:
#print("new min for layer %s, row %s - %s < %s" % (j, k, row_min, mins[j][k]))
mins[j][k] = row_min
# save the layer data
# sample_sequence_layer.npy
npy_filename = "%s_%02d_%s.npy" % (s, new_index, j + 1)
npy_path = os.path.join(NPY_DIRECTORY, npy_filename)
np.save(npy_path, l)
print("write npy to %s" % (npy_path))
return mins, maxes
def threshold_iad(c3d_activation_map, info_values, sem):
'''
Get the IAD for the activation map and record the highest and lowest observed values.
'''
thresholded_data = thresholding(c3d_activation_map[0],
info_values["data_ratio"],
compression_method, "norm")
#write file
filename = new_dir + str(info_values["uid"]).zfill(6) + ".tfrecord"
write_to_tfrecord(filename, thresholded_data, info_values)
sem.release()
def convert_to_IAD_input(directory, layers, sample_names, labels,
compression_method, thresholding_approach):
'''
Provides the training input for the ITR network by generating an IAD from the
activation map of the C3D network. Outputs two dictionaries. The first contains
the placeholders that will be used when evaluating the full ITR model. The second
contains information about the observation being read (ie. true labels, number of
prompts, file name, etc).
-placeholders: the list of placeholders used by the network
-tf_records: the TFRecord data source to read from
-sess: the tensorflow Session
-c3d_model: the c3d network model
'''
num_layers = 5
assert len(layers) % num_layers == 0
# assert (len(layers) / num_layers) == len(sample_names), "layers list and sample_names list have different lengths (%s/%s)" % (len(layers), len(sample_names))
# print("sample_names = %s" % (sample_names))
file_list = os.listdir(directory)
file_list = [x for x in file_list if 'tfrecord' in x]
for i, s in enumerate(sample_names):
# get a list of files matching this name from the directory
matching_samples = []
pattern = s + "*.tfrecord"
for f in file_list:
if fnmatch.fnmatch(f, pattern):
matching_samples.append(f)
# if there are already samples, just pick the next index
if len(matching_samples) > 0:
last_index = int(
sorted(matching_samples)[-1].split('_')[4].split('.')[0])
else:
last_index = -1
new_index = last_index + 1
video_name = os.path.join(directory,
"%s_%02d.tfrecord" % (s, new_index))
s_index = i * num_layers
sample_layers = layers[s_index:s_index + num_layers]
assert len(
sample_layers
) == num_layers, "sample_layers has invalid length - %s" % len(
sample_layers)
thresholded_data = []
for l in sample_layers:
layer_data = np.squeeze(l, axis=0)
thresholded_data.append(
thresholding(layer_data, compression_method,
thresholding_approach))
# print("thresholded_data shape = %s" % str(thresholded_data.shape))
# generate the tfrecord
ex = make_sequence_example(thresholded_data, labels[i], s,
compression_method["value"])
print("write tfrecord to: %s" % video_name)
writer = tf.python_io.TFRecordWriter(video_name)
writer.write(ex.SerializeToString())
# generate images with 5% chance
if random.random() < 0.05:
for i, d in enumerate(thresholded_data):
img_name = os.path.join(directory,
"%s_%02d_%s.jpg" % (s, new_index, i))
print("write test image to: %s" % img_name)
#print("single layer type = %s, shape = %s" % (type(d), str(thresholded_data[layer_to_test].shape)))
pixels = np.squeeze(d, axis=2)
rescaled = (255.0 / pixels.max() *
(pixels - pixels.min())).astype(np.uint8)
img = Image.fromarray(rescaled)
img = img.convert("L")
# img = img.resize((img.width * 10, img.height))
img.save(img_name, quality=95)