def get_reader():
# Convert feature_names and feature_sizes to lists of values.
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
return reader
def evaluate():
tf.set_random_seed(0) # for reproducibility
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
model = find_class_by_name(FLAGS.model, [labels_autoencoder])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
if FLAGS.eval_data_pattern is "":
raise IOError("'eval_data_pattern' was not specified. " +
"Nothing to evaluate.")
build_graph(reader=reader,
model=model,
eval_data_pattern=FLAGS.eval_data_pattern,
label_loss_fn=label_loss_fn,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_batch = tf.get_collection("video_id_batch")[0]
prediction_batch = tf.get_collection("predictions")[0]
label_batch = tf.get_collection("labels")[0]
loss = tf.get_collection("loss")[0]
summary_op = tf.get_collection("summary_op")[0]
saver = tf.train.Saver(tf.global_variables())
summary_writer = tf.summary.FileWriter(FLAGS.train_dir,
graph=tf.get_default_graph())
evl_metrics = eval_util.EvaluationMetrics(reader.num_classes,
FLAGS.top_k)
last_global_step_val = -1
while True:
last_global_step_val = evaluation_loop(video_id_batch,
prediction_batch,
label_batch, loss,
summary_op, saver,
summary_writer, evl_metrics,
last_global_step_val)
if FLAGS.run_once:
break
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
if FLAGS.output_dir is "":
raise ValueError("'output_dir' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
if FLAGS.distill_data_pattern is not None:
predictions_patterns = FLAGS.distill_data_pattern.strip().split(",")
predictions_readers = []
for pattern in predictions_patterns:
predictions_readers.append(
readers.YT8MAggregatedFeatureReader(
feature_names=["predictions"], feature_sizes=[4716]))
else:
predictions_patterns = None
predictions_readers = None
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
transformer_fn = find_class_by_name(FLAGS.feature_transformer,
[feature_transform])
build_graph(reader=reader,
input_data_pattern=FLAGS.input_data_pattern,
model=model,
distill_readers=predictions_readers,
distill_data_patterns=predictions_patterns,
batch_size=FLAGS.batch_size,
transformer_class=transformer_fn)
saver = tf.train.Saver(max_to_keep=3,
keep_checkpoint_every_n_hours=10000000000)
inference(saver, FLAGS.model_checkpoint_path, FLAGS.output_dir,
FLAGS.batch_size, FLAGS.top_k)
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
if FLAGS.input_model_tgz:
if FLAGS.train_dir:
raise ValueError(
"You cannot supply --train_dir if supplying " "--input_model_tgz"
)
# Untar.
if not os.path.exists(FLAGS.untar_model_dir):
os.makedirs(FLAGS.untar_model_dir)
tarfile.open(FLAGS.input_model_tgz).extractall(FLAGS.untar_model_dir)
FLAGS.train_dir = FLAGS.untar_model_dir
flags_dict_file = os.path.join(FLAGS.train_dir, "model_flags.json")
if not file_io.file_exists(flags_dict_file):
raise IOError("Cannot find %s. Did you run eval.py?" % flags_dict_file)
flags_dict = json.loads(file_io.FileIO(flags_dict_file, "r").read())
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"]
)
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes
)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes
)
if FLAGS.output_file is "":
raise ValueError(
"'output_file' was not specified. " "Unable to continue with inference."
)
if FLAGS.input_data_pattern is "":
raise ValueError(
"'input_data_pattern' was not specified. "
"Unable to continue with inference."
)
inference(
reader,
FLAGS.train_dir,
FLAGS.input_data_pattern,
FLAGS.output_file,
FLAGS.batch_size,
FLAGS.top_k,
)
def get_reader():
# Convert feature_names and feature_sizes to lists of values.
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
random_selection = 0
if FLAGS.select_randomly: random_selection = 2
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes, random_selection=random_selection,
negative_sampling=FLAGS.negative_sampling, percentage_negative=FLAGS.percentage_negative)
return reader
def evaluate():
"""Starts main evaluation loop."""
tf.compat.v1.set_random_seed(0) # for reproducibility
# Write json of flags
model_flags_path = os.path.join(FLAGS.train_dir, "model_flags.json")
if not file_io.file_exists(model_flags_path):
raise IOError(("Cannot find file %s. Did you run train.py on the same "
"--train_dir?") % model_flags_path)
flags_dict = json.loads(file_io.FileIO(model_flags_path, mode="r").read())
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names,
feature_sizes=feature_sizes,
segment_labels=FLAGS.segment_labels)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
model = find_class_by_name(flags_dict["model"],
[frame_level_models, video_level_models])()
label_loss_fn = find_class_by_name(flags_dict["label_loss"],
[losses])()
if not FLAGS.input_data_pattern:
raise IOError("'input_data_pattern' was not specified. Nothing to "
"evaluate.")
build_graph(reader=reader,
model=model,
input_data_pattern=FLAGS.input_data_pattern,
label_loss_fn=label_loss_fn,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size)
logging.info("built inference graph")
# A dict of tensors to be run in Session.
saver = tf.compat.v1.train.Saver(tf.compat.v1.global_variables())
inference(saver, reader, FLAGS.train_dir, FLAGS.output_dir,
FLAGS.batch_size, FLAGS.top_k)
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
if FLAGS.input_model_tgz:
if FLAGS.train_dir:
raise ValueError("You cannot supply --train_dir if supplying "
"--input_model_tgz")
# Untar.
if not os.path.exists(FLAGS.untar_model_dir):
os.makedirs(FLAGS.untar_model_dir)
tarfile.open(FLAGS.input_model_tgz).extractall(FLAGS.untar_model_dir)
FLAGS.train_dir = FLAGS.untar_model_dir
flags_dict_file = os.path.join(FLAGS.train_dir, "model_flags.json")
if not os.path.exists(flags_dict_file):
raise IOError("Cannot find %s. Did you run eval.py?" % flags_dict_file)
flags_dict = json.loads(open(flags_dict_file).read())
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
all_data_patterns = []
with open(FLAGS.data_config) as f:
all_data_patterns = f.read().splitlines()
all_readers = []
for i in xrange(len(all_data_patterns)):
reader = readers.EnsembleReader(feature_names=["predictions"],
feature_sizes=["3862"])
all_readers.append(reader)
# if FLAGS.output_file is "":
# raise ValueError("'output_file' was not specified. "
# "Unable to continue with inference.")
# if FLAGS.input_data_pattern is "":
# raise ValueError("'input_data_pattern' was not specified. "
# "Unable to continue with inference.")
inference(all_readers, FLAGS.train_dir, all_data_patterns,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
def build_model(self):
"""Find the model and build the graph."""
# Convert feature_names and feature_sizes to lists of values.
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
if FLAGS.frame_only:
reader = readers.YT8MFrameFeatureOnlyReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
# Find the model.
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
optimizer_class = find_class_by_name(FLAGS.optimizer, [tf.train])
build_graph(
reader=reader,
model=model,
optimizer_class=optimizer_class,
clip_gradient_norm=FLAGS.clip_gradient_norm,
train_data_pattern=FLAGS.train_data_pattern,
label_loss_fn=label_loss_fn,
base_learning_rate=FLAGS.base_learning_rate,
learning_rate_decay=FLAGS.learning_rate_decay,
learning_rate_decay_examples=FLAGS.learning_rate_decay_examples,
regularization_penalty=FLAGS.regularization_penalty,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.num_epochs)
logging.info("%s: Built graph.", task_as_string(self.task))
all_vars = tf.trainable_variables()
restore_list = [
var for var in all_vars if 'notrestore' not in var.name
]
return tf.train.Saver(var_list=restore_list), tf.train.Saver(
max_to_keep=2, keep_checkpoint_every_n_hours=1)
def get_reader():
# Convert feature_names and feature_sizes to lists of values.
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names,
feature_sizes=feature_sizes,
is_training=True,
data_augmentation=FLAGS.data_augmentation)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
return reader
def get_params(flags_dict):
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
model = find_class_by_name(flags_dict["model"],
[frame_level_models, video_level_models])()
# a model instance is created
label_loss_fn = find_class_by_name(flags_dict["label_loss"], [losses])()
return model, label_loss_fn, reader
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_dir is "":
raise ValueError("'output_dir' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
if FLAGS.input_data_pattern is "":
raise IOError("'input_data_pattern' was not specified. " +
"Nothing to evaluate.")
build_graph(
reader=reader,
model=model,
eval_data_pattern=FLAGS.input_data_pattern,
label_loss_fn=label_loss_fn,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_batch = tf.get_collection("video_id_batch")[0]
prediction_batch = tf.get_collection("predictions")[0]
label_batch = tf.get_collection("labels")[0]
saver = tf.train.Saver(tf.global_variables())
inference(video_id_batch, prediction_batch, label_batch, saver, FLAGS.output_dir)
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("START FLAGS===========================")
print("train_dir: " + str(FLAGS.train_dir))
print("output_file: " + str(FLAGS.output_file))
print("input_data_pattern: " + str(FLAGS.input_data_pattern))
print("frame_features: " + str(FLAGS.frame_features))
print("batch_size: " + str(FLAGS.batch_size))
print("feature_names: " + str(FLAGS.feature_names))
print("feature_sizes: " + str(FLAGS.feature_sizes))
print("c_vars: " + str(FLAGS.c_vars))
print("num_readers: " + str(FLAGS.num_readers))
print("top_k: " + str(FLAGS.top_k))
print("layers_keep_probs: " + str(FLAGS.layers_keep_probs))
print("gpu_only: " + str(FLAGS.gpu_only))
print("END FLAGS ============================")
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names,
feature_sizes=feature_sizes,
num_classes=FLAGS.truncated_num_classes,
decode_zlib=FLAGS.decode_zlib)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names,
feature_sizes=feature_sizes,
feature_calcs=FLAGS.c_vars,
feature_remove=FLAGS.r_vars,
num_classes=FLAGS.truncated_num_classes,
decode_zlib=FLAGS.decode_zlib)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
inference(reader, FLAGS.train_dir, FLAGS.input_data_pattern,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
def get_reader():
# Convert feature_names and feature_sizes to lists of values.
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
#print('FEATURE NAMES',feature_names,'\n\n\n\n\n')
#print('FEATURES SIZE',feature_sizes,'\n\n\n\n\n')
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names,
feature_sizes=feature_sizes,
segment_labels=FLAGS.segment_labels)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
print(reader)
return reader
def get_reader():
# Convert feature_names and feature_sizes to lists of values.
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(
num_classes=FLAGS.truncated_num_classes,
decode_zlib=FLAGS.decode_zlib,
feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
num_classes=FLAGS.truncated_num_classes,
decode_zlib=FLAGS.decode_zlib,
feature_names=feature_names,
feature_sizes=feature_sizes)
return reader
def evaluate():
tf.set_random_seed(0) # for reproducibility
# Write json of flags
model_flags_path = os.path.join(FLAGS.train_dir, "model_flags.json")
if not file_io.file_exists(model_flags_path):
raise IOError(("Cannot find file %s. Did you run train.py on the same "
"--train_dir?") % model_flags_path)
flags_dict = json.loads(file_io.FileIO(model_flags_path, mode="r").read())
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
model = find_class_by_name(flags_dict["model"],
[frame_level_models, video_level_models])()
label_loss_fn = find_class_by_name(flags_dict["label_loss"], [losses])()
if FLAGS.eval_data_pattern is "":
raise IOError("'eval_data_pattern' was not specified. " +
"Nothing to evaluate.")
build_graph(
reader=reader,
model=model,
eval_data_pattern=FLAGS.eval_data_pattern,
label_loss_fn=label_loss_fn,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
summary_writer = tf.summary.FileWriter(
FLAGS.train_dir, graph=tf.get_default_graph())
evl_metrics = eval_util.EvaluationMetrics(reader.num_classes, FLAGS.top_k)
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
#if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names,
feature_sizes=feature_sizes,
segment_labels=FLAGS.segment_labels)
#else:
# reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
# feature_sizes=feature_sizes,
# segment_labels=FLAGS.segment_labels)
if FLAGS.output_dir is "":
raise ValueError("'output_dir' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
transformer_fn = find_class_by_name(FLAGS.feature_transformer,
[feature_transform])
build_graph(reader,
model,
input_data_pattern=FLAGS.input_data_pattern,
batch_size=FLAGS.batch_size,
transformer_class=transformer_fn)
saver = tf.train.Saver(max_to_keep=3,
keep_checkpoint_every_n_hours=10000000000)
inference(saver, FLAGS.train_dir, FLAGS.output_dir, FLAGS.batch_size,
FLAGS.top_k)
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
if FLAGS.output_file is "":
raise ValueError("'output_file' was not specified. "
"Unable to continue with inference.")
if FLAGS.input_data_pattern is "":
raise ValueError("'input_data_pattern' was not specified. "
"Unable to continue with inference.")
def get_reader():
"""
Convert feature_names and feature_sizes to lists of values.
:return: A reader to read data from tfrecords.
"""
# Convert csv format into a list of strings or list of integers. Added by Sophie.
# For example, feature_names = 'mean_rgb, mean_audio' => ['mean_rgb', 'mean_audio'].
# feature_sizes = '1024, 128' => [1024, 128].
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
# using frame-level features. Added by Sophie.
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
# using video-level features. Added by Sophie.
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
return reader
def predict_init(self):
# logging.set_verbosity(tf.logging.INFO)
self.log.info("Initing pre_model and reader")
if self.pred_flags.input_model_tgz:
if self.pred_flags.train_dir:
raise ValueError("You cannot supply --train_dir if supplying "
"--input_model_tgz")
# Untar.
if not os.path.exists(self.pred_flags.untar_model_dir):
os.makedirs(self.pred_flags.untar_model_dir)
tarfile.open(self.pred_flags.input_model_tgz).extractall(self.pred_flags.untar_model_dir)
self.pred_flags.train_dir = self.pred_flags.untar_model_dir
flags_dict_file = os.path.join(self.pred_flags.train_dir, "model_flags.json")
if not os.path.exists(flags_dict_file):
raise IOError("Cannot find %s. Did you run eval.py?" % flags_dict_file)
flags_dict = json.loads(open(flags_dict_file).read())
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes,
prepare_distill=True)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
# if pred_flags.output_file is "":
# raise ValueError("'output_file' was not specified. "
# "Unable to continue with inference.")
# if self.pred_flags.input_data_pattern is "":
# raise ValueError("'input_data_pattern' was not specified. "
# "Unable to continue with inference.")
self.log.info("Successful init pre_model and reader")
return reader
def evaluate(model, checkpoint_name, k_frame):
tf.set_random_seed(0) # for reproducibility
# Write json of flags
model_flags_path = os.path.join(FLAGS.train_dir, "model_flags.json")
if not os.path.exists(model_flags_path):
raise IOError(("Cannot find file %s. Did you run train.py on the same "
"--train_dir?") % model_flags_path)
flags_dict = json.loads(open(model_flags_path).read())
with tf.Graph().as_default() as graph:
# scope_name
# if FLAGS.model_type=='KD':
# scope_name='student'
# with tf.variable_scope(, reuse=False):
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
label_loss_fn = find_class_by_name(flags_dict["label_loss"],
[losses])()
if FLAGS.eval_data_pattern is "":
raise IOError("'eval_data_pattern' was not specified. " +
"Nothing to evaluate.")
scope_name = 'teacher'
if k_frame != 300:
scope_name = 'student'
with tf.variable_scope(scope_name, reuse=None):
build_graph(reader=reader,
model=model,
eval_data_pattern=FLAGS.eval_data_pattern,
label_loss_fn=label_loss_fn,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size,
k_frame=k_frame)
# print("See variables in student in EVAL")
# for i in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,scope='student'):
# print(i) # i.name if you want just a name
logging.info("built evaluation graph")
video_id_batch = tf.get_collection("video_id_batch")[0]
prediction_batch = tf.get_collection("predictions")[0]
label_batch = tf.get_collection("labels")[0]
loss = tf.get_collection("loss")[0]
summary_op = tf.get_collection("summary_op")[0]
saver = tf.train.Saver(tf.global_variables())
if k_frame != 300:
student_variables = tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES, scope='student')
saver = tf.train.Saver(var_list=student_variables)
# student_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='student')
# print("See variables in student")
# for i in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='student'):
# print(i) # i.name if you want just a name
# student_saver = tf.train.Saver(var_list=student_variables)
summary_writer = tf.summary.FileWriter(FLAGS.train_dir,
graph=tf.get_default_graph())
evl_metrics = eval_util.EvaluationMetrics(reader.num_classes,
FLAGS.top_k)
last_global_step_val = -1
while True:
last_global_step_val, EvalGAP = evaluation_loop(
video_id_batch, prediction_batch, label_batch, loss,
summary_op, saver, summary_writer, evl_metrics,
last_global_step_val, checkpoint_name)
if FLAGS.run_once:
break
return EvalGAP
def evaluate():
"""Starts main evaluation loop."""
tf.set_random_seed(0) # for reproducibility
# Write json of flags
model_flags_path = os.path.join(FLAGS.train_dir, "model_flags.json")
if not file_io.file_exists(model_flags_path):
raise IOError(("Cannot find file %s. Did you run train.py on the same "
"--train_dir?") % model_flags_path)
flags_dict = json.loads(file_io.FileIO(model_flags_path, mode="r").read())
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names,
feature_sizes=feature_sizes,
segment_labels=FLAGS.segment_labels)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
model = find_class_by_name(
flags_dict["model"],
[frame_level_models, video_level_models, nextvlad])()
label_loss_fn = find_class_by_name(flags_dict["label_loss"],
[losses])()
if not FLAGS.eval_data_pattern:
raise IOError("'eval_data_pattern' was not specified. Nothing to "
"evaluate.")
build_graph(reader=reader,
model=model,
eval_data_pattern=FLAGS.eval_data_pattern,
label_loss_fn=label_loss_fn,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
# A dict of tensors to be run in Session.
fetches = {
"video_id": tf.get_collection("video_id_batch")[0],
"predictions": tf.get_collection("predictions")[0],
"labels": tf.get_collection("labels")[0],
"loss": tf.get_collection("loss")[0],
"summary": tf.get_collection("summary_op")[0]
}
if FLAGS.segment_labels:
fetches["label_weights"] = tf.get_collection("label_weights")[0]
saver = tf.train.Saver(tf.global_variables())
summary_writer = tf.summary.FileWriter(os.path.join(
FLAGS.train_dir, "eval"),
graph=tf.get_default_graph())
evl_metrics = eval_util.EvaluationMetrics(reader.num_classes,
FLAGS.top_k, None)
last_global_step_val = -1
cur_checkpoint_file = FLAGS.checkpoint_file
while True:
checkpoint_file = 'model.ckpt-{}'.format(cur_checkpoint_file)
checkpoint = get_checkpoint(checkpoint_file)
try:
last_global_step_val = evaluation_loop(fetches, saver,
summary_writer,
evl_metrics, checkpoint,
last_global_step_val)
cur_checkpoint_file += 200
except:
logging.info(
'Find invalid checkpoint, try again using checkpoint: {}'.
format(cur_checkpoint_file))
break
if FLAGS.run_once:
break
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
print("tensorflow version: %s" % tf.__version__)
is_chief = (FLAGS.task == 0)
# Recover session
saver = None
latest_checkpoint = tf.train.latest_checkpoint(FLAGS.train_dir)
if FLAGS.start_new_model:
logging.info("'start_new_model' flag is set. Removing existing train dir.")
try:
gfile.DeleteRecursively(FLAGS.train_dir)
except:
logging.error(
"Failed to delete directory " + FLAGS.train_dir +
" when starting a new model. Please delete it manually and" +
" try again.")
elif not latest_checkpoint:
logging.info("No checkpoint file found. Building a new model.")
else:
meta_filename = latest_checkpoint + ".meta"
if not gfile.Exists(meta_filename):
logging.info("No meta graph file found. Building a new model.")
else:
logging.info("Restoring from meta graph file %s", meta_filename)
saver = tf.train.import_meta_graph(meta_filename)
if not saver:
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
if FLAGS.frame_features:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names,
feature_sizes=feature_sizes)
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
optimizer_class = find_class_by_name(FLAGS.optimizer, [tf.train])
build_graph(reader=reader,
model=model,
optimizer_class=optimizer_class,
clip_gradient_norm=FLAGS.clip_gradient_norm,
train_data_pattern=FLAGS.train_data_pattern,
label_loss_fn=label_loss_fn,
base_learning_rate=FLAGS.base_learning_rate,
learning_rate_decay=FLAGS.learning_rate_decay,
learning_rate_decay_examples=FLAGS.learning_rate_decay_examples,
regularization_penalty=FLAGS.regularization_penalty,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.num_epochs)
logging.info("built graph")
saver = tf.train.Saver(max_to_keep=0, keep_checkpoint_every_n_hours=0.25)
train_loop(is_chief=is_chief,
train_dir=FLAGS.train_dir,
saver=saver,
master=FLAGS.master)
def evaluate():
tf.set_random_seed(0) # for reproducibility
# Write json of flags
model_flags_path = os.path.join(FLAGS.train_dir, "model_flags.json")
if not file_io.file_exists(model_flags_path):
raise IOError(("Cannot find file %s. Did you run train.py on the same "
"--train_dir?") % model_flags_path)
flags_dict = json.loads(file_io.FileIO(model_flags_path, "r").read())
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
model = frame_level_models.NetVLADModelLF()
model_lst = [model]
if FLAGS.ensemble_num > 1:
for ensemble_idx in range(1, FLAGS.ensemble_num):
model2 = frame_level_models.NetVLADModelLF()
model_lst.append(model2)
label_loss_fn = losses.CrossEntropyLoss()
if FLAGS.eval_data_pattern is "":
raise IOError("'eval_data_pattern' was not specified. " +
"Nothing to evaluate.")
video_id_batch = build_graph(reader=reader,
model=model_lst,
eval_data_pattern=FLAGS.eval_data_pattern,
label_loss_fn=label_loss_fn,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
if not FLAGS.force_output_model_name:
video_id_batch = tf.get_collection("video_id_batch")[0]
prediction_batch = tf.get_collection("predictions")[0]
label_batch = tf.get_collection("labels")[0]
loss = tf.get_collection("loss")[0]
summary_op = tf.get_collection("summary_op")[0]
saver = tf.train.Saver(tf.global_variables())
summary_writer = tf.summary.FileWriter(FLAGS.train_dir,
graph=tf.get_default_graph())
evl_metrics = eval_util.EvaluationMetrics(reader.num_classes,
FLAGS.top_k)
last_global_step_val = -1
#with tf.device("/gpu:0"):
while True:
last_global_step_val = evaluation_loop(video_id_batch,
prediction_batch,
label_batch, loss,
summary_op, saver,
summary_writer, evl_metrics,
last_global_step_val)
if FLAGS.run_once:
break
def inference_loop():
model_path = FLAGS.model_file
checkpoint_file = os.path.join(model_path, "inference_model",
"inference_model")
print(checkpoint_file)
# assert os.path.isfile(checkpoint_file + ".meta"), "Specified model does not exist."
if not gfile.Exists(checkpoint_file + ".meta"):
raise IOError("Cannot find %s. Did you run eval.py?" % checkpoint_file)
# model_flags_path = os.path.join(os.path.dirname(model_path), "model_flags.json")
model_flags_path = os.path.join(model_path, "model_flags.json")
directory = FLAGS.output_dir # We will store the predictions here.
if not os.path.exists(directory):
os.makedirs(directory)
else:
raise IOError("Output path exists! path='" + directory + "'")
# if not gfile.Exists(model_flags_path):#os.path.exists(model_flags_path):
# raise IOError(("Cannot find file %s. Did you run train.py on the same "
# "--train_dir?") % model_flags_path)
# flags_dict = json.loads(open(model_flags_path).read())
if not file_io.file_exists(model_flags_path):
raise IOError("Cannot find %s. Did you run eval.py?" %
model_flags_path)
flags_dict = json.loads(file_io.FileIO(model_flags_path, "r").read())
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
raise NotImplementedError
video_ids_batch, inputs_batch, labels_batch, num_frames = fetc_inputs(
reader, FLAGS.input_data_pattern, FLAGS.batch_size, FLAGS.num_readers)
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
# video_id_batch, video_batch, num_frames_batch = get_input_data_tensors(
# reader, FLAGS.input_data_pattern, FLAGS.batch_size)
# meta_graph_location = model_path + ".meta"
meta_graph_location = checkpoint_file + ".meta"
logging.info("loading meta-graph: " + meta_graph_location)
with tf.device("/cpu:0"): #gpu
saver = tf.train.import_meta_graph(meta_graph_location,
clear_devices=True)
saver.restore(sess, checkpoint_file)
input_tensor = tf.get_collection("input_batch_raw")[0]
num_frames_tensor = tf.get_collection("num_frames")[0]
predictions_batch = tf.get_collection("predictions")[0]
# Workaround for num_epochs issue.
def set_up_init_ops(variables):
init_op_list = []
for variable in list(variables):
if "train_input" in variable.name:
init_op_list.append(tf.assign(variable, 1))
variables.remove(variable)
init_op_list.append(tf.variables_initializer(variables))
return init_op_list
sess.run(
set_up_init_ops(tf.get_collection_ref(
tf.GraphKeys.LOCAL_VARIABLES)))
# Start the queue runners.
fetches1 = [video_ids_batch, labels_batch, inputs_batch, num_frames]
fetches2 = [predictions_batch]
coord = tf.train.Coordinator()
start_time = time.time()
video_ids = []
video_labels = []
video_rgbs = []
video_audios = []
video_predictions = []
video_num_frames = []
filenum = 0
num_examples_processed = 0
total_num_examples_processed = 0
import csv
import urllib2
# import numpy as np
whitelisted_cls_mask = np.zeros((3862, ), dtype=np.float32)
url = 'http://storage.googleapis.com/youtube8m-lijun-mlengine/segment_label_ids.csv'
response = urllib2.urlopen(url)
fobj = csv.reader(response)
for line in fobj:
try:
cls_id = int(line[0])
whitelisted_cls_mask[cls_id] = 1.
except ValueError:
# Simply skip the non-integer line.
continue
response.close()
# print('mask',whitelisted_cls_mask)
# print('mask_shape',whitelisted_cls_mask.shape)
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(
qr.create_threads(sess,
coord=coord,
daemon=True,
start=True))
while not coord.should_stop():
ids_val = None
ids_val, labels_val, inputs_val, num_frames_val = sess.run(
fetches1)
# print('labels_val',labels_val.shape)
target = np.matmul(labels_val, whitelisted_cls_mask)
# print('target',target.shape)
target_index = target > 0.
labels_val = labels_val[target_index, :]
inputs_val = inputs_val[target_index, :, :]
num_frames_val = num_frames_val[target_index]
ids_val = ids_val[target_index]
results = get_segments(inputs_val[:, 20:, :],
num_frames_val - 20 - 20, 5, labels_val)
results["video_segment_ids"][:, 1] = results[
"video_segment_ids"][:, 1] + 20
video_segment_ids = results["video_segment_ids"]
video_id_batch_val = ids_val[video_segment_ids[:, 0]]
video_id_batch_val = np.array([
"%s:%d" % (x, y)
for x, y in zip(video_id_batch_val, video_segment_ids[:,
1])
])
ids_val = video_id_batch_val
inputs_val = results['video_batch']
num_frames_val = results['num_frames_batch']
labels_val = results['labels_batch']
# print(results['video_batch'].shape)
# print(results['video_segment_ids'])
# print(results['num_frames_batch'])
print(video_id_batch_val)
# rgbs_val, audios_val = quant_inpt_val[:, :, :1024].copy(), quant_inpt_val[:, :, 1024:].copy()
rgbs_val, audios_val = inputs_val[:, :, :1024].copy(
), inputs_val[:, :, 1024:].copy()
predictions_val = sess.run(fetches2,
feed_dict={
input_tensor: inputs_val,
num_frames_tensor:
num_frames_val
})[0]
# print('inputs_val',inputs_val)
# print('inputs_val',inputs_val.shape)
# print('num_frames',num_frames_val)
# print('num_frames',num_frames_val.shape)
# print('ids_val',ids_val)
# print('ids_val',ids_val.shape)
# print('labels_val',labels_val)
# print('labels_val',labels_val.shape)
# print('predictions_val',predictions_val)
# print('predictions_val',predictions_val.shape)
# print('predictions_val',predictions_val.shape)
# print('whitelisted_cls_mask sum',np.sum(whitelisted_cls_mask))
predictions_val = predictions_val * whitelisted_cls_mask
video_max = np.amax(predictions_val, axis=1)
sorted_idx = np.argsort(-video_max)[:64]
# print('index',video_max[sorted_idx])
maxIndex = np.argmax(predictions_val, axis=1)
select = labels_val[np.arange(len(labels_val)), maxIndex]
if len(ids_val[select]) >= 64: #256
ids_val = ids_val[select]
labels_val = labels_val[select, :]
rgbs_val = rgbs_val[select, :, :]
audios_val = audios_val[select, :, :]
predictions_val = predictions_val[select, :]
num_frames_val = num_frames_val[select]
inputs_val = inputs_val[select, :, :]
video_max = np.amax(predictions_val, axis=1)
sorted_idx = np.argsort(-video_max)[:64]
print('saved segments are truly predicted')
ids_val = ids_val[sorted_idx]
labels_val = labels_val[sorted_idx, :]
rgbs_val = rgbs_val[sorted_idx, :, :]
audios_val = audios_val[sorted_idx, :, :]
predictions_val = predictions_val[sorted_idx, :]
num_frames_val = num_frames_val[sorted_idx]
# video_max_matrix=np.repeat(np.amax(predictions_val,axis=1)[:,np.newaxis],3862,axis=1)
predictions_val = predictions_val >= (
np.ones_like(predictions_val) * 0.5) #video_max_matrix
# video_max_matrix=np.repeat(np.amax(predictions_val,axis=1)[:,np.newaxis],3862,axis=1)
# predictions_val=predictions_val>=video_max_matrix
# print('boolean_prediction',predictions_val)
# print(predictions_val.shape)
video_ids.append(ids_val) #ids_val
video_labels.append(labels_val) #labels_val
video_rgbs.append(rgbs_val) #rgbs_val
video_audios.append(audios_val) #audios_val
video_predictions.append(predictions_val) #predictions_val
video_num_frames.append(num_frames_val) #num_frames_val
num_examples_processed += len(ids_val)
ids_shape = ids_val.shape[0]
inputs_shape = inputs_val[sorted_idx, :, :].shape[0]
predictions_shape = predictions_val.shape[0]
assert ids_shape == inputs_shape == predictions_shape, "tensor ids(%d), inputs(%d) and predictions(%d) should have equal rows" % (
ids_shape, inputs_shape, predictions_shape)
ids_val = None
if num_examples_processed >= FLAGS.file_size:
assert num_examples_processed == FLAGS.file_size, "num_examples_processed should be equal to %d" % FLAGS.file_size
video_ids = np.concatenate(video_ids, axis=0)
# video_labels = np.concatenate(video_labels, axis=0)
video_labels = np.concatenate(video_predictions, axis=0)
video_rgbs = np.concatenate(video_rgbs, axis=0)
video_audios = np.concatenate(video_audios, axis=0)
video_num_frames = np.concatenate(video_num_frames, axis=0)
video_predictions = np.concatenate(video_predictions,
axis=0)
write_to_record(
video_ids, video_labels, video_rgbs, video_audios,
video_num_frames, filenum,
num_examples_processed) #, video_predictions
video_ids = []
video_labels = []
video_rgbs = []
video_audios = []
video_predictions = []
video_num_frames = []
filenum += 1
total_num_examples_processed += num_examples_processed
now = time.time()
logging.info("num examples processed: " +
str(total_num_examples_processed) +
" elapsed seconds: " +
"{0:.2f}".format(now - start_time))
num_examples_processed = 0
except tf.errors.OutOfRangeError as e:
if ids_val is not None:
video_ids.append(ids_val)
video_labels.append(labels_val)
video_rgbs.append(rgbs_val)
video_audios.append(audios_val)
video_predictions.append(predictions_val)
video_num_frames.append(num_frames_val)
num_examples_processed += len(ids_val)
if 0 < num_examples_processed <= FLAGS.file_size:
video_ids = np.concatenate(video_ids, axis=0)
# video_labels = np.concatenate(video_labels, axis=0)
video_labels = np.concatenate(video_predictions, axis=0)
video_rgbs = np.concatenate(video_rgbs, axis=0)
video_audios = np.concatenate(video_audios, axis=0)
video_num_frames = np.concatenate(video_num_frames, axis=0)
video_predictions = np.concatenate(video_predictions, axis=0)
write_to_record(video_ids, video_labels, video_rgbs,
video_audios, video_num_frames, filenum,
num_examples_processed) #, video_predictions
total_num_examples_processed += num_examples_processed
now = time.time()
logging.info("num examples processed: " +
str(total_num_examples_processed) +
" elapsed seconds: " +
"{0:.2f}".format(now - start_time))
logging.info("Done with inference. %d samples was written to %s" %
(total_num_examples_processed, FLAGS.output_dir))
# except Exception as e: # pylint: disable=broad-except
# logging.info("Unexpected exception: " + str(e))
finally:
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
def evaluate():
tf.compat.v1.set_random_seed(0) # for reproducibility
# Write json of flags
model_flags_path = os.path.join(FLAGS.train_dir, "model_flags.json")
if not file_io.file_exists(model_flags_path):
raise IOError(("Cannot find file %s. Did you run train.py on the same "
"--train_dir?") % model_flags_path)
flags_dict = json.loads(open(model_flags_path).read())
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names,
feature_sizes=feature_sizes,
segment_labels=FLAGS.segment_labels,
)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
model = find_class_by_name(
flags_dict["model"],
[frame_level_models, video_level_models, nextvlad])()
label_loss_fn = find_class_by_name(flags_dict["label_loss"],
[losses])()
if FLAGS.eval_data_pattern:
raise IOError("'eval_data_pattern' was not specified. " +
"Nothing to evaluate.")
build_graph(
reader=reader,
model=model,
eval_data_pattern=FLAGS.eval_data_pattern,
label_loss_fn=label_loss_fn,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size,
)
logging.info("built evaluation graph")
# xxx 2018
# video_id_batch = tf.compat.v1.get_collection("video_id_batch")[0]
# prediction_batch = tf.compat.v1.get_collection("predictions")[0]
# label_batch = tf.compat.v1.get_collection("labels")[0]
# loss = tf.compat.v1.get_collection("loss")[0]
# summary_op = tf.compat.v1.get_collection("summary_op")[0]
# A dict of tensors to be run in Session.
fetches = {
"video_id": tf.compat.v1.get_collection("video_id_batch")[0],
"predictions": tf.compat.v1.get_collection("predictions")[0],
"labels": tf.compat.v1.get_collection("labels")[0],
"loss": tf.compat.v1.get_collection("loss")[0],
"summary": tf.compat.v1.get_collection("summary_op")[0],
}
if FLAGS.segment_labels:
fetches["label_weights"] = tf.compat.v1.get_collection(
"label_weights")[0]
saver = tf.compat.v1.train.Saver(tf.compat.v1.global_variables())
summary_writer = tf.compat.v1.summary.FileWriter(
os.path.join(FLAGS.train_dir, "eval"),
graph=tf.compat.v1.get_default_graph(),
)
evl_metrics = eval_util.EvaluationMetrics(reader.num_classes,
FLAGS.top_k, None)
last_global_step_val = -1
while True:
last_global_step_val = evaluation_loop(fetches, saver,
summary_writer, evl_metrics,
last_global_step_val)
if FLAGS.run_once:
break
def inference_loop():
model_path = FLAGS.model_file
assert os.path.isfile(model_path +
".meta"), "Specified model does not exist."
model_flags_path = os.path.join(os.path.dirname(model_path),
"model_flags.json")
directory = FLAGS.output_dir # We will store the predictions here.
if not os.path.exists(directory):
os.makedirs(directory)
else:
raise IOError("Output path exists! path='" + directory + "'")
if not os.path.exists(model_flags_path):
raise IOError(("Cannot find file %s. Did you run train.py on the same "
"--train_dir?") % model_flags_path)
flags_dict = json.loads(open(model_flags_path).read())
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes,
prepare_distill=True)
else:
raise NotImplementedError
video_ids_batch, inputs_batch, labels_batch, num_frames, quant_inpt = fetc_inputs(
reader, FLAGS.input_data_pattern, FLAGS.batch_size, FLAGS.num_readers)
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
meta_graph_location = model_path + ".meta"
logging.info("loading meta-graph: " + meta_graph_location)
with tf.device("/gpu:0"):
saver = tf.train.import_meta_graph(meta_graph_location,
clear_devices=True)
saver.restore(sess, model_path)
input_tensor = tf.get_collection("input_batch_raw")[0]
num_frames_tensor = tf.get_collection("num_frames")[0]
predictions_batch = tf.get_collection("predictions")[0]
# Workaround for num_epochs issue.
def set_up_init_ops(variables):
init_op_list = []
for variable in list(variables):
if "train_input" in variable.name:
init_op_list.append(tf.assign(variable, 1))
variables.remove(variable)
init_op_list.append(tf.variables_initializer(variables))
return init_op_list
sess.run(
set_up_init_ops(tf.get_collection_ref(
tf.GraphKeys.LOCAL_VARIABLES)))
# Start the queue runners.
fetches1 = [
video_ids_batch, labels_batch, inputs_batch, num_frames, quant_inpt
]
fetches2 = [predictions_batch]
coord = tf.train.Coordinator()
start_time = time.time()
video_ids = []
video_labels = []
video_rgbs = []
video_audios = []
video_predictions = []
video_num_frames = []
filenum = 0
num_examples_processed = 0
total_num_examples_processed = 0
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(
qr.create_threads(sess,
coord=coord,
daemon=True,
start=True))
while not coord.should_stop():
ids_val = None
ids_val, labels_val, inputs_val, num_frames_val, quant_inpt_val = sess.run(
fetches1)
rgbs_val, audios_val = quant_inpt_val[:, :, :1024].copy(
), quant_inpt_val[:, :, 1024:].copy()
predictions_val = sess.run(fetches2,
feed_dict={
input_tensor: inputs_val,
num_frames_tensor:
num_frames_val
})[0]
video_ids.append(ids_val)
video_labels.append(labels_val)
video_rgbs.append(rgbs_val)
video_audios.append(audios_val)
video_predictions.append(predictions_val)
video_num_frames.append(num_frames_val)
num_examples_processed += len(ids_val)
ids_shape = ids_val.shape[0]
inputs_shape = inputs_val.shape[0]
predictions_shape = predictions_val.shape[0]
assert ids_shape == inputs_shape == predictions_shape, "tensor ids(%d), inputs(%d) and predictions(%d) should have equal rows" % (
ids_shape, inputs_shape, predictions_shape)
ids_val = None
if num_examples_processed >= FLAGS.file_size:
assert num_examples_processed == FLAGS.file_size, "num_examples_processed should be equal to %d" % FLAGS.file_size
video_ids = np.concatenate(video_ids, axis=0)
video_labels = np.concatenate(video_labels, axis=0)
video_rgbs = np.concatenate(video_rgbs, axis=0)
video_audios = np.concatenate(video_audios, axis=0)
video_num_frames = np.concatenate(video_num_frames, axis=0)
video_predictions = np.concatenate(video_predictions,
axis=0)
write_to_record(video_ids, video_labels, video_rgbs,
video_audios, video_predictions,
video_num_frames, filenum,
num_examples_processed)
video_ids = []
video_labels = []
video_rgbs = []
video_audios = []
video_predictions = []
video_num_frames = []
filenum += 1
total_num_examples_processed += num_examples_processed
now = time.time()
logging.info("num examples processed: " +
str(num_examples_processed) +
" elapsed seconds: " +
"{0:.2f}".format(now - start_time))
num_examples_processed = 0
except tf.errors.OutOfRangeError as e:
if ids_val is not None:
video_ids.append(ids_val)
video_labels.append(labels_val)
video_rgbs.append(rgbs_val)
video_audios.append(audios_val)
video_predictions.append(predictions_val)
video_num_frames.append(num_frames_val)
num_examples_processed += len(ids_val)
if 0 < num_examples_processed <= FLAGS.file_size:
video_ids = np.concatenate(video_ids, axis=0)
video_labels = np.concatenate(video_labels, axis=0)
video_rgbs = np.concatenate(video_rgbs, axis=0)
video_audios = np.concatenate(video_audios, axis=0)
video_num_frames = np.concatenate(video_num_frames, axis=0)
video_predictions = np.concatenate(video_predictions, axis=0)
write_to_record(video_ids, video_labels, video_rgbs,
video_audios, video_predictions,
video_num_frames, filenum,
num_examples_processed)
total_num_examples_processed += num_examples_processed
now = time.time()
logging.info("num examples processed: " +
str(total_num_examples_processed) +
" elapsed seconds: " +
"{0:.2f}".format(now - start_time))
logging.info("Done with inference. %d samples was written to %s" %
(total_num_examples_processed, FLAGS.output_dir))
# except Exception as e: # pylint: disable=broad-except
# logging.info("Unexpected exception: " + str(e))
finally:
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
def evaluate():
ema_tensors = None
if FLAGS.use_EMA:
latest_checkpoint = get_latest_checkpoint()
assert latest_checkpoint, "No checkpoint found"
with tf.device("/cpu:0"):
saver = tf.train.import_meta_graph(latest_checkpoint + ".meta", clear_devices=True)
# saver.restore(sess, "../trained_models/attention_frames_v0_EMA/model.ckpt-15512")
xvars = tf.get_collection("ema_vars")
assert len(xvars) > 0, "No EMA shadow variables found. Did you train with EMA?"
ema_tensors = list(set([x.name for x in xvars]))
tf.reset_default_graph()
tf.set_random_seed(0) # for reproducibility
# Write json of flags
model_flags_path = os.path.join(FLAGS.train_dir, "model_flags.json")
if not os.path.exists(model_flags_path):
raise IOError(("Cannot find file %s. Did you run train.py on the same "
"--train_dir?") % model_flags_path)
flags_dict = json.loads(open(model_flags_path).read())
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
model = find_class_by_name(flags_dict["model"],
[frame_level_models, video_level_models])()
label_loss_fn = find_class_by_name(flags_dict["label_loss"], [losses])()
if FLAGS.eval_data_pattern is "":
raise IOError("'eval_data_pattern' was not specified. " +
"Nothing to evaluate.")
build_graph(
reader=reader,
model=model,
eval_data_pattern=FLAGS.eval_data_pattern,
label_loss_fn=label_loss_fn,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_batch = tf.get_collection("video_id_batch")[0]
prediction_batch = tf.get_collection("predictions")[0]
label_batch = tf.get_collection("labels")[0]
loss = tf.get_collection("loss")[0]
summary_op = tf.get_collection("summary_op")[0]
saver = tf.train.Saver(tf.global_variables())
summary_writer = tf.summary.FileWriter(
FLAGS.train_dir, graph=tf.get_default_graph())
evl_metrics = eval_util.EvaluationMetrics(reader.num_classes, FLAGS.top_k)
last_global_step_val = -1
while True:
last_global_step_val = evaluation_loop(video_id_batch, prediction_batch,
label_batch, loss, summary_op,
saver, summary_writer, evl_metrics,
last_global_step_val, ema_tensors)
if FLAGS.run_once:
break
def evaluate():
tf.set_random_seed(0) # for reproducibility
# Write json of flags
model_flags_path = os.path.join(FLAGS.train_dir, "model_flags.json")
# if not os.path.exists(model_flags_path):
if not file_io.file_exists(model_flags_path):
raise IOError(("Cannot find file %s. Did you run train.py on the same "
"--train_dir?") % model_flags_path)
# flags_dict = json.loads(open(model_flags_path).read())
flags_dict = json.loads(file_io.FileIO(model_flags_path, mode='r').read())
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
model = find_class_by_name(flags_dict["model"],
[frame_level_models, video_level_models])()
label_loss_fn = find_class_by_name(flags_dict["label_loss"],
[losses])()
if FLAGS.eval_data_pattern is "":
raise IOError("'eval_data_pattern' was not specified. " +
"Nothing to evaluate.")
build_graph(reader=reader,
model=model,
eval_data_pattern=FLAGS.eval_data_pattern,
label_loss_fn=label_loss_fn,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_batch = tf.get_collection("video_id_batch")[0]
prediction_batch = tf.get_collection("predictions")[0]
label_batch = tf.get_collection("labels")[0]
loss = tf.get_collection("loss")[0]
summary_op = tf.get_collection("summary_op")[0]
saver = tf.train.Saver(tf.global_variables())
summary_writer = tf.summary.FileWriter(FLAGS.train_dir,
graph=tf.get_default_graph())
evl_metrics = eval_util.EvaluationMetrics(reader.num_classes,
FLAGS.top_k)
last_global_step_val = -1
while True:
last_global_step_val = evaluation_loop(video_id_batch,
prediction_batch,
label_batch, loss,
summary_op, saver,
summary_writer, evl_metrics,
last_global_step_val)
if FLAGS.run_once:
break
def evaluate():
model_path = FLAGS.model_file
assert os.path.isfile(model_path +
".meta"), "Specified model does not exist."
model_flags_path = os.path.join(os.path.dirname(model_path),
"model_flags.json")
if not os.path.exists(model_flags_path):
raise IOError(("Cannot find file %s. Did you run train.py on the same "
"--train_dir?") % model_flags_path)
flags_dict = json.loads(open(model_flags_path).read())
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
raise NotImplementedError
video_id_batch, model_input_raw, labels_batch, num_frames = fetc_inputs(
reader, FLAGS.eval_data_pattern, FLAGS.batch_size, FLAGS.num_readers)
evl_metrics = GapScore(reader.num_classes, FLAGS.top_k)
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
meta_graph_location = model_path + ".meta"
logging.info("loading meta-graph: " + meta_graph_location)
with tf.device("/gpu:0"):
saver = tf.train.import_meta_graph(meta_graph_location,
clear_devices=True)
saver.restore(sess, model_path)
input_tensor = tf.get_collection("input_batch_raw")[0]
num_frames_tensor = tf.get_collection("num_frames")[0]
predictions_tensor = tf.get_collection("predictions")[0]
# loss = tf.get_collection("loss")[0]
# Workaround for num_epochs issue.
def set_up_init_ops(variables):
init_op_list = []
for variable in list(variables):
if "train_input" in variable.name:
init_op_list.append(tf.assign(variable, 1))
variables.remove(variable)
init_op_list.append(tf.variables_initializer(variables))
return init_op_list
sess.run(
set_up_init_ops(tf.get_collection_ref(
tf.GraphKeys.LOCAL_VARIABLES)))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
num_examples_processed = 0
start_time = time.time()
try:
while not coord.should_stop():
xvideo_id_batch, xmodel_input_raw, xlabels_batch, xnum_frames = sess.run(
[
video_id_batch, model_input_raw, labels_batch,
num_frames
])
predictions_val = sess.run([predictions_tensor],
feed_dict={
input_tensor: xmodel_input_raw,
num_frames_tensor: xnum_frames
})[0]
now = time.time()
num_examples_processed += len(predictions_val)
# num_classes = predictions_val.shape[1]
logging.info("num examples processed: " +
str(num_examples_processed) +
" elapsed seconds: " +
"{0:.2f}".format(now - start_time))
# Collect metrics
evl_metrics.accumulate(predictions_val, xlabels_batch)
except tf.errors.OutOfRangeError:
logging.info('Done with inference.')
finally:
coord.request_stop()
coord.join(threads)
sess.close()
logging.info('Calculating statistics.')
epoch_info_dict = evl_metrics.get()
logging.info('GAP Score: {}'.format(epoch_info_dict))
def main(stage, split_id=""):
feature_names = "rgb"
feature_sizes = "1024"
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
feature_names, feature_sizes)
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names,
feature_sizes=feature_sizes,
)
# data_pattern = "/data/uts700/linchao/yt8m/data/{0}/{0}*.tfrecord".format(stage)
data_pattern = "/data/uts700/linchao/yt8m/data/splits/{0}/{1}/{0}*.tfrecord".format(
stage, split_id)
# data_pattern = "/data/uts700/linchao/yt8m/data/splits/train/5/traincc.tfrecord"#.format(stage, split_id)
num_readers = 3
batch_size = 128
input_shuffle = False
files = gfile.Glob(data_pattern)
if not files:
raise IOError("Unable to find the evaluation files.")
filename_queue = tf.train.string_input_producer(files,
shuffle=input_shuffle,
num_epochs=1)
eval_data = [
reader.prepare_reader(filename_queue) for _ in xrange(num_readers)
]
# eval_data = reader.prepare_reader(filename_queue)
if input_shuffle:
eval_data = tf.train.shuffle_batch_join(eval_data,
batch_size=batch_size,
capacity=5 * batch_size,
min_after_dequeue=batch_size,
allow_smaller_final_batch=True)
else:
eval_data = tf.train.batch_join(eval_data,
batch_size=batch_size,
capacity=5 * batch_size,
allow_smaller_final_batch=True)
video_id_batch, model_input_raw, labels_batch, num_frames = eval_data
inputs = {
'video_id': video_id_batch,
'input_raw': model_input_raw,
'labels': labels_batch,
'num_frames': num_frames
}
task = Stats()
task = HDFS(stage, split_id)
with tf.Session() as sess:
sess.run([tf.local_variables_initializer()])
coord = tf.train.Coordinator()
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(
qr.create_threads(sess,
coord=coord,
daemon=True,
start=True))
examples_processed = 0
cnt = 0
while not coord.should_stop():
if not task.run(sess, inputs):
break
examples_processed += batch_size
cnt += 1
if cnt % 5 == 0:
print("examples processed: {}".format(examples_processed))
except tf.errors.OutOfRangeError as e:
logging.info(
"Done with batched inference. Now calculating global performance "
"metrics.")
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
task.done()
