def evaluate():
tf.set_random_seed(0) # for reproducibility
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
# prepare a reader for each single model prediction result
all_readers = []
all_patterns = FLAGS.eval_data_patterns
all_patterns = map(lambda x: x.strip(),
all_patterns.strip().strip(",").split(","))
for i in xrange(len(all_patterns)):
reader = readers.EnsembleReader(feature_names=feature_names,
feature_sizes=feature_sizes)
all_readers.append(reader)
input_reader = None
input_data_pattern = None
if FLAGS.input_data_pattern is not None:
input_reader = readers.EnsembleReader(feature_names=["input"],
feature_sizes=[1024 + 128])
input_data_pattern = FLAGS.input_data_pattern
# find the model
model = find_class_by_name(FLAGS.model, [ensemble_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
if FLAGS.eval_data_patterns is "":
raise IOError("'eval_data_patterns' was not specified. " +
"Nothing to evaluate.")
build_graph(all_readers=all_readers,
input_reader=input_reader,
all_eval_data_patterns=all_patterns,
input_data_pattern=input_data_pattern,
model=model,
label_loss_fn=label_loss_fn,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_batch = tf.get_collection("video_id_batch")[0]
prediction_batch = tf.get_collection("predictions")[0]
label_batch = tf.get_collection("labels")[0]
loss = tf.get_collection("loss")[0]
summary_op = tf.get_collection("summary_op")[0]
saver = tf.train.Saver(tf.global_variables())
summary_writer = tf.summary.FileWriter(FLAGS.train_dir,
graph=tf.get_default_graph())
evl_metrics = eval_util.EvaluationMetrics(FLAGS.num_classes,
FLAGS.top_k)
last_global_step_val = -1
last_global_step_val = evaluation_loop(video_id_batch,
prediction_batch, label_batch,
loss, summary_op, saver,
summary_writer, evl_metrics,
last_global_step_val)
def build_model(self):
"""Find the model and build the graph."""
# Convert feature_names and feature_sizes to lists of values.
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
# prepare a reader for each single model prediction result
all_readers = []
all_patterns = FLAGS.train_data_patterns
all_patterns = map(lambda x: x.strip(),
all_patterns.strip().strip(",").split(","))
for i in xrange(len(all_patterns)):
all_readers.append(
readers.EnsembleReader(feature_names=feature_names,
feature_sizes=feature_sizes))
input_reader = None
input_data_pattern = None
if FLAGS.input_data_pattern is not None:
input_reader = readers.EnsembleReader(feature_names=["input"],
feature_sizes=[1024 + 128])
input_data_pattern = FLAGS.input_data_pattern
# Find the model.
model = find_class_by_name(FLAGS.model, [ensemble_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
optimizer_class = find_class_by_name(FLAGS.optimizer, [tf.train])
build_graph(
all_readers=all_readers,
all_train_data_patterns=all_patterns,
input_reader=input_reader,
input_data_pattern=input_data_pattern,
model=model,
optimizer_class=optimizer_class,
clip_gradient_norm=FLAGS.clip_gradient_norm,
label_loss_fn=label_loss_fn,
base_learning_rate=FLAGS.base_learning_rate,
learning_rate_decay=FLAGS.learning_rate_decay,
learning_rate_decay_examples=FLAGS.learning_rate_decay_examples,
regularization_penalty=FLAGS.regularization_penalty,
batch_size=FLAGS.batch_size,
num_epochs=FLAGS.num_epochs)
logging.info("%s: Built graph.", task_as_string(self.task))
return tf.train.Saver(
max_to_keep=3,
keep_checkpoint_every_n_hours=FLAGS.keep_checkpoint_every_n_hours)
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
# prepare a reader for each single model prediction result
all_readers = []
all_patterns = FLAGS.input_data_patterns
all_patterns = map(lambda x: x.strip(), all_patterns.strip().strip(",").split(","))
for i in xrange(len(all_patterns)):
reader = readers.EnsembleReader(
feature_names=feature_names, feature_sizes=feature_sizes)
all_readers.append(reader)
input_reader = None
input_data_pattern = None
if FLAGS.input_data_pattern is not None:
input_reader = readers.EnsembleReader(
feature_names=["input"], feature_sizes=[1024+128])
input_data_pattern = FLAGS.input_data_pattern
model = find_class_by_name(FLAGS.model, [ensemble_level_models])()
label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])()
if FLAGS.input_data_patterns is "":
raise IOError("'input_data_patterns' was not specified. " +
"Nothing to evaluate.")
build_graph(
all_readers=all_readers,
input_reader=input_reader,
all_data_patterns=all_patterns,
input_data_pattern=input_data_pattern,
model=model,
label_loss_fn=label_loss_fn,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_batch = tf.get_collection("video_id_batch")[0]
prediction_batch = tf.get_collection("predictions")[0]
label_batch = tf.get_collection("labels")[0]
saver = tf.train.Saver(tf.global_variables())
inference_loop(video_id_batch, prediction_batch,
label_batch, saver,
FLAGS.output_dir, FLAGS.batch_size)
def check_video_id():
tf.set_random_seed(0) # for reproducibility
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.feature_names, FLAGS.feature_sizes)
# prepare a reader for each single model prediction result
all_readers = []
all_patterns = FLAGS.eval_data_patterns
all_patterns = map(lambda x: x.strip(),
all_patterns.strip().strip(",").split(","))
for i in xrange(len(all_patterns)):
reader = readers.EnsembleReader(feature_names=feature_names,
feature_sizes=feature_sizes)
all_readers.append(reader)
input_reader = None
input_data_pattern = None
if FLAGS.input_data_pattern is not None:
input_reader = readers.EnsembleReader(
feature_names=["mean_rgb", "mean_audio"],
feature_sizes=[1024, 128])
# input_reader = readers.EnsembleReader(
# feature_names=["input"], feature_sizes=[1024+128])
input_data_pattern = FLAGS.input_data_pattern
if FLAGS.eval_data_patterns is "":
raise IOError("'eval_data_patterns' was not specified. " +
"Nothing to evaluate.")
build_graph(all_readers=all_readers,
input_reader=input_reader,
input_data_pattern=input_data_pattern,
all_eval_data_patterns=all_patterns,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_mismatch = tf.get_collection("video_id_mismatch")[0]
input_distance = tf.get_collection("input_distance")[0]
actual_batch_size = tf.get_collection("actual_batch_size")[0]
check_loop(video_id_mismatch, input_distance, actual_batch_size,
all_patterns)
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
if FLAGS.input_data_pattern is "":
raise IOError("'input_data_pattern' was not specified. " +
"Nothing to evaluate.")
if FLAGS.prediction_data_pattern is "":
raise IOError("'prediction_data_pattern' was not specified. " +
"Nothing to evaluate.")
# convert feature_names and feature_sizes to lists of values
input_feature_names, input_feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.input_feature_names, FLAGS.input_feature_sizes)
input_reader = readers.EnsembleReader(
feature_names=input_feature_names,
feature_sizes=input_feature_sizes)
prediction_feature_names, prediction_feature_sizes = utils.GetListOfFeatureNamesAndSizes(
FLAGS.prediction_feature_names, FLAGS.prediction_feature_sizes)
prediction_reader = readers.EnsembleReader(
feature_names=prediction_feature_names,
feature_sizes=prediction_feature_sizes)
build_graph(input_reader=input_reader,
prediction_reader=prediction_reader,
input_data_pattern=FLAGS.input_data_pattern,
prediction_data_pattern=FLAGS.prediction_data_pattern,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_ids_equal = tf.get_collection("video_ids_equal")[0]
labels_equal = tf.get_collection("labels_equal")[0]
video_ids_batch = tf.get_collection("video_ids_batch")[0]
labels_batch = tf.get_collection("labels_batch")[0]
inputs_batch = tf.get_collection("inputs_batch")[0]
predictions_batch = tf.get_collection("predictions_batch")[0]
inference_loop(video_ids_batch, labels_batch, inputs_batch,
predictions_batch, video_ids_equal, labels_equal,
FLAGS.output_dir, FLAGS.batch_size)
def main(unused_argv):
logging.set_verbosity(tf.logging.INFO)
if FLAGS.input_model_tgz:
if FLAGS.train_dir:
raise ValueError("You cannot supply --train_dir if supplying "
"--input_model_tgz")
# Untar.
if not os.path.exists(FLAGS.untar_model_dir):
os.makedirs(FLAGS.untar_model_dir)
tarfile.open(FLAGS.input_model_tgz).extractall(FLAGS.untar_model_dir)
FLAGS.train_dir = FLAGS.untar_model_dir
flags_dict_file = os.path.join(FLAGS.train_dir, "model_flags.json")
if not os.path.exists(flags_dict_file):
raise IOError("Cannot find %s. Did you run eval.py?" % flags_dict_file)
flags_dict = json.loads(open(flags_dict_file).read())
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(feature_names=feature_names,
feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
all_data_patterns = []
with open(FLAGS.data_config) as f:
all_data_patterns = f.read().splitlines()
all_readers = []
for i in xrange(len(all_data_patterns)):
reader = readers.EnsembleReader(feature_names=["predictions"],
feature_sizes=["3862"])
all_readers.append(reader)
# if FLAGS.output_file is "":
# raise ValueError("'output_file' was not specified. "
# "Unable to continue with inference.")
# if FLAGS.input_data_pattern is "":
# raise ValueError("'input_data_pattern' was not specified. "
# "Unable to continue with inference.")
inference(all_readers, FLAGS.train_dir, all_data_patterns,
FLAGS.output_file, FLAGS.batch_size, FLAGS.top_k)
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object, ), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.", task_as_string(task),
tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
model = find_class_by_name(FLAGS.model, [ensemble_model])()
all_data_patterns = []
with open(FLAGS.data_config) as f:
all_data_patterns = f.read().splitlines()
all_readers = []
for i in xrange(len(all_data_patterns)):
reader = readers.EnsembleReader(
feature_names=[FLAGS.feature_names],
feature_sizes=[FLAGS.feature_sizes])
all_readers.append(reader)
# model_exporter = export_model.ModelExporter(
# frame_features=FLAGS.frame_features,
# model=model,
# all_readers=all_readers)
Trainer(cluster, task, FLAGS.train_dir, model, all_readers, None,
FLAGS.log_device_placement, FLAGS.max_steps,
FLAGS.export_model_steps).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
def evaluate():
tf.set_random_seed(0) # for reproducibility
# Write json of flags
model_flags_path = os.path.join(FLAGS.train_dir, "model_flags.json")
if not file_io.file_exists(model_flags_path):
raise IOError(("Cannot find file %s. Did you run train.py on the same "
"--train_dir?") % model_flags_path)
flags_dict = json.loads(file_io.FileIO(model_flags_path, mode="r").read())
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
flags_dict["feature_names"], flags_dict["feature_sizes"])
if flags_dict["frame_features"]:
reader = readers.YT8MFrameFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
else:
reader = readers.YT8MAggregatedFeatureReader(
feature_names=feature_names, feature_sizes=feature_sizes)
all_eval_data_patterns = []
with open(FLAGS.eval_data_config) as f:
all_eval_data_patterns = f.read().splitlines()
all_readers = []
for i in xrange(len(all_eval_data_patterns)):
reader = readers.EnsembleReader(feature_names=["predictions"],
feature_sizes=["3862"])
all_readers.append(reader)
model = find_class_by_name(flags_dict["model"], [ensemble_model])()
label_loss_fn = find_class_by_name(flags_dict["label_loss"],
[losses])()
# if FLAGS.eval_data_pattern is "":
# raise IOError("'eval_data_pattern' was not specified. " +
# "Nothing to evaluate.")
build_graph(all_readers=all_readers,
all_eval_data_patterns=all_eval_data_patterns,
model=model,
label_loss_fn=label_loss_fn,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_batch = tf.get_collection("video_id_batch")[0]
prediction_batch = tf.get_collection("predictions")[0]
label_batch = tf.get_collection("labels")[0]
loss = tf.get_collection("loss")[0]
summary_op = tf.get_collection("summary_op")[0]
saver = tf.train.Saver(tf.global_variables())
summary_writer = tf.summary.FileWriter(FLAGS.train_dir,
graph=tf.get_default_graph())
evl_metrics = eval_util.EvaluationMetrics(reader.num_classes,
FLAGS.top_k)
last_global_step_val = -1
while True:
last_global_step_val = evaluation_loop(video_id_batch,
prediction_batch,
label_batch, loss,
summary_op, saver,
summary_writer, evl_metrics,
last_global_step_val)
if FLAGS.run_once:
break
def evaluate():
tf.set_random_seed(0) # for reproducibility
# Write json of flags
# model_flags_path = os.path.join(FLAGS.train_dir, "model_flags.json")
# if not file_io.file_exists(model_flags_path):
# raise IOError(("Cannot find file %s. Did you run train.py on the same "
# "--train_dir?") % model_flags_path)
# flags_dict = json.loads(file_io.FileIO(model_flags_path, mode="r").read())
all_eval_data_patterns = []
with open(FLAGS.eval_data_config) as f:
all_eval_data_patterns = f.read().splitlines()
with tf.Graph().as_default():
# convert feature_names and feature_sizes to lists of values
# feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes(
# flags_dict["feature_names"], flags_dict["feature_sizes"])
# prepare a reader for each single model prediction result
all_readers = []
for i in xrange(len(all_eval_data_patterns)):
reader = readers.EnsembleReader(
feature_names=[FLAGS.feature_names], feature_sizes=[FLAGS.feature_sizes])
all_readers.append(reader)
input_reader = None
input_data_pattern = None
# model = find_class_by_name(flags_dict["model"],
# [frame_level_models, video_level_models])()
model = ensemble_model.MeanModel()
label_loss_fn = find_class_by_name("CrossEntropyLoss", [losses])()
build_graph(
all_readers=all_readers,
all_eval_data_patterns = all_eval_data_patterns,
model=model,
label_loss_fn=label_loss_fn,
num_readers=FLAGS.num_readers,
batch_size=FLAGS.batch_size)
logging.info("built evaluation graph")
video_id_batch = tf.get_collection("video_id_batch")[0]
prediction_batch = tf.get_collection("predictions")[0]
label_batch = tf.get_collection("labels")[0]
loss = tf.get_collection("loss")[0]
summary_op = tf.get_collection("summary_op")[0]
saver = tf.train.Saver(tf.global_variables())
summary_writer = tf.summary.FileWriter(
FLAGS.train_dir, graph=tf.get_default_graph())
evl_metrics = eval_util.EvaluationMetrics(reader.num_classes, FLAGS.top_k)
last_global_step_val = -1
while True:
last_global_step_val = evaluation_loop(video_id_batch, prediction_batch,
label_batch, loss, summary_op,
saver, summary_writer, evl_metrics,
last_global_step_val)
if FLAGS.run_once:
break