def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object, ), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.", task_as_string(task),
tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
model = find_class_by_name(FLAGS.model, [mnist_models])()
reader = get_reader()
model_exporter = export_model.ModelExporter(model=model, reader=reader)
Trainer(cluster, task, FLAGS.train_dir, model, reader, model_exporter,
FLAGS.log_device_placement, FLAGS.max_steps,
FLAGS.export_model_steps).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object, ), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.", task_as_string(task),
tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
model = frame_level_models.NetVLADModelLF()
model_lst = [model]
if FLAGS.ensemble_num > 1:
for ensemble_idx in range(1, FLAGS.ensemble_num):
model2 = frame_level_models.NetVLADModelLF()
model_lst.append(model2)
reader = get_reader()
model_exporter = export_model.ModelExporter(
frame_features=FLAGS.frame_features,
model=model,
reader=reader,
cluster_size=FLAGS.netvlad_cluster_size
if type(FLAGS.netvlad_cluster_size) is int else
FLAGS.netvlad_cluster_size[0],
hidden_size=FLAGS.netvlad_hidden_size
if type(FLAGS.netvlad_hidden_size) is int else
FLAGS.netvlad_hidden_size[0])
Trainer(cluster, task, FLAGS.train_dir, model_lst, reader,
model_exporter, FLAGS.log_device_placement, FLAGS.max_steps,
FLAGS.export_model_steps).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object, ), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.", task_as_string(task),
tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
reader = get_reader()
model_exporter = export_model.ModelExporter(
frame_features=FLAGS.frame_features, model=model, reader=reader)
mat_dir = '/home/weimin/yt8m/code/youtube-8m/'
with open(mat_dir + 'rule1_mat.npz', 'rb') as f:
rule1_mat = np.load(f)
with open(mat_dir + 'rule2_mat.npz', 'rb') as f:
rule2_mat = np.load(f)
print("\n\nLoaded!\n\n")
Trainer(cluster, task, FLAGS.train_dir, model, reader, model_exporter,
FLAGS.log_device_placement, FLAGS.max_steps,
FLAGS.export_model_steps, rule1_mat,
rule2_mat).run(start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
def main(unused_argv):
# Safety first
if FLAGS.ema_source:
assert not os.path.isdir(FLAGS.train_dir), "For your safety create a new model."
assert os.path.isfile(FLAGS.ema_source + ".meta"), "Unkown EMA seed model."
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object,), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.",
task_as_string(task), tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
reader = get_reader()
model_exporter = export_model.ModelExporter(
frame_features=FLAGS.frame_features,
model=model,
reader=reader)
Trainer(cluster, task, FLAGS.train_dir, model, reader, model_exporter,
FLAGS.log_device_placement, FLAGS.max_steps,
FLAGS.export_model_steps).run(start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
#Store flags in file
flag_lines = []
flag_lines.append("START FLAGS =====================================")
flag_lines.append("train_dir: " + str(FLAGS.train_dir))
flag_lines.append("train_data_pattern: " + str(FLAGS.train_data_pattern))
flag_lines.append("train_data_pattern2: " + str(FLAGS.train_data_pattern2))
flag_lines.append("train_data_pattern3: " + str(FLAGS.train_data_pattern3))
flag_lines.append("eval_data_pattern: " + str(FLAGS.eval_data_pattern))
flag_lines.append("feature_names: " + str(FLAGS.feature_names))
flag_lines.append("feature_sizes: " + str(FLAGS.feature_sizes))
flag_lines.append("frame_features: " + str(FLAGS.frame_features))
flag_lines.append("model: " + str(FLAGS.model))
flag_lines.append("start_new_model: " + str(FLAGS.start_new_model))
flag_lines.append("l2_penalty: " + str(FLAGS.l2_penalty))
flag_lines.append("c_vars: " + str(FLAGS.c_vars))
flag_lines.append("r_vars: " + str(FLAGS.r_vars))
flag_lines.append("loss_epsilon: " + str(FLAGS.loss_epsilon))
flag_lines.append("batch_size: " + str(FLAGS.batch_size))
flag_lines.append("label_loss: " + str(FLAGS.label_loss))
flag_lines.append("regularization_penalty: " +
str(FLAGS.regularization_penalty))
flag_lines.append("base_learning_rate: " + str(FLAGS.base_learning_rate))
flag_lines.append("learning_rate_decay: " + str(FLAGS.learning_rate_decay))
flag_lines.append("learning_rate_decay_examples: " +
str(FLAGS.learning_rate_decay_examples))
flag_lines.append("num_epochs: " + str(FLAGS.num_epochs))
flag_lines.append("max_steps: " + str(FLAGS.max_steps))
flag_lines.append("export_model_steps: " + str(FLAGS.export_model_steps))
flag_lines.append("save_model_minutes: " + str(FLAGS.save_model_minutes))
flag_lines.append("gpu_only: " + str(FLAGS.gpu_only))
flag_lines.append("num_readers: " + str(FLAGS.num_readers))
flag_lines.append("optimizer: " + str(FLAGS.optimizer))
flag_lines.append("clip_gradient_norm: " + str(FLAGS.clip_gradient_norm))
flag_lines.append("log_device_placement: " +
str(FLAGS.log_device_placement))
flag_lines.append("apply_global_normalization: " +
str(FLAGS.apply_global_normalization))
flag_lines.append("apply_batch_l2_normalization: " +
str(FLAGS.apply_batch_l2_normalization))
flag_lines.append("restart_learning_rate: " +
str(FLAGS.restart_learning_rate))
flag_lines.append("model_export_gpu_only: " +
str(FLAGS.model_export_gpu_only))
flag_lines.append("layers_keep_probs: " + str(FLAGS.layers_keep_probs))
flag_lines.append("truncated_num_classes: " +
str(FLAGS.truncated_num_classes))
flag_lines.append("decode_zlib: " + str(FLAGS.decode_zlib))
flag_lines.append("fold: " + str(FLAGS.fold))
flag_lines.append("ema_halflife: " + str(FLAGS.ema_halflife))
flag_lines.append("use_ema: " + str(FLAGS.use_ema))
flag_lines.append("END FLAGS =======================================")
if not os.path.exists(FLAGS.train_dir):
os.makedirs(FLAGS.train_dir)
with open(os.path.join(FLAGS.train_dir, 'flags.cfg'), 'a') as ff:
for line in flag_lines:
ff.write(line + '\n')
print(line)
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object, ), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.", task_as_string(task),
tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
model = find_class_by_name(FLAGS.model, [
frame_level_models, video_level_models, xp_frame_level_models,
xp_video_level_models
])()
reader = get_reader()
# Exporter only needs layer_keep_probs for predictions, so those should always be 1.0
model_exporter = export_model.ModelExporter(
frame_features=FLAGS.frame_features,
model=model,
reader=reader,
gpu_only=FLAGS.model_export_gpu_only)
Trainer(cluster, task, FLAGS.train_dir, model, reader, model_exporter,
FLAGS.log_device_placement, FLAGS.max_steps,
FLAGS.export_model_steps,
FLAGS.gpu_only).run(start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object, ), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.", task_as_string(task),
tf.__version__)
# if os.path.exists(FLAGS.train_dir):
# shutil.rmtree(FLAGS.train_dir)
# subdirs = [x for x in os.listdir(FLAGS.Ensemble_Models) if os.path.isdir(x)]
# subdirs = [os.path.join(FLAGS.Ensemble_Models, x) for x in FLAGS.model_path.split(',')]
subdirs = os.listdir(FLAGS.Ensemble_Models)
flags_dict = []
save_flag = True
for subdir in subdirs:
model_flags_path = os.path.join(
os.path.join(FLAGS.Ensemble_Models, subdir), "model_flags.json")
print "Load model from " + model_flags_path + "\n"
flags_dict.append(
json.loads(file_io.FileIO(model_flags_path, mode="r").read()))
# save model_flags.json to inference dictory
if save_flag:
train_dir = os.path.join(FLAGS.Ensemble_Models, "train_log")
train_dir = train_dir + datetime.strftime(datetime.now(),
"%m%d%H%M")
if os.path.exists(train_dir) == False:
os.mkdir(train_dir)
shutil.copyfile(model_flags_path,
os.path.join(train_dir, "model_flags.json"))
save_flag = False
models = []
model_nums = len(subdirs)
for m in range(model_nums):
model = get_params(flags_dict[m])
models.append(model)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
# model = find_class_by_name(FLAGS.model,
# [frame_level_models, video_level_models])()
reader = get_reader()
model_exporter = export_model.ModelExporter(
frame_features=FLAGS.frame_features,
model=models[1], # anyone is OK
reader=reader)
# model_exporter=None
with tf.Graph().as_default() as graph:
Trainer(cluster, task, train_dir, subdirs, models, reader, graph,
model_exporter, FLAGS.log_device_placement,
FLAGS.max_steps, FLAGS.export_model_steps).run(
start_new_model=FLAGS.start_new_model)
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))
def main(unused_argv):
# Load the environment.
env = json.loads(os.environ.get("TF_CONFIG", "{}"))
# Load the cluster data from the environment.
cluster_data = env.get("cluster", None)
cluster = tf.train.ClusterSpec(cluster_data) if cluster_data else None
# Load the task data from the environment.
task_data = env.get("task", None) or {"type": "master", "index": 0}
task = type("TaskSpec", (object, ), task_data)
# Logging the version.
logging.set_verbosity(tf.logging.INFO)
logging.info("%s: Tensorflow version: %s.", task_as_string(task),
tf.__version__)
# Dispatch to a master, a worker, or a parameter server.
if not cluster or task.type == "master" or task.type == "worker":
model = find_class_by_name(FLAGS.model,
[frame_level_models, video_level_models])()
reader = get_reader()
model_exporter = export_model.ModelExporter(
frame_features=FLAGS.frame_features, model=model, reader=reader)
b = os.listdir(r'E:/ucf50_train/25grouptf')
c = []
for row in b:
file_path = os.path.join('E:/ucf50_train/25grouptf', row)
c.append(file_path)
sum_accuracy = 0
c = np.array(c)
kf = KFold(n_splits=25)
for train_index, test_index in kf.split(c):
X_train, X_test = c[train_index], c[test_index]
Trainer(cluster, task, FLAGS.train_dir, X_train, model, reader,
model_exporter, FLAGS.log_device_placement,
FLAGS.max_steps, FLAGS.export_model_steps).run(
start_new_model=FLAGS.start_new_model)
accuracy_performance = inference(reader, FLAGS.train_dir, X_test,
FLAGS.batch_size, FLAGS.top_k)
sum_accuracy += accuracy_performance
f = open("prediction_accuracy_vlad.txt", 'a')
f.write(str(accuracy_performance))
f.write('\n')
print(
'---------------------------accuracy:{0:.3f}----------------------------------------'
.format(accuracy_performance))
total_accuracy = sum_accuracy / 25
print(
'---------------------------accuracy_final:{0:.3f}----------------------------------------'
.format(total_accuracy))
elif task.type == "ps":
ParameterServer(cluster, task).run()
else:
raise ValueError("%s: Invalid task_type: %s." %
(task_as_string(task), task.type))