def main(unused_argv):
print("Using TensorFlow version %s" % tf.__version__)
version = tf.__version__.split(".")
assert 1 == int(version[0]) and 14 <= int(version[1]), "Need TensorFlow r1.14 or Later."
if FLAGS.data_dir is None:
raise ValueError("Must specify prediction data_file by --data_dir")
print('Model type: {}'.format(FLAGS.model_type))
model_dir = os.path.join(FLAGS.model_dir, FLAGS.model_type)
print('Model directory: {}'.format(model_dir))
# model = build_estimator(model_dir, FLAGS.model_type)
model = build_custom_estimator(model_dir, FLAGS.model_type)
tf.logging.info('Build estimator: {}'.format(model))
tf.logging.info('='*30+'START PREDICTION'+'='*30)
t0 = time.time()
predictions = model.predict(input_fn=lambda: input_fn(FLAGS.data_dir, FLAGS.image_data_dir, 'pred', FLAGS.batch_size),
predict_keys=None,
hooks=None,
checkpoint_path=FLAGS.checkpoint_path) # defaults None to use latest_checkpoint
tf.logging.info('='*30+'FINISH PREDICTION, TAKE {} mins'.format(elapse_time(t0))+'='*30)
for pred_dict in predictions: # dict{probabilities, classes, class_ids}
class_id = pred_dict['class_ids'][0]
probability = pred_dict['probabilities'][class_id]
print('\nPrediction is "{}" ({:.1f}%)'.format(class_id, 100 * probability))
def main(unused_argv):
print("Using TensorFlow version %s" % tf.__version__)
assert "1.4" <= tf.__version__, "TensorFlow r1.4 or later is needed"
if FLAGS.data_dir is None:
raise ValueError("Must specify prediction data_file by --data_dir")
print('Model type: {}'.format(FLAGS.model_type))
model_dir = os.path.join(FLAGS.model_dir, FLAGS.model_type)
print('Model directory: {}'.format(model_dir))
model = build_custom_estimator(model_dir, FLAGS.model_type)
tf.logging.info('Build estimator: {}'.format(model))
tf.logging.info('='*30+'START PREDICTION'+'='*30)
t0 = time.time()
predictions = model.predict(input_fn=lambda: input_fn(FLAGS.data_dir, 'pred', FLAGS.batch_size),
predict_keys=None,
hooks=None,
checkpoint_path=FLAGS.checkpoint_path) # defaults None to use latest_checkpoint
tf.logging.info('='*30+'FINISH PREDICTION, TAKE {} mins'.format(elapse_time(t0))+'='*30)
class_list = []
for pred_dict in predictions: # dict{probabilities, classes, class_ids}
# class_id = pred_dict['class_ids'][0]
# probability = pred_dict['probabilities'][class_id]
# class_list.append([class_id,probability])
# pd.DataFrame(class_list,columns=['class_ids','probabilities']).to_csv('final_prob_train.csv')
prob_list = pred_dict['probabilities']
max_prob_list = [(i,x) for i,x in enumerate(prob_list)] #if x>sorted(prob_list)[-4]
class_list.append(str(max_prob_list))
pd.DataFrame(class_list,columns=['prob_dict']).to_csv('final_prob.csv')
def main(unused_argv):
print("Using TensorFlow version %s" % tf.__version__)
# assert "1.4" <= tf.__version__, "TensorFlow r1.4 or later is needed"
# if FLAGS.is_distribution:
# print("Using distribution tensoflow. Job_name:{} Task_index:{}"
# .format(CONFIG.distribution["job_name"], CONFIG.distribution["task_index"]))
print('Model type: {}'.format(FLAGS.model_type))
model_dir = os.path.join(FLAGS.model_dir, FLAGS.model_type)
print('Model directory: {}'.format(model_dir))
model = build_custom_estimator(model_dir, FLAGS.model_type)
tf.compat.v1.logging.info('Build estimator: {}'.format(model))
# checkpoint_path = FLAGS.checkpoint_path or model.latest_checkpoint()
# if checkpoint_path is None:
# raise ValueError('No model checkpoint found, please check the model dir.')
# tf.logging.info('Using model checkpoint: {}'.format(checkpoint_path))
# print('\n')
tf.compat.v1.logging.info('='*30+' START TESTING'+'='*30)
s_time = time.time()
results = model.evaluate(input_fn=lambda: input_fn(FLAGS.test_data, FLAGS.image_test_data, 'eval', FLAGS.batch_size),
steps=None, # Number of steps for which to evaluate model.
hooks=None,
checkpoint_path=FLAGS.checkpoint_path, # If None, the latest checkpoint is used.
name=None)
tf.compat.v1.logging.info('='*30+'FINISH TESTING, TAKE {}'.format(elapse_time(s_time))+'='*30)
# predictions = model.predict(
# input_fn=lambda: input_fn(FLAGS.data_dir, FLAGS.image_data_dir, 'pred', FLAGS.batch_size),
# predict_keys=None,
# hooks=None,
# checkpoint_path=FLAGS.checkpoint_path)
# Display evaluation metrics
print('-' * 80)
for key in sorted(results):
print('%s: %s' % (key, results[key]))
def main(unused_argv):
print("Using TensorFlow version %s" % tf.__version__)
# assert "1.4" <= tf.__version__, "TensorFlow r1.4 or later is needed"
if FLAGS.data_dir is None:
raise ValueError("Must specify prediction data_file by --data_dir")
print('Model type: {}'.format(FLAGS.model_type))
model_dir = os.path.join(FLAGS.model_dir, FLAGS.model_type)
print('Model directory: {}'.format(model_dir))
# model = build_estimator(model_dir, FLAGS.model_type)
model = build_custom_estimator(model_dir, FLAGS.model_type)
tf.compat.v1.logging.info('Build estimator: {}'.format(model))
# weights and other parameters (e.g. Adagrad) of the model
name_ls = model.get_variable_names()
print_shape = True
total_linear_weights = 0
for name in name_ls:
if print_shape:
shape = model.get_variable_value(name).shape
print(name, "\t", shape)
if name[:6] == "linear" and \
(name[-7:] == "weights"or name[-4:] == "bias"):
total_linear_weights += np.prod(shape)
else:
print(name)
if print_shape:
print("Total parameters in linear model: {}".format(
total_linear_weights))
# embedding layer look up
sample_embedding = model.get_variable_value(
'dnn/input_from_feature_columns/input_layer/ad_cates_embedding/embedding_weights'
)
ids = [10, 20, 30]
with tf.compat.v1.Session() as sess:
lookup = tf.nn.embedding_lookup(params=sample_embedding,
ids=ids).eval()
print(lookup)
# predictions
tf.compat.v1.logging.info('=' * 30 + 'START PREDICTION' + '=' * 30)
t0 = time.time()
predictions = model.predict(input_fn=lambda: input_fn(
FLAGS.data_dir, FLAGS.image_data_dir, 'pred', FLAGS.batch_size),
predict_keys=None,
hooks=None,
checkpoint_path=FLAGS.checkpoint_path
) # defaults None to use latest_checkpoint
for pred_dict in predictions: # dict{probabilities, classes, class_ids}
class_id = pred_dict['class_ids'][0]
probability = pred_dict['probabilities'][class_id]
print('\nPrediction is "{}" ({:.1f}%)'.format(class_id,
100 * probability))
tf.compat.v1.logging.info(
'=' * 30 + 'FINISH PREDICTION, TAKE {} mins'.format(elapse_time(t0)) +
'=' * 30)
def main(unused_argv):
print("Using TensorFlow version %s" % tf.__version__)
assert "1.4" <= tf.__version__, "Need TensorFlow r1.4 or later."
print("Using Train config: {}".format(CONFIG.train))
print('Model type: {}'.format(FLAGS.model_type))
model_dir = os.path.join(FLAGS.model_dir, FLAGS.model_type)
print('Model directory: {}'.format(model_dir))
if not FLAGS.keep_train:
# Clean up the model directory if not keep training
shutil.rmtree(model_dir, ignore_errors=True)
print('Remove model directory: {}'.format(model_dir))
# model = build_estimator(model_dir, FLAGS.model_type)
model = build_custom_estimator(model_dir, FLAGS.model_type)
tf.logging.info('Build estimator: {}'.format(model))
if CONFIG.distribution["is_distribution"]:
print("Using PID: {}".format(os.getpid()))
cluster = CONFIG.distribution["cluster"]
job_name = CONFIG.distribution["job_name"]
task_index = CONFIG.distribution["task_index"]
print("Using Distributed TensorFlow. Local host: {} Job_name: {} Task_index: {}"
.format(cluster[job_name][task_index], job_name, task_index))
cluster = tf.train.ClusterSpec(CONFIG.distribution["cluster"])
server = tf.train.Server(cluster,
job_name=job_name,
task_index=task_index)
if job_name == 'ps':
# wait for incoming connection forever
server.join()
# sess = tf.Session(server.target)
# queue = create_done_queue(task_index, num_workers)
# for i in range(num_workers):
# sess.run(queue.dequeue())
# print("ps {} received worker {} done".format(task_index, i)
# print("ps {} quitting".format(task_index))
else: # TODO：supervisor & MonotoredTrainingSession & experiment (deprecated)
train(model)
# train_and_eval(model)
# Each worker only needs to contact the PS task(s) and the local worker task.
# config = tf.ConfigProto(device_filters=[
# '/job:ps', '/job:worker/task:%d' % arguments.task_index])
# with tf.device(tf.train.replica_device_setter(
# worker_device="/job:worker/task:%d" % task_index,
# cluster=cluster)):
# e = _create_experiment_fn()
# e.train_and_evaluate() # call estimator's train() and evaluate() method
# hooks = [tf.train.StopAtStepHook(last_step=10000)]
# with tf.train.MonitoredTrainingSession(
# master=server.target,
# is_chief=(task_index == 0),
# checkpoint_dir=args.model_dir,
# hooks=hooks) as mon_sess:
# while not mon_sess.should_stop():
# # mon_sess.run()
# classifier.fit(input_fn=train_input_fn, steps=1)
else:
# local run
train(model)
def main(_):
if FLAGS.model_version <= 0:
print('Please specify a positive value for version number.')
sys.exit(-1)
# tf.estimator.export.build_parsing_serving_input_receiver_fn
# tf.estimator.export.build_raw_serving_input_receiver_fn
# If these utilities do not meet your needs, you are free to write your own serving_input_receiver_fn()
# feature_spec = {
# "some_feature": tf.FixedLenFeature([], dtype=tf.string, default_value=""),
# "some_feature": tf.VarLenFeature(dtype=tf.string),
# }
#
# def _serving_input_receiver_fn():
# serialized_tf_example = tf.placeholder(dtype=tf.string, shape=None,
# name='input_example_tensor')
# # key (e.g. 'examples') should be same with the inputKey when you
# # buid the request for prediction
# receiver_tensors = {'examples': serialized_tf_example}
# features = tf.parse_example(serialized_tf_example, your_feature_spec)
# return tf.estimator.export.ServingInputReceiver(features, receiver_tensors)
# estimator.export_savedmodel(export_dir, _serving_input_receiver_fn)
wide_columns, deep_columns = _build_model_columns()
feature_columns = wide_columns + deep_columns
# for f in feature_columns:
# print(f._parse_example_spec)
# A dict mapping each feature key to a FixedLenFeature or VarLenFeature value.
feature_spec = tf.feature_column.make_parse_example_spec(feature_columns)
serving_input_receiver_fn = \
tf.estimator.export.build_parsing_serving_input_receiver_fn(feature_spec)
model_dir = os.path.join(model_base_dir, FLAGS.model_type)
export_dir = os.path.join(FLAGS.export_dir, FLAGS.model_type)
print("building custom estimator")
model = build_custom_estimator(model_dir, FLAGS.model_type)
print("exporting saved model")
model.export_savedmodel(export_dir,
serving_input_receiver_fn,
as_text=CONF['as_text'],
checkpoint_path=FLAGS.checkpoint_path)
def main(unused_argv):
CONFIG = Config()
print("Using TensorFlow Version %s" % tf.__version__)
assert "1.4" <= tf.__version__, "Need TensorFlow r1.4 or Later."
print('\nModel Type: {}'.format(FLAGS.model_type))
model_dir = os.path.join(FLAGS.model_dir, FLAGS.model_type)
print('\nModel Directory: {}'.format(model_dir))
print("\nUsing Train Config:")
for k, v in CONFIG.train.items():
print('{}: {}'.format(k, v))
print("\nUsing Model Config:")
for k, v in CONFIG.model.items():
print('{}: {}'.format(k, v))
if not FLAGS.keep_train:
# Clean up the model directory if not keep training
shutil.rmtree(model_dir, ignore_errors=True)
print('Remove model directory: {}'.format(model_dir))
model = build_custom_estimator(model_dir, FLAGS.model_type)
tf.logging.info('Build estimator: {}'.format(model))
train_and_eval_api(model)
