def build_and_test_estimator(self, model_type):
"""Ensure that model trains and minimizes loss."""
model = build_estimator(self.temp_dir, model_type)
# Train for 1 step to initialize model and evaluate initial loss
model.train(
input_fn=lambda: input_fn(TEST_CSV, None, 'eval', batch_size=1),
steps=1)
initial_results = model.evaluate(
input_fn=lambda: input_fn(TEST_CSV, None, 'eval', batch_size=1))
# Train for 100 epochs at batch size 3 and evaluate final loss
model.train(
input_fn=lambda: input_fn(TEST_CSV, None, 'eval', batch_size=8))
final_results = model.evaluate(
input_fn=lambda: input_fn(TEST_CSV, None, 'eval', batch_size=1))
logging.info('\n%s initial results: %s', model_type, initial_results)
logging.info('\n%s final results: %s', model_type, final_results)
# Ensure loss has decreased, while accuracy and both AUCs have increased.
self.assertLess(final_results['loss'], initial_results['loss'])
self.assertGreaterEqual(final_results['auc'], initial_results['auc'])
self.assertGreaterEqual(final_results['auc_precision_recall'],
initial_results['auc_precision_recall'])
self.assertGreaterEqual(final_results['accuracy'],
initial_results['accuracy'])
def main(unused_argv):
print("Using TensorFlow version %s" % tf.__version__)
assert "1.4" <= tf.__version__, "TensorFlow r1.4 or later is needed"
# if FLAGS.is_distribution:
# print("Using distribution tensoflow. Job_name:{} Task_index:{}"
# .format(CONFIG.distribution["job_name"], CONFIG.distribution["task_index"]))
print('Model type: {}'.format(FLAGS.model_type))
model_dir = os.path.join(FLAGS.model_dir, FLAGS.model_type)
print('Model directory: {}'.format(model_dir))
model = build_estimator(model_dir, FLAGS.model_type)
tf.logging.info('Build estimator: {}'.format(model))
checkpoint_path = FLAGS.checkpoint_path or model.latest_checkpoint()
if checkpoint_path is None:
raise ValueError(
'No model checkpoint found, please check the model dir.')
tf.logging.info('Using model checkpoint: {}'.format(checkpoint_path))
print('\n')
tf.logging.info('=' * 30 + ' START TESTING' + '=' * 30)
s_time = time.time()
results = model.evaluate(
input_fn=lambda: input_fn(FLAGS.test_data, FLAGS.image_test_data,
'eval', FLAGS.batch_size),
steps=None, # Number of steps for which to evaluate model.
hooks=None,
checkpoint_path=FLAGS.
checkpoint_path, # If None, the latest checkpoint is used.
name=None)
tf.logging.info('=' * 30 +
'FINISH TESTING, TAKE {}'.format(elapse_time(s_time)) +
'=' * 30)
# Display evaluation metrics
print('-' * 80)
for key in sorted(results):
print('%s: %s' % (key, results[key]))
def main(unused_argv):
print("Using TensorFlow version %s" % tf.__version__)
assert "1.4" <= tf.__version__, "TensorFlow r1.4 or later is needed"
if FLAGS.data_dir is None:
raise ValueError("Must specify prediction data_file by --data_dir")
print('Model type: {}'.format(FLAGS.model_type))
model_dir = os.path.join(FLAGS.model_dir, FLAGS.model_type)
print('Model directory: {}'.format(model_dir))
model = build_estimator(model_dir, FLAGS.model_type)
tf.logging.info('Build estimator: {}'.format(model))
tf.logging.info('=' * 30 + 'START PREDICTION' + '=' * 30)
t0 = time.time()
predictions = model.predict(input_fn=lambda: input_fn(
FLAGS.data_dir, FLAGS.image_data_dir, 'pred', FLAGS.batch_size),
predict_keys=None,
hooks=None,
checkpoint_path=FLAGS.checkpoint_path
) # defaults None to use latest_checkpoint
tf.logging.info('=' * 30 +
'FINISH PREDICTION, TAKE {} mins'.format(elapse_time(t0)) +
'=' * 30)
for pred_dict in predictions: # dict{probabilities, classes, class_ids}
class_id = pred_dict['class_ids'][0]
probability = pred_dict['probabilities'][class_id]
print('\nPrediction is "{}" ({:.1f}%)'.format(class_id,
100 * probability))
def main(unused_argv):
print("Using TensorFlow version %s" % tf.__version__)
assert "1.4" <= tf.__version__, "TensorFlow r1.4 or later is needed"
# if FLAGS.is_distribution:
# print("Using distribution tensoflow. Job_name:{} Task_index:{}"
# .format(CONFIG.distribution["job_name"], CONFIG.distribution["task_index"]))
# model info
print('Model type: {}'.format(FLAGS.model_type))
model_dir = os.path.join(FLAGS.model_dir, FLAGS.model_type)
print('Model directory: {}'.format(model_dir))
model = build_estimator(model_dir, FLAGS.model_type)
tf.logging.info('Build estimator: {}'.format(model))
checkpoint_path = FLAGS.checkpoint_path or model.latest_checkpoint()
if checkpoint_path is None:
raise ValueError('No model checkpoint found, please check the model dir.')
tf.logging.info('Using model checkpoint: {}'.format(checkpoint_path))
print('-' * 80)
tf.logging.info('='*30+' START PREDICTION'+'='*30)
t0 = time.time()
predictions = model.predict(input_fn=lambda: input_fn(FLAGS.data_dir, FLAGS.image_data_dir, 'pred', FLAGS.batch_size),
predict_keys=None,
hooks=None,
checkpoint_path=checkpoint_path) # defaults None to use latest_checkpoint
tf.logging.info('='*30+'FINISH PREDICTION, TAKE {} mins'.format(elapse_time(t0))+'='*30)
for pred_dict in predictions: # dict{probabilities, classes, class_ids}
class_id = pred_dict['class_ids'][0]
probability = pred_dict['probabilities'][class_id]
print('\nPrediction is "{}" ({:.1f}%)'.format(class_id, 100 * probability))
def main():
wide_columns, deep_columns = build_model_columns()
feature_columns = wide_columns + deep_columns
feature_spec = tf.feature_column.make_parse_example_spec(feature_columns)
print(feature_spec)
serving_input_receiver_fn = \
tf.estimator.export.build_parsing_serving_input_receiver_fn(feature_spec)
export_dir = EXPORT_MODEL
model = build_estimator()
model.export_savedmodel(export_dir,
serving_input_receiver_fn,
as_text=False)
def main():
CONFIG = Config()
model_conf = CONFIG.read_model_conf()['model_conf']
traindata_list = FileListGenerator(model_conf['data_dir_train']).generate()
testdata_list = FileListGenerator(model_conf['data_dir_pred']).generate()
model = build_estimator()
traindata = next(traindata_list)
testdata = next(testdata_list)
t0 = time.time()
tf.logging.info('Start training {}'.format(traindata))
model.train(input_fn=lambda: input_fn(traindata, 'train'),
hooks=None,
steps=None,
max_steps=None,
saving_listeners=None)
t1 = time.time()
tf.logging.info('Finish training {}, take {} mins'.format(
traindata, float((t1 - t0) / 60)))
tf.logging.info('Start evaluating {}'.format(testdata))
t2 = time.time()
results = model.evaluate(
input_fn=lambda: input_fn(testdata, 'eval'),
steps=None, # Number of steps for which to evaluate model.
hooks=None,
checkpoint_path=None, # latest checkpoint in model_dir is used.
name=None)
t3 = time.time()
tf.logging.info('Finish evaluation {}, take {} mins'.format(
testdata, float((t3 - t2) / 60)))
# Display evaluation metrics
for key in sorted(results):
print('{}: {}'.format(key, results[key]))
def main():
CONFIG = Config()
model_conf = CONFIG.read_model_conf()['model_conf']
model = build_estimator()
predictions = model.predict(input_fn=lambda: input_fn('/home/leadtek/zhangqifan/reflux_user_pro/data/pred_data/all_data.csv','pred'),
predict_keys=None,
hooks=None,
checkpoint_path=None) # defaults None to use latest_checkpoint
res = []
for pred_dict in predictions: # dict{probabilities, classes, class_ids}
opt = []
class_id = pred_dict['class_ids'][0]
opt.append(class_id)
probability = pred_dict['probabilities']
opt.append(probability[1])
res.append(opt)
# print('class_id:',class_id,'probability:',probability)
res_df = pd.DataFrame(res, columns=['class_id','probability'])
x = res_df[res_df['class_id'].isin([1])]
sample = pd.read_csv("/home/leadtek/zhangqifan/reflux_user_pro/data/opt_all_data.csv",sep=' ')
res_sample = pd.concat([sample,res_df],axis=1)
res_sample.to_csv(r"/home/leadtek/zhangqifan/reflux_user_pro/res.csv", header=True, index=False,
sep=' ')