def main(unused_argv): print("Using TensorFlow version %s" % tf.__version__) version = tf.__version__.split(".") assert 1 == int(version[0]) and 14 <= int(version[1]), "Need TensorFlow r1.14 or Later." if FLAGS.data_dir is None: raise ValueError("Must specify prediction data_file by --data_dir") print('Model type: {}'.format(FLAGS.model_type)) model_dir = os.path.join(FLAGS.model_dir, FLAGS.model_type) print('Model directory: {}'.format(model_dir)) # model = build_estimator(model_dir, FLAGS.model_type) model = build_custom_estimator(model_dir, FLAGS.model_type) tf.logging.info('Build estimator: {}'.format(model)) tf.logging.info('='*30+'START PREDICTION'+'='*30) t0 = time.time() predictions = model.predict(input_fn=lambda: input_fn(FLAGS.data_dir, FLAGS.image_data_dir, 'pred', FLAGS.batch_size), predict_keys=None, hooks=None, checkpoint_path=FLAGS.checkpoint_path) # defaults None to use latest_checkpoint tf.logging.info('='*30+'FINISH PREDICTION, TAKE {} mins'.format(elapse_time(t0))+'='*30) for pred_dict in predictions: # dict{probabilities, classes, class_ids} class_id = pred_dict['class_ids'][0] probability = pred_dict['probabilities'][class_id] print('\nPrediction is "{}" ({:.1f}%)'.format(class_id, 100 * probability))
def main(unused_argv): print("Using TensorFlow version %s" % tf.__version__) assert "1.4" <= tf.__version__, "TensorFlow r1.4 or later is needed" if FLAGS.data_dir is None: raise ValueError("Must specify prediction data_file by --data_dir") print('Model type: {}'.format(FLAGS.model_type)) model_dir = os.path.join(FLAGS.model_dir, FLAGS.model_type) print('Model directory: {}'.format(model_dir)) model = build_custom_estimator(model_dir, FLAGS.model_type) tf.logging.info('Build estimator: {}'.format(model)) tf.logging.info('='*30+'START PREDICTION'+'='*30) t0 = time.time() predictions = model.predict(input_fn=lambda: input_fn(FLAGS.data_dir, 'pred', FLAGS.batch_size), predict_keys=None, hooks=None, checkpoint_path=FLAGS.checkpoint_path) # defaults None to use latest_checkpoint tf.logging.info('='*30+'FINISH PREDICTION, TAKE {} mins'.format(elapse_time(t0))+'='*30) class_list = [] for pred_dict in predictions: # dict{probabilities, classes, class_ids} # class_id = pred_dict['class_ids'][0] # probability = pred_dict['probabilities'][class_id] # class_list.append([class_id,probability]) # pd.DataFrame(class_list,columns=['class_ids','probabilities']).to_csv('final_prob_train.csv') prob_list = pred_dict['probabilities'] max_prob_list = [(i,x) for i,x in enumerate(prob_list)] #if x>sorted(prob_list)[-4] class_list.append(str(max_prob_list)) pd.DataFrame(class_list,columns=['prob_dict']).to_csv('final_prob.csv')
def main(unused_argv): print("Using TensorFlow version %s" % tf.__version__) # assert "1.4" <= tf.__version__, "TensorFlow r1.4 or later is needed" # if FLAGS.is_distribution: # print("Using distribution tensoflow. Job_name:{} Task_index:{}" # .format(CONFIG.distribution["job_name"], CONFIG.distribution["task_index"])) print('Model type: {}'.format(FLAGS.model_type)) model_dir = os.path.join(FLAGS.model_dir, FLAGS.model_type) print('Model directory: {}'.format(model_dir)) model = build_custom_estimator(model_dir, FLAGS.model_type) tf.compat.v1.logging.info('Build estimator: {}'.format(model)) # checkpoint_path = FLAGS.checkpoint_path or model.latest_checkpoint() # if checkpoint_path is None: # raise ValueError('No model checkpoint found, please check the model dir.') # tf.logging.info('Using model checkpoint: {}'.format(checkpoint_path)) # print('\n') tf.compat.v1.logging.info('='*30+' START TESTING'+'='*30) s_time = time.time() results = model.evaluate(input_fn=lambda: input_fn(FLAGS.test_data, FLAGS.image_test_data, 'eval', FLAGS.batch_size), steps=None, # Number of steps for which to evaluate model. hooks=None, checkpoint_path=FLAGS.checkpoint_path, # If None, the latest checkpoint is used. name=None) tf.compat.v1.logging.info('='*30+'FINISH TESTING, TAKE {}'.format(elapse_time(s_time))+'='*30) # predictions = model.predict( # input_fn=lambda: input_fn(FLAGS.data_dir, FLAGS.image_data_dir, 'pred', FLAGS.batch_size), # predict_keys=None, # hooks=None, # checkpoint_path=FLAGS.checkpoint_path) # Display evaluation metrics print('-' * 80) for key in sorted(results): print('%s: %s' % (key, results[key]))
def main(unused_argv): print("Using TensorFlow version %s" % tf.__version__) # assert "1.4" <= tf.__version__, "TensorFlow r1.4 or later is needed" if FLAGS.data_dir is None: raise ValueError("Must specify prediction data_file by --data_dir") print('Model type: {}'.format(FLAGS.model_type)) model_dir = os.path.join(FLAGS.model_dir, FLAGS.model_type) print('Model directory: {}'.format(model_dir)) # model = build_estimator(model_dir, FLAGS.model_type) model = build_custom_estimator(model_dir, FLAGS.model_type) tf.compat.v1.logging.info('Build estimator: {}'.format(model)) # weights and other parameters (e.g. Adagrad) of the model name_ls = model.get_variable_names() print_shape = True total_linear_weights = 0 for name in name_ls: if print_shape: shape = model.get_variable_value(name).shape print(name, "\t", shape) if name[:6] == "linear" and \ (name[-7:] == "weights"or name[-4:] == "bias"): total_linear_weights += np.prod(shape) else: print(name) if print_shape: print("Total parameters in linear model: {}".format( total_linear_weights)) # embedding layer look up sample_embedding = model.get_variable_value( 'dnn/input_from_feature_columns/input_layer/ad_cates_embedding/embedding_weights' ) ids = [10, 20, 30] with tf.compat.v1.Session() as sess: lookup = tf.nn.embedding_lookup(params=sample_embedding, ids=ids).eval() print(lookup) # predictions tf.compat.v1.logging.info('=' * 30 + 'START PREDICTION' + '=' * 30) t0 = time.time() predictions = model.predict(input_fn=lambda: input_fn( FLAGS.data_dir, FLAGS.image_data_dir, 'pred', FLAGS.batch_size), predict_keys=None, hooks=None, checkpoint_path=FLAGS.checkpoint_path ) # defaults None to use latest_checkpoint for pred_dict in predictions: # dict{probabilities, classes, class_ids} class_id = pred_dict['class_ids'][0] probability = pred_dict['probabilities'][class_id] print('\nPrediction is "{}" ({:.1f}%)'.format(class_id, 100 * probability)) tf.compat.v1.logging.info( '=' * 30 + 'FINISH PREDICTION, TAKE {} mins'.format(elapse_time(t0)) + '=' * 30)
def main(unused_argv): print("Using TensorFlow version %s" % tf.__version__) assert "1.4" <= tf.__version__, "Need TensorFlow r1.4 or later." print("Using Train config: {}".format(CONFIG.train)) print('Model type: {}'.format(FLAGS.model_type)) model_dir = os.path.join(FLAGS.model_dir, FLAGS.model_type) print('Model directory: {}'.format(model_dir)) if not FLAGS.keep_train: # Clean up the model directory if not keep training shutil.rmtree(model_dir, ignore_errors=True) print('Remove model directory: {}'.format(model_dir)) # model = build_estimator(model_dir, FLAGS.model_type) model = build_custom_estimator(model_dir, FLAGS.model_type) tf.logging.info('Build estimator: {}'.format(model)) if CONFIG.distribution["is_distribution"]: print("Using PID: {}".format(os.getpid())) cluster = CONFIG.distribution["cluster"] job_name = CONFIG.distribution["job_name"] task_index = CONFIG.distribution["task_index"] print("Using Distributed TensorFlow. Local host: {} Job_name: {} Task_index: {}" .format(cluster[job_name][task_index], job_name, task_index)) cluster = tf.train.ClusterSpec(CONFIG.distribution["cluster"]) server = tf.train.Server(cluster, job_name=job_name, task_index=task_index) if job_name == 'ps': # wait for incoming connection forever server.join() # sess = tf.Session(server.target) # queue = create_done_queue(task_index, num_workers) # for i in range(num_workers): # sess.run(queue.dequeue()) # print("ps {} received worker {} done".format(task_index, i) # print("ps {} quitting".format(task_index)) else: # TODO:supervisor & MonotoredTrainingSession & experiment (deprecated) train(model) # train_and_eval(model) # Each worker only needs to contact the PS task(s) and the local worker task. # config = tf.ConfigProto(device_filters=[ # '/job:ps', '/job:worker/task:%d' % arguments.task_index]) # with tf.device(tf.train.replica_device_setter( # worker_device="/job:worker/task:%d" % task_index, # cluster=cluster)): # e = _create_experiment_fn() # e.train_and_evaluate() # call estimator's train() and evaluate() method # hooks = [tf.train.StopAtStepHook(last_step=10000)] # with tf.train.MonitoredTrainingSession( # master=server.target, # is_chief=(task_index == 0), # checkpoint_dir=args.model_dir, # hooks=hooks) as mon_sess: # while not mon_sess.should_stop(): # # mon_sess.run() # classifier.fit(input_fn=train_input_fn, steps=1) else: # local run train(model)
def main(_): if FLAGS.model_version <= 0: print('Please specify a positive value for version number.') sys.exit(-1) # tf.estimator.export.build_parsing_serving_input_receiver_fn # tf.estimator.export.build_raw_serving_input_receiver_fn # If these utilities do not meet your needs, you are free to write your own serving_input_receiver_fn() # feature_spec = { # "some_feature": tf.FixedLenFeature([], dtype=tf.string, default_value=""), # "some_feature": tf.VarLenFeature(dtype=tf.string), # } # # def _serving_input_receiver_fn(): # serialized_tf_example = tf.placeholder(dtype=tf.string, shape=None, # name='input_example_tensor') # # key (e.g. 'examples') should be same with the inputKey when you # # buid the request for prediction # receiver_tensors = {'examples': serialized_tf_example} # features = tf.parse_example(serialized_tf_example, your_feature_spec) # return tf.estimator.export.ServingInputReceiver(features, receiver_tensors) # estimator.export_savedmodel(export_dir, _serving_input_receiver_fn) wide_columns, deep_columns = _build_model_columns() feature_columns = wide_columns + deep_columns # for f in feature_columns: # print(f._parse_example_spec) # A dict mapping each feature key to a FixedLenFeature or VarLenFeature value. feature_spec = tf.feature_column.make_parse_example_spec(feature_columns) serving_input_receiver_fn = \ tf.estimator.export.build_parsing_serving_input_receiver_fn(feature_spec) model_dir = os.path.join(model_base_dir, FLAGS.model_type) export_dir = os.path.join(FLAGS.export_dir, FLAGS.model_type) print("building custom estimator") model = build_custom_estimator(model_dir, FLAGS.model_type) print("exporting saved model") model.export_savedmodel(export_dir, serving_input_receiver_fn, as_text=CONF['as_text'], checkpoint_path=FLAGS.checkpoint_path)
def main(unused_argv): CONFIG = Config() print("Using TensorFlow Version %s" % tf.__version__) assert "1.4" <= tf.__version__, "Need TensorFlow r1.4 or Later." print('\nModel Type: {}'.format(FLAGS.model_type)) model_dir = os.path.join(FLAGS.model_dir, FLAGS.model_type) print('\nModel Directory: {}'.format(model_dir)) print("\nUsing Train Config:") for k, v in CONFIG.train.items(): print('{}: {}'.format(k, v)) print("\nUsing Model Config:") for k, v in CONFIG.model.items(): print('{}: {}'.format(k, v)) if not FLAGS.keep_train: # Clean up the model directory if not keep training shutil.rmtree(model_dir, ignore_errors=True) print('Remove model directory: {}'.format(model_dir)) model = build_custom_estimator(model_dir, FLAGS.model_type) tf.logging.info('Build estimator: {}'.format(model)) train_and_eval_api(model)