def train(): # pack tf_dist_conf if 'TF_CONFIG' in os.environ: tf_dist_conf = os.environ['TF_CONFIG'] conf = json.loads(tf_dist_conf) if conf['task']['type'] == 'ps': is_ps = True else: is_ps = False if conf['task']['type'] == 'master': conf['task']['type'] = 'chief' conf['cluster']['chief'] = conf['cluster']['master'] del conf['cluster'][ 'master'] # delete all conf setting about 'master', trans to 'chief' print(conf) os.environ['TF_CONFIG'] = json.dumps(conf) else: print('tf_config not exists in os.environ, task over.') return if is_ps: distribution = tf.distribute.experimental.ParameterServerStrategy() else: distribution = tf.distribute.experimental.MultiWorkerMirroredStrategy() run_config = tf.estimator.RunConfig(train_distribute=distribution, save_checkpoints_steps=10, keep_checkpoint_max=3, model_dir='/data/output/run_output') # model_fn = my_model_fn(num_gpus=0) _hparams = hparams.HParams() # variable_strategy = 'CPU' # num_gpus = 0 estimator = tf.estimator.Estimator( model_fn=my_model_fn, config=run_config, params=_hparams, ) data_dir = _hparams.tfrecord_dir BATCH_SIZE = _hparams.batch_size # 16 # EPOCHS = 5 STEPS = _hparams.steps # 2000 train_spec = tf.estimator.TrainSpec( input_fn=lambda: define_input_fn.my_input_fn( data_dir=data_dir, subset='train', batch_size=BATCH_SIZE), max_steps=STEPS) eval_spec = tf.estimator.EvalSpec( input_fn=lambda: define_input_fn.my_input_fn( data_dir=data_dir, subset='val', batch_size=BATCH_SIZE), steps=1, start_delay_secs=1) tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def main(): # load params _hparams = hparams.HParams() # _hparams.reload_params(config.config_fpath) tf.keras.optimizers.Adadelta() tf.train.AdadeltaOptimizer() # Session configuration. sess_config = tf.ConfigProto( allow_soft_placement=True, log_device_placement=False, intra_op_parallelism_threads=0, gpu_options=tf.GPUOptions(force_gpu_compatible=False)) # sess_config.gpu_options.per_process_gpu_memory_fraction=0.4 run_config = tf.estimator.RunConfig( session_config=sess_config, save_checkpoints_steps=_hparams.save_checkpoints_steps, keep_checkpoint_max=_hparams.keep_checkpoint_max, model_dir=config.model_save_dir) estimator = tf.estimator.Estimator( model_fn=my_model_fn, config=run_config, params=_hparams, ) # train_setting BATCH_SIZE = _hparams.batch_size * _hparams.gpu_nums # 16 # EPOCHS = 5 train_steps = _hparams.train_steps # 2000 eval_steps = _hparams.eval_steps tfrecord_dir = config.tfrecord_dir train_spec = tf.estimator.TrainSpec( input_fn=lambda: define_input_fn.my_input_fn(data_dir=tfrecord_dir, batch_size=BATCH_SIZE), max_steps=train_steps) eval_spec = tf.estimator.EvalSpec( input_fn=lambda: define_input_fn.my_input_fn(data_dir=tfrecord_dir, batch_size=BATCH_SIZE), steps=eval_steps, start_delay_secs=1) tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def main(): # Session configuration. sess_config = tf.ConfigProto( allow_soft_placement=True, log_device_placement=False, intra_op_parallelism_threads=0, gpu_options=tf.GPUOptions(force_gpu_compatible=False)) # sess_config.gpu_options.per_process_gpu_memory_fraction=0.4 run_config = tf.estimator.RunConfig(session_config=sess_config, save_checkpoints_steps=10, keep_checkpoint_max=3, model_dir='/data/output/crnn_name_ckp') # model_fn = my_model_fn(num_gpus=0) _hparams = hparams.HParams() # variable_strategy = 'CPU' # num_gpus = 0 estimator = tf.estimator.Estimator( model_fn=my_model_fn, config=run_config, params=_hparams, ) BATCH_SIZE = _hparams.batch_size # 16 # EPOCHS = 5 STEPS = _hparams.steps # 2000 tfrecord_dir = _hparams.tfrecord_dir train_spec = tf.estimator.TrainSpec( input_fn=lambda: define_input_fn.my_input_fn( data_dir=tfrecord_dir, subset='train', batch_size=BATCH_SIZE), max_steps=STEPS) eval_spec = tf.estimator.EvalSpec( input_fn=lambda: define_input_fn.my_input_fn( data_dir=tfrecord_dir, subset='val', batch_size=BATCH_SIZE), steps=1, start_delay_secs=1) tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def main(): # Session configuration. sess_config = tf.ConfigProto( allow_soft_placement=True, log_device_placement=False, intra_op_parallelism_threads=0, gpu_options=tf.GPUOptions(force_gpu_compatible=False)) # sess_config.gpu_options.per_process_gpu_memory_fraction=0.4 run_config = tf.estimator.RunConfig(session_config=sess_config, save_checkpoints_steps=100, keep_checkpoint_max=3, model_dir='./run_output') # model_fn = my_model_fn(num_gpus=0) _hparams = hparams.HParams() # variable_strategy = 'CPU' # num_gpus = 0 estimator = tf.estimator.Estimator( model_fn=my_model_fn, config=run_config, params=_hparams, ) # add metrics def my_acc(labels, predictions): acc = tf.metrics.accuracy(labels=labels, predictions=predictions) return {'acc': acc} estimator = tf.contrib.estimator.add_metrics(estimator, my_acc) data_dir = _hparams.tfrecord_dir BATCH_SIZE = _hparams.batch_size # 16 # EPOCHS = 5 STEPS = _hparams.steps # 2000 train_spec = tf.estimator.TrainSpec( input_fn=lambda: define_input_fn.my_input_fn( data_dir=data_dir, subset='train', batch_size=BATCH_SIZE), max_steps=STEPS) eval_spec = tf.estimator.EvalSpec( input_fn=lambda: define_input_fn.my_input_fn( data_dir=data_dir, subset='val', batch_size=BATCH_SIZE), steps=1, start_delay_secs=1) tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec) print('ckp_path: ', estimator.latest_checkpoint()) predictions = estimator.predict( input_fn=lambda: define_input_fn.my_input_fn(batch_size=1), yield_single_examples=True) # pred_result = load_tf_data.sparse_tensor_to_str(next(predictions)) pred_res_num = next(predictions) print('pred_res_num: ', pred_res_num) int_to_char = load_tf_data.char_dict.int_to_char pred_res_str = ''.join([int_to_char[int] for int in pred_res_num]) print('prediction: ', pred_res_str)
def main(): # Session configuration. sess_config = tf.ConfigProto( allow_soft_placement=True, log_device_placement=False, intra_op_parallelism_threads=0, gpu_options=tf.GPUOptions(force_gpu_compatible=False)) # sess_config.gpu_options.per_process_gpu_memory_fraction=0.4 run_config = tf.estimator.RunConfig(session_config=sess_config, save_checkpoints_steps=100, keep_checkpoint_max=3, model_dir='./run_output') # model_fn = my_model_fn(num_gpus=0) _hparams = hparams.HParams() # variable_strategy = 'CPU' # num_gpus = 0 estimator = tf.estimator.Estimator( model_fn=my_model_fn, config=run_config, params=_hparams, ) data_dir = _hparams.tfrecord_dir BATCH_SIZE = _hparams.batch_size # 16 # EPOCHS = 5 STEPS = _hparams.steps # 2000 train_spec = tf.estimator.TrainSpec( input_fn=lambda: define_input_fn.my_input_fn( data_dir=data_dir, subset='train', batch_size=BATCH_SIZE), max_steps=STEPS) eval_spec = tf.estimator.EvalSpec( input_fn=lambda: define_input_fn.my_input_fn( data_dir=data_dir, subset='val', batch_size=BATCH_SIZE), steps=1, start_delay_secs=1) tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec) print('ckp_path: ', estimator.latest_checkpoint()) test_img_dir = './real_data/name_crop_result' img_fn_list = os.listdir(test_img_dir) for img_fn in img_fn_list: test_img_path = os.path.join(test_img_dir, img_fn) # test_img_path = './real_data/name_crop_result/1_55998336_IDFront_name_0.jpg' print(test_img_path) img_cv2 = cv2.imread(test_img_path) # predictions = estimator.predict(input_fn=lambda: define_input_fn.input_fn_for_inference(img_cv2), # yield_single_examples=True) test_images = [cv2.resize(img_cv2, (100, 32))] test_images = np.asarray(test_images) test_labels = np.asarray(['']) test_input_fn = tf.estimator.inputs.numpy_input_fn(x=test_images, y=test_labels, batch_size=1, num_epochs=1, shuffle=False, queue_capacity=1, num_threads=1) predictions = estimator.predict(input_fn=test_input_fn, yield_single_examples=True) # pred_result = load_tf_data.sparse_tensor_to_str(next(predictions)) pred_res_num = next(predictions) print('pred_res_num: ', pred_res_num) int_to_char = load_tf_data.char_dict.int_to_char pred_res_str = ''.join([int_to_char[int] for int in pred_res_num]) print('prediction: ', pred_res_str)