def train():
# pack tf_dist_conf
if 'TF_CONFIG' in os.environ:
tf_dist_conf = os.environ['TF_CONFIG']
conf = json.loads(tf_dist_conf)
if conf['task']['type'] == 'ps':
is_ps = True
else:
is_ps = False
if conf['task']['type'] == 'master':
conf['task']['type'] = 'chief'
conf['cluster']['chief'] = conf['cluster']['master']
del conf['cluster'][
'master'] # delete all conf setting about 'master', trans to 'chief'
print(conf)
os.environ['TF_CONFIG'] = json.dumps(conf)
else:
print('tf_config not exists in os.environ, task over.')
return
if is_ps:
distribution = tf.distribute.experimental.ParameterServerStrategy()
else:
distribution = tf.distribute.experimental.MultiWorkerMirroredStrategy()
run_config = tf.estimator.RunConfig(train_distribute=distribution,
save_checkpoints_steps=10,
keep_checkpoint_max=3,
model_dir='/data/output/run_output')
# model_fn = my_model_fn(num_gpus=0)
_hparams = hparams.HParams()
# variable_strategy = 'CPU'
# num_gpus = 0
estimator = tf.estimator.Estimator(
model_fn=my_model_fn,
config=run_config,
params=_hparams,
)
data_dir = _hparams.tfrecord_dir
BATCH_SIZE = _hparams.batch_size # 16
# EPOCHS = 5
STEPS = _hparams.steps # 2000
train_spec = tf.estimator.TrainSpec(
input_fn=lambda: define_input_fn.my_input_fn(
data_dir=data_dir, subset='train', batch_size=BATCH_SIZE),
max_steps=STEPS)
eval_spec = tf.estimator.EvalSpec(
input_fn=lambda: define_input_fn.my_input_fn(
data_dir=data_dir, subset='val', batch_size=BATCH_SIZE),
steps=1,
start_delay_secs=1)
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def main():
# load params
_hparams = hparams.HParams()
# _hparams.reload_params(config.config_fpath)
tf.keras.optimizers.Adadelta()
tf.train.AdadeltaOptimizer()
# Session configuration.
sess_config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=False,
intra_op_parallelism_threads=0,
gpu_options=tf.GPUOptions(force_gpu_compatible=False))
# sess_config.gpu_options.per_process_gpu_memory_fraction=0.4
run_config = tf.estimator.RunConfig(
session_config=sess_config,
save_checkpoints_steps=_hparams.save_checkpoints_steps,
keep_checkpoint_max=_hparams.keep_checkpoint_max,
model_dir=config.model_save_dir)
estimator = tf.estimator.Estimator(
model_fn=my_model_fn,
config=run_config,
params=_hparams,
)
# train_setting
BATCH_SIZE = _hparams.batch_size * _hparams.gpu_nums # 16
# EPOCHS = 5
train_steps = _hparams.train_steps # 2000
eval_steps = _hparams.eval_steps
tfrecord_dir = config.tfrecord_dir
train_spec = tf.estimator.TrainSpec(
input_fn=lambda: define_input_fn.my_input_fn(data_dir=tfrecord_dir,
batch_size=BATCH_SIZE),
max_steps=train_steps)
eval_spec = tf.estimator.EvalSpec(
input_fn=lambda: define_input_fn.my_input_fn(data_dir=tfrecord_dir,
batch_size=BATCH_SIZE),
steps=eval_steps,
start_delay_secs=1)
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def main():
# Session configuration.
sess_config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=False,
intra_op_parallelism_threads=0,
gpu_options=tf.GPUOptions(force_gpu_compatible=False))
# sess_config.gpu_options.per_process_gpu_memory_fraction=0.4
run_config = tf.estimator.RunConfig(session_config=sess_config,
save_checkpoints_steps=10,
keep_checkpoint_max=3,
model_dir='/data/output/crnn_name_ckp')
# model_fn = my_model_fn(num_gpus=0)
_hparams = hparams.HParams()
# variable_strategy = 'CPU'
# num_gpus = 0
estimator = tf.estimator.Estimator(
model_fn=my_model_fn,
config=run_config,
params=_hparams,
)
BATCH_SIZE = _hparams.batch_size # 16
# EPOCHS = 5
STEPS = _hparams.steps # 2000
tfrecord_dir = _hparams.tfrecord_dir
train_spec = tf.estimator.TrainSpec(
input_fn=lambda: define_input_fn.my_input_fn(
data_dir=tfrecord_dir, subset='train', batch_size=BATCH_SIZE),
max_steps=STEPS)
eval_spec = tf.estimator.EvalSpec(
input_fn=lambda: define_input_fn.my_input_fn(
data_dir=tfrecord_dir, subset='val', batch_size=BATCH_SIZE),
steps=1,
start_delay_secs=1)
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def main():
# Session configuration.
sess_config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=False,
intra_op_parallelism_threads=0,
gpu_options=tf.GPUOptions(force_gpu_compatible=False))
# sess_config.gpu_options.per_process_gpu_memory_fraction=0.4
run_config = tf.estimator.RunConfig(session_config=sess_config,
save_checkpoints_steps=100,
keep_checkpoint_max=3,
model_dir='./run_output')
# model_fn = my_model_fn(num_gpus=0)
_hparams = hparams.HParams()
# variable_strategy = 'CPU'
# num_gpus = 0
estimator = tf.estimator.Estimator(
model_fn=my_model_fn,
config=run_config,
params=_hparams,
)
# add metrics
def my_acc(labels, predictions):
acc = tf.metrics.accuracy(labels=labels, predictions=predictions)
return {'acc': acc}
estimator = tf.contrib.estimator.add_metrics(estimator, my_acc)
data_dir = _hparams.tfrecord_dir
BATCH_SIZE = _hparams.batch_size # 16
# EPOCHS = 5
STEPS = _hparams.steps # 2000
train_spec = tf.estimator.TrainSpec(
input_fn=lambda: define_input_fn.my_input_fn(
data_dir=data_dir, subset='train', batch_size=BATCH_SIZE),
max_steps=STEPS)
eval_spec = tf.estimator.EvalSpec(
input_fn=lambda: define_input_fn.my_input_fn(
data_dir=data_dir, subset='val', batch_size=BATCH_SIZE),
steps=1,
start_delay_secs=1)
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
print('ckp_path: ', estimator.latest_checkpoint())
predictions = estimator.predict(
input_fn=lambda: define_input_fn.my_input_fn(batch_size=1),
yield_single_examples=True)
# pred_result = load_tf_data.sparse_tensor_to_str(next(predictions))
pred_res_num = next(predictions)
print('pred_res_num: ', pred_res_num)
int_to_char = load_tf_data.char_dict.int_to_char
pred_res_str = ''.join([int_to_char[int] for int in pred_res_num])
print('prediction: ', pred_res_str)
def main():
# Session configuration.
sess_config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=False,
intra_op_parallelism_threads=0,
gpu_options=tf.GPUOptions(force_gpu_compatible=False))
# sess_config.gpu_options.per_process_gpu_memory_fraction=0.4
run_config = tf.estimator.RunConfig(session_config=sess_config,
save_checkpoints_steps=100,
keep_checkpoint_max=3,
model_dir='./run_output')
# model_fn = my_model_fn(num_gpus=0)
_hparams = hparams.HParams()
# variable_strategy = 'CPU'
# num_gpus = 0
estimator = tf.estimator.Estimator(
model_fn=my_model_fn,
config=run_config,
params=_hparams,
)
data_dir = _hparams.tfrecord_dir
BATCH_SIZE = _hparams.batch_size # 16
# EPOCHS = 5
STEPS = _hparams.steps # 2000
train_spec = tf.estimator.TrainSpec(
input_fn=lambda: define_input_fn.my_input_fn(
data_dir=data_dir, subset='train', batch_size=BATCH_SIZE),
max_steps=STEPS)
eval_spec = tf.estimator.EvalSpec(
input_fn=lambda: define_input_fn.my_input_fn(
data_dir=data_dir, subset='val', batch_size=BATCH_SIZE),
steps=1,
start_delay_secs=1)
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
print('ckp_path: ', estimator.latest_checkpoint())
test_img_dir = './real_data/name_crop_result'
img_fn_list = os.listdir(test_img_dir)
for img_fn in img_fn_list:
test_img_path = os.path.join(test_img_dir, img_fn)
# test_img_path = './real_data/name_crop_result/1_55998336_IDFront_name_0.jpg'
print(test_img_path)
img_cv2 = cv2.imread(test_img_path)
# predictions = estimator.predict(input_fn=lambda: define_input_fn.input_fn_for_inference(img_cv2),
# yield_single_examples=True)
test_images = [cv2.resize(img_cv2, (100, 32))]
test_images = np.asarray(test_images)
test_labels = np.asarray([''])
test_input_fn = tf.estimator.inputs.numpy_input_fn(x=test_images,
y=test_labels,
batch_size=1,
num_epochs=1,
shuffle=False,
queue_capacity=1,
num_threads=1)
predictions = estimator.predict(input_fn=test_input_fn,
yield_single_examples=True)
# pred_result = load_tf_data.sparse_tensor_to_str(next(predictions))
pred_res_num = next(predictions)
print('pred_res_num: ', pred_res_num)
int_to_char = load_tf_data.char_dict.int_to_char
pred_res_str = ''.join([int_to_char[int] for int in pred_res_num])
print('prediction: ', pred_res_str)