def train():
# pack tf_dist_conf
if 'TF_CONFIG' in os.environ:
tf_dist_conf = os.environ['TF_CONFIG']
conf = json.loads(tf_dist_conf)
if conf['task']['type'] == 'ps':
is_ps = True
else:
is_ps = False
if conf['task']['type'] == 'master':
conf['task']['type'] = 'chief'
conf['cluster']['chief'] = conf['cluster']['master']
del conf['cluster'][
'master'] # delete all conf setting about 'master', trans to 'chief'
print(conf)
os.environ['TF_CONFIG'] = json.dumps(conf)
else:
print('tf_config not exists in os.environ, task over.')
return
if is_ps:
distribution = tf.distribute.experimental.ParameterServerStrategy()
else:
distribution = tf.distribute.experimental.MultiWorkerMirroredStrategy()
run_config = tf.estimator.RunConfig(train_distribute=distribution,
save_checkpoints_steps=10,
keep_checkpoint_max=3,
model_dir='/data/output/run_output')
# model_fn = my_model_fn(num_gpus=0)
_hparams = hparams.HParams()
# variable_strategy = 'CPU'
# num_gpus = 0
estimator = tf.estimator.Estimator(
model_fn=my_model_fn,
config=run_config,
params=_hparams,
)
data_dir = _hparams.tfrecord_dir
BATCH_SIZE = _hparams.batch_size # 16
# EPOCHS = 5
STEPS = _hparams.steps # 2000
train_spec = tf.estimator.TrainSpec(
input_fn=lambda: define_input_fn.my_input_fn(
data_dir=data_dir, subset='train', batch_size=BATCH_SIZE),
max_steps=STEPS)
eval_spec = tf.estimator.EvalSpec(
input_fn=lambda: define_input_fn.my_input_fn(
data_dir=data_dir, subset='val', batch_size=BATCH_SIZE),
steps=1,
start_delay_secs=1)
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def main():
# load params
_hparams = hparams.HParams()
# _hparams.reload_params(config.config_fpath)
tf.keras.optimizers.Adadelta()
tf.train.AdadeltaOptimizer()
# Session configuration.
sess_config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=False,
intra_op_parallelism_threads=0,
gpu_options=tf.GPUOptions(force_gpu_compatible=False))
# sess_config.gpu_options.per_process_gpu_memory_fraction=0.4
run_config = tf.estimator.RunConfig(
session_config=sess_config,
save_checkpoints_steps=_hparams.save_checkpoints_steps,
keep_checkpoint_max=_hparams.keep_checkpoint_max,
model_dir=config.model_save_dir)
estimator = tf.estimator.Estimator(
model_fn=my_model_fn,
config=run_config,
params=_hparams,
)
# train_setting
BATCH_SIZE = _hparams.batch_size * _hparams.gpu_nums # 16
# EPOCHS = 5
train_steps = _hparams.train_steps # 2000
eval_steps = _hparams.eval_steps
test_img_dir = '/data/data/weche_train_data/images'
img_fn_list = os.listdir(test_img_dir)[0:1]
for img_fn in img_fn_list:
print('img_fn: ', img_fn)
img_fpath = os.path.join(test_img_dir, img_fn)
img_cv2 = cv2.imread(img_fpath)
img_cv2 = cv2.resize(img_cv2, (config.img_w, config.img_h))
img_data = np.asarray(img_cv2, dtype=np.float)
img_data = np.expand_dims(img_data, axis=0)
test_images = np.asarray(img_data, dtype=np.float32)
test_labels = np.asarray([''] * img_data.shape[0])
test_input_fn = tf.estimator.inputs.numpy_input_fn(x=test_images,
y=test_labels,
batch_size=1,
num_epochs=1,
shuffle=False,
queue_capacity=1,
num_threads=1)
predictions = estimator.predict(input_fn=test_input_fn,
yield_single_examples=False)
_predictions = next(predictions)
pred_boxes = _predictions['boxes']
print('boxes: ', pred_boxes)
print('labels: ', _predictions['labels'])
print('scores: ', _predictions['scores'])
def main():
# Session configuration.
sess_config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=False,
intra_op_parallelism_threads=0,
gpu_options=tf.GPUOptions(force_gpu_compatible=False))
# sess_config.gpu_options.per_process_gpu_memory_fraction=0.4
run_config = tf.estimator.RunConfig(session_config=sess_config,
save_checkpoints_steps=100,
keep_checkpoint_max=3,
model_dir='./run_output')
# model_fn = my_model_fn(num_gpus=0)
_hparams = hparams.HParams()
# variable_strategy = 'CPU'
# num_gpus = 0
estimator = tf.estimator.Estimator(
model_fn=my_model_fn,
config=run_config,
params=_hparams,
)
BATCH_SIZE = _hparams.batch_size # 16
EPOCHS = 5
STEPS = _hparams.steps # 2000
train_spec = tf.estimator.TrainSpec(
input_fn=lambda: crnn_estimator.my_input_fn(subset='train',
batch_size=BATCH_SIZE),
max_steps=STEPS)
eval_spec = tf.estimator.EvalSpec(
input_fn=lambda: crnn_estimator.my_input_fn(subset='val',
batch_size=BATCH_SIZE),
steps=1,
start_delay_secs=1)
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
print('ckp_path: ', estimator.latest_checkpoint())
predictions = estimator.predict(
input_fn=lambda: crnn_estimator.my_input_fn(batch_size=1),
yield_single_examples=True)
# pred_result = load_tf_data.sparse_tensor_to_str(next(predictions))
pred_res_num = next(predictions)
print('pred_res_num: ', pred_res_num)
int_to_char = load_tf_data.char_dict.int_to_char
pred_res_str = ''.join([int_to_char[int] for int in pred_res_num])
print('prediction: ', pred_res_str)
def _build_model(self):
hyparams = hparams.HParams()
hyparams.embedding_dim = 100
X = T.imatrix('X')
M = T.matrix('M')
y = T.ivector('y')
print "building LSTM model..."
clf = build_model(hyparams, self.vocab, self.nclass, invar=X, maskvar=M)
read_model_from_file(clf, self.model_name)
print "model built."
test_output = lasagne.layers.get_output(clf['softmax'], deterministic=True)
val_cost_func = lasagne.objectives.categorical_crossentropy(test_output, y).mean()
preds = T.argmax(test_output, axis=1)
val_acc_func = T.mean(T.eq(preds, y), dtype=theano.config.floatX)
val_func = theano.function([X, M, y], [val_cost_func, val_acc_func, preds], allow_input_downcast=True)
self.pred = val_func
def main():
# load params
_hparams = hparams.HParams()
# _hparams.reload_params(config.config_fpath)
tf.keras.optimizers.Adadelta()
tf.train.AdadeltaOptimizer()
# Session configuration.
sess_config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=False,
intra_op_parallelism_threads=0,
gpu_options=tf.GPUOptions(force_gpu_compatible=False))
# sess_config.gpu_options.per_process_gpu_memory_fraction=0.4
run_config = tf.estimator.RunConfig(
session_config=sess_config,
save_checkpoints_steps=_hparams.save_checkpoints_steps,
keep_checkpoint_max=_hparams.keep_checkpoint_max,
model_dir=config.model_save_dir)
estimator = tf.estimator.Estimator(
model_fn=my_model_fn,
config=run_config,
params=_hparams,
)
# train_setting
BATCH_SIZE = _hparams.batch_size * _hparams.gpu_nums # 16
# EPOCHS = 5
train_steps = _hparams.train_steps # 2000
eval_steps = _hparams.eval_steps
tfrecord_dir = config.tfrecord_dir
train_spec = tf.estimator.TrainSpec(
input_fn=lambda: define_input_fn.my_input_fn(data_dir=tfrecord_dir,
batch_size=BATCH_SIZE),
max_steps=train_steps)
eval_spec = tf.estimator.EvalSpec(
input_fn=lambda: define_input_fn.my_input_fn(data_dir=tfrecord_dir,
batch_size=BATCH_SIZE),
steps=eval_steps,
start_delay_secs=1)
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def main():
# Session configuration.
sess_config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=False,
intra_op_parallelism_threads=0,
gpu_options=tf.GPUOptions(force_gpu_compatible=False))
# sess_config.gpu_options.per_process_gpu_memory_fraction=0.4
run_config = tf.estimator.RunConfig(
session_config=sess_config,
save_checkpoints_steps=10,
keep_checkpoint_max=3,
model_dir='/data/output/crnn_name_v1.0')
# model_fn = my_model_fn(num_gpus=0)
_hparams = hparams.HParams()
# variable_strategy = 'CPU'
# num_gpus = 0
estimator = tf.estimator.Estimator(
model_fn=my_model_fn,
config=run_config,
params=_hparams,
)
BATCH_SIZE = _hparams.batch_size # 16
# EPOCHS = 5
STEPS = _hparams.steps # 2000
tfrecord_dir = '/data/data/tfrecords'
train_spec = tf.estimator.TrainSpec(
input_fn=lambda: crnn_estimator.my_input_fn(
data_dir=tfrecord_dir, subset='train', batch_size=BATCH_SIZE),
max_steps=STEPS)
eval_spec = tf.estimator.EvalSpec(
input_fn=lambda: crnn_estimator.my_input_fn(
data_dir=tfrecord_dir, subset='val', batch_size=BATCH_SIZE),
steps=1,
start_delay_secs=1)
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def main():
_hparams = hparams.HParams()
img_dir = '/data/data/ticket_train_data/images'
train_json_path = '/data/data/ticket_train_data/labels/train.json'
val_json_path = '/data/data/ticket_train_data/labels/val.json'
batch_size = 8
save_dir = _hparams.tfrecord_dir
if not os.path.exists(save_dir):
os.mkdir(save_dir)
# train_data
gen_tfrecords(img_dir=img_dir,
json_fpath=train_json_path,
batch_size=batch_size,
save_dir=save_dir,
data_type='train')
# val_data
gen_tfrecords(img_dir=img_dir,
json_fpath=val_json_path,
batch_size=batch_size,
save_dir=save_dir,
data_type='val')
"""
2019.12 by liu
"""
import tensorflow as tf
import hparams
import numpy as np
import data_loader
hp = hparams.HParams().hparams
def get_token_embedding(vocab_size, embeding_size, zero_pad=True):
"""
建立word_embedding [vocab_size, embedding_size]
:param vocab_size:
:param embeding_size:
:param zero_pad: if True 即矩阵第一行的值为0, PAD_ID = 0 则最后得到的词向量的值pad值对应的值为0, 方便后序对mask的处理
:return:
"""
with tf.variable_scope("word_embedding"):
embedding = tf.get_variable(name="embedding",
dtype=tf.float32,
shape=[vocab_size, embeding_size],
initializer=tf.contrib.layers.xavier_initializer())
if zero_pad:
embedding = tf.concat([tf.zeros(shape=[1, embeding_size]),
embedding[1:, :]], axis=0)
return embedding
#!/usr/bin/env python # -*- coding: utf-8 -*- # @Time : 17-9-22 下午3:25 # @Author : Luo Yao # @Site : http://github.com/TJCVRS # @File : config.py # @IDE: PyCharm Community Edition """ Set some global configuration """ from easydict import EasyDict as edict import hparams _hparams = hparams.HParams() __C = edict() # Consumers can get config by: from config import cfg cfg = __C # Train options __C.TRAIN = edict() # Set the shadownet training epochs __C.TRAIN.EPOCHS = 3 # Set the display step __C.TRAIN.DISPLAY_STEP = 1 # Set the test display step during training process __C.TRAIN.TEST_DISPLAY_STEP = 2 # Set the momentum parameter of the optimizer __C.TRAIN.MOMENTUM = 0.9 # Set the initial learning rate
def main():
# Session configuration.
sess_config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=False,
intra_op_parallelism_threads=0,
gpu_options=tf.GPUOptions(force_gpu_compatible=False))
# sess_config.gpu_options.per_process_gpu_memory_fraction=0.4
run_config = tf.estimator.RunConfig(session_config=sess_config,
save_checkpoints_steps=100,
keep_checkpoint_max=3,
model_dir='./run_output')
# model_fn = my_model_fn(num_gpus=0)
_hparams = hparams.HParams()
# variable_strategy = 'CPU'
# num_gpus = 0
estimator = tf.estimator.Estimator(
model_fn=my_model_fn,
config=run_config,
params=_hparams,
)
data_dir = _hparams.tfrecord_dir
BATCH_SIZE = _hparams.batch_size # 16
# EPOCHS = 5
STEPS = _hparams.steps # 2000
train_spec = tf.estimator.TrainSpec(
input_fn=lambda: define_input_fn.my_input_fn(
data_dir=data_dir, subset='train', batch_size=BATCH_SIZE),
max_steps=STEPS)
eval_spec = tf.estimator.EvalSpec(
input_fn=lambda: define_input_fn.my_input_fn(
data_dir=data_dir, subset='val', batch_size=BATCH_SIZE),
steps=1,
start_delay_secs=1)
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
print('ckp_path: ', estimator.latest_checkpoint())
test_img_dir = './real_data/name_crop_result'
img_fn_list = os.listdir(test_img_dir)
for img_fn in img_fn_list:
test_img_path = os.path.join(test_img_dir, img_fn)
# test_img_path = './real_data/name_crop_result/1_55998336_IDFront_name_0.jpg'
print(test_img_path)
img_cv2 = cv2.imread(test_img_path)
# predictions = estimator.predict(input_fn=lambda: define_input_fn.input_fn_for_inference(img_cv2),
# yield_single_examples=True)
test_images = [cv2.resize(img_cv2, (100, 32))]
test_images = np.asarray(test_images)
test_labels = np.asarray([''])
test_input_fn = tf.estimator.inputs.numpy_input_fn(x=test_images,
y=test_labels,
batch_size=1,
num_epochs=1,
shuffle=False,
queue_capacity=1,
num_threads=1)
predictions = estimator.predict(input_fn=test_input_fn,
yield_single_examples=True)
# pred_result = load_tf_data.sparse_tensor_to_str(next(predictions))
pred_res_num = next(predictions)
print('pred_res_num: ', pred_res_num)
int_to_char = load_tf_data.char_dict.int_to_char
pred_res_str = ''.join([int_to_char[int] for int in pred_res_num])
print('prediction: ', pred_res_str)