def train(config):
# load Vocab
src_vocab = data_reader.Vocab(vocab_limits=config['src_vocab_size'])
src_vocab.load_metadata(config['metadata']['src'])
config['src_vocab_size'] = src_vocab.vocab_size()
tgt_vocab = data_reader.Vocab(vocab_limits=config['tgt_vocab_size'])
tgt_vocab.load_metadata(config['metadata']['tgt'])
config['tgt_vocab_size'] = tgt_vocab.vocab_size()
tf.logging.info(config)
initializer = tf.random_uniform_initializer(-config['init_scale'],
config['init_scale'])
# create models
with tf.name_scope('Train'):
opt, lr = optimizer.get_optimizer(config['optimizer'],
config['learning_rate'])
with tf.variable_scope("Model", reuse=None, initializer=initializer):
train_model = model.Model(is_training=True,
config=config,
seq_length=config['tgt_length'] - 1,
optimizer=opt,
lr=lr)
with tf.name_scope('Test'):
with tf.variable_scope("Model", reuse=True):
test_model = model.Model(is_training=False,
config=config,
seq_length=1)
sv = tf.train.Supervisor(logdir=config['logdir'])
sess_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=True)
sess_config.gpu_options.allow_growth = True
sess_config.gpu_options.per_process_gpu_memory_fraction = 0.9
# load Data
train_data = data_reader.DataReader(
src_data=config['train_data']['src'][0],
tgt_data=config['train_data']['tgt'][0],
src_vocab=src_vocab,
tgt_vocab=tgt_vocab,
src_length=config['src_length'],
tgt_length=config['tgt_length'],
batch_size=config['batch_size'])
tf.logging.info('Start Sess')
with sv.managed_session(config=sess_config) as sess:
for i in range(config['n_epoch']):
lr_decay = config['lr_decay']**max(i + 1 - config['decay_epoch'],
0)
train_model.assign_lr(sess, config['learning_rate'] * lr_decay)
tf.logging.info('Iter %d Start, Learning_rate: %.4f' %
(i, sess.run(train_model.lr)))
loss = run_epoch(sess, train_model, train_data, is_training = True, \
t_model = test_model, src_vocab = src_vocab, tgt_vocab = tgt_vocab)
tf.logging.info('Iter %d: training_loss: %.4f' %
(i, np.power(2, loss)))
def gen(config, gen_file):
# load Vocab
src_vocab = data_reader.Vocab(vocab_limits=config['src_vocab_size'])
src_vocab.load_metadata(config['metadata']['src'])
config['src_vocab_size'] = src_vocab.vocab_size()
tgt_vocab = data_reader.Vocab(vocab_limits=config['tgt_vocab_size'])
tgt_vocab.load_metadata(config['metadata']['tgt'])
config['tgt_vocab_size'] = tgt_vocab.vocab_size()
tf.logging.info(config)
# create model
with tf.name_scope('Genearte'):
with tf.variable_scope("Model", reuse=None):
gen_model = model.Model(is_training=False,
config=config,
seq_length=1)
sv = tf.train.Supervisor(logdir=config['logdir'])
sess_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
sess_config.gpu_options.allow_growth = True
sess_config.gpu_options.per_process_gpu_memory_fraction = 0.5
tf.logging.info('Start Sess')
with sv.managed_session(config=sess_config) as sess:
for line in open(gen_file):
words = line.strip().split(' ')
words.insert(0, "<s>")
words.append("</s>")
masks = [1] * len(words)
if len(words) < config['src_length']:
masks.extend([0] * (config['src_length'] - len(words)))
words.extend(["</s>"] * (config['src_length'] - len(words)))
src_vec = [src_vocab.char2id(c) for c in words]
src_vec = np.array(src_vec)
masks = np.array(masks)
sample, scores = sample_run(sess,
gen_model,
src_vec,
masks,
bos_id=tgt_vocab.char2id("<s>"),
eos_id=tgt_vocab.char2id("</s>"))
out_words = [tgt_vocab.id2char(c) for c in sample[0]]
tf.logging.info('Input: %s' % line.strip())
tf.logging.info('Output: %s' % (" ".join(out_words)))
def train(config):
# load Vocab
vocab = data_reader.Vocab(vocab_limits=config['vocab_size'])
vocab.load_metadata(config['metadata'])
config['vocab_size'] = vocab.vocab_size()
tf.logging.info(config)
initializer = tf.random_uniform_initializer(-config['init_scale'],
config['init_scale'])
# create models
with tf.name_scope('Train'):
opt, lr = optimizer.get_optimizer("sgd", config['learning_rate'])
with tf.variable_scope("Model", reuse=None, initializer=initializer):
train_model = model.Model(is_training = True, \
config = config, \
optimizer = opt,
lr = lr)
with tf.name_scope('Generate'):
generate_config = copy.deepcopy(config)
generate_config['batch_size'] = 1
generate_config['seq_length'] = 1
with tf.variable_scope("Model", reuse=True, initializer=initializer):
gen_model = model.Model(is_training=False, config=generate_config)
sv = tf.train.Supervisor(logdir=config['logdir'])
sess_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
sess_config.gpu_options.allow_growth = True
sess_config.gpu_options.per_process_gpu_memory_fraction = 0.5
tf.logging.info('Start Sess')
with sv.managed_session(config=sess_config) as sess:
for i in range(config['n_epoch']):
lr_decay = config['lr_decay']**max(i + 1 - config['decay_epoch'],
0)
train_model.assign_lr(sess, config['learning_rate'] * lr_decay)
tf.logging.info('Iter %d Start, Learning_rate: %.4f' %
(i, sess.run(train_model.lr)))
costs, speed = run_epoch(sess,
train_model,
datapath=config['train_data'][0],
config=config,
is_training=True,
gen_model=gen_model,
vocab=vocab)
tf.logging.info(
'Iter %d: training_loss:%.4f, speed %.4f words/sec' %
(i, np.exp(costs), speed))
def convert_model(config):
vocab = data_reader.Vocab(vocab_limits=config['vocab_size'])
vocab.load_metadata(config['metadata'])
config['vocab_size'] = vocab.vocab_size()
tf.logging.info(config)
with tf.name_scope('Eval'):
config['batch_size'] = 100
config['seq_length'] = 100
with tf.variable_scope("Model", reuse=None):
eval_model = model.Model(is_training=False, config=config)
with tf.Session() as sess:
saver = tf.train.Saver()
ckpt = tf.train.latest_checkpoint(config['logdir'])
tf.logging.info(ckpt)
saver.restore(sess, ckpt)
export_path = os.path.join(config['logdir'], 'exported/2')
builder = tf.saved_model.builder.SavedModelBuilder(export_path)
in_x = tf.saved_model.utils.build_tensor_info(eval_model.input_data)
out_y = tf.saved_model.utils.build_tensor_info(eval_model.target_data)
pred = tf.saved_model.utils.build_tensor_info(eval_model.target_probs)
pred_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={
'x': in_x,
'y': out_y
},
outputs={'pred': pred},
method_name=tf.saved_model.signature_constants.
PREDICT_METHOD_NAME))
legacy_init_op = tf.group(tf.tables_initializer(),
name='legacy_init_op')
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
'predict':
pred_signature,
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
pred_signature,
},
legacy_init_op=legacy_init_op)
builder.save()
def gen(config):
vocab = data_reader.Vocab(vocab_limits=config['vocab_size'])
vocab.load_metadata(config['metadata'])
config['vocab_size'] = vocab.vocab_size()
tf.logging.info(config)
with tf.name_scope('Generate'):
config['batch_size'] = 1
config['seq_length'] = 1
with tf.variable_scope("Model", reuse=None):
gen_model = model.Model(is_training=False, config=config)
with tf.Session() as sess:
saver = tf.train.Saver()
ckpt = tf.train.latest_checkpoint(config['logdir'])
tf.logging.info(ckpt)
saver.restore(sess, ckpt)
sample(sess, model=gen_model, vocab=vocab)
def rename_model(config):
vocab = data_reader.Vocab(vocab_limits=config['vocab_size'])
vocab.load_metadata(config['metadata'])
config['vocab_size'] = vocab.vocab_size()
tf.logging.info(config)
config['batch_size'] = 100
config['seq_length'] = 100
with tf.variable_scope("Model", reuse=None):
eval_model = model.Model(is_training=False, config=config)
with tf.Session() as sess:
saver = tf.train.Saver()
ckpt = tf.train.latest_checkpoint(config['logdir'])
tf.logging.info(ckpt)
saver.restore(sess, ckpt)
tf.train.write_graph(sess.graph_def,
config['logdir'],
"nn_model.pbtxt",
as_text=True)
def eval(config):
vocab = data_reader.Vocab(vocab_limits=config['vocab_size'])
vocab.load_metadata(config['metadata'])
config['vocab_size'] = vocab.vocab_size()
tf.logging.info(config)
batch_size = 100
with tf.name_scope('Eval'):
config['batch_size'] = batch_size
config['seq_length'] = 1
with tf.variable_scope("Model", reuse=None):
eval_model = model.Model(is_training=False, config=config)
with tf.Session() as sess:
saver = tf.train.Saver()
ckpt = tf.train.latest_checkpoint(config['logdir'])
tf.logging.info(ckpt)
saver.restore(sess, ckpt)
out = open('%s.out3' % FLAGS.evalfile, 'w')
index = None
max_len = 0
cnt = 0
sens = [[]] * batch_size
base = [None] * batch_size
lens = [0] * batch_size
for line in open(FLAGS.evalfile):
vec = line.strip().split()
now_idx = vec[0]
if index != now_idx:
for i in range(batch_size):
sens[i].extend([vocab.char2id("</s>")] *
(max_len - len(sens[i])))
scores = sen_score(sess,
input=sens,
lens=lens,
model=eval_model)
for i in range(batch_size):
if base[i] == None: break
out.write('%s\t%d\t%.4f\n' % (base[i], lens[i], scores[i]))
max_len = 0
cnt = 0
sens = [[]] * batch_size
base = [None] * batch_size
lens = [0] * batch_size
index = now_idx
sen = list(''.join(vec[1:]).decode('utf-8'))
sen.insert(0, "<s>")
sen.append("</s>")
base[cnt] = line.strip()
sens[cnt] = [vocab.char2id(c) for c in sen]
lens[cnt] = len(sens[cnt])
max_len = max(len(sens[cnt]), max_len)
cnt += 1
for i in range(batch_size):
sens[i].extend([vocab.char2id("</s>")] * (max_len - len(sens[i])))
scores = sen_score(sess, input=sens, lens=lens, model=eval_model)
for i in range(batch_size):
if base[i] == None: break
out.write('%s\t%d\t%.4f\n' % (base[i], lens[i], scores[i]))
def cluster_train(config):
# init for cluster
cluster_conf = config["cluster_conf"]
cluster = tf.train.ClusterSpec(cluster_conf)
server = tf.train.Server(cluster,
job_name=FLAGS.job_name,
task_index=FLAGS.task_id)
n_ps = len(cluster_conf['ps'])
n_workers = len(cluster_conf['worker'])
if FLAGS.job_name == "ps": server.join()
is_chief = (FLAGS.task_id == 0)
# load Vocab
vocab = data_reader.Vocab(vocab_limits=config['vocab_size'])
vocab.load_metadata(config['metadata'])
config['vocab_size'] = vocab.vocab_size()
tf.logging.info(config)
# load Data
train_data = data_reader.DataReader(config['train_data'][FLAGS.task_id],
vocab=vocab,
batch_size=config['batch_size'],
seq_length=config['seq_length'])
initializer = tf.random_uniform_initializer(-config['init_scale'],
config['init_scale'])
# create models
with tf.device(
tf.train.replica_device_setter(
worker_device="/job:worker/task:%d" % FLAGS.task_id,
cluster=cluster)):
with tf.name_scope('Train'):
opt, lr = optimizer.get_optimizer("sgd", config['learning_rate'])
sync_opt = tf.train.SyncReplicasOptimizer(
opt,
replicas_to_aggregate=n_workers * config['sync_freq'],
total_num_replicas=n_workers)
with tf.variable_scope("Model",
reuse=None,
initializer=initializer):
train_model = model.Model(is_training = True, \
config = config, \
optimizer = sync_opt,
lr = lr)
with tf.name_scope('Generate'):
generate_config = copy.deepcopy(config)
generate_config['batch_size'] = 1
generate_config['seq_length'] = 1
with tf.variable_scope("Model",
reuse=True,
initializer=initializer):
gen_model = model.Model(is_training=False,
config=generate_config)
chief_queue_runner = train_model.optimizer.get_chief_queue_runner()
init_tokens_op = train_model.optimizer.get_init_tokens_op()
sv = tf.train.Supervisor(is_chief=is_chief, logdir=config['logdir'])
sess_config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=True,
device_filters=["/job:ps",
"/job:worker/task:%d" % FLAGS.task_id])
sess_config.gpu_options.allow_growth = True
sess_config.gpu_options.per_process_gpu_memory_fraction = 0.5
tf.logging.info('Start Sess')
with sv.prepare_or_wait_for_session(server.target,
config=sess_config) as sess:
if is_chief:
sv.start_queue_runners(sess, [chief_queue_runner])
sess.run(init_tokens_op)
for i in range(config['n_epoch']):
lr_decay = config['lr_decay']**max(i + 1 - config['decay_epoch'],
0)
train_model.assign_lr(sess, config['learning_rate'] * lr_decay)
tf.logging.info('Iter %d Start, Learning_rate: %.4f' %
(i, sess.run(train_model.lr)))
costs, speed = run_epoch(sess,
train_model,
train_data,
is_training=True,
gen_model=gen_model,
vocab=vocab)
tf.logging.info(
'Iter %d: training_loss:%.4f, speed %.4f words/sec' %
(i, np.exp(costs), speed))
def eval(infile, outfile, config):
# init vocab
vocab = data_reader.Vocab(vocab_limits = config['vocab_size'])
vocab.load_metadata(config['metadata'])
config['vocab_size'] = vocab.vocab_size()
tf.logging.info(config)
seq_length = config['seq_length']
out = open(outfile, 'w')
index = None
lens = []
in_ids = []
out_ids = []
base = []
total_timer = time_control()
send_timer = time_control()
total_timer.start()
for line in open(infile):
vec = line.strip().split()
if index != vec[0]:
if index != None:
send_timer.start()
scores = client_batch(input = in_ids, output = out_ids, lens = lens,
batch_size = len(in_ids), seq_length = seq_length)
send_timer.stop()
for i in range(len(in_ids)):
if base[i] == None: break
out.write('%s\t%d\t%.4f\n' % (base[i], lens[i], scores[i]))
index = vec[0]
lens = []
in_ids = []
out_ids = []
base = []
sen = list(''.join(vec[1:]).decode('utf-8'))
sen.insert(0, "<s>")
sen.append("</s>")
lens.append(len(sen))
sen.extend(["</s>"] * (seq_length - len(sen)))
base.append(line.strip())
in_ids.append([vocab.char2id(c) for c in sen])
out_ids.append(in_ids[-1][1:])
out_ids[-1].append(vocab.char2id("</s>"))
send_timer.start()
scores = client_batch(input = in_ids, output = out_ids, lens = lens,
batch_size = len(in_ids), seq_length = seq_length)
send_timer.stop()
for i in range(len(in_ids)):
if base[i] == None: break
out.write('%s\t%d\t%.4f\n' % (base[i], lens[i], scores[i]))
total_timer.stop()
print 'total_timer: %.2f' % total_timer.get_total_time()
print 'send_timer: %.2f' % send_timer.get_total_time()
print 'client_timer: %.2f' % client_timer.get_total_time()
