def train(config): # load Vocab src_vocab = data_reader.Vocab(vocab_limits=config['src_vocab_size']) src_vocab.load_metadata(config['metadata']['src']) config['src_vocab_size'] = src_vocab.vocab_size() tgt_vocab = data_reader.Vocab(vocab_limits=config['tgt_vocab_size']) tgt_vocab.load_metadata(config['metadata']['tgt']) config['tgt_vocab_size'] = tgt_vocab.vocab_size() tf.logging.info(config) initializer = tf.random_uniform_initializer(-config['init_scale'], config['init_scale']) # create models with tf.name_scope('Train'): opt, lr = optimizer.get_optimizer(config['optimizer'], config['learning_rate']) with tf.variable_scope("Model", reuse=None, initializer=initializer): train_model = model.Model(is_training=True, config=config, seq_length=config['tgt_length'] - 1, optimizer=opt, lr=lr) with tf.name_scope('Test'): with tf.variable_scope("Model", reuse=True): test_model = model.Model(is_training=False, config=config, seq_length=1) sv = tf.train.Supervisor(logdir=config['logdir']) sess_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=True) sess_config.gpu_options.allow_growth = True sess_config.gpu_options.per_process_gpu_memory_fraction = 0.9 # load Data train_data = data_reader.DataReader( src_data=config['train_data']['src'][0], tgt_data=config['train_data']['tgt'][0], src_vocab=src_vocab, tgt_vocab=tgt_vocab, src_length=config['src_length'], tgt_length=config['tgt_length'], batch_size=config['batch_size']) tf.logging.info('Start Sess') with sv.managed_session(config=sess_config) as sess: for i in range(config['n_epoch']): lr_decay = config['lr_decay']**max(i + 1 - config['decay_epoch'], 0) train_model.assign_lr(sess, config['learning_rate'] * lr_decay) tf.logging.info('Iter %d Start, Learning_rate: %.4f' % (i, sess.run(train_model.lr))) loss = run_epoch(sess, train_model, train_data, is_training = True, \ t_model = test_model, src_vocab = src_vocab, tgt_vocab = tgt_vocab) tf.logging.info('Iter %d: training_loss: %.4f' % (i, np.power(2, loss)))
def gen(config, gen_file): # load Vocab src_vocab = data_reader.Vocab(vocab_limits=config['src_vocab_size']) src_vocab.load_metadata(config['metadata']['src']) config['src_vocab_size'] = src_vocab.vocab_size() tgt_vocab = data_reader.Vocab(vocab_limits=config['tgt_vocab_size']) tgt_vocab.load_metadata(config['metadata']['tgt']) config['tgt_vocab_size'] = tgt_vocab.vocab_size() tf.logging.info(config) # create model with tf.name_scope('Genearte'): with tf.variable_scope("Model", reuse=None): gen_model = model.Model(is_training=False, config=config, seq_length=1) sv = tf.train.Supervisor(logdir=config['logdir']) sess_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) sess_config.gpu_options.allow_growth = True sess_config.gpu_options.per_process_gpu_memory_fraction = 0.5 tf.logging.info('Start Sess') with sv.managed_session(config=sess_config) as sess: for line in open(gen_file): words = line.strip().split(' ') words.insert(0, "<s>") words.append("</s>") masks = [1] * len(words) if len(words) < config['src_length']: masks.extend([0] * (config['src_length'] - len(words))) words.extend(["</s>"] * (config['src_length'] - len(words))) src_vec = [src_vocab.char2id(c) for c in words] src_vec = np.array(src_vec) masks = np.array(masks) sample, scores = sample_run(sess, gen_model, src_vec, masks, bos_id=tgt_vocab.char2id("<s>"), eos_id=tgt_vocab.char2id("</s>")) out_words = [tgt_vocab.id2char(c) for c in sample[0]] tf.logging.info('Input: %s' % line.strip()) tf.logging.info('Output: %s' % (" ".join(out_words)))
def train(config): # load Vocab vocab = data_reader.Vocab(vocab_limits=config['vocab_size']) vocab.load_metadata(config['metadata']) config['vocab_size'] = vocab.vocab_size() tf.logging.info(config) initializer = tf.random_uniform_initializer(-config['init_scale'], config['init_scale']) # create models with tf.name_scope('Train'): opt, lr = optimizer.get_optimizer("sgd", config['learning_rate']) with tf.variable_scope("Model", reuse=None, initializer=initializer): train_model = model.Model(is_training = True, \ config = config, \ optimizer = opt, lr = lr) with tf.name_scope('Generate'): generate_config = copy.deepcopy(config) generate_config['batch_size'] = 1 generate_config['seq_length'] = 1 with tf.variable_scope("Model", reuse=True, initializer=initializer): gen_model = model.Model(is_training=False, config=generate_config) sv = tf.train.Supervisor(logdir=config['logdir']) sess_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) sess_config.gpu_options.allow_growth = True sess_config.gpu_options.per_process_gpu_memory_fraction = 0.5 tf.logging.info('Start Sess') with sv.managed_session(config=sess_config) as sess: for i in range(config['n_epoch']): lr_decay = config['lr_decay']**max(i + 1 - config['decay_epoch'], 0) train_model.assign_lr(sess, config['learning_rate'] * lr_decay) tf.logging.info('Iter %d Start, Learning_rate: %.4f' % (i, sess.run(train_model.lr))) costs, speed = run_epoch(sess, train_model, datapath=config['train_data'][0], config=config, is_training=True, gen_model=gen_model, vocab=vocab) tf.logging.info( 'Iter %d: training_loss:%.4f, speed %.4f words/sec' % (i, np.exp(costs), speed))
def convert_model(config): vocab = data_reader.Vocab(vocab_limits=config['vocab_size']) vocab.load_metadata(config['metadata']) config['vocab_size'] = vocab.vocab_size() tf.logging.info(config) with tf.name_scope('Eval'): config['batch_size'] = 100 config['seq_length'] = 100 with tf.variable_scope("Model", reuse=None): eval_model = model.Model(is_training=False, config=config) with tf.Session() as sess: saver = tf.train.Saver() ckpt = tf.train.latest_checkpoint(config['logdir']) tf.logging.info(ckpt) saver.restore(sess, ckpt) export_path = os.path.join(config['logdir'], 'exported/2') builder = tf.saved_model.builder.SavedModelBuilder(export_path) in_x = tf.saved_model.utils.build_tensor_info(eval_model.input_data) out_y = tf.saved_model.utils.build_tensor_info(eval_model.target_data) pred = tf.saved_model.utils.build_tensor_info(eval_model.target_probs) pred_signature = ( tf.saved_model.signature_def_utils.build_signature_def( inputs={ 'x': in_x, 'y': out_y }, outputs={'pred': pred}, method_name=tf.saved_model.signature_constants. PREDICT_METHOD_NAME)) legacy_init_op = tf.group(tf.tables_initializer(), name='legacy_init_op') builder.add_meta_graph_and_variables( sess, [tf.saved_model.tag_constants.SERVING], signature_def_map={ 'predict': pred_signature, tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: pred_signature, }, legacy_init_op=legacy_init_op) builder.save()
def gen(config): vocab = data_reader.Vocab(vocab_limits=config['vocab_size']) vocab.load_metadata(config['metadata']) config['vocab_size'] = vocab.vocab_size() tf.logging.info(config) with tf.name_scope('Generate'): config['batch_size'] = 1 config['seq_length'] = 1 with tf.variable_scope("Model", reuse=None): gen_model = model.Model(is_training=False, config=config) with tf.Session() as sess: saver = tf.train.Saver() ckpt = tf.train.latest_checkpoint(config['logdir']) tf.logging.info(ckpt) saver.restore(sess, ckpt) sample(sess, model=gen_model, vocab=vocab)
def rename_model(config): vocab = data_reader.Vocab(vocab_limits=config['vocab_size']) vocab.load_metadata(config['metadata']) config['vocab_size'] = vocab.vocab_size() tf.logging.info(config) config['batch_size'] = 100 config['seq_length'] = 100 with tf.variable_scope("Model", reuse=None): eval_model = model.Model(is_training=False, config=config) with tf.Session() as sess: saver = tf.train.Saver() ckpt = tf.train.latest_checkpoint(config['logdir']) tf.logging.info(ckpt) saver.restore(sess, ckpt) tf.train.write_graph(sess.graph_def, config['logdir'], "nn_model.pbtxt", as_text=True)
def eval(config): vocab = data_reader.Vocab(vocab_limits=config['vocab_size']) vocab.load_metadata(config['metadata']) config['vocab_size'] = vocab.vocab_size() tf.logging.info(config) batch_size = 100 with tf.name_scope('Eval'): config['batch_size'] = batch_size config['seq_length'] = 1 with tf.variable_scope("Model", reuse=None): eval_model = model.Model(is_training=False, config=config) with tf.Session() as sess: saver = tf.train.Saver() ckpt = tf.train.latest_checkpoint(config['logdir']) tf.logging.info(ckpt) saver.restore(sess, ckpt) out = open('%s.out3' % FLAGS.evalfile, 'w') index = None max_len = 0 cnt = 0 sens = [[]] * batch_size base = [None] * batch_size lens = [0] * batch_size for line in open(FLAGS.evalfile): vec = line.strip().split() now_idx = vec[0] if index != now_idx: for i in range(batch_size): sens[i].extend([vocab.char2id("</s>")] * (max_len - len(sens[i]))) scores = sen_score(sess, input=sens, lens=lens, model=eval_model) for i in range(batch_size): if base[i] == None: break out.write('%s\t%d\t%.4f\n' % (base[i], lens[i], scores[i])) max_len = 0 cnt = 0 sens = [[]] * batch_size base = [None] * batch_size lens = [0] * batch_size index = now_idx sen = list(''.join(vec[1:]).decode('utf-8')) sen.insert(0, "<s>") sen.append("</s>") base[cnt] = line.strip() sens[cnt] = [vocab.char2id(c) for c in sen] lens[cnt] = len(sens[cnt]) max_len = max(len(sens[cnt]), max_len) cnt += 1 for i in range(batch_size): sens[i].extend([vocab.char2id("</s>")] * (max_len - len(sens[i]))) scores = sen_score(sess, input=sens, lens=lens, model=eval_model) for i in range(batch_size): if base[i] == None: break out.write('%s\t%d\t%.4f\n' % (base[i], lens[i], scores[i]))
def cluster_train(config): # init for cluster cluster_conf = config["cluster_conf"] cluster = tf.train.ClusterSpec(cluster_conf) server = tf.train.Server(cluster, job_name=FLAGS.job_name, task_index=FLAGS.task_id) n_ps = len(cluster_conf['ps']) n_workers = len(cluster_conf['worker']) if FLAGS.job_name == "ps": server.join() is_chief = (FLAGS.task_id == 0) # load Vocab vocab = data_reader.Vocab(vocab_limits=config['vocab_size']) vocab.load_metadata(config['metadata']) config['vocab_size'] = vocab.vocab_size() tf.logging.info(config) # load Data train_data = data_reader.DataReader(config['train_data'][FLAGS.task_id], vocab=vocab, batch_size=config['batch_size'], seq_length=config['seq_length']) initializer = tf.random_uniform_initializer(-config['init_scale'], config['init_scale']) # create models with tf.device( tf.train.replica_device_setter( worker_device="/job:worker/task:%d" % FLAGS.task_id, cluster=cluster)): with tf.name_scope('Train'): opt, lr = optimizer.get_optimizer("sgd", config['learning_rate']) sync_opt = tf.train.SyncReplicasOptimizer( opt, replicas_to_aggregate=n_workers * config['sync_freq'], total_num_replicas=n_workers) with tf.variable_scope("Model", reuse=None, initializer=initializer): train_model = model.Model(is_training = True, \ config = config, \ optimizer = sync_opt, lr = lr) with tf.name_scope('Generate'): generate_config = copy.deepcopy(config) generate_config['batch_size'] = 1 generate_config['seq_length'] = 1 with tf.variable_scope("Model", reuse=True, initializer=initializer): gen_model = model.Model(is_training=False, config=generate_config) chief_queue_runner = train_model.optimizer.get_chief_queue_runner() init_tokens_op = train_model.optimizer.get_init_tokens_op() sv = tf.train.Supervisor(is_chief=is_chief, logdir=config['logdir']) sess_config = tf.ConfigProto( allow_soft_placement=True, log_device_placement=True, device_filters=["/job:ps", "/job:worker/task:%d" % FLAGS.task_id]) sess_config.gpu_options.allow_growth = True sess_config.gpu_options.per_process_gpu_memory_fraction = 0.5 tf.logging.info('Start Sess') with sv.prepare_or_wait_for_session(server.target, config=sess_config) as sess: if is_chief: sv.start_queue_runners(sess, [chief_queue_runner]) sess.run(init_tokens_op) for i in range(config['n_epoch']): lr_decay = config['lr_decay']**max(i + 1 - config['decay_epoch'], 0) train_model.assign_lr(sess, config['learning_rate'] * lr_decay) tf.logging.info('Iter %d Start, Learning_rate: %.4f' % (i, sess.run(train_model.lr))) costs, speed = run_epoch(sess, train_model, train_data, is_training=True, gen_model=gen_model, vocab=vocab) tf.logging.info( 'Iter %d: training_loss:%.4f, speed %.4f words/sec' % (i, np.exp(costs), speed))
def eval(infile, outfile, config): # init vocab vocab = data_reader.Vocab(vocab_limits = config['vocab_size']) vocab.load_metadata(config['metadata']) config['vocab_size'] = vocab.vocab_size() tf.logging.info(config) seq_length = config['seq_length'] out = open(outfile, 'w') index = None lens = [] in_ids = [] out_ids = [] base = [] total_timer = time_control() send_timer = time_control() total_timer.start() for line in open(infile): vec = line.strip().split() if index != vec[0]: if index != None: send_timer.start() scores = client_batch(input = in_ids, output = out_ids, lens = lens, batch_size = len(in_ids), seq_length = seq_length) send_timer.stop() for i in range(len(in_ids)): if base[i] == None: break out.write('%s\t%d\t%.4f\n' % (base[i], lens[i], scores[i])) index = vec[0] lens = [] in_ids = [] out_ids = [] base = [] sen = list(''.join(vec[1:]).decode('utf-8')) sen.insert(0, "<s>") sen.append("</s>") lens.append(len(sen)) sen.extend(["</s>"] * (seq_length - len(sen))) base.append(line.strip()) in_ids.append([vocab.char2id(c) for c in sen]) out_ids.append(in_ids[-1][1:]) out_ids[-1].append(vocab.char2id("</s>")) send_timer.start() scores = client_batch(input = in_ids, output = out_ids, lens = lens, batch_size = len(in_ids), seq_length = seq_length) send_timer.stop() for i in range(len(in_ids)): if base[i] == None: break out.write('%s\t%d\t%.4f\n' % (base[i], lens[i], scores[i])) total_timer.stop() print 'total_timer: %.2f' % total_timer.get_total_time() print 'send_timer: %.2f' % send_timer.get_total_time() print 'client_timer: %.2f' % client_timer.get_total_time()