def main(_):
config = trfnce.Config(data)
config.structure_type = 'mix'
config.embedding_dim = 128
config.cnn_filters = [(i, 128) for i in range(1, 5)]
config.cnn_hidden = 128
config.cnn_layers = 1
config.cnn_skip_connection = False
config.cnn_residual = True
config.cnn_activation = 'relu'
config.rnn_hidden_layers = 1
config.rnn_hidden_size = 128
config.attention = True
config.batch_size = 100
config.noise_factor = 2
config.noise_sampler = 2
config.init_weight = 0.1
config.optimize_method = ['sgd', 'sgd']
config.lr_param = trfbase.LearningRateEpochDelay(1e-2, 0.5)
config.lr_zeta = trfbase.LearningRateEpochDelay(1e-2, 0.5)
config.max_epoch = 10
# config.dropout = 0.75
# config.init_zeta = config.get_initial_logz(0)
config.update_zeta = True
config.write_dbg = False
config.pprint()
q_config = run_lstmlm.small_config(data)
# q_config = None
name = create_name(config, q_config)
logdir = 'trf_nce/' + name
wb.mkdir(logdir, is_recreate=True)
sys.stdout = wb.std_log(os.path.join(logdir, 'trf.log'))
print(logdir)
data.write_vocab(logdir + '/vocab.txt')
data.write_data(data.datas[1], logdir + '/valid.id')
data.write_data(data.datas[2], logdir + '/test.id')
# wb.rmdir(logdirs)
with tf.Graph().as_default():
if q_config is None:
m = trfnce.TRF(config, data, logdir=logdir, device='/gpu:0')
else:
m = trfnce.TRF(config,
data,
logdir=logdir,
device='/gpu:1',
q_model=lstmlm.LM(q_config, device='/gpu:1'))
# noise_lstm = lstmlm.LM(run_lstmlm_withBegToken.small_config(data), device='/gpu:1')
# m.lstm = noise_lstm
sv = tf.train.Supervisor(logdir=os.path.join(logdir, 'logs'),
global_step=m.train_net.global_step)
sv.summary_writer.add_graph(
tf.get_default_graph()) # write the graph to logs
session_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
session_config.gpu_options.allow_growth = True
with sv.managed_session(config=session_config) as session:
m.set_session(session)
if m.q_model is not None:
print('load lstmlm for q model')
m.q_model.restore(
session, './lstm/' + run_lstmlm.create_name(q_config) +
'/model.ckpt')
m.train(
sv,
session,
print_per_epoch=0.1,
operation=task.Ops(m),
# nbest=nbest,
# lmscale_vec=np.linspace(1, 20, 20)
)
def main(_):
data = reader.Data().load_raw_data(reader.ptb_raw_dir(),
add_beg_token='<s>',
add_end_token='</s>',
add_unknwon_token='<unk>')
nbest = reader.NBest(*reader.wsj0_nbest())
nbest_list = data.load_data(nbest.nbest, is_nbest=True)
print('nbest list info=', wb.TxtInfo(nbest.nbest))
config = trfnce.Config(data)
config.structure_type = 'rnn'
config.embedding_dim = 200
config.rnn_hidden_layers = 2
config.rnn_hidden_size = 200
config.batch_size = 20
config.noise_factor = 100
config.noise_sampler = 2
config.init_weight = 0.1
config.lr_param = trfbase.LearningRateTime(1e-3)
config.max_epoch = 100
# config.dropout = 0.75
# config.init_zeta = config.get_initial_logz(20)
config.update_zeta = False
config.write_dbg = False
config.pprint()
name = create_name(config)
logdir = 'trf_nce/' + name
wb.mkdir(logdir, is_recreate=True)
sys.stdout = wb.std_log(os.path.join(logdir, 'trf.log'))
print(logdir)
data.write_vocab(logdir + '/vocab.txt')
data.write_data(data.datas[1], logdir + '/valid.id')
data.write_data(data.datas[2], logdir + '/test.id')
data.write_data(nbest_list, logdir + '/nbest.id')
# wb.rmdir(logdirs)
with tf.Graph().as_default():
m = trfnce.TRF(config, data, logdir=logdir, device='/gpu:0')
# noise_lstm = lstmlm.LM(run_lstmlm_withBegToken.small_config(data), device='/gpu:1')
# m.lstm = noise_lstm
sv = tf.train.Supervisor(logdir=os.path.join(logdir, 'logs'),
global_step=m.train_net.global_step)
sv.summary_writer.add_graph(
tf.get_default_graph()) # write the graph to logs
session_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
session_config.gpu_options.allow_growth = True
with sv.managed_session(config=session_config) as session:
m.set_session(session)
# print('load lstmlm for noise generator')
# noise_lstm.restore(session,
# './lstm/' + run_lstmlm_withBegToken.create_name(noise_lstm.config) + '/model.ckpt')
m.train(sv,
session,
print_per_epoch=0.1,
nbest=nbest,
nbest_list=nbest_list)
def main(_):
config = trfnce.Config(data)
config.structure_type = 'cnn'
config.embedding_dim = 200
config.cnn_filters = [(i, 100) for i in range(1, 11)]
config.cnn_width = 3
config.cnn_layers = 3
config.cnn_hidden = 200
config.rnn_hidden_layers = 2
config.rnn_hidden_size = 200
config.rnn_predict = True
config.batch_size = 10
config.noise_factor = 10
config.noise_sampler = 'lstm:lstm/lstm_e200_h200x2/model.ckpt'
config.init_weight = 0.1
config.optimize_method = ['adam', 'adam']
config.lr_param = trfbase.LearningRateEpochDelay(0.001)
config.lr_zeta = trfbase.LearningRateEpochDelay(0.01)
config.max_epoch = 100
# config.dropout = 0.75
# config.init_zeta = config.get_initial_logz(20)
config.update_zeta = True
config.write_dbg = False
config.print()
# q_config = run_lstmlm.small_config(data)
q_config = None
name = create_name(config, q_config)
logdir = 'trf_nce/' + name
wb.mkdir(logdir, is_recreate=True)
sys.stdout = wb.std_log(os.path.join(logdir, 'trf.log'))
print(logdir)
data.write_vocab(logdir + '/vocab.txt')
data.write_data(data.datas[1], logdir + '/valid.id')
data.write_data(data.datas[2], logdir + '/test.id')
# wb.rmdir(logdirs)
with tf.Graph().as_default():
if q_config is None:
m = trfnce.TRF(config, data, logdir=logdir, device='/gpu:0')
else:
m = trfnce.TRF(config,
data,
logdir=logdir,
device='/gpu:0',
q_model=lstmlm.LM(q_config, device='/gpu:0'))
# s1 = trfnce.NoiseSamplerNgram(config, data, 2)
# s2 = trfnce.NoiseSamplerLSTMEval(config, data, config.noise_sampler.split(':')[-1])
sv = tf.train.Supervisor(logdir=os.path.join(logdir, 'logs'),
global_step=m.train_net.global_step)
sv.summary_writer.add_graph(
tf.get_default_graph()) # write the graph to logs
session_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
session_config.gpu_options.allow_growth = True
with sv.managed_session(config=session_config) as session:
with session.as_default():
if m.q_model is not None:
print('load lstmlm for q model')
m.q_model.restore(
session, './lstm/' + run_lstmlm.create_name(q_config) +
'/model.ckpt')
m.train(
sv,
session,
print_per_epoch=0.1,
operation=task.Ops(m),
)