test_label_preds = {}
best_scores = {}
if mode == 'train':
if not os.path.exists(model_dir):
os.mkdir(model_dir)
for target_label in labels:
model_file = os.path.join(model_dir,
'checkpoint_{}.mdl'.format(target_label))
model = models[target_label]
optimizer = optimizers[target_label]
# TODO: combine init_dataset() and shuffle_dataset()
dev_set.init_dataset(target_label)
test_set.init_dataset(target_label)
(
dev_tids, dev_tokens, dev_labels, dev_lens
) = dev_set.get_dataset(max_seq_len, volatile=True, gpu=use_gpu)
(
test_tids, test_tokens, test_labels, test_lens
) = test_set.get_dataset(max_seq_len, volatile=True, gpu=use_gpu)
best_dev_fscore = 0.0
best_test_scores = None
for epoch in range(max_epoch):
epoch_start_time = current_time()
epoch_loss = 0.0
train_set.shuffle_dataset(target_label, balance=True)
batch_num = train_set.batch_num(batch_size)
for batch_idx in range(batch_num):
optimizer.zero_grad()
(
batch_tids, batch_tokens, batch_labels, batch_lens
def pretrain(opts):
# dataset and iterator
dataset = Dataset(opts.rawdata_path)
dataset_train = dataset.get_dataset(opts)
iterator = tf.data.Iterator.from_structure(dataset_train.output_types,
dataset_train.output_shapes)
v_a, v_b, label = iterator.get_next()
# network
outputs, training = get_network(tf.concat((v_a, v_b), axis=0), opts)
# save and load
saver = tf.train.Saver(var_list=tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES, scope=opts.network))
# loss
loss, accuracy = cross_entropy_loss(outputs, label, opts)
# summary
writer_train = tf.summary.FileWriter(
os.path.join(opts.output_path, opts.time, 'logs'),
tf.get_default_graph())
summary_op = tf.summary.merge_all()
# optimizer
global_step = tf.Variable(0, trainable=False)
lr = tf.train.exponential_decay(opts.lr,
global_step,
dataset_train.length * 67,
0.1,
staircase=True)
train_op = tf.train.AdamOptimizer(learning_rate=lr).minimize(
loss, global_step=global_step, colocate_gradients_with_ops=True)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
train_op = tf.group(update_ops + [train_op])
# main loop
with tf.Session(config=tf.ConfigProto(log_device_placement=False,
allow_soft_placement=True)) as sess:
sess.run(tf.global_variables_initializer())
print('training loop start')
start_train = time.clock()
for epoch in range(1, opts.epochs + 1):
print('epoch: %d' % epoch)
start_ep = time.clock()
# train
print('training')
sess.run(iterator.make_initializer(dataset_train))
while True:
try:
summary_train, _ = sess.run([summary_op, train_op],
feed_dict={training: True})
writer_train.add_summary(
summary_train, tf.train.global_step(sess, global_step))
except tf.errors.OutOfRangeError:
break
print('step: %d' % tf.train.global_step(sess, global_step))
# save model
if epoch % opts.save_freq == 0 or epoch == opts.epochs:
print('save model')
saver.save(
sess,
os.path.join(opts.output_path, opts.time,
'pretrainmodel.ckpt'))
print("epoch end, elapsed time: %ds, total time: %ds" %
(time.clock() - start_train, time.clock() - start_ep))
print('training loop end')
writer_train.close()
