def train():
loss = network.compute_loss(correct_X, wrong_X)
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(init_learning_rate,
global_step,
steps_per_epoch,
0.794,
staircase=True)
optim = tf.train.AdamOptimizer(init_learning_rate)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_step = optim.minimize(loss, global_step=global_step)
saver = tf.train.Saver(save_relative_paths=True)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(num_steps):
for start, end in zip(
range(0, num_train_samples, BATCH_SIZE),
range(BATCH_SIZE, num_train_samples + 1,
BATCH_SIZE)): # batches
feed_dict = {
correct_X: corr_fts_train[start:end],
wrong_X: wrong_fts_train[start:end]
}
_, loss_Val = sess.run([train_step, loss], feed_dict=feed_dict)
print('Epoch: %d Step: %d Loss: %f' %
(i // steps_per_epoch, i, loss_Val))
if i % steps_per_epoch == 0 and i > 0:
print('Saving checkpoint at step %d' % i)
import network
import reader
import util
import tensorflow as tf
import time
import numpy as np
from scipy import misc
r = reader.Reader('./data/train/', 'annotations.pkl', 32)
l_rate = 1e-1
inflow, kmaps, amaps = network.vanilla()
ref_kmap, ref_amap, k_loss, a_loss, loss = network.compute_loss(kmaps, amaps)
train_step = tf.train.AdagradOptimizer(l_rate).minimize(loss)
sess = tf.Session()
init_op = tf.global_variables_initializer()
sess.run(init_op)
for i in range(1000000):
tic = time.time()
img, kmap, amap = r.next_batch()
_, batch_k_loss, batch_a_loss = \
sess.run([train_step, k_loss, a_loss], feed_dict={inflow:img, ref_kmap:kmap, ref_amap:amap})
toc = time.time()
interval = (toc - tic) * 1000
r.index = 0 # train on the first batch for sanity check
print('Iter %d, k loss %g, a loss %g, timecost %g ms' % \
(i, batch_k_loss, batch_a_loss, interval))
model_path = './model/' + model_name + '/'
sess = tf.Session()
l_rate = float(sys.argv[2])
######################################################################
r = reader.RectReader('./data/train/',
'annotations_large.pkl',
16,
h=368,
w=208)
# r = reader.RectReader('./data/train/', 'annotations_large.pkl', 16)
# r = reader.RectReader('./data/validate/', 'val_anno.pkl', 16)
inflow, kmaps, amaps = network.a13()
k_ref, a_ref, k_loss, a_loss, loss = network.compute_loss(kmaps, amaps, 0.5)
######################################################################
with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
train_step = tf.train.AdagradOptimizer(l_rate).minimize(loss)
depth = len(kmaps)
tf.summary.scalar('total loss', loss)
tf.summary.scalar('kmap loss', k_loss)
tf.summary.scalar('amap loss', a_loss)
tf.summary.scalar('learning rate', l_rate)
tf.summary.scalar('avg k loss', k_loss / depth)
tf.summary.scalar('avg a loss', a_loss / depth)
tf.summary.scalar('avg loss', loss / depth)
import time
import numpy as np
import scipy.misc as misc
import sys
import os
model_name = sys.argv[1]
model_path = './model/' + model_name + '/'
l_rate = float(sys.argv[2])
r = reader.DAReader('./data/train/', 'annotations_new.pkl', 16)
sess = tf.Session()
inflow, dmaps, amaps = network.a3()
d_ref, a_ref, d_loss, a_loss, loss = network.compute_loss(dmaps, amaps, 0.5)
train_step = tf.train.AdagradOptimizer(l_rate).minimize(loss)
tf.summary.scalar('total loss', loss)
tf.summary.scalar('dmap loss', d_loss)
tf.summary.scalar('amap loss', a_loss)
tf.summary.scalar('learning rate', l_rate)
merged = tf.summary.merge_all()
writer = tf.summary.FileWriter(model_path, sess.graph)
global_step = tf.Variable(0, name='global_step', trainable=False)
one_step_op = global_step.assign_add(1)
init_op = tf.global_variables_initializer()
sess.run(init_op)
params['batch_size'],
device,
negative_table,
n_negative=5)
output_dir = pathlib.Path(params['output_dir'])
output_dir.mkdir(exist_ok=True, parents=True)
vocab.save(output_dir.as_posix())
print(model)
optimizer = optim.Adam(model.parameters())
start_at = time.time()
accum_loss = 0.0
for batch_id, batch in enumerate(dataloader):
loss = compute_loss(model, batch, optimizer=optimizer, is_train=True)
accum_loss += loss
if batch_id % 1000 == 0:
end_at = time.time()
elapsed = end_at - start_at
print(
f'Batch #{batch_id}, Loss:{accum_loss:.2f}, Elapsed:{elapsed:.2f}'
)
model_fname = params['algorithm'] + f'{batch_id:08d}.pth'
model_fpath = pathlib.PurePath(output_dir, model_fname).as_posix()
embedding_matrix = model.embedding.weight.data.cpu().numpy()
torch.save(embedding_matrix, model_fpath)
print(f'saved model into {model_fpath}')
start_at = end_at