def sg_init(sess):
r""" Initializes session variables.
Args:
sess: Session to initialize.
"""
# initialize variables
sess.run(tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer()))
def genIt(name='bird'):
z = tf.random_normal((batch_size, rand_dim))
gen = generator(z)
with tf.Session() as sess:
sess.run(
tf.group(tf.global_variables_initializer(),
tf.sg_phase().assign(False)))
tf.sg_restore(sess,
tf.train.latest_checkpoint('asset/train/gan'),
category=['generator', 'discriminator'])
fake_features = []
for i in range(100):
fake_features.append(sess.run(gen))
np.save('../data/fake_' + name + '_negative.npy',
np.array(fake_features).reshape((-1, 4096)))
def test(tfname, weightPaths, steps=100000, Var=["NNReg"], lll=2000):
tf.Graph()
x, y = read_from_tfrecords(tfname, ["source", "target"], 10,
[[1070, 3], [1070, 3]])
global_step = tf.Variable(1, trainable=False, name='global_step')
print(x.shape, y.shape)
x = np.loadtxt('EM.txt', dtype='float32') / 1500
y = np.loadtxt('FM.txt', dtype='float32')[:, :100] / 1500
x = tf.convert_to_tensor(np.expand_dims(np.rollaxis(x, axis=0), axis=0))
y = tf.convert_to_tensor(np.expand_dims(np.rollaxis(y, axis=0), axis=0))
print(x.shape, y.shape)
yp = Net(x, x, y) + x
tmp_var_list = {}
for j in Var:
for i in tf.global_variables():
if i.name.startswith(j):
tmp_var_list[i.name[:-2]] = i
saver = tf.train.Saver(tmp_var_list)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
path = weightPaths + "model.ckpt-{}".format(steps)
Sour = []
Targ = []
Trans_S = []
with tf.Session() as sess:
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
saver.restore(sess, path)
for i in tqdm.tqdm(range(lll)):
S, T, TS = sess.run([x, y, yp])
Sour.append(S)
Targ.append(T)
Trans_S.append(TS)
coord.request_stop()
coord.join(threads)
return Sour, Targ, Trans_S
def testIt():
data = raw
positive = np.array(data.label_train) > 0
x = tf.placeholder(tf.float32, [None, 4096])
y = tf.placeholder(tf.float32)
disc_real = discriminator(x)
accuracy = tf.reduce_mean(
tf.cast(tf.equal(tf.cast(disc_real > 0.5, "float"), y), tf.float32))
np.set_printoptions(precision=3, suppress=True)
with tf.Session() as sess:
sess.run(
tf.group(tf.global_variables_initializer(),
tf.sg_phase().assign(False)))
# restore parameters
tf.sg_restore(sess,
tf.train.latest_checkpoint('asset/train/gan'),
category=['generator', 'discriminator'])
ans = sess.run(disc_real, feed_dict={x: np.array(data.test)})
print np.sum(ans > 0.5)
np.save('dm_bird.npy', ans)
def test(tfname, weightPaths, steps=100000, Var=["NNReg"], lll=2000):
tf.Graph()
x, y = read_from_tfrecords(tfname, ["source", "target"], 10,
[[91, 2], [91, 2]])
global_step = tf.Variable(1, trainable=False, name='global_step')
yp = Net(x, x, y) + x
tmp_var_list = {}
for j in Var:
for i in tf.global_variables():
if i.name.startswith(j):
tmp_var_list[i.name[:-2]] = i
saver = tf.train.Saver(tmp_var_list)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
path = weightPaths + "model.ckpt-{}".format(steps)
Sour = []
Targ = []
Trans_S = []
with tf.Session() as sess:
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
saver.restore(sess, path)
for i in tqdm.tqdm(range(lll)):
S, T, TS = sess.run([x, y, yp])
Sour.append(S)
Targ.append(T)
Trans_S.append(TS)
coord.request_stop()
coord.join(threads)
return Sour, Targ, Trans_S
def generate():
dev = '/cpu:0'
with tf.device(dev):
mydir = 'tfrc150char_wrd0704'
files = [f for f in listdir(mydir) if isfile(join(mydir, f))]
tfrecords_filename = []
tfrecords_filename = [join(mydir, 'short_infer3.tfrecords')
] #[join(mydir, f) for f in tfrecords_filename]
tfrecords_filename_inf = [join(mydir, '11_3.tfrecords')]
print(tfrecords_filename)
filename_queue = tf.train.string_input_producer(tfrecords_filename,
num_epochs=num_epochs,
shuffle=True,
capacity=1)
infer_queue = tf.train.string_input_producer(tfrecords_filename_inf,
num_epochs=num_epochs,
shuffle=True,
capacity=1)
optim = tf.train.AdamOptimizer(learning_rate=0.0001,
beta1=0.9,
beta2=0.99)
# Calculate the gradients for each model tower.
tower_grads = []
reuse_vars = False
with tf.variable_scope("dec_lstm") as scp:
dec_cell = BasicLSTMCell2(Hp.w_emb_size,
Hp.rnn_hd,
state_is_tuple=True)
with tf.variable_scope("contx_lstm") as scp:
cell = BasicLSTMCell2(Hp.hd, Hp.rnn_hd, state_is_tuple=True)
rnn_cell = tf.contrib.rnn.DropoutWrapper(
cell,
input_keep_prob=Hp.keep_prob,
output_keep_prob=Hp.keep_prob)
(words, chars) = read_and_decode(filename_queue,
Hp.batch_size * Hp.num_gpus)
words_splits = tf.split(axis=0,
num_or_size_splits=Hp.num_gpus,
value=words)
chars_splits = tf.split(axis=0,
num_or_size_splits=Hp.num_gpus,
value=chars)
word_emb = np.loadtxt("glove300d_0704.txt")
Hp.word_vs = word_emb.shape[0]
# --------------------------------------------------------------------------------
with tf.name_scope('%s_%d' % ("tower", 0)) as scope:
rnn_state = tower_infer_enc(chars_splits[0],
scope,
rnn_cell,
dec_cell,
word_emb,
out_reuse_vars=False,
dev='/cpu:0')
chars_pl = tf.placeholder(tf.int32, shape=(None, Hp.c_maxlen))
rnn_state_pl1 = [
tf.placeholder(tf.float32, shape=(None, Hp.rnn_hd)),
tf.placeholder(tf.float32, shape=(None, Hp.rnn_hd))
]
rnn_state_pl = tf.contrib.rnn.LSTMStateTuple(
rnn_state_pl1[0], rnn_state_pl1[1])
final_ids, rnn_state_dec = tower_infer_dec(chars_pl,
scope,
rnn_cell,
dec_cell,
word_emb,
rnn_state_pl,
out_reuse_vars=False,
dev='/cpu:0')
# --------------------------------------------------------------------------------
saver = tf.train.Saver(tf.trainable_variables())
session_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
session_config.gpu_options.per_process_gpu_memory_fraction = 0.94
session_config.gpu_options.allow_growth = False
restore_dir = 'tnsrbrd/hin17d08m_1313g2' # lec30d07m_1634g2 lec04d07m_2006g2 lec28d07m_1221g2 lec31d07m_1548g2
csv_file = join(restore_dir, time.strftime("hin%dd%mm_%H%M.csv"))
csv_f = open(csv_file, 'a')
csv_writer = csv.writer(csv_f)
with tf.Session(config=session_config) as sess:
sess.run(
tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer()))
tf.train.start_queue_runners(sess=sess)
saver.restore(sess,
tf.train.latest_checkpoint(
join(restore_dir,
'last_chpt'))) # lec04d07m_2006g2
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for ep in range(num_epochs):
tf.sg_set_infer(sess)
rnn_state_val, w_txt, ch_txt = sess.run(
[rnn_state, words_splits[0], chars_splits[0]],
feed_dict={Hp.keep_prob: 1.0})
predictions = [] #[w_txt[:,2,:]]
for idx in range(3):
char_inpt = word2char_ids(
ids_val) if idx != 0 else ch_txt[:, 2, :]
ids_val, rnn_state_val = sess.run(
[final_ids, rnn_state_dec],
feed_dict={
Hp.keep_prob: 1.0,
rnn_state_pl1[0]: rnn_state_val[0],
rnn_state_pl1[1]: rnn_state_val[1],
chars_pl: char_inpt
})
temp = np.zeros((Hp.batch_size, Hp.w_maxlen))
for b in range(Hp.batch_size):
stop_ind = np.where(ids_val[b] == 2)[0]
if stop_ind.size > 0:
stop_ind = stop_ind[0]
ids_val[b, stop_ind +
1:] = ids_val[b, stop_ind + 1:] * 0
temp[:, :ids_val.shape[1]] = ids_val
predictions.append(temp)
# predictions are decode_sent x b x w_maxlen
predictions = np.array(predictions)
in_batches = [w_txt[b, :, :] for b in range(Hp.batch_size)]
res_batches = [
predictions[:, b, :] for b in range(Hp.batch_size)
]
for b in range(Hp.batch_size):
in_paragraph = idxword2txt(in_batches[b])
print("\n INPUT SAMPLE \n")
print(in_paragraph)
res_paragraph = idxword2txt(res_batches[b])
print("\n RESULTS \n")
print(res_paragraph)
csv_writer.writerow([
" ".join(in_paragraph[:3]), " ".join(in_paragraph[3:]),
" ".join(res_paragraph)
])
csv_f.close()
def sg_optim(loss, **kwargs):
r"""Applies gradients to variables.
Args:
loss: A 0-D `Tensor` containing the value to minimize. list of 0-D tensor for Multiple GPU
kwargs:
optim: A name for optimizer. 'MaxProp' (default), 'AdaMax', 'Adam', 'RMSProp' or 'sgd'.
lr: A Python Scalar (optional). Learning rate. Default is .001.
beta1: A Python Scalar (optional). Default is .9.
beta2: A Python Scalar (optional). Default is .99.
momentum : A Python Scalar for RMSProp optimizer (optional). Default is 0.
category: A string or string list. Specifies the variables that should be trained (optional).
Only if the name of a trainable variable starts with `category`, it's value is updated.
Default is '', which means all trainable variables are updated.
"""
opt = tf.sg_opt(kwargs)
# default training options
opt += tf.sg_opt(optim='MaxProp', lr=0.001, beta1=0.9, beta2=0.99, momentum=0., category='')
# select optimizer
if opt.optim == 'MaxProp':
optim = tf.sg_optimize.MaxPropOptimizer(learning_rate=opt.lr, beta2=opt.beta2)
elif opt.optim == 'AdaMax':
optim = tf.sg_optimize.AdaMaxOptimizer(learning_rate=opt.lr, beta1=opt.beta1, beta2=opt.beta2)
elif opt.optim == 'Adam':
optim = tf.train.AdamOptimizer(learning_rate=opt.lr, beta1=opt.beta1, beta2=opt.beta2)
elif opt.optim == 'RMSProp':
optim = tf.train.RMSPropOptimizer(learning_rate=opt.lr, decay=opt.beta1, momentum=opt.momentum)
else:
optim = tf.train.GradientDescentOptimizer(learning_rate=opt.lr)
# get trainable variables
if isinstance(opt.category, (tuple, list)):
var_list = []
for cat in opt.category:
var_list.extend([t for t in tf.trainable_variables() if t.name.startswith(cat)])
else:
var_list = [t for t in tf.trainable_variables() if t.name.startswith(opt.category)]
#
# calc gradient
#
# multiple GPUs case
if isinstance(loss, (tuple, list)):
gradients = []
# loop for each GPU tower
for i, loss_ in enumerate(loss):
# specify device
with tf.device('/gpu:%d' % i):
# give new scope only to operation
with tf.name_scope('gpu_%d' % i):
# add gradient calculation operation for each GPU tower
gradients.append(tf.gradients(loss_, var_list))
# averaging gradient
gradient = []
for grad in zip(*gradients):
gradient.append(tf.add_n(grad) / len(loss))
# single GPU case
else:
gradient = tf.gradients(loss, var_list)
gradient, _ = tf.clip_by_global_norm(gradient, opt.clip_grad_norm)
# gradient update op
with tf.device('/gpu:0'):
grad_var = [(g, v) for g, v in zip(gradient, var_list)]
grad_op = optim.apply_gradients(grad_var, global_step=tf.sg_global_step())
# add summary using last tower value
for g, v in grad_var:
# exclude batch normal statics
if 'mean' not in v.name and 'variance' not in v.name \
and 'beta' not in v.name and 'gamma' not in v.name:
tf.sg_summary_gradient(v, g)
# extra update ops within category ( for example, batch normal running stat update )
if isinstance(opt.category, (tuple, list)):
update_op = []
for cat in opt.category:
update_op.extend([t for t in tf.get_collection(tf.GraphKeys.UPDATE_OPS) if t.name.startswith(cat)])
else:
update_op = [t for t in tf.get_collection(tf.GraphKeys.UPDATE_OPS) if t.name.startswith(opt.category)]
return tf.group(*([grad_op] + update_op))
def wrapper(**kwargs):
r""" Manages arguments of `tf.sg_opt`.
Args:
**kwargs:
lr: A Python Scalar (optional). Learning rate. Default is .001.
save_dir: A string. The root path to which checkpoint and log files are saved.
Default is `asset/train`.
max_ep: A positive integer. Maximum number of epochs. Default is 1000.
ep_size: A positive integer. Number of Total batches in an epoch.
For proper display of log. Default is 1e5.
save_interval: A Python scalar. The interval of saving checkpoint files.
By default, for every 600 seconds, a checkpoint file is written.
log_interval: A Python scalar. The interval of recoding logs.
By default, for every 60 seconds, logging is executed.
max_keep: A positive integer. Maximum number of recent checkpoints to keep. Default is 5.
keep_interval: A Python scalar. How often to keep checkpoints. Default is 1 hour.
eval_metric: A list of tensors containing the value to evaluate. Default is [].
tqdm: Boolean. If True (Default), progress bars are shown. If False, a series of loss
will be shown on the console.
"""
opt = tf.sg_opt(kwargs)
# default training options
opt += tf.sg_opt(lr=0.001,
save_dir='asset/train',
max_ep=1000, ep_size=100000,
save_interval=600, log_interval=60,
eval_metric=[],
max_keep=5, keep_interval=1,
tqdm=True)
# training epoch and loss
epoch, loss = -1, None
# checkpoint saver
saver = tf.train.Saver(max_to_keep=opt.max_keep,
keep_checkpoint_every_n_hours=opt.keep_interval)
# add evaluation summary
for m in opt.eval_metric:
tf.sg_summary_metric(m)
# summary writer
log_dir = opt.save_dir + '/run-%02d%02d-%02d%02d' % tuple(tf.time.localtime(tf.time.time()))[1:5]
summary_writer = tf.summary.FileWriter(log_dir)
# console logging function
def console_log(sess_):
if epoch >= 0:
tf.sg_info('\tEpoch[%03d:gs=%d] - loss = %s' %
(epoch, sess_.run(tf.sg_global_step()),
('NA' if loss is None else '%8.6f' % loss)))
local_init_op = tf.group(tf.sg_phase().assign(True), tf.tables_initializer(), tf.local_variables_initializer())
# create supervisor
sv = tf.train.Supervisor(logdir=opt.save_dir,
saver=saver,
save_model_secs=opt.save_interval,
summary_writer=summary_writer,
save_summaries_secs=opt.log_interval,
global_step=tf.sg_global_step(),
local_init_op=local_init_op)
# create session
with sv.managed_session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
# console logging loop
if not opt.tqdm:
sv.loop(opt.log_interval, console_log, args=(sess,))
# get start epoch
_step = sess.run(tf.sg_global_step())
ep = _step // opt.ep_size
best_f1 = 0
# check if already finished
if ep <= opt.max_ep:
# logging
tf.sg_info('Training started from epoch[%03d]-step[%d].' % (ep, _step))
# epoch loop
for ep in range(ep, opt.max_ep + 1):
# update epoch info
start_step = sess.run(tf.sg_global_step()) % opt.ep_size
epoch = ep
# create progressbar iterator
if opt.tqdm:
iterator = tf.tqdm(range(start_step, opt.ep_size), total=opt.ep_size, initial=start_step,
desc='train', ncols=70, unit='b', leave=False)
else:
iterator = range(start_step, opt.ep_size)
# batch loop
for _ in iterator:
# exit loop
if sv.should_stop():
break
# call train function
batch_loss = func(sess, opt)
# loss history update
if batch_loss is not None and \
not np.isnan(batch_loss.all()) and not np.isinf(batch_loss.all()):
if loss is None:
loss = np.mean(batch_loss)
else:
loss = loss * 0.9 + np.mean(batch_loss) * 0.1
# log epoch information
console_log(sess)
f1_stat = show_metrics(sv, sess, opt.eval_metric[2], opt.eval_metric[3], ep, opt.val_ep_size,
'val', use_tqdm=True)
if f1_stat > best_f1:
best_f1 = f1_stat
max_model_file = opt.save_dir + max_model_name
# save last version
saver.save(sess, max_model_file)
print("Improved F1 score, max model saved in file: %s" % max_model_file)
print('Test metrics:')
show_metrics(sv, sess, opt.test_metric[0], opt.test_metric[1], ep, opt.test_ep_size,
'test', use_tqdm=True)
# save last version
saver.save(sess, opt.save_dir + '/model.ckpt', global_step=sess.run(tf.sg_global_step()))
# logging
tf.sg_info('Training finished at epoch[%d]-step[%d].' % (ep, sess.run(tf.sg_global_step())))
else:
tf.sg_info('Training already finished at epoch[%d]-step[%d].' %
(ep - 1, sess.run(tf.sg_global_step())))
def sg_init(sess):
# initialize variables
sess.run(
tf.group(tf.initialize_all_variables(),
tf.initialize_local_variables()))
def train():
tf.Graph()
tf.set_random_seed(888)
print("*****************************************")
print("Training started with random seed: {}".format(111))
print("Batch started with random seed: {}".format(111))
#read data
x,y=read_from_tfrecords(tfname,
["source","target"], batSize, [[s1,2],[s2,2]])
global_step = tf.Variable(1, trainable=False,name='global_step')
yp=Net(x,x,y)+x
Loss=chamfer_loss(yp,y)
#Learning Rate****************************************************************************
lr = tf.train.exponential_decay(learningRate, global_step,
batSize, learningRateDecay, staircase=False)
# Optimization Algo************************************************************************
train_step = tf.train.AdamOptimizer(learning_rate=lr,
beta1=adam_beta1,
beta2=adam_beta2
).minimize(Loss,global_step=global_step)
saver = tf.train.Saver(max_to_keep=int(maxKeepWeights))
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
# Continue Training************************************************************************
if len(conWeightPath)>0:
print("Continue Training...")
tmp_var_list={}
if len(conWeightVar)==0:
print("For all variables")
globals()['conWeightVar']={''}
else:
print("Training variables: {}".format(conWeightVar))
for j in conWeightVar:
for i in tf.global_variables():
if i.name.startswith(j):
tmp_var_list[i.name[:-2]] = i
saver1=tf.train.Saver(tmp_var_list)
# Training**********************************************************************************
with tf.Session() as sess:
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
# Read Weight******************************
if len(conWeightPath)>0:
print(conWeightPath)
if stepsContinue==-1:
STEPS=sorted([int(i.split("/")[-1].split(".")[1].split("-")[-1]) for i in glob.glob(conWeightPath+"/*meta")])
print("hahaha",STEPS)
globals()['stepsContinue']=STEPS[-1]
wtt=glob.glob(conWeightPath+"/*{}*meta".format(stepsContinue))[0][:-5]
print("Reading Weight:{}".format(wtt))
saver1.restore(sess,wtt)
print('Weight is successfully updated from: {}'.format(wtt))
#*******************************************
stepst = sess.run(global_step)
for t in tqdm.tqdm(range(stepst,int(maxStep)+1)):
_= sess.run([train_step])
if t % saveStep==0:
if not os.path.exists(dirSave):
os.makedirs(dirSave)
saver.save(sess, dirSave + '/model.ckpt', global_step=t)
coord.request_stop()
coord.join(threads)