def generate(sample_image):
start_time = time.time()
g = ModelGraph()
with tf.Session() as sess:
# We need to initialize variables in this case because the Variable `generator/x` will not restored.
tf.sg_init(sess)
vars = [v for v in tf.global_variables() if "generator" not in v.name]
saver = tf.train.Saver(vars)
saver.restore(sess, tf.train.latest_checkpoint('asset/train/ckpt'))
i = 0
while True:
mse, _ = sess.run([g.mse, g.train_gen],
{g.y: transform_image(sample_image)}) # (16, 28)
if time.time() - start_time > 60: # Save every 60 seconds
gen_image = sess.run(g.x)
gen_image = np.squeeze(gen_image)
misc.imsave('gen_images/%s/gen_%.2f.jpg' % (label, mse),
gen_image)
start_time = time.time()
i += 1
if i == 60: break # Finish after 1 hour
def get_loss(opt):
# encode audio feature
logit = get_logit(opt.input[opt.gpu_index], voca_size=voca_size)
for i in tf.get_collection("regularization_losses"):
print(i)
print('--------------------')
train_list = tf.trainable_variables()
var_list = tf.global_variables()
real_var_list = []
for item in var_list:
# print(item)
if 'W' in item.name:
real_var_list.append(item)
loss = logit.sg_ctc(target=opt.target[opt.gpu_index],
seq_len=opt.seq_len[opt.gpu_index])
# print(loss)
# tf.add_to_collection("losses", loss)
# losses = tf.get_collection("losses")
# losses += tf.get_collection("regularization_losses")
# for i in tf.get_collection("losses"):
# print(i.name)
# print('++++++++++++++++++++')
# total_loss = tf.add_n(losses, name='total_loss')
# for item in real_var_list:
# loss += 0.03 * tf.nn.l2_loss(item)
# for i in tf.get_collection("regularization_losses"):
# loss += 0.03 * i
regular_loss = tf.sg_regularizer_loss(0.03)
loss += regular_loss
return loss
def discriminator(tensor):
# reuse flag
reuse = len([
t for t in tf.global_variables() if t.name.startswith('discriminator')
]) > 0
with tf.sg_context(name='discriminator',
size=4,
stride=2,
act='leaky_relu',
reuse=reuse):
# shared part
shared = (tensor.sg_conv(dim=64, name='conv1').sg_conv(
dim=128, name='conv2').sg_flatten().sg_dense(dim=1024, name='fc1'))
# discriminator end
disc = shared.sg_dense(dim=1, act='linear', name='disc').sg_squeeze()
# shared recognizer part
recog_shared = shared.sg_dense(dim=128, name='recog')
# categorical auxiliary classifier end
cat = recog_shared.sg_dense(dim=cat_dim, act='linear', name='cat')
# continuous auxiliary classifier end
con = recog_shared.sg_dense(dim=con_dim, act='sigmoid', name='con')
return disc, cat, con
def sg_restore(sess, save_path, category=''):
r""" Restores previously saved variables.
Args:
sess: A `Session` to use to restore the parameters.
save_path: Path where parameters were previously saved.
category: A `String` to filter variables starts with given category.
Returns:
"""
# to list
if not isinstance(category, (tuple, list)):
category = [category]
# make variable list to load
var_list = {}
for cat in category:
for t in tf.global_variables():
if t.name.startswith(cat):
var_list[t.name[:-2]] = t
# restore parameters
saver = tf.train.Saver(var_list)
saver.restore(sess, save_path)
def discriminator(tensor):
# reuse flag
reuse = len([
t for t in tf.global_variables() if t.name.startswith('discriminator')
]) > 0
with tf.sg_context(name='discriminator',
size=(4, 1),
stride=(2, 1),
act='leaky_relu',
bn=False,
reuse=reuse):
# shared part
shared = (tensor.sg_conv(dim=32, name='conv1').sg_conv(
dim=64, name='conv2').sg_conv(
dim=128, name='conv3').sg_flatten().sg_dense(dim=1024,
name='fc1'))
# discriminator end
disc = shared.sg_dense(dim=1, act='linear', bn=False,
name='disc').sg_squeeze()
# continuous auxiliary classifier end
con = (shared.sg_dense(dim=128, name='recog').sg_dense(dim=con_dim,
act='sigmoid',
bn=False,
name='con'))
return disc, con
def discriminator(x):
reuse = len([t for t in tf.global_variables() if t.name.startswith('discriminator')]) > 0
with tf.sg_context(name='discriminator', size=4, stride=2, act='leaky_relu', bn=True, reuse=reuse):
res = (x.sg_conv(dim=64, name='conv_1')
.sg_conv(dim=128, name='conv_2')
.sg_upconv(dim=64, name='conv_3')
.sg_upconv(dim=1, act='linear', name='conv_4'))
return res
def discriminator(tensor):
# reuse flag
reuse = len([t for t in tf.global_variables() if t.name.startswith('discriminator')]) > 0
with tf.sg_context(name='discriminator', size=4, stride=2, act='leaky_relu', reuse=reuse):
res = (tensor
.sg_conv(dim=64, name='conv1')
.sg_conv(dim=128, name='conv2')
.sg_flatten()
.sg_dense(dim=1024, name='fc1')
.sg_dense(dim=1, act='linear', name='fc2')
.sg_squeeze())
return res
def generator(x):
reuse = len([t for t in tf.global_variables() if t.name.startswith('generator')]) > 0
with tf.sg_context(name='generator', size=4, stride=2, act='leaky_relu', bn=True, reuse=reuse):
# generator network
res = (x.sg_dense(dim=1024, name='fc_1')
.sg_dense(dim=7*7*128, name='fc_2')
.sg_reshape(shape=(-1, 7, 7, 128))
.sg_upconv(dim=64, name='conv_1')
.sg_upconv(dim=1, act='sigmoid', bn=False, name='conv_2'))
return res
def generator(tensor):
# reuse flag
reuse = len([t for t in tf.global_variables() if t.name.startswith('generator')]) > 0
with tf.sg_context(name='generator', size=4, stride=2, act='relu', bn=True, reuse=reuse):
res = (tensor
.sg_dense(dim=1024, name='fc1')
.sg_dense(dim=7*7*128, name='fc2')
.sg_reshape(shape=(-1, 7, 7, 128))
.sg_upconv(dim=64, name='conv1')
.sg_upconv(dim=1, act='sigmoid', bn=False, name='conv2'))
return res
def discriminator(tensor):
reuse = len([
t for t in tf.global_variables() if t.name.startswith('discriminator')
]) > 0
with tf.sg_context(name='discriminator',
size=4,
stride=2,
act='leaky_relu',
bn=True,
reuse=reuse):
res = (tensor.sg_dense(dim=4096, name='fc1').sg_dense(
dim=512, name='fc2').sg_dense(dim=1,
act='sigmoid',
bn=False,
name='fc3').sg_squeeze())
return res
def generator(tensor):
reuse = len(
[t
for t in tf.global_variables() if t.name.startswith('generator')]) > 0
with tf.sg_context(name='generator',
size=4,
stride=2,
act='leaky_relu',
bn=True,
reuse=reuse):
res = (tensor.sg_dense(dim=1024, name='fc1').sg_dense(dim=4096,
act='relu',
bn=False,
name='fc3'))
return res
def test(tfname, weightPaths, steps=100000, Var=["NNReg"], lll=2000):
tf.Graph()
x, y = read_from_tfrecords(tfname, ["source", "target"], 10,
[[1070, 3], [1070, 3]])
global_step = tf.Variable(1, trainable=False, name='global_step')
print(x.shape, y.shape)
x = np.loadtxt('EM.txt', dtype='float32') / 1500
y = np.loadtxt('FM.txt', dtype='float32')[:, :100] / 1500
x = tf.convert_to_tensor(np.expand_dims(np.rollaxis(x, axis=0), axis=0))
y = tf.convert_to_tensor(np.expand_dims(np.rollaxis(y, axis=0), axis=0))
print(x.shape, y.shape)
yp = Net(x, x, y) + x
tmp_var_list = {}
for j in Var:
for i in tf.global_variables():
if i.name.startswith(j):
tmp_var_list[i.name[:-2]] = i
saver = tf.train.Saver(tmp_var_list)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
path = weightPaths + "model.ckpt-{}".format(steps)
Sour = []
Targ = []
Trans_S = []
with tf.Session() as sess:
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
saver.restore(sess, path)
for i in tqdm.tqdm(range(lll)):
S, T, TS = sess.run([x, y, yp])
Sour.append(S)
Targ.append(T)
Trans_S.append(TS)
coord.request_stop()
coord.join(threads)
return Sour, Targ, Trans_S
def test(tfname, weightPaths, steps=100000, Var=["NNReg"], lll=2000):
tf.Graph()
x, y = read_from_tfrecords(tfname, ["source", "target"], 10,
[[91, 2], [91, 2]])
global_step = tf.Variable(1, trainable=False, name='global_step')
yp = Net(x, x, y) + x
tmp_var_list = {}
for j in Var:
for i in tf.global_variables():
if i.name.startswith(j):
tmp_var_list[i.name[:-2]] = i
saver = tf.train.Saver(tmp_var_list)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
path = weightPaths + "model.ckpt-{}".format(steps)
Sour = []
Targ = []
Trans_S = []
with tf.Session() as sess:
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
saver.restore(sess, path)
for i in tqdm.tqdm(range(lll)):
S, T, TS = sess.run([x, y, yp])
Sour.append(S)
Targ.append(T)
Trans_S.append(TS)
coord.request_stop()
coord.join(threads)
return Sour, Targ, Trans_S
def wrapper(tensor, **kwargs):
r"""Manages arguments of `tf.sg_opt`.
Args:
tensor: automatically passed by decorator
kwargs:
in_dim: An integer. The size of input dimension, which is set to the last one by default.
dim: An integer. The size of output dimension. Has the same value as in_dim by default.
ln: Boolean. If True, layer normalization is applied.
bias: Boolean. If True, biases are added. As a default, it is set to True
name: A name for the layer. As a default, the function name is assigned.
reuse: `True` or `None`; if `True`, we go into reuse mode for this `layer` scope
as well as all sub-scopes; if `None`, we just inherit the parent scope reuse.
"""
# kwargs parsing
opt = tf.sg_opt(kwargs) + _context
# set default argument
try:
shape = tensor.get_shape().as_list()
# dropout off
opt += tf.sg_opt(shape=shape,
in_dim=shape[-1],
dim=shape[-1],
dout=0)
# disable bias when normalization on
opt += tf.sg_opt(bias=not opt.ln)
finally:
pass
# automatic layer naming
if opt.name is None:
# layer function name will be used as layer name
opt.name = func.__name__.replace('sg_', 'lyr-')
# find existing layer names
exist_layers = []
for t in tf.global_variables():
scope_name = tf.get_variable_scope().name
prefix = scope_name + '/' if len(scope_name) > 0 else ''
i = t.name.rfind(prefix + opt.name)
if i >= 0:
exist_layers.append(t.name[i:].split('/')[-2])
exist_layers = list(set(exist_layers))
# layer name numbering
if len(exist_layers) == 0:
opt.name += '_1'
else:
opt.name += '_%d' % (
max([int(n.split('_')[-1]) for n in exist_layers]) + 1)
# all layer variables start with 'lyr-' prefix
with tf.variable_scope(opt.name, reuse=opt.reuse) as scope:
# call layer function
out = func(tensor, opt)
# apply dropout
if opt.dout:
out = tf.cond(_phase, lambda: tf.nn.dropout(out, 1 - opt.dout),
lambda: out)
# rename tensor
out = tf.identity(out, 'out')
# add final output summary
if scope.reuse:
tf.sg_summary_activation(out)
# save node info for reuse
out._sugar = tf.sg_opt(func=func,
arg=tf.sg_opt(kwargs) + _context,
prev=tensor,
is_layer=True,
name=opt.name)
# inject reuse function
out.sg_reuse = types.MethodType(sg_reuse, out)
return out
def wrapper(tensor, **kwargs):
r"""Manages arguments of `tf.sg_opt`.
Args:
tensor: A `tensor` (automatically passed by decorator).
kwargs:
shape: A list of integers. The shape of `tensor`. Inferred if not specified.
in_dim: An integer. The size of input dimension, which is set to the last one by default.
dim: An integer. The size of output dimension. Has the same value as in_dim by default.
bn: Boolean. If True, batch normalization is applied.
ln: Boolean. If True, layer normalization is applied.
dout: A float of range [0, 100). A dropout rate. Set to 0 by default.
bias: Boolean. If True, biases are added. As a default, it is set to True
name: A name for the layer. As a default, the function name is assigned.
act: A name of activation function. e.g., `sigmoid`, `tanh`, etc.
reuse: `True` or `None`; if `True`, we go into reuse mode for this `layer` scope
as well as all sub-scopes; if `None`, we just inherit the parent scope reuse.
"""
from . import sg_initializer as init
from . import sg_activation
# kwargs parsing
opt = tf.sg_opt(kwargs) + _context
# set default argument
try:
shape = tensor.get_shape().as_list()
# batch normalization off, layer normalization off, dropout off
opt += tf.sg_opt(shape=shape,
in_dim=shape[-1],
dim=shape[-1],
bn=False,
ln=False,
dout=0)
assert not (
opt.bn and opt.ln
), 'one of batch normalization and layer normalization is available.'
# disable bias when normalization on
opt += tf.sg_opt(bias=not (opt.bn or opt.ln))
finally:
pass
# automatic layer naming
if opt.name is None:
# layer function name will be used as layer name
opt.name = func.__name__.replace('sg_', '')
# find existing layer names
exist_layers = []
for t in tf.global_variables():
scope_name = tf.get_variable_scope().name
prefix = scope_name + '/' if len(scope_name) > 0 else ''
i = t.name.rfind(prefix + opt.name)
if i >= 0:
exist_layers.append(t.name[i:].split('/')[-2])
exist_layers = list(set(exist_layers))
# layer name numbering
if len(exist_layers) == 0:
opt.name += '_1'
else:
opt.name += '_%d' % (
max([int(n.split('_')[-1]) for n in exist_layers]) + 1)
# all layer variables start with 'lyr-' prefix
with tf.variable_scope(opt.name, reuse=opt.reuse) as scope:
# call layer function
out = func(tensor, opt)
# apply batch normalization
if opt.bn:
# offset, scale parameter
beta = init.constant('beta', opt.dim, summary=False)
gamma = init.constant('gamma', opt.dim, value=1, summary=False)
# offset, scale parameter
mean_running = init.constant('mean', opt.dim, summary=False)
variance_running = init.constant('variance',
opt.dim,
value=1,
summary=False)
# calc batch mean, variance
mean, variance = tf.nn.moments(
out, axes=range(len(out.get_shape()) - 1))
# update running mean, variance
def update_running_stat():
decay = 0.99
update_op = [
mean_running.assign(mean_running * decay + mean *
(1 - decay)),
variance_running.assign(variance_running * decay +
variance * (1 - decay))
]
with tf.control_dependencies(update_op):
return tf.identity(mean), tf.identity(variance)
# select mean, variance by training phase
m, v = tf.cond(
_phase,
update_running_stat, # updated running stat and batch mean, variance
lambda:
(mean_running, variance_running)) # saved mean, variance
# apply batch normalization
out = tf.nn.batch_normalization(out, m, v, beta, gamma,
tf.sg_eps)
# apply normalization parameters
if opt.ln:
# offset, scale parameter
beta = init.constant('beta', opt.dim, summary=False)
gamma = init.constant('gamma', opt.dim, value=1, summary=False)
# calc layer mean, variance for final axis
mean, variance = tf.nn.moments(out,
axes=[len(out.get_shape()) - 1],
keep_dims=True)
# apply normalization
out = (out - mean) / tf.sqrt(variance + tf.sg_eps)
# apply parameter
out = gamma * out + beta
# apply activation
if opt.act:
out = getattr(sg_activation, 'sg_' + opt.act.lower())(out)
# apply dropout
if opt.dout:
out = tf.cond(_phase, lambda: tf.nn.dropout(out, 1 - opt.dout),
lambda: out)
# rename tensor
out = tf.identity(out, 'out')
# add final output summary
if not scope.reuse:
tf.sg_summary_activation(out)
# save node info for reuse
out._sugar = tf.sg_opt(func=func,
arg=tf.sg_opt(kwargs) + _context,
prev=tensor,
is_layer=True,
name=opt.name)
# inject reuse function
out.sg_reuse = types.MethodType(sg_reuse, out)
return out
def wrapper(tensor, **kwargs):
r"""Manages arguments of `tf.sg_opt`.
Args:
tensor: A `tensor` (automatically passed by decorator).
kwargs:
shape: A list of integers. The shape of `tensor`. Inferred if not specified.
in_dim: An integer. The size of input dimension, which is set to the last one by default.
dim: An integer. The size of output dimension. Has the same value as in_dim by default.
bn: Boolean. If True, batch normalization is applied.
ln: Boolean. If True, layer normalization is applied.
scale: If true, multiple by a trainable gamma variable. When the activation is
linear (relu included), this can be disabled because it can be implicitly
learned by the next layer. The default is True.
dout: A float of range [0, 100). A dropout rate. Set to 0 by default.
bias: Boolean. If True, biases are added. As a default, it is set to True
name: A name for the layer. As a default, the function name is assigned.
act: A name of activation function. e.g., `sigmoid`, `tanh`, etc.
reuse: `True` or `None`; if `True`, we go into reuse mode for this `layer` scope
as well as all sub-scopes; if `None`, we just inherit the parent scope reuse.
regularizer: A string. None, 'l1' or 'l2'. The default is None
summary: If True, summaries are added. The default is True.
"""
from . import sg_initializer as init
from . import sg_activation
# kwargs parsing
opt = tf.sg_opt(kwargs) + sg_get_context()
# set default argument
try:
shape = tensor.get_shape().as_list()
# batch normalization off, layer normalization off, dropout off
opt += tf.sg_opt(shape=shape,
in_dim=shape[-1],
dim=shape[-1],
bn=False,
ln=False,
dout=0,
summary=True,
scale=True)
if opt.regularizer == 'l1':
opt.regularizer = lambda x: tf.reduce_mean(tf.abs(x))
elif opt.regularizer == 'l2':
opt.regularizer = lambda x: tf.square(
tf.reduce_mean(tf.square(x)))
else:
opt.regularizer = None
assert not (
opt.bn and opt.ln
), 'one of batch normalization and layer normalization is available.'
# disable bias when normalization on
opt += tf.sg_opt(bias=not (opt.bn or opt.ln))
finally:
pass
# automatic layer naming
if opt.name is None:
# layer function name will be used as layer name
opt.name = func.__name__.replace('sg_', '')
# find existing layer names
exist_layers = []
for t in tf.global_variables():
scope_name = tf.get_variable_scope().name
prefix = scope_name + '/' if len(scope_name) > 0 else ''
i = t.name.rfind(prefix + opt.name)
if i >= 0:
exist_layers.append(t.name[i:].split('/')[-2])
exist_layers = list(set(exist_layers))
# layer name numbering
if len(exist_layers) == 0:
opt.name += '_1'
else:
opt.name += '_%d' % (
max([int(n.split('_')[-1]) for n in exist_layers]) + 1)
with tf.variable_scope(opt.name, reuse=opt.reuse) as scope:
# call layer function
out = func(tensor, opt)
out_shape = out.get_shape()
# apply batch normalization
if opt.bn:
beta = init.constant('beta', opt.dim, summary=opt.summary)
gamma = init.constant('gamma',
opt.dim,
value=1,
summary=opt.summary,
trainable=opt.scale)
# offset, scale parameter ( for inference )
mean_running = init.constant('mean',
opt.dim,
trainable=False,
summary=opt.summary)
variance_running = init.constant('variance',
opt.dim,
value=1,
trainable=False,
summary=opt.summary)
# use fused batch norm if ndims in [2, 3, 4]
if out_shape.ndims in [2, 3, 4]:
# add HW dims if necessary, fused_batch_norm requires shape to be NHWC
if out_shape.ndims == 2:
out = tf.expand_dims(out, axis=1)
out = tf.expand_dims(out, axis=2)
elif out_shape.ndims == 3:
out = tf.expand_dims(out, axis=2)
fused_eps = tf.sg_eps if tf.sg_eps > 1e-5 else 1e-5
out, mean, variance = tf.cond(
_phase,
lambda: tf.nn.fused_batch_norm(
out, gamma, beta, epsilon=fused_eps),
lambda: tf.nn.fused_batch_norm(out,
gamma,
beta,
mean=mean_running,
variance=
variance_running,
epsilon=fused_eps,
is_training=False),
)
# restore original shape if HW dims was added
if out_shape.ndims == 2:
out = tf.squeeze(out, axis=[1, 2])
elif out_shape.ndims == 3:
out = tf.squeeze(out, axis=2)
# fallback to naive batch norm
else:
mean, variance = tf.nn.moments(
out, axes=list(range(len(out.get_shape()) - 1)))
out = tf.cond(
_phase, lambda: tf.nn.batch_normalization(
out, mean, variance, beta, gamma, tf.sg_eps),
lambda: tf.nn.batch_normalization(
out, mean_running, variance_running, beta, gamma,
tf.sg_eps))
decay = 0.99
tf.add_to_collection(
tf.GraphKeys.UPDATE_OPS,
mean_running.assign(mean_running * decay + mean *
(1 - decay)))
tf.add_to_collection(
tf.GraphKeys.UPDATE_OPS,
variance_running.assign(variance_running * decay +
variance * (1 - decay)))
# apply layer normalization
if opt.ln:
# offset, scale parameter
beta = init.constant('beta', opt.dim, summary=opt.summary)
if opt.scale:
gamma = init.constant('gamma',
opt.dim,
value=1,
summary=opt.summary)
# calc layer mean, variance for final axis
mean, variance = tf.nn.moments(out,
axes=[len(out.get_shape()) - 1],
keep_dims=True)
# apply normalization
out = (out - mean) / tf.sqrt(variance + tf.sg_eps)
# apply parameter
if opt.scale:
out = gamma * out + beta
else:
out = out + beta
# apply activation
if opt.act:
out = getattr(sg_activation, 'sg_' + opt.act.lower())(out)
# apply dropout
if opt.dout:
out = tf.cond(_phase, lambda: tf.nn.dropout(out, 1 - opt.dout),
lambda: out)
# rename tensor
out = tf.identity(out, 'out')
# add final output summary
if opt.summary:
tf.sg_summary_activation(out)
# save node info for reuse
out._sugar = tf.sg_opt(func=func,
arg=tf.sg_opt(kwargs) + sg_get_context(),
prev=tensor,
is_layer=True,
name=opt.name)
# inject reuse function
out.sg_reuse = types.MethodType(sg_reuse, out)
return out
def train():
tf.Graph()
tf.set_random_seed(888)
print("*****************************************")
print("Training started with random seed: {}".format(111))
print("Batch started with random seed: {}".format(111))
#read data
x,y=read_from_tfrecords(tfname,
["source","target"], batSize, [[s1,2],[s2,2]])
global_step = tf.Variable(1, trainable=False,name='global_step')
yp=Net(x,x,y)+x
Loss=chamfer_loss(yp,y)
#Learning Rate****************************************************************************
lr = tf.train.exponential_decay(learningRate, global_step,
batSize, learningRateDecay, staircase=False)
# Optimization Algo************************************************************************
train_step = tf.train.AdamOptimizer(learning_rate=lr,
beta1=adam_beta1,
beta2=adam_beta2
).minimize(Loss,global_step=global_step)
saver = tf.train.Saver(max_to_keep=int(maxKeepWeights))
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
# Continue Training************************************************************************
if len(conWeightPath)>0:
print("Continue Training...")
tmp_var_list={}
if len(conWeightVar)==0:
print("For all variables")
globals()['conWeightVar']={''}
else:
print("Training variables: {}".format(conWeightVar))
for j in conWeightVar:
for i in tf.global_variables():
if i.name.startswith(j):
tmp_var_list[i.name[:-2]] = i
saver1=tf.train.Saver(tmp_var_list)
# Training**********************************************************************************
with tf.Session() as sess:
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
# Read Weight******************************
if len(conWeightPath)>0:
print(conWeightPath)
if stepsContinue==-1:
STEPS=sorted([int(i.split("/")[-1].split(".")[1].split("-")[-1]) for i in glob.glob(conWeightPath+"/*meta")])
print("hahaha",STEPS)
globals()['stepsContinue']=STEPS[-1]
wtt=glob.glob(conWeightPath+"/*{}*meta".format(stepsContinue))[0][:-5]
print("Reading Weight:{}".format(wtt))
saver1.restore(sess,wtt)
print('Weight is successfully updated from: {}'.format(wtt))
#*******************************************
stepst = sess.run(global_step)
for t in tqdm.tqdm(range(stepst,int(maxStep)+1)):
_= sess.run([train_step])
if t % saveStep==0:
if not os.path.exists(dirSave):
os.makedirs(dirSave)
saver.save(sess, dirSave + '/model.ckpt', global_step=t)
coord.request_stop()
coord.join(threads)
