util.toMagenta("loading pretrained ({0}) to fine-tune...".format(
opt.load)))
summaryWriter.add_graph(sess.graph)
print(util.toMagenta("start training..."))
chunkResumeN, chunkMaxN = opt.fromIt // opt.itPerChunk, opt.toIt // opt.itPerChunk
# training loop
for c in range(chunkResumeN, chunkMaxN):
dataloader.shipChunk()
dataloader.thread = threading.Thread(target=dataloader.loadChunk,
args=[opt])
dataloader.thread.start()
for i in range(c * opt.itPerChunk, (c + 1) * opt.itPerChunk):
lr = opt.lr * opt.lrDecay**(i // opt.lrStep)
# make training batch
batch = data.makeBatch(opt, dataloader, PH)
batch[lr_PH] = lr
# run one step
runList = [optim, loss, loss_depth, loss_mask, maskLogit]
_, l, ld, lm, ml = sess.run(runList, feed_dict=batch)
if (i + 1) % 20 == 0:
print(
"it. {0}/{1}, lr={2}, loss={4} ({5},{6}), time={3}".format(
util.toCyan("{0}".format(i + 1)), opt.toIt,
util.toYellow("{0:.0e}".format(lr)),
util.toGreen("{0:.2f}".format(time.time() -
timeStart)),
util.toRed("{0:.2f}".format(l)),
util.toRed("{0:.2f}".format(ld)),
util.toRed("{0:.2f}".format(lm))))
if (i + 1) % 100 == 0:
with tf.Session(config=tfConfig) as sess:
sess.run(tf.global_variables_initializer())
summaryWriter.add_graph(sess.graph)
if opt.fromIt!=0:
util.restoreModelFromIt(opt,sess,saver_D,"D",opt.fromIt)
print(util.toMagenta("resuming from iteration {0}...".format(opt.fromIt)))
elif opt.loadD:
util.restoreModel(opt,sess,saver_D,opt.loadD,"D")
print(util.toMagenta("loading pretrained D {0}...".format(opt.loadD)))
print(util.toMagenta("start training..."))
# training loop
for i in range(opt.fromIt,opt.toIt):
lrD = opt.lrD*opt.lrDdecay**(i//opt.lrDstep)
# make training batch
batch = data.makeBatch(opt,trainData,PH)
batch[lrD_PH] = lrD
# update discriminator
runList = [optimD,loss_D,grad_D_norm_mean]
for u in range(opt.updateD):
_,ld,gdn = sess.run(runList,feed_dict=batch)
if (i+1)%10==0:
print("it.{0}/{1} lr={3}(GP),{4}(D) loss={5}(GP),{6}(D) norm={7} time={2}"
.format(util.toCyan("{0}".format(i+1)),
opt.toIt,
util.toGreen("{0:.2f}".format(time.time()-timeStart)),
util.toYellow("X"),
util.toYellow("{0:.0e}".format(lrD)),
util.toRed("X"),
util.toRed("{0:.4f}".format(ld)),
util.toBlue("{0:.4f}".format(gdn))))
print(util.toYellow("======= TRAINING START ======="))
timeStart = time.time()
# start session
tfConfig = tf.ConfigProto(allow_soft_placement=True)
tfConfig.gpu_options.allow_growth = True
with tf.Session(config=tfConfig) as sess:
sess.run(tf.global_variables_initializer())
summaryWriter.add_graph(sess.graph)
# training loop
for i in range(toIt):
lrGP = lrGP*lrGPdecay**(i//lrGPstep)
batch = data.makeBatch(batchSize,trainData,PH)
batch[lrGP_PH] = lrGP
if(i<1200):
optim = optimGP1
else:
optim = optimGP2
'''
if(i<2000):
optim = optimGP1
elif(i<10000):
optim = optimGP2
else:
optim = optimGP3
'''
runList = [optim,loss_GP,vars_all,summary_op,imageWarped,StandardData]
_,lg,var,summary_res,image,stimage = sess.run(runList,feed_dict=batch)