print(util.toMagenta("loading pretrained D {0}...".format(opt.loadD)))
print(util.toMagenta("start training..."))
# training loop
for i in range(opt.fromIt,opt.toIt):
lrD = opt.lrD*opt.lrDdecay**(i//opt.lrDstep)
# make training batch
batch = data.makeBatch(opt,trainData,PH)
batch[lrD_PH] = lrD
# update discriminator
runList = [optimD,loss_D,grad_D_norm_mean]
for u in range(opt.updateD):
_,ld,gdn = sess.run(runList,feed_dict=batch)
if (i+1)%10==0:
print("it.{0}/{1} lr={3}(GP),{4}(D) loss={5}(GP),{6}(D) norm={7} time={2}"
.format(util.toCyan("{0}".format(i+1)),
opt.toIt,
util.toGreen("{0:.2f}".format(time.time()-timeStart)),
util.toYellow("X"),
util.toYellow("{0:.0e}".format(lrD)),
util.toRed("X"),
util.toRed("{0:.4f}".format(ld)),
util.toBlue("{0:.4f}".format(gdn))))
# if (i+1)%20==0:
# runList = [summaryLossTrain,summaryGradTrain]
# sl,sg = sess.run(runList,feed_dict=batch)
# summaryWriter.add_summary(sl,i+1)
# summaryWriter.add_summary(sg,i+1)
# if (i+1)%200==0:
# si = sess.run(summaryImageTrain,feed_dict=batch)
# summaryWriter.add_summary(si,i+1)
# make training batch
batch = data.makeBatch(opt,trainData,PH)
batch[lrGP_PH] = lrGP
batch[lrD_PH] = lrD
# update geometric predictor
runList = [optimGP,loss_GP,imageComp]
for u in range(opt.updateGP):
_,lg,ic = sess.run(runList,feed_dict=batch)
# update discriminator
batch[imageComp] = data.updateHistory(opt,ic)
runList = [optimD,loss_D,grad_D_norm_mean]
for u in range(opt.updateD):
_,ld,gdn = sess.run(runList,feed_dict=batch)
if (i+1)%10==0:
print("it.{0}/{1} lr={3}(GP),{4}(D) loss={5}(GP),{6}(D) norm={7} time={2}"
.format(util.toCyan("{0}".format((opt.warpN-1)*opt.toIt+i+1)),
opt.warpN*opt.toIt,
util.toGreen("{0:.2f}".format(time.time()-timeStart)),
util.toYellow("{0:.0e}".format(lrGP)),
util.toYellow("{0:.0e}".format(lrD)),
util.toRed("{0:.4f}".format(lg)),
util.toRed("{0:.4f}".format(ld)),
util.toBlue("{0:.4f}".format(gdn))))
if (i+1)%20==0:
runList = [summaryLossTrain,summaryGradTrain]
sl,sg = sess.run(runList,feed_dict=batch)
summaryWriter.add_summary(sl,(opt.warpN-1)*opt.toIt+i+1)
summaryWriter.add_summary(sg,(opt.warpN-1)*opt.toIt+i+1)
if (i+1)%200==0:
si = sess.run(summaryImageTrain,feed_dict=batch)
summaryWriter.add_summary(si,(opt.warpN-1)*opt.toIt+i+1)
dataloader.shipChunk()
dataloader.thread = threading.Thread(target=dataloader.loadChunk,
args=[opt])
dataloader.thread.start()
for i in range(c * opt.itPerChunk, (c + 1) * opt.itPerChunk):
lr = opt.lr * opt.lrDecay**(i // opt.lrStep)
# make training batch
batch = data.makeBatch(opt, dataloader, PH)
batch[lr_PH] = lr
# run one step
runList = [optim, loss, loss_depth, loss_mask, maskLogit]
_, l, ld, lm, ml = sess.run(runList, feed_dict=batch)
if (i + 1) % 20 == 0:
print(
"it. {0}/{1}, lr={2}, loss={4} ({5},{6}), time={3}".format(
util.toCyan("{0}".format(i + 1)), opt.toIt,
util.toYellow("{0:.0e}".format(lr)),
util.toGreen("{0:.2f}".format(time.time() -
timeStart)),
util.toRed("{0:.2f}".format(l)),
util.toRed("{0:.2f}".format(ld)),
util.toRed("{0:.2f}".format(lm))))
if (i + 1) % 100 == 0:
summaryWriter.add_summary(
sess.run(summaryLoss, feed_dict=batch), i + 1)
if (i + 1) % 500 == 0:
summaryWriter.add_summary(
sess.run(summaryImage, feed_dict=batch), i + 1)
if (i + 1) % 2000 == 0:
util.saveModel(opt, sess, saver, i + 1)
print(
# for each CAD model in data chunk
for i in range(testN):
m = idx[i]
CAD = dataloader.CADs[m]
points24 = np.zeros([opt.inputViewN,1],dtype=np.object)
# make test batch
batch = { inputImage: dataChunk["image_in"][i] }
# evaluate network
runList = [XYZid,ML]
xyz,ml = sess.run(runList,feed_dict=batch)
for a in range(opt.inputViewN):
xyz1 = xyz[a].T # [VHW,3]
ml1 = ml[a].reshape([-1]) # [VHW]
points24[a,0] = xyz1[ml1>0]
# output results
scipy.io.savemat("results_{0}/{1}/{2}.mat".format(opt.group,opt.load,CAD),{
"image": dataChunk["image_in"][i],
"pointcloud": points24
})
pointMeanN = np.array([len(p) for p in points24[:,0]]).mean()
print("{0}/{1} ({2}) done (average {3} points), time={4}"
.format(util.toCyan("{0}".format(m+1)),
util.toCyan("{0}".format(CADN)),
CAD,
util.toBlue("{0:.2f}".format(pointMeanN)),
util.toGreen("{0:.2f}".format(time.time()-timeStart))))
if c!=chunkN-1: dataloader.thread.join()
print(util.toYellow("======= EVALUATION DONE ======="))
