print(util.toMagenta("loading pretrained D {0}...".format(opt.loadD))) print(util.toMagenta("start training...")) # training loop for i in range(opt.fromIt,opt.toIt): lrD = opt.lrD*opt.lrDdecay**(i//opt.lrDstep) # make training batch batch = data.makeBatch(opt,trainData,PH) batch[lrD_PH] = lrD # update discriminator runList = [optimD,loss_D,grad_D_norm_mean] for u in range(opt.updateD): _,ld,gdn = sess.run(runList,feed_dict=batch) if (i+1)%10==0: print("it.{0}/{1} lr={3}(GP),{4}(D) loss={5}(GP),{6}(D) norm={7} time={2}" .format(util.toCyan("{0}".format(i+1)), opt.toIt, util.toGreen("{0:.2f}".format(time.time()-timeStart)), util.toYellow("X"), util.toYellow("{0:.0e}".format(lrD)), util.toRed("X"), util.toRed("{0:.4f}".format(ld)), util.toBlue("{0:.4f}".format(gdn)))) # if (i+1)%20==0: # runList = [summaryLossTrain,summaryGradTrain] # sl,sg = sess.run(runList,feed_dict=batch) # summaryWriter.add_summary(sl,i+1) # summaryWriter.add_summary(sg,i+1) # if (i+1)%200==0: # si = sess.run(summaryImageTrain,feed_dict=batch) # summaryWriter.add_summary(si,i+1)
# make training batch batch = data.makeBatch(opt,trainData,PH) batch[lrGP_PH] = lrGP batch[lrD_PH] = lrD # update geometric predictor runList = [optimGP,loss_GP,imageComp] for u in range(opt.updateGP): _,lg,ic = sess.run(runList,feed_dict=batch) # update discriminator batch[imageComp] = data.updateHistory(opt,ic) runList = [optimD,loss_D,grad_D_norm_mean] for u in range(opt.updateD): _,ld,gdn = sess.run(runList,feed_dict=batch) if (i+1)%10==0: print("it.{0}/{1} lr={3}(GP),{4}(D) loss={5}(GP),{6}(D) norm={7} time={2}" .format(util.toCyan("{0}".format((opt.warpN-1)*opt.toIt+i+1)), opt.warpN*opt.toIt, util.toGreen("{0:.2f}".format(time.time()-timeStart)), util.toYellow("{0:.0e}".format(lrGP)), util.toYellow("{0:.0e}".format(lrD)), util.toRed("{0:.4f}".format(lg)), util.toRed("{0:.4f}".format(ld)), util.toBlue("{0:.4f}".format(gdn)))) if (i+1)%20==0: runList = [summaryLossTrain,summaryGradTrain] sl,sg = sess.run(runList,feed_dict=batch) summaryWriter.add_summary(sl,(opt.warpN-1)*opt.toIt+i+1) summaryWriter.add_summary(sg,(opt.warpN-1)*opt.toIt+i+1) if (i+1)%200==0: si = sess.run(summaryImageTrain,feed_dict=batch) summaryWriter.add_summary(si,(opt.warpN-1)*opt.toIt+i+1)
dataloader.shipChunk() dataloader.thread = threading.Thread(target=dataloader.loadChunk, args=[opt]) dataloader.thread.start() for i in range(c * opt.itPerChunk, (c + 1) * opt.itPerChunk): lr = opt.lr * opt.lrDecay**(i // opt.lrStep) # make training batch batch = data.makeBatch(opt, dataloader, PH) batch[lr_PH] = lr # run one step runList = [optim, loss, loss_depth, loss_mask, maskLogit] _, l, ld, lm, ml = sess.run(runList, feed_dict=batch) if (i + 1) % 20 == 0: print( "it. {0}/{1}, lr={2}, loss={4} ({5},{6}), time={3}".format( util.toCyan("{0}".format(i + 1)), opt.toIt, util.toYellow("{0:.0e}".format(lr)), util.toGreen("{0:.2f}".format(time.time() - timeStart)), util.toRed("{0:.2f}".format(l)), util.toRed("{0:.2f}".format(ld)), util.toRed("{0:.2f}".format(lm)))) if (i + 1) % 100 == 0: summaryWriter.add_summary( sess.run(summaryLoss, feed_dict=batch), i + 1) if (i + 1) % 500 == 0: summaryWriter.add_summary( sess.run(summaryImage, feed_dict=batch), i + 1) if (i + 1) % 2000 == 0: util.saveModel(opt, sess, saver, i + 1) print(
# for each CAD model in data chunk for i in range(testN): m = idx[i] CAD = dataloader.CADs[m] points24 = np.zeros([opt.inputViewN,1],dtype=np.object) # make test batch batch = { inputImage: dataChunk["image_in"][i] } # evaluate network runList = [XYZid,ML] xyz,ml = sess.run(runList,feed_dict=batch) for a in range(opt.inputViewN): xyz1 = xyz[a].T # [VHW,3] ml1 = ml[a].reshape([-1]) # [VHW] points24[a,0] = xyz1[ml1>0] # output results scipy.io.savemat("results_{0}/{1}/{2}.mat".format(opt.group,opt.load,CAD),{ "image": dataChunk["image_in"][i], "pointcloud": points24 }) pointMeanN = np.array([len(p) for p in points24[:,0]]).mean() print("{0}/{1} ({2}) done (average {3} points), time={4}" .format(util.toCyan("{0}".format(m+1)), util.toCyan("{0}".format(CADN)), CAD, util.toBlue("{0:.2f}".format(pointMeanN)), util.toGreen("{0:.2f}".format(time.time()-timeStart)))) if c!=chunkN-1: dataloader.thread.join() print(util.toYellow("======= EVALUATION DONE ======="))