opt.group, opt.model))
print(util.toYellow("======= TRAINING START ======="))
timeStart = time.time()
# start session
tfConfig = tf.ConfigProto(allow_soft_placement=True)
tfConfig.gpu_options.allow_growth = True
with tf.Session(config=tfConfig) as sess:
sess.run(tf.global_variables_initializer())
if opt.fromIt != 0:
util.restoreModelFromIt(opt, sess, saver, opt.fromIt)
print(
util.toMagenta("resuming from iteration {0}...".format(
opt.fromIt)))
elif opt.load:
util.restoreModel(opt, sess, saver)
print(
util.toMagenta("loading pretrained ({0}) to fine-tune...".format(
opt.load)))
summaryWriter.add_graph(sess.graph)
print(util.toMagenta("start training..."))
chunkResumeN, chunkMaxN = opt.fromIt // opt.itPerChunk, opt.toIt // opt.itPerChunk
# training loop
for c in range(chunkResumeN, chunkMaxN):
dataloader.shipChunk()
dataloader.thread = threading.Thread(target=dataloader.loadChunk,
args=[opt])
dataloader.thread.start()
for i in range(c * opt.itPerChunk, (c + 1) * opt.itPerChunk):
lr = opt.lr * opt.lrDecay**(i // opt.lrStep)
saver_D = tf.train.Saver(var_list=varsD,max_to_keep=20)
summaryWriter = tf.summary.FileWriter(main_folder + "summary_{0}/{1}".format(opt.group,opt.name))
print(util.toYellow("======= TRAINING START ======="))
timeStart = time.time()
# start session
tfConfig = tf.ConfigProto(allow_soft_placement=True)
tfConfig.gpu_options.allow_growth = True
with tf.Session(config=tfConfig) as sess:
sess.run(tf.global_variables_initializer())
summaryWriter.add_graph(sess.graph)
if opt.fromIt!=0:
util.restoreModelFromIt(opt,sess,saver_D,"D",opt.fromIt)
print(util.toMagenta("resuming from iteration {0}...".format(opt.fromIt)))
elif opt.loadD:
util.restoreModel(opt,sess,saver_D,opt.loadD,"D")
print(util.toMagenta("loading pretrained D {0}...".format(opt.loadD)))
print(util.toMagenta("start training..."))
# training loop
for i in range(opt.fromIt,opt.toIt):
lrD = opt.lrD*opt.lrDdecay**(i//opt.lrDstep)
# make training batch
batch = data.makeBatch(opt,trainData,PH)
batch[lrD_PH] = lrD
# update discriminator
runList = [optimD,loss_D,grad_D_norm_mean]
for u in range(opt.updateD):
_,ld,gdn = sess.run(runList,feed_dict=batch)
if (i+1)%10==0:
print("it.{0}/{1} lr={3}(GP),{4}(D) loss={5}(GP),{6}(D) norm={7} time={2}"
# load data
print(util.toMagenta("loading test data..."))
path = "./my_data"
bags = np.load("{0}/bags_smaller_new_128.npy".format(path))
# prepare model saver/summary writer
saver_GP = tf.train.Saver(var_list=varsGP)
print(util.toYellow("======= EVALUATION START ======="))
timeStart = time.time()
# start session
tfConfig = tf.ConfigProto(allow_soft_placement=True)
tfConfig.gpu_options.allow_growth = True
with tf.Session(config=tfConfig) as sess:
sess.run(tf.global_variables_initializer())
util.restoreModel(opt, sess, saver_GP, opt.loadGP, "GP")
print(util.toMagenta("start evaluation..."))
# create directories for test image output
os.makedirs("eval_{0}".format(opt.loadGP), exist_ok=True)
testImage = util.imread(opt.loadImage)
# resize input image size to 128*128
import cv2
testImage = cv2.resize(testImage, dsize=(128, 128))
# end of resize code
batch = data.makeBatchEval(opt, testImage, bags, PH)
runList = [imageCompAll[0], imageCompAll[-1]]
ic0, icf = sess.run(runList, feed_dict=batch)
for b in range(opt.batchSize):
tfConfig = tf.ConfigProto(allow_soft_placement=True)
tfConfig.gpu_options.allow_growth = True
with tf.Session(config=tfConfig) as sess:
sess.run(tf.global_variables_initializer())
summaryWriter.add_graph(sess.graph)
if opt.fromIt!=0:
util.restoreModelFromIt(opt,sess,saver_GP,"GP",opt.fromIt)
util.restoreModelFromIt(opt,sess,saver_D,"D",opt.fromIt)
print(util.toMagenta("resuming from iteration {0}...".format(opt.fromIt)))
elif opt.warpN>1:
util.restoreModelPrevStage(opt,sess,saver_GPprev,"GP")
print(util.toMagenta("loading GP from previous warp..."))
if opt.loadGP=="prev":
util.restoreModelPrevStage(opt,sess,saver_GPcur,"GP")
elif opt.loadGP is not None:
util.restoreModel(opt,sess,saver_GPcur,opt.loadGP,"GP")
print(util.toMagenta("loading pretrained GP ({0})...".format(opt.loadGP)))
util.restoreModelPrevStage(opt,sess,saver_D,"D")
print(util.toMagenta("continue to train D..."))
else:
if opt.loadGP:
util.restoreModel(opt,sess,saver_GPcur,opt.loadGP,"GP")
print(util.toMagenta("loading pretrained GP ({0})...".format(opt.loadGP)))
if opt.loadD:
util.restoreModel(opt,sess,saver_D,opt.loadD,"D")
print(util.toMagenta("loading pretrained D ({0})...".format(opt.loadD)))
print(util.toMagenta("start training..."))
# training loop
for i in range(opt.fromIt,opt.toIt):
lrGP = opt.lrGP*opt.lrGPdecay**(i//opt.lrGPstep)
# load data
print(util.toMagenta("loading MNIST dataset..."))
trainData,validData,testData = data.loadMNIST("data/MNIST.npz")
# prepare model saver/summary writer
saver = tf.train.Saver(max_to_keep=20)
print(util.toYellow("======= EVALUATION START ======="))
timeStart = time.time()
# start session
tfConfig = tf.ConfigProto(allow_soft_placement=True)
tfConfig.gpu_options.allow_growth = True
with tf.Session(config=tfConfig) as sess:
sess.run(tf.global_variables_initializer())
util.restoreModel(opt,sess,saver,opt.toIt)
for idx in range(0,len(validData["image"])):
singleImage = validData["image"][idx]
singleImage = np.reshape(singleImage, [1, 28, 28, 1])
feed_dict = {image: singleImage}
imageValue, imageWrapValue = sess.run([image, imageWarp], feed_dict=feed_dict)
# pdb.set_trace()
imageValue = np.reshape(imageValue, [28,28])
imageWrapValue = np.reshape(imageWrapValue, [28,28])
fig = plt.figure()
ax1 = fig.add_subplot(121)
ax1.imshow(imageValue)
ax2 = fig.add_subplot(122)
ax2.imshow(imageWrapValue)
# load data
print(util.toMagenta("loading MNIST dataset..."))
trainData, testData = data.loadMNIST(opt, "data")
# visdom visualizer
vis = util.Visdom(opt)
print(util.toYellow("======= TRAINING START ======="))
timeStart = time.time()
# start session
with torch.cuda.device(0):
geometric.train()
classifier.train()
if opt.fromIt != 0:
util.restoreModel(opt, geometric, classifier, opt.fromIt)
print(
util.toMagenta("resuming from iteration {0}...".format(
opt.fromIt)))
print(util.toMagenta("start training..."))
# training loop
for i in range(opt.fromIt, opt.toIt):
lrGP = opt.lrGP * opt.lrDecay**(i // opt.lrStep)
lrC = opt.lrC * opt.lrDecay**(i // opt.lrStep)
# make training batch
batch = data.makeBatch(opt, trainData)
image = batch["image"].unsqueeze(dim=1)
label = batch["label"]
# generate perturbation
pInit = data.genPerturbations(opt)
trainData,validData,testData = data.loadMNIST("data/MNIST.npz")
# prepare model saver/summary writer
saver = tf.train.Saver(max_to_keep=20)
summaryWriter = tf.summary.FileWriter("summary_{0}/{1}".format(opt.group,opt.model))
print(util.toYellow("======= TRAINING START ======="))
timeStart = time.time()
# start session
tfConfig = tf.ConfigProto(allow_soft_placement=True)
tfConfig.gpu_options.allow_growth = True
with tf.Session(config=tfConfig) as sess:
sess.run(tf.global_variables_initializer())
summaryWriter.add_graph(sess.graph)
if opt.fromIt!=0:
util.restoreModel(opt,sess,saver,opt.fromIt)
print(util.toMagenta("resuming from iteration {0}...".format(opt.fromIt)))
print(util.toMagenta("start training..."))
# training loop
for i in range(opt.fromIt,opt.toIt):
lrGP = opt.lrGP*opt.lrGPdecay**(i//opt.lrGPstep)
lrC = opt.lrC*opt.lrCdecay**(i//opt.lrCstep)
# make training batch
batch = data.makeBatch(opt,trainData,PH)
batch[lrGP_PH] = lrGP
batch[lrC_PH] = lrC
# run one step
_,l = sess.run([optim,loss],feed_dict=batch)
if (i+1)%100==0:
print("it. {0}/{1} lr={3}(GP),{4}(C), loss={5}, time={2}"
