def testing(seman_in):
label_images1 = Variable(
torch.unsqueeze(torch.from_numpy(
helper.get_semantic_map(seman_in)).float().permute(2, 0, 1),
dim=0))
global D_m
global D
global count
D = []
D_m = []
count = 0
label_images = torch.cat(
(label_images1, (1 - label_images1.sum(1)).unsqueeze(1)), 1)
label_images = label_images #.cuda()
res = 256
G_temp = recursive_img(label_images, res)
model = cascaded_model(D_m)
model = model.cuda()
model.load_state_dict(torch.load('mynet_200epoch_CRN.pth'))
model = model.cpu().eval()
G = model(D, label_images)
Generator = G.permute(0, 2, 3, 1)
Generator = Generator
Generator = Generator.data.numpy()
output = np.minimum(np.maximum(Generator, 0.0), 255.0)
scipy.misc.toimage(output[2, :, :, :], cmin=0, cmax=255).save("val3.jpg")
def testing(seman_in):
label_images1 = Variable(
torch.unsqueeze(torch.from_numpy(
helper.get_semantic_map(seman_in)).float().permute(2, 0, 1),
dim=0))
label_images = torch.cat(
(label_images1, (1 - label_images1.sum(1)).unsqueeze(1)), 1)
label_images = label_images # .cuda()
res = 256
model = cascaded_model(label_images, res)
model = model.cuda()
model.load_state_dict(torch.load('crn0/mynet_200epoch_CRN.pth'))
model = model.cpu().eval()
G = model(label_images)
Generator = G.permute(0, 2, 3, 1)
Generator = Generator
Generator = Generator.data.numpy()
output = np.minimum(np.maximum(Generator, 0.0), 255.0)
result = scipy.misc.toimage(output[2, :, :, :], cmin=0, cmax=255)
result.save('torch-result.jpg')
scipy.misc.toimage(output[2, :, :, :], cmin=0, cmax=255).save("val4.jpg")
saver = tf.train.Saver(max_to_keep=1000)
if is_training:
g_loss = np.zeros(3000, dtype=float)
input_images = [None] * 3000
label_images = [None] * 3000
for epoch in range(1, 21):
if os.path.isdir("result_512p/%04d" % epoch):
continue
cnt = 0
for ind in np.random.permutation(2975) + 1:
st = time.time()
cnt += 1
if input_images[ind] is None:
label_images[ind] = helper.get_semantic_map(
"data/cityscapes/Label512Full/%08d.png" %
ind) #training label
input_images[ind] = np.expand_dims(
np.float32(
scipy.misc.imread(
"data/cityscapes/RGB512Full_vivid/%08d.png" %
ind)),
axis=0
) #training image with vivid appearance. see "optional_preprocessing"
_, G_current, l0, l1, l2, l3, l4, l5 = sess.run(
[G_opt, G_loss, p0, p1, p2, p3, p4, p5],
feed_dict={
label:
np.concatenate(
(label_images[ind],
np.expand_dims(1 - np.sum(label_images[ind], axis=3),
st=time.time()
for batch_count in range(0, batch_size):
if(ind>=total_number):
ind = 0
cnt += 1
rand_folder = np.random.permutation(num_folder) + 1
try:
dic = scipy.io.loadmat(mat_path +"%02d"%rand_folder[0]+"/%08d.mat" % tmp_list[ind])
except:
print("cannot load" + "%08d.mat" % tmp_list[ind])
print(mat_path +"%02d"%rand_folder[0]+"/%08d.mat" % tmp_list[ind])
ind = ind + 1
global_ind = global_ind + 1
continue
tmp_label[batch_count, :, :, :] = helper.get_semantic_map(
label_path + "/%02d" % rand_folder[0] + "/%08d.png" % tmp_list[ind])
tmp_image[batch_count, :, :, :] = np.expand_dims(
np.float32(cv2.imread(rgb_path+"/%08d.png" % tmp_list[ind])), axis=0)
tmp_label_gt[batch_count, :,:,:] = helper.get_semantic_map(
data_path + label_gt_folder + "/%08d.png" % tmp_list[ind])
tmp_proposal[batch_count,:,:,:] = (np.expand_dims(dic['proposal'],axis = 0).astype(np.float32))/255.0#+ np.expand_dims(dic2['proposal'],axis = 0)
tmp_z = np.sum(tmp_proposal[batch_count, :, :, :], axis=-1)
tmp_z[np.where(tmp_z > 0)] = 1
tmp_mask[batch_count, :, :, :] = np.expand_dims(tmp_z, axis=3).astype(np.float32)
z = np.concatenate((tmp_label, np.expand_dims(1 - np.sum(tmp_label, axis=1), axis=1)), axis=1)
z = z.transpose([0,2,3,1])
saver = tf.train.Saver(max_to_keep=1000)
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state("gta_demo/final")
if ckpt:
print('loaded ' + ckpt.model_checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
#Find test semantic layouts
if not os.path.isdir("gta_demo/result"):
os.makedirs("gta_demo/result")
lst = os.listdir("./datasets/GTA/Label256Full/")
lst = [x for x in lst if x.endswith(".png")]
#Testing
for ind in lst:
semantic = helper.get_semantic_map("./datasets/GTA/Label256Full/%s" % ind)
semantic = np.concatenate(
(semantic, np.expand_dims(1 - np.sum(semantic, axis=3), axis=3)),
axis=3)
output = np.empty([9, 256, 512, 3], dtype=np.float32)
for i in range(9):
semantic1 = np.concatenate(
(semantic,
np.random.randn(semantic.shape[0], semantic.shape[1],
semantic.shape[2], num_noise)),
axis=3)
output[i] = sess.run(generator, feed_dict={label: semantic1})
output = np.minimum(np.maximum(output, 0.0), 255.0)
upper = np.concatenate(
(output[0, :, :, :], output[1, :, :, :], output[2, :, :, :]), axis=1)
middle = np.concatenate(
print('loaded ' + ckpt.model_checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
if is_training:
input_images = [None] * 20000
label_images = [None] * 20000
for epoch in range(1, 401):
if os.path.isdir("result_GTA/%04d" % epoch):
continue
g_loss = np.zeros(20000, dtype=float)
cnt = 0
for ind in np.random.permutation(12403) + 1:
st = time.time()
cnt += 1
if input_images[ind] is None:
label_images[ind] = helper.get_semantic_map(
"data/GTA/Label256Full/%08d.png" % ind)
input_images[ind] = np.expand_dims(np.float32(
scipy.misc.imread("data/GTA/RGB256Full/%08d.png" % ind)),
axis=0)
_, G_current, l0, l1, l2, l3, l4, l5 = sess.run(
[G_opt, G_loss, p0, p1, p2, p3, p4, p5],
feed_dict={
label:
np.concatenate(
(label_images[ind],
np.expand_dims(1 - np.sum(label_images[ind], axis=3),
axis=3)),
axis=3),
real_image:
input_images[ind],
lr:
bottom = np.concatenate(
(output[6, :, :, :], output[7, :, :, :], output[8, :, :, :]),
axis=1)
scipy.misc.toimage(
np.concatenate((upper, middle, bottom), axis=0),
cmin=0,
cmax=255).save("result_256p/%04d/%06d_output.jpg" %
(epoch, ind))
if not os.path.isdir("result_256p/final"):
os.makedirs("result_256p/final")
for ind in range(100001, 100501):
if not os.path.isfile(
"data/cityscapes/Label512Full/%08d.png" % ind): #test label
continue
bsemantic = helper.get_semantic_map(
"data/cityscapes/Label512Full/%08d.png" % ind) #test label
b, h, w, c = bsemantic.shape
semantic = np.zeros([b, h // 2, w // 2, c])
for i in range(c):
for j in range(b):
semantic[j, :, :,
i] = cv2.resize(bsemantic[j, :, :, i],
(semantic.shape[2], semantic.shape[1]),
cv2.INTER_NEAREST)
output = sess.run(
generator,
feed_dict={
label:
np.concatenate(
(semantic, np.expand_dims(1 - np.sum(semantic, axis=3),
objectlist = ['road', 'building', 'sky', 'other', 'sidewalk']
objectid = [0, 2, 10, 19, 1]
objectnum = 5
rarity_bin = []
for i in range(objectnum):
rarity_bin.append(
np.load("rarity/kdecolor_rarity_bin_%d.npy" % objectid[i]))
#Start training
for ind in range(reinit_point + 11):
for i in range(K):
#Sample image based on rarity bin
idx = np.searchsorted(rarity_bin[(i) % objectnum],
np.random.rand()) + 1
semantic = helper.get_semantic_map(labelroot +
"/%08d.png" % idx) #test label
semantic = np.concatenate(
(semantic, np.expand_dims(1 - np.sum(semantic, axis=3),
axis=3)),
axis=3)
label_images[i] = semantic
input_images[i] = np.float32(
scipy.misc.imread(imageroot +
"/%08d.png" % idx)) #training image
loss_masks[i] = rarity_mask[idx - 1]
mindist = np.inf
#Nearest Neighbor search
for j in range(nn_num):
semantic = label_images[i]
for epoch in range(1, epoch_range):
print_message('%s ---------------> %d' %
(getFormattedTimeStr(), epoch))
if os.path.isdir("%s/%04d" % (checkpoint_name, epoch)):
continue
cnt = 0
for ind in np.random.permutation(MAX_TRAINING_IMG):
if (cnt > img_cnt_range):
break
# A weird naming thing going on here, this is a temp fix
# I think they use 0-2975 for training
# and 100001-100501 for test/validation
st = time.time()
cnt += 1
if input_images[ind] is None:
label_images[ind] = helper.get_semantic_map(
"data/train-256p/%08d.png" % ind) # training label
input_images[ind] = np.expand_dims(np.float32(
scipy.misc.imread("data/train-256p/%08d.png" % ind)),
axis=0) # training image
input_images[ind] = input_images[ind][:, :, :, :3]
_, G_current, l0, l1, l2, l3, l4, l5 = sess.run(
[G_opt, G_loss, p0, p1, p2, p3, p4, p5],
feed_dict={
label:
np.concatenate(
(label_images[ind],
np.expand_dims(1 - np.sum(label_images[ind], axis=3),
axis=3)),
axis=3),
real_image:
input_images[ind],
rand_folder = np.random.permutation(num_folder) + 1
try:
dic = scipy.io.loadmat(mat_path + "%02d" % rand_folder[0] +
"/%08d.mat" % tmp_list[ind])
except:
print("cannot load" + "%08d.mat" % tmp_list[ind])
print(mat_path + "%02d" % rand_folder[0] +
"/%08d.mat" % tmp_list[ind])
ind = ind + 1
global_ind = global_ind + 1
continue
cnt += 1
num = dic['proposal'].shape[0]
tmp_label[:, :, :, :] = helper.get_semantic_map(label_path +
"/%08d.png" %
tmp_list[ind])
tmp_image[:, :, :, :] = np.expand_dims(np.float32(
cv2.imread(rgb_path + "/%08d.png" % tmp_list[ind])),
axis=0)
if (num <= num_proposal):
tmp_mask[0:num, :, :, :] = np.expand_dims(dic['mask'],
axis=3).astype(
np.float32)
tmp_proposal[0:num, :, :, :] = (dic['proposal']).astype(
np.float32) / 255.0
if (num > num_proposal):
d = np.random.permutation(num)
tmp_mask[:, :, :, :] = np.expand_dims(
dic['mask'], axis=3
).astype(
print('loaded '+ckpt.model_checkpoint_path)
saver.restore(sess,ckpt.model_checkpoint_path)
if is_training:
g_loss=np.zeros(3000,dtype=float)
input_images=[None]*3000
label_images=[None]*3000
for epoch in range(1,201):
if os.path.isdir("result_256p/%04d"%epoch):
continue
cnt=0
for ind in np.random.permutation(2975)+1:
st=time.time()
cnt+=1
if input_images[ind] is None:
label_images[ind]=helper.get_semantic_map("data/cityscapes/Label256Full/%08d.png"%ind)#training label
input_images[ind]=np.expand_dims(np.float32(scipy.misc.imread("data/cityscapes/RGB256Full/%08d.png"%ind)),axis=0)#training image
_,G_current,l0,l1,l2,l3,l4,l5=sess.run([G_opt,G_loss,p0,p1,p2,p3,p4,p5],feed_dict={label:np.concatenate((label_images[ind],np.expand_dims(1-np.sum(label_images[ind],axis=3),axis=3)),axis=3),real_image:input_images[ind],lr:1e-4})#may try lr:min(1e-6*np.power(1.1,epoch-1),1e-4 if epoch>100 else 1e-3) in case lr:1e-4 is not good
g_loss[ind]=G_current
print("%d %d %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f"%(epoch,cnt,np.mean(g_loss[np.where(g_loss)]),np.mean(l0),np.mean(l1),np.mean(l2),np.mean(l3),np.mean(l4),np.mean(l5),time.time()-st))
os.makedirs("result_256p/%04d"%epoch)
target=open("result_256p/%04d/score.txt"%epoch,'w')
target.write("%f"%np.mean(g_loss[np.where(g_loss)]))
target.close()
saver.save(sess,"result_256p/model.ckpt")
if epoch%20==0:
saver.save(sess,"result_256p/%04d/model.ckpt"%epoch)
for ind in range(100001,100051):
if not os.path.isfile("data/cityscapes/Label256Full/%08d.png"%ind):#test label
continue
semantic=helper.get_semantic_map("data/cityscapes/Label256Full/%08d.png"%ind)#test label
def training(M):
res = 256
label_dir = 'Label256Full'
l = os.listdir(label_dir)
for epoch in range(200):
running_loss = 0
c_t = 0
for I in enumerate(l):
c_t += 1
global D_m
global D
global count
D = []
D_m = []
count = 0
J = str.replace(I[1], 'gtFine_color.png', 'leftImg8bit.png')
label_images1 = Variable(
torch.unsqueeze(torch.from_numpy(
helper.get_semantic_map('Label256Full/' +
I[1])).float().permute(2, 0, 1),
dim=0)) #.cuda()#training label
input_images = Variable(
torch.unsqueeze(torch.from_numpy(io.imread("RGB256Full/" +
J)).float(),
dim=0).permute(0, 3, 1, 2))
label_images = torch.cat(
(label_images1, (1 - label_images1.sum(1)).unsqueeze(1)), 1)
input_images = input_images.cuda()
label_images = label_images.cuda()
G_temp = recursive_img(label_images, res)
if M == 0:
model = cascaded_model(D_m)
model = model.cuda()
# model.load_state_dict(torch.load('mynet_updated.pth')) # if u want to resume training from a pretrained model then add the .pth file here
optimizer = optim.Adam(model.parameters(),
lr=0.0001,
betas=(0.9, 0.999),
eps=1e-08,
weight_decay=0)
optimizer.zero_grad()
Generator = model(D, label_images)
Loss = loss_function(input_images, Generator, label_images, D)
Loss.backward()
optimizer.step()
M = 1
running_loss += Loss.data[0]
print(epoch, c_t, Loss.data[0])
del D
del D_m
del count
del Loss, label_images, G_temp, input_images
shuffle(l)
epoch_loss = running_loss / 2975.0 #can replace the 2975 with c_t for generalization
print(epoch, epoch_loss)
if epoch % 2 == 0:
Generator = Generator.permute(0, 2, 3, 1)
Generator = Generator.cpu()
Generator = Generator.data.numpy()
output = np.minimum(np.maximum(Generator, 0.0), 255.0)
scipy.misc.toimage(output[0, :, :, :], cmin=0,
cmax=255).save("%06d_output_real.jpg" % epoch)
#epoch_acc = running_corrects / 2975.0
# return Loss
best_model_wts = model.state_dict()
model.load_state_dict(best_model_wts)
return model
print('loaded '+ckpt.model_checkpoint_path)
saver.restore(sess,ckpt.model_checkpoint_path)
if is_training:
input_images=[None]*20000
label_images=[None]*20000
for epoch in range(1,401):
if os.path.isdir("result_GTA/%04d"%epoch):
continue
g_loss=np.zeros(20000,dtype=float)
cnt=0
for ind in np.random.permutation(12403)+1:
st=time.time()
cnt+=1
if input_images[ind] is None:
label_images[ind]=helper.get_semantic_map("data/GTA/Label256Full/%08d.png"%ind)
input_images[ind]=np.expand_dims(np.float32(scipy.misc.imread("data/GTA/RGB256Full/%08d.png"%ind)),axis=0)
_,G_current,l0,l1,l2,l3,l4,l5=sess.run([G_opt,G_loss,p0,p1,p2,p3,p4,p5],feed_dict={label:np.concatenate((label_images[ind],np.expand_dims(1-np.sum(label_images[ind],axis=3),axis=3)),axis=3),real_image:input_images[ind],lr:1e-4})
g_loss[ind]=G_current
print("%d %d %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f"%(epoch,cnt,np.mean(g_loss[np.where(g_loss)]),np.mean(l0),np.mean(l1),np.mean(l2),np.mean(l3),np.mean(l4),np.mean(l5),time.time()-st))
if cnt>3000:
break
os.makedirs("result_GTA/%04d"%epoch)#save models
target=open("result_GTA/%04d/score.txt"%epoch,'w')
target.write("%f"%np.mean(g_loss[np.where(g_loss)]))
target.close()
saver.save(sess,"result_GTA/model.ckpt")
if epoch%100==0:
saver.save(sess,"result_GTA/%04d/model.ckpt"%epoch)
for ind in range(12403-49,12403+1)+range(100001,100051):#print intermediate results
semantic=helper.get_semantic_map("data/GTA/Label256Full/%08d.png"%ind)
