def test(filenameSave, model, dataloader_test, args):
for step, (images, labels, filename, filenameGt) in enumerate(dataloader_test):
if (args.cuda):
images = images.cuda()
# labels = labels.cuda()
inputs = Variable(images)
# targets = Variable(labels)
with torch.no_grad():
outputs = model(inputs)
label = outputs[0].max(0)[1].byte().cpu().data
# label_cityscapes = cityscapes_trainIds2labelIds(label.unsqueeze(0))
label_color = Colorize()(label.unsqueeze(0))
# image_transform(label.byte()).save(filenameSave)
# label_save = ToPILImage()(label_color)
label_save = label_color.numpy()
label_save = label_save.transpose(1, 2, 0)
# label_save.save(filenameSave)
images = images.cpu().numpy().squeeze(axis=0).transpose(1, 2, 0)
images = (images*255).astype(np.uint8)
for i in range(len(filename)):
fileSave = '../eval/'+ args.savedir + "/" + filename[i].split("leftImg8bit/")[1]
os.makedirs(os.path.dirname(fileSave), exist_ok=True)
output = cv2.addWeighted(images, 0.4, label_save, 0.6, 0)
cv2.imwrite(fileSave,output)
def inference(model, args):
image_folder = "/media/pandongwei/ExtremeSSD/work_relative/extract_img/2020.10.16_1/"
video_save_path = "/home/pandongwei/work_repository/erfnet_pytorch/eval/"
# parameters about saving video
fourcc = cv2.VideoWriter_fourcc(*'XVID')
out = cv2.VideoWriter(video_save_path + 'output_new.avi', fourcc, 10.0, (640, 480))
cuda = True
model.eval()
paths = []
for root, dirs, files in os.walk(image_folder, topdown=True):
for file in files:
image_path = os.path.join(image_folder, file)
paths.append(image_path)
paths.sort()
font = cv2.FONT_HERSHEY_SIMPLEX
angle_pre = 0
for i, path in enumerate(paths):
start_time = time.time()
image = cv2.imread(path)
image = (image / 255.).astype(np.float32)
image = ToTensor()(image).unsqueeze(0)
if (cuda):
image = image.cuda()
input = Variable(image)
with torch.no_grad():
output = model(input)
label = output[0].max(0)[1].byte().cpu().data
# label_cityscapes = cityscapes_trainIds2labelIds(label.unsqueeze(0))
label_color = Colorize()(label.unsqueeze(0))
label_save = label_color.numpy()
label_save = label_save.transpose(1, 2, 0)
# 加上路径规划
label_save, angle = perception_to_angle(label_save, angle_pre)
# 加一个滤波以防止角度突然跳变 TODO
if abs(angle - angle_pre) > 10:
angle = angle_pre
angle_pre = angle
# label_save.save(filenameSave)
image = image.cpu().numpy().squeeze(axis=0).transpose(1, 2, 0)
image = (image * 255).astype(np.uint8)
output = cv2.addWeighted(image, 0.5, label_save, 0.5, 0)
cv2.putText(output,str(round(angle,3)),(50,50),cv2.FONT_HERSHEY_SIMPLEX,2,(0,0,0),2)
#output = np.hstack([label_save, image])
out.write(output)
print(i, " time: %.2f s" % (time.time() - start_time))
out.release()
def inference(model, image, angle_pre):
# pre-process
image = (image / 255.).astype(np.float32)
image = ToTensor()(image).unsqueeze(0)
image = image.cuda()
input = Variable(image)
# inference
with torch.no_grad():
output = model(input)
# post-process
label = output[0].max(0)[1].byte().cpu().data
label_color = Colorize()(label.unsqueeze(0))
label_save = label_color.numpy()
label_save = label_save.transpose(1, 2, 0)
# 加上路径规划
label_save, angle = perception_to_angle(label_save, angle_pre)
return angle
def callback(self, oimg):
try:
#if you want to save images and labels ,please uncomment following codes(No.1 to No.4).
#NO.1
write_image_name = "image_" + str(self.count) + ".jpg"
#No.2
write_label_name = "label_" + str(self.count) + ".jpg"
oimg_b = bytes(oimg.data)
np_arr = np.fromstring(oimg_b, np.uint8)
img = cv2.imdecode(np_arr, cv2.IMREAD_COLOR)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
#No.3
#cv2.imwrite("/home/amsl/images/output/seg_pub_2/image/" + write_image_name, img)
image = PIL_Image.fromarray(img)
#image = image.crop((0, 120, 640, 480))
image = image.crop((0, 10, 640, 330))
#image = image.resize((1024,512),PIL_Image.NEAREST)
image = image.resize((1024,512),PIL_Image.NEAREST)
#img_size = image.shape
image = ToTensor()(image)
image = torch.Tensor(np.array([image.numpy()]))
image = image.cuda()
input_image = Variable(image)
with torch.no_grad():
output_image = self.model(input_image)
label = output_image[0].max(0)[1].byte().cpu().data
label_color = Colorize()(label.unsqueeze(0))
label_pub = ToPILImage()(label_color)
#label_pub = label_pub.resize((1024, 512),PIL_Image.NEAREST)
#label_pub = label_pub.resize((1024, 512),PIL_Image.LANCZOS)
label_pub = np.asarray(label_pub)
#show label.
#plt.imshow(label_pub)
#plt.pause(0.001)
#No.4
#cv2.imwrite("/home/amsl/images/output/seg_pub_2/label/" + write_label_name, label_pub)
self.pub_seg.publish(self.bridge.cv2_to_imgmsg(label_pub, "bgr8"))
print("published")
self.count += 1
except CvBridgeError as e:
print(e)
def vis_using_Colorize(indir_list, outdir):
indir = indir_list[0]
# outdir = os.path.join(os.path.split(indir)[0], "vis_labels")
mkdir_if_not_exist(outdir)
for one_file in tqdm(os.listdir(indir)):
fullpath = os.path.join(indir, one_file)
hard_to_see_img = m.imread(fullpath)
# outputs = outputs[0, :19].data.max(0)[1]
# outputs = outputs.view(1, outputs.size()[0], outputs.size()[1])
outputs = hard_to_see_img # TODO this should be fixed
output = Colorize()(outputs)
output = np.transpose(output.cpu().numpy(), (1, 2, 0))
img = Image.fromarray(output, "RGB")
img = img.resize(hard_to_see_img.shape, Image.NEAREST)
outfn = os.path.join(outdir, one_file)
plt.savefig(outfn, transparent=True, bbox_inches='tight', pad_inches=0)
img.save(outfn)
def ensemble_predict(prob_fns, outfile='sample.png',
out_npy_file='sample.npy', out_vis_file='sample_vis.png',
method='averaging', out_shape=(2048, 1024)):
"""Output predict file from two npy files by the given method."""
probs = [np.load(prob_fn) for prob_fn in prob_fns]
if method == 'averaging':
prob = sum(probs) / len(probs)
elif method == 'nms':
prob = np.max(probs, 0)
# -- output npy --
np.save(out_npy_file, prob)
# -- output label-predict --
pred = prob[:N_CLASS].argmax(0)
img = Image.fromarray(np.uint8(pred))
img = img.resize(out_shape, Image.NEAREST)
img.save(outfile)
# -- output vis-predict --
# ToTensor function: ndarray -> Tensor
# * H, W, C -> C, H, W
# * 0, 255 -> 0, 1
# prob.shape == (20, 512, 1024)
prob_for_tensor = np.transpose(prob[:N_CLASS], (1, 2, 0))
prob_tensor = ToTensor()(prob_for_tensor)
pred = prob_tensor.max(0)[1]
pred = pred.view(1, pred.size()[0], pred.size()[1])
# Colorize function: Tensor -> Tensor
vis_tensor = Colorize()(pred)
# Tensor -> ndarray -> image(save)
vis = np.transpose(vis_tensor.numpy(), (1, 2, 0))
vis_img = Image.fromarray(vis, 'RGB')
vis_img = vis_img.resize(out_shape, Image.NEAREST)
vis_img.save(out_vis_file)
def talker(args):
NUM_CLASSES = 4
color_transform = Colorize(NUM_CLASSES)
# Load Model
savedir = f'../save/{args.savedir}'
if not os.path.exists(savedir):
os.makedirs(savedir)
model = ERFNet(NUM_CLASSES)
if args.cuda:
model = torch.nn.DataParallel(model).cuda()
model_dir = args.model_dir
def load_my_state_dict(model, state_dict):
# state_dict = state_dict["state_dict"]
own_state = model.state_dict()
for name, param in state_dict.items():
if name not in own_state:
continue
own_state[name].copy_(param)
return model
model = load_my_state_dict(model, torch.load(model_dir))
model.eval()
# parameters about saving video
video_save_path = args.video_save_path
fourcc = cv2.VideoWriter_fourcc(*'XVID')
out = cv2.VideoWriter(video_save_path + 'output_new.avi', fourcc, 10.0,
(640, 480))
ic = image_converter(model, out)
rospy.init_node('talker', anonymous=True)
try:
rospy.spin()
except KeyboardInterrupt:
print("Shutting down")
out.release()
def main(args):
modelpath = args.loadDir + args.loadModel
weightspath = args.loadDir + args.loadWeights
print("Loading model: " + modelpath)
print("Loading weights: " + weightspath)
model = Net(NUM_CLASSES)
model = torch.nn.DataParallel(model)
if (not args.cpu):
model = model.cuda()
def load_my_state_dict(model, state_dict):
own_state = model.state_dict()
for a in own_state.keys():
print(a)
for a in state_dict.keys():
print(a)
print('-----------')
for name, param in state_dict.items():
if name not in own_state:
continue
own_state[name].copy_(param)
return model
model = load_my_state_dict(model, torch.load(weightspath))
print("Model and weights LOADED successfully")
model.eval()
if (not os.path.exists(args.datadir)):
print("Error: datadir could not be loaded")
loader = DataLoader(cityscapes(args.datadir,
input_transform_cityscapes,
subset=args.subset),
num_workers=args.num_workers,
batch_size=args.batch_size,
shuffle=False)
# For visualizer:
# must launch in other window "python3.6 -m visdom.server -port 8097"
# and access localhost:8097 to see it
if (args.visualize):
vis = visdom.Visdom()
time_all = []
with torch.no_grad():
for step, (images, filename) in enumerate(loader):
images = images.cuda()
start_time = time.time()
outputs = model(images)
fwt = time.time() - start_time
time_all.append(fwt)
label = outputs[0].cpu().max(0)[1].data.byte()
label_color = Colorize()(label.unsqueeze(0))
filenameSave = "./save_color/" + filename[0].split(
"leftImg8bit/")[1]
os.makedirs(os.path.dirname(filenameSave), exist_ok=True)
label_save = ToPILImage()(label_color)
label_save.save(filenameSave)
print(step, filenameSave)
print("FPS (Mean: %.4f)" % (1.0000 /
(sum(time_all) / len(time_all))))
from torchvision.transforms import Compose, CenterCrop, Normalize, Resize, Pad
from torchvision.transforms import ToTensor, ToPILImage
from dataset import VOC12, cityscapes
from transform import Relabel, ToLabel, Colorize
from visualize import Dashboard
import importlib
from iouEval import iouEval, getColorEntry
from shutil import copyfile
NUM_CHANNELS = 3
NUM_CLASSES = 34 #pascal=22, cityscapes=20
color_transform = Colorize(NUM_CLASSES)
image_transform = ToPILImage()
#Augmentations - different function implemented to perform random augments on both image and target
class MyCoTransform(object):
def __init__(self, enc, augment=True, height=512):
self.enc=enc
self.augment = augment
self.height = height
pass
def __call__(self, input, target):
# do something to both images
input = Resize(self.height, Image.BILINEAR)(input)
target = Resize(self.height, Image.NEAREST)(target)
if(self.augment):
from dataset import CityScapes,CityScapes_validation from network import LinkNet34 from criterion import CrossEntropyLoss2d from transform import Relabel, ToLabel, Colorize #import deeplab_resnet import torch.nn.functional as F #from accuracy_metrics import pixel_accuracy,mean_accuracy,mean_IU from accuracy_metrics import pixel_accuracy NUM_CHANNELS = 3 NUM_CLASSES = 35 #6 for brats color_transform = Colorize() image_transform = ToPILImage() input_transform = Compose([ #CenterCrop(256), #Scale(240), Resize((512,1024),Image.NEAREST), ToTensor(), Normalize([73.158359/255.0, 82.908917/255.0, 72.392398/255.0], [11.847663/255.0, 10.710858/255.0, 10.358618/255.0]), ]) input_transform1 = Compose([ #CenterCrop(256), ToTensor(), ]) target_transform = Compose([
optimizer_feat = torch.optim.Adam(res101.parameters(), lr=1e-4)
for t in range(10):
for i, (img, label) in enumerate(loader):
img = img.cuda()
label = label[0].cuda()
label = Variable(label)
input = Variable(img)
feats = res101(input)
output = seg(feats)
seg.zero_grad()
res101.zero_grad()
loss = criterion(output, label)
loss.backward()
optimizer_feat.step()
optimizer_seg.step()
## see
input = make_image_grid(img, mean, std)
label = make_label_grid(label.data)
label = Colorize()(label).type(torch.FloatTensor)
output = make_label_grid(torch.max(output, dim=1)[1].data)
output = Colorize()(output).type(torch.FloatTensor)
writer.add_image('image', input, i)
writer.add_image('label', label, i)
writer.add_image('pred', output, i)
writer.add_scalar('loss', loss.data[0], i)
print "epoch %d step %d, loss=%.4f" % (t, i, loss.data.cpu()[0])
def main(args):
modelpath = args.loadDir + args.loadModel
weightspath = args.loadDir + args.loadWeights
print("Loading model: " + modelpath)
print("Loading weights: " + weightspath)
#Import ERFNet model from the folder
#Net = importlib.import_module(modelpath.replace("/", "."), "ERFNet")
model = ERFNet(NUM_CLASSES)
model = torch.nn.DataParallel(model)
if (not args.cpu):
model = model.cuda()
#model.load_state_dict(torch.load(args.state))
#model.load_state_dict(torch.load(weightspath)) #not working if missing key
fourcc = cv2.VideoWriter_fourcc(*'MP4V') # Save as video
out = cv2.VideoWriter('output.mp4', fourcc, 20.0, (640, 352))
def load_my_state_dict(
model, state_dict
): #custom function to load model when not all dict elements
own_state = model.state_dict()
for name, param in state_dict.items():
if name not in own_state:
continue
own_state[name].copy_(param)
return model
model = load_my_state_dict(
model, torch.load(weightspath, map_location=torch.device('cpu')))
print("Model and weights LOADED successfully")
model.eval()
# loader = DataLoader(cityscapes(args.datadir, input_transform_cityscapes, target_transform_cityscapes, subset=args.subset),
# num_workers=args.num_workers, batch_size=args.batch_size, shuffle=False)
# For visualizer:
# must launch in other window "python3.6 -m visdom.server -port 8097"
# and access localhost:8097 to see it
# if (args.visualize):
# vis = visdom.Visdom()
# for step, (images, labels, filename, filenameGt) in enumerate(loader):
# cap = cv2.VideoCapture(0)
cap = cv2.VideoCapture('project_video_trimmed.mp4')
while (True):
# Capture frame-by-frame
ret, images = cap.read()
# print(images.shape)
images = trans(images)
images = images.float()
images = images.view((1, 3, 352, 640)) # vidoe
# Our operations on the frame come here
if (not args.cpu):
images = images.cuda()
inputs = Variable(images)
with torch.no_grad():
outputs = model(inputs)
label = outputs[0].max(0)[1].byte().cpu().data
#label_cityscapes = cityscapes_trainIds2labelIds(label.unsqueeze(0))
label_color = Colorize()(label.unsqueeze(0))
frame = label_color.numpy().transpose(1, 2, 0)
# label_save = ToPILImage()(label_color)
# label_save.save("result_1.png")
# Display the resulting frame
cv2.imshow('Segmented Image', frame)
out.write(frame) # To save video file
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
out.release()
cv2.destroyAllWindows()
if (args.visualize):
vis.image(label_color.numpy())
print(step, filenameSave)
def main(args, get_dataset):
modelpath = args.loadDir + args.loadModel
weightspath = args.loadDir + args.loadWeights
print("Loading model: " + modelpath)
print("Loading weights: " + weightspath)
NUM_CLASSES = get_dataset.num_labels
model = Net(NUM_CLASSES, args.em_dim, args.resnet)
model = torch.nn.DataParallel(model)
if (not args.cpu):
model = model.cuda()
def load_my_state_dict(
model, state_dict
): #custom function to load model when not all dict elements
own_state = model.state_dict()
for a in own_state.keys():
print(a)
for a in state_dict.keys():
print(a)
print('-----------')
for name, param in state_dict.items():
if name not in own_state:
continue
own_state[name].copy_(param)
return model
model = load_my_state_dict(model, torch.load(weightspath))
print("Model and weights LOADED successfully")
model.eval()
if (not os.path.exists(args.datadir)):
print("Error: datadir could not be loaded")
loader = DataLoader(cityscapes(args.datadir,
input_transform_cityscapes,
subset=args.subset),
num_workers=args.num_workers,
batch_size=args.batch_size,
shuffle=False)
# For visualizer:
# must launch in other window "python3.6 -m visdom.server -port 8097"
# and access localhost:8097 to see it
if (args.visualize):
vis = visdom.Visdom()
with torch.set_grad_enabled(False):
for step, (images, filename) in enumerate(loader):
images = images.cuda()
outputs = model(images, enc=False)
outputs = outputs['MAP']
label = outputs[0].cpu().max(0)[1].data.byte()
label_color = Colorize()(label.unsqueeze(0))
filenameSave = "./save_color/" + filename[0].split(
"leftImg8bit/")[1]
os.makedirs(os.path.dirname(filenameSave), exist_ok=True)
label_save = ToPILImage()(label_color)
label_save.save(filenameSave)
label = outputs[1].cpu().max(0)[1].data.byte()
label_color = Colorize()(label.unsqueeze(0))
filenameSave = "./save_color/" + filename[1].split(
"leftImg8bit/")[1]
os.makedirs(os.path.dirname(filenameSave), exist_ok=True)
label_save = ToPILImage()(label_color)
label_save.save(filenameSave)
print(step, filenameSave)
def main(args):
modelpath = args.loadDir + args.loadModel
weightspath = args.loadDir + args.loadWeights
print("Loading model: " + modelpath)
print("Loading weights: " + weightspath)
model = Net(NUM_CLASSES)
model = torch.nn.DataParallel(model)
if (not args.cpu):
model = model.cuda()
# model.load_state_dict(torch.load(args.state))
# model.load_state_dict(torch.load(weightspath)) #not working if missing key
def load_my_state_dict(
model, state_dict
): # custom function to load model when not all dict elements
own_state = model.state_dict()
for name, param in state_dict.items():
if name not in own_state:
continue
own_state[name].copy_(param)
return model
model = load_my_state_dict(model, torch.load(weightspath))
print("Model and weights LOADED successfully")
model.eval()
if (not os.path.exists(args.datadir)):
print("Error: datadir could not be loaded")
# loader = DataLoader(
# cityscapes('/home/liqi/PycharmProjects/LEDNet/4.png', input_transform_cityscapes, target_transform_cityscapes, subset=args.subset),
# num_workers=args.num_workers, batch_size=1 ,shuffle=False)
input_transform_cityscapes = Compose([
Resize((512, 1024), Image.BILINEAR),
ToTensor(),
# Normalize([.485, .456, .406], [.229, .224, .225]),
])
name = "4.png"
with open(image_path_city('/home/gongyiqun/images', name), 'rb') as f:
images = load_image(f).convert('RGB')
images = input_transform_cityscapes(images)
# For visualizer:
# must launch in other window "python3.6 -m visdom.server -port 8097"
# and access localhost:8097 to see it
if (args.visualize):
vis = visdom.Visdom()
if (not args.cpu):
images = images.cuda()
# labels = labels.cuda()
a = torch.unsqueeze(images, 0)
inputs = Variable(a)
# targets = Variable(labels)
with torch.no_grad():
outputs = model(inputs)
label = outputs[0].max(0)[1].byte().cpu().data
# label_cityscapes = cityscapes_trainIds2labelIds(label.unsqueeze(0))
label_color = Colorize()(label.unsqueeze(0))
filenameSave = "./save_color/" + "Others/" + name
os.makedirs(os.path.dirname(filenameSave), exist_ok=True)
# image_transform(label.byte()).save(filenameSave)
label_save = ToPILImage()(label_color)
label_save = label_save.resize((1241, 376), Image.BILINEAR)
# label_save = cv2.resize(label_save, (376, 1224),interpolation=cv2.INTER_AREA)
label_save.save(filenameSave)
if (args.visualize):
vis.image(label_color.numpy())
def main(args, get_dataset):
modelpath = args.loadDir + args.loadModel
weightspath = args.loadDir + args.loadWeights
print("Loading model: " + modelpath)
print("Loading weights: " + weightspath)
NUM_CLASSES = get_dataset.num_labels
model = Net(NUM_CLASSES, args.em_dim, args.resnet)
model = torch.nn.DataParallel(model)
if (not args.cpu):
model = model.cuda()
def load_my_state_dict(
model, state_dict
): #custom function to load model when not all dict elements
own_state = model.state_dict()
for a in own_state.keys():
print(a)
for a in state_dict.keys():
print(a)
print('-----------')
for name, param in state_dict.items():
if name not in own_state:
continue
own_state[name].copy_(param)
return model
model = load_my_state_dict(model, torch.load(weightspath))
print("Model and weights LOADED successfully")
model.eval()
if (not os.path.exists(args.datadir)):
print("Error: datadir could not be loaded")
loader = DataLoader(cityscapes(args.datadir,
input_transform_cityscapes,
subset=args.subset),
num_workers=args.num_workers,
batch_size=args.batch_size,
shuffle=False)
log_dir = "./save_eval_iou/"
os.makedirs(log_dir, exist_ok=True)
logger = create_logger(log_dir)
fused_iou_eval = FusedIoU(NUM_CLASSES['MAP'], WILDPASS_IN_MAP_ID,
REMAP_MAP_IDD, WILDPASS_REMAP_ID, logger)
with torch.set_grad_enabled(False):
step = 0
for step, (images, filename) in enumerate(loader):
images = images.cuda()
outputs = model(images, enc=False)
pred_MAP = outputs['MAP'].data.cpu().numpy()
pred_IDD = outputs['IDD20K'].data.cpu().numpy()
# --- evaluate
label = np.asarray(
Image.open(filename[0].replace('/leftImg8bit/', '/gtFine/')))
label = np.expand_dims(label, axis=0)
if args.is_fuse:
outputs = fused_iou_eval.fuse(label, pred_MAP, pred_IDD,
filename)
else:
fused_iou_eval.add_batch(np.argmax(pred_MAP, axis=1),
label) # only MAP
#outputs = outputs['MAP']
outputs = torch.from_numpy(outputs[None, ...])
label_color = Colorize()(outputs)
filenameSave = "./save_color/" + filename[0].split(
"leftImg8bit/")[1]
os.makedirs(os.path.dirname(filenameSave), exist_ok=True)
label_save = ToPILImage()(label_color)
label_save.save(filenameSave)
print(step, filenameSave)
print('processed step {}, file {}.'.format(
step, filename[0].split("leftImg8bit/val/cs/")[1]))
logger.info('========== evaluator using onehot ==========')
miou, iou, macc, acc = fused_iou_eval.get_iou()
logger.info('-----------Acc of each classes-----------')
for i, c_acc in enumerate(acc):
name = WILDPASS_IN_MAP_NAME[i] if len(acc) == len(
WILDPASS_IN_MAP_NAME) else str(i)
logger.info('ID= {:>10s}: {:.2f}'.format(name, c_acc * 100.0))
logger.info("Acc of {} images :{:.2f}".format(str(step + 1),
macc * 100.0))
logger.info('-----------IoU of each classes-----------')
for i, c_iou in enumerate(iou):
name = WILDPASS_IN_MAP_NAME[i] if len(iou) == len(
WILDPASS_IN_MAP_NAME) else str(i)
logger.info('ID= {:>10s}: {:.2f}'.format(name, c_iou * 100.0))
logger.info("mIoU of {} images :{:.2f}".format(str(step + 1),
miou * 100.0))
def train(args, rmodel, model, enc=False):
best_acc = 0
weight = classWeights(NUM_CLASSES)
assert os.path.exists(
args.datadir), "Error: datadir (dataset directory) could not be loaded"
co_transform = MyCoTransform(augment=True, height=args.height)
co_transform_val = MyCoTransform(augment=False, height=args.height)
dataset_train = cityscapes(args.datadir, co_transform, 'train')
dataset_val = cityscapes(args.datadir, co_transform_val, 'val')
loader = DataLoader(dataset_train,
num_workers=args.num_workers,
batch_size=args.batch_size,
shuffle=True)
loader_val = DataLoader(dataset_val,
num_workers=args.num_workers,
batch_size=args.batch_size,
shuffle=False)
if args.cuda:
weight = weight.cuda()
rcriterion = torch.nn.L1Loss()
savedir = '/home/shyam.nandan/NewExp/F_erfnet_pytorch_ours_w_gt_v2_multiply/save/' + args.savedir #change path
if (enc):
automated_log_path = savedir + "/automated_log_encoder.txt"
modeltxtpath = savedir + "/model_encoder.txt"
else:
automated_log_path = savedir + "/automated_log.txt"
modeltxtpath = savedir + "/model.txt"
if (not os.path.exists(automated_log_path)):
with open(automated_log_path, "a") as myfile:
myfile.write(
"Epoch\t\tTrain-loss\t\tTest-loss\t\tTrain-IoU\t\tTest-IoU\t\tlearningRate"
)
with open(modeltxtpath, "w") as myfile:
myfile.write(str(model))
optimizer = Adam(model.parameters(),
5e-4, (0.9, 0.999),
eps=1e-08,
weight_decay=2e-4) ##
roptimizer = Adam(rmodel.parameters(), 2e-4,
(0.9, 0.999)) ## restoration scheduler
start_epoch = 1
scheduler = lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.99)
rscheduler = lr_scheduler.StepLR(roptimizer, step_size=30,
gamma=0.5) ## Restoration schedular
for epoch in range(start_epoch, args.num_epochs + 1):
print("----- TRAINING - EPOCH", epoch, "-----")
scheduler.step() ## scheduler 2
rscheduler.step()
epoch_loss = []
time_train = []
doIouTrain = args.iouTrain
doIouVal = args.iouVal
if (doIouTrain):
iouEvalTrain = iouEval(NUM_CLASSES)
usedLr = 0
rusedLr = 0
for param_group in optimizer.param_groups:
print("Segmentation LEARNING RATE: ", param_group['lr'])
usedLr = float(param_group['lr'])
for param_group in roptimizer.param_groups:
print("Restoration LEARNING RATE: ", param_group['lr'])
rusedLr = float(param_group['lr'])
model.eval()
epoch_loss_val = []
time_val = []
if (doIouVal):
iouEvalVal = iouEval(NUM_CLASSES)
for step, (timages, images, labels, filename) in enumerate(loader_val):
start_time = time.time()
if args.cuda:
images = images.cuda()
labels = labels.cuda()
timages = timages.cuda()
inputs = Variable(
timages, volatile=True
) #volatile flag makes it free backward or outputs for eval
itargets = Variable(images, volatile=True)
targets = Variable(labels, volatile=True)
ss_inputs = rmodel(inputs, flag=0, r_fb1=0, r_fb2=0)
outs = model(ss_inputs, only_encode=enc)
tminus_outs = outs.detach()
tplus_outs = outs.detach()
for num_feedback in range(3):
optimizer.zero_grad()
roptimizer.zero_grad()
ss_inputs = rmodel(inputs,
flag=1,
r_fb1=(tplus_outs - tminus_outs),
r_fb2=ss_inputs.detach())
loss = rcriterion(ss_inputs, itargets)
outs = model(ss_inputs.detach(), only_encode=enc)
tminus_outs = tplus_outs
tplus_outs = outs.detach()
outputs = outs
del outs, tminus_outs, tplus_outs
gc.collect()
Gamma = [0, 0, 0]
Alpha = [1, 1, 1]
loss = CB_iFl(outputs,
targets[:, 0],
weight,
gamma=Gamma[0],
alpha=Alpha[0])
epoch_loss_val.append(loss.data[0])
time_val.append(time.time() - start_time)
if (doIouVal):
#start_time_iou = time.time()
iouEvalVal_img = iouEval(NUM_CLASSES)
iouEvalVal_img.addBatch(
outputs.max(1)[1].unsqueeze(1).data, targets.data)
iouEvalVal.addBatch(
outputs.max(1)[1].unsqueeze(1).data, targets.data)
#print ("Time to add confusion matrix: ", time.time() - start_time_iou)
label_color = Colorize()(
outputs[0].max(0)[1].byte().cpu().data.unsqueeze(0))
label_save = ToPILImage()(label_color)
filenameSave = '../save_color_restored_joint_afl_CBFL/' + filename[
0].split('/')[-2]
im_iou, _ = iouEvalVal_img.getIoU()
if not os.path.exists(filenameSave):
os.makedirs(filenameSave)
#Uncomment to save output
#label_save.save(filenameSave+ '/' + str(" %6.4f " %im_iou[0].data.numpy()) + '_' + filename[0].split('/')[-1])
if args.steps_loss > 0 and step % args.steps_loss == 0:
average = sum(epoch_loss_val) / len(epoch_loss_val)
print('Val loss: ', average, 'Epoch: ', epoch, 'Step: ',
step)
average_epoch_loss_val = sum(epoch_loss_val) / len(epoch_loss_val)
iouVal = 0
if (doIouVal):
iouVal, iou_classes = iouEvalVal.getIoU()
iouStr = getColorEntry(iouVal) + '{:0.2f}'.format(
iouVal * 100) + '\033[0m'
print(iouVal, iou_classes, iouStr)
return (model)
def main(args):
modelpath = args.loadDir + args.loadModel
weightspath = args.loadDir + args.loadWeights
if (not os.path.exists(args.datadir)):
print("Error: datadir could not be loaded")
else:
print("Loading model: " + modelpath)
print("Loading weights: " + weightspath)
# Import ERFNET
model = ERFNet(NUM_CLASSES)
model = torch.nn.DataParallel(model)
if (not args.cpu):
model = model.cuda()
def load_my_state_dict(
model, state_dict
): #custom function to load model when not all dict elements
own_state = model.state_dict()
for name, param in state_dict.items():
if name not in own_state:
continue
own_state[name].copy_(param)
return model
model = load_my_state_dict(model, torch.load(weightspath))
print("Model and weights LOADED successfully")
# Set model to Evaluation mode
model.eval()
# Setup the dataset loader
### RELLIS-3D Dataloader
enc = False
loader_test = custom_datasets.setup_loaders(args, enc)
# For visualizer:
# must launch in other window "python3.6 -m visdom.server -port 8097"
# and access localhost:8097 to see it
if (args.visualize):
vis = visdom.Visdom()
for step, (images, labels, img_name, _) in enumerate(loader_test):
start_time = time.time()
if (not args.cpu):
images = images.cuda()
#labels = labels.cuda()
inputs = Variable(images)
with torch.no_grad():
outputs = model(inputs)
label = outputs[0].max(0)[1].byte().cpu().data
label_color = Colorize()(label.unsqueeze(0))
eval_save_path = "./save_colour_rellis/"
if not os.path.exists(eval_save_path):
os.makedirs(eval_save_path)
_, file_name = os.path.split(img_name[0])
file_name = file_name + ".png"
#image_transform(label.byte()).save(filenameSave)
label_save = ToPILImage()(label_color)
label_save.save(os.path.join(eval_save_path, file_name))
if (args.visualize):
vis.image(label_color.numpy())
if step != 0: #first run always takes some time for setup
fwt = time.time() - start_time
time_train.append(fwt)
print("Forward time per img (b=%d): %.3f (Mean: %.3f)" %
(args.batch_size, fwt / args.batch_size,
sum(time_train) / len(time_train) / args.batch_size))
print(step, os.path.join(eval_save_path, file_name))
def main(args):
modelpath = args.loadDir + args.loadModel
#weightspath = args.loadDir + args.loadWeights #TODO
weightspath = "../save/feriburgForest_3/model_best.pth"
print("Loading model: " + modelpath)
print("Loading weights: " + weightspath)
#Import ERFNet model from the folder
#Net = importlib.import_module(modelpath.replace("/", "."), "ERFNet")
model = ERFNet(NUM_CLASSES)
model = torch.nn.DataParallel(model)
if (not args.cpu):
model = model.cuda()
#model.load_state_dict(torch.load(args.state))
#model.load_state_dict(torch.load(weightspath)) #not working if missing key
def load_my_state_dict(
model, state_dict
): #custom function to load model when not all dict elements
own_state = model.state_dict()
for name, param in state_dict.items():
if name not in own_state:
continue
own_state[name].copy_(param)
return model
model = load_my_state_dict(model, torch.load(weightspath))
print("Model and weights LOADED successfully")
model.eval()
if (not os.path.exists(args.datadir)):
print("Error: datadir could not be loaded")
loader = DataLoader(freiburgForest(args.datadir,
input_transform_cityscapes,
target_transform_cityscapes,
subset=args.subset),
num_workers=args.num_workers,
batch_size=args.batch_size,
shuffle=False)
# For visualizer:
# must launch in other window "python3.6 -m visdom.server -port 8097"
# and access localhost:8097 to see it
if (args.visualize):
vis = visdom.Visdom()
for step, (images, labels, filename, filenameGt) in enumerate(loader):
if (not args.cpu):
images = images.cuda()
#labels = labels.cuda()
inputs = Variable(images)
#targets = Variable(labels)
with torch.no_grad():
outputs = model(inputs)
print(outputs.shape)
label = outputs[0].max(0)[1].byte().cpu().data
#label_cityscapes = cityscapes_trainIds2labelIds(label.unsqueeze(0))
label_color = Colorize()(label.unsqueeze(0))
filenameSave = "./freiburgforest_1/" + filename[0].split(
"freiburg_forest_annotated/")[1]
os.makedirs(os.path.dirname(filenameSave), exist_ok=True)
#image_transform(label.byte()).save(filenameSave)
print(label_color.shape)
# label_save = ToPILImage()(label_color)
label_save = label_color.numpy()
label_save = label_save.transpose(1, 2, 0)
print(label_save.shape)
# label_save.save(filenameSave)
images = images.cpu().numpy().squeeze(axis=0)
images = images.transpose(1, 2, 0)
# print(images.shape)
# print(label_save.shape)
plt.figure(figsize=(10.24, 5.12), dpi=100)
plt.imshow(images)
plt.imshow(label_save, alpha=0.6)
plt.axis('off')
# plt.show()
plt.savefig(filenameSave, dpi=100)
plt.close()
if (args.visualize):
vis.image(label_color.numpy())
print(step, filenameSave)
NUM_CHANNELS = 3
# NUM_CLASSES = 20 #pascal=22, cityscapes=20
NUM_HISTOGRAMS = 5
NUM_IMG_PER_EPOCH = 5
# Optimizer params.
LEARNING_RATE=5e-4
BETAS=(0.9, 0.999)
OPT_EPS=1e-08
WEIGHT_DECAY=1e-6
DISCOUNT_RATE_START=0.1
DISCOUNT_RATE=0.01
MAX_CONSISTENCY_EPOCH=30
DISCOUNT_RATE_START_EPOCH=50
color_transform_target = Colorize(1.0, 2.0, remove_negative=True, extend=True, white_val=1.0) # min_val, max_val, remove negative
color_transform_output = Colorize(1.0, 2.0, remove_negative=False, extend=True, white_val=1.0) # Automatic color based on tensor min/max val
# color_transform_output = ColorizeMinMax() # Automatic color based on tensor min/max val
image_transform = ToPILImage()
#Augmentations - different function implemented to perform random augments on both image and target
class MyCoTransform(object):
def __init__(self, enc, augment=True, height=512):
self.enc=enc
self.augment = augment
self.height = height
self.rotation_angle = 5.0
self.affine_angle = 5.0
def main(args):
modelpath = args.loadDir + args.loadModel
weightspath = args.loadDir + args.loadWeights
print("Loading model: " + modelpath)
print("Loading weights: " + weightspath)
resnet = resnet18(pretrained=True, efficient=False, use_bn=True)
model = Net(resnet, size=(512, 1024 * 4), num_classes=NUM_CLASSES)
model = torch.nn.DataParallel(model)
if (not args.cpu):
model = model.cuda()
#model.load_state_dict(torch.load(args.state))
#model.load_state_dict(torch.load(weightspath)) #not working if missing key
def load_my_state_dict(
model, state_dict
): #custom function to load model when not all dict elements
own_state = model.state_dict()
for a, b in zip(own_state.keys(), state_dict.keys()):
print(a, ' ', b)
print('-----------')
for name, param in state_dict.items():
# print('#####', name)
name = name[7:]
if name not in own_state:
print('{} not in own_state'.format(name))
continue
#if name not in except_list:
own_state[name].copy_(param)
return model
model = load_my_state_dict(model, torch.load(weightspath))
print("Model and weights LOADED successfully")
model.eval()
if (not os.path.exists(args.datadir)):
print("Error: datadir could not be loaded")
loader = DataLoader(cityscapes(args.datadir,
input_transform_cityscapes,
subset=args.subset),
num_workers=args.num_workers,
batch_size=args.batch_size,
shuffle=False)
# For visualizer:
# must launch in other window "python3.6 -m visdom.server -port 8097"
# and access localhost:8097 to see it
if (args.visualize):
vis = visdom.Visdom()
time_all = []
with torch.no_grad():
for step, (images, filename) in enumerate(loader):
images = images.cuda()
start_time = time.time()
outputs = model(images)
fwt = time.time() - start_time
time_all.append(fwt)
label = outputs[0].cpu().max(0)[1].data.byte()
label_color = Colorize()(label.unsqueeze(0))
filenameSave = "./save_color/" + filename[0].split(
"leftImg8bit/")[1]
os.makedirs(os.path.dirname(filenameSave), exist_ok=True)
label_save = ToPILImage()(label_color)
label_save.save(filenameSave)
print(step, filenameSave)
print("Forward time per img (Mean: %.4f)" %
(1 / (sum(time_all) / len(time_all))))
model.load_state_dict(torch.load("./pth/fcn-deconv-40.pth"))
model.eval()
# 10 13 48 86 101
img = Image.open("./data/VOC2012test/JPEGImages/2008_000101.jpg").convert("RGB")
original_size = img.size
img.save("original.png")
img = img.resize((256, 256), Image.BILINEAR)
img = ToTensor()(img)
img = Variable(img).unsqueeze(0)
outputs = model(img)
# 22 256 256
for i, output in enumerate(outputs):
output = output[0].data.max(0)[1]
output = Colorize()(output)
output = np.transpose(output.numpy(), (1, 2, 0))
img = Image.fromarray(output, "RGB")
if i == 0:
img = img.resize(original_size, Image.NEAREST)
img.save("test-%d.png" % i)
'''
for index, (imgs, name, size) in tqdm(enumerate(testloader)):
imgs = Variable(imgs.cuda())
outputs = model(imgs)
output = outputs[0][0].data.max(0)[1]
output = Colorize()(output)
print(output)
with torch.no_grad():
for sample in tqdm(dataset_it):
im = sample['image']
instances = sample['instance'].squeeze()
output = model(im)
instance_map, predictions = cluster.cluster(output[0], threshold=0.9)
visualizer.display(im, 'image')
visualizer.display(instance_map.cpu(), 'pred')
label1 = instance_map.cpu()
label_color = Colorize()(label1.unsqueeze(0))
sigma = output[0][2].cpu()
sigma = (sigma - sigma.min())/(sigma.max() - sigma.min())
sigma[instances == 0] = 0
visualizer.display(sigma, 'sigma')
label2=sigma
label2.squeeze_()
seed = torch.sigmoid(output[0][3]).cpu()
visualizer.display(seed, 'seed')
label3 = seed.cpu()
label3.squeeze_()
comp = model.get_computations(True)
print(comp)
print(sum(comp))
if cuda:
model = model.cuda()
model.eval() # Set in evaluation mode
Tensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor
print ('\nPerforming object detection:')
bar = progressbar.ProgressBar(0, len(valloader), redirect_stdout=False)
for batch_i, (imgs, targets) in enumerate(valloader):
# Configure input
input_imgs = imgs.type(Tensor)
# Get detections
with torch.no_grad():
pred = model(input_imgs)
_, predClass = torch.max(pred, 1)
mask = Colorize(predClass.cpu().squeeze()).permute(1,2,0).numpy()
cv2.imwrite('output/%d.png' % (batch_i),mask)
# Log progress
bar.update(batch_i)
bar.finish()
imgs = Variable(imgs.cuda())
start_time = time.time()
outputs_4, outputs_2, outputs = model(imgs)
end_time = time.time()
if j > 9:
avr_time += (end_time - start_time)
if j == 109:
print avr_time/100.0
exit()
# 22 256 256
filename = list(names)[0]
for i, (output_4, output_2, output) in enumerate(zip(outputs_4, outputs_2, outputs)):
output_4 = output_4.data.max(0)[1]
output_4 = Colorize()(output_4)
output_4 = np.transpose(output_4.numpy(), (1, 2, 0))
img_4 = Image.fromarray(output_4, "RGB")
output_2 = output_2.data.max(0)[1]
output_2 = Colorize()(output_2)
output_2 = np.transpose(output_2.numpy(), (1, 2, 0))
img_2 = Image.fromarray(output_2, "RGB")
output = output.data.max(0)[1]
output = Colorize()(output)
output = np.transpose(output.numpy(), (1, 2, 0))
img = Image.fromarray(output, "RGB")
if i == 0:
outputs[cnt] = lab[0]
outputs[cnt + 1] = lab[1]
cnt += 2
beg = time.clock()
pred = model(inputs)
t += time.clock() - beg
_, predClass = torch.max(pred, 1)
bSize = inputs.data.size()[0]
running_acc += torch.sum(predClass == outputs).item() * outSize * 100
for j in range(bSize):
img = Image.fromarray(
Colorize(predClass.data[j]).permute(1, 2,
0).numpy().astype('uint8'))
img.save(outDir + "%d.png" % (imgCnt + j))
imgCnt += bSize
maskPred = torch.zeros(numClass, bSize, int(labSize[0]),
int(labSize[1])).long()
maskLabel = torch.zeros(numClass, bSize, int(labSize[0]),
int(labSize[1])).long()
for currClass in range(numClass):
maskPred[currClass] = predClass == currClass
maskLabel[currClass] = outputs == currClass
for labIdx in range(numClass):
labCnts[labIdx] += torch.sum(maskLabel[labIdx]).item()
for predIdx in range(numClass):
inter = torch.sum(maskPred[predIdx] & maskLabel[labIdx]).item()
from torchvision.transforms import Compose, CenterCrop, Normalize, Resize, Pad
from torchvision.transforms import ToTensor, ToPILImage
import matplotlib.pyplot as plt
from dataset import geoMat
from transform import Colorize
from visualize import Dashboard
from tensorboardX import SummaryWriter
import importlib
from iouEval import iouEval, getColorEntry
from shutil import copyfile
mean_and_var = 2
color_transform = Colorize(mean_and_var)
#Augmentations - different function implemented to perform random augments on both image and target
class MyCoTransform(object):
def __init__(self, augment=True, rescale=True, size=104):
self.augment = augment
self.size = (size, size)
self.rescale = rescale
pass
def __call__(self, input, target):
# do something to both images
# input = Resize(self.size, Image.BILINEAR)(input)
def main(args):
modelpath = args.loadDir + args.loadModel
weightspath = args.loadDir + args.loadWeights
print("Loading model: " + modelpath)
print("Loading weights: " + weightspath)
model = Net(NUM_CLASSES)
model = torch.nn.DataParallel(model)
if (not args.cpu):
model = model.cuda()
#model.load_state_dict(torch.load(args.state))
#model.load_state_dict(torch.load(weightspath)) #not working if missing key
def load_my_state_dict(
model, state_dict
): #custom function to load model when not all dict elements
own_state = model.state_dict()
for name, param in state_dict.items():
if name not in own_state:
continue
own_state[name].copy_(param)
return model
model = load_my_state_dict(model, torch.load(weightspath))
print("Model and weights LOADED successfully")
model.eval()
if (not os.path.exists(args.datadir)):
print("Error: datadir could not be loaded")
loader = DataLoader(cityscapes(args.datadir,
input_transform_cityscapes,
target_transform_cityscapes,
subset=args.subset),
num_workers=args.num_workers,
batch_size=args.batch_size,
shuffle=False)
# For visualizer:
# must launch in other window "python3.6 -m visdom.server -port 8097"
# and access localhost:8097 to see it
if (args.visualize):
vis = visdom.Visdom()
for step, (images, labels, filename, filenameGt) in enumerate(loader):
if (not args.cpu):
images = images.cuda()
#labels = labels.cuda()
inputs = Variable(images)
#targets = Variable(labels)
with torch.no_grad():
outputs = model(inputs)
label = outputs[0].max(0)[1].byte().cpu().data
#label_cityscapes = cityscapes_trainIds2labelIds(label.unsqueeze(0))
label_color = Colorize()(label.unsqueeze(0))
filenameSave = "./save_color/" + filename[0].split("leftImg8bit/")[1]
os.makedirs(os.path.dirname(filenameSave), exist_ok=True)
#image_transform(label.byte()).save(filenameSave)
label_save = ToPILImage()(label_color)
label_save.save(filenameSave)
if (args.visualize):
vis.image(label_color.numpy())
print(step, filenameSave)
loss.backward()
optimizer_seg.step()
optimizer_feat.step()
print "epoch is:[{}|{}],index is:[{}|{}],loss:{}".\
format(epoch,epoch_num,i,len(dataloader),loss)
win = visutils.visualize_loss(epoch, loss.cpu().detach(), env, win)
if epoch % 40 == 0:
#save model
torch.save(vgg16.state_dict(),
'%s/vgg16_%03d.pkl' % (result_directory, epoch))
torch.save(Seg.state_dict(),
'%s/Seg_%03d.pkl' % (result_directory, epoch))
#save result
input = make_image_grid(img, mean, std)
label = make_label_grid(labels.data)
label = Colorize()(label).type(torch.FloatTensor)
output = make_label_grid(torch.max(logits, dim=1)[1].data)
output = Colorize()(output).type(torch.FloatTensor)
vutils.save_image(
label.cpu().detach(),
'%s/labels_epoch_%03d.png' % (result_directory, epoch))
vutils.save_image(
output.cpu().detach(),
'%s/output_epoch_%03d.png' % (result_directory, epoch))
vutils.save_image(input.cpu().detach(),
'%s/img_epoch_%03d.png' % (result_directory, epoch))
