class SGNetAgent(BaseAgent):
"""
This class will be responsible for handling the whole process of our architecture.
"""
def __init__(self, config):
super().__init__(config)
## Select network
if config.spatial_information == 'depth' and config.os == 16 and config.network == "SGNet" and config.mode != "measure_speed":
from graphs.models.SGNet.SGNet import SGNet
elif config.spatial_information == 'depth' and config.os == 16 and config.network == "SGNet":
from graphs.models.SGNet.SGNet_fps import SGNet
elif config.spatial_information == 'depth' and config.os == 16 and config.network == "SGNet_ASPP" and config.mode != "measure_speed":
from graphs.models.SGNet.SGNet_ASPP import SGNet
elif config.spatial_information == 'depth' and config.os == 16 and config.network == "SGNet_ASPP":
from graphs.models.SGNet.SGNet_ASPP_fps import SGNet
random.seed(self.config.seed)
os.environ['PYTHONHASHSEED'] = str(self.config.seed)
np.random.seed(self.config.seed)
torch.manual_seed(self.config.seed)
torch.cuda.manual_seed(self.config.seed)
torch.cuda.manual_seed_all(self.config.seed)
cudnn.enabled = True
cudnn.benchmark = True
cudnn.deterministic = True
os.environ["CUDA_VISIBLE_DEVICES"] = config.gpu
# create data loader
if config.dataset == "NYUD":
self.testloader = data.DataLoader(NYUDataset_val_full(
self.config.val_list_path),
batch_size=1,
shuffle=False,
pin_memory=True)
# Create an instance from the Model
self.logger.info("Loading encoder pretrained in imagenet...")
self.model = SGNet(self.config.num_classes)
print(self.model)
self.model.cuda()
self.model.train()
self.model.float()
print(config.gpu)
if config.mode != 'measure_speed':
self.model = DataParallelModel(self.model, device_ids=[0])
print('parallel....................')
total = sum([param.nelement() for param in self.model.parameters()])
print(' + Number of params: %.2fM' % (total / 1e6))
print_cuda_statistics()
def load_checkpoint(self, filename):
try:
self.logger.info("Loading checkpoint '{}'".format(filename))
checkpoint = torch.load(filename)
self.current_epoch = checkpoint['epoch']
self.current_iteration = checkpoint['iteration']
self.model.load_state_dict(checkpoint['state_dict'])
# self.optimizer.load_state_dict(checkpoint['optimizer'])
self.logger.info(
"Checkpoint loaded successfully from '{}' at (epoch {}) at (iteration {})\n"
.format(filename, checkpoint['epoch'],
checkpoint['iteration']))
except OSError as e:
self.logger.info(
"No checkpoint exists from '{}'. Skipping...".format(
self.config.checkpoint_dir))
self.logger.info("**First time to train**")
def run(self):
"""
This function will the operator
:return:
"""
assert self.config.mode in [
'train', 'test', 'measure_speed', 'train_iters'
]
try:
if self.config.mode == 'test':
self.test()
elif self.config.mode == 'measure_speed':
with torch.no_grad():
self.measure_speed(input_size=[1, 3, 480, 640])
except KeyboardInterrupt:
self.logger.info("You have entered CTRL+C.. Wait to finalize")
def test(self):
tqdm_batch = tqdm(self.testloader,
total=len(self.testloader),
desc="Testing...")
self.model.eval()
metrics = IOUMetric(self.config.num_classes)
loss_val = 0
metrics = IOUMetric(self.config.num_classes)
palette = get_palette(256)
# if (not os.path.exists(self.config.output_img_dir)):
# os.mkdir(self.config.output_img_dir)
# if (not os.path.exists(self.config.output_gt_dir)):
# os.mkdir(self.config.output_gt_dir)
if (not os.path.exists(self.config.output_predict_dir)):
os.mkdir(self.config.output_predict_dir)
self.load_checkpoint(self.config.trained_model_path)
index = 0
for batch_val in tqdm_batch:
image = batch_val['image'].cuda()
label = batch_val['seg'].cuda()
label = torch.squeeze(label, 1).long()
HHA = batch_val['HHA'].cuda()
depth = batch_val['depth'].cuda()
size = np.array([label.size(1), label.size(2)])
input_size = (label.size(1), label.size(2))
with torch.no_grad():
if self.config.ms:
output = predict_multiscale(self.model, image, depth,
input_size, [0.8, 1.0, 2.0],
self.config.num_classes, False)
else:
output = predict_multiscale(self.model, image, depth,
input_size, [1.0],
self.config.num_classes, False)
seg_pred = np.asarray(np.argmax(output, axis=2), dtype=np.int)
output_im = Image.fromarray(
np.asarray(np.argmax(output, axis=2), dtype=np.uint8))
output_im.putpalette(palette)
output_im.save(self.config.output_predict_dir + '/' +
str(index) + '.png')
seg_gt = np.asarray(label[0].cpu().numpy(), dtype=np.int)
ignore_index = seg_gt != 255
seg_gt = seg_gt[ignore_index]
seg_pred = seg_pred[ignore_index]
metrics.add_batch(seg_pred, seg_gt, ignore_index=255)
index = index + 1
acc, acc_cls, iu, mean_iu, fwavacc = metrics.evaluate()
print({
'meanIU': mean_iu,
'IU_array': iu,
'acc': acc,
'acc_cls': acc_cls
})
pass
def finalize(self):
"""
Finalize all the operations of the 2 Main classes of the process the operator and the data loader
:return:
"""
# TODO
pass
def measure_speed(self, input_size, iteration=500):
"""
Measure the speed of model
:return: speed_time
fps
"""
self.model.eval()
input = torch.randn(*input_size).cuda()
depth = torch.randn(*input_size).cuda()
HHA = torch.randn(*input_size).cuda()
for _ in range(100):
self.model(input, depth)
print('=========Speed Testing=========')
torch.cuda.synchronize()
torch.cuda.synchronize()
t_start = time.time()
for _ in range(iteration):
x = self.model(input, depth)
torch.cuda.synchronize()
elapsed_time = time.time() - t_start
speed_time = elapsed_time / iteration * 1000
fps = iteration / elapsed_time
print(iteration)
print('Elapsed Time: [%.2f s / %d iter]' % (elapsed_time, iteration))
print('Speed Time: %.2f ms / iter FPS: %.2f' % (speed_time, fps))
return speed_time, fps
def main():
writer = SummaryWriter(args.snapshot_dir)
if not args.gpu == 'None':
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
cudnn.enabled = True
xlsor = XLSor(num_classes=args.num_classes)
print(xlsor)
saved_state_dict = torch.load(args.restore_from)
new_params = xlsor.state_dict().copy()
for i in saved_state_dict:
i_parts = i.split('.')
if not i_parts[0] == 'fc':
new_params['.'.join(i_parts[0:])] = saved_state_dict[i]
xlsor.load_state_dict(new_params)
model = DataParallelModel(xlsor)
model.train()
model.float()
model.cuda()
criterion = Criterion()
criterion = DataParallelCriterion(criterion)
criterion.cuda()
cudnn.benchmark = True
if not os.path.exists(args.snapshot_dir):
os.makedirs(args.snapshot_dir)
trainloader = data.DataLoader(XRAYDataSet(args.data_dir,
args.data_list,
max_iters=args.num_steps *
args.batch_size,
crop_size=input_size,
scale=args.random_scale,
mirror=args.random_mirror,
mean=IMG_MEAN),
batch_size=args.batch_size,
shuffle=True,
num_workers=16,
pin_memory=True)
optimizer = optim.SGD(
[{
'params': filter(lambda p: p.requires_grad, xlsor.parameters()),
'lr': args.learning_rate
}],
lr=args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
optimizer.zero_grad()
interp = nn.Upsample(size=input_size, mode='bilinear', align_corners=True)
for i_iter, batch in enumerate(trainloader):
i_iter += args.start_iters
images, labels, _, _ = batch
images = images.cuda()
labels = labels.float().cuda()
if torch_ver == "0.3":
images = Variable(images)
labels = Variable(labels)
optimizer.zero_grad()
lr = adjust_learning_rate(optimizer, i_iter)
preds = model(images, args.recurrence)
loss = criterion(preds, labels)
loss.backward()
optimizer.step()
if i_iter % 100 == 0:
writer.add_scalar('learning_rate', lr, i_iter)
writer.add_scalar('loss', loss.data.cpu().numpy(), i_iter)
if i_iter % 100 == 0:
images_inv = inv_preprocess(images, args.save_num_images, IMG_MEAN)
if isinstance(preds, list):
preds = preds[0]
if isinstance(preds, list):
preds = preds[0]
preds = interp(preds)
for index, img in enumerate(images_inv):
writer.add_image('Images/' + str(index),
torch.from_numpy(img / 255.).permute(2, 0, 1),
i_iter)
writer.add_image('Labels/' + str(index), labels[index], i_iter)
writer.add_image('preds/' + str(index),
(preds[index] > 0.5).float(), i_iter)
print('iter = {} of {} completed, loss = {}'.format(
i_iter, args.num_steps,
loss.data.cpu().numpy()))
if i_iter >= args.num_steps - 1:
print('save model ...')
torch.save(
xlsor.state_dict(),
osp.join(args.snapshot_dir,
'XLSor_' + str(args.num_steps) + '.pth'))
break
if i_iter % args.save_pred_every == 0:
print('taking snapshot ...')
torch.save(
xlsor.state_dict(),
osp.join(args.snapshot_dir, 'XLSor_' + str(i_iter) + '.pth'))
end = timeit.default_timer()
print(end - start, 'seconds')
def main():
writer = SummaryWriter(args.snapshot_dir)
if not args.gpu == 'None':
os.environ["CUDA_VISIBLE_DEVICES"]=args.gpu
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
cudnn.enabled = True
xlsor = XLSor(num_classes=args.num_classes)
print(xlsor)
saved_state_dict = torch.load(args.restore_from)
new_params = xlsor.state_dict().copy()
for i in saved_state_dict:
i_parts = i.split('.')
if not i_parts[0]=='fc':
new_params['.'.join(i_parts[0:])] = saved_state_dict[i]
xlsor.load_state_dict(new_params)
model = DataParallelModel(xlsor)
model.train()
model.float()
model.cuda()
criterion = Criterion()
criterion = DataParallelCriterion(criterion)
criterion.cuda()
cudnn.benchmark = True
if not os.path.exists(args.snapshot_dir):
os.makedirs(args.snapshot_dir)
trainloader = data.DataLoader(XRAYDataSet(args.data_dir, args.data_list, max_iters=args.num_steps*args.batch_size, crop_size=input_size,
scale=args.random_scale, mirror=args.random_mirror, mean=IMG_MEAN),
batch_size=args.batch_size, shuffle=True, num_workers=16, pin_memory=True)
optimizer = optim.SGD([{'params': filter(lambda p: p.requires_grad, xlsor.parameters()), 'lr': args.learning_rate }],
lr=args.learning_rate, momentum=args.momentum,weight_decay=args.weight_decay)
optimizer.zero_grad()
interp = nn.Upsample(size=input_size, mode='bilinear', align_corners=True)
for i_iter, batch in enumerate(trainloader):
i_iter += args.start_iters
images, labels, _, _ = batch
images = images.cuda()
labels = labels.float().cuda()
if torch_ver == "0.3":
images = Variable(images)
labels = Variable(labels)
optimizer.zero_grad()
lr = adjust_learning_rate(optimizer, i_iter)
preds = model(images, args.recurrence)
loss = criterion(preds, labels)
loss.backward()
optimizer.step()
if i_iter % 100 == 0:
writer.add_scalar('learning_rate', lr, i_iter)
writer.add_scalar('loss', loss.data.cpu().numpy(), i_iter)
if i_iter % 100 == 0:
images_inv = inv_preprocess(images, args.save_num_images, IMG_MEAN)
if isinstance(preds, list):
preds = preds[0]
if isinstance(preds, list):
preds = preds[0]
preds = interp(preds)
for index, img in enumerate(images_inv):
writer.add_image('Images/'+str(index), torch.from_numpy(img/255.).permute(2,0,1), i_iter)
writer.add_image('Labels/'+str(index), labels[index], i_iter)
writer.add_image('preds/'+str(index), (preds[index]>0.5).float(), i_iter)
print('iter = {} of {} completed, loss = {}'.format(i_iter, args.num_steps, loss.data.cpu().numpy()))
if i_iter >= args.num_steps-1:
print('save model ...')
torch.save(xlsor.state_dict(),osp.join(args.snapshot_dir, 'XLSor_'+str(args.num_steps)+'.pth'))
break
if i_iter % args.save_pred_every == 0:
print('taking snapshot ...')
torch.save(xlsor.state_dict(),osp.join(args.snapshot_dir, 'XLSor_'+str(i_iter)+'.pth'))
end = timeit.default_timer()
print(end-start,'seconds')
def main():
writer = SummaryWriter(args.snapshot_dir)
if not args.gpu == 'None':
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
cudnn.enabled = True
deeplab = Res_Deeplab(num_classes=args.num_classes)
print(deeplab)
saved_state_dict = torch.load(args.restore_from)
new_params = deeplab.state_dict().copy()
for i in saved_state_dict:
i_parts = i.split('.')
if not i_parts[0] == 'fc':
new_params['.'.join(i_parts[0:])] = saved_state_dict[i]
deeplab.load_state_dict(new_params)
model = DataParallelModel(deeplab)
model.train()
model.float()
# model.apply(set_bn_momentum)
model.cuda()
criterion = CriterionDSN() # CriterionCrossEntropy()
criterion = DataParallelCriterion(criterion)
criterion.cuda()
cudnn.benchmark = True
if not os.path.exists(args.snapshot_dir):
os.makedirs(args.snapshot_dir)
trainloader = data.DataLoader(
ModaDataset(
args.data_dir,
args.list_path,
max_iters=args.num_steps * args.batch_size,
# mirror=args.random_mirror,
mirror=True,
rotate=True,
mean=IMG_MEAN),
batch_size=args.batch_size,
shuffle=True,
num_workers=4,
pin_memory=True)
optimizer = optim.SGD(
[{
'params': filter(lambda p: p.requires_grad, deeplab.parameters()),
'lr': args.learning_rate
}],
lr=args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
optimizer.zero_grad()
print('start training!')
for i_iter, batch in enumerate(trainloader):
i_iter += args.start_iters
images, labels, _, _ = batch
images = images.cuda()
labels = labels.long().cuda()
optimizer.zero_grad()
lr = adjust_learning_rate(optimizer, i_iter)
# preds = model(images, args.recurrence)
preds = model(images)
loss = criterion(preds, labels)
loss.backward()
optimizer.step()
if i_iter % 100 == 0:
writer.add_scalar('learning_rate', lr, i_iter)
writer.add_scalar('loss', loss.data.cpu().numpy(), i_iter)
# if i_iter % 5000 == 0:
# images_inv = inv_preprocess(images, args.save_num_images, IMG_MEAN)
# labels_colors = decode_labels(labels, args.save_num_images, args.num_classes)
# if isinstance(preds, list):
# preds = preds[0]
# preds_colors = decode_predictions(preds, args.save_num_images, args.num_classes)
# for index, (img, lab) in enumerate(zip(images_inv, labels_colors)):
# writer.add_image('Images/'+str(index), img, i_iter)
# writer.add_image('Labels/'+str(index), lab, i_iter)
# writer.add_image('preds/'+str(index), preds_colors[index], i_iter)
print('iter = {} of {} completed, loss = {}'.format(
i_iter, args.num_steps,
loss.data.cpu().numpy()))
if i_iter >= args.num_steps - 1:
print('save model ...')
torch.save(
deeplab.state_dict(),
osp.join(args.snapshot_dir,
'CS_scenes_' + str(args.num_steps) + '.pth'))
break
if i_iter % args.save_pred_every == 0:
print('taking snapshot ...')
torch.save(
deeplab.state_dict(),
osp.join(args.snapshot_dir,
'CS_scenes_' + str(i_iter) + '.pth'))
end = timeit.default_timer()
print(end - start, 'seconds')
