def _val_impl(args, model_specs, logger):
if len(args.output) > 0:
_make_dirs(args.output)
# dataiter
dataset_specs_tgt = get_dataset_specs_tgt(args, model_specs)
scale, mean_, _ = _get_scalemeanstd()
if scale > 0:
mean_ /= scale
margs = argparse.Namespace(**model_specs)
dargs = argparse.Namespace(**dataset_specs_tgt)
mod = _get_module(args, margs, dargs)
addr_weights = args.weights # first weights should be xxxx_ep-0000.params!
addr_output = args.output
# current round index
cround = args.idx_round
net_args = None
net_auxs = None
###
if addr_weights is not None:
net_args, net_auxs = mxutil.load_params_from_file(addr_weights)
######
save_dir = osp.join(args.output, str(cround), 'results')
save_dir_self_train = osp.join(args.output, str(cround), 'self_train')
# pseudo labels
save_dir_pseudo_labelIds = osp.join(save_dir_self_train, 'pseudo_labelIds')
save_dir_pseudo_color = osp.join(save_dir_self_train, 'pseudo_color')
# without sp
save_dir_nplabelIds = osp.join(save_dir, 'nplabelIds')
save_dir_npcolor = osp.join(save_dir, 'npcolor')
# probability map
save_dir_probmap = osp.join(args.output, 'probmap')
save_dir_stats = osp.join(args.output, 'stats')
_make_dirs(save_dir)
_make_dirs(save_dir_pseudo_labelIds)
_make_dirs(save_dir_pseudo_color)
_make_dirs(save_dir_nplabelIds)
_make_dirs(save_dir_npcolor)
_make_dirs(save_dir_probmap)
_make_dirs(save_dir_stats)
if args.with_prior == 'True':
# with sp
save_dir_splabelIds = osp.join(save_dir_self_train, 'splabelIds')
save_dir_spcolor = osp.join(save_dir_self_train, 'spcolor')
_make_dirs(save_dir_splabelIds)
_make_dirs(save_dir_spcolor)
if args.kc_policy == 'cb':
# reweighted prediction map
save_dir_rwlabelIds = osp.join(save_dir_self_train, 'rwlabelIds')
save_dir_rwcolor = osp.join(save_dir_self_train, 'rwcolor')
_make_dirs(save_dir_rwlabelIds)
_make_dirs(save_dir_rwcolor)
######
dataset_tgt = model_specs['dataset_tgt']
image_list_tgt, label_gt_list_tgt, image_tgt_list = parse_split_file_tgt(
margs.dataset_tgt, args.split_tgt)
has_gt = args.split_tgt in (
'train',
'val',
)
crop_sizes = sorted([int(_) for _ in args.test_scales.split(',')])[::-1]
crop_size = crop_sizes[0]
assert len(crop_sizes) == 1, 'multi-scale testing not implemented'
label_stride = margs.feat_stride
x_num = len(image_list_tgt)
do_forward = True
# for all images that has the same resolution
if do_forward:
batch = None
transformers = [ts.Scale(crop_size, Image.CUBIC, False)]
transformers += _get_transformer_image()
transformer = ts.Compose(transformers)
scorer_np = ScoreUpdater(dargs.valid_labels_tgt, margs.classes, x_num,
logger)
scorer_np.reset()
# with prior
if args.with_prior == 'True':
scorer = ScoreUpdater(dargs.valid_labels_tgt, margs.classes, x_num,
logger)
scorer.reset()
done_count = 0 # for multi-scale testing
num_classes = margs.classes
init_tgt_port = float(args.init_tgt_port)
# class-wise
cls_exist_array = np.zeros([1, num_classes], dtype=int)
cls_thresh = np.zeros([num_classes
]) # confidence thresholds for all classes
cls_size = np.zeros([num_classes]) # number of predictions in each class
array_pixel = 0.0
# prior
if args.with_prior == 'True':
in_path_prior = 'spatial_prior/{}/prior_array.mat'.format(args.dataset)
sprior = scipy.io.loadmat(in_path_prior)
prior_array = sprior["prior_array"].astype(np.float32)
#prior_array = np.maximum(prior_array,0)
############################ network forward
for i in xrange(x_num):
start = time.time()
############################ network forward on single image (from official ResNet-38 implementation)
sample_name = osp.splitext(osp.basename(image_list_tgt[i]))[0]
im_path = osp.join(args.data_root_tgt, image_list_tgt[i])
rim = np.array(Image.open(im_path).convert('RGB'), np.uint8)
if do_forward:
im = transformer(rim)
imh, imw = im.shape[:2]
# init
if batch is None:
if dargs.ident_size:
input_h = make_divisible(imh, margs.feat_stride)
input_w = make_divisible(imw, margs.feat_stride)
else:
input_h = input_w = make_divisible(crop_size,
margs.feat_stride)
label_h, label_w = input_h / label_stride, input_w / label_stride
test_steps = args.test_steps
pred_stride = label_stride / test_steps
pred_h, pred_w = label_h * test_steps, label_w * test_steps
input_data = np.zeros((1, 3, input_h, input_w), np.single)
input_label = 255 * np.ones((1, label_h * label_w), np.single)
dataiter_tgt = mx.io.NDArrayIter(input_data, input_label)
batch = dataiter_tgt.next()
mod.bind(dataiter_tgt.provide_data,
dataiter_tgt.provide_label,
for_training=False,
force_rebind=True)
if not mod.params_initialized:
mod.init_params(arg_params=net_args, aux_params=net_auxs)
nim = np.zeros((3, imh + label_stride, imw + label_stride),
np.single)
sy = sx = label_stride // 2
nim[:, sy:sy + imh, sx:sx + imw] = im.transpose(2, 0, 1)
net_preds = np.zeros((margs.classes, pred_h, pred_w), np.single)
sy = sx = pred_stride // 2 + np.arange(test_steps) * pred_stride
for ix in xrange(test_steps):
for iy in xrange(test_steps):
input_data = np.zeros((1, 3, input_h, input_w), np.single)
input_data[0, :, :imh, :imw] = nim[:, sy[iy]:sy[iy] + imh,
sx[ix]:sx[ix] + imw]
batch.data[0] = mx.nd.array(input_data)
mod.forward(batch, is_train=False)
this_call_preds = mod.get_outputs()[0].asnumpy()[0]
if args.test_flipping:
batch.data[0] = mx.nd.array(input_data[:, :, :, ::-1])
mod.forward(batch, is_train=False)
# average the original and flipped image prediction
this_call_preds = 0.5 * (
this_call_preds +
mod.get_outputs()[0].asnumpy()[0][:, :, ::-1])
net_preds[:, iy:iy + pred_h:test_steps,
ix:ix + pred_w:test_steps] = this_call_preds
interp_preds_np = interp_preds_as(rim.shape[:2], net_preds,
pred_stride, imh, imw)
########################### #save predicted labels and confidence score vectors in target domains
# no prior prediction with trainIDs
pred_label_np = interp_preds_np.argmax(0)
# no prior prediction with labelIDs
if dargs.id_2_label_tgt is not None:
pred_label_np = dargs.id_2_label_tgt[pred_label_np]
# no prior color prediction
im_to_save_np = Image.fromarray(pred_label_np.astype(np.uint8))
im_to_save_npcolor = im_to_save_np.copy()
if dargs.cmap is not None:
im_to_save_npcolor.putpalette(dargs.cmap.ravel())
# save no prior prediction with labelIDs and colors
out_path_np = osp.join(save_dir_nplabelIds,
'{}.png'.format(sample_name))
out_path_npcolor = osp.join(save_dir_npcolor,
'{}.png'.format(sample_name))
im_to_save_np.save(out_path_np)
im_to_save_npcolor.save(out_path_npcolor)
# with prior
if args.with_prior == 'True':
probmap = np.multiply(prior_array,
interp_preds_np).astype(np.float32)
elif args.with_prior == 'False':
probmap = interp_preds_np.copy().astype(np.float32)
pred_label = probmap.argmax(0)
probmap_max = np.amax(probmap, axis=0)
############################ save confidence scores of target domain as class-wise vectors
for idx_cls in np.arange(0, num_classes):
idx_temp = pred_label == idx_cls
sname = 'array_cls' + str(idx_cls)
if not (sname in locals()):
exec("%s = np.float32(0)" % sname)
if idx_temp.any():
cls_exist_array[0, idx_cls] = 1
probmap_max_cls_temp = probmap_max[idx_temp].astype(np.float32)
len_cls = probmap_max_cls_temp.size
# downsampling by rate 4
probmap_cls = probmap_max_cls_temp[0:len_cls:4]
exec("%s = np.append(%s,probmap_cls)" % (sname, sname))
############################ save prediction
# save prediction probablity map
out_path_probmap = osp.join(save_dir_probmap,
'{}.npy'.format(sample_name))
np.save(out_path_probmap, probmap.astype(np.float32))
# save predictions with spatial priors, if sp exist.
if args.with_prior == 'True':
if dargs.id_2_label_tgt is not None:
pred_label = dargs.id_2_label_tgt[pred_label]
im_to_save_sp = Image.fromarray(pred_label.astype(np.uint8))
im_to_save_spcolor = im_to_save_sp.copy()
if dargs.cmap is not None: # save color seg map
im_to_save_spcolor.putpalette(dargs.cmap.ravel())
out_path_sp = osp.join(save_dir_splabelIds,
'{}.png'.format(sample_name))
out_path_spcolor = osp.join(save_dir_spcolor,
'{}.png'.format(sample_name))
im_to_save_sp.save(out_path_sp)
im_to_save_spcolor.save(out_path_spcolor)
# log information
done_count += 1
if not has_gt:
logger.info('Done {}/{} with speed: {:.2f}/s'.format(
i + 1, x_num, 1. * done_count / (time.time() - start)))
continue
if args.split_tgt in ('train', 'val'):
# evaluate with ground truth
label_path = osp.join(args.data_root_tgt, label_gt_list_tgt[i])
label = np.array(Image.open(label_path), np.uint8)
if args.with_prior == 'True':
scorer.update(pred_label, label, i)
scorer_np.update(pred_label_np, label, i)
# save target training list
fout = 'issegm/data_list/{}/{}_training_gpu{}.lst'.format(
args.dataset_tgt, args.split_tgt, args.gpus)
fo = open(fout, "w")
for idx_image in range(x_num):
sample_name = osp.splitext(osp.basename(image_list_tgt[idx_image]))[0]
fo.write(
image_tgt_list[idx_image] + '\t' +
osp.join(save_dir_pseudo_labelIds, '{}.png'.format(sample_name)) +
'\n')
fo.close()
############################ kc generation
start_sort = time.time()
# threshold for each class
if args.kc_policy == 'global':
for idx_cls in np.arange(0, num_classes):
tname = 'array_cls' + str(idx_cls)
exec(
"array_pixel = np.append(array_pixel,%s)" % tname
) # reverse=False for ascending losses and reverse=True for descending confidence
array_pixel = sorted(array_pixel, reverse=True)
len_cls = len(array_pixel)
len_thresh = int(math.floor(len_cls * init_tgt_port))
cls_size[:] = len_cls
cls_thresh[:] = array_pixel[len_thresh - 1].copy()
array_pixel = 0.0
if args.kc_policy == 'cb':
for idx_cls in np.arange(0, num_classes):
tname = 'array_cls' + str(idx_cls)
if cls_exist_array[0, idx_cls] == 1:
exec(
"%s = sorted(%s,reverse=True)" % (tname, tname)
) # reverse=False for ascending losses and reverse=True for descending confidence
exec("len_cls = len(%s)" % tname)
cls_size[idx_cls] = len_cls
len_thresh = int(math.floor(len_cls * init_tgt_port))
if len_thresh != 0:
exec("cls_thresh[idx_cls] = %s[len_thresh-1].copy()" %
tname)
exec("%s = %d" % (tname, 0.0))
# threshold for mine_id with priority
mine_id_priority = np.nonzero(
cls_size / np.sum(cls_size) < args.mine_thresh)[0]
# chosen mine_id
mine_id_all = np.argsort(cls_size / np.sum(cls_size))
mine_id = mine_id_all[:args.mine_id_number]
print(mine_id)
np.save(save_dir_stats + '/mine_id.npy', mine_id)
np.save(save_dir_stats + '/mine_id_priority.npy', mine_id_priority)
np.save(save_dir_stats + '/cls_thresh.npy', cls_thresh)
np.save(save_dir_stats + '/cls_size.npy', cls_size)
logger.info('Kc determination done in %.2f s.', time.time() - start_sort)
############################ pseudo-label generation
for i in xrange(x_num):
sample_name = osp.splitext(osp.basename(image_list_tgt[i]))[0]
sample_pseudo_label_name = osp.join(save_dir_pseudo_labelIds,
'{}.png'.format(sample_name))
sample_pseudocolor_label_name = osp.join(save_dir_pseudo_color,
'{}.png'.format(sample_name))
out_path_probmap = osp.join(save_dir_probmap,
'{}.npy'.format(sample_name))
probmap = np.load(out_path_probmap)
rw_probmap = np.zeros(probmap.shape, np.single)
cls_thresh[
cls_thresh ==
0] = 1.0 # cls_thresh = 0 means there is no prediction in this class
############# pseudo-label assignment
for idx_cls in np.arange(0, num_classes):
cls_thresh_temp = cls_thresh[idx_cls]
cls_probmap = probmap[idx_cls, :, :]
cls_rw_probmap = np.true_divide(cls_probmap, cls_thresh_temp)
rw_probmap[idx_cls, :, :] = cls_rw_probmap.copy()
rw_probmap_max = np.amax(rw_probmap, axis=0)
pseudo_label = np.argmax(rw_probmap, axis=0)
############# pseudo-label selection
idx_unconfid = rw_probmap_max < 1
idx_confid = rw_probmap_max >= 1
# pseudo-labels with labelID
pseudo_label = pseudo_label.astype(np.uint8)
pseudo_label_labelID = dargs.id_2_label_tgt[pseudo_label]
rw_pred_label = pseudo_label_labelID.copy()
# ignore label assignment, compatible with labelIDs
pseudo_label_labelID[idx_unconfid] = 0
############# save pseudo-label
im_to_save_pseudo = Image.fromarray(
pseudo_label_labelID.astype(np.uint8))
im_to_save_pseudocol = im_to_save_pseudo.copy()
if dargs.cmap is not None: # save segmentation prediction with color
im_to_save_pseudocol.putpalette(dargs.cmap.ravel())
out_path_pseudo = osp.join(save_dir_pseudo_labelIds,
'{}.png'.format(sample_name))
out_path_colpseudo = osp.join(save_dir_pseudo_color,
'{}.png'.format(sample_name))
im_to_save_pseudo.save(out_path_pseudo)
im_to_save_pseudocol.save(out_path_colpseudo)
############# save reweighted pseudo-label in cbst
if args.kc_policy == 'cb':
im_to_save_rw = Image.fromarray(rw_pred_label.astype(np.uint8))
im_to_save_rwcolor = im_to_save_rw.copy()
if dargs.cmap is not None:
im_to_save_rwcolor.putpalette(dargs.cmap.ravel())
out_path_rw = osp.join(save_dir_rwlabelIds,
'{}.png'.format(sample_name))
out_path_rwcolor = osp.join(save_dir_rwcolor,
'{}.png'.format(sample_name))
# save no prior prediction with labelIDs and colors
im_to_save_rw.save(out_path_rw)
im_to_save_rwcolor.save(out_path_rwcolor)
## remove probmap folder
import shutil
shutil.rmtree(save_dir_probmap)
def do(args, model_specs, logger):
meta = get_dataset_specs(args, model_specs)
id_2_label = meta['id_2_label']
cmap = meta['cmap']
input_h = 1024
input_w = 2048
classes = model_specs['classes']
label_stride = model_specs['feat_stride']
start_idx = args.start
end_idx = args.end
print('{}-{}\n'.format(start_idx, end_idx))
image_list = []
idx = 0
with open(args.file_list) as f:
for item in f.readlines():
if idx < start_idx:
idx += 1
continue
if idx > end_idx:
break
item = item.strip()
image_list.append(os.path.join(args.data_root, item))
idx += 1
net_args, net_auxs = mxutil.load_params_from_file(args.weights)
net = fcrna_model_a1(classes, label_stride, bootstrapping=True)
if net is None:
raise NotImplementedError('Unknown network')
contexts = [mx.gpu(int(_)) for _ in args.gpus.split(',')]
mod = mx.mod.Module(net, context=contexts)
crop_size = 2048
save_dir = args.output
x_num = len(image_list)
transformers = [ts.Scale(crop_size, Image.CUBIC, False)]
transformers += _get_transformer_image()
transformer = ts.Compose(transformers)
start = time.time()
for i in range(x_num):
time1 = time.time()
sample_name = osp.splitext(osp.basename(image_list[i]))[0]
out_path = osp.join(save_dir, '{}.png'.format(sample_name))
if os.path.exists(out_path):
continue
im_path = osp.join(args.data_root, image_list[i])
rim = np.array(Image.open(im_path).convert('RGB'), np.uint8)
h, w = rim.shape[:2]
need_resize = False
if h != input_h or w != input_w:
need_resize = True
im = np.array(
Image.fromarray(rim.astype(np.uint8, copy=False)).resize(
(input_w, input_h), Image.NEAREST))
else:
im = rim
im = transformer(im)
imh, imw = im.shape[:2]
# init
label_h, label_w = input_h / label_stride, input_w / label_stride
test_steps = 1
pred_stride = label_stride / test_steps
pred_h, pred_w = label_h * test_steps, label_w * test_steps
input_data = np.zeros((1, 3, input_h, input_w), np.single)
input_label = 255 * np.ones((1, label_h * label_w), np.single)
dataiter = mx.io.NDArrayIter(input_data, input_label)
batch = dataiter.next()
mod.bind(dataiter.provide_data,
dataiter.provide_label,
for_training=False,
force_rebind=True)
if not mod.params_initialized:
mod.init_params(arg_params=net_args, aux_params=net_auxs)
nim = np.zeros((3, imh + label_stride, imw + label_stride), np.single)
sy = sx = label_stride / 2
nim[:, sy:sy + imh, sx:sx + imw] = im.transpose(2, 0, 1)
net_preds = np.zeros((classes, pred_h, pred_w), np.single)
# sy = sx = pred_stride // 2 + np.arange(test_steps) * pred_stride
# sy = sx = sy[0]
input_data = np.zeros((1, 3, input_h, input_w), np.single)
input_data[0, :, :imh, :imw] = nim[:, sy:sy + imh, sx:sx + imw]
batch.data[0] = mx.nd.array(input_data)
mod.forward(batch, is_train=False)
this_call_preds = mod.get_outputs()[0].asnumpy()[0]
if args.test_flipping:
batch.data[0] = mx.nd.array(input_data[:, :, :, ::-1])
mod.forward(batch, is_train=False)
this_call_preds = 0.5 * (
this_call_preds +
mod.get_outputs()[0].asnumpy()[0][:, :, ::-1])
net_preds[:, 0:0 + pred_h:test_steps,
0:0 + pred_w:test_steps] = this_call_preds
# compute pixel-wise predictions
interp_preds = interp_preds_as(rim.shape[:2], net_preds, pred_stride,
imh, imw)
pred_label = interp_preds.argmax(0)
if id_2_label is not None:
pred_label = id_2_label[pred_label]
# save predicted labels into an image
im_to_save = Image.fromarray(pred_label.astype(np.uint8))
if cmap is not None:
im_to_save.putpalette(cmap.ravel())
if need_resize:
im_to_save = im_to_save.resize((w, h), Image.NEAREST)
im_to_save.save(out_path)
time2 = time.time()
print("{}/{} {} finish in {} s\n".format(i, x_num, out_path,
time2 - time1))
logger.info('Done in %.2f s.', time.time() - start)
def _train_impl(args, model_specs, logger):
if len(args.output) > 0:
_make_dirs(args.output)
# dataiter
dataset_specs_tgt = get_dataset_specs_tgt(args, model_specs)
scale, mean_, _ = _get_scalemeanstd()
if scale > 0:
mean_ /= scale
margs = argparse.Namespace(**model_specs)
dargs = argparse.Namespace(**dataset_specs_tgt)
# number of list_lines
split_filename = 'issegm/data_list/{}/{}.lst'.format(
margs.dataset, args.split)
num_source = 0
with open(split_filename) as f:
for item in f.readlines():
num_source = num_source + 1
#
batches_per_epoch = num_source // args.batch_images
# optimizer
assert args.to_epoch is not None
if args.stop_epoch is not None:
assert args.stop_epoch > args.from_epoch and args.stop_epoch <= args.to_epoch
else:
args.stop_epoch = args.to_epoch
from_iter = args.from_epoch * batches_per_epoch
to_iter = args.to_epoch * batches_per_epoch
lr_params = model_specs['lr_params']
base_lr = lr_params['base']
if lr_params['type'] == 'fixed':
scheduler = FixedScheduler()
elif lr_params['type'] == 'step':
left_step = []
for step in lr_params['args']['step']:
if from_iter > step:
base_lr *= lr_params['args']['factor']
continue
left_step.append(step - from_iter)
model_specs['lr_params']['step'] = left_step
scheduler = mx.lr_scheduler.MultiFactorScheduler(**lr_params['args'])
elif lr_params['type'] == 'linear':
scheduler = LinearScheduler(updates=to_iter + 1,
frequency=50,
stop_lr=min(base_lr / 100., 1e-6),
offset=from_iter)
elif lr_params['type'] == 'poly':
scheduler = PolyScheduler(updates=to_iter + 1,
frequency=50,
stop_lr=min(base_lr / 100., 1e-8),
power=0.9,
offset=from_iter)
initializer = mx.init.Xavier(rnd_type='gaussian',
factor_type='in',
magnitude=2)
optimizer_params = {
'learning_rate': base_lr,
'momentum': 0.9,
'wd': args.weight_decay,
'lr_scheduler': scheduler,
'rescale_grad': 1.0 / len(args.gpus.split(',')),
}
data_src_port = args.init_src_port
data_src_num = int(num_source * data_src_port)
mod = _get_module(args, margs, dargs)
addr_weights = args.weights # first weights should be xxxx_ep-0000.params!
addr_output = args.output
# initializer
net_args = None
net_auxs = None
###
if addr_weights is not None:
net_args, net_auxs = mxutil.load_params_from_file(addr_weights)
####################################### training model
to_model = osp.join(addr_output, str(args.idx_round),
'{}_ep'.format(args.model))
dataiter = FileIter(
dataset=margs.dataset,
split=args.split,
data_root=args.data_root,
num_sel_source=data_src_num,
num_source=num_source,
seed_int=args.seed_int,
dataset_tgt=args.dataset_tgt,
split_tgt=args.split_tgt,
data_root_tgt=args.data_root_tgt,
sampler='random',
batch_images=args.batch_images,
meta=dataset_specs_tgt,
rgb_mean=mean_,
feat_stride=margs.feat_stride,
label_stride=margs.feat_stride,
origin_size=args.origin_size,
origin_size_tgt=args.origin_size_tgt,
crop_size=args.crop_size,
scale_rate_range=[float(_) for _ in args.scale_rate_range.split(',')],
transformer=None,
transformer_image=ts.Compose(_get_transformer_image()),
prefetch_threads=args.prefetch_threads,
prefetcher_type=args.prefetcher,
)
dataiter.reset()
mod.fit(
dataiter,
eval_metric=_get_metric(),
batch_end_callback=mx.callback.log_train_metric(10, auto_reset=False),
epoch_end_callback=mx.callback.do_checkpoint(to_model),
kvstore=args.kvstore,
optimizer='sgd',
optimizer_params=optimizer_params,
initializer=initializer,
arg_params=net_args,
aux_params=net_auxs,
allow_missing=args.from_epoch == 0,
begin_epoch=args.from_epoch,
num_epoch=args.stop_epoch,
)
def _val_impl(args, model_specs, logger):
assert args.prefetch_threads == 1
assert args.weights is not None
margs = argparse.Namespace(**model_specs)
dargs = argparse.Namespace(**get_dataset_specs(args, model_specs))
image_list, label_list = parse_split_file(margs.dataset, args.split)
net_args, net_auxs = mxutil.load_params_from_file(args.weights)
net = None
mod = _get_module(args, margs, dargs, net)
has_gt = args.split in (
'train',
'val',
)
crop_sizes = sorted([int(_) for _ in args.test_scales.split(',')])[::-1]
# TODO: multi-scale testing
assert len(crop_sizes) == 1, 'multi-scale testing not implemented'
label_stride = margs.feat_stride
crop_size = crop_sizes[0]
save_dir = osp.join(args.output, osp.splitext(args.log_file)[0])
_make_dirs(save_dir)
x_num = len(image_list)
do_forward = True
if do_forward:
batch = None
transformers = [ts.Scale(crop_size, Image.CUBIC, False)]
transformers += _get_transformer_image()
transformer = ts.Compose(transformers)
scorer = ScoreUpdater(dargs.valid_labels, margs.classes, x_num, logger)
scorer.reset()
start = time.time()
done_count = 0
for i in xrange(x_num):
sample_name = osp.splitext(osp.basename(image_list[i]))[0]
# skip computed images
if args.save_predictions:
pred_save_path = osp.join(save_dir, 'predictions',
'{}.h5'.format(sample_name))
if osp.isfile(pred_save_path):
logger.info('Skipped {} {}/{}'.format(sample_name, i + 1,
x_num))
continue
im_path = osp.join(args.data_root, image_list[i])
rim = np.array(Image.open(im_path).convert('RGB'), np.uint8)
if do_forward:
im = transformer(rim)
imh, imw = im.shape[:2]
# init
if batch is None:
if dargs.ident_size:
input_h = make_divisible(imh, margs.feat_stride)
input_w = make_divisible(imw, margs.feat_stride)
else:
input_h = input_w = make_divisible(crop_size,
margs.feat_stride)
label_h, label_w = input_h / label_stride, input_w / label_stride
test_steps = args.test_steps
pred_stride = label_stride / test_steps
pred_h, pred_w = label_h * test_steps, label_w * test_steps
input_data = np.zeros((1, 3, input_h, input_w), np.single)
input_label = 255 * np.ones((1, label_h * label_w), np.single)
dataiter = mx.io.NDArrayIter(input_data, input_label)
batch = dataiter.next()
mod.bind(dataiter.provide_data,
dataiter.provide_label,
for_training=False,
force_rebind=True)
if not mod.params_initialized:
mod.init_params(arg_params=net_args, aux_params=net_auxs)
nim = np.zeros((3, imh + label_stride, imw + label_stride),
np.single)
sy = sx = label_stride // 2
nim[:, sy:sy + imh, sx:sx + imw] = im.transpose(2, 0, 1)
net_preds = np.zeros((margs.classes, pred_h, pred_w), np.single)
sy = sx = pred_stride // 2 + np.arange(test_steps) * pred_stride
for ix in xrange(test_steps):
for iy in xrange(test_steps):
input_data = np.zeros((1, 3, input_h, input_w), np.single)
input_data[0, :, :imh, :imw] = nim[:, sy[iy]:sy[iy] + imh,
sx[ix]:sx[ix] + imw]
batch.data[0] = mx.nd.array(input_data)
mod.forward(batch, is_train=False)
this_call_preds = mod.get_outputs()[0].asnumpy()[0]
if args.test_flipping:
batch.data[0] = mx.nd.array(input_data[:, :, :, ::-1])
mod.forward(batch, is_train=False)
this_call_preds = 0.5 * (
this_call_preds +
mod.get_outputs()[0].asnumpy()[0][:, :, ::-1])
net_preds[:, iy:iy + pred_h:test_steps,
ix:ix + pred_w:test_steps] = this_call_preds
# save predicted probabilities
if args.save_predictions:
_make_dirs(osp.dirname(pred_save_path))
tmp = (rim.shape[:2], net_preds.astype(np.float16), pred_stride,
imh, imw)
util.h5py_save(pred_save_path, *tmp)
if args.save_results:
# compute pixel-wise predictions
interp_preds = interp_preds_as(rim.shape[:2], net_preds,
pred_stride, imh, imw)
pred_label = interp_preds.argmax(0)
if dargs.id_2_label is not None:
pred_label = dargs.id_2_label[pred_label]
# save predicted labels into an image
out_path = osp.join(save_dir, '{}.png'.format(sample_name))
im_to_save = Image.fromarray(pred_label.astype(np.uint8))
if dargs.cmap is not None:
im_to_save.putpalette(dargs.cmap.ravel())
im_to_save.save(out_path)
else:
assert not has_gt
done_count += 1
if not has_gt:
logger.info('Done {}/{} with speed: {:.2f}/s'.format(
i + 1, x_num, 1. * done_count / (time.time() - start)))
continue
label_path = osp.join(args.data_root, label_list[i])
label = np.array(Image.open(label_path), np.uint8)
# save correctly labeled pixels into an image
out_path = osp.join(save_dir, 'correct', '{}.png'.format(sample_name))
_make_dirs(osp.dirname(out_path))
invalid_mask = np.logical_not(np.in1d(
label, dargs.valid_labels)).reshape(label.shape)
Image.fromarray(
(invalid_mask * 255 + (label == pred_label) * 127).astype(
np.uint8)).save(out_path)
scorer.update(pred_label, label, i)
logger.info('Done in %.2f s.', time.time() - start)
def _train_impl(args, model_specs, logger):
if len(args.output) > 0:
_make_dirs(args.output)
# dataiter
dataset_specs = get_dataset_specs(args, model_specs)
scale, mean_, _ = _get_scalemeanstd()
if scale > 0:
mean_ /= scale
margs = argparse.Namespace(**model_specs)
dargs = argparse.Namespace(**dataset_specs)
dataiter = FileIter(
dataset=margs.dataset,
split=args.split,
data_root=args.data_root,
sampler='random',
batch_images=args.batch_images,
meta=dataset_specs,
rgb_mean=mean_,
feat_stride=margs.feat_stride,
label_stride=margs.feat_stride,
origin_size=args.origin_size,
crop_size=args.crop_size,
scale_rate_range=[float(_) for _ in args.scale_rate_range.split(',')],
transformer=None,
transformer_image=ts.Compose(_get_transformer_image()),
prefetch_threads=args.prefetch_threads,
prefetcher_type=args.prefetcher,
)
dataiter.reset()
# optimizer
assert args.to_epoch is not None
if args.stop_epoch is not None:
assert args.stop_epoch > args.from_epoch and args.stop_epoch <= args.to_epoch
else:
args.stop_epoch = args.to_epoch
from_iter = args.from_epoch * dataiter.batches_per_epoch
to_iter = args.to_epoch * dataiter.batches_per_epoch
lr_params = model_specs['lr_params']
base_lr = lr_params['base']
if lr_params['type'] == 'fixed':
scheduler = FixedScheduler()
elif lr_params['type'] == 'step':
left_step = []
for step in lr_params['args']['step']:
if from_iter > step:
base_lr *= lr_params['args']['factor']
continue
left_step.append(step - from_iter)
model_specs['lr_params']['step'] = left_step
scheduler = mx.lr_scheduler.MultiFactorScheduler(**lr_params['args'])
elif lr_params['type'] == 'linear':
scheduler = LinearScheduler(updates=to_iter + 1,
frequency=50,
stop_lr=min(base_lr / 100., 1e-6),
offset=from_iter)
optimizer_params = {
'learning_rate': base_lr,
'momentum': 0.9,
'wd': args.weight_decay,
'lr_scheduler': scheduler,
'rescale_grad': 1.0 / len(args.gpus.split(',')),
}
# initializer
net_args = None
net_auxs = None
if args.weights is not None:
net_args, net_auxs = mxutil.load_params_from_file(args.weights)
initializer = mx.init.Xavier(rnd_type='gaussian',
factor_type='in',
magnitude=2)
#
to_model = osp.join(args.output, '{}_ep'.format(args.model))
mod = _get_module(args, margs, dargs)
mod.fit(
dataiter,
eval_metric=_get_metric(),
batch_end_callback=mx.callback.Speedometer(dataiter.batch_size, 1),
epoch_end_callback=mx.callback.do_checkpoint(to_model),
kvstore=args.kvstore,
optimizer='sgd',
optimizer_params=optimizer_params,
initializer=initializer,
arg_params=net_args,
aux_params=net_auxs,
allow_missing=args.from_epoch == 0,
begin_epoch=args.from_epoch,
num_epoch=args.stop_epoch,
)
def _val_impl(args, model_specs, logger):
assert args.prefetch_threads == 1
assert args.weights is not None
net_args, net_auxs = util.load_params_from_file(args.weights)
mod = _get_module(model_specs)
has_gt = args.split in (
'train',
'val',
)
scale_, mean_, std_ = _get_scalemeanstd()
if args.test_scales is None:
crop_sizes = [model_specs['crop_size']]
else:
crop_sizes = sorted([int(_)
for _ in args.test_scales.split(',')])[::-1]
batch_images = args.batch_images
if has_gt:
gt_labels = np.array(
parse_split_file(model_specs['split_filename'], args.data_root)[1])
save_dir = os.path.join(args.output, os.path.splitext(args.log_file)[0])
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
preds = []
for crop_size in crop_sizes:
save_path = os.path.join(save_dir, 'preds_sz{}'.format(crop_size))
if os.path.isfile(save_path):
logger.info('File %s exists, skipped crop size %d', save_path,
crop_size)
with open(save_path) as f:
preds.append(cPickle.load(f))
continue
ts_list = [
ts.Scale(crop_size),
ts.ThreeCrops(crop_size)
if args.test_3crops else ts.CenterCrop(crop_size),
]
if scale_ > 0:
ts_list.append(ts.ListInput(ts.ColorScale(np.single(scale_))))
ts_list += [ts.ListInput(ts.ColorNormalize(mean_, std_))]
transformer = ts.Compose(ts_list)
dataiter = FileIter(
split_filename=model_specs['split_filename'],
data_root=args.data_root,
has_gt=has_gt,
batch_images=batch_images,
transformer=transformer,
prefetch_threads=args.prefetch_threads,
prefetcher_type=args.prefetcher,
)
dataiter.reset()
mod.bind(dataiter.provide_data,
dataiter.provide_label,
for_training=False,
force_rebind=True)
if not mod.params_initialized:
mod.init_params(arg_params=net_args, aux_params=net_auxs)
this_call_preds = []
start = time.time()
counter = [0, 0]
for nbatch, batch in enumerate(dataiter):
mod.forward(batch, is_train=False)
outputs = mod.get_outputs()[0].asnumpy()
outputs = outputs.reshape(
(batch_images, -1, model_specs['classes'])).mean(1)
this_call_preds.append(outputs)
if args.test_flipping:
batch.data[0] = mx.nd.flip(batch.data[0], axis=3)
mod.forward(batch, is_train=False)
outputs = mod.get_outputs()[0].asnumpy()
outputs = outputs.reshape(
(batch_images, -1, model_specs['classes'])).mean(1)
this_call_preds[-1] = (this_call_preds[-1] + outputs) / 2
score_str = ''
if has_gt:
counter[0] += batch_images
counter[1] += (this_call_preds[-1].argmax(1) ==
gt_labels[nbatch * batch_images:(nbatch + 1) *
batch_images]).sum()
score_str = ', Top1 {:.4f}%'.format(100.0 * counter[1] /
counter[0])
logger.info('Crop size {}, done {}/{} at speed: {:.2f}/s{}'.\
format(crop_size, nbatch+1, dataiter.batches_per_epoch, 1.*(nbatch+1)*batch_images / (time.time()-start), score_str))
logger.info('Done crop size {} in {:.4f}s.'.format(
crop_size,
time.time() - start))
this_call_preds = np.vstack(this_call_preds)
with open(save_path, 'wb') as f:
cPickle.dump(this_call_preds, f)
preds.append(this_call_preds)
for num_sizes in set((
1,
len(crop_sizes),
)):
for this_pred_inds in itertools.combinations(xrange(len(crop_sizes)),
num_sizes):
this_pred = np.mean([preds[_] for _ in this_pred_inds], axis=0)
this_pred_label = this_pred.argsort(1)[:, -1 - np.arange(5)]
logger.info('Done testing crop_size %s',
[crop_sizes[_] for _ in this_pred_inds])
if has_gt:
top1 = 100. * (this_pred_label[:, 0]
== gt_labels).sum() / gt_labels.size
top5 = 100. * sum(
map(lambda x, y: y in x.tolist(), this_pred_label,
gt_labels)) / gt_labels.size
logger.info('Top1 %.4f%%, Top5 %.4f%%', top1, top5)
else:
# TODO: Save predictions for submission
raise NotImplementedError('Save predictions for submission')
def _val_impl(args, model_specs, logger):
assert args.prefetch_threads == 1
assert args.weights is not None
margs = argparse.Namespace(**model_specs)
dargs = argparse.Namespace(**get_dataset_specs(args, model_specs))
image_list, label_list = parse_split_file(margs.dataset, args.split)
net_args, net_auxs = mxutil.load_params_from_file(args.weights)
net = None
mod = _get_module(args, margs, dargs, net)
has_gt = args.split in (
'train',
'val',
)
crop_sizes = sorted([int(_) for _ in args.test_scales.split(',')])[::-1]
crop_size = crop_sizes[0]
# TODO: multi-scale testing
assert len(crop_sizes) == 1, 'multi-scale testing not implemented'
label_stride = margs.feat_stride
save_dir_color = osp.join(args.output, 'color')
save_dir_labelId = osp.join(args.output, 'labelId')
save_dir_trainId = osp.join(args.output, 'trainId')
_make_dirs(save_dir_color)
_make_dirs(save_dir_labelId)
_make_dirs(save_dir_trainId)
x_num = len(image_list)
do_forward = True
# for all images that has the same resolution
if do_forward:
batch = None
transformers = [ts.Scale(crop_size, Image.CUBIC, False)]
transformers += _get_transformer_image()
transformer = ts.Compose(transformers)
scorer = ScoreUpdater(dargs.valid_labels, margs.classes, x_num, logger)
scorer.reset()
start = time.time()
done_count = 0
############################
#for i in xrange(x_num):
for i in range(5):
############################ network forward on single image
sample_name = osp.splitext(osp.basename(image_list[i]))[0]
im_path = osp.join(args.data_root, image_list[i])
rim = np.array(Image.open(im_path).convert('RGB'), np.uint8)
if do_forward:
im = transformer(rim)
print('Image shape :', im.shape, rim.shape)
imh, imw = im.shape[:2]
# init
if batch is None:
if dargs.ident_size:
input_h = make_divisible(imh, margs.feat_stride)
input_w = make_divisible(imw, margs.feat_stride)
else:
input_h = input_w = make_divisible(crop_size,
margs.feat_stride)
label_h, label_w = input_h / label_stride, input_w / label_stride
test_steps = args.test_steps
pred_stride = label_stride / test_steps
pred_h, pred_w = label_h * test_steps, label_w * test_steps
input_data = np.zeros((1, 3, input_h, input_w), np.single)
input_label = 255 * np.ones((1, label_h * label_w), np.single)
dataiter = mx.io.NDArrayIter(input_data, input_label)
batch = dataiter.next()
mod.bind(dataiter.provide_data,
dataiter.provide_label,
for_training=False,
force_rebind=True)
# since we could use different transformers, but the same variables in the loop.
if not mod.params_initialized:
mod.init_params(arg_params=net_args, aux_params=net_auxs)
nim = np.zeros((3, imh + label_stride, imw + label_stride),
np.single)
sy = sx = label_stride // 2
nim[:, sy:sy + imh, sx:sx + imw] = im.transpose(2, 0, 1)
nim = im.transpose(2, 0, 1)
net_preds = np.zeros((margs.classes, pred_h, pred_w), np.single)
#sy = sx = pred_stride // 2 + np.arange(test_steps) * pred_stride
patchSize = 512
nPatches = 100
pixelCount = np.ones((imh, imw))
interp_preds = np.zeros((19, imh, imw))
net_preds = np.zeros((margs.classes, pred_h, pred_w), np.single)
for ix in xrange(test_steps):
for iy in xrange(test_steps):
input_data = np.zeros((1, 3, input_h, input_w), np.single)
#print('input_data shape:', input_data.shape, imh, imw, nim.shape, sx, sy)
#input_data[0, :, :imh, :imw] = nim[:, sy[iy]:sy[iy] + imh, sx[ix]:sx[ix] + imw]
input_data[0, :, :imh, :imw] = nim
batch.data[0] = mx.nd.array(input_data)
#print('shape :', batch.data[0].shape)
mod.forward(batch, is_train=False)
this_call_preds = mod.get_outputs()[0].asnumpy()[0]
#print('shape :', this_call_preds.shape)
if args.test_flipping:
batch.data[0] = mx.nd.array(input_data[:, :, :, ::-1])
mod.forward(batch, is_train=False)
# average the original and flipped image prediction
this_call_preds = 0.5 * (
this_call_preds +
mod.get_outputs()[0].asnumpy()[0][:, :, ::-1])
net_preds[:, iy:iy + pred_h:test_steps,
ix:ix + pred_w:test_steps] = this_call_preds
# batch = None
# print('I am here')
interp_preds = interp_preds_as(rim.shape[:2], net_preds,
pred_stride, imh, imw)
startRow = np.random.randint(0, imh - patchSize - 4, size=nPatches)
startCol = np.random.randint(0, imw - patchSize - 4, size=nPatches)
#print(startRow, startCol)
for patch in range(nPatches):
for ix in xrange(test_steps):
for iy in xrange(test_steps):
#input_data = np.zeros((1, 3, input_h, input_w), np.single)
#print('input_data shape:', input_data.shape, imh, imw, nim.shape, sx, sy)
#input_data[0, :, :imh, :imw] = nim[:, sy[iy]:sy[iy] + imh, sx[ix]:sx[ix] + imw]
input_data = np.zeros((1, 3, patchSize, patchSize),
np.single)
pixelCount_temp = np.zeros((imh, imw))
#print('input_data shape:', input_data.shape, imh, imw, nim.shape, sx, sy)
#rowIdx = sy[iy] + startRow[patch]
#colIdx = sx[ix] + startCol[patch]
rowIdx = startRow[patch]
colIdx = startCol[patch]
input_data[0, :, :, :] = nim[:,
rowIdx:rowIdx + patchSize,
colIdx:colIdx + patchSize]
pixelCount_temp[rowIdx:rowIdx + patchSize,
colIdx:colIdx + patchSize] = 1
pixelCount += pixelCount_temp
batch.data[0] = mx.nd.array(input_data)
#print('shape :', batch.data[0].shape)
#np.save('/media/itu/CV_Lab/JavedData/cbst-master/debug_own/imBatch/' + str(ix)+ str(iy)+ '.npy', batch.data[0].asnumpy())
mod.forward(batch, is_train=False)
this_call_preds = mod.get_outputs()[0].asnumpy()[0]
#np.save('/media/itu/CV_Lab/JavedData/cbst-master/debug_own/imBatch/' + str(ix)+ str(iy)+ '.npy', this_call_preds)
#print('shape :', this_call_preds.shape)
if args.test_flipping:
batch.data[0] = mx.nd.array(
input_data[:, :, :, ::-1])
mod.forward(batch, is_train=False)
# average the original and flipped image prediction
this_call_preds = 0.5 * (
this_call_preds +
mod.get_outputs()[0].asnumpy()[0][:, :, ::-1])
#net_preds[:, iy:iy + pred_h:test_steps, ix:ix + pred_w:test_steps] = this_call_preds
interp_preds_temp = interp_preds_as(input_data.shape[2:],
this_call_preds,
pred_stride, patchSize,
patchSize)
#print(rowIdx, colIdx)
interp_preds[:, rowIdx:rowIdx + patchSize,
colIdx:colIdx + patchSize] += interp_preds_temp
interp_preds = interp_preds / pixelCount
# batch = None
#print('I am here')
#interp_preds = interp_preds_as(rim.shape[:2], net_preds, pred_stride, imh, imw)
if args.save_results:
# compute pixel-wise predictions
pred_label = interp_preds.argmax(0)
# with trainIDs
pred_label_trainId = pred_label.copy()
# with labelIDs
if dargs.id_2_label is not None:
pred_label = dargs.id_2_label[pred_label_trainId]
# #
# save predicted label with trainIDs, labelIDs, colors
out_path_trainId = osp.join(save_dir_trainId,
'{}.png'.format(sample_name))
out_path_labelId = osp.join(save_dir_labelId,
'{}.png'.format(sample_name))
out_path_color = osp.join(save_dir_color,
'{}.png'.format(sample_name))
im_to_save_trainId = Image.fromarray(
pred_label_trainId.astype(np.uint8))
im_to_save = Image.fromarray(pred_label.astype(np.uint8))
im_to_save_labelId = im_to_save.copy()
# save color prediction
im_to_save.putpalette(dargs.cmap.ravel())
im_to_save.save(out_path_color)
im_to_save_trainId.save(out_path_trainId)
im_to_save_labelId.save(out_path_labelId)
else:
assert not has_gt
done_count += 1
if not has_gt:
logger.info('Done {}/{} with speed: {:.2f}/s'.format(
i + 1, x_num, 1. * done_count / (time.time() - start)))
continue
if args.split in ('train', 'train+', 'val'):
label_path = osp.join(args.data_root, label_list[i])
label = np.array(Image.open(label_path), np.uint8)
# save correctly labeled pixels into an image
out_path = osp.join(save_dir_color, 'correct',
'{}.png'.format(sample_name))
_make_dirs(osp.dirname(out_path))
invalid_mask = np.logical_not(np.in1d(
label, dargs.valid_labels)).reshape(label.shape)
Image.fromarray(
(invalid_mask * 255 + (label == pred_label) * 127).astype(
np.uint8)).save(out_path)
scorer.update(pred_label, label, i)
print('End is here :', scorer.overall_scores(logger))
logger.info('Done in %.2f s.', time.time() - start)
