def eval_prepared(self, num_gpu, batch_size):
if self.min_num_workers == 0:
return MapData(self, self.ex_process.val_process)
else:
return MultiProcessMapData(self, max(num_gpu,
self.min_num_workers),
self.ex_process.val_process)
def get_plain_train_dataflow(batch_size=2):
# no aspect ratio grouping
print("In train dataflow")
roidbs = list(itertools.chain.from_iterable(DatasetRegistry.get(x).training_roidbs() for x in cfg.DATA.TRAIN))
print_class_histogram(roidbs)
print("Done loading roidbs")
# Filter out images that have no gt boxes, but this filter shall not be applied for testing.
# The model does support training with empty images, but it is not useful for COCO.
num = len(roidbs)
roidbs = list(filter(lambda img: len(img["boxes"][img["is_crowd"] == 0]) > 0, roidbs))
logger.info(
"Filtered {} images which contain no non-crowd groudtruth boxes. Total #images for training: {}".format(
num - len(roidbs), len(roidbs)
)
)
ds = DataFromList(roidbs, shuffle=True)
preprocess = TrainingDataPreprocessor()
buffer_size = cfg.DATA.NUM_WORKERS * 20
ds = MultiProcessMapData(ds, cfg.DATA.NUM_WORKERS, preprocess, buffer_size=buffer_size)
ds.reset_state()
dataiter = ds.__iter__()
return dataiter
def prepared(self, num_gpu, batch_size, eval=False):
# use a single process version to debug if needed
if self.min_num_workers == 0:
ds = MapData(self, self.ex_process.train_process)
else:
ds = MultiProcessMapData(self, max(num_gpu, self.min_num_workers),
self.ex_process.train_process)
return BatchData(ds, batch_size)
def get_train_dataflow():
"""
Return a training dataflow. Each datapoint consists of the following:
An image: (h, w, 3),
1 or more pairs of (anchor_labels, anchor_boxes):
anchor_labels: (h', w', NA)
anchor_boxes: (h', w', NA, 4)
gt_boxes: (N, 4)
gt_labels: (N,)
If MODE_MASK, gt_masks: (N, h, w)
"""
roidbs = list(
itertools.chain.from_iterable(
DatasetRegistry.get(x).training_roidbs() for x in cfg.DATA.TRAIN))
print(
"---------------------------------------------------------------- data.py:343"
)
print_class_histogram(roidbs)
# Filter out images that have no gt boxes, but this filter shall not be applied for testing.
# The model does support training with empty images, but it is not useful for COCO.
num = len(roidbs)
roidbs = list(
filter(lambda img: len(img["boxes"][img["is_crowd"] == 0]) > 0,
roidbs))
logger.info(
"Filtered {} images which contain no non-crowd groudtruth boxes. Total #images for training: {}"
.format(num - len(roidbs), len(roidbs)))
ds = DataFromList(roidbs, shuffle=True)
preprocess = TrainingDataPreprocessor(cfg)
if cfg.DATA.NUM_WORKERS > 0:
if cfg.TRAINER == "horovod":
# one dataflow for each process, therefore don't need large buffer
buffer_size = cfg.DATA.NUM_WORKERS * 10
ds = MultiThreadMapData(ds,
cfg.DATA.NUM_WORKERS,
preprocess,
buffer_size=buffer_size)
# MPI does not like fork()
else:
buffer_size = cfg.DATA.NUM_WORKERS * 20
ds = MultiProcessMapData(ds,
cfg.DATA.NUM_WORKERS,
preprocess,
buffer_size=buffer_size)
else:
ds = MapData(ds, preprocess)
return ds
def get_train_dataflow_coco(add_mask=False):
"""
Return a training dataflow. Each datapoint is:
image, fm_labels, fm_boxes, gt_boxes, gt_class [, masks]
"""
imgs = COCODetection.load_many(config.BASEDIR,
config.TRAIN_DATASET,
add_gt=True,
add_mask=add_mask)
# Valid training images should have at least one fg box.
# But this filter shall not be applied for testing.
imgs = list(filter(lambda img: len(img['boxes']) > 0,
imgs)) # log invalid training
ds = DataFromList(imgs, shuffle=True)
aug = imgaug.AugmentorList([
CustomResize(config.SHORT_EDGE_SIZE, config.MAX_SIZE),
imgaug.Flip(horiz=True)
])
def preprocess(img):
print("start preproc coco")
start = time.time()
if config.USE_SECOND_HEAD:
fname, boxes, klass, second_klass, is_crowd = img['file_name'], img['boxes'], img['class'], \
img['second_class'], img['is_crowd']
else:
fname, boxes, klass, is_crowd = img['file_name'], img[
'boxes'], img['class'], img['is_crowd']
second_klass = None
res = preproc_img(fname, boxes, klass, second_klass, is_crowd, aug)
if res is None:
print("coco: preproc_img returned None on", fname)
return None
ret, params = res
im = ret[0]
boxes = ret[3]
# masks
if add_mask:
# augmentation will modify the polys in-place
segmentation = copy.deepcopy(img.get('segmentation', None))
segmentation = [
segmentation[k] for k in range(len(segmentation))
if not is_crowd[k]
]
assert len(segmentation) == len(boxes), (len(segmentation),
len(boxes))
# one image-sized binary mask per box
masks = []
for polys in segmentation:
polys = [aug.augment_coords(p, params) for p in polys]
masks.append(
segmentation_to_mask(polys, im.shape[0], im.shape[1]))
masks = np.asarray(masks, dtype='uint8') # values in {0, 1}
ret.append(masks)
# from viz import draw_annotation, draw_mask
# viz = draw_annotation(im, boxes, klass)
# for mask in masks:
# viz = draw_mask(viz, mask)
# tpviz.interactive_imshow(viz)
end = time.time()
elapsed = end - start
print("coco example done, elapsed:", elapsed)
return ret
#ds = MapData(ds, preprocess)
ds = MultiProcessMapData(ds,
nr_proc=4,
map_func=preprocess,
buffer_size=20)
return ds
def get_train_dataflow_mapillary(add_mask=False, map_to_coco=False):
train_img_path = config.MAPILLARY_PATH + "training/images/"
train_label_path = config.MAPILLARY_PATH + "training/instances/"
imgs = glob.glob(train_img_path + "*.jpg")
ds = DataFromList(imgs, shuffle=True)
aug = imgaug.AugmentorList([
CustomResize(config.SHORT_EDGE_SIZE, config.MAX_SIZE),
imgaug.Flip(horiz=True)
])
def preprocess(fname):
print("start preproc mapillary")
start = time.time()
label_fname = fname.replace(train_img_path,
train_label_path).replace(".jpg", ".png")
pil_label = Image.open(label_fname)
label = np.array(pil_label)
instances = np.unique(label)
instance_classes = [x // 256 for x in instances]
# filter by categories we use
instances_valid = [
cls in config.MAPILLARY_CAT_IDS_TO_USE for cls in instance_classes
]
instances = [
inst for inst, valid in zip(instances, instances_valid) if valid
]
instance_classes = [
cls for cls, valid in zip(instance_classes, instances_valid)
if valid
]
if len(instances) == 0:
print("no instances")
pil_label.close()
return None
if map_to_coco:
instance_classes = [
config.MAPILLARY_TO_COCO_MAP[cls] for cls in instance_classes
]
instance_classes = [
config.VOID_LABEL if cls == config.VOID_LABEL else
COCOMeta.category_id_to_class_id[cls]
for cls in instance_classes
]
else:
# remap to contiguous numbers starting with 1
instance_classes = [
config.MAPILLARY_CAT_IDS_TO_USE.index(cls) + 1
for cls in instance_classes
]
masks = np.array([label == inst for inst in instances], dtype=np.uint8)
#import cProfile
#start1 = time.time()
boxes1 = np.array(
[get_bbox_from_segmentation_mask(mask) for mask in masks],
dtype=np.float32)
#boxes1_time = time.time() - start1
#pr = cProfile.Profile()
#pr.enable()
#start1 = time.time()
#boxes2 = get_bboxes_from_segmentation_masks(masks)
#print("boxes1", boxes1_time, "boxes2", time.time() - start1)
#pr.disable()
#pr.print_stats(sort="cumulative")
#assert (boxes1 == boxes2).all(), (boxes1, boxes2)
boxes = boxes1
second_klass = np.array(instance_classes, dtype=np.int)
klass = np.ones_like(second_klass)
is_crowd = np.zeros_like(second_klass)
res = preproc_img(fname, boxes, klass, second_klass, is_crowd, aug)
if res is None:
print("mapillary: preproc_img returned None on", fname)
pil_label.close()
return None
ret, params = res
if add_mask:
do_flip, h, w = params[1]
assert do_flip in (True, False), do_flip
# augment label
label = np.array(pil_label.resize((w, h), Image.NEAREST))
if do_flip:
label = label[:, ::-1]
# create augmented masks
masks = np.array([label == inst for inst in instances],
dtype=np.uint8)
ret.append(masks)
end = time.time()
elapsed = end - start
print("mapillary example done, elapsed:", elapsed)
VISUALIZE = False
if VISUALIZE:
from viz import draw_annotation, draw_mask
config.CLASS_NAMES = [str(idx) for idx in range(81)]
im = ret[0]
boxes = ret[3]
draw_klass = ret[-2]
viz = draw_annotation(im, boxes, draw_klass)
for mask in masks:
viz = draw_mask(viz, mask)
tpviz.interactive_imshow(viz)
pil_label.close()
return ret
#ds = MapData(ds, preprocess)
ds = MultiProcessMapData(ds,
nr_proc=8,
map_func=preprocess,
buffer_size=35)
return ds
def get_train_dataflow_w_unlabeled(load_path):
"""
Return a training dataflow. Each datapoint consists of the following:
An image: (h, w, 3),
1 or more pairs of (anchor_labels, anchor_boxes):
anchor_labels: (h', w', NA)
anchor_boxes: (h', w', NA, 4)
gt_boxes: (N, 4)
gt_labels: (N,)
If MODE_MASK, gt_masks: (N, h, w)
"""
assert os.path.isfile(load_path), "{} does not find".format(load_path)
roidbs = list(
itertools.chain.from_iterable(
DatasetRegistry.get(x).training_roidbs() for x in cfg.DATA.TRAIN))
print_class_histogram(roidbs)
if "VOC" in cfg.DATA.TRAIN[0]:
roidbs_u = list(
itertools.chain.from_iterable(
DatasetRegistry.get(x).training_roidbs()
for x in cfg.DATA.UNLABEL))
unlabled2017_used = False
else:
unlabled2017_used = np.any(["@" not in x for x in cfg.DATA.TRAIN])
def prase_name(x):
if not unlabled2017_used:
assert "@" in load_path, (
"{}: Did you use wrong pseudo_data.py for "
"this model?").format(load_path)
return x + "-unlabeled"
else:
# return coco2017 unlabeled data
return "coco_unlabeled2017"
roidbs_u = list(
itertools.chain.from_iterable(
DatasetRegistry.get(prase_name(x)).training_roidbs()
for x in cfg.DATA.TRAIN))
print_class_histogram(roidbs_u)
# Filter out images that have no gt boxes, but this filter shall not be applied for testing.
# The model does support training with empty images, but it is not useful for COCO.
def remove_no_box_data(_roidbs, filter_fn, dset):
num = len(_roidbs)
_roidbs = filter_fn(_roidbs)
logger.info(
"Filtered {} images which contain no non-crowd groudtruth boxes. Total {} #images for training: {}"
.format(num - len(_roidbs), dset, len(_roidbs)))
return _roidbs
roidbs = remove_no_box_data(
roidbs, lambda x: list(
filter(lambda img: len(img["boxes"][img["is_crowd"] == 0]) > 0, x)
), "labeled")
# load unlabeled
if unlabled2017_used:
assert "@" not in load_path, "Did you use the wrong pseudo path"
pseudo_targets = dd.io.load(load_path)
logger.info("Loaded {} pseudo targets from {}".format(
len(pseudo_targets), load_path))
roidbs_u = remove_no_box_data(
roidbs_u, lambda x: list(
filter(
lambda img: len(pseudo_targets[img["image_id"]]["boxes"]) > 0,
x)), "unlabeled")
preprocess = TrainingDataPreprocessorSSlAug(
cfg, confidence=cfg.TRAIN.CONFIDENCE, pseudo_targets=pseudo_targets)
ds = DataFrom2List(roidbs, roidbs_u, shuffle=True)
if cfg.DATA.NUM_WORKERS > 0:
if cfg.TRAINER == "horovod":
buffer_size = cfg.DATA.NUM_WORKERS * 10
ds = MultiThreadMapData(ds,
cfg.DATA.NUM_WORKERS,
preprocess,
buffer_size=buffer_size)
else:
buffer_size = cfg.DATA.NUM_WORKERS * 20
ds = MultiProcessMapData(ds,
cfg.DATA.NUM_WORKERS,
preprocess,
buffer_size=buffer_size)
else:
ds = MapData(ds, preprocess)
return ds
def get_train_dataflow(add_mask=False):
"""
Return a training dataflow. Each datapoint is:
image, fm_labels, fm_boxes, gt_boxes, gt_class [, masks]
"""
imgs = COCODetection.load_many(config.BASEDIR,
config.TRAIN_DATASET,
add_gt=True,
add_mask=add_mask)
"""
To train on your own data, change this to your loader.
Produce "igms" as a list of dict, in the dict the following keys are needed for training:
height, width: integer
file_name: str
boxes: kx4 floats
class: k integers
is_crowd: k booleans. Use k False if you don't know what it means.
segmentation: k numpy arrays. Each array is a polygon of shape Nx2.
If your segmentation annotations are masks rather than polygons,
either convert it, or the augmentation code below will need to be
changed or skipped accordingly.
"""
# Valid training images should have at least one fg box.
# But this filter shall not be applied for testing.
imgs = list(filter(lambda img: len(img['boxes']) > 0,
imgs)) # log invalid training
ds = DataFromList(imgs, shuffle=True)
aug = imgaug.AugmentorList([
CustomResize(config.SHORT_EDGE_SIZE, config.MAX_SIZE),
imgaug.Flip(horiz=True)
])
def preprocess(img):
fname, boxes, klass, is_crowd = img['file_name'], img['boxes'], img[
'class'], img['is_crowd']
im = cv2.imread(fname, cv2.IMREAD_COLOR)
assert im is not None, fname
im = im.astype('float32')
# assume floatbox as input
assert boxes.dtype == np.float32
# augmentation:
im, params = aug.augment_return_params(im)
points = box_to_point8(boxes)
points = aug.augment_coords(points, params)
boxes = point8_to_box(points)
# rpn anchor:
try:
fm_labels, fm_boxes = get_rpn_anchor_input(im, boxes, is_crowd)
boxes = boxes[is_crowd == 0] # skip crowd boxes in training target
klass = klass[is_crowd == 0]
if not len(boxes):
raise MalformedData("No valid gt_boxes!")
except MalformedData as e:
log_once(
"Input {} is filtered for training: {}".format(fname, str(e)),
'warn')
return None
ret = [im, fm_labels, fm_boxes, boxes, klass]
if add_mask:
# augmentation will modify the polys in-place
segmentation = copy.deepcopy(img.get('segmentation', None))
segmentation = [
segmentation[k] for k in range(len(segmentation))
if not is_crowd[k]
]
assert len(segmentation) == len(boxes)
# Apply augmentation on polygon coordinates.
# And produce one image-sized binary mask per box.
masks = []
for polys in segmentation:
polys = [aug.augment_coords(p, params) for p in polys]
masks.append(
segmentation_to_mask(polys, im.shape[0], im.shape[1]))
masks = np.asarray(masks, dtype='uint8') # values in {0, 1}
ret.append(masks)
# from viz import draw_annotation, draw_mask
# viz = draw_annotation(im, boxes, klass)
# for mask in masks:
# viz = draw_mask(viz, mask)
# tpviz.interactive_imshow(viz)
return ret
ds = MultiProcessMapData(ds, 3, preprocess)
return ds
