def _import(src_file, instance_data, load_data_callback=None):
with TemporaryDirectory() as tmp_dir:
Archive(src_file.name).extractall(tmp_dir)
image_info = {}
frames = [YoloExtractor.name_from_path(osp.relpath(p, tmp_dir))
for p in glob(osp.join(tmp_dir, '**', '*.txt'), recursive=True)]
root_hint = find_dataset_root(
[DatasetItem(id=frame) for frame in frames], instance_data)
for frame in frames:
frame_info = None
try:
frame_id = match_dm_item(DatasetItem(id=frame), instance_data,
root_hint=root_hint)
frame_info = instance_data.frame_info[frame_id]
except Exception: # nosec
pass
if frame_info is not None:
image_info[frame] = (frame_info['height'], frame_info['width'])
dataset = Dataset.import_from(tmp_dir, 'yolo',
env=dm_env, image_info=image_info)
if load_data_callback is not None:
load_data_callback(dataset, instance_data)
import_dm_annotations(dataset, instance_data)
def _import(src_file, task_data):
with TemporaryDirectory() as tmp_dir:
Archive(src_file.name).extractall(tmp_dir)
image_info = {}
frames = [
YoloExtractor.name_from_path(osp.relpath(p, tmp_dir))
for p in glob(osp.join(tmp_dir, '**', '*.txt'), recursive=True)
]
root_hint = find_dataset_root(
[DatasetItem(id=frame) for frame in frames], task_data)
for frame in frames:
frame_info = None
try:
frame_id = match_dm_item(DatasetItem(id=frame),
task_data,
root_hint=root_hint)
frame_info = task_data.frame_info[frame_id]
except Exception:
pass
if frame_info is not None:
image_info[frame] = (frame_info['height'], frame_info['width'])
dataset = dm_env.make_importer('yolo')(tmp_dir, image_info=image_info) \
.make_dataset()
import_dm_annotations(dataset, task_data)
def _import(src_file, instance_data, load_data_callback=None):
with TemporaryDirectory() as tmp_dir:
Archive(src_file.name).extractall(tmp_dir)
image_meta_path = osp.join(tmp_dir, OpenImagesPath.ANNOTATIONS_DIR,
DEFAULT_IMAGE_META_FILE_NAME)
image_meta = None
if not osp.isfile(image_meta_path):
image_meta = {}
item_ids = list(find_item_ids(tmp_dir))
root_hint = find_dataset_root(
[DatasetItem(id=item_id) for item_id in item_ids],
instance_data)
for item_id in item_ids:
frame_info = None
try:
frame_id = match_dm_item(DatasetItem(id=item_id),
instance_data, root_hint)
frame_info = instance_data.frame_info[frame_id]
except Exception: # nosec
pass
if frame_info is not None:
image_meta[item_id] = (frame_info['height'],
frame_info['width'])
dataset = Dataset.import_from(tmp_dir,
'open_images',
image_meta=image_meta,
env=dm_env)
dataset.transform('masks_to_polygons')
if load_data_callback is not None:
load_data_callback(dataset, instance_data)
import_dm_annotations(dataset, instance_data)
def load(file_object, annotations):
from defusedxml import ElementTree
context = ElementTree.iterparse(file_object, events=("start", "end"))
context = iter(context)
ev, _ = next(context)
supported_shapes = ('box', 'polygon', 'polyline', 'points', 'cuboid')
track = None
shape = None
tag = None
image_is_opened = False
attributes = None
for ev, el in context:
if ev == 'start':
if el.tag == 'track':
track = annotations.Track(
label=el.attrib['label'],
group=int(el.attrib.get('group_id', 0)),
source=el.attrib.get('source', 'manual'),
shapes=[],
)
elif el.tag == 'image':
image_is_opened = True
frame_id = annotations.abs_frame_id(
match_dm_item(DatasetItem(
id=osp.splitext(el.attrib['name'])[0],
attributes={'frame': el.attrib['id']},
image=el.attrib['name']),
task_data=annotations))
elif el.tag in supported_shapes and (track is not None
or image_is_opened):
attributes = []
shape = {
'attributes': attributes,
'points': [],
}
elif el.tag == 'tag' and image_is_opened:
attributes = []
tag = {
'frame': frame_id,
'label': el.attrib['label'],
'group': int(el.attrib.get('group_id', 0)),
'attributes': attributes,
'source': str(el.attrib.get('source', 'manual'))
}
elif ev == 'end':
if el.tag == 'attribute' and attributes is not None:
attributes.append(
annotations.Attribute(
name=el.attrib['name'],
value=el.text or "",
))
if el.tag in supported_shapes:
if track is not None:
shape['frame'] = el.attrib['frame']
shape['outside'] = el.attrib['outside'] == "1"
shape['keyframe'] = el.attrib['keyframe'] == "1"
else:
shape['frame'] = frame_id
shape['label'] = el.attrib['label']
shape['group'] = int(el.attrib.get('group_id', 0))
shape['source'] = str(el.attrib.get('source', 'manual'))
shape['type'] = 'rectangle' if el.tag == 'box' else el.tag
shape['occluded'] = el.attrib['occluded'] == '1'
shape['z_order'] = int(el.attrib.get('z_order', 0))
if el.tag == 'box':
shape['points'].append(el.attrib['xtl'])
shape['points'].append(el.attrib['ytl'])
shape['points'].append(el.attrib['xbr'])
shape['points'].append(el.attrib['ybr'])
elif el.tag == 'cuboid':
shape['points'].append(el.attrib['xtl1'])
shape['points'].append(el.attrib['ytl1'])
shape['points'].append(el.attrib['xbl1'])
shape['points'].append(el.attrib['ybl1'])
shape['points'].append(el.attrib['xtr1'])
shape['points'].append(el.attrib['ytr1'])
shape['points'].append(el.attrib['xbr1'])
shape['points'].append(el.attrib['ybr1'])
shape['points'].append(el.attrib['xtl2'])
shape['points'].append(el.attrib['ytl2'])
shape['points'].append(el.attrib['xbl2'])
shape['points'].append(el.attrib['ybl2'])
shape['points'].append(el.attrib['xtr2'])
shape['points'].append(el.attrib['ytr2'])
shape['points'].append(el.attrib['xbr2'])
shape['points'].append(el.attrib['ybr2'])
else:
for pair in el.attrib['points'].split(';'):
shape['points'].extend(map(float, pair.split(',')))
if track is not None:
if shape["keyframe"]:
track.shapes.append(annotations.TrackedShape(**shape))
else:
annotations.add_shape(annotations.LabeledShape(**shape))
shape = None
elif el.tag == 'track':
annotations.add_track(track)
track = None
elif el.tag == 'image':
image_is_opened = False
elif el.tag == 'tag':
annotations.add_tag(annotations.Tag(**tag))
tag = None
el.clear()
def _import(src_file, task_data):
with TemporaryDirectory() as tmp_dir:
Archive(src_file.name).extractall(tmp_dir)
dataset = Dataset.import_from(tmp_dir, 'mots', env=dm_env)
dataset.transform('masks_to_polygons')
tracks = {}
label_cat = dataset.categories()[AnnotationType.label]
root_hint = find_dataset_root(dataset, task_data)
shift = 0
for item in dataset:
frame_number = task_data.abs_frame_id(
match_dm_item(item, task_data, root_hint=root_hint))
track_ids = set()
for ann in item.annotations:
if ann.type != AnnotationType.polygon:
continue
track_id = ann.attributes['track_id']
group_id = track_id
if track_id in track_ids:
# use negative id for tracks with the same id on the same frame
shift -= 1
track_id = shift
else:
track_ids.add(track_id)
shape = task_data.TrackedShape(
type='polygon',
points=ann.points,
occluded=ann.attributes.get('occluded') == True,
outside=False,
keyframe=True,
z_order=ann.z_order,
frame=frame_number,
attributes=[],
source='manual',
group=group_id)
# build trajectories as lists of shapes in track dict
if track_id not in tracks:
tracks[track_id] = task_data.Track(
label_cat.items[ann.label].name, 0, 'manual', [])
tracks[track_id].shapes.append(shape)
for track in tracks.values():
track.shapes.sort(key=lambda t: t.frame)
# insert outside=True in skips between the frames track is visible
prev_shape_idx = 0
prev_shape = track.shapes[0]
for shape in track.shapes[1:]:
has_skip = task_data.frame_step < shape.frame - prev_shape.frame
if has_skip and not prev_shape.outside:
prev_shape = prev_shape._replace(outside=True,
frame=prev_shape.frame +
task_data.frame_step)
prev_shape_idx += 1
track.shapes.insert(prev_shape_idx, prev_shape)
prev_shape = shape
prev_shape_idx += 1
# Append a shape with outside=True to finish the track
last_shape = track.shapes[-1]
if last_shape.frame + task_data.frame_step <= \
int(task_data.meta['task']['stop_frame']):
track.shapes.append(
last_shape._replace(outside=True,
frame=last_shape.frame +
task_data.frame_step))
task_data.add_track(track)
