def get(self, export_id):
""" Returns exports """
export = ExportModel.objects(id=export_id).first()
if export is None:
return {"message": "Invalid export ID"}, 400
dataset = current_user.datasets.filter(id=export.dataset_id).first()
if dataset is None:
return {"message": "Invalid dataset ID"}, 400
time_delta = datetime.datetime.utcnow() - export.created_at
d = fix_ids(export)
d['ago'] = query_util.td_format(time_delta)
return d
def export_annotations(task_id,
dataset_id,
categories,
with_empty_images=False):
task = TaskModel.objects.get(id=task_id)
dataset = DatasetModel.objects.get(id=dataset_id)
task.update(status="PROGRESS")
socket = create_socket()
task.info("Beginning Export (COCO Format)")
db_categories = CategoryModel.objects(id__in=categories, deleted=False) \
.only(*CategoryModel.COCO_PROPERTIES)
db_images = ImageModel.objects(
deleted=False, dataset_id=dataset.id).only(*ImageModel.COCO_PROPERTIES)
db_annotations = AnnotationModel.objects(deleted=False,
category_id__in=categories)
total_items = db_categories.count()
coco = {'images': [], 'categories': [], 'annotations': []}
total_items += db_images.count()
progress = 0
# iterate though all categoires and upsert
category_names = []
for category in fix_ids(db_categories):
if len(category.get('keypoint_labels', [])) > 0:
category['keypoints'] = category.pop('keypoint_labels', [])
category['skeleton'] = category.pop('keypoint_edges', [])
else:
if 'keypoint_edges' in category:
del category['keypoint_edges']
if 'keypoint_labels' in category:
del category['keypoint_labels']
task.info(f"Adding category: {category.get('name')}")
coco.get('categories').append(category)
category_names.append(category.get('name'))
progress += 1
task.set_progress((progress / total_items) * 100, socket=socket)
total_annotations = db_annotations.count()
total_images = db_images.count()
for image in db_images:
image = fix_ids(image)
progress += 1
task.set_progress((progress / total_items) * 100, socket=socket)
annotations = db_annotations.filter(image_id=image.get('id'))\
.only(*AnnotationModel.COCO_PROPERTIES)
annotations = fix_ids(annotations)
if len(annotations) == 0:
if with_empty_images:
coco.get('images').append(image)
continue
num_annotations = 0
for annotation in annotations:
has_keypoints = len(annotation.get('keypoints', [])) > 0
has_segmentation = len(annotation.get('segmentation', [])) > 0
if has_keypoints or has_segmentation:
if not has_keypoints:
if 'keypoints' in annotation:
del annotation['keypoints']
else:
arr = np.array(annotation.get('keypoints', []))
arr = arr[2::3]
annotation['num_keypoints'] = len(arr[arr > 0])
num_annotations += 1
coco.get('annotations').append(annotation)
task.info(
f"Exporting {num_annotations} annotations for image {image.get('id')}"
)
coco.get('images').append(image)
task.info(
f"Done export {total_annotations} annotations and {total_images} images from {dataset.name}"
)
timestamp = time.time()
directory = f"{dataset.directory}.exports/"
file_path = f"{directory}coco-{timestamp}.json"
if not os.path.exists(directory):
os.makedirs(directory)
task.info(f"Writing export to file {file_path}")
with open(file_path, 'w') as fp:
json.dump(coco, fp)
task.info("Creating export object")
export = ExportModel(dataset_id=dataset.id,
path=file_path,
tags=["COCO", *category_names])
export.save()
task.set_progress(100, socket=socket)
def get_dataset_coco(dataset):
"""
Generates coco for all images in dataset
:param dataset: DatasetModel
:return: Coco in dictionary format
"""
categories = CategoryModel.objects(deleted=False) \
.exclude('deleted_date').in_bulk(dataset.categories).items()
dataset = fix_ids(dataset)
images = ImageModel.objects(deleted=False, dataset_id=dataset.get('id')).exclude('deleted_date')
all_annotations = AnnotationModel.objects(deleted=False).exclude('deleted_date', 'paper_object')
coco = {
'images': [],
'categories': [],
'annotations': []
}
for category in categories:
category = fix_ids(category[1])
del category['deleted']
if len(category.get('keypoint_labels', [])) > 0:
category['keypoints'] = category.pop('keypoint_labels')
category['skeleton'] = category.pop('keypoint_edges')
else:
del category['keypoint_edges']
del category['keypoint_labels']
coco.get('categories').append(category)
for image in images:
annotations = all_annotations.filter(image_id=image.id)
if annotations.count() == 0:
continue
annotations = fix_ids(annotations.all())
for annotation in annotations:
has_keypoints = len(annotation.get('keypoints', [])) > 0
has_segmentation = len(annotation.get('segmentation', [])) > 0
if has_keypoints or has_segmentation:
del annotation['deleted']
if not has_keypoints:
del annotation['keypoints']
else:
arr = np.array(annotation.get('keypoints', []))
arr = arr[2::3]
annotation['num_keypoints'] = len(arr[arr > 0])
coco.get('annotations').append(annotation)
image = fix_ids(image)
del image['deleted']
coco.get('images').append(image)
return coco
def get_image_coco(image_id):
"""
Generates coco for an image
:param image: ImageModel
:return: Coco in dictionary format
"""
image = ImageModel.objects(id=image_id)\
.only(*ImageModel.COCO_PROPERTIES)
image = fix_ids(image)[0]
dataset = DatasetModel.objects(id=image.get('dataset_id')).first()
bulk_categories = CategoryModel.objects(id__in=dataset.categories, deleted=False) \
.only(*CategoryModel.COCO_PROPERTIES)
print(bulk_categories)
db_annotations = AnnotationModel.objects(deleted=False, image_id=image_id)
categories = []
annotations = []
for category in fix_ids(bulk_categories):
category_annotations = db_annotations\
.filter(category_id=category.get('id'))\
.only(*AnnotationModel.COCO_PROPERTIES)
if category_annotations.count() == 0:
continue
category_annotations = fix_ids(category_annotations)
for annotation in category_annotations:
has_segmentation = len(annotation.get('segmentation', [])) > 0
has_keypoints = len(annotation.get('keypoints', [])) > 0
if has_segmentation or has_keypoints:
if has_keypoints:
arr = np.array(annotation.get('keypoints', []))
arr = arr[2::3]
annotation['num_keypoints'] = len(arr[arr > 0])
annotations.append(annotation)
if len(category.get('keypoint_labels')) > 0:
category['keypoints'] = category.pop('keypoint_labels')
category['skeleton'] = category.pop('keypoint_edges')
else:
del category['keypoint_edges']
del category['keypoint_labels']
categories.append(category)
coco = {
"images": [image],
"categories": categories,
"annotations": annotations
}
return coco
def collect_coco_annotations(task, categories, dataset, socket):
"""
Getting all coco labels from current dataset and creating a dict from it
:return: COCO labels from current dataset as dict with fields: "images", "categories",
"annotations".
"""
# Getting coco annotations
task.info("===== Getting COCO annotations =====")
db_categories = CategoryModel.objects(id__in=categories, deleted=False) \
.only(*CategoryModel.COCO_PROPERTIES)
db_images = ImageModel.objects(deleted=False, dataset_id=dataset.id) \
.only(*ImageModel.COCO_PROPERTIES)
db_annotations = AnnotationModel.objects(deleted=False, category_id__in=categories)
total_items = db_categories.count()
coco = {
'images': [],
'categories': [],
'annotations': []
}
total_items += db_images.count()
progress = 0
# iterate though all categoires and upsert
category_names = []
for category in fix_ids(db_categories):
if len(category.get('keypoint_labels', [])) > 0:
category['keypoints'] = category.pop('keypoint_labels', [])
category['skeleton'] = category.pop('keypoint_edges', [])
else:
if 'keypoint_edges' in category:
del category['keypoint_edges']
if 'keypoint_labels' in category:
del category['keypoint_labels']
task.info(f"Adding category: {category.get('name')}")
coco.get('categories').append(category)
category_names.append(category.get('name'))
progress += 1
task.set_progress((progress / total_items) * 50, socket=socket)
total_annotations = db_annotations.count()
total_images = db_images.count()
for img_counter, image in enumerate(fix_ids(db_images)):
progress += 1
task.set_progress((progress / total_items) * 50, socket=socket)
annotations = db_annotations.filter(image_id=image.get('id')) \
.only(*AnnotationModel.COCO_PROPERTIES)
annotations = fix_ids(annotations)
num_annotations = 0
for annotation in annotations:
has_keypoints = len(annotation.get('keypoints', [])) > 0
has_segmentation = len(annotation.get('segmentation', [])) > 0
if has_keypoints or has_segmentation:
if not has_keypoints:
if 'keypoints' in annotation:
del annotation['keypoints']
else:
arr = np.array(annotation.get('keypoints', []))
arr = arr[2::3]
annotation['num_keypoints'] = len(arr[arr > 0])
num_annotations += 1
coco.get('annotations').append(annotation)
task.info(f"Exporting {num_annotations} annotations for image {image.get('id')} ({img_counter+1}/{total_images})")
coco.get('images').append(image)
task.info(f"Done export {total_annotations} annotations and {total_images} images from {dataset.name}")
return coco, category_names