def get(self, export_id): """ Returns exports """ export = ExportModel.objects(id=export_id).first() if export is None: return {"message": "Invalid export ID"}, 400 dataset = current_user.datasets.filter(id=export.dataset_id).first() if dataset is None: return {"message": "Invalid dataset ID"}, 400 time_delta = datetime.datetime.utcnow() - export.created_at d = fix_ids(export) d['ago'] = query_util.td_format(time_delta) return d
def export_annotations(task_id, dataset_id, categories, with_empty_images=False): task = TaskModel.objects.get(id=task_id) dataset = DatasetModel.objects.get(id=dataset_id) task.update(status="PROGRESS") socket = create_socket() task.info("Beginning Export (COCO Format)") db_categories = CategoryModel.objects(id__in=categories, deleted=False) \ .only(*CategoryModel.COCO_PROPERTIES) db_images = ImageModel.objects( deleted=False, dataset_id=dataset.id).only(*ImageModel.COCO_PROPERTIES) db_annotations = AnnotationModel.objects(deleted=False, category_id__in=categories) total_items = db_categories.count() coco = {'images': [], 'categories': [], 'annotations': []} total_items += db_images.count() progress = 0 # iterate though all categoires and upsert category_names = [] for category in fix_ids(db_categories): if len(category.get('keypoint_labels', [])) > 0: category['keypoints'] = category.pop('keypoint_labels', []) category['skeleton'] = category.pop('keypoint_edges', []) else: if 'keypoint_edges' in category: del category['keypoint_edges'] if 'keypoint_labels' in category: del category['keypoint_labels'] task.info(f"Adding category: {category.get('name')}") coco.get('categories').append(category) category_names.append(category.get('name')) progress += 1 task.set_progress((progress / total_items) * 100, socket=socket) total_annotations = db_annotations.count() total_images = db_images.count() for image in db_images: image = fix_ids(image) progress += 1 task.set_progress((progress / total_items) * 100, socket=socket) annotations = db_annotations.filter(image_id=image.get('id'))\ .only(*AnnotationModel.COCO_PROPERTIES) annotations = fix_ids(annotations) if len(annotations) == 0: if with_empty_images: coco.get('images').append(image) continue num_annotations = 0 for annotation in annotations: has_keypoints = len(annotation.get('keypoints', [])) > 0 has_segmentation = len(annotation.get('segmentation', [])) > 0 if has_keypoints or has_segmentation: if not has_keypoints: if 'keypoints' in annotation: del annotation['keypoints'] else: arr = np.array(annotation.get('keypoints', [])) arr = arr[2::3] annotation['num_keypoints'] = len(arr[arr > 0]) num_annotations += 1 coco.get('annotations').append(annotation) task.info( f"Exporting {num_annotations} annotations for image {image.get('id')}" ) coco.get('images').append(image) task.info( f"Done export {total_annotations} annotations and {total_images} images from {dataset.name}" ) timestamp = time.time() directory = f"{dataset.directory}.exports/" file_path = f"{directory}coco-{timestamp}.json" if not os.path.exists(directory): os.makedirs(directory) task.info(f"Writing export to file {file_path}") with open(file_path, 'w') as fp: json.dump(coco, fp) task.info("Creating export object") export = ExportModel(dataset_id=dataset.id, path=file_path, tags=["COCO", *category_names]) export.save() task.set_progress(100, socket=socket)
def get_dataset_coco(dataset): """ Generates coco for all images in dataset :param dataset: DatasetModel :return: Coco in dictionary format """ categories = CategoryModel.objects(deleted=False) \ .exclude('deleted_date').in_bulk(dataset.categories).items() dataset = fix_ids(dataset) images = ImageModel.objects(deleted=False, dataset_id=dataset.get('id')).exclude('deleted_date') all_annotations = AnnotationModel.objects(deleted=False).exclude('deleted_date', 'paper_object') coco = { 'images': [], 'categories': [], 'annotations': [] } for category in categories: category = fix_ids(category[1]) del category['deleted'] if len(category.get('keypoint_labels', [])) > 0: category['keypoints'] = category.pop('keypoint_labels') category['skeleton'] = category.pop('keypoint_edges') else: del category['keypoint_edges'] del category['keypoint_labels'] coco.get('categories').append(category) for image in images: annotations = all_annotations.filter(image_id=image.id) if annotations.count() == 0: continue annotations = fix_ids(annotations.all()) for annotation in annotations: has_keypoints = len(annotation.get('keypoints', [])) > 0 has_segmentation = len(annotation.get('segmentation', [])) > 0 if has_keypoints or has_segmentation: del annotation['deleted'] if not has_keypoints: del annotation['keypoints'] else: arr = np.array(annotation.get('keypoints', [])) arr = arr[2::3] annotation['num_keypoints'] = len(arr[arr > 0]) coco.get('annotations').append(annotation) image = fix_ids(image) del image['deleted'] coco.get('images').append(image) return coco
def get_image_coco(image_id): """ Generates coco for an image :param image: ImageModel :return: Coco in dictionary format """ image = ImageModel.objects(id=image_id)\ .only(*ImageModel.COCO_PROPERTIES) image = fix_ids(image)[0] dataset = DatasetModel.objects(id=image.get('dataset_id')).first() bulk_categories = CategoryModel.objects(id__in=dataset.categories, deleted=False) \ .only(*CategoryModel.COCO_PROPERTIES) print(bulk_categories) db_annotations = AnnotationModel.objects(deleted=False, image_id=image_id) categories = [] annotations = [] for category in fix_ids(bulk_categories): category_annotations = db_annotations\ .filter(category_id=category.get('id'))\ .only(*AnnotationModel.COCO_PROPERTIES) if category_annotations.count() == 0: continue category_annotations = fix_ids(category_annotations) for annotation in category_annotations: has_segmentation = len(annotation.get('segmentation', [])) > 0 has_keypoints = len(annotation.get('keypoints', [])) > 0 if has_segmentation or has_keypoints: if has_keypoints: arr = np.array(annotation.get('keypoints', [])) arr = arr[2::3] annotation['num_keypoints'] = len(arr[arr > 0]) annotations.append(annotation) if len(category.get('keypoint_labels')) > 0: category['keypoints'] = category.pop('keypoint_labels') category['skeleton'] = category.pop('keypoint_edges') else: del category['keypoint_edges'] del category['keypoint_labels'] categories.append(category) coco = { "images": [image], "categories": categories, "annotations": annotations } return coco
def collect_coco_annotations(task, categories, dataset, socket): """ Getting all coco labels from current dataset and creating a dict from it :return: COCO labels from current dataset as dict with fields: "images", "categories", "annotations". """ # Getting coco annotations task.info("===== Getting COCO annotations =====") db_categories = CategoryModel.objects(id__in=categories, deleted=False) \ .only(*CategoryModel.COCO_PROPERTIES) db_images = ImageModel.objects(deleted=False, dataset_id=dataset.id) \ .only(*ImageModel.COCO_PROPERTIES) db_annotations = AnnotationModel.objects(deleted=False, category_id__in=categories) total_items = db_categories.count() coco = { 'images': [], 'categories': [], 'annotations': [] } total_items += db_images.count() progress = 0 # iterate though all categoires and upsert category_names = [] for category in fix_ids(db_categories): if len(category.get('keypoint_labels', [])) > 0: category['keypoints'] = category.pop('keypoint_labels', []) category['skeleton'] = category.pop('keypoint_edges', []) else: if 'keypoint_edges' in category: del category['keypoint_edges'] if 'keypoint_labels' in category: del category['keypoint_labels'] task.info(f"Adding category: {category.get('name')}") coco.get('categories').append(category) category_names.append(category.get('name')) progress += 1 task.set_progress((progress / total_items) * 50, socket=socket) total_annotations = db_annotations.count() total_images = db_images.count() for img_counter, image in enumerate(fix_ids(db_images)): progress += 1 task.set_progress((progress / total_items) * 50, socket=socket) annotations = db_annotations.filter(image_id=image.get('id')) \ .only(*AnnotationModel.COCO_PROPERTIES) annotations = fix_ids(annotations) num_annotations = 0 for annotation in annotations: has_keypoints = len(annotation.get('keypoints', [])) > 0 has_segmentation = len(annotation.get('segmentation', [])) > 0 if has_keypoints or has_segmentation: if not has_keypoints: if 'keypoints' in annotation: del annotation['keypoints'] else: arr = np.array(annotation.get('keypoints', [])) arr = arr[2::3] annotation['num_keypoints'] = len(arr[arr > 0]) num_annotations += 1 coco.get('annotations').append(annotation) task.info(f"Exporting {num_annotations} annotations for image {image.get('id')} ({img_counter+1}/{total_images})") coco.get('images').append(image) task.info(f"Done export {total_annotations} annotations and {total_images} images from {dataset.name}") return coco, category_names