def __init__(self, org_id: str, data_set_name: str) -> None:
"""Construct Loader."""
self.org_id = org_id
self.data_set_name = data_set_name
self.api_client = RedBrickApi(cache=False)
print_info("Retrieving dataset ...")
# Dataset info
try:
dataset = self.api_client.get_datapointset(self.org_id, self.data_set_name)[
"dataPointSet"
]
except Exception as err:
print_error(err)
return
print_info("Dataset successfully retrieved!")
self.org_id = dataset["orgId"]
self.data_set_name = dataset["name"]
self.data_type = dataset["dataType"]
self.datapoint_count = dataset["datapointCount"]
self.desc = dataset["desc"]
self.createdAt = dataset["createdAt"]
self.createdBy = dataset["createdBy"]
self.status = dataset["status"]
def __init__(self, org_id: str, label_set_name: str) -> None:
"""Construct Loader."""
self.org_id = org_id
self.label_set_name = label_set_name
self.api_client = RedBrickApi(cache=False)
print_info("Counting available datapoints...")
# All datapoints in labelset
try:
self.dp_ids, custom_group = self.api_client.get_datapoint_ids(
self.org_id, self.label_set_name)
except Exception as err:
print_error(err)
return
self.task_type = custom_group.task_type
self.data_type = custom_group.data_type
self.taxonomy: Dict[str, int] = custom_group.taxonomy
print_info("Number of Datapoints = %s" % len(self.dp_ids))
# Update taxonomy mapper if segmentation
if self.task_type == "SEGMENTATION":
self.taxonomy_update_segmentation()
# Get all users
try:
self.users = self.api_client.get_members(self.org_id)
except Exception as err:
print_error(err)
return
def taxonomy_update_segmentation(self) -> None:
"""
Fix the taxonomy mapper object to be 1-indexed for
segmentation projects.
"""
for key in self.taxonomy.keys():
self.taxonomy[key] += 1
if self.taxonomy[key] == 0:
print_error("Taxonomy class id's must be 0 indexed. \
Please contact [email protected] for help.")
exit(1)
# Add a background class for segmentation
self.taxonomy["background"] = 0
def _create_segmentation_labels(self, labels: List) -> ImageSegmentation:
"""Create Segmentations labels."""
segment_remote_labels: List[ImageSegmentationRemoteLabel] = []
try:
segment_remote_labels = [
ImageSegmentationRemoteLabel.from_dict(label)
for label in labels
]
except Exception as err:
print_error(err)
# Create label object
return ImageSegmentation(remote_labels=segment_remote_labels,
classes=self.taxonomy)
def export(self, format: str = "redbrick") -> str:
"""Export."""
if self.data_type == "IMAGE":
export_img: ExportImage = ExportImage(format=format, labelset=self)
export_img.export()
return export_img.cache_dir
elif self.data_type == "VIDEO":
export_vid: ExportVideo = ExportVideo(format=format, labelset=self)
export_vid.export()
return export_vid.cache_dir
else:
err = ValueError("%s data type not supported! Please reach out to \
[email protected]" % self.data_type)
print_error(err)
return ""
def _create_bbox_labels(self, labels: List) -> ImageBoundingBox:
"""Create Bbox type labels."""
bbox_remote_labels: List[ImageBoundingBoxRemoteLabel] = []
try:
bbox_remote_labels = [
ImageBoundingBoxRemoteLabel.from_dict(label)
for label in labels
]
except Exception as err:
print(err)
print_error(
"Parsing error. Please reach out to [email protected]")
# Create label object
return ImageBoundingBox(labels=bbox_remote_labels)
def _taxonomy_update_segmentation(
self, tax_map: Dict[str, int]) -> Dict[str, int]:
"""
Fix the taxonomy mapper object to be 1-indexed for
segmentation projects.
"""
for key in tax_map.keys():
tax_map[key] += 1
if tax_map[key] == 0:
print_error("Taxonomy class id's must be 0 indexed. \
Please contact [email protected] for help.")
exit(1)
# Add a background class for segmentation
tax_map["background"] = 0
return tax_map
def upload_items_with_labels(
self, items: str, storage_id: str, label_set_name: str, task_type: str
) -> Union[Dict, None]:
"""Upload a list of items with labels to the backend."""
# Getting item list presign
itemsListUploadInfo_ = self.api_client.get_itemListUploadPresign(
org_id=self.org_id, file_name="upload-sdk.json"
)["itemListUploadPresign"]
presignedUrl_ = itemsListUploadInfo_["presignedUrl"]
filePath_ = itemsListUploadInfo_["filePath"]
fileName_ = itemsListUploadInfo_["fileName"]
uploadId_ = itemsListUploadInfo_["uploadId"]
createdAt_ = itemsListUploadInfo_["createdAt"]
# Uploading items to presigned url
print_info("Uploading file '{}'".format(items))
with open(items, "rb") as f:
json_payload = json.load(f)
response = requests.put(presignedUrl_, json=json_payload)
# Call item list upload success
if response.ok:
itemsListUploadSuccessInput_ = {
"orgId": self.org_id,
"filePath": filePath_,
"fileName": fileName_,
"uploadId": uploadId_,
"taskType": task_type,
"dataType": self.data_type,
"storageId": storage_id,
"dpsName": self.data_set_name,
"cstName": label_set_name,
}
uploadSuccessPayload_ = self.api_client.itemListUploadSuccess(
org_id=self.org_id,
itemsListUploadSuccessInput=itemsListUploadSuccessInput_,
)["itemListUploadSuccess"]
importId_ = uploadSuccessPayload_["upload"]["importId"]
print_info(
"Upload is processing, this is your importId: {}".format(importId_)
)
return uploadSuccessPayload_
else:
print_error("Something went wrong uploading your file {}.".format(items))
return None
def export(
self,
download_data: bool = False,
single_json: bool = False,
use_name: bool = False,
export_format: str = "redbrick",
) -> None:
# Create LabelsetIterator
labelsetIter = LabelsetIterator(org_id=self.org_id,
label_set_name=self.label_set_name)
# Validation of optional parameters
task_type = labelsetIter.customGroup["taskType"]
if export_format not in ["redbrick", "png", "coco"]:
print_error(
f'Export format "{export_format}" not valid, please use '
f'"redbrick", "coco" or "png"')
return
if export_format in ["png", "coco"] and task_type not in [
"SEGMENTATION",
"MULTI",
"POLYGON",
]:
print_error(
'Export format "png" and "coco" is only valid for segmentation '
'and polygon tasks. Please use "redbrick"')
return
# Create target_dir if it doesn't exists
if not os.path.exists(self.target_dir):
os.makedirs(self.target_dir)
# Create folder 'data' inside target_dir
if download_data:
target_data_dir = os.path.join(self.target_dir, "data")
if not os.path.exists(target_data_dir):
os.makedirs(target_data_dir)
# Create masks folder
if export_format in ["png", "coco"] and task_type in [
"SEGMENTATION",
"MULTI",
"POLYGON",
]:
target_masks_dir = os.path.join(self.target_dir, "masks")
if not os.path.exists(target_masks_dir):
os.makedirs(target_masks_dir)
# Saving summary.json file
self._save_summary_json(labelsetIter)
if download_data:
print_info("Exporting datapoints and data to dir: {}".format(
self.target_dir))
else:
print_info("Exporting datapoints to dir: {}".format(
self.target_dir))
# If we are exporting image segmentation
if export_format in ["png", "coco"]:
if task_type in ["SEGMENTATION", "POLYGON"]:
self._export_image_segmentation(labelsetIter, download_data,
single_json, use_name,
export_format)
else:
self._export_multi(labelsetIter, download_data, single_json,
use_name, export_format)
elif labelsetIter.customGroup["dataType"] == "VIDEO":
self._export_video(labelsetIter, download_data, single_json,
use_name, export_format)
else:
self._export_image(labelsetIter, download_data, single_json,
use_name, export_format)
