def __init__(self, org_id: str, data_set_name: str) -> None:
"""Construct Loader."""
self.org_id = org_id
self.data_set_name = data_set_name
self.api_client = RedBrickApi(cache=False)
print_info("Retrieving dataset ...")
# Dataset info
try:
dataset = self.api_client.get_datapointset(self.org_id, self.data_set_name)[
"dataPointSet"
]
except Exception as err:
print_error(err)
return
print_info("Dataset successfully retrieved!")
self.org_id = dataset["orgId"]
self.data_set_name = dataset["name"]
self.data_type = dataset["dataType"]
self.datapoint_count = dataset["datapointCount"]
self.desc = dataset["desc"]
self.createdAt = dataset["createdAt"]
self.createdBy = dataset["createdBy"]
self.status = dataset["status"]
def __init__(self, org_id: str, label_set_name: str) -> None:
"""Construct Loader."""
self.org_id = org_id
self.label_set_name = label_set_name
self.api_client = RedBrickApi(cache=False)
print_info("Counting available datapoints...")
# All datapoints in labelset
try:
self.dp_ids, custom_group = self.api_client.get_datapoint_ids(
self.org_id, self.label_set_name)
except Exception as err:
print_error(err)
return
self.task_type = custom_group.task_type
self.data_type = custom_group.data_type
self.taxonomy: Dict[str, int] = custom_group.taxonomy
print_info("Number of Datapoints = %s" % len(self.dp_ids))
# Update taxonomy mapper if segmentation
if self.task_type == "SEGMENTATION":
self.taxonomy_update_segmentation()
# Get all users
try:
self.users = self.api_client.get_members(self.org_id)
except Exception as err:
print_error(err)
return
def upload_items_with_labels(
self, items: str, storage_id: str, label_set_name: str, task_type: str
) -> Union[Dict, None]:
"""Upload a list of items with labels to the backend."""
# Getting item list presign
itemsListUploadInfo_ = self.api_client.get_itemListUploadPresign(
org_id=self.org_id, file_name="upload-sdk.json"
)["itemListUploadPresign"]
presignedUrl_ = itemsListUploadInfo_["presignedUrl"]
filePath_ = itemsListUploadInfo_["filePath"]
fileName_ = itemsListUploadInfo_["fileName"]
uploadId_ = itemsListUploadInfo_["uploadId"]
createdAt_ = itemsListUploadInfo_["createdAt"]
# Uploading items to presigned url
print_info("Uploading file '{}'".format(items))
with open(items, "rb") as f:
json_payload = json.load(f)
response = requests.put(presignedUrl_, json=json_payload)
# Call item list upload success
if response.ok:
itemsListUploadSuccessInput_ = {
"orgId": self.org_id,
"filePath": filePath_,
"fileName": fileName_,
"uploadId": uploadId_,
"taskType": task_type,
"dataType": self.data_type,
"storageId": storage_id,
"dpsName": self.data_set_name,
"cstName": label_set_name,
}
uploadSuccessPayload_ = self.api_client.itemListUploadSuccess(
org_id=self.org_id,
itemsListUploadSuccessInput=itemsListUploadSuccessInput_,
)["itemListUploadSuccess"]
importId_ = uploadSuccessPayload_["upload"]["importId"]
print_info(
"Upload is processing, this is your importId: {}".format(importId_)
)
return uploadSuccessPayload_
else:
print_error("Something went wrong uploading your file {}.".format(items))
return None
def create_datapoint(self,
name: str,
items: List[str],
labels: List[Dict] = None) -> Dict:
"""Create a datapoint in the backend"""
datapoint_ = self.api_client.createDatapoint(
org_id=self.org_id,
items=items,
name=name,
data_set_name=self.data_set_name,
storage_id=self.storage_id,
label_set_name=self.label_set_name,
labels=labels,
)
print_info("Datapoint successfully created. Datapoint id: {}".format(
datapoint_["createDatapoint"]["dpId"]))
return datapoint_["createDatapoint"]
def show_data(self) -> None:
"""Visualize the data."""
if self.data_type == "VIDEO":
print_info("Visualizing first 20 frames...")
num_dps = self.number_of_datapoints()
if not num_dps:
return
idx = random.randint(0, num_dps - 1)
self[idx].show_data()
return
# Image data type
print_info("Visualizing data and labels...")
# Prepare figure
num_dps = self.number_of_datapoints()
cols = int(min(2, num_dps))
rows = int(min(2, math.ceil(num_dps / cols)))
fig = plt.figure()
# Generate random index list
list_len = np.min([rows * cols, num_dps])
indexes = random.sample(range(0, list_len), list_len)
# Iterate through axes
for i, idx in enumerate(indexes):
ax = fig.add_subplot(rows, cols, i + 1)
self[idx].show_data(ax=ax) # type: ignore
plt.tight_layout()
plt.show()
def export(
self,
download_data: bool = False,
single_json: bool = False,
use_name: bool = False,
export_format: str = "redbrick",
) -> None:
# Create LabelsetIterator
labelsetIter = LabelsetIterator(org_id=self.org_id,
label_set_name=self.label_set_name)
# Validation of optional parameters
task_type = labelsetIter.customGroup["taskType"]
if export_format not in ["redbrick", "png", "coco"]:
print_error(
f'Export format "{export_format}" not valid, please use '
f'"redbrick", "coco" or "png"')
return
if export_format in ["png", "coco"] and task_type not in [
"SEGMENTATION",
"MULTI",
"POLYGON",
]:
print_error(
'Export format "png" and "coco" is only valid for segmentation '
'and polygon tasks. Please use "redbrick"')
return
# Create target_dir if it doesn't exists
if not os.path.exists(self.target_dir):
os.makedirs(self.target_dir)
# Create folder 'data' inside target_dir
if download_data:
target_data_dir = os.path.join(self.target_dir, "data")
if not os.path.exists(target_data_dir):
os.makedirs(target_data_dir)
# Create masks folder
if export_format in ["png", "coco"] and task_type in [
"SEGMENTATION",
"MULTI",
"POLYGON",
]:
target_masks_dir = os.path.join(self.target_dir, "masks")
if not os.path.exists(target_masks_dir):
os.makedirs(target_masks_dir)
# Saving summary.json file
self._save_summary_json(labelsetIter)
if download_data:
print_info("Exporting datapoints and data to dir: {}".format(
self.target_dir))
else:
print_info("Exporting datapoints to dir: {}".format(
self.target_dir))
# If we are exporting image segmentation
if export_format in ["png", "coco"]:
if task_type in ["SEGMENTATION", "POLYGON"]:
self._export_image_segmentation(labelsetIter, download_data,
single_json, use_name,
export_format)
else:
self._export_multi(labelsetIter, download_data, single_json,
use_name, export_format)
elif labelsetIter.customGroup["dataType"] == "VIDEO":
self._export_video(labelsetIter, download_data, single_json,
use_name, export_format)
else:
self._export_image(labelsetIter, download_data, single_json,
use_name, export_format)
def export(self) -> None:
"""Export the images and labels."""
print_info("Cacheing labels, and data...")
self.cache()
print_info("Exported complete in %s" % self.cache_dir)