def test_get_labels_with_right_params(self, mock_response):
id = uuid.uuid4()
project_id = uuid.uuid4()
response = mock_response.return_value
response.status_code = 200
response.json.return_value = {
'data': {
'labels': {
'count': 100,
'edges': [{
'id': id,
'projectId': project_id
}]
}
}
}
manager = LabelManager()
count, labels = manager.get_labels(project_id, page=0, page_size=10)
self.assertEqual(count, 100)
self.assertEqual(len(labels), 1)
label = labels[0]
self.assertEqual(label.id, id)
self.assertEqual(label.project_id, project_id)
def test_get_0_labels_with_right_params(self, mock_response):
response = mock_response.return_value
response.status_code = 200
response.json.return_value = {
'data': {
'labels': {
'count': 0,
'edges': []
}
}
}
manager = LabelManager()
count, labels = manager.get_labels(project_id=uuid.uuid4(),
page=0,
page_size=10)
self.assertEqual(count, 0)
self.assertEqual(len(labels), 0)
def get_data_page(self,
page_idx,
page_size=10,
num_data=None,
tags=[],
dataset=None,
data_key=None,
**kwargs):
if num_data is None:
num_data = self.get_num_data(tags=tags, **kwargs)
num_pages = math.ceil(float(num_data) / page_size)
if page_idx >= num_pages:
print('[WARNING] Index out of bounds. Empty list returned')
return []
workapp = self._project.workapp
if workapp == 'video-siesta':
# Video
command = spb.Command(type='describe_videolabel')
tags = [{'name': tag} for tag in tags]
option = {'project_id': self._project.id, 'tags': tags, **kwargs}
data_page, _ = spb.run(command=command,
option=option,
page=page_idx + 1,
page_size=page_size)
for data in data_page:
yield VideoDataHandle(data, self._project)
else:
manager = LabelManager()
tags = [{'name': tag} for tag in tags]
option = {
'project_id': self._project.id,
'tags': tags,
'page': page_idx + 1,
'page_size': page_size,
'dataset': dataset,
'data_key': data_key,
**kwargs
}
count, data_page = manager.get_labels(**option)
for data in data_page:
yield DataHandle(data, self._project)
class LabelData():
def __init__(self):
self.label_manager = LabelManager()
def upload_data(self, project, dataset_name, directory_path, include_label,
is_forced):
imgs_path = recursive_glob_image_files(directory_path)
if not is_forced:
if not click.confirm(
f"Uploading {len(imgs_path)} data and {len(recursive_glob_label_files(directory_path)) if include_label else 0 } labels to dataset '{dataset_name}' under project '{project.name}'. Proceed?"
):
return
asset_images = []
manager = Manager()
if len(imgs_path) != 0:
for key in imgs_path:
file_name = key
asset_image = {
'file': imgs_path[key],
'file_name': file_name,
'data_key': key,
'dataset': dataset_name
}
asset_images.append(asset_image)
data_results = manager.list([manager.dict()] * len(asset_images))
console.print(f"Uploading data:")
with Pool(NUM_MULTI_PROCESS) as p:
list(
tqdm.tqdm(p.imap(
_upload_asset,
zip([project.id] * len(asset_images), asset_images,
data_results)),
total=len(asset_images)))
else:
data_results = [{}]
label_results = None
if include_label:
labels_path = recursive_glob_label_files(directory_path)
console.print(f"Uploading labels:")
if len(labels_path) != 0:
label_results = manager.list([manager.dict()] *
len(labels_path))
with Pool(NUM_MULTI_PROCESS) as p:
list(
tqdm.tqdm(p.imap(
_update_label,
zip([self.label_manager] * len(labels_path),
labels_path, [project.id] * len(labels_path),
[project.label_interface] * len(labels_path),
[dataset_name] * len(labels_path),
label_results)),
total=len(labels_path)))
else:
label_results = [{}]
console.print('\n[b blue]** Result Summary **[/b blue]')
success_data_count = len(asset_images) - len(data_results[0])
data_success_ratio = round(
success_data_count / len(asset_images) *
100, 2) if len(data_results[0]) != 0 else 100
console.print(
f'Successful upload of {success_data_count} out of {len(asset_images)} data. ({data_success_ratio}%) - [b red]{len(data_results[0])} ERRORS[/b red]'
)
if include_label:
success_label_count = len(labels_path) - len(label_results[0])
label_success_ratio = round(
success_label_count / len(labels_path) *
100, 2) if len(label_results[0]) != 0 else 100
console.print(
f'Successful upload of {success_label_count} out of {len(labels_path)} labels. ({label_success_ratio}%) - [b red]{len(label_results[0])} ERRORS[/b red]'
)
self._print_error_table(data_error_results=dict(data_results[0]),
label_error_results=dict(label_results[0]))
else:
self._print_error_table(data_error_results=dict(data_results[0]))
def upload_label(self, project, dataset_name, directory_path, is_forced):
labels_path = recursive_glob_label_files(directory_path)
if not is_forced:
if not click.confirm(
f"Uploading {len(labels_path)} labels to project '{project.name}'. Proceed?"
):
return
if len(labels_path) != 0:
manager = Manager()
label_results = manager.list([manager.dict()] * len(labels_path))
with Pool(NUM_MULTI_PROCESS) as p:
list(
tqdm.tqdm(p.imap(
_update_label,
zip([self.label_manager] * len(labels_path),
labels_path, [project.id] * len(labels_path),
[project.label_interface] * len(labels_path),
[dataset_name] * len(labels_path), label_results)),
total=len(labels_path)))
else:
label_results = [{}]
console.print('\n[b blue]** Result Summary **[/b blue]')
success_label_count = len(labels_path) - len(label_results[0])
success_label_ratio = round(
success_label_count / len(labels_path) *
100, 2) if len(labels_path) != 0 else 100
console.print(
f'Successful upload of {success_label_count} out of {len(labels_path)} labels. ({success_label_ratio}%) - [b red]{len(label_results[0])} ERRORS[/b red]'
)
self._print_error_table(label_error_results=dict(label_results[0]))
def download(self, project, directory_path, is_forced):
process_results = []
success_data_count = 0
success_labels_count = 0
data_error_results = {}
label_error_results = {}
# get data count
main_label_count, labels = self.label_manager.get_labels(
project.id, label_type='DEFAULT', page=1, page_size=1)
# get labels count
total_label_count, labels = self.label_manager.get_labels(project.id,
page=1,
page_size=1)
#Download project configuration
try:
project_config_path = os.path.join(directory_path, 'project.json')
with open(project_config_path, 'w') as input:
json.dump(project.label_interface, input, indent=4)
is_download_project_config = True
except Exception as e:
is_download_project_config = False
if main_label_count != 0:
page_length = int(
main_label_count / LABEL_DESCRIBE_PAGE_SIZE
) if main_label_count % LABEL_DESCRIBE_PAGE_SIZE == 0 else int(
main_label_count / LABEL_DESCRIBE_PAGE_SIZE) + 1
if not is_forced:
if not click.confirm(
f"Downloading {main_label_count} data and {total_label_count} labels from project '{project.name}' to '{directory_path}'. Proceed?"
):
return
manager = Manager()
label_results_dict = manager.dict()
label_results = manager.list([label_results_dict] * page_length)
process_results = manager.list(range(page_length))
with Pool(NUM_MULTI_PROCESS) as p:
list(
tqdm.tqdm(p.imap(
_download_worker,
zip([self.label_manager] * page_length,
[project.id] * page_length, range(page_length),
[directory_path] * page_length, label_results,
[process_results] * page_length)),
total=page_length))
for key in label_results_dict.keys():
results_error = label_results_dict[key]['error']
success_data_count += 1
success_labels_count += label_results_dict[key][
'success_labels_count']
if 'data' in results_error:
data_error_results[key] = results_error['data']
if 'label' in results_error:
label_error_results[key] = results_error['label']
console.print('\n[b blue]** Result Summary **[/b blue]')
console.print(
f'Download of project configuration - {"[b blue]Success[/b blue]" if is_download_project_config else "[b red]Fail[/b red]"}'
)
console.print(
f'Successful download of {success_labels_count} out of {total_label_count} labels. ({round(success_labels_count/total_label_count*100,2)}%) - [b red]{total_label_count - success_labels_count} ERRORS[/b red]'
)
console.print(
f'Successful download of {success_data_count} out of {main_label_count} data. ({round(success_data_count/main_label_count*100,2)}%) - [b red]{main_label_count - success_data_count} ERRORS[/b red]'
)
# TODO: Need to refactor Get Labels API Fail Logic
for process_result in process_results:
is_process_fail = False
if not process_result:
is_process_fail = True
if is_process_fail:
console.print(
f'[b red]Failed Download Labels from API. Please retry download.[/b red]'
)
self._print_error_table(
data_error_results=data_error_results,
label_error_results=label_error_results,
)
def _print_error_table(self,
data_error_results=None,
label_error_results=None):
results = {}
if isinstance(data_error_results, dict):
for key in data_error_results:
results[key] = {}
results[key]['data'] = data_error_results[key]
results[key]['label'] = None
if isinstance(label_error_results, dict):
for key in label_error_results:
if key in results:
results[key]['label'] = label_error_results[key]
else:
results[key] = {
'label': label_error_results[key],
'data': None
}
if not next(iter(results), None):
return
console.print('\n[b red]** Error Table **[/b red]')
page = 1
page_length = math.ceil(len(results) / 10)
while True:
table = rich.table.Table(show_header=True,
header_style="bold magenta")
table.add_column("FILE NAME")
if isinstance(data_error_results, dict):
table.add_column("DATA UPLOAD")
if isinstance(label_error_results, dict):
table.add_column("LABEL UPLOAD")
for _ in range(10):
key = next(iter(results), None)
if not key:
break
if isinstance(data_error_results, dict) and isinstance(
label_error_results, dict):
data = results[key]['data']
label = results[key]['label']
table.add_row(key, f"{data if data else '-'}",
f"{label if label else '-'}")
elif isinstance(data_error_results, dict):
data = results[key]['data']
table.add_row(key, f"{data if data else '-'}")
else:
label = results[key]['label']
table.add_row(key, f"{label if label else '-'}")
del results[key]
console.print(table)
if not next(iter(results), None):
break
else:
click.echo(
f'Press any button to continue to the next page ({page}/{page_length}). Otherwise press ‘Q’ to quit.',
nl=False)
key = click.getchar()
click.echo()
if key == 'q' or key == 'Q':
break
console.log(f'[b]Check the log file for more details[/b]')
console.log(f'- {simple_logger.handlers[0].baseFilename}')
console.log(f'- {logger.handlers[0].baseFilename}')