def _update_label(args):
[label_path, project_id, dataset, result] = args
data_key = ".".join(label_path.split(".")[:-1])
if not os.path.isfile(label_path):
_set_error_result(data_key, result, 'Label json file is not existed.')
return
option = {
'project_id': project_id,
'dataset': dataset,
'data_key': data_key
}
try:
command = spb.Command(type='describe_label')
described_labels, _ = spb.run(command=command,
option=option,
page_size=1,
page=1)
described_label = described_labels[
0] if described_labels and described_labels[0] else None
if described_label is None:
_set_error_result(data_key, result, 'Label cannot be described.')
return
if described_label.data_key != option[
'data_key'] and described_label.dataset != option['dataset']:
_set_error_result(data_key, result,
'Described label does not match to upload.')
return
except Exception as e:
_set_error_result(data_key, result, str(e), e)
return
label = {
"id": described_label.id,
"project_id": project_id,
"tags": [tag.get_datas(tag) for tag in described_label.tags],
"result": described_label.result,
}
try:
with open(label_path) as json_file:
json_data = json.load(json_file)
if json_data['result'] is None:
return
if 'result' in json_data:
label['result'] = json_data['result']
if 'tags' in json_data:
label['tags'] = json_data['tags']
command = spb.Command(type='update_label')
label = spb.run(command=command, option=label)
with open(label_path, 'w') as f:
f.write(label.toJson())
except Exception as e:
_set_error_result(data_key, result, str(e), e)
def _update_label(args):
[label_path, project_id, label_interface, dataset, result] = args
data_key = ".".join(label_path.split(".")[:-1])
if not os.path.isfile(label_path):
_set_error_result(data_key, result, 'Label json file is not existed.')
return
option = {
'project_id': project_id,
'dataset': dataset,
'data_key': data_key
}
try:
command = spb.Command(type='describe_videolabel')
described_labels, _ = spb.run(command=command, option=option, page_size=1, page=1)
described_label = described_labels[0] if described_labels and described_labels[0] else None
if described_label is None:
_set_error_result(data_key, result, 'Label cannot be described.')
return
if described_label.data_key != option['data_key'] and described_label.dataset != option['dataset']:
_set_error_result(data_key, result, 'Described label does not match to upload.')
return
except Exception as e:
_set_error_result(data_key, result, str(e), e)
return
label = {
"id": described_label.id,
"project_id": project_id,
"tags": [tag.get_datas(tag) for tag in described_label.tags]
}
try:
with open(label_path) as json_file:
json_data = json.load(json_file)
# TODO: NEED info.json VALIDATION
if json_data['result'] is None:
return
label_info = build_label_info(label_interface=label_interface, result=json_data['result'])
info_json = label_info.build_info()
write_response = requests.put(described_label.info_write_presigned_url ,data=json.dumps(info_json))
if 'tags' in json_data:
label['tags'] = json_data['tags']
command = spb.Command(type='update_videolabel')
label = spb.run(command=command, option=label, optional={'info': json.dumps(info_json)})
with open(label_path, 'w') as f:
f.write(label.toJson())
except Exception as e:
_set_error_result(data_key, result, str(e), e)
def upload_image(self, path, dataset_name, key=None, name=None):
if not os.path.isfile(path):
print('[WARNING] Invalid path. Upload failed')
return
if name is None:
name = path.split('/')[-1]
if key is None:
key = name
command = spb.Command(type='create_data')
option = {
'file': path,
'file_name': name,
'dataset': dataset_name,
'data_key': key
}
try:
return spb.run(command=command,
optional={'projectId': self._project.id},
option=option)
except Exception as e:
print('[WARNING] Duplicate data key. Upload failed')
def _upload_to_suite(self, info=None):
command = spb.Command(type='update_videolabel')
if info is None:
_ = spb.run(command=command, option=self._data)
else:
_ = spb.run(command=command,
option=self._data,
optional={'info': json.dumps(info)})
def get_num_data(self, tags=[], **kwargs):
command = spb.Command(type='describe_label')
tags = [{'name': tag} for tag in tags]
option = {'project_id': self._project.id, 'tags': tags, **kwargs}
_, num_data = spb.run(command=command, option=option, page=1, page_size=1)
if num_data == 0:
print('[WARNING] Data list is empty')
return num_data
def _get_project(cls, project_name):
command = spb.Command(type='describe_project')
projects, num_of_projects = spb.run(command=command, option={'name': project_name}, page=1, page_size=1)
if num_of_projects == 0:
print('[WARNING] Project {} not found'.format(project_name))
return None
return projects[0]
def _upload_asset(args):
logging.debug(f'Uploading Asset: {args}')
[project_id, asset_image, result] = args
try:
command = spb.Command(type='create_data')
spb.run(command=command,
option=asset_image,
optional={'projectId': project_id})
except Exception as e:
_set_error_result(asset_image['data_key'], result, str(e), e)
pass
def download(self, project, directory_path, is_forced):
command = spb.Command(type='describe_label')
_, label_count = spb.run(command=command,
option={'project_id': project.id},
page_size=1,
page=1)
if label_count != 0:
page_length = int(
label_count / LABEL_DESCRIBE_PAGE_SIZE
) if label_count % LABEL_DESCRIBE_PAGE_SIZE == 0 else int(
label_count / LABEL_DESCRIBE_PAGE_SIZE) + 1
if not is_forced:
if not click.confirm(
f"Downloading {label_count} data and {label_count} labels from project '{project.name}' to '{directory_path}'. Proceed?"
):
return
manager = Manager()
results = manager.list([manager.dict()] * page_length)
with Pool(NUM_MULTI_PROCESS) as p:
list(
tqdm.tqdm(p.imap(
_download_worker,
zip([project.id] * page_length, range(page_length),
[directory_path] * page_length, results)),
total=page_length))
results = results[0]
data_results = {}
label_results = {}
if len(results) > 0:
for key in results.keys():
if 'data' in results[key]:
data_results[key] = results[key]['data']
if 'label' in results[key]:
label_results[key] = results[key]['label']
else:
label_results = {}
data_results = {}
console.print('\n[b blue]** Result Summary **[/b blue]')
label_success_count = label_count - len(label_results)
console.print(
f'Successful download of {label_success_count} out of {label_count} labels. ({round(label_success_count/label_count*100,2)}%) - [b red]{len(label_results)} ERRORS[/b red]'
)
data_success_count = label_count - len(data_results)
console.print(
f'Successful download of {data_success_count} out of {label_count} data. ({round(data_success_count/label_count*100,2)}%) - [b red]{len(data_results)} ERRORS[/b red]'
)
self._print_error_table(label_results=label_results,
data_results=data_results)
def download(self, project, directory_path, is_forced):
command = spb.Command(type='describe_videolabel')
_, label_count = spb.run(command=command, option={
'project_id' : project.id
}, page_size = 1, page = 1)
#Download project configuration
try:
project_config_path = os.path.join(directory_path, 'project.json')
with open(project_config_path, 'w') as input:
json.dump(project.label_interface, input, indent=4)
is_download_project_config = True
except Exception as e:
is_download_project_config = False
if label_count != 0:
page_length = int(label_count/LABEL_DESCRIBE_PAGE_SIZE) if label_count % LABEL_DESCRIBE_PAGE_SIZE == 0 else int(label_count/LABEL_DESCRIBE_PAGE_SIZE)+1
if not is_forced:
if not click.confirm(f"Downloading {label_count} data and {label_count} labels from project '{project.name}' to '{directory_path}'. Proceed?"):
return
manager = Manager()
results = manager.list([manager.dict()]*page_length)
# inputs = zip([project.id] * page_length, range(page_length), [directory_path] * page_length, results)
# for i in inputs:
# _download_worker(i)
with Pool(NUM_MULTI_PROCESS) as p:
list(tqdm.tqdm(p.imap(_download_worker, zip([project.id] * page_length, range(page_length), [directory_path] * page_length, results)), total=page_length))
results = results[0]
data_results = {}
label_results = {}
if len(results) > 0:
for key in results.keys():
if 'data' in results[key]:
data_results[key] = results[key]['data']
if 'label' in results[key]:
label_results[key] = results[key]['label']
else:
label_results = {}
data_results = {}
console.print('\n[b blue]** Result Summary **[/b blue]')
console.print(f'Download of project configuration - {"[b blue]Success[/b blue]" if is_download_project_config else "[b red]Fail[/b red]"}')
label_success_count = label_count - len(label_results)
console.print(f'Successful download of {label_success_count} out of {label_count} labels. ({round(label_success_count/label_count*100,2)}%) - [b red]{len(label_results)} ERRORS[/b red]')
data_success_count = label_count - len(data_results)
console.print(f'Successful download of {data_success_count} out of {label_count} data. ({round(data_success_count/label_count*100,2)}%) - [b red]{len(data_results)} ERRORS[/b red]')
self._print_error_table(label_results=label_results, data_results=data_results)
def get_data_page(self, page_idx, page_size=10, num_data=None, tags=[], **kwargs):
if num_data is None:
num_data = self.get_num_data(tags=tags, **kwargs)
num_pages = math.ceil(float(num_data) / page_size)
if page_idx >= num_pages:
print('[WARNING] Index out of bounds. Empty list returned')
return []
command = spb.Command(type='describe_label')
tags = [{'name': tag} for tag in tags]
option = {'project_id': self._project.id, 'tags': tags, **kwargs}
data_page, _ = spb.run(command=command, option=option, page=page_idx+1, page_size=page_size)
for data in data_page:
yield DataHandle(data, self._project)
def get_data(self, data_idx, num_data=None, tags=[], **kwargs):
if num_data is None:
num_data = self.get_num_data(tags=tags, **kwargs)
if data_idx >= num_data:
print('[WARNING] Index out of bounds. None returned')
return None
command = spb.Command(type='describe_label')
tags = [{'name': tag} for tag in tags]
option = {'project_id': self._project.id, 'tags': tags, **kwargs}
data, _ = spb.run(command=command,
option=option,
page=data_idx + 1,
page_size=1)
return DataHandle(data[0], self._project)
def _download_worker(args):
[project_id, page_idx, directory_path, result] = args
command = spb.Command(type='describe_videolabel')
labels, _ = spb.run(command=command, option={
'project_id' : project_id
}, page_size = LABEL_DESCRIBE_PAGE_SIZE, page = page_idx + 1)
for label in labels:
error = {}
path = os.path.join(label.dataset, label.data_key[1:]) if label.data_key.startswith('/') else os.path.join(label.dataset, label.data_key)
path = os.path.join(directory_path, path)
os.makedirs(os.path.dirname(path), exist_ok=True)
label_error = None
data_error = None
try:
label_json_path = f'{path}.json'
open(label_json_path, 'w').write(label.toJson())
except Exception as e:
error = {'label':str(e)}
label_error = error
try:
custom_signed_url = json.loads(label.data_url)
base_url = custom_signed_url['base_url']
query = custom_signed_url['query']
file_infos = custom_signed_url['file_infos']
for idx, file_info in enumerate(file_infos, 1):
file_path = f'{path}/{file_info["file_name"]}'
ext = file_info['file_name'].split('.')[1]
new_path = os.path.split(file_path)[0]
if not os.path.exists(new_path):
os.makedirs(new_path)
file_url = '{}image_{:08d}.{}?{}'.format(base_url, idx, ext, query)
r = requests.get(file_url, allow_redirects=True)
open(file_path, 'wb').write(r.content)
except Exception as e:
error.update({'data':str(e)})
data_error = error
if len(error) > 0:
result[f'{label.dataset}/{label.data_key}'] = error
if label_error:
_ = dict()
_set_error_result(f'{label.dataset}/{label.data_key}', _, str(label_error), label_error)
if data_error:
_ = dict()
_set_error_result(f'{label.dataset}/{label.data_key}', _, str(data_error), data_error)
def _upload_asset(args):
logging.debug(f'Uploading Asset: {args}')
[project_id, asset_video, result] = args
try:
command = spb.Command(type='create_videodata')
result = spb.run(command=command, option=asset_video, optional={'projectId': project_id})
# TODO: Perhaps move this logic elsewhere
file_infos = json.loads(result.file_infos)
for file_info in file_infos:
path = asset_video['files']['path']
file_name = file_info['file_name']
file_path = F'{path}/{file_name}'
data = open(file_path,'rb').read()
response = requests.put(file_info['presigned_url'],data=data)
except Exception as e:
_set_error_result(asset_video['data_key'], result, str(e), e)
pass
def get_data_page(self,
page_idx,
page_size=10,
num_data=None,
tags=[],
dataset=None,
data_key=None,
**kwargs):
if num_data is None:
num_data = self.get_num_data(tags=tags, **kwargs)
num_pages = math.ceil(float(num_data) / page_size)
if page_idx >= num_pages:
print('[WARNING] Index out of bounds. Empty list returned')
return []
workapp = self._project.workapp
if workapp == 'video-siesta':
# Video
command = spb.Command(type='describe_videolabel')
tags = [{'name': tag} for tag in tags]
option = {'project_id': self._project.id, 'tags': tags, **kwargs}
data_page, _ = spb.run(command=command,
option=option,
page=page_idx + 1,
page_size=page_size)
for data in data_page:
yield VideoDataHandle(data, self._project)
else:
manager = LabelManager()
tags = [{'name': tag} for tag in tags]
option = {
'project_id': self._project.id,
'tags': tags,
'page': page_idx + 1,
'page_size': page_size,
'dataset': dataset,
'data_key': data_key,
**kwargs
}
count, data_page = manager.get_labels(**option)
for data in data_page:
yield DataHandle(data, self._project)
def upload_video(self, path, dataset_name, key=None):
if not os.path.isdir(path):
print('[WARNING] Invalid path. Upload failed')
return
# TODO: support_img_format is const
support_img_format = ('png', 'jpg', 'bmp', 'jpeg', 'tiff', 'tif')
file_names = [
os.path.basename(file_path)
for file_path in glob.glob(os.path.join(path, '*'))
if file_path.lower().endswith(support_img_format)
]
if len(file_names) == 0:
print('[WARNING] Invalid path. Upload failed')
return
if key is None:
key = os.path.split(path)[-1]
asset_video = {
'dataset': dataset_name,
'data_key': key,
'files': {
'path': path,
'file_names': natsorted(file_names),
},
}
try:
command = spb.Command(type='create_videodata')
result = spb.run(command=command,
option=asset_video,
optional={'projectId': self._project.id})
file_infos = json.loads(result.file_infos)
for file_info in file_infos:
file_name = file_info['file_name']
file_path = os.path.join(path, file_name)
data = open(file_path, 'rb').read()
response = requests.put(file_info['presigned_url'], data=data)
except Exception as e:
print('[WARNING] Duplicate data key. Upload failed')
def _download_worker(args):
[project_id, page_idx, directory_path, result] = args
command = spb.Command(type='describe_label')
labels, _ = spb.run(command=command,
option={'project_id': project_id},
page_size=LABEL_DESCRIBE_PAGE_SIZE,
page=page_idx + 1)
for label in labels:
error = {}
path = os.path.join(
label.dataset, label.data_key[1:]) if label.data_key.startswith(
'/') else os.path.join(label.dataset, label.data_key)
path = os.path.join(directory_path, path)
os.makedirs(os.path.dirname(path), exist_ok=True)
label_error = None
data_error = None
try:
label_json_path = f'{path}.json'
open(label_json_path, 'w').write(label.toJson())
except Exception as e:
error = {'label': str(e)}
label_error = error
try:
data_url = label.data_url
path = f'{path}'
r = requests.get(data_url, allow_redirects=True)
open(path, 'wb').write(r.content)
except Exception as e:
error.update({'data': str(e)})
data_error = error
if len(error) > 0:
result[f'{label.dataset}/{label.data_key}'] = error
if label_error:
_ = dict()
_set_error_result(f'{label.dataset}/{label.data_key}', _,
str(label_error), label_error)
if data_error:
_ = dict()
_set_error_result(f'{label.dataset}/{label.data_key}', _,
str(data_error), data_error)
def _get_projects(self, name=None, page=None, page_size=None):
command = spb.Command(type='describe_project')
if name is not None:
return spb.run(command=command, option={'name': name})
else:
return spb.run(command=command, page=page, page_size=page_size)
def _upload_to_suite(self):
command = spb.Command(type='update_label')
_ = spb.run(command=command, option=self._data)
