def _upload_to_suite(self, info=None): command = spb.Command(type='update_videolabel') if info is None: _ = spb.run(command=command, option=self._data) else: _ = spb.run(command=command, option=self._data, optional={'info': json.dumps(info)})
def _update_label(args): [label_path, project_id, dataset, result] = args data_key = ".".join(label_path.split(".")[:-1]) if not os.path.isfile(label_path): _set_error_result(data_key, result, 'Label json file is not existed.') return option = { 'project_id': project_id, 'dataset': dataset, 'data_key': data_key } try: command = spb.Command(type='describe_label') described_labels, _ = spb.run(command=command, option=option, page_size=1, page=1) described_label = described_labels[ 0] if described_labels and described_labels[0] else None if described_label is None: _set_error_result(data_key, result, 'Label cannot be described.') return if described_label.data_key != option[ 'data_key'] and described_label.dataset != option['dataset']: _set_error_result(data_key, result, 'Described label does not match to upload.') return except Exception as e: _set_error_result(data_key, result, str(e), e) return label = { "id": described_label.id, "project_id": project_id, "tags": [tag.get_datas(tag) for tag in described_label.tags], "result": described_label.result, } try: with open(label_path) as json_file: json_data = json.load(json_file) if json_data['result'] is None: return if 'result' in json_data: label['result'] = json_data['result'] if 'tags' in json_data: label['tags'] = json_data['tags'] command = spb.Command(type='update_label') label = spb.run(command=command, option=label) with open(label_path, 'w') as f: f.write(label.toJson()) except Exception as e: _set_error_result(data_key, result, str(e), e)
def _upload_asset(args): logging.debug(f'Uploading Asset: {args}') [project_id, asset_image, result] = args try: command = spb.Command(type='create_data') spb.run(command=command, option=asset_image, optional={'projectId': project_id}) except Exception as e: _set_error_result(asset_image['data_key'], result, str(e), e) pass
def _update_label(args): [label_path, project_id, label_interface, dataset, result] = args data_key = ".".join(label_path.split(".")[:-1]) if not os.path.isfile(label_path): _set_error_result(data_key, result, 'Label json file is not existed.') return option = { 'project_id': project_id, 'dataset': dataset, 'data_key': data_key } try: command = spb.Command(type='describe_videolabel') described_labels, _ = spb.run(command=command, option=option, page_size=1, page=1) described_label = described_labels[0] if described_labels and described_labels[0] else None if described_label is None: _set_error_result(data_key, result, 'Label cannot be described.') return if described_label.data_key != option['data_key'] and described_label.dataset != option['dataset']: _set_error_result(data_key, result, 'Described label does not match to upload.') return except Exception as e: _set_error_result(data_key, result, str(e), e) return label = { "id": described_label.id, "project_id": project_id, "tags": [tag.get_datas(tag) for tag in described_label.tags] } try: with open(label_path) as json_file: json_data = json.load(json_file) # TODO: NEED info.json VALIDATION if json_data['result'] is None: return label_info = build_label_info(label_interface=label_interface, result=json_data['result']) info_json = label_info.build_info() write_response = requests.put(described_label.info_write_presigned_url ,data=json.dumps(info_json)) if 'tags' in json_data: label['tags'] = json_data['tags'] command = spb.Command(type='update_videolabel') label = spb.run(command=command, option=label, optional={'info': json.dumps(info_json)}) with open(label_path, 'w') as f: f.write(label.toJson()) except Exception as e: _set_error_result(data_key, result, str(e), e)
def test_describe_project(): spb.client() command = Command(type='describe_project') projects = spb.run(command=command) for item in projects: print(item)
def upload_image(self, path, dataset_name, key=None, name=None): if not os.path.isfile(path): print('[WARNING] Invalid path. Upload failed') return if name is None: name = path.split('/')[-1] if key is None: key = name command = spb.Command(type='create_data') option = { 'file': path, 'file_name': name, 'dataset': dataset_name, 'data_key': key } try: return spb.run(command=command, optional={'projectId': self._project.id}, option=option) except Exception as e: print('[WARNING] Duplicate data key. Upload failed')
def _get_project(cls, project_name): command = spb.Command(type='describe_project') projects, num_of_projects = spb.run(command=command, option={'name': project_name}, page=1, page_size=1) if num_of_projects == 0: print('[WARNING] Project {} not found'.format(project_name)) return None return projects[0]
def get_num_data(self, tags=[], **kwargs): command = spb.Command(type='describe_label') tags = [{'name': tag} for tag in tags] option = {'project_id': self._project.id, 'tags': tags, **kwargs} _, num_data = spb.run(command=command, option=option, page=1, page_size=1) if num_data == 0: print('[WARNING] Data list is empty') return num_data
def download(self, project, directory_path, is_forced): command = spb.Command(type='describe_videolabel') _, label_count = spb.run(command=command, option={ 'project_id' : project.id }, page_size = 1, page = 1) #Download project configuration try: project_config_path = os.path.join(directory_path, 'project.json') with open(project_config_path, 'w') as input: json.dump(project.label_interface, input, indent=4) is_download_project_config = True except Exception as e: is_download_project_config = False if label_count != 0: page_length = int(label_count/LABEL_DESCRIBE_PAGE_SIZE) if label_count % LABEL_DESCRIBE_PAGE_SIZE == 0 else int(label_count/LABEL_DESCRIBE_PAGE_SIZE)+1 if not is_forced: if not click.confirm(f"Downloading {label_count} data and {label_count} labels from project '{project.name}' to '{directory_path}'. Proceed?"): return manager = Manager() results = manager.list([manager.dict()]*page_length) # inputs = zip([project.id] * page_length, range(page_length), [directory_path] * page_length, results) # for i in inputs: # _download_worker(i) with Pool(NUM_MULTI_PROCESS) as p: list(tqdm.tqdm(p.imap(_download_worker, zip([project.id] * page_length, range(page_length), [directory_path] * page_length, results)), total=page_length)) results = results[0] data_results = {} label_results = {} if len(results) > 0: for key in results.keys(): if 'data' in results[key]: data_results[key] = results[key]['data'] if 'label' in results[key]: label_results[key] = results[key]['label'] else: label_results = {} data_results = {} console.print('\n[b blue]** Result Summary **[/b blue]') console.print(f'Download of project configuration - {"[b blue]Success[/b blue]" if is_download_project_config else "[b red]Fail[/b red]"}') label_success_count = label_count - len(label_results) console.print(f'Successful download of {label_success_count} out of {label_count} labels. ({round(label_success_count/label_count*100,2)}%) - [b red]{len(label_results)} ERRORS[/b red]') data_success_count = label_count - len(data_results) console.print(f'Successful download of {data_success_count} out of {label_count} data. ({round(data_success_count/label_count*100,2)}%) - [b red]{len(data_results)} ERRORS[/b red]') self._print_error_table(label_results=label_results, data_results=data_results)
def download(self, project, directory_path, is_forced): command = spb.Command(type='describe_label') _, label_count = spb.run(command=command, option={'project_id': project.id}, page_size=1, page=1) if label_count != 0: page_length = int( label_count / LABEL_DESCRIBE_PAGE_SIZE ) if label_count % LABEL_DESCRIBE_PAGE_SIZE == 0 else int( label_count / LABEL_DESCRIBE_PAGE_SIZE) + 1 if not is_forced: if not click.confirm( f"Downloading {label_count} data and {label_count} labels from project '{project.name}' to '{directory_path}'. Proceed?" ): return manager = Manager() results = manager.list([manager.dict()] * page_length) with Pool(NUM_MULTI_PROCESS) as p: list( tqdm.tqdm(p.imap( _download_worker, zip([project.id] * page_length, range(page_length), [directory_path] * page_length, results)), total=page_length)) results = results[0] data_results = {} label_results = {} if len(results) > 0: for key in results.keys(): if 'data' in results[key]: data_results[key] = results[key]['data'] if 'label' in results[key]: label_results[key] = results[key]['label'] else: label_results = {} data_results = {} console.print('\n[b blue]** Result Summary **[/b blue]') label_success_count = label_count - len(label_results) console.print( f'Successful download of {label_success_count} out of {label_count} labels. ({round(label_success_count/label_count*100,2)}%) - [b red]{len(label_results)} ERRORS[/b red]' ) data_success_count = label_count - len(data_results) console.print( f'Successful download of {data_success_count} out of {label_count} data. ({round(data_success_count/label_count*100,2)}%) - [b red]{len(data_results)} ERRORS[/b red]' ) self._print_error_table(label_results=label_results, data_results=data_results)
def get_data_page(self, page_idx, page_size=10, num_data=None, tags=[], **kwargs): if num_data is None: num_data = self.get_num_data(tags=tags, **kwargs) num_pages = math.ceil(float(num_data) / page_size) if page_idx >= num_pages: print('[WARNING] Index out of bounds. Empty list returned') return [] command = spb.Command(type='describe_label') tags = [{'name': tag} for tag in tags] option = {'project_id': self._project.id, 'tags': tags, **kwargs} data_page, _ = spb.run(command=command, option=option, page=page_idx+1, page_size=page_size) for data in data_page: yield DataHandle(data, self._project)
def get_data(self, data_idx, num_data=None, tags=[], **kwargs): if num_data is None: num_data = self.get_num_data(tags=tags, **kwargs) if data_idx >= num_data: print('[WARNING] Index out of bounds. None returned') return None command = spb.Command(type='describe_label') tags = [{'name': tag} for tag in tags] option = {'project_id': self._project.id, 'tags': tags, **kwargs} data, _ = spb.run(command=command, option=option, page=data_idx + 1, page_size=1) return DataHandle(data[0], self._project)
def _download_worker(args): [project_id, page_idx, directory_path, result] = args command = spb.Command(type='describe_videolabel') labels, _ = spb.run(command=command, option={ 'project_id' : project_id }, page_size = LABEL_DESCRIBE_PAGE_SIZE, page = page_idx + 1) for label in labels: error = {} path = os.path.join(label.dataset, label.data_key[1:]) if label.data_key.startswith('/') else os.path.join(label.dataset, label.data_key) path = os.path.join(directory_path, path) os.makedirs(os.path.dirname(path), exist_ok=True) label_error = None data_error = None try: label_json_path = f'{path}.json' open(label_json_path, 'w').write(label.toJson()) except Exception as e: error = {'label':str(e)} label_error = error try: custom_signed_url = json.loads(label.data_url) base_url = custom_signed_url['base_url'] query = custom_signed_url['query'] file_infos = custom_signed_url['file_infos'] for idx, file_info in enumerate(file_infos, 1): file_path = f'{path}/{file_info["file_name"]}' ext = file_info['file_name'].split('.')[1] new_path = os.path.split(file_path)[0] if not os.path.exists(new_path): os.makedirs(new_path) file_url = '{}image_{:08d}.{}?{}'.format(base_url, idx, ext, query) r = requests.get(file_url, allow_redirects=True) open(file_path, 'wb').write(r.content) except Exception as e: error.update({'data':str(e)}) data_error = error if len(error) > 0: result[f'{label.dataset}/{label.data_key}'] = error if label_error: _ = dict() _set_error_result(f'{label.dataset}/{label.data_key}', _, str(label_error), label_error) if data_error: _ = dict() _set_error_result(f'{label.dataset}/{label.data_key}', _, str(data_error), data_error)
def _upload_asset(args): logging.debug(f'Uploading Asset: {args}') [project_id, asset_video, result] = args try: command = spb.Command(type='create_videodata') result = spb.run(command=command, option=asset_video, optional={'projectId': project_id}) # TODO: Perhaps move this logic elsewhere file_infos = json.loads(result.file_infos) for file_info in file_infos: path = asset_video['files']['path'] file_name = file_info['file_name'] file_path = F'{path}/{file_name}' data = open(file_path,'rb').read() response = requests.put(file_info['presigned_url'],data=data) except Exception as e: _set_error_result(asset_video['data_key'], result, str(e), e) pass
def get_data_page(self, page_idx, page_size=10, num_data=None, tags=[], dataset=None, data_key=None, **kwargs): if num_data is None: num_data = self.get_num_data(tags=tags, **kwargs) num_pages = math.ceil(float(num_data) / page_size) if page_idx >= num_pages: print('[WARNING] Index out of bounds. Empty list returned') return [] workapp = self._project.workapp if workapp == 'video-siesta': # Video command = spb.Command(type='describe_videolabel') tags = [{'name': tag} for tag in tags] option = {'project_id': self._project.id, 'tags': tags, **kwargs} data_page, _ = spb.run(command=command, option=option, page=page_idx + 1, page_size=page_size) for data in data_page: yield VideoDataHandle(data, self._project) else: manager = LabelManager() tags = [{'name': tag} for tag in tags] option = { 'project_id': self._project.id, 'tags': tags, 'page': page_idx + 1, 'page_size': page_size, 'dataset': dataset, 'data_key': data_key, **kwargs } count, data_page = manager.get_labels(**option) for data in data_page: yield DataHandle(data, self._project)
def upload_video(self, path, dataset_name, key=None): if not os.path.isdir(path): print('[WARNING] Invalid path. Upload failed') return # TODO: support_img_format is const support_img_format = ('png', 'jpg', 'bmp', 'jpeg', 'tiff', 'tif') file_names = [ os.path.basename(file_path) for file_path in glob.glob(os.path.join(path, '*')) if file_path.lower().endswith(support_img_format) ] if len(file_names) == 0: print('[WARNING] Invalid path. Upload failed') return if key is None: key = os.path.split(path)[-1] asset_video = { 'dataset': dataset_name, 'data_key': key, 'files': { 'path': path, 'file_names': natsorted(file_names), }, } try: command = spb.Command(type='create_videodata') result = spb.run(command=command, option=asset_video, optional={'projectId': self._project.id}) file_infos = json.loads(result.file_infos) for file_info in file_infos: file_name = file_info['file_name'] file_path = os.path.join(path, file_name) data = open(file_path, 'rb').read() response = requests.put(file_info['presigned_url'], data=data) except Exception as e: print('[WARNING] Duplicate data key. Upload failed')
def _download_worker(args): [project_id, page_idx, directory_path, result] = args command = spb.Command(type='describe_label') labels, _ = spb.run(command=command, option={'project_id': project_id}, page_size=LABEL_DESCRIBE_PAGE_SIZE, page=page_idx + 1) for label in labels: error = {} path = os.path.join( label.dataset, label.data_key[1:]) if label.data_key.startswith( '/') else os.path.join(label.dataset, label.data_key) path = os.path.join(directory_path, path) os.makedirs(os.path.dirname(path), exist_ok=True) label_error = None data_error = None try: label_json_path = f'{path}.json' open(label_json_path, 'w').write(label.toJson()) except Exception as e: error = {'label': str(e)} label_error = error try: data_url = label.data_url path = f'{path}' r = requests.get(data_url, allow_redirects=True) open(path, 'wb').write(r.content) except Exception as e: error.update({'data': str(e)}) data_error = error if len(error) > 0: result[f'{label.dataset}/{label.data_key}'] = error if label_error: _ = dict() _set_error_result(f'{label.dataset}/{label.data_key}', _, str(label_error), label_error) if data_error: _ = dict() _set_error_result(f'{label.dataset}/{label.data_key}', _, str(data_error), data_error)
def _get_projects(self, name=None, page=None, page_size=None): command = spb.Command(type='describe_project') if name is not None: return spb.run(command=command, option={'name': name}) else: return spb.run(command=command, page=page, page_size=page_size)
def _upload_to_suite(self): command = spb.Command(type='update_label') _ = spb.run(command=command, option=self._data)