class DatasetApi(ModuleApi): _info_sequence = [ ApiField.ID, ApiField.NAME, ApiField.DESCRIPTION, ApiField.SIZE, ApiField.PROJECT_ID, ApiField.IMAGES_COUNT, ApiField.CREATED_AT, ApiField.UPDATED_AT ] Info = namedtuple('DatasetInfo', [camel_to_snake(name) for name in _info_sequence]) def get_list(self, project_id, filters=None): return self.get_list_all_pages('datasets.list', { ApiField.PROJECT_ID: project_id, ApiField.FILTER: filters or [] }) def get_info_by_id(self, id): return self._get_info_by_id(id, 'datasets.info') def create(self, project_id, name, description=""): response = self.api.post( 'datasets.add', { ApiField.PROJECT_ID: project_id, ApiField.NAME: name, ApiField.DESCRIPTION: description }) return self._convert_json_info(response.json()) def _get_update_method(self): return 'datasets.editInfo'
class AgentApi(ModuleApi): class Status(Enum): WAITING = 'waiting' RUNNING = 'running' _info_sequence = [ ApiField.ID, ApiField.NAME, ApiField.TOKEN, ApiField.STATUS, ApiField.USER_ID, ApiField.TEAM_ID, ApiField.CAPABILITIES, ApiField.CREATED_AT, ApiField.UPDATED_AT ] Info = namedtuple('AgentInfo', [camel_to_snake(name) for name in _info_sequence]) def get_list(self, team_id, filters=None): return self.get_list_all_pages('agents.list', { 'teamId': team_id, "filter": filters or [] }) def get_info_by_id(self, id): return self._get_info_by_id(id, 'agent.info') def get_status(self, id): status_str = self.get_info_by_id(id).status return self.Status(status_str) def raise_for_status(self, status): pass
class WorkspaceApi(ModuleApi): _info_sequence = [ ApiField.ID, ApiField.NAME, ApiField.DESCRIPTION, ApiField.TEAM_ID, ApiField.CREATED_AT, ApiField.UPDATED_AT ] Info = namedtuple('WorkspaceInfo', [camel_to_snake(name) for name in _info_sequence]) def get_list(self, team_id, filters=None): return self.get_list_all_pages('workspaces.list', { ApiField.TEAM_ID: team_id, ApiField.FILTER: filters or [] }) def get_info_by_id(self, id): return self._get_info_by_id(id, 'workspaces.info') def create(self, team_id, name, description=""): response = self.api.post( 'workspaces.add', { ApiField.TEAM_ID: team_id, ApiField.NAME: name, ApiField.DESCRIPTION: description }) return self._convert_json_info(response.json()) def _get_update_method(self): return 'workspaces.editInfo'
def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) try: cls.InfoType = namedtuple( cls.info_tuple_name(), [camel_to_snake(name) for name in cls.info_sequence()]) except NotImplementedError: pass
def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) try: field_names = [] for name in cls.info_sequence(): if type(name) is str: field_names.append(camel_to_snake(name)) elif type(name) is tuple and type(name[1]) is str: field_names.append(name[1]) else: raise RuntimeError('Can not parse field {!r}'.format(name)) cls.InfoType = namedtuple(cls.info_tuple_name(), field_names) except NotImplementedError: pass
class ProjectApi(ModuleApi): _info_sequence = [ ApiField.ID, ApiField.NAME, ApiField.DESCRIPTION, ApiField.SIZE, ApiField.README, ApiField.WORKSPACE_ID, ApiField.CREATED_AT, ApiField.UPDATED_AT ] Info = namedtuple('ProjectInfo', [camel_to_snake(name) for name in _info_sequence]) def get_list(self, workspace_id, filters=None): return self.get_list_all_pages('projects.list', { ApiField.WORKSPACE_ID: workspace_id, "filter": filters or [] }) def get_info_by_id(self, id): return self._get_info_by_id(id, 'projects.info') def get_meta(self, id): response = self.api.post('projects.meta', {'id': id}) return response.json() def create(self, workspace_id, name, description=""): response = self.api.post( 'projects.add', { ApiField.WORKSPACE_ID: workspace_id, ApiField.NAME: name, ApiField.DESCRIPTION: description }) return self._convert_json_info(response.json()) def _get_update_method(self): return 'projects.editInfo' def update_meta(self, id, meta): self.api.post('projects.meta.update', { ApiField.ID: id, ApiField.META: meta }) def _clone_api_method_name(self): return 'projects.clone' def get_datasets_count(self, id): datasets = self.api.dataset.get_list(id) return len(datasets) def get_images_count(self, id): datasets = self.api.dataset.get_list(id) return sum([dataset.images_count for dataset in datasets])
class PluginApi(ModuleApi): _info_sequence = [ApiField.ID, ApiField.NAME, ApiField.DESCRIPTION, ApiField.TYPE, ApiField.DEFAULT_VERSION, ApiField.DOCKER_IMAGE, ApiField.README, ApiField.CONFIGS, ApiField.VERSIONS, ApiField.CREATED_AT, ApiField.UPDATED_AT] Info = namedtuple('PluginInfo', [camel_to_snake(name) for name in _info_sequence]) def get_list(self, team_id, filters=None): return self.get_list_all_pages('plugins.list', {ApiField.TEAM_ID: team_id, ApiField.FILTER: filters or []}) def get_info_by_id(self, team_id, plugin_id): filters = [{"field": ApiField.ID, "operator": "=", "value": plugin_id}] return self._get_info_by_filters(team_id, filters)
class AnnotationApi(ModuleApi): _info_sequence = [ApiField.IMAGE_ID, ApiField.IMAGE_NAME, ApiField.ANNOTATION, ApiField.CREATED_AT, ApiField.UPDATED_AT] Info = namedtuple('AnnotationInfo', [camel_to_snake(name) for name in _info_sequence]) def get_list(self, dataset_id, filters=None, progress_cb=None): return self.get_list_all_pages('annotations.list', {ApiField.DATASET_ID: dataset_id, ApiField.FILTER: filters or []}, progress_cb) def download(self, image_id): response = self.api.post('annotations.info', {ApiField.IMAGE_ID: image_id}) return self._convert_json_info(response.json()) def download_batch(self, dataset_id, image_ids, progress_cb=None): id_to_ann = {} for batch in batched(image_ids): results = self.api.post('annotations.bulk.info', data={ApiField.DATASET_ID: dataset_id, ApiField.IMAGE_IDS: batch}).json() for ann_dict in results: ann_info = self._convert_json_info(ann_dict) id_to_ann[ann_info.image_id] = ann_info if progress_cb is not None: progress_cb(len(batch)) ordered_results = [id_to_ann[image_id] for image_id in image_ids] return ordered_results def upload_path(self, img_id, ann_path): self.upload_paths([img_id], [ann_path]) def upload_paths(self, img_ids, ann_paths, progress_cb=None): # img_ids from the same dataset def read_json(ann_path): with open(ann_path) as json_file: return json.load(json_file) self._upload_batch(read_json, img_ids, ann_paths, progress_cb) def upload_json(self, img_id, ann_json): self.upload_jsons([img_id], [ann_json]) def upload_jsons(self, img_ids, ann_jsons, progress_cb=None): # img_ids from the same dataset self._upload_batch(lambda x : x, img_ids, ann_jsons, progress_cb) def upload_ann(self, img_id, ann): self.upload_anns([img_id], [ann]) def upload_anns(self, img_ids, anns, progress_cb=None): # img_ids from the same dataset self._upload_batch(Annotation.to_json, img_ids, anns, progress_cb) def _upload_batch(self, func_ann_to_json, img_ids, anns, progress_cb=None): # img_ids from the same dataset if len(img_ids) == 0: return if len(img_ids) != len(anns): raise RuntimeError("Can not match \"img_ids\" and \"anns\" lists, len(img_ids) != len(anns)") dataset_id = self.api.image.get_info_by_id(img_ids[0]).dataset_id for batch in batched(list(zip(img_ids, anns))): data = [{ApiField.IMAGE_ID: img_id, ApiField.ANNOTATION: func_ann_to_json(ann)} for img_id, ann in batch] self.api.post('annotations.bulk.add', data={ApiField.DATASET_ID: dataset_id, ApiField.ANNOTATIONS: data}) if progress_cb is not None: progress_cb(len(batch)) def get_info_by_id(self, id): raise RuntimeError('Method is not supported') def get_info_by_name(self, parent_id, name): raise RuntimeError('Method is not supported') def exists(self, parent_id, name): raise RuntimeError('Method is not supported') def get_free_name(self, parent_id, name): raise RuntimeError('Method is not supported') def _add_sort_param(self, data): return data
class ImageApi(ModuleApi): _info_sequence = [ ApiField.ID, ApiField.NAME, ApiField.LINK, ApiField.HASH, ApiField.MIME, ApiField.EXT, ApiField.SIZE, ApiField.WIDTH, ApiField.HEIGHT, ApiField.LABELS_COUNT, ApiField.DATASET_ID, ApiField.CREATED_AT, ApiField.UPDATED_AT ] Info = namedtuple('ImageInfo', [camel_to_snake(name) for name in _info_sequence]) def get_list(self, dataset_id, filters=None): return self.get_list_all_pages('images.list', { ApiField.DATASET_ID: dataset_id, ApiField.FILTER: filters or [] }) def get_info_by_id(self, id): return self._get_info_by_id(id, 'images.info') # @TODO: reimplement to new method images.bulk.info def get_info_by_id_batch(self, ids): results = [] if len(ids) == 0: return results dataset_id = self.get_info_by_id(ids[0]).dataset_id for batch in batched(ids): filters = [{ "field": ApiField.ID, "operator": "in", "value": batch }] results.extend( self.get_list_all_pages('images.list', { ApiField.DATASET_ID: dataset_id, ApiField.FILTER: filters })) return results def _download(self, id): response = self.api.post('images.download', {ApiField.ID: id}) return response def download_np(self, id): response = self._download(id) img = sly_image.read_bytes(response.content) return img def download_path(self, id, path): response = self._download(id) ensure_base_path(path) with open(path, 'wb') as fd: for chunk in response.iter_content(chunk_size=1024 * 1024): fd.write(chunk) def _download_batch(self, dataset_id, ids): for batch_ids in batched(ids): response = self.api.post('images.bulk.download', { ApiField.DATASET_ID: dataset_id, ApiField.IMAGE_IDS: batch_ids }) decoder = MultipartDecoder.from_response(response) for idx, part in enumerate(decoder.parts): img_id = int( re.findall( 'name="(.*)"', part.headers[b'Content-Disposition'].decode('utf-8')) [0]) yield img_id, part def download_paths(self, dataset_id, ids, paths, progress_cb=None): if len(ids) == 0: return if len(ids) != len(paths): raise RuntimeError( "Can not match \"ids\" and \"paths\" lists, len(ids) != len(paths)" ) id_to_path = {id: path for id, path in zip(ids, paths)} #debug_ids = [] for img_id, resp_part in self._download_batch(dataset_id, ids): #debug_ids.append(img_id) with open(id_to_path[img_id], 'wb') as w: w.write(resp_part.content) if progress_cb is not None: progress_cb(1) #if ids != debug_ids: # raise RuntimeError("images.bulk.download: imageIds order is broken") def download_nps(self, dataset_id, ids, progress_cb=None): images = [] if len(ids) == 0: return images id_to_img = {} for img_id, resp_part in self._download_batch(dataset_id, ids): id_to_img[img_id] = sly_image.read_bytes(resp_part.content) if progress_cb is not None: progress_cb(1) images = [id_to_img[id] for id in ids] return images def check_existing_hashes(self, hashes): results = [] if len(hashes) == 0: return results for hashes_batch in batched(hashes, batch_size=900): response = self.api.post('images.internal.hashes.list', hashes_batch) results.extend(response.json()) return results def check_image_uploaded(self, hash): response = self.api.post('images.internal.hashes.list', [hash]) results = response.json() if len(results) == 0: return False else: return True def _upload_data_bulk(self, func_item_to_byte_stream, func_item_hash, items, progress_cb): hashes = [] if len(items) == 0: return hashes hash_to_items = defaultdict(list) for idx, item in enumerate(items): item_hash = func_item_hash(item) hashes.append(item_hash) hash_to_items[item_hash].append(item) unique_hashes = set(hashes) remote_hashes = self.check_existing_hashes(list(unique_hashes)) new_hashes = unique_hashes - set(remote_hashes) if progress_cb is not None: progress_cb(len(remote_hashes)) # upload only new images to supervisely server items_to_upload = [] for hash in new_hashes: items_to_upload.extend(hash_to_items[hash]) for batch in batched(items_to_upload): content_dict = {} for idx, item in enumerate(batch): content_dict["{}-file".format(idx)] = ( str(idx), func_item_to_byte_stream(item), 'image/*') encoder = MultipartEncoder(fields=content_dict) self.api.post('images.bulk.upload', encoder) if progress_cb is not None: progress_cb(len(batch)) return hashes def upload_path(self, dataset_id, name, path): return self.upload_paths(dataset_id, [name], [path])[0] def upload_paths(self, dataset_id, names, paths, progress_cb=None): def path_to_bytes_stream(path): return open(path, 'rb') hashes = self._upload_data_bulk(path_to_bytes_stream, get_file_hash, paths, progress_cb) return self.upload_hashes(dataset_id, names, hashes) def upload_np(self, dataset_id, name, img): return self.upload_nps(dataset_id, [name], [img])[0] def upload_nps(self, dataset_id, names, imgs, progress_cb=None): def img_to_bytes_stream(item): img, name = item[0], item[1] img_bytes = sly_image.write_bytes(img, get_file_ext(name)) return io.BytesIO(img_bytes) def img_to_hash(item): img, name = item[0], item[1] return sly_image.get_hash(img, get_file_ext(name)) img_name_list = list(zip(imgs, names)) hashes = self._upload_data_bulk(img_to_bytes_stream, img_to_hash, img_name_list, progress_cb) return self.upload_hashes(dataset_id, names, hashes) def upload_link(self, dataset_id, name, link): return self.upload_links(dataset_id, [name], [link])[0] def upload_links(self, dataset_id, names, links, progress_cb=None): return self._upload_bulk_add(lambda item: (ApiField.LINK, item), dataset_id, names, links, progress_cb) def upload_hash(self, dataset_id, name, hash): return self.upload_hashes(dataset_id, [name], [hash])[0] def upload_hashes(self, dataset_id, names, hashes, progress_cb=None): return self._upload_bulk_add(lambda item: (ApiField.HASH, item), dataset_id, names, hashes, progress_cb) def upload_id(self, dataset_id, name, id): return self.upload_ids(dataset_id, [name], [id])[0] def upload_ids(self, dataset_id, names, ids, progress_cb=None): # all ids have to be from single dataset infos = self.get_info_by_id_batch(ids) hashes = [info.hash for info in infos] return self.upload_hashes(dataset_id, names, hashes, progress_cb) def _upload_bulk_add(self, func_item_to_kv, dataset_id, names, items, progress_cb=None): results = [] if len(names) == 0: return results if len(names) != len(items): raise RuntimeError( "Can not match \"names\" and \"items\" lists, len(names) != len(items)" ) for batch in batched(list(zip(names, items))): images = [] for name, item in batch: item_tuple = func_item_to_kv(item) #@TODO: 'title' -> ApiField.NAME images.append({'title': name, item_tuple[0]: item_tuple[1]}) response = self.api.post('images.bulk.add', { ApiField.DATASET_ID: dataset_id, ApiField.IMAGES: images }) if progress_cb is not None: progress_cb(len(images)) results.extend([ self._convert_json_info(info_json) for info_json in response.json() ]) name_to_res = {img_info.name: img_info for img_info in results} ordered_results = [name_to_res[name] for name in names] return ordered_results #@TODO: reimplement def _convert_json_info(self, info: dict): if info is None: return None temp_ext = None field_values = [] for field_name in self.__class__._info_sequence: if field_name == ApiField.EXT: continue field_values.append(info[field_name]) if field_name == ApiField.MIME: temp_ext = info[field_name].split('/')[1] field_values.append(temp_ext) for idx, field_name in enumerate(self.__class__._info_sequence): if field_name == ApiField.NAME: cur_ext = get_file_ext(field_values[idx]) if not cur_ext: field_values[idx] = "{}.{}".format(field_values[idx], temp_ext) break cur_ext = cur_ext.replace(".", "").lower() if temp_ext == 'jpeg' and cur_ext in ['jpg', 'jpeg']: break if temp_ext not in field_values[idx]: field_values[idx] = "{}.{}".format(field_values[idx], temp_ext) break return self.__class__.Info._make(field_values)
class AnnotationApi(ModuleApi): _info_sequence = [ ApiField.IMAGE_ID, ApiField.IMAGE_NAME, ApiField.ANNOTATION, ApiField.CREATED_AT, ApiField.UPDATED_AT ] Info = namedtuple('AnnotationInfo', [camel_to_snake(name) for name in _info_sequence]) def get_list(self, dataset_id, filters=None, progress_cb=None): return self.get_list_all_pages('annotations.list', { ApiField.DATASET_ID: dataset_id, ApiField.FILTER: filters or [] }, progress_cb) def download(self, image_id): response = self.api.post('annotations.info', {ApiField.IMAGE_ID: image_id}) return self._convert_json_info(response.json()) def download_batch(self, dataset_id, image_ids, progress_cb=None): filters = [{ "field": ApiField.IMAGE_ID, "operator": "in", "value": image_ids }] return self.get_list_all_pages('annotations.list', { ApiField.DATASET_ID: dataset_id, ApiField.FILTER: filters }, progress_cb) # @TODO: no errors from api if annotation is not valid def upload(self, image_id: int, ann: dict): self.api.post('annotations.add', data={ ApiField.IMAGE_ID: image_id, ApiField.ANNOTATION: ann }) def upload_batch_paths(self, dataset_id, img_ids, ann_paths, progress_cb=None): MAX_BATCH_SIZE = 50 for batch in batched(list(zip(img_ids, ann_paths)), MAX_BATCH_SIZE): data = [] for img_id, ann_path in batch: with open(ann_path) as json_file: ann_json = json.load(json_file) data.append({ ApiField.IMAGE_ID: img_id, ApiField.ANNOTATION: ann_json }) self.api.post('annotations.bulk.add', data={ ApiField.DATASET_ID: dataset_id, ApiField.ANNOTATIONS: data }) if progress_cb is not None: progress_cb(len(batch)) def upload_batch(self, img_ids, anns, progress_cb=None): raise NotImplementedError() def get_info_by_id(self, id): raise RuntimeError('Method is not supported') def get_info_by_name(self, parent_id, name): raise RuntimeError('Method is not supported') def exists(self, parent_id, name): raise RuntimeError('Method is not supported') def get_free_name(self, parent_id, name): raise RuntimeError('Method is not supported') def _add_sort_param(self, data): return data
class NeuralNetworkApi(ModuleApi): _info_sequence = [ ApiField.ID, ApiField.NAME, ApiField.DESCRIPTION, ApiField.CONFIG, ApiField.HASH, ApiField.ONLY_TRAIN, ApiField.PLUGIN_ID, ApiField.PLUGIN_VERSION, ApiField.SIZE, ApiField.WEIGHTS_LOCATION, ApiField.README, ApiField.TASK_ID, ApiField.USER_ID, ApiField.TEAM_ID, ApiField.WORKSPACE_ID, ApiField.CREATED_AT, ApiField.UPDATED_AT ] Info = namedtuple('ModelInfo', [camel_to_snake(name) for name in _info_sequence]) def get_list(self, workspace_id, filters=None): return self.get_list_all_pages('models.list', { ApiField.WORKSPACE_ID: workspace_id, ApiField.FILTER: filters or [] }) def get_info_by_id(self, id): return self._get_info_by_id(id, 'models.info') def download(self, id): response = self.api.post('models.download', {ApiField.ID: id}, stream=True) return response def download_to_tar(self, workspace_id, name, tar_path, progress_cb=None): model = self.get_info_by_name(workspace_id, name) response = self.download(model.id) ensure_base_path(tar_path) with open(tar_path, 'wb') as fd: for chunk in response.iter_content(chunk_size=1024 * 1024): fd.write(chunk) if progress_cb is not None: read_mb = len(chunk) / 1024.0 / 1024.0 progress_cb(read_mb) def download_to_dir(self, workspace_id, name, directory, progress_cb=None): model_tar = os.path.join(directory, rand_str(10) + '.tar') self.download_to_tar(workspace_id, name, model_tar, progress_cb) model_dir = os.path.join(directory, name) with tarfile.open(model_tar) as archive: archive.extractall(model_dir) silent_remove(model_tar) return model_dir def generate_hash(self, task_id): response = self.api.post('models.hash.create', {ApiField.TASK_ID: task_id}) return response.json() def upload(self, hash, archive_path, progress_cb=None): encoder = MultipartEncoder({ 'hash': hash, 'weights': (os.path.basename(archive_path), open(archive_path, 'rb'), 'application/x-tar') }) def callback(monitor): read_mb = monitor.bytes_read / 1024.0 / 1024.0 if progress_cb is not None: progress_cb(read_mb) monitor = MultipartEncoderMonitor(encoder, callback) self.api.post('models.upload', monitor) def inference_remote_image(self, id, image_hash, ann=None, meta=None, mode=None, ext=None): data = { "request_type": "inference", "meta": meta or ProjectMeta().to_json(), "annotation": ann or None, "mode": mode or {}, "image_hash": image_hash } fake_img_data = sly_image.write_bytes(np.zeros([5, 5, 3]), '.jpg') encoder = MultipartEncoder({ 'id': str(id).encode('utf-8'), 'data': json.dumps(data), 'image': ("img", fake_img_data, "") }) response = self.api.post('models.infer', MultipartEncoderMonitor(encoder)) return response.json() def inference(self, id, img, ann=None, meta=None, mode=None, ext=None): data = { "request_type": "inference", "meta": meta or ProjectMeta().to_json(), "annotation": ann or None, "mode": mode or {}, } img_data = sly_image.write_bytes(img, ext or '.jpg') encoder = MultipartEncoder({ 'id': str(id).encode('utf-8'), 'data': json.dumps(data), 'image': ("img", img_data, "") }) response = self.api.post('models.infer', MultipartEncoderMonitor(encoder)) return response.json() def get_output_meta(self, id, input_meta=None, inference_mode=None): data = { "request_type": "get_out_meta", "meta": input_meta or ProjectMeta().to_json(), "mode": inference_mode or {} } encoder = MultipartEncoder({ 'id': str(id).encode('utf-8'), 'data': json.dumps(data) }) response = self.api.post('models.infer', MultipartEncoderMonitor(encoder)) response_json = response.json() if 'out_meta' in response_json: return response_json['out_meta'] return response.json() def get_deploy_tasks(self, model_id): response = self.api.post('models.info.deployed', {'id': model_id}) return [task[ApiField.ID] for task in response.json()] def _clone_api_method_name(self): return 'models.clone'
class ImageApi(ModuleApi): _info_sequence = [ ApiField.ID, ApiField.NAME, ApiField.LINK, ApiField.HASH, ApiField.MIME, ApiField.EXT, ApiField.SIZE, ApiField.WIDTH, ApiField.HEIGHT, ApiField.LABELS_COUNT, ApiField.DATASET_ID, ApiField.CREATED_AT, ApiField.UPDATED_AT ] Info = namedtuple('ImageInfo', [camel_to_snake(name) for name in _info_sequence]) def get_list(self, dataset_id, filters=None): return self.get_list_all_pages('images.list', { ApiField.DATASET_ID: dataset_id, ApiField.FILTER: filters or [] }) def get_info_by_id(self, id): return self._get_info_by_id(id, 'images.info') def download(self, id): response = self.api.post('images.download', {ApiField.ID: id}) return response def download_np(self, id): response = self.download(id) image_bytes = np.asarray(bytearray(response.content), dtype="uint8") img = image.read_bytes(image_bytes) return img def download_to_file(self, id, path): response = self.download(id) ensure_base_path(path) with open(path, 'wb') as fd: for chunk in response.iter_content(chunk_size=1024 * 1024): fd.write(chunk) def download_batch(self, dataset_id, ids, paths, progress_cb=None): id_to_path = {id: path for id, path in zip(ids, paths)} MAX_BATCH_SIZE = 50 for batch_ids in batched(ids, MAX_BATCH_SIZE): response = self.api.post('images.bulk.download', { ApiField.DATASET_ID: dataset_id, ApiField.IMAGE_IDS: batch_ids }) decoder = MultipartDecoder.from_response(response) for idx, part in enumerate(decoder.parts): img_id = int( re.findall( 'name="(.*)"', part.headers[b'Content-Disposition'].decode('utf-8')) [0]) with open(id_to_path[img_id], 'wb') as w: w.write(part.content) progress_cb(1) def check_existing_hashes(self, hashes): BATCH_SIZE = 900 results = [] for hashes_batch in batched(hashes, BATCH_SIZE): response = self.api.post('images.internal.hashes.list', hashes_batch) results.extend(response.json()) return results def check_image_uploaded(self, hash): response = self.api.post('images.internal.hashes.list', [hash]) results = response.json() if len(results) == 0: return False else: return True def upload_np(self, img_np, ext='.png'): data = image.write_bytes(img_np, ext) return self.upload(data) def upload_path(self, img_path): data = open(img_path, 'rb').read() return self.upload(data) def upload(self, data): response = self.api.post('images.upload', data) return response.json()[ApiField.HASH] def upload_link(self, link): response = self.api.post('images.remote.upsert', {ApiField.LINK: link}) return response.json()[ApiField.ID] def upload_batch_paths(self, img_paths, progress_cb=None): MAX_BATCH_SIZE = 50 for batch_paths in batched(img_paths, MAX_BATCH_SIZE): content_dict = {} for idx, path in enumerate(batch_paths): content_dict["{}-file".format(idx)] = (str(idx), open(path, 'rb'), 'image/*') encoder = MultipartEncoder(fields=content_dict) self.api.post('images.bulk.upload', encoder) if progress_cb is not None: progress_cb(len(batch_paths)) def add(self, dataset_id, name, hash): response = self.api.post( 'images.add', { ApiField.DATASET_ID: dataset_id, ApiField.HASH: hash, ApiField.NAME: name }) return self._convert_json_info(response.json()) def add_link(self, dataset_id, name, link): response = self.api.post( 'images.add', { ApiField.DATASET_ID: dataset_id, ApiField.LINK: link, ApiField.NAME: name }) return self._convert_json_info(response.json()) def add_batch(self, dataset_id, names, hashes, progress_cb=None): # @TOD0: ApiField.NAME images = [{ 'title': name, ApiField.HASH: hash } for name, hash in zip(names, hashes)] response = self.api.post('images.bulk.add', { ApiField.DATASET_ID: dataset_id, ApiField.IMAGES: images }) if progress_cb is not None: progress_cb(len(images)) return [ self._convert_json_info(info_json) for info_json in response.json() ] def _convert_json_info(self, info: dict): if info is None: return None field_values = [] for field_name in self.__class__._info_sequence: if field_name == ApiField.EXT: continue field_values.append(info[field_name]) if field_name == ApiField.MIME: field_values.append(info[field_name].split('/')[1]) return self.__class__.Info._make(field_values)