def download_project(self, parent_dir, project, datasets, download_images=True): project_info = self.api.simple_request('GetProjectMeta', api_proto.Project, api_proto.Id(id=project.id)) pr_writer = sly.ProjectWriterFS(parent_dir, project_info.title) pr_meta = sly.ProjectMeta(sly.json_loads(project_info.meta)) pr_writer.write_meta(pr_meta) image_id_to_ds = {} for dataset in datasets: image_array = self.api.simple_request('GetDatasetImages', api_proto.ImageArray, api_proto.Id(id=dataset.id)) image_id_to_ds.update( {img_id: dataset.title for img_id in image_array.images}) if download_images is True: self._download_images(pr_writer, image_id_to_ds) self._download_annotations(pr_writer, image_id_to_ds)
def download_data_sources(self, only_meta=False): self.logger.info("download_data_sources started") data_sources = sly.get_data_sources(self.info['graph']) for proj, datasets in data_sources.items(): pr_name = proj pr_proto = self.api.simple_request( 'GetProjectByName', api_proto.Project, api_proto.Project(title=pr_name)) if pr_proto.id == -1: self.logger.critical('Project not found', extra={'project_name': pr_name}) raise RuntimeError('Project not found') datasets_proto_arr = [] if datasets != "*": for ds_name in datasets: ds_proto = self.api.simple_request( 'GetDatasetByName', api_proto.Dataset, api_proto.ProjectDataset( project=api_proto.Project(id=pr_proto.id), dataset=api_proto.Dataset(title=ds_name))) if ds_proto.id == -1: self.logger.critical('Dataset not found', extra={ 'project_id': pr_proto.id, 'project_title': pr_name, 'dataset_title': ds_name }) raise RuntimeError('Dataset not found') datasets_proto_arr.append( api_proto.Dataset(id=ds_proto.id, title=ds_name)) else: datasets_proto = self.api.simple_request( 'GetProjectDatasets', api_proto.DatasetArray, api_proto.Id(id=pr_proto.id)) datasets_proto_arr = datasets_proto.datasets if only_meta is True: project_info = self.api.simple_request( 'GetProjectMeta', api_proto.Project, api_proto.Id(id=pr_proto.id)) pr_writer = sly.ProjectWriterFS(self.dir_data, project_info.title) pr_meta = sly.ProjectMeta(sly.json_loads(project_info.meta)) pr_writer.write_meta(pr_meta) else: self.data_mgr.download_project( self.dir_data, pr_proto, datasets_proto_arr, download_images=self.download_images)
def download_import_files(self, task_id, data_dir): import_struct = self.api.simple_request('GetImportStructure', api_proto.ListFiles, api_proto.Id(id=task_id)) progress = sly.ProgressCounter(subtask_name='Downloading', total_cnt=len(import_struct.files), ext_logger=self.logger, report_limit=int( len(import_struct.files) / 10)) def close_fh(fh): fpath = fh.file_path if fh.close_and_check(): progress.iter_done_report() else: self.logger.warning('file was skipped while downloading', extra={'file_path': fpath}) file_handler = None for chunk in self.api.get_stream_with_data( 'GetImportFiles', api_proto.ChunkFile, api_proto.ImportRequest(task_id=task_id, files=import_struct.files)): new_fpath = chunk.file.path if new_fpath: # non-empty if file_handler is not None: close_fh(file_handler) real_fpath = osp.join(data_dir, new_fpath.lstrip('/')) self.logger.trace('download import file', extra={'file_path': real_fpath}) file_handler = ChunkedFileWriter(file_path=real_fpath) file_handler.write(chunk.chunk) close_fh(file_handler)
def upload_project(self, dir_results, pr_name, no_image_files): self.logger.info("upload_result_project started") root_path, project_name = sly.ProjectFS.split_dir_project(dir_results) project_fs = sly.ProjectFS.from_disk(root_path, project_name, by_annotations=True) project_meta = sly.ProjectMeta.from_dir(dir_results) project_meta_str = project_meta.to_json_str() project_id, remote_pr_name = self._create_project(pr_name, project_meta_str) ds_name_to_id = {} for local_ds_name in project_fs.pr_structure.datasets.keys(): ds_id, remote_ds_name = self._create_dataset(project_id, local_ds_name) ds_name_to_id[local_ds_name] = ds_id project_items = self._construct_project_items_to_upload(project_fs, project_id, ds_name_to_id) if len(project_items) == 0: raise RuntimeError('Empty result project') # upload images at first if not no_image_files: all_img_paths = [it.img_path for it in project_items] all_img_hashes = [it.ia_data['image_hash'] for it in project_items] self._write_images_to_agent_storage(all_img_paths, all_img_hashes) if constants.UPLOAD_RESULT_IMAGES: remote_images = self.api.simple_request('FindImagesExist', api_proto.ImagesHashes, api_proto.ImagesHashes(images_hashes=all_img_hashes)) img_hashes_to_upload = set(all_img_hashes) - set(remote_images.images_hashes) to_upload_imgs = list(filter(lambda x: x.ia_data['image_hash'] in img_hashes_to_upload, project_items)) img_paths = [it.img_path for it in to_upload_imgs] self.upload_images_to_remote( fpaths=img_paths, infos=[it.ia_data['img_proto_info'] for it in to_upload_imgs] ) # add images to project obtained_img_ids = self.api.simple_request('AddImages', api_proto.ImageArray, api_proto.ImagesInfo( infos=[it.ia_data['img_proto_info'] for it in project_items] )) # and upload anns self._upload_annotations_to_remote(project_id=project_id, img_ids=obtained_img_ids.images, img_names=[it.image_name for it in project_items], ann_paths=[it.ann_path for it in project_items]) self.api.simple_request('SetProjectFinished', api_proto.Empty, api_proto.Id(id=project_id)) return project_id