def download_project(self,
parent_dir,
project,
datasets,
download_images=True):
project_info = self.api.simple_request('GetProjectMeta',
api_proto.Project,
api_proto.Id(id=project.id))
pr_writer = sly.ProjectWriterFS(parent_dir, project_info.title)
pr_meta = sly.ProjectMeta(sly.json_loads(project_info.meta))
pr_writer.write_meta(pr_meta)
image_id_to_ds = {}
for dataset in datasets:
image_array = self.api.simple_request('GetDatasetImages',
api_proto.ImageArray,
api_proto.Id(id=dataset.id))
image_id_to_ds.update(
{img_id: dataset.title
for img_id in image_array.images})
if download_images is True:
self._download_images(pr_writer, image_id_to_ds)
self._download_annotations(pr_writer, image_id_to_ds)
def download_data_sources(self, only_meta=False):
self.logger.info("download_data_sources started")
data_sources = sly.get_data_sources(self.info['graph'])
for proj, datasets in data_sources.items():
pr_name = proj
pr_proto = self.api.simple_request(
'GetProjectByName', api_proto.Project,
api_proto.Project(title=pr_name))
if pr_proto.id == -1:
self.logger.critical('Project not found',
extra={'project_name': pr_name})
raise RuntimeError('Project not found')
datasets_proto_arr = []
if datasets != "*":
for ds_name in datasets:
ds_proto = self.api.simple_request(
'GetDatasetByName', api_proto.Dataset,
api_proto.ProjectDataset(
project=api_proto.Project(id=pr_proto.id),
dataset=api_proto.Dataset(title=ds_name)))
if ds_proto.id == -1:
self.logger.critical('Dataset not found',
extra={
'project_id': pr_proto.id,
'project_title': pr_name,
'dataset_title': ds_name
})
raise RuntimeError('Dataset not found')
datasets_proto_arr.append(
api_proto.Dataset(id=ds_proto.id, title=ds_name))
else:
datasets_proto = self.api.simple_request(
'GetProjectDatasets', api_proto.DatasetArray,
api_proto.Id(id=pr_proto.id))
datasets_proto_arr = datasets_proto.datasets
if only_meta is True:
project_info = self.api.simple_request(
'GetProjectMeta', api_proto.Project,
api_proto.Id(id=pr_proto.id))
pr_writer = sly.ProjectWriterFS(self.dir_data,
project_info.title)
pr_meta = sly.ProjectMeta(sly.json_loads(project_info.meta))
pr_writer.write_meta(pr_meta)
else:
self.data_mgr.download_project(
self.dir_data,
pr_proto,
datasets_proto_arr,
download_images=self.download_images)
def download_import_files(self, task_id, data_dir):
import_struct = self.api.simple_request('GetImportStructure',
api_proto.ListFiles,
api_proto.Id(id=task_id))
progress = sly.ProgressCounter(subtask_name='Downloading',
total_cnt=len(import_struct.files),
ext_logger=self.logger,
report_limit=int(
len(import_struct.files) / 10))
def close_fh(fh):
fpath = fh.file_path
if fh.close_and_check():
progress.iter_done_report()
else:
self.logger.warning('file was skipped while downloading',
extra={'file_path': fpath})
file_handler = None
for chunk in self.api.get_stream_with_data(
'GetImportFiles', api_proto.ChunkFile,
api_proto.ImportRequest(task_id=task_id,
files=import_struct.files)):
new_fpath = chunk.file.path
if new_fpath: # non-empty
if file_handler is not None:
close_fh(file_handler)
real_fpath = osp.join(data_dir, new_fpath.lstrip('/'))
self.logger.trace('download import file',
extra={'file_path': real_fpath})
file_handler = ChunkedFileWriter(file_path=real_fpath)
file_handler.write(chunk.chunk)
close_fh(file_handler)
def upload_project(self, dir_results, pr_name, no_image_files):
self.logger.info("upload_result_project started")
root_path, project_name = sly.ProjectFS.split_dir_project(dir_results)
project_fs = sly.ProjectFS.from_disk(root_path, project_name, by_annotations=True)
project_meta = sly.ProjectMeta.from_dir(dir_results)
project_meta_str = project_meta.to_json_str()
project_id, remote_pr_name = self._create_project(pr_name, project_meta_str)
ds_name_to_id = {}
for local_ds_name in project_fs.pr_structure.datasets.keys():
ds_id, remote_ds_name = self._create_dataset(project_id, local_ds_name)
ds_name_to_id[local_ds_name] = ds_id
project_items = self._construct_project_items_to_upload(project_fs, project_id, ds_name_to_id)
if len(project_items) == 0:
raise RuntimeError('Empty result project')
# upload images at first
if not no_image_files:
all_img_paths = [it.img_path for it in project_items]
all_img_hashes = [it.ia_data['image_hash'] for it in project_items]
self._write_images_to_agent_storage(all_img_paths, all_img_hashes)
if constants.UPLOAD_RESULT_IMAGES:
remote_images = self.api.simple_request('FindImagesExist', api_proto.ImagesHashes,
api_proto.ImagesHashes(images_hashes=all_img_hashes))
img_hashes_to_upload = set(all_img_hashes) - set(remote_images.images_hashes)
to_upload_imgs = list(filter(lambda x: x.ia_data['image_hash'] in img_hashes_to_upload, project_items))
img_paths = [it.img_path for it in to_upload_imgs]
self.upload_images_to_remote(
fpaths=img_paths,
infos=[it.ia_data['img_proto_info'] for it in to_upload_imgs]
)
# add images to project
obtained_img_ids = self.api.simple_request('AddImages', api_proto.ImageArray, api_proto.ImagesInfo(
infos=[it.ia_data['img_proto_info'] for it in project_items]
))
# and upload anns
self._upload_annotations_to_remote(project_id=project_id,
img_ids=obtained_img_ids.images,
img_names=[it.image_name for it in project_items],
ann_paths=[it.ann_path for it in project_items])
self.api.simple_request('SetProjectFinished', api_proto.Empty, api_proto.Id(id=project_id))
return project_id