def upload_nn(self, nn_id, nn_hash): local_service_log = {'nn_id': nn_id, 'nn_hash': nn_hash} storage_nn_dir = self.storage.nns.check_storage_object(nn_hash) if storage_nn_dir is None: self.logger.critical('NN_NOT_FOUND', extra=local_service_log) local_tar_path = os.path.join(constants.AGENT_TMP_DIR, sly.generate_random_string(30) + '.tar') sly.archive_directory(storage_nn_dir, local_tar_path) freader = ChunkedFileReader(local_tar_path, constants.NETW_CHUNK_SIZE) progress = sly.ProgressCounter("Upload NN", freader.splitter.chunk_cnt, ext_logger=self.logger) def chunk_generator(): for chunk_bytes in freader: current_chunk = api_proto.Chunk(buffer=chunk_bytes, total_size=freader.file_size) yield api_proto.ChunkModel(chunk=current_chunk, model=api_proto.ModelDescription(id=nn_id, hash=nn_hash)) progress.iter_done_report() try: self.api.put_stream_with_data('UploadModel', api_proto.Empty, chunk_generator(), addit_headers={'x-model-hash': nn_hash}) finally: sly.silent_remove(local_tar_path) self.logger.info('NN_UPLOADED', extra=local_service_log)
def upload_archive(self, dir_to_archive, archive_name): local_tar_path = os.path.join(constants.AGENT_TMP_DIR, sly.generate_random_string(30) + '.tar') self.logger.info("PACK_TO_ARCHIVE ...") sly.archive_directory(dir_to_archive, local_tar_path) freader = ChunkedFileReader(local_tar_path, constants.NETW_CHUNK_SIZE) progress = sly.ProgressCounter("Upload archive", freader.splitter.chunk_cnt, ext_logger=self.logger) def chunk_generator(): for chunk_bytes in freader: current_chunk = api_proto.Chunk(buffer=chunk_bytes, total_size=freader.file_size) yield current_chunk progress.iter_done_report() try: self.api.put_stream_with_data( 'UploadArchive', api_proto.Empty, chunk_generator(), addit_headers={'x-archive-name': archive_name}) finally: sly.silent_remove(local_tar_path) self.logger.info('ARCHIVE_UPLOADED', extra={'archive_name': archive_name})
def download_import_files(self, task_id, data_dir): import_struct = self.api.simple_request('GetImportStructure', api_proto.ListFiles, api_proto.Id(id=task_id)) progress = sly.ProgressCounter(subtask_name='Downloading', total_cnt=len(import_struct.files), ext_logger=self.logger, report_limit=int(len(import_struct.files) / 10)) def close_fh(fh): fpath = fh.file_path if fh.close_and_check(): progress.iter_done_report() else: self.logger.warning('file was skipped while downloading', extra={'file_path': fpath}) file_handler = None for chunk in self.api.get_stream_with_data('GetImportFiles', api_proto.ChunkFile, api_proto.ImportRequest(task_id=task_id, files=import_struct.files)): new_fpath = chunk.file.path if new_fpath: # non-empty if file_handler is not None: close_fh(file_handler) real_fpath = osp.join(data_dir, new_fpath.lstrip('/')) self.logger.trace('download import file', extra={'file_path': real_fpath}) file_handler = ChunkedFileWriter(file_path=real_fpath) file_handler.write(chunk.chunk) close_fh(file_handler)
def chunk_generator(): progress = sly.ProgressCounter('Upload annotations', len(img_ids), ext_logger=self.logger) for batch_some in sly.batched( list(zip(img_ids, img_names, ann_paths)), constants.BATCH_SIZE_UPLOAD_ANNOTATIONS): for img_id, img_name, ann_path in batch_some: proto_img = api_proto.Image(id=img_id, title=img_name, project_id=project_id) freader = ChunkedFileReader(ann_path, constants.NETW_CHUNK_SIZE) for chunk_bytes in freader: current_chunk = api_proto.Chunk( buffer=chunk_bytes, total_size=freader.file_size) yield api_proto.ChunkImage(chunk=current_chunk, image=proto_img) self.logger.trace('annotation is uploaded', extra={ 'img_name': img_name, 'img_path': ann_path }) progress.iter_done_report()
def _write_images_to_agent_storage(self, src_paths, hashes): self.logger.info("WRITE_IMAGES_TO_LOCAL_CACHE", extra={'cnt_images': len(src_paths)}) progress = sly.ProgressCounter("Add images to local storage", len(src_paths), ext_logger=self.logger) self.storage.images.write_objects(zip(src_paths, hashes), progress)
def _download_annotations(self, pr_writer, image_id_to_ds): progress = sly.ProgressCounter('Download annotations', len(image_id_to_ds), ext_logger=self.logger) for batch_img_ids in sly.batched( list(image_id_to_ds.keys()), constants.BATCH_SIZE_DOWNLOAD_ANNOTATIONS): for chunk in self.api.get_stream_with_data( 'DownloadAnnotations', api_proto.ChunkImage, api_proto.ImageArray(images=batch_img_ids)): img_id = chunk.image.id ds_name = image_id_to_ds[img_id] self.logger.trace('download_annotations', extra={'img_id': img_id}) fh = ChunkedFileWriter(file_path=pr_writer.get_ann_path( ds_name, chunk.image.title)) fh.write(chunk.chunk) progress.iter_done_report() if not fh.close_and_check(): self.logger.warning('ann was skipped while downloading', extra={ 'img_id': img_id, 'ann_path': fh.file_path })
def _get_images_from_agent_storage(self, pr_writer, image_id_to_ds, img_infos): progress = sly.ProgressCounter('Copying local images', len(img_infos), ext_logger=self.logger) dst_paths_hashes = [(pr_writer.get_img_path(image_id_to_ds[info.id], info.title, info.ext), info.hash) for info in img_infos] written_img_paths = set(self.storage.images.read_objects(dst_paths_hashes, progress)) written_hashes = [p_h[1] for p_h in dst_paths_hashes if p_h[0] in written_img_paths] return written_hashes
def _docker_pull(self): self.logger.info('Docker image will be pulled', extra={'image_name': self.docker_image_name}) _ = sly.ProgressCounter('Pulling image...', 1, ext_logger=self.logger) pulled_img = self._docker_api.images.pull(self.docker_image_name) self.logger.info( 'Docker image has been pulled', extra={'pulled': { 'tags': pulled_img.tags, 'id': pulled_img.id }})
def _download_images_from_remote(self, pr_writer, image_id_to_ds, img_infos): if len(img_infos) == 0: return infos_with_paths = [(info, pr_writer.get_img_path(image_id_to_ds[info.id], info.title, info.ext)) for info in img_infos] hash2path = {x[0].hash: x[1] for x in infos_with_paths} # for unique hashes unique_hashes = list(hash2path.keys()) ready_paths = [] ready_hashes = [] progress = sly.ProgressCounter('Download remote images', len(unique_hashes), ext_logger=self.logger) def close_fh(fh): fpath = fh.file_path if fh.close_and_check(): ready_paths.append(fpath) ready_hashes.append(img_hash) progress.iter_done_report() else: self.logger.warning('file was skipped while downloading', extra={'img_path': fpath, 'img_hash': img_hash}) # download by unique hashes for batch_img_hashes in sly.batched(unique_hashes, constants.BATCH_SIZE_DOWNLOAD_IMAGES): file_handler = None img_hash = None for chunk in self.api.get_stream_with_data('DownloadImages', api_proto.ChunkImage, api_proto.ImagesHashes(images_hashes=batch_img_hashes)): if chunk.image.hash: # non-empty hash means beginning of new image if file_handler is not None: close_fh(file_handler) img_hash = chunk.image.hash self.logger.trace('download_images', extra={'img_hash': img_hash}) dst_fpath = hash2path[img_hash] file_handler = ChunkedFileWriter(file_path=dst_fpath) file_handler.write(chunk.chunk) close_fh(file_handler) # must be not None # process non-unique hashes for info, dst_path in infos_with_paths: origin_path = hash2path[info.hash] if (origin_path != dst_path) and osp.isfile(origin_path): sly.ensure_base_path(dst_path) sly.copy_file(origin_path, dst_path) self._write_images_to_agent_storage(ready_paths, ready_hashes)
def _clean_unused_objects(self, name, storage, get_used_hashes_method): self.logger.info('Will clean unused {}.'.format(name)) used_hashes_exts = self.data_mgr.download_object_hashes( get_used_hashes_method) persistent_objs = set((storage.get_storage_path(hash_, suffix), suffix) for hash_, suffix in used_hashes_exts) self.logger.info('Will scan existing {}.'.format(name)) found_objs = set(storage.list_objects()) cnt_extra = { 'found_obj_cnt': len(found_objs), 'persist_obj_cnt': len(persistent_objs) } self.logger.info('Existing {} are listed.'.format(name), extra=cnt_extra) missing_objs = persistent_objs - found_objs cnt_extra = {**cnt_extra, 'miss_obj_cnt': len(missing_objs)} if len(missing_objs) == 0: self.logger.info('There are no missing persistent objects.', extra=cnt_extra) else: self.logger.warn('Missing persistent objects are found.', extra=cnt_extra) temp_objs = found_objs - persistent_objs cnt_extra = {**cnt_extra, 'temp_obj_cnt': len(temp_objs)} self.logger.info('Temporary objects count is determined.', extra=cnt_extra) progress = sly.ProgressCounter(subtask_name='Cleaning {}'.format(name), total_cnt=len(temp_objs), ext_logger=self.logger) removed_cnt = 0 for st_path, suffix in temp_objs: if storage.remove_object(st_path, suffix): removed_cnt += 1 progress.iter_done_report() if removed_cnt != len(temp_objs): self.logger.warn('Not all objects have been removed.') self.logger.info('Unused {} are cleaned.'.format(name), extra={ 'temp_obj_cnt': len(temp_objs), 'removed_cnt': removed_cnt })
def upload_images_to_remote(self, fpaths, infos): progress = sly.ProgressCounter('Upload images', len(fpaths), ext_logger=self.logger) for batch_paths_infos in sly.batched(list(zip(fpaths, infos)), constants.BATCH_SIZE_UPLOAD_IMAGES): def chunk_generator(): for fpath, proto_img_info in batch_paths_infos: self.logger.trace('image upload start', extra={'img_path': fpath}) freader = ChunkedFileReader(fpath, constants.NETW_CHUNK_SIZE) for chunk_bytes in freader: current_chunk = api_proto.Chunk(buffer=chunk_bytes, total_size=freader.file_size) yield api_proto.ChunkImage(chunk=current_chunk, image=proto_img_info) self.logger.trace('image uploaded', extra={'img_path': fpath}) progress.iter_done_report() self.api.put_stream_with_data('UploadImages', api_proto.Empty, chunk_generator()) self.logger.debug('Batch of images has been sent.', extra={'batch_len': len(batch_paths_infos)})
def _exif_checking_step(self): self.logger.info('Check orientation of images') root_path, project_name = sly.ProjectFS.split_dir_project(self.dir_res_project) project_fs = sly.ProjectFS.from_disk(root_path, project_name, by_annotations=True) progress = sly.ProgressCounter('EXIF checking', project_fs.image_cnt, ext_logger=self.logger) for descr in project_fs: img_path = descr.img_path exif_data = pyexiv2.metadata.ImageMetadata(img_path) exif_data.read() if exif_data.get_orientation() != 1: raise RuntimeError('Wrong image orientation in EXIF. Image name: {}'.format(osp.basename(img_path))) progress.iter_done_report()
def _clean_tasks_dir(self): self.logger.info('Will remove obsolete tasks data.') task_dir = constants.AGENT_TASKS_DIR task_names = os.listdir(task_dir) self.logger.info('Obtained list of subdirs.', extra={'dir_cnt': len(task_names)}) progress = sly.ProgressCounter(subtask_name='Task dir checking', total_cnt=len(task_names), ext_logger=self.logger) for subdir_n in task_names: dir_task = osp.join(task_dir, subdir_n) try: TaskDirCleaner(dir_task).clean_forced() except Exception: self.logger.warn('Unable to delete task dir.', extra={'dir_task': dir_task}, exc_info=True) progress.iter_done_report() self.logger.info('Obsolete tasks data has been removed.')
def chunk_generator(): progress = sly.ProgressCounter('Upload images', len(fpaths), ext_logger=self.logger) for batch_paths_infos in sly.batched( list(zip(fpaths, infos)), constants.BATCH_SIZE_UPLOAD_IMAGES): for fpath, proto_img_info in batch_paths_infos: self.logger.trace('image upload start', extra={'img_path': fpath}) freader = ChunkedFileReader(fpath, constants.NETW_CHUNK_SIZE) for chunk_bytes in freader: current_chunk = api_proto.Chunk( buffer=chunk_bytes, total_size=freader.file_size) yield api_proto.ChunkImage(chunk=current_chunk, image=proto_img_info) self.logger.trace('image uploaded', extra={'img_path': fpath}) progress.iter_done_report()