def upload_nn(self, nn_id, nn_hash):
local_service_log = {'nn_id': nn_id, 'nn_hash': nn_hash}
storage_nn_dir = self.storage.nns.check_storage_object(nn_hash)
if storage_nn_dir is None:
self.logger.critical('NN_NOT_FOUND', extra=local_service_log)
local_tar_path = os.path.join(constants.AGENT_TMP_DIR, sly.generate_random_string(30) + '.tar')
sly.archive_directory(storage_nn_dir, local_tar_path)
freader = ChunkedFileReader(local_tar_path, constants.NETW_CHUNK_SIZE)
progress = sly.ProgressCounter("Upload NN", freader.splitter.chunk_cnt, ext_logger=self.logger)
def chunk_generator():
for chunk_bytes in freader:
current_chunk = api_proto.Chunk(buffer=chunk_bytes, total_size=freader.file_size)
yield api_proto.ChunkModel(chunk=current_chunk,
model=api_proto.ModelDescription(id=nn_id, hash=nn_hash))
progress.iter_done_report()
try:
self.api.put_stream_with_data('UploadModel', api_proto.Empty, chunk_generator(),
addit_headers={'x-model-hash': nn_hash})
finally:
sly.silent_remove(local_tar_path)
self.logger.info('NN_UPLOADED', extra=local_service_log)
def upload_archive(self, dir_to_archive, archive_name):
local_tar_path = os.path.join(constants.AGENT_TMP_DIR,
sly.generate_random_string(30) + '.tar')
self.logger.info("PACK_TO_ARCHIVE ...")
sly.archive_directory(dir_to_archive, local_tar_path)
freader = ChunkedFileReader(local_tar_path, constants.NETW_CHUNK_SIZE)
progress = sly.ProgressCounter("Upload archive",
freader.splitter.chunk_cnt,
ext_logger=self.logger)
def chunk_generator():
for chunk_bytes in freader:
current_chunk = api_proto.Chunk(buffer=chunk_bytes,
total_size=freader.file_size)
yield current_chunk
progress.iter_done_report()
try:
self.api.put_stream_with_data(
'UploadArchive',
api_proto.Empty,
chunk_generator(),
addit_headers={'x-archive-name': archive_name})
finally:
sly.silent_remove(local_tar_path)
self.logger.info('ARCHIVE_UPLOADED',
extra={'archive_name': archive_name})
def download_import_files(self, task_id, data_dir):
import_struct = self.api.simple_request('GetImportStructure', api_proto.ListFiles, api_proto.Id(id=task_id))
progress = sly.ProgressCounter(subtask_name='Downloading',
total_cnt=len(import_struct.files),
ext_logger=self.logger,
report_limit=int(len(import_struct.files) / 10))
def close_fh(fh):
fpath = fh.file_path
if fh.close_and_check():
progress.iter_done_report()
else:
self.logger.warning('file was skipped while downloading', extra={'file_path': fpath})
file_handler = None
for chunk in self.api.get_stream_with_data('GetImportFiles',
api_proto.ChunkFile,
api_proto.ImportRequest(task_id=task_id, files=import_struct.files)):
new_fpath = chunk.file.path
if new_fpath: # non-empty
if file_handler is not None:
close_fh(file_handler)
real_fpath = osp.join(data_dir, new_fpath.lstrip('/'))
self.logger.trace('download import file', extra={'file_path': real_fpath})
file_handler = ChunkedFileWriter(file_path=real_fpath)
file_handler.write(chunk.chunk)
close_fh(file_handler)
def chunk_generator():
progress = sly.ProgressCounter('Upload annotations',
len(img_ids),
ext_logger=self.logger)
for batch_some in sly.batched(
list(zip(img_ids, img_names, ann_paths)),
constants.BATCH_SIZE_UPLOAD_ANNOTATIONS):
for img_id, img_name, ann_path in batch_some:
proto_img = api_proto.Image(id=img_id,
title=img_name,
project_id=project_id)
freader = ChunkedFileReader(ann_path,
constants.NETW_CHUNK_SIZE)
for chunk_bytes in freader:
current_chunk = api_proto.Chunk(
buffer=chunk_bytes, total_size=freader.file_size)
yield api_proto.ChunkImage(chunk=current_chunk,
image=proto_img)
self.logger.trace('annotation is uploaded',
extra={
'img_name': img_name,
'img_path': ann_path
})
progress.iter_done_report()
def _write_images_to_agent_storage(self, src_paths, hashes):
self.logger.info("WRITE_IMAGES_TO_LOCAL_CACHE",
extra={'cnt_images': len(src_paths)})
progress = sly.ProgressCounter("Add images to local storage",
len(src_paths),
ext_logger=self.logger)
self.storage.images.write_objects(zip(src_paths, hashes), progress)
def _download_annotations(self, pr_writer, image_id_to_ds):
progress = sly.ProgressCounter('Download annotations',
len(image_id_to_ds),
ext_logger=self.logger)
for batch_img_ids in sly.batched(
list(image_id_to_ds.keys()),
constants.BATCH_SIZE_DOWNLOAD_ANNOTATIONS):
for chunk in self.api.get_stream_with_data(
'DownloadAnnotations', api_proto.ChunkImage,
api_proto.ImageArray(images=batch_img_ids)):
img_id = chunk.image.id
ds_name = image_id_to_ds[img_id]
self.logger.trace('download_annotations',
extra={'img_id': img_id})
fh = ChunkedFileWriter(file_path=pr_writer.get_ann_path(
ds_name, chunk.image.title))
fh.write(chunk.chunk)
progress.iter_done_report()
if not fh.close_and_check():
self.logger.warning('ann was skipped while downloading',
extra={
'img_id': img_id,
'ann_path': fh.file_path
})
def _get_images_from_agent_storage(self, pr_writer, image_id_to_ds, img_infos):
progress = sly.ProgressCounter('Copying local images', len(img_infos), ext_logger=self.logger)
dst_paths_hashes = [(pr_writer.get_img_path(image_id_to_ds[info.id], info.title, info.ext), info.hash)
for info in img_infos]
written_img_paths = set(self.storage.images.read_objects(dst_paths_hashes, progress))
written_hashes = [p_h[1] for p_h in dst_paths_hashes if p_h[0] in written_img_paths]
return written_hashes
def _docker_pull(self):
self.logger.info('Docker image will be pulled',
extra={'image_name': self.docker_image_name})
_ = sly.ProgressCounter('Pulling image...', 1, ext_logger=self.logger)
pulled_img = self._docker_api.images.pull(self.docker_image_name)
self.logger.info(
'Docker image has been pulled',
extra={'pulled': {
'tags': pulled_img.tags,
'id': pulled_img.id
}})
def _download_images_from_remote(self, pr_writer, image_id_to_ds, img_infos):
if len(img_infos) == 0:
return
infos_with_paths = [(info, pr_writer.get_img_path(image_id_to_ds[info.id], info.title, info.ext))
for info in img_infos]
hash2path = {x[0].hash: x[1] for x in infos_with_paths} # for unique hashes
unique_hashes = list(hash2path.keys())
ready_paths = []
ready_hashes = []
progress = sly.ProgressCounter('Download remote images', len(unique_hashes), ext_logger=self.logger)
def close_fh(fh):
fpath = fh.file_path
if fh.close_and_check():
ready_paths.append(fpath)
ready_hashes.append(img_hash)
progress.iter_done_report()
else:
self.logger.warning('file was skipped while downloading',
extra={'img_path': fpath, 'img_hash': img_hash})
# download by unique hashes
for batch_img_hashes in sly.batched(unique_hashes, constants.BATCH_SIZE_DOWNLOAD_IMAGES):
file_handler = None
img_hash = None
for chunk in self.api.get_stream_with_data('DownloadImages',
api_proto.ChunkImage,
api_proto.ImagesHashes(images_hashes=batch_img_hashes)):
if chunk.image.hash: # non-empty hash means beginning of new image
if file_handler is not None:
close_fh(file_handler)
img_hash = chunk.image.hash
self.logger.trace('download_images', extra={'img_hash': img_hash})
dst_fpath = hash2path[img_hash]
file_handler = ChunkedFileWriter(file_path=dst_fpath)
file_handler.write(chunk.chunk)
close_fh(file_handler) # must be not None
# process non-unique hashes
for info, dst_path in infos_with_paths:
origin_path = hash2path[info.hash]
if (origin_path != dst_path) and osp.isfile(origin_path):
sly.ensure_base_path(dst_path)
sly.copy_file(origin_path, dst_path)
self._write_images_to_agent_storage(ready_paths, ready_hashes)
def _clean_unused_objects(self, name, storage, get_used_hashes_method):
self.logger.info('Will clean unused {}.'.format(name))
used_hashes_exts = self.data_mgr.download_object_hashes(
get_used_hashes_method)
persistent_objs = set((storage.get_storage_path(hash_, suffix), suffix)
for hash_, suffix in used_hashes_exts)
self.logger.info('Will scan existing {}.'.format(name))
found_objs = set(storage.list_objects())
cnt_extra = {
'found_obj_cnt': len(found_objs),
'persist_obj_cnt': len(persistent_objs)
}
self.logger.info('Existing {} are listed.'.format(name),
extra=cnt_extra)
missing_objs = persistent_objs - found_objs
cnt_extra = {**cnt_extra, 'miss_obj_cnt': len(missing_objs)}
if len(missing_objs) == 0:
self.logger.info('There are no missing persistent objects.',
extra=cnt_extra)
else:
self.logger.warn('Missing persistent objects are found.',
extra=cnt_extra)
temp_objs = found_objs - persistent_objs
cnt_extra = {**cnt_extra, 'temp_obj_cnt': len(temp_objs)}
self.logger.info('Temporary objects count is determined.',
extra=cnt_extra)
progress = sly.ProgressCounter(subtask_name='Cleaning {}'.format(name),
total_cnt=len(temp_objs),
ext_logger=self.logger)
removed_cnt = 0
for st_path, suffix in temp_objs:
if storage.remove_object(st_path, suffix):
removed_cnt += 1
progress.iter_done_report()
if removed_cnt != len(temp_objs):
self.logger.warn('Not all objects have been removed.')
self.logger.info('Unused {} are cleaned.'.format(name),
extra={
'temp_obj_cnt': len(temp_objs),
'removed_cnt': removed_cnt
})
def upload_images_to_remote(self, fpaths, infos):
progress = sly.ProgressCounter('Upload images', len(fpaths), ext_logger=self.logger)
for batch_paths_infos in sly.batched(list(zip(fpaths, infos)), constants.BATCH_SIZE_UPLOAD_IMAGES):
def chunk_generator():
for fpath, proto_img_info in batch_paths_infos:
self.logger.trace('image upload start', extra={'img_path': fpath})
freader = ChunkedFileReader(fpath, constants.NETW_CHUNK_SIZE)
for chunk_bytes in freader:
current_chunk = api_proto.Chunk(buffer=chunk_bytes, total_size=freader.file_size)
yield api_proto.ChunkImage(chunk=current_chunk, image=proto_img_info)
self.logger.trace('image uploaded', extra={'img_path': fpath})
progress.iter_done_report()
self.api.put_stream_with_data('UploadImages', api_proto.Empty, chunk_generator())
self.logger.debug('Batch of images has been sent.', extra={'batch_len': len(batch_paths_infos)})
def _exif_checking_step(self):
self.logger.info('Check orientation of images')
root_path, project_name = sly.ProjectFS.split_dir_project(self.dir_res_project)
project_fs = sly.ProjectFS.from_disk(root_path, project_name, by_annotations=True)
progress = sly.ProgressCounter('EXIF checking', project_fs.image_cnt, ext_logger=self.logger)
for descr in project_fs:
img_path = descr.img_path
exif_data = pyexiv2.metadata.ImageMetadata(img_path)
exif_data.read()
if exif_data.get_orientation() != 1:
raise RuntimeError('Wrong image orientation in EXIF. Image name: {}'.format(osp.basename(img_path)))
progress.iter_done_report()
def _clean_tasks_dir(self):
self.logger.info('Will remove obsolete tasks data.')
task_dir = constants.AGENT_TASKS_DIR
task_names = os.listdir(task_dir)
self.logger.info('Obtained list of subdirs.',
extra={'dir_cnt': len(task_names)})
progress = sly.ProgressCounter(subtask_name='Task dir checking',
total_cnt=len(task_names),
ext_logger=self.logger)
for subdir_n in task_names:
dir_task = osp.join(task_dir, subdir_n)
try:
TaskDirCleaner(dir_task).clean_forced()
except Exception:
self.logger.warn('Unable to delete task dir.',
extra={'dir_task': dir_task},
exc_info=True)
progress.iter_done_report()
self.logger.info('Obsolete tasks data has been removed.')
def chunk_generator():
progress = sly.ProgressCounter('Upload images',
len(fpaths),
ext_logger=self.logger)
for batch_paths_infos in sly.batched(
list(zip(fpaths, infos)),
constants.BATCH_SIZE_UPLOAD_IMAGES):
for fpath, proto_img_info in batch_paths_infos:
self.logger.trace('image upload start',
extra={'img_path': fpath})
freader = ChunkedFileReader(fpath,
constants.NETW_CHUNK_SIZE)
for chunk_bytes in freader:
current_chunk = api_proto.Chunk(
buffer=chunk_bytes, total_size=freader.file_size)
yield api_proto.ChunkImage(chunk=current_chunk,
image=proto_img_info)
self.logger.trace('image uploaded',
extra={'img_path': fpath})
progress.iter_done_report()
