def download_project(self,
parent_dir,
project,
datasets,
download_images=True):
project_info = self.api.simple_request('GetProjectMeta',
api_proto.Project,
api_proto.Id(id=project.id))
pr_writer = sly.ProjectWriterFS(parent_dir, project_info.title)
pr_meta = sly.ProjectMeta(sly.json_loads(project_info.meta))
pr_writer.write_meta(pr_meta)
image_id_to_ds = {}
for dataset in datasets:
image_array = self.api.simple_request('GetDatasetImages',
api_proto.ImageArray,
api_proto.Id(id=dataset.id))
image_id_to_ds.update(
{img_id: dataset.title
for img_id in image_array.images})
if download_images is True:
self._download_images(pr_writer, image_id_to_ds)
self._download_annotations(pr_writer, image_id_to_ds)
def parse_log_line(cls, log_line):
msg = ''
try:
jlog = json_loads(log_line)
msg = jlog['message']
del jlog['message']
except (KeyError, ValueError, TypeError):
jlog = {'cont_msg': str(log_line), 'event_type': EventType.LOGJ}
if 'event_type' not in jlog:
jlog['event_type'] = EventType.LOGJ
return msg, jlog
def download_data_sources(self, only_meta=False):
self.logger.info("download_data_sources started")
data_sources = sly.get_data_sources(self.info['graph'])
for proj, datasets in data_sources.items():
pr_name = proj
pr_proto = self.api.simple_request(
'GetProjectByName', api_proto.Project,
api_proto.Project(title=pr_name))
if pr_proto.id == -1:
self.logger.critical('Project not found',
extra={'project_name': pr_name})
raise RuntimeError('Project not found')
datasets_proto_arr = []
if datasets != "*":
for ds_name in datasets:
ds_proto = self.api.simple_request(
'GetDatasetByName', api_proto.Dataset,
api_proto.ProjectDataset(
project=api_proto.Project(id=pr_proto.id),
dataset=api_proto.Dataset(title=ds_name)))
if ds_proto.id == -1:
self.logger.critical('Dataset not found',
extra={
'project_id': pr_proto.id,
'project_title': pr_name,
'dataset_title': ds_name
})
raise RuntimeError('Dataset not found')
datasets_proto_arr.append(
api_proto.Dataset(id=ds_proto.id, title=ds_name))
else:
datasets_proto = self.api.simple_request(
'GetProjectDatasets', api_proto.DatasetArray,
api_proto.Id(id=pr_proto.id))
datasets_proto_arr = datasets_proto.datasets
if only_meta is True:
project_info = self.api.simple_request(
'GetProjectMeta', api_proto.Project,
api_proto.Id(id=pr_proto.id))
pr_writer = sly.ProjectWriterFS(self.dir_data,
project_info.title)
pr_meta = sly.ProjectMeta(sly.json_loads(project_info.meta))
pr_writer.write_meta(pr_meta)
else:
self.data_mgr.download_project(
self.dir_data,
pr_proto,
datasets_proto_arr,
download_images=self.download_images)
def task_main_func(self):
try:
self.logger.info('IMAGE_STREAMER_INITIALIZED')
for gen_event in self.api.get_endless_stream(
'GetGeneralEventsStream', api_proto.GeneralEvent,
api_proto.Empty()):
event_obj = {
'request_id': gen_event.request_id,
'data': sly.json_loads(gen_event.data.decode('utf-8')),
}
self.logger.debug('GET_STREAM_IMAGE_CALL', extra=event_obj)
self.thread_pool.submit(sly.function_wrapper_nofail,
self.stream_image, event_obj)
except Exception:
extra = {
'event_type': EventType.TASK_CRASHED,
'error': traceback.format_exc().split('\n')
}
self.logger.critical("IMAGE_STREAMER_CRASHED", extra={**extra})
def get_new_task(self):
for task in self.api.get_endless_stream('GetNewTask', api_proto.Task,
api_proto.Empty()):
task_msg = sly.json_loads(task.data)
self.logger.debug('GET_NEW_TASK', extra={'task_msg': task_msg})
self.start_task(task_msg)