def download_project(self, parent_dir, project, datasets, download_images=True): project_info = self.api.simple_request('GetProjectMeta', api_proto.Project, api_proto.Id(id=project.id)) pr_writer = sly.ProjectWriterFS(parent_dir, project_info.title) pr_meta = sly.ProjectMeta(sly.json_loads(project_info.meta)) pr_writer.write_meta(pr_meta) image_id_to_ds = {} for dataset in datasets: image_array = self.api.simple_request('GetDatasetImages', api_proto.ImageArray, api_proto.Id(id=dataset.id)) image_id_to_ds.update( {img_id: dataset.title for img_id in image_array.images}) if download_images is True: self._download_images(pr_writer, image_id_to_ds) self._download_annotations(pr_writer, image_id_to_ds)
def parse_log_line(cls, log_line): msg = '' try: jlog = json_loads(log_line) msg = jlog['message'] del jlog['message'] except (KeyError, ValueError, TypeError): jlog = {'cont_msg': str(log_line), 'event_type': EventType.LOGJ} if 'event_type' not in jlog: jlog['event_type'] = EventType.LOGJ return msg, jlog
def download_data_sources(self, only_meta=False): self.logger.info("download_data_sources started") data_sources = sly.get_data_sources(self.info['graph']) for proj, datasets in data_sources.items(): pr_name = proj pr_proto = self.api.simple_request( 'GetProjectByName', api_proto.Project, api_proto.Project(title=pr_name)) if pr_proto.id == -1: self.logger.critical('Project not found', extra={'project_name': pr_name}) raise RuntimeError('Project not found') datasets_proto_arr = [] if datasets != "*": for ds_name in datasets: ds_proto = self.api.simple_request( 'GetDatasetByName', api_proto.Dataset, api_proto.ProjectDataset( project=api_proto.Project(id=pr_proto.id), dataset=api_proto.Dataset(title=ds_name))) if ds_proto.id == -1: self.logger.critical('Dataset not found', extra={ 'project_id': pr_proto.id, 'project_title': pr_name, 'dataset_title': ds_name }) raise RuntimeError('Dataset not found') datasets_proto_arr.append( api_proto.Dataset(id=ds_proto.id, title=ds_name)) else: datasets_proto = self.api.simple_request( 'GetProjectDatasets', api_proto.DatasetArray, api_proto.Id(id=pr_proto.id)) datasets_proto_arr = datasets_proto.datasets if only_meta is True: project_info = self.api.simple_request( 'GetProjectMeta', api_proto.Project, api_proto.Id(id=pr_proto.id)) pr_writer = sly.ProjectWriterFS(self.dir_data, project_info.title) pr_meta = sly.ProjectMeta(sly.json_loads(project_info.meta)) pr_writer.write_meta(pr_meta) else: self.data_mgr.download_project( self.dir_data, pr_proto, datasets_proto_arr, download_images=self.download_images)
def task_main_func(self): try: self.logger.info('IMAGE_STREAMER_INITIALIZED') for gen_event in self.api.get_endless_stream( 'GetGeneralEventsStream', api_proto.GeneralEvent, api_proto.Empty()): event_obj = { 'request_id': gen_event.request_id, 'data': sly.json_loads(gen_event.data.decode('utf-8')), } self.logger.debug('GET_STREAM_IMAGE_CALL', extra=event_obj) self.thread_pool.submit(sly.function_wrapper_nofail, self.stream_image, event_obj) except Exception: extra = { 'event_type': EventType.TASK_CRASHED, 'error': traceback.format_exc().split('\n') } self.logger.critical("IMAGE_STREAMER_CRASHED", extra={**extra})
def get_new_task(self): for task in self.api.get_endless_stream('GetNewTask', api_proto.Task, api_proto.Empty()): task_msg = sly.json_loads(task.data) self.logger.debug('GET_NEW_TASK', extra={'task_msg': task_msg}) self.start_task(task_msg)