def get_image_and_ann(): mkdir(image_dir_path) mkdir(ann_dir) image_path = os.path.join(image_dir_path, image_name) api.image.download_path(image_id, image_path) image_ext_to_png(image_path) mask_color, mask_label, poly_json = from_ann_to_cityscapes_mask( ann, name2id, app_logger, train_val_flag) # dump_json_file(poly_json, # os.path.join(ann_dir, get_file_name(base_image_name) + cityscapes_polygons_suffix)) # write( # os.path.join(ann_dir, # get_file_name(base_image_name) + cityscapes_color_suffix), mask_color) # write( # os.path.join(ann_dir, # get_file_name(base_image_name) + cityscapes_labels_suffix), mask_label) dump_json_file( poly_json, os.path.join( ann_dir, get_file_name(base_image_name).replace('_leftImg8bit', '') + cityscapes_polygons_suffix)) write( os.path.join( ann_dir, get_file_name(base_image_name).replace('_leftImg8bit', '') + cityscapes_color_suffix), mask_color) write( os.path.join( ann_dir, get_file_name(base_image_name).replace('_leftImg8bit', '') + cityscapes_labels_suffix), mask_label)
def _create(self): if dir_exists(self.directory): raise RuntimeError( "Can not create new project {!r}. Directory {!r} already exists" .format(self.name, self.directory)) mkdir(self.directory) self.set_meta(ProjectMeta())
def save_project_as_pascal_voc_detection(save_path, project: Project): # Create root pascal 'datasets' folders for dataset in project.datasets: pascal_dataset_path = os.path.join(save_path, dataset.name) pascal_dataset_relative_path = os.path.relpath(pascal_dataset_path, save_path) images_dir = os.path.join(pascal_dataset_path, 'JPEGImages') anns_dir = os.path.join(pascal_dataset_path, 'Annotations') lists_dir = os.path.join(pascal_dataset_path, 'ImageSets/Layout') fs_utils.mkdir(pascal_dataset_path) for subdir in [ 'ImageSets', # Train list, Val list, etc. 'ImageSets/Layout', 'Annotations', 'JPEGImages' ]: fs_utils.mkdir(os.path.join(pascal_dataset_path, subdir)) samples_by_tags = defaultdict(list) # TRAIN: [img_1, img2, ..] for item_name in dataset: img_path, ann_path = dataset.get_item_paths(item_name) no_ext_name = fs_utils.get_file_name(item_name) pascal_img_path = os.path.join(images_dir, no_ext_name + OUT_IMG_EXT) pascal_ann_path = os.path.join(anns_dir, no_ext_name + XML_EXT) if item_name.endswith(OUT_IMG_EXT): fs_utils.copy_file(img_path, pascal_img_path) else: img = image_utils.read(img_path) image_utils.write(pascal_img_path, img) ann = Annotation.load_json_file(ann_path, project_meta=project.meta) # Read tags for images lists generation for tag in ann.img_tags: samples_by_tags[tag.name].append( (no_ext_name, len(ann.labels))) writer = pascal_voc_writer.Writer( path=pascal_dataset_relative_path, width=ann.img_size[1], height=ann.img_size[0]) for label in ann.labels: obj_class = label.obj_class rect: Rectangle = label.geometry.to_bbox() writer.addObject(name=obj_class.name, xmin=rect.left, ymin=rect.top, xmax=rect.right, ymax=rect.bottom) writer.save(pascal_ann_path) save_images_lists(lists_dir, samples_by_tags)
def _create(self): if dir_exists(self.directory): if len(list_files_recursively(self.directory)) > 0: raise RuntimeError( "Cannot create new project {!r}. Directory {!r} already exists and is not empty" .format(self.name, self.directory)) else: mkdir(self.directory) self.set_meta(ProjectMeta())
def _create(self): ''' Creates a leaf directory and empty meta.json file. Generate exception error if project directory already exists and is not empty. ''' if dir_exists(self.directory): if len(list_files_recursively(self.directory)) > 0: raise RuntimeError( "Cannot create new project {!r}. Directory {!r} already exists and is not empty" .format(self.name, self.directory)) else: mkdir(self.directory) self.set_meta(ProjectMeta())
def __init__(self, logger=None, task_id=None, server_address=None, agent_token=None, ignore_errors=False, ignore_task_id=False): self._ignore_task_id = ignore_task_id self.logger = take_with_default(logger, default_logger) self._ignore_errors = ignore_errors self.task_id = take_with_default(task_id, os.environ["TASK_ID"]) self.server_address = take_with_default(server_address, os.environ[SERVER_ADDRESS]) self.agent_token = take_with_default(agent_token, os.environ[AGENT_TOKEN]) self.public_api = Api.from_env(ignore_task_id=self._ignore_task_id) self._app_url = self.public_api.app.get_url(self.task_id) self._session_dir = "/sessions/{}".format(self.task_id) debug_app_dir = os.environ.get("DEBUG_APP_DIR", "") if debug_app_dir != "": self._session_dir = debug_app_dir mkdir(self.data_dir) self.api = AgentAPI(token=self.agent_token, server_address=self.server_address, ext_logger=self.logger) self.api.add_to_metadata('x-task-id', str(self.task_id)) self.callbacks = {} self.processing_queue = queue.Queue() #(maxsize=self.QUEUE_MAX_SIZE) self.logger.debug('App is created', extra={ "task_id": self.task_id, "server_address": self.server_address }) self._ignore_stop_for_debug = False self._error = None self.stop_event = asyncio.Event() self.executor = concurrent.futures.ThreadPoolExecutor() self.loop = asyncio.get_event_loop() # May want to catch other signals too signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT, signal.SIGQUIT) for s in signals: self.loop.add_signal_handler( s, lambda s=s: asyncio.create_task(self._shutdown(signal=s))) # comment out the line below to see how unhandled exceptions behave self.loop.set_exception_handler(self.handle_exception)
def _create(self): mkdir(self.ann_dir) mkdir(self.img_dir)
from supervisely_lib.io.fs import mkdir my_app = sly.AppService() api: sly.Api = my_app.public_api task_id = os.environ["TASK_ID"] user_id = os.environ["context.userId"] team_id = int(os.environ['context.teamId']) workspace_id = int(os.environ['context.workspaceId']) project_id = int(os.environ['context.projectId']) user = api.user.get_info_by_id(user_id) project = api.project.get_info_by_id(project_id) meta_json = api.project.get_meta(project_id) meta = sly.ProjectMeta.from_json(meta_json) storage_dir = os.path.join(my_app.data_dir, "storage_dir") coco_base_dir = os.path.join(storage_dir, project.name) sly_base_dir = os.path.join(storage_dir, "sly_base_dir") mkdir(storage_dir, True) mkdir(coco_base_dir) mkdir(sly_base_dir) isObjectDetection = True isStuffSegmentation = False # isKeypointDetection = False # isPanopticSegmentation = False # isImageCaptioning = False
PROJECT_ID = int(os.environ["modal.state.slyProjectId"]) LOG_LEVEL = str(os.environ["LOG_LEVEL"]) OPTIONS = os.environ['modal.state.Options'] BATCH_SIZE = int(os.environ['modal.state.batchSize']) SELECTED_DATASETS = json.loads(os.environ["modal.state.selectedDatasets"].replace("'", '"')) ALL_DATASETS = os.getenv("modal.state.allDatasets").lower() in ('true', '1', 't') if ALL_DATASETS: SELECTED_DATASETS = [dataset.name for dataset in api.dataset.get_list(PROJECT_ID)] need_download_threshold = 0.15 storage_dir = os.path.join(my_app.data_dir, "sly_base_sir") mkdir(storage_dir, True) video_dir = os.path.join(storage_dir, "video") mkdir(video_dir) img_dir = os.path.join(storage_dir, "images") mkdir(img_dir) project = api.project.get_info_by_id(PROJECT_ID) if project is None: raise RuntimeError("Project {!r} not found".format(project.name)) if project.type != str(sly.ProjectType.VIDEOS): raise TypeError("Project type is {!r}, but have to be {!r}".format(project.type, sly.ProjectType.VIDEOS)) meta_json = api.project.get_meta(project.id) meta = sly.ProjectMeta.from_json(meta_json) if "object_id" not in [tag.name for tag in meta.tag_metas]:
def convert_project(dest_dir, result_dir, app_logger): datasets_paths = glob(dest_dir + "/*/") if len(datasets_paths) == 0: g.logger.warn('There are no datasets in project') meta_json = sly.json.load_json_file(os.path.join(dest_dir, 'meta.json')) meta = sly.ProjectMeta.from_json(meta_json) for ds_path in datasets_paths: ds_name = ds_path.split('/')[-2] anns_paths = glob(ds_path + "ann" + "/*") progress = sly.Progress('Processing Video', len(anns_paths), app_logger) for ann_path in anns_paths: ann_json = sly.json.load_json_file(ann_path) ann = sly.VideoAnnotation.from_json(ann_json, meta) video_name = sly.io.fs.get_file_name(ann_path) video_path = os.path.join(ds_path, "video", video_name) video_info = sly.video.get_info(video_path)['streams'][0] curr_objs_geometry_types = [obj.obj_class.geometry_type for obj in ann.objects] if os.environ['modal.state.shapes'] == "false" and Rectangle not in curr_objs_geometry_types: g.logger.warn('Video {} does not contain figures with shape Rectangle'.format(video_name)) continue if len(ann.figures) > 0: result_images = os.path.join(result_dir, ds_name, "train", get_file_name(video_name), g.images_dir_name) result_anns = os.path.join(result_dir, ds_name, "train", get_file_name(video_name), g.ann_dir_name) seq_path = os.path.join(result_dir, ds_name, "train", get_file_name(video_name), g.seq_name) mkdir(result_images) mkdir(result_anns) if len(ann.figures) == 0: result_images = os.path.join(result_dir, ds_name, "test", get_file_name(video_name), g.images_dir_name) seq_path = os.path.join(result_dir, ds_name, "test", get_file_name(video_name), g.seq_name) mkdir(result_images) with open(seq_path, 'a') as f: f.write('[Sequence]\n') f.write('name={}\n'.format(get_file_name(video_name))) f.write('imDir={}\n'.format(g.images_dir_name)) f.write('frameRate={}\n'.format(round(1 / video_info['framesToTimecodes'][1]))) f.write('seqLength={}\n'.format(video_info['framesCount'])) f.write('imWidth={}\n'.format(video_info['width'])) f.write('imHeight={}\n'.format(video_info['height'])) f.write('imExt={}\n'.format(g.image_ext)) id_to_video_obj = {} for idx, curr_video_obj in enumerate(ann.objects): id_to_video_obj[curr_video_obj] = idx + 1 for frame_index, frame in enumerate(ann.frames): for figure in frame.figures: if os.environ['modal.state.shapes'] == "false" and figure.video_object.obj_class.geometry_type != Rectangle: continue rectangle_geom = figure.geometry.to_bbox() left = rectangle_geom.left top = rectangle_geom.top width = rectangle_geom.width height = rectangle_geom.height conf_val = 1 for curr_tag in figure.video_object.tags: if g.conf_tag_name == curr_tag.name and ( curr_tag.frame_range is None or frame_index in range(curr_tag.frame_range[0], curr_tag.frame_range[1] + 1)): conf_val = 0 curr_gt_data = '{},{},{},{},{},{},{},{},{},{}\n'.format(frame_index + 1, id_to_video_obj[figure.video_object], left, top, width - 1, height - 1, conf_val, -1, -1, -1) filename = 'gt_{}.txt'.format(figure.parent_object.obj_class.name) with open(os.path.join(result_anns, filename), 'a') as f: # gt_path f.write(curr_gt_data) if frame_index == ann.frames_count: break vidcap = cv2.VideoCapture(video_path) success, image = vidcap.read() count = 1 while success: image_name = str(count).zfill(6) + g.image_ext image_path = os.path.join(result_images, image_name) cv2.imwrite(image_path, image) success, image = vidcap.read() count += 1 progress.iter_done_report()
def _prepare_next_dir(self): self._idx += 1 self._subdir = '{:08}'.format(self._idx) self._odir = os.path.join(self._base_out_dir, self._subdir) sly_fs.mkdir(self._odir)
def _create(self): ''' Creates a leaf directory and all intermediate ones for items and annatations. ''' mkdir(self.ann_dir) mkdir(self.item_dir)
def export_coco(api: sly.Api, task_id, context, state, app_logger): datasets = [ds for ds in api.dataset.get_list(g.project_id)] for dataset in datasets: coco_dataset_dir = os.path.join(g.coco_base_dir, dataset.name) mkdir(os.path.join(coco_dataset_dir)) ann_dir = os.path.join(g.coco_base_dir, 'annotations') mkdir(ann_dir) images = api.image.get_list(dataset.id) for batch in sly.batched(images): image_ids = [image_info.id for image_info in batch] image_names = [image_info.name for image_info in batch] image_paths = [ os.path.join(coco_dataset_dir, image_info.name) for image_info in batch ] api.image.download_paths(dataset.id, image_ids, image_paths) ann_infos = api.annotation.download_batch(dataset.id, image_ids) anns = [ sly.Annotation.from_json(x.annotation, g.meta) for x in ann_infos ] meta = convert_geometry.prepare_meta(g.meta) new_anns = [ convert_geometry.convert_annotation(ann, meta) for ann in anns ] data = dict( info=dict( description=None, url=None, version=1.0, year=dataset.created_at[:4], contributor=g.user.name, date_created=dataset.created_at, ), licenses=[dict( url=None, id=0, name=None, )], images=[ # license, url, file_name, height, width, date_captured, id ], type="instances", annotations=[ # segmentation, area, iscrowd, image_id, bbox, category_id, id ], categories=[ # supercategory, id, name ], ) for image_info, ann in zip(batch, new_anns): data["images"].append( dict( license=None, url=image_info. full_storage_url, # coco_url, flickr_url filename=image_info.name, height=image_info.height, width=image_info.width, date_captured=image_info.created_at, id=image_info.id, )) for label in ann.labels: segmentation = label.geometry.to_json( )["points"]["exterior"] segmentation = [ coord for sublist in segmentation for coord in sublist ] bbox = label.geometry.to_bbox().to_json( )["points"]["exterior"] bbox = [coord for sublist in bbox for coord in sublist] x, y, max_x, max_y = bbox width = max_x - x height = max_y - y bbox = (x, y, width, height) data["annotations"].append( dict( segmentation=[segmentation], area=label.geometry.area, # wrong? iscrowd=0, image_id=image_info.id, bbox=bbox, category_id=None, id=None, # label.id? )) data["categories"].append( dict(supercategory=None, id=None, name=label.obj_class.name)) dump_json_file(data, os.path.join(ann_dir, f"instances_{dataset.name}.json")) g.my_app.stop()
import supervisely_lib as sly from supervisely_lib.io.fs import mkdir my_app = sly.AppService() api: sly.Api = my_app.public_api TEAM_ID = int(os.environ['context.teamId']) WORKSPACE_ID = int(os.environ['context.workspaceId']) PROJECT_ID = int(os.environ['modal.state.slyProjectId']) TASK_ID = int(os.environ["TASK_ID"]) images_dir_name = 'img1' ann_dir_name = 'gt' image_ext = '.jpg' seq_name = 'seqinfo.ini' conf_tag_name = 'ignore_conf' logger = sly.logger project = api.project.get_info_by_id(PROJECT_ID) project_name = project.name archive_name = '{}_{}_{}.tar.gz'.format(TASK_ID, PROJECT_ID, project_name) result_archive = os.path.join(my_app.data_dir, archive_name) storage_dir = os.path.join(my_app.data_dir, "mot_exporter") mot_base_dir = os.path.join(storage_dir, "mot_base_dir") sly_base_dir = os.path.join(storage_dir, "supervisely") mkdir(storage_dir, remove_content_if_exists=True) mkdir(sly_base_dir) mkdir(mot_base_dir)
def _create(self): mkdir(self.ann_dir) mkdir(self.item_dir)