def _convert_json_info(self, info: dict): if info is None: return None temp_ext = None field_values = [] for field_name in self.__class__._info_sequence: if field_name == ApiField.EXT: continue field_values.append(info[field_name]) if field_name == ApiField.MIME: temp_ext = info[field_name].split('/')[1] field_values.append(temp_ext) for idx, field_name in enumerate(self.__class__._info_sequence): if field_name == ApiField.NAME: cur_ext = get_file_ext(field_values[idx]) if not cur_ext: field_values[idx] = "{}.{}".format(field_values[idx], temp_ext) break cur_ext = cur_ext.replace(".", "").lower() if temp_ext == 'jpeg' and cur_ext in ['jpg', 'jpeg']: break if temp_ext not in field_values[idx]: field_values[idx] = "{}.{}".format(field_values[idx], temp_ext) break return self.__class__.Info._make(field_values)
def _convert_json_info(self, info: dict, skip_missing=True): if info is None: return None temp_ext = None field_values = [] for field_name in self.info_sequence(): if field_name == ApiField.EXT: continue if skip_missing is True: val = info.get(field_name, None) else: val = info[field_name] field_values.append(val) if field_name == ApiField.MIME: temp_ext = val.split('/')[1] field_values.append(temp_ext) for idx, field_name in enumerate(self.info_sequence()): if field_name == ApiField.NAME: cur_ext = get_file_ext(field_values[idx]).replace(".", "").lower() if not cur_ext: field_values[idx] = "{}.{}".format(field_values[idx], temp_ext) break if temp_ext == 'jpeg' and cur_ext in ['jpg', 'jpeg', 'mpo']: break if temp_ext != cur_ext: field_values[idx] = "{}.{}".format(field_values[idx], temp_ext) break return self.InfoType(*field_values)
def _read(self): if not dir_exists(self.img_dir): raise FileNotFoundError('Image directory not found: {!r}'.format( self.img_dir)) if not dir_exists(self.ann_dir): raise FileNotFoundError( 'Annotation directory not found: {!r}'.format(self.ann_dir)) ann_paths = list_files(self.ann_dir, [ANN_EXT]) img_paths = list_files(self.img_dir, image.SUPPORTED_IMG_EXTS) ann_names = set(get_file_name(path) for path in ann_paths) img_names = { get_file_name(path): get_file_ext(path) for path in img_paths } if len(img_names) == 0 or len(ann_names) == 0: raise RuntimeError('Dataset {!r} is empty'.format(self.name)) if ann_names != set(img_names.keys()): raise RuntimeError( 'File names in dataset {!r} are inconsistent'.format( self.name)) self._items_exts = img_names
def gen_video_stream_name(file_name, stream_index): ''' Create name to video stream from given filename and index of stream :param file_name: str :param stream_index: int :return: str ''' return "{}_stream_{}_{}{}".format(get_file_name(file_name), stream_index, rand_str(5), get_file_ext(file_name))
def generate_names(base_name, count): name = sly_fs.get_file_name(base_name) ext = sly_fs.get_file_ext(base_name) names = [base_name] for idx in range(1, count): names.append('{}_{:02d}{}'.format(name, idx, ext)) return names
def _get_free_name(exist_check_fn, name): res_title = name suffix = 1 name_without_ext = get_file_name(name) ext = get_file_ext(name) while exist_check_fn(res_title): res_title = '{}_{:03d}{}'.format(name_without_ext, suffix, ext) suffix += 1 return res_title
def generate_free_name(used_names, possible_name, with_ext=False): res_name = possible_name new_suffix = 1 while res_name in set(used_names): if with_ext is True: res_name = '{}_{:02d}{}'.format( sly_fs.get_file_name(possible_name), new_suffix, sly_fs.get_file_ext(possible_name)) else: res_name = '{}_{:02d}'.format(possible_name, new_suffix) new_suffix += 1 return res_name
def download(self, team_id, remote_path, local_save_path, cache: FileCache = None, progress_cb=None): if cache is None: self._download(team_id, remote_path, local_save_path, progress_cb) else: file_info = self.get_info_by_path(team_id, remote_path) if file_info.hash is None: self._download(team_id, remote_path, local_save_path, progress_cb) else: cache_path = cache.check_storage_object(file_info.hash, get_file_ext(remote_path)) if cache_path is None: # file not in cache self._download(team_id, remote_path, local_save_path, progress_cb) if file_info.hash != get_file_hash(local_save_path): raise KeyError(f"Remote and local hashes are different (team id: {team_id}, file: {remote_path})") cache.write_object(local_save_path, file_info.hash) else: cache.read_object(file_info.hash, local_save_path) if progress_cb is not None: progress_cb(get_file_size(local_save_path))
def load_font(font_file_name: str, font_size: int = 12) -> ImageFont.FreeTypeFont: """ Set global font true-type for drawing. Args: font_file_name: name of font file (example: 'DejaVuSansMono.ttf') font_size: selected font size Returns: loaded from file font """ if get_file_ext(font_file_name) == FONT_EXTENSION: font_path = _get_font_path_by_name(font_file_name) if (font_path is not None) and file_exists(font_path): return ImageFont.truetype(font_path, font_size, encoding='utf-8') else: raise ValueError( 'Font file "{}" not found in system paths. Try to set another font.' .format(font_file_name)) else: raise ValueError('Supported only TrueType fonts!')
def get_free_name(self, team_id, path): directory = Path(path).parent name = get_file_name(path) ext = get_file_ext(path) res_name = name suffix = 0 def _combine(suffix: int = None): res = "{}/{}".format(directory, res_name) if suffix is not None: res += "_{:03d}".format(suffix) if ext: res += "{}".format(ext) return res res_path = _combine() while self.exists(team_id, res_path): res_path = _combine(suffix) suffix += 1 return res_path
def _add_img_file(self, item_name, img_path): img_ext = get_file_ext(img_path) dst_img_path = self.deprecated_make_img_path(item_name, img_ext) if img_path != dst_img_path: # used only for agent + api during download project copy_file(img_path, dst_img_path) self._items_exts[item_name] = img_ext
def gen_video_stream_name(file_name, stream_index): return "{}_stream_{}_{}{}".format(get_file_name(file_name), stream_index, rand_str(5), get_file_ext(file_name))
parser = argparse.ArgumentParser(description='Inference REST client for standalone Supervisely models.') parser.add_argument('--server-url', required=True) parser.add_argument('--request-type', required=True, choices=SUPPORTED_REQUEST_TYPES) parser.add_argument('--in-image', default='') parser.add_argument('--out-json', default='') args = parser.parse_args() request_url = args.server_url + '/' + MODEL + '/' + args.request_type response_json = None if args.request_type == GET_OUTPUT_META: response = requests.post(request_url) elif args.request_type == INFERENCE: with open(args.in_image, 'rb') as fin: img_bytes = fin.read() img_ext = sly_fs.get_file_ext(args.in_image) encoder = MultipartEncoder({IMAGE: (args.in_image, io.BytesIO(img_bytes), 'application/octet-stream')}) response = requests.post(request_url, data=encoder, headers={'Content-Type': encoder.content_type}) else: raise ValueError( 'Unknown model request type: {!r}. Only the following request types are supported: {!r}.'.format( args.request_type, SUPPORTED_REQUEST_TYPES)) response.raise_for_status() response_str = json.dumps(response.json(), indent=4, sort_keys=True) if args.out_json: with open(args.out_json, 'w') as fout: fout.write(response_str) else: print(response_str)
def img_to_hash(item): img, name = item[0], item[1] return sly_image.get_hash(img, get_file_ext(name))
def img_to_bytes_stream(item): img, name = item[0], item[1] img_bytes = sly_image.write_bytes(img, get_file_ext(name)) return io.BytesIO(img_bytes)
def from_sl_to_cityscapes(api: sly.Api, task_id, context, state, app_logger): def get_image_and_ann(): mkdir(image_dir_path) mkdir(ann_dir) image_path = os.path.join(image_dir_path, image_name) api.image.download_path(image_id, image_path) image_ext_to_png(image_path) mask_color, mask_label, poly_json = from_ann_to_cityscapes_mask( ann, name2id, app_logger, train_val_flag) # dump_json_file(poly_json, # os.path.join(ann_dir, get_file_name(base_image_name) + cityscapes_polygons_suffix)) # write( # os.path.join(ann_dir, # get_file_name(base_image_name) + cityscapes_color_suffix), mask_color) # write( # os.path.join(ann_dir, # get_file_name(base_image_name) + cityscapes_labels_suffix), mask_label) dump_json_file( poly_json, os.path.join( ann_dir, get_file_name(base_image_name).replace('_leftImg8bit', '') + cityscapes_polygons_suffix)) write( os.path.join( ann_dir, get_file_name(base_image_name).replace('_leftImg8bit', '') + cityscapes_color_suffix), mask_color) write( os.path.join( ann_dir, get_file_name(base_image_name).replace('_leftImg8bit', '') + cityscapes_labels_suffix), mask_label) project_name = api.project.get_info_by_id(PROJECT_ID).name ARCHIVE_NAME = '{}_{}_Cityscapes.tar.gz'.format(PROJECT_ID, project_name) meta_json = api.project.get_meta(PROJECT_ID) meta = sly.ProjectMeta.from_json(meta_json) has_bitmap_poly_shapes = False for obj_class in meta.obj_classes: if obj_class.geometry_type not in possible_geometries: app_logger.warn( f'Cityscapes format supports only bitmap and polygon classes, {obj_class.geometry_type} will be skipped' ) else: has_bitmap_poly_shapes = True if has_bitmap_poly_shapes is False: raise Exception( 'Input project does not contain bitmap or polygon classes') my_app.stop() RESULT_ARCHIVE = os.path.join(my_app.data_dir, ARCHIVE_NAME) RESULT_DIR = os.path.join(my_app.data_dir, RESULT_DIR_NAME) result_images_train = os.path.join(RESULT_DIR, images_dir_name, default_dir_train) result_images_val = os.path.join(RESULT_DIR, images_dir_name, default_dir_val) result_images_test = os.path.join(RESULT_DIR, images_dir_name, default_dir_test) result_anns_train = os.path.join(RESULT_DIR, annotations_dir_name, default_dir_train) result_anns_val = os.path.join(RESULT_DIR, annotations_dir_name, default_dir_val) result_anns_test = os.path.join(RESULT_DIR, annotations_dir_name, default_dir_test) sly.fs.mkdir(RESULT_DIR) app_logger.info("Cityscapes Dataset folder has been created") class_to_id = [] name2id = {} for idx, obj_class in enumerate(meta.obj_classes): if obj_class.geometry_type not in possible_geometries: continue curr_class = {} curr_class['name'] = obj_class.name curr_class['id'] = idx + 1 curr_class['color'] = obj_class.color class_to_id.append(curr_class) name2id[obj_class.name] = (idx + 1, idx + 1, idx + 1) dump_json_file(class_to_id, os.path.join(RESULT_DIR, 'class_to_id.json')) app_logger.info("Writing classes with colors to class_to_id.json file") datasets = api.dataset.get_list(PROJECT_ID) for dataset in datasets: images_dir_path_train = os.path.join(result_images_train, dataset.name) images_dir_path_val = os.path.join(result_images_val, dataset.name) images_dir_path_test = os.path.join(result_images_test, dataset.name) anns_dir_path_train = os.path.join(result_anns_train, dataset.name) anns_dir_path_val = os.path.join(result_anns_val, dataset.name) anns_dir_path_test = os.path.join(result_anns_test, dataset.name) images = api.image.get_list(dataset.id) progress = sly.Progress( 'Convert images and anns from dataset {}'.format(dataset.name), len(images), app_logger) if len(images) < 3: app_logger.warn( 'Number of images in {} dataset is less then 3, val and train directories for this dataset will not be created' .format(dataset.name)) image_ids = [image_info.id for image_info in images] base_image_names = [image_info.name for image_info in images] # image_names = [ # get_file_name(image_info.name) + cityscapes_images_suffix + get_file_ext(image_info.name) for # image_info in images # ] image_names = [ get_file_name(image_info.name.replace('_leftImg8bit', '')) + \ cityscapes_images_suffix + get_file_ext(image_info.name) for image_info in images ] ann_infos = api.annotation.download_batch(dataset.id, image_ids) anns = [ sly.Annotation.from_json(ann_info.annotation, meta) for ann_info in ann_infos ] splitter = get_tags_splitter(anns) curr_splitter = {'train': 0, 'val': 0, 'test': 0} for ann, image_id, image_name, base_image_name in zip( anns, image_ids, image_names, base_image_names): train_val_flag = True try: split_name = ann.img_tags.get('split').value if split_name == 'train': image_dir_path = images_dir_path_train ann_dir = anns_dir_path_train elif split_name == 'val': image_dir_path = images_dir_path_val ann_dir = anns_dir_path_val else: image_dir_path = images_dir_path_test ann_dir = anns_dir_path_test train_val_flag = False except: ann_tags = [tag.name for tag in ann.img_tags] separator_tags = list(set(ann_tags) & set(possible_tags)) if len(separator_tags) > 1: app_logger.warn( '''There are more then one separator tag for {} image. {} tag will be used for split'''.format( image_name, separator_tags[0])) if len(separator_tags) >= 1: if separator_tags[0] == 'train': image_dir_path = images_dir_path_train ann_dir = anns_dir_path_train elif separator_tags[0] == 'val': image_dir_path = images_dir_path_val ann_dir = anns_dir_path_val else: image_dir_path = images_dir_path_test ann_dir = anns_dir_path_test train_val_flag = False if len(separator_tags) == 0: if curr_splitter['test'] == splitter['test']: curr_splitter = {'train': 0, 'val': 0, 'test': 0} if curr_splitter['train'] < splitter['train']: curr_splitter['train'] += 1 image_dir_path = images_dir_path_train ann_dir = anns_dir_path_train elif curr_splitter['val'] < splitter['val']: curr_splitter['val'] += 1 image_dir_path = images_dir_path_val ann_dir = anns_dir_path_val elif curr_splitter['test'] < splitter['test']: curr_splitter['test'] += 1 image_dir_path = images_dir_path_test ann_dir = anns_dir_path_test train_val_flag = False get_image_and_ann() progress.iter_done_report() sly.fs.archive_directory(RESULT_DIR, RESULT_ARCHIVE) app_logger.info("Result directory is archived") upload_progress = [] remote_archive_path = "/cityscapes_format/{}/{}".format( task_id, ARCHIVE_NAME) def _print_progress(monitor, upload_progress): if len(upload_progress) == 0: upload_progress.append( sly.Progress(message="Upload {!r}".format(ARCHIVE_NAME), total_cnt=monitor.len, ext_logger=app_logger, is_size=True)) upload_progress[0].set_current_value(monitor.bytes_read) file_info = api.file.upload( team_id=TEAM_ID, src=RESULT_ARCHIVE, dst=remote_archive_path, progress_cb=lambda m: _print_progress(m, upload_progress)) app_logger.info("Uploaded to Team-Files: {!r}".format( file_info.full_storage_url)) api.task.set_output_archive(task_id, file_info.id, ARCHIVE_NAME, file_url=file_info.full_storage_url) my_app.stop()
def image_ext_to_png(im_path): if get_file_ext(im_path) != '.png': im = Image.open(im_path).convert('RGB') im.save(im_path[:-1 * len(get_file_ext(im_path))] + '.png') silent_remove(im_path)
def write(path, img): ensure_base_path(path) validate_ext(get_file_ext(path)) img = cv2.cvtColor(img.astype(np.uint8), cv2.COLOR_RGB2BGR) return cv2.imwrite(path, img)
def _get_suffix(self, path): return sly_fs.get_file_ext(path)