def get_related_images(self, item_name):
results = []
path = self.get_related_images_path(item_name)
if dir_exists(path):
files = list_files(path, SUPPORTED_IMG_EXTS)
for file in files:
img_meta_path = os.path.join(path, get_file_name_with_ext(file)+".json")
img_meta = {}
if file_exists(img_meta_path):
img_meta = load_json_file(img_meta_path)
results.append((file, img_meta))
return results
def download_file(path2file, app, team_id):
assert app, 'App object should be passed'
assert team_id, 'team_id should be passed'
remote_path = path2file
local_file_path = os.path.join(app.data_dir,
get_file_name_with_ext(remote_path))
if os.path.exists(local_file_path):
return local_file_path
try:
app.public_api.file.download(team_id=team_id,
remote_path=remote_path,
local_save_path=local_file_path)
return local_file_path
except:
raise FileNotFoundError('FileNotFoundError')
def _upload_legacy(self, team_id, src, dst):
def path_to_bytes_stream(path):
return open(path, 'rb')
item = get_file_name_with_ext(dst)
content_dict = {}
content_dict[ApiField.NAME] = item
dst_dir = os.path.dirname(dst)
if not dst_dir.endswith(os.path.sep):
dst_dir += os.path.sep
content_dict[ApiField.PATH] = dst_dir # os.path.basedir ...
content_dict["file"] = (item, path_to_bytes_stream(src), mimetypes.MimeTypes().guess_type(src)[0])
encoder = MultipartEncoder(fields=content_dict)
resp = self._api.post("file-storage.upload?teamId={}".format(team_id), encoder)
return resp.json()
def upload_bulk(self, team_id, src_paths, dst_paths, progress_cb=None):
def path_to_bytes_stream(path):
return open(path, 'rb')
content_dict = []
for idx, (src, dst) in enumerate(zip(src_paths, dst_paths)):
name = get_file_name_with_ext(dst)
content_dict.append((ApiField.NAME, name))
dst_dir = os.path.dirname(dst)
if not dst_dir.endswith(os.path.sep):
dst_dir += os.path.sep
content_dict.append((ApiField.PATH, dst_dir))
content_dict.append(
("file", (name, path_to_bytes_stream(src),
mimetypes.MimeTypes().guess_type(src)[0])))
encoder = MultipartEncoder(fields=content_dict)
# progress = None
# if progress_logger is not None:
# progress = Progress("Uploading", encoder.len, progress_logger, is_size=True)
#
# def _print_progress(monitor):
# if progress is not None:
# progress.set_current_value(monitor.bytes_read)
# print(monitor.bytes_read, monitor.len)
# last_percent = 0
# def _progress_callback(monitor):
# cur_percent = int(monitor.bytes_read * 100.0 / monitor.len)
# if cur_percent - last_percent > 10 or cur_percent == 100:
# api.task.set_fields(task_id, [{"field": "data.previewProgress", "payload": cur_percent}])
# last_percent = cur_percent
if progress_cb is None:
data = encoder
else:
data = MultipartEncoderMonitor(encoder, progress_cb)
resp = self._api.post(
"file-storage.bulk.upload?teamId={}".format(team_id), data)
results = [
self._convert_json_info(info_json) for info_json in resp.json()
]
return results
def import_cityscapes(api: sly.Api, task_id, context, state, app_logger):
tag_metas = sly.TagMetaCollection()
obj_classes = sly.ObjClassCollection()
dataset_names = []
storage_dir = my_app.data_dir
if INPUT_DIR:
cur_files_path = INPUT_DIR
extract_dir = os.path.join(
storage_dir,
str(Path(cur_files_path).parent).lstrip("/"))
input_dir = os.path.join(extract_dir, Path(cur_files_path).name)
archive_path = os.path.join(
storage_dir,
cur_files_path + ".tar") # cur_files_path.split("/")[-2] + ".tar"
project_name = Path(cur_files_path).name
else:
cur_files_path = INPUT_FILE
extract_dir = os.path.join(storage_dir, get_file_name(cur_files_path))
archive_path = os.path.join(storage_dir,
get_file_name_with_ext(cur_files_path))
project_name = get_file_name(INPUT_FILE)
input_dir = os.path.join(storage_dir,
get_file_name(cur_files_path)) # extract_dir
api.file.download(TEAM_ID, cur_files_path, archive_path)
if tarfile.is_tarfile(archive_path):
with tarfile.open(archive_path) as archive:
archive.extractall(extract_dir)
else:
raise Exception("No such file".format(INPUT_FILE))
new_project = api.project.create(WORKSPACE_ID,
project_name,
change_name_if_conflict=True)
tags_template = os.path.join(input_dir, "gtFine", "*")
tags_paths = glob.glob(tags_template)
tags = [os.path.basename(tag_path) for tag_path in tags_paths]
if train_tag in tags and val_tag not in tags:
split_train = True
elif trainval_tag in tags and val_tag not in tags:
split_train = True
else:
split_train = False
search_fine = os.path.join(input_dir, "gtFine", "*", "*",
"*_gt*_polygons.json")
files_fine = glob.glob(search_fine)
files_fine.sort()
search_imgs = os.path.join(input_dir, "leftImg8bit", "*", "*",
"*_leftImg8bit" + IMAGE_EXT)
files_imgs = glob.glob(search_imgs)
files_imgs.sort()
if len(files_fine) == 0 or len(files_imgs) == 0:
raise Exception('Input cityscapes format not correct')
samples_count = len(files_fine)
progress = sly.Progress('Importing images', samples_count)
images_pathes_for_compare = []
images_pathes = {}
images_names = {}
anns_data = {}
ds_name_to_id = {}
if samples_count > 2:
random_train_indexes = get_split_idxs(samples_count, samplePercent)
for idx, orig_ann_path in enumerate(files_fine):
parent_dir, json_filename = os.path.split(
os.path.abspath(orig_ann_path))
dataset_name = os.path.basename(parent_dir)
if dataset_name not in dataset_names:
dataset_names.append(dataset_name)
ds = api.dataset.create(new_project.id,
dataset_name,
change_name_if_conflict=True)
ds_name_to_id[dataset_name] = ds.id
images_pathes[dataset_name] = []
images_names[dataset_name] = []
anns_data[dataset_name] = []
orig_img_path = json_path_to_image_path(orig_ann_path)
images_pathes_for_compare.append(orig_img_path)
if not file_exists(orig_img_path):
logger.warn(
'Image for annotation {} not found is dataset {}'.format(
orig_ann_path.split('/')[-1], dataset_name))
continue
images_pathes[dataset_name].append(orig_img_path)
images_names[dataset_name].append(
sly.io.fs.get_file_name_with_ext(orig_img_path))
tag_path = os.path.split(parent_dir)[0]
train_val_tag = os.path.basename(tag_path)
if split_train is True and samples_count > 2:
if (train_val_tag == train_tag) or (train_val_tag == trainval_tag):
if idx in random_train_indexes:
train_val_tag = train_tag
else:
train_val_tag = val_tag
# tag_meta = sly.TagMeta(train_val_tag, sly.TagValueType.NONE)
tag_meta = sly.TagMeta('split', sly.TagValueType.ANY_STRING)
if not tag_metas.has_key(tag_meta.name):
tag_metas = tag_metas.add(tag_meta)
# tag = sly.Tag(tag_meta)
tag = sly.Tag(meta=tag_meta, value=train_val_tag)
json_data = json.load(open(orig_ann_path))
ann = sly.Annotation.from_img_path(orig_img_path)
for obj in json_data['objects']:
class_name = obj['label']
if class_name == 'out of roi':
polygon = obj['polygon'][:5]
interiors = [obj['polygon'][5:]]
else:
polygon = obj['polygon']
if len(polygon) < 3:
logger.warn(
'Polygon must contain at least 3 points in ann {}, obj_class {}'
.format(orig_ann_path, class_name))
continue
interiors = []
interiors = [convert_points(interior) for interior in interiors]
polygon = sly.Polygon(convert_points(polygon), interiors)
if city_classes_to_colors.get(class_name, None):
obj_class = sly.ObjClass(
name=class_name,
geometry_type=sly.Polygon,
color=city_classes_to_colors[class_name])
else:
new_color = generate_rgb(city_colors)
city_colors.append(new_color)
obj_class = sly.ObjClass(name=class_name,
geometry_type=sly.Polygon,
color=new_color)
ann = ann.add_label(sly.Label(polygon, obj_class))
if not obj_classes.has_key(class_name):
obj_classes = obj_classes.add(obj_class)
ann = ann.add_tag(tag)
anns_data[dataset_name].append(ann)
progress.iter_done_report()
out_meta = sly.ProjectMeta(obj_classes=obj_classes, tag_metas=tag_metas)
api.project.update_meta(new_project.id, out_meta.to_json())
for ds_name, ds_id in ds_name_to_id.items():
dst_image_infos = api.image.upload_paths(ds_id, images_names[ds_name],
images_pathes[ds_name])
dst_image_ids = [img_info.id for img_info in dst_image_infos]
api.annotation.upload_anns(dst_image_ids, anns_data[ds_name])
stat_dct = {
'samples': samples_count,
'src_ann_cnt': len(files_fine),
'src_img_cnt': len(files_imgs)
}
logger.info('Found img/ann pairs.', extra=stat_dct)
images_without_anns = set(files_imgs) - set(images_pathes_for_compare)
if len(images_without_anns) > 0:
logger.warn('Found source images without corresponding annotations:')
for im_path in images_without_anns:
logger.warn('Annotation not found {}'.format(im_path))
logger.info('Found classes.',
extra={
'cnt':
len(obj_classes),
'classes':
sorted([obj_class.name for obj_class in obj_classes])
})
logger.info('Created tags.',
extra={
'cnt':
len(out_meta.tag_metas),
'tags':
sorted([tag_meta.name for tag_meta in out_meta.tag_metas])
})
my_app.stop()