def chunk_generator():
progress = sly.ProgressCounter('Upload annotations',
len(img_ids),
ext_logger=self.logger)
for batch_some in sly.batched(
list(zip(img_ids, img_names, ann_paths)),
constants.BATCH_SIZE_UPLOAD_ANNOTATIONS):
for img_id, img_name, ann_path in batch_some:
proto_img = api_proto.Image(id=img_id,
title=img_name,
project_id=project_id)
freader = ChunkedFileReader(ann_path,
constants.NETW_CHUNK_SIZE)
for chunk_bytes in freader:
current_chunk = api_proto.Chunk(
buffer=chunk_bytes, total_size=freader.file_size)
yield api_proto.ChunkImage(chunk=current_chunk,
image=proto_img)
self.logger.trace('annotation is uploaded',
extra={
'img_name': img_name,
'img_path': ann_path
})
progress.iter_done_report()
def add_metadata_to_images(api, path_to_files, dataset_id, app_logger):
path_to_images = [
sly.fs.get_file_name(json_name)
for json_name in os.listdir(path_to_files)
]
images = api.image.get_list(dataset_id)
image_names = [image_info.name for image_info in images]
matches = list(set(path_to_images) & set(image_names))
if len(path_to_images) != len(matches):
app_logger.warn(
'{} metadata files were given, {} matches image names in dataset'.
format(len(path_to_images), len(matches)))
progress = sly.Progress('Uploading metadata to images', len(images),
app_logger)
for batch in sly.batched(images):
for image_info in batch:
if image_info.name not in path_to_images:
app_logger.warn(
'Metadata file for image {} was not found in directory {}'.
format(image_info.name, path_to_files))
continue
meta = load_json_file(
os.path.join(path_to_files, image_info.name + '.json'))
if RESOLVE == "merge":
meta_copy = meta.copy()
for key in meta.keys():
if key in image_info.meta:
meta_copy[key + "-original"] = image_info.meta[key]
meta = {**image_info.meta, **meta_copy}
api.image.update_meta(image_info.id, meta)
progress.iters_done_report(len(batch))
def _download_annotations(self, pr_writer, image_id_to_ds):
progress = sly.ProgressCounter('Download annotations',
len(image_id_to_ds),
ext_logger=self.logger)
for batch_img_ids in sly.batched(
list(image_id_to_ds.keys()),
constants.BATCH_SIZE_DOWNLOAD_ANNOTATIONS):
for chunk in self.api.get_stream_with_data(
'DownloadAnnotations', api_proto.ChunkImage,
api_proto.ImageArray(images=batch_img_ids)):
img_id = chunk.image.id
ds_name = image_id_to_ds[img_id]
self.logger.trace('download_annotations',
extra={'img_id': img_id})
fh = ChunkedFileWriter(file_path=pr_writer.get_ann_path(
ds_name, chunk.image.title))
fh.write(chunk.chunk)
progress.iter_done_report()
if not fh.close_and_check():
self.logger.warning('ann was skipped while downloading',
extra={
'img_id': img_id,
'ann_path': fh.file_path
})
def upload_pred_vis():
submitted = False
for i in range(5):
paths = [
x for x in Path(g.local_artifacts_dir).glob('test*.jpg')
if x.exists()
]
if len(paths) % 2 != 0:
time.sleep(
3
) # wait while thread in YOLOv5 script produce visualization: test batch + prediction
continue
_paths = [str(path) for path in paths]
_paths.sort()
sync_bindings = []
for batch in sly.batched(_paths, 2):
sync_bindings.append([
sly.fs.get_file_name_with_ext(batch[0]),
sly.fs.get_file_name_with_ext(batch[1])
])
api.task.set_field(task_id, "data.syncBindings", sync_bindings)
_upload_data_vis("data.predVis", paths, 2)
submitted = True
break
if submitted is False:
sly.logger.warn(
"Test batch visualizations (labels + predictions) are not ready, see them in artifacts "
"directory after training ")
pass
def _assign_tag(task_id, api: sly.Api, split, tag_metas, new_project, created_datasets, progress):
for dataset_id, images in split.items():
dataset = api.dataset.get_info_by_id(dataset_id)
if dataset.name not in created_datasets:
new_dataset = api.dataset.create(new_project.id, dataset.name)
created_datasets[dataset.name] = new_dataset
new_dataset = created_datasets[dataset.name]
for batch in sly.batched(images):
image_ids = [image_info.id for image_info in batch]
image_names = [image_info.name for image_info in batch]
ann_infos = api.annotation.download_batch(dataset.id, image_ids)
new_annotations = []
for ann_info in ann_infos:
ann_json = ann_info.annotation
new_ann = sly.Annotation.from_json(ann_json, META_ORIGINAL)
for tag_meta in tag_metas:
new_ann = new_ann.add_tag(sly.Tag(tag_meta))
new_annotations.append(new_ann)
new_images = api.image.upload_ids(new_dataset.id, image_names, image_ids)
new_image_ids = [image_info.id for image_info in new_images]
api.annotation.upload_anns(new_image_ids, new_annotations)
progress.iters_done_report(len(batch))
progress_percent = int(progress.current * 100 / progress.total)
api.task.set_field(task_id, "data.progress", progress_percent)
fields = [
{"field": "data.progressCurrent", "payload": progress.current},
{"field": "data.progressTotal", "payload": progress.total},
{"field": "data.progress", "payload": progress_percent},
]
api.task.set_fields(task_id, fields)
def create_foreground(api: sly.Api, task_id, context, state, app_logger):
global meta
project_info = api.project.get_info_by_id(project_id)
meta = create_classes(api)
progress = sly.Progress("Processing", project_info.items_count)
for dataset in api.dataset.get_list(project_id):
images_infos = api.image.get_list(dataset.id)
for batch_infos in sly.batched(images_infos, 20):
ids = []
names = []
local_paths = []
for info in batch_infos:
ids.append(info.id)
names.append(info.name)
local_paths.append(os.path.join(app.data_dir, info.name))
api.image.download_paths(dataset.id, ids, local_paths)
anns_infos = api.annotation.download_batch(dataset.id, ids)
anns = [
sly.Annotation.from_json(info.annotation, meta)
for info in anns_infos
]
res_ids = []
res_anns = []
for img_id, img_name, img_path, ann in zip(ids, names, local_paths,
anns):
img = sly.image.read(img_path, remove_alpha_channel=False)
if len(img.shape) == 3:
if img.shape[2] != 4:
sly.logger.warn(
f"Image {img_name} (id={img_id}) does not have alpha channel, will be skipped"
)
continue
else:
sly.logger.warn(
f"Image {img_name} (id={img_id}) does not have alpha channel, will be skipped"
)
continue
fg, fuzzy = get_masks(img)
new_ann = ann.add_label(fg)
if fuzzy is not None:
new_ann = new_ann.add_label(fuzzy)
res_ids.append(img_id)
res_anns.append(new_ann)
api.annotation.upload_anns(res_ids, res_anns)
for img_path in local_paths:
sly.fs.silent_remove(img_path)
progress.iters_done_report(len(batch_infos))
api.task.set_output_project(task_id, project_id)
app.stop()
def get_dataset_images_hashes(self, dataset_id):
image_array = self.api.simple_request('GetDatasetImages', sly.api_proto.ImageArray, sly.api_proto.Id(id=dataset_id))
img_hashes = []
for batch_img_ids in sly.batched(list(image_array.images), constants.BATCH_SIZE_GET_IMAGES_INFO()):
images_info_proto = self.api.simple_request('GetImagesInfo', sly.api_proto.ImagesInfo,
sly.api_proto.ImageArray(images=batch_img_ids))
img_hashes.extend([(info.hash, info.ext) for info in images_info_proto.infos])
return img_hashes
def copy_project(api: sly.Api, task_id, context, state, app_logger):
project2 = api2.project.get_info_by_id(PROJECT_ID2)
if project2 is None:
raise RuntimeError(
f"Project with id={PROJECT_ID2} not found on remote Supervisely instance"
)
if project2.type != str(sly.ProjectType.IMAGES):
raise TypeError(
"This version supports only images project. Please, submit a feature request to Supervisely dev"
"team to add support of other types of projects (videos, 3d, dicom, ...)"
)
meta2_json = api2.project.get_meta(project2.id)
project = api.project.create(WORKSPACE_ID,
project2.name,
project2.type,
project2.description,
change_name_if_conflict=True)
api.project.update_meta(project.id, meta2_json)
progress = sly.Progress("Import",
api2.project.get_images_count(project2.id))
for dataset2 in api2.dataset.get_list(project2.id):
dataset = api.dataset.create(project.id, dataset2.name,
dataset2.description)
images2 = api2.image.get_list(dataset2.id)
for batch2 in sly.batched(images2, batch_size=10):
ids2 = []
names = []
paths = []
metas = []
for image_info2 in batch2:
ids2.append(image_info2.id)
names.append(image_info2.name)
paths.append(os.path.join(my_app.data_dir, image_info2.name))
metas.append(image_info2.meta)
api2.image.download_paths(dataset2.id, ids2, paths)
anns2 = api2.annotation.download_batch(dataset2.id, ids2)
anns2 = [ann2.annotation for ann2 in anns2]
batch = api.image.upload_paths(dataset.id,
names,
paths,
metas=metas)
ids = [image_info.id for image_info in batch]
api.annotation.upload_jsons(ids, anns2)
for p in paths:
sly.fs.silent_remove(p)
progress.iters_done_report(len(batch2))
api.task.set_output_project(task_id, project.id, project.name)
my_app.stop()
def apply_model(api: sly.Api, task_id, context, state, app_logger):
def _update_progress(progress):
fields = [
{"field": "data.progress", "payload": int(progress.current * 100 / progress.total)},
{"field": "data.progressCurrent", "payload": progress.current},
{"field": "data.progressTotal", "payload": progress.total},
]
api.task.set_fields(task_id, fields)
try:
inf_setting = yaml.safe_load(state["settings"])
except Exception as e:
inf_setting = {}
app_logger.warn(repr(e))
global project_meta
res_project_meta = project_meta.clone()
res_project = api.project.create(workspace_id, state["resProjectName"], change_name_if_conflict=True)
api.project.update_meta(res_project.id, res_project_meta.to_json())
progress = sly.Progress("Inference", len(input_images), need_info_log=True)
for dataset in input_datasets:
res_dataset = api.dataset.create(res_project.id, dataset.name, dataset.description)
images = api.image.get_list(dataset.id)
for batch in sly.batched(images, batch_size=10):
image_ids, res_names, res_metas = [], [], []
for image_info in batch:
image_ids.append(image_info.id)
res_names.append(image_info.name)
res_metas.append(image_info.meta)
_, res_anns, final_project_meta = apply_model_to_images(api, state, dataset.id, image_ids, inf_setting)
if res_project_meta != final_project_meta:
res_project_meta = final_project_meta
api.project.update_meta(res_project.id, res_project_meta.to_json())
res_images_infos = api.image.upload_ids(res_dataset.id, res_names, image_ids, metas=res_metas)
res_ids = [image_info.id for image_info in res_images_infos]
api.annotation.upload_anns(res_ids, res_anns)
progress.iters_done_report(len(res_ids))
if progress.need_report():
_update_progress(progress)
res_project = api.project.get_info_by_id(res_project.id) # to refresh reference_image_url
fields = [
{"field": "data.resProjectId", "payload": res_project.id},
{"field": "data.resProjectName", "payload": res_project.name},
{"field": "data.resProjectPreviewUrl", "payload": api.image.preview_url(res_project.reference_image_url, 100, 100)},
]
api.task.set_fields(task_id, fields)
api.task.set_output_project(task_id, res_project.id, res_project.name)
my_app.stop()
def _download_images(self, pr_writer, image_id_to_ds):
# get full info (hash, title, ext)
img_infos = []
for batch_img_ids in sly.batched(list(image_id_to_ds.keys()), constants.BATCH_SIZE_GET_IMAGES_INFO):
images_info_proto = self.api.simple_request('GetImagesInfo',
api_proto.ImagesInfo,
api_proto.ImageArray(images=batch_img_ids))
img_infos.extend(images_info_proto.infos)
written_hashes = set(self._get_images_from_agent_storage(pr_writer, image_id_to_ds, img_infos))
img_infos_to_download = [x for x in img_infos if x.hash not in written_hashes]
self._download_images_from_remote(pr_writer, image_id_to_ds, img_infos_to_download)
def _download_images_from_remote(self, pr_writer, image_id_to_ds, img_infos):
if len(img_infos) == 0:
return
infos_with_paths = [(info, pr_writer.get_img_path(image_id_to_ds[info.id], info.title, info.ext))
for info in img_infos]
hash2path = {x[0].hash: x[1] for x in infos_with_paths} # for unique hashes
unique_hashes = list(hash2path.keys())
ready_paths = []
ready_hashes = []
progress = sly.ProgressCounter('Download remote images', len(unique_hashes), ext_logger=self.logger)
def close_fh(fh):
fpath = fh.file_path
if fh.close_and_check():
ready_paths.append(fpath)
ready_hashes.append(img_hash)
progress.iter_done_report()
else:
self.logger.warning('file was skipped while downloading',
extra={'img_path': fpath, 'img_hash': img_hash})
# download by unique hashes
for batch_img_hashes in sly.batched(unique_hashes, constants.BATCH_SIZE_DOWNLOAD_IMAGES):
file_handler = None
img_hash = None
for chunk in self.api.get_stream_with_data('DownloadImages',
api_proto.ChunkImage,
api_proto.ImagesHashes(images_hashes=batch_img_hashes)):
if chunk.image.hash: # non-empty hash means beginning of new image
if file_handler is not None:
close_fh(file_handler)
img_hash = chunk.image.hash
self.logger.trace('download_images', extra={'img_hash': img_hash})
dst_fpath = hash2path[img_hash]
file_handler = ChunkedFileWriter(file_path=dst_fpath)
file_handler.write(chunk.chunk)
close_fh(file_handler) # must be not None
# process non-unique hashes
for info, dst_path in infos_with_paths:
origin_path = hash2path[info.hash]
if (origin_path != dst_path) and osp.isfile(origin_path):
sly.ensure_base_path(dst_path)
sly.copy_file(origin_path, dst_path)
self._write_images_to_agent_storage(ready_paths, ready_hashes)
def init(api: sly.Api, task_id, context, state, app_logger):
global project_info, meta
project_info = api.project.get_info_by_id(project_id)
meta = sly.ProjectMeta.from_json(api.project.get_meta(project_id))
same_tags = defaultdict(list)
progress = sly.Progress("Initializing image examples",
project_info.items_count,
need_info_log=True)
for dataset in api.dataset.get_list(project_id):
images_infos = api.image.get_list(dataset.id)
for batch in sly.batched(images_infos):
img_ids = [info.id for info in batch]
ann_infos = api.annotation.download_batch(dataset.id, img_ids)
anns = [
sly.Annotation.from_json(ann_info.annotation, meta)
for ann_info in ann_infos
]
for img_info, ann in zip(batch, anns):
for tag in ann.img_tags:
tag: sly.Tag
same_tags[tag.get_compact_str()].append({
"url": img_info.full_storage_url,
"figures": [],
"tag": {
**tag.to_json(), "color":
sly.color.rgb2hex(
meta.get_tag_meta(tag.name).color)
},
"_tag": tag
})
progress.iters_done_report(len(batch))
index = 0
for key, cards in same_tags.items():
for card in cards:
tag = card.pop('_tag', None)
gallery["content"]["annotations"][str(index)] = card
gallery["content"]["layout"][index % CNT_GRID_COLUMNS].append(
str(index))
gallery2tag[str(index)] = tag
index += 1
api.task.set_field(task_id, "data.gallery", gallery)
if len(gallery2tag) > 0:
api.task.set_field(task_id, "state.selectedItem", '0')
sly.logger.info("Initialization finished")
def upload_images_to_remote(self, fpaths, infos):
progress = sly.ProgressCounter('Upload images', len(fpaths), ext_logger=self.logger)
for batch_paths_infos in sly.batched(list(zip(fpaths, infos)), constants.BATCH_SIZE_UPLOAD_IMAGES):
def chunk_generator():
for fpath, proto_img_info in batch_paths_infos:
self.logger.trace('image upload start', extra={'img_path': fpath})
freader = ChunkedFileReader(fpath, constants.NETW_CHUNK_SIZE)
for chunk_bytes in freader:
current_chunk = api_proto.Chunk(buffer=chunk_bytes, total_size=freader.file_size)
yield api_proto.ChunkImage(chunk=current_chunk, image=proto_img_info)
self.logger.trace('image uploaded', extra={'img_path': fpath})
progress.iter_done_report()
self.api.put_stream_with_data('UploadImages', api_proto.Empty, chunk_generator())
self.logger.debug('Batch of images has been sent.', extra={'batch_len': len(batch_paths_infos)})
def inference_model_batch(model, images_nps, topn=5):
"""Inference image(s) with the classifier.
Args:
model (nn.Module): The loaded classifier.
img (str/ndarray): The image filename or loaded image.
Returns:
result (list of dict): The classification results that contains
`class_name`, `pred_label` and `pred_score`.
"""
cfg = model.cfg
device = next(model.parameters()).device # model device
# build the data pipeline
if cfg.data.test.pipeline[0]['type'] == 'LoadImageFromFile':
cfg.data.test.pipeline.pop(0)
test_pipeline = Compose(cfg.data.test.pipeline)
with torch.no_grad():
inference_results = []
for images_batch in sly.batched(images_nps, g.batch_size):
data = [dict(img=img) for img in images_batch]
data = [test_pipeline(row) for row in data]
data = collate(data, samples_per_gpu=1)
if next(model.parameters()).is_cuda:
# scatter to specified GPU
data = scatter(data, [device])[0]
batch_scores = np.asarray(model(return_loss=False, **data))
batch_top_indexes = batch_scores.argsort(axis=1)[:,
-topn:][:, ::-1]
for scores, top_indexes in zip(batch_scores, batch_top_indexes):
inference_results.append({
'label':
top_indexes.astype(int).tolist(),
'score':
scores[top_indexes].astype(float).tolist(),
'class':
np.asarray(model.CLASSES)[top_indexes].tolist()
})
return inference_results
def cache_annotations(api: sly.Api, task_id, context, state, app_logger):
progress = sly.Progress("Cache annotations", project_info.items_count)
for dataset in api.dataset.get_list(project_id):
images = api.image.get_list(dataset.id)
for batch in sly.batched(images):
image_ids = [image_info.id for image_info in batch]
ann_infos = api.annotation.download_batch(dataset.id, image_ids)
for image_id, image_info, ann_info in zip(image_ids, batch,
ann_infos):
ann = sly.Annotation.from_json(ann_info.annotation, meta)
anns[image_id] = ann
images_info[image_id] = image_info
for label in ann.labels:
labels[label.obj_class.name][image_id].append(label)
progress.iters_done_report(len(batch))
progress = sly.Progress("App is ready", 1)
progress.iter_done_report()
def chunk_generator():
progress = sly.ProgressCounter('Upload images',
len(fpaths),
ext_logger=self.logger)
for batch_paths_infos in sly.batched(
list(zip(fpaths, infos)),
constants.BATCH_SIZE_UPLOAD_IMAGES):
for fpath, proto_img_info in batch_paths_infos:
self.logger.trace('image upload start',
extra={'img_path': fpath})
freader = ChunkedFileReader(fpath,
constants.NETW_CHUNK_SIZE)
for chunk_bytes in freader:
current_chunk = api_proto.Chunk(
buffer=chunk_bytes, total_size=freader.file_size)
yield api_proto.ChunkImage(chunk=current_chunk,
image=proto_img_info)
self.logger.trace('image uploaded',
extra={'img_path': fpath})
progress.iter_done_report()
def create_reference_file(api: sly.Api, task_id, context, state, app_logger):
global PROJECT, JSON_PATH_REMOTE
PROJECT = api.project.get_info_by_id(PROJECT_ID)
read_and_validate_project_meta()
file_remote = "/reference_items/{}_{}.json".format(PROJECT.id,
PROJECT.name)
app_logger.info("Remote file path: {!r}".format(file_remote))
if api.file.exists(TEAM_ID, file_remote):
raise FileExistsError(
"File {!r} already exists in Team Files. Make sure you want to replace it. "
"Please, remove it manually and run the app again.".format(
file_remote))
result = {
"project_id": PROJECT.id,
"project_name": PROJECT.name,
"project_url": api.project.url(PROJECT_ID),
"reference_tag_name": TAG_NAME,
"key_image_field": KEY_IMAGE_FIELD,
"all_keys": [],
"references": defaultdict(list)
}
progress = sly.Progress("Processing",
PROJECT.images_count,
ext_logger=app_logger)
for dataset in api.dataset.get_list(PROJECT.id):
ds_images = api.image.get_list(dataset.id)
for batch in sly.batched(ds_images):
image_ids = [image_info.id for image_info in batch]
image_names = [image_info.name for image_info in batch]
ann_infos = api.annotation.download_batch(dataset.id, image_ids)
anns = [
sly.Annotation.from_json(ann_info.annotation, META)
for ann_info in ann_infos
]
for image_info, image_id, image_name, ann in zip(
batch, image_ids, image_names, anns):
for label in ann.labels:
tag = label.tags.get(TAG_NAME)
if tag is None:
continue
key_tag = image_info.meta.get(KEY_IMAGE_FIELD)
if key_tag is None:
app_logger.warn(
"Object has reference tag, but image doesn't have key field. Object is skipped",
extra={
"figure_id": label.geometry.sly_id,
"image_id": image_id,
"image_name": image_name,
"dataset_name": dataset.name
})
continue
rect: sly.Rectangle = label.geometry.to_bbox()
reference = {
"image_id":
image_id,
"image_name":
image_name,
"dataset_name":
dataset.name,
"image_preview_url":
api.image.url(TEAM_ID, WORKSPACE_ID, PROJECT.id,
dataset.id, image_id),
"image_url":
image_info.full_storage_url,
KEY_IMAGE_FIELD:
key_tag,
"bbox": [rect.top, rect.left, rect.bottom, rect.right],
"geometry":
label.geometry.to_json()
}
result["references"][key_tag].append(reference)
progress.iters_done_report(len(batch))
result["all_keys"] = list(result["references"].keys())
file_local = os.path.join(my_app.data_dir, file_remote.lstrip("/"))
app_logger.info("Local file path: {!r}".format(file_local))
sly.fs.ensure_base_path(file_local)
sly.json.dump_json_file(result, file_local)
file_info = api.file.upload(TEAM_ID, file_local, file_remote)
api.task._set_custom_output(task_id,
file_info.id,
sly.fs.get_file_name_with_ext(file_remote),
description="JSON with reference items",
icon="zmdi zmdi-collection-text")
#zmdi-receipt
#zmdi-ungroup
#zmdi-collection-text
app_logger.info("Local file successfully uploaded to team files")
my_app.stop()
def start_import(api: sly.Api, task_id, context, state, app_logger):
fields = [
{"field": "data.destinationError", "payload": ""},
{"field": "data.uploadError", "payload": ""},
{"field": "data.uploadStarted", "payload": True},
{"field": "data.uploadedCount", "payload": 0},
{"field": "data.totalCount", "payload": 0},
{"field": "data.uploadProgress", "payload": 0},
{"field": "data.uploadDsName", "payload": ""},
{"field": "data.uploadedDsCount", "payload": 0},
{"field": "data.totalDsCount", "payload": 0},
{"field": "data.uploadDsProgress", "payload": 0},
]
api.app.set_fields(task_id, fields)
remote_dir = state["remoteDir"]
listing_flags = state["listingFlags"]
workspace_name = state["workspaceName"]
project_name = state["projectName"] #slugify(state["projectName"], lowercase=False, save_order=True)
if project_name == "":
_show_error(api, task_id, "data.destinationError", "Project name is not defined", app_logger)
return
#@TODO: will be added in future releases
add_to_existing_project = False #state["addToExisting"]
existing_meta = None
try:
workspace = api.workspace.get_info_by_name(TEAM_ID, workspace_name)
if workspace is None:
workspace = api.workspace.create(TEAM_ID, workspace_name)
app_logger.info("Workspace {!r} is created".format(workspace.name))
else:
app_logger.info("Workspace {!r} already exists".format(workspace.name))
project = api.project.get_info_by_name(workspace.id, project_name)
if project is None:
project = api.project.create(workspace.id, project_name)
app_logger.info("Project {!r} is created".format(project.name))
else:
_show_error(api, task_id, "data.destinationError", "Project {!r} already exists".format(project.name), app_logger)
return
if add_to_existing_project is False:
app_logger.warn("Project {!r} already exists. Allow add to existing project or change the name of "
"destination project. We recommend to upload to new project. Thus the existing project "
"will be safe. New name will be generated".format(project.name))
project = api.project.create(workspace.id, project_name, change_name_if_conflict=True)
else:
existing_meta_json = api.project.get_meta(project.id)
existing_meta = sly.ProjectMeta.from_json(existing_meta_json)
update_res_project_icon = None
fields = [
{"field": "data.resultProject", "payload": project.name},
{"field": "data.resultProjectId", "payload": project.id},
# {"field": "data.resultProjectPreviewUrl", "payload": 0},
]
api.app.set_fields(task_id, fields)
resp = requests.get(urljoin(remote_dir, 'meta.json'))
meta_json = resp.json()
meta = sly.ProjectMeta.from_json(meta_json)
if existing_meta is not None:
meta = existing_meta.merge(meta)
api.project.update_meta(project.id, meta.to_json())
datasets_to_upload = []
for ds_info, flags in zip(listing, listing_flags):
dataset_name = ds_info['name']
if flags["selected"] is False:
app_logger.info("Folder {!r} is not selected, it will be skipped".format(dataset_name))
continue
if flags["disabled"] is True:
app_logger.info("File {!r} is skipped".format(dataset_name))
continue
datasets_to_upload.append(dataset_name)
api.task.set_field(task_id, "data.totalDsCount", len(datasets_to_upload))
for index, dataset_name in enumerate(datasets_to_upload):
dataset = api.dataset.get_info_by_name(project.id, dataset_name)
if dataset is None:
dataset = api.dataset.create(project.id, dataset_name)
app_logger.info("Dataset {!r} is created".format(dataset.name))
else:
app_logger.warn("Dataset {!r} already exists. Uploading is skipped".format(dataset.name))
_increment_ds_progress(task_id, api, index + 1, len(datasets_to_upload))
continue
#img_dir = reduce(urljoin, [remote_dir, dataset_name, 'img'])
#ann_dir = reduce(urljoin, [remote_dir, dataset_name, 'ann'])
img_dir = os.path.join(remote_dir, dataset_name, 'img/')
ann_dir = os.path.join(remote_dir, dataset_name, 'ann/')
cwd, img_listing = htmllistparse.fetch_listing(img_dir, timeout=30)
uploaded_to_dataset = 0
fields = [
{"field": "data.totalCount", "payload": len(img_listing)},
{"field": "data.uploadDsName", "payload": dataset.name},
]
api.app.set_fields(task_id, fields)
task_progress = sly.Progress("Uploading dataset {!r}".format(dataset.name), len(img_listing))
for batch in sly.batched(img_listing, batch_size=50):
try:
names = []
image_urls_batch = []
annotations_batch = []
for file_entry in batch:
name = file_entry.name
try:
img_url = urljoin(img_dir, name) #'https://i.imgur.com/uFYNj9Z.jpg'
ann_url = urljoin(ann_dir, name + sly.ANN_EXT)
resp = requests.get(ann_url)
if resp.status_code == 404:
ann_url = urljoin(ann_dir, sly.fs.get_file_name(name) + sly.ANN_EXT)
resp = requests.get(ann_url)
resp.raise_for_status()
ann_json = resp.json()
ann = sly.Annotation.from_json(ann_json, meta)
except Exception as e:
app_logger.warn("Image {!r} and annotation {!r} are skipped due to error: {}"
.format(img_url, ann_url, repr(e)))
continue
names.append(name)
image_urls_batch.append(img_url)
annotations_batch.append(ann)
img_infos = api.image.upload_links(dataset.id, names, image_urls_batch)
uploaded_ids = [img_info.id for img_info in img_infos]
api.annotation.upload_anns(uploaded_ids, annotations_batch)
uploaded_to_dataset += len(uploaded_ids)
except Exception as e:
app_logger.warn("Batch ({} items) of images is skipped due to error: {}"
.format(len(batch), repr(e)))
finally:
task_progress.iters_done_report(len(batch))
_increment_task_progress(task_id, api, task_progress)
#only once + to check the image urls are loaded correctly
if update_res_project_icon is None:
pinfo = api.project.get_info_by_id(project.id)
if pinfo.reference_image_url is None:
raise RuntimeError("Preview image is not accessible. Check that image URLs are public.")
update_res_project_icon = api.image.preview_url(pinfo.reference_image_url, 100, 100),
api.task.set_field(task_id, "data.resultProjectPreviewUrl", update_res_project_icon)
_increment_ds_progress(task_id, api, index + 1, len(datasets_to_upload))
app_logger.info("Dataset {!r} is uploaded: {} images with annotations"
.format(dataset.name, uploaded_to_dataset))
except Exception as e:
app_logger.error(repr(e))
api.task.set_field(task_id, "data.uploadError", repr(e))
api.task.set_output_project(task_id, project.id, project.name)
my_app.stop()
def main():
api = sly.Api.from_env()
# read source project
src_project = api.project.get_info_by_id(PROJECT_ID)
if src_project.type != str(sly.ProjectType.IMAGES):
raise RuntimeError("Project {!r} has type {!r}. App works only with type {!r}"
.format(src_project.name, src_project.type, sly.ProjectType.IMAGES))
src_project_meta_json = api.project.get_meta(src_project.id)
src_project_meta = sly.ProjectMeta.from_json(src_project_meta_json)
# create destination project
DST_PROJECT_NAME = "{} (rasterized)".format(src_project.name)
dst_project = api.project.create(WORKSPACE_ID, DST_PROJECT_NAME, description="rasterized", change_name_if_conflict=True)
sly.logger.info('Destination project is created.', extra={'project_id': dst_project.id, 'project_name': dst_project.name})
# mapping polygons -> bitmaps
new_classes_lst = []
for cls in src_project_meta.obj_classes:
if need_convert(cls.geometry_type):
new_class = cls.clone(geometry_type=sly.Bitmap)
else:
new_class = cls.clone()
new_classes_lst.append(new_class)
dst_classes = sly.ObjClassCollection(new_classes_lst)
# create destination meta
dst_project_meta = src_project_meta.clone(obj_classes=dst_classes)
api.project.update_meta(dst_project.id, dst_project_meta.to_json())
def convert_to_nonoverlapping(src_ann: sly.Annotation) -> sly.Annotation:
common_img = np.zeros(src_ann.img_size, np.int32) # size is (h, w)
for idx, lbl in enumerate(src_ann.labels, start=1):
if need_convert(lbl.obj_class.geometry_type):
if allow_render_non_spatial_for_any_shape(lbl) == True:
lbl.draw(common_img, color=idx)
else:
sly.logger.warn(
"Object of class {!r} (class shape: {!r}) has non spatial shape {!r}. It will not be rendered."
.format(lbl.obj_class.name,
lbl.obj_class.geometry_type.geometry_name(),
lbl.geometry.geometry_name()))
new_labels = []
for idx, lbl in enumerate(src_ann.labels, start=1):
new_cls = dst_project_meta.obj_classes.get(lbl.obj_class.name)
if not need_convert(lbl.obj_class.geometry_type):
new_lbl = lbl.clone(obj_class=new_cls)
new_labels.append(new_lbl)
else:
if allow_render_non_spatial_for_any_shape(lbl) == False:
continue
mask = common_img == idx
if np.any(mask): # figure may be entirely covered by others
g = lbl.geometry
new_bmp = sly.Bitmap(data=mask,
labeler_login=g.labeler_login,
updated_at=g.updated_at,
created_at=g.created_at)
new_lbl = lbl.clone(geometry=new_bmp, obj_class=new_cls)
new_labels.append(new_lbl)
return src_ann.clone(labels=new_labels)
for ds_info in api.dataset.get_list(src_project.id):
ds_progress = sly.Progress('Processing dataset: {!r}/{!r}'.format(src_project.name, ds_info.name),
total_cnt=ds_info.images_count)
dst_dataset = api.dataset.create(dst_project.id, ds_info.name)
img_infos_all = api.image.get_list(ds_info.id)
for img_infos in sly.batched(img_infos_all):
img_names, img_ids, img_metas = zip(*((x.name, x.id, x.meta) for x in img_infos))
ann_infos = api.annotation.download_batch(ds_info.id, img_ids)
anns = [sly.Annotation.from_json(x.annotation, src_project_meta) for x in ann_infos]
new_anns = [convert_to_nonoverlapping(ann) for ann in anns]
new_img_infos = api.image.upload_ids(dst_dataset.id, img_names, img_ids, metas=img_metas)
new_img_ids = [x.id for x in new_img_infos]
api.annotation.upload_anns(new_img_ids, new_anns)
ds_progress.iters_done_report(len(img_infos))
api.task.set_output_project(task_id, dst_project.id, dst_project.name)
def calc(api: sly.Api, task_id, context, state, app_logger):
global progress, sum_class_area_per_image, sum_class_count_per_image, count_images_with_class
workspace = api.workspace.get_info_by_id(WORKSPACE_ID)
project = None
datasets = None
if DATASET_ID is not None:
dataset = api.dataset.get_info_by_id(DATASET_ID)
datasets = [dataset]
project = api.project.get_info_by_id(PROJECT_ID)
if datasets is None:
datasets = api.dataset.get_list(PROJECT_ID)
ds_images, sample_count = sample_images(api, datasets)
fields = [
{
"field": "data.projectName",
"payload": project.name
},
{
"field": "data.projectId",
"payload": project.id
},
{
"field": "data.projectPreviewUrl",
"payload": api.image.preview_url(project.reference_image_url, 100,
100)
},
{
"field": "data.samplePercent",
"payload": SAMPLE_PERCENT
},
{
"field": "data.sampleCount",
"payload": sample_count
},
]
api.task.set_fields(task_id, fields)
meta_json = api.project.get_meta(project.id)
meta = sly.ProjectMeta.from_json(meta_json)
colors_warning = meta.obj_classes.validate_classes_colors()
# list classes
class_names = ["unlabeled"]
class_colors = [[0, 0, 0]]
class_indices_colors = [[0, 0, 0]]
_name_to_index = {}
table_columns = [
"image id", "image", "dataset", "height", "width", "channels",
"unlabeled"
]
for idx, obj_class in enumerate(meta.obj_classes):
class_names.append(obj_class.name)
class_colors.append(obj_class.color)
class_index = idx + 1
class_indices_colors.append([class_index, class_index, class_index])
_name_to_index[obj_class.name] = class_index
table_columns.append(get_col_name_area(obj_class.name,
obj_class.color))
table_columns.append(
get_col_name_count(obj_class.name, obj_class.color))
sum_class_area_per_image = [0] * len(class_names)
sum_class_count_per_image = [0] * len(class_names)
count_images_with_class = [0] * len(class_names)
#count_images_with_class[0] = 1 # for unlabeled
api.task.set_field(task_id, "data.table.columns", table_columns)
all_stats = []
task_progress = sly.Progress("Stats", sample_count, app_logger)
for dataset_id, images in ds_images.items():
dataset = api.dataset.get_info_by_id(dataset_id)
for batch in sly.batched(images, batch_size=BATCH_SIZE):
batch_stats = []
image_ids = [image_info.id for image_info in batch]
ann_infos = api.annotation.download_batch(dataset_id, image_ids)
ann_jsons = [ann_info.annotation for ann_info in ann_infos]
for info, ann_json in zip(batch, ann_jsons):
ann = sly.Annotation.from_json(ann_json, meta)
render_idx_rgb = np.zeros(ann.img_size + (3, ), dtype=np.uint8)
render_idx_rgb[:] = BG_COLOR
ann.draw_class_idx_rgb(render_idx_rgb, _name_to_index)
stat_area = sly.Annotation.stat_area(render_idx_rgb,
class_names,
class_indices_colors)
stat_count = ann.stat_class_count(class_names)
if stat_area["unlabeled"] > 0:
stat_count["unlabeled"] = 1
table_row = []
table_row.append(info.id)
table_row.append(
'<a href="{0}" rel="noopener noreferrer" target="_blank">{1}</a>'
.format(
api.image.url(TEAM_ID, WORKSPACE_ID, project.id,
dataset.id, info.id), info.name))
table_row.append(dataset.name)
area_unl = stat_area["unlabeled"] if not np.isnan(
stat_area["unlabeled"]) else 0
table_row.extend([
stat_area["height"], stat_area["width"],
stat_area["channels"],
round(area_unl, 2)
])
resolutions_count["{}x{}x{}".format(
stat_area["height"], stat_area["width"],
stat_area["channels"])] += 1
for idx, class_name in enumerate(class_names):
cur_area = stat_area[class_name] if not np.isnan(
stat_area[class_name]) else 0
cur_count = stat_count[class_name] if not np.isnan(
stat_count[class_name]) else 0
sum_class_area_per_image[idx] += cur_area
sum_class_count_per_image[idx] += cur_count
count_images_with_class[
idx] += 1 if stat_count[class_name] > 0 else 0
if class_name == "unlabeled":
continue
table_row.append(round(cur_area, 2))
table_row.append(round(cur_count, 2))
if len(table_row) != len(table_columns):
raise RuntimeError("Values for some columns are missed")
batch_stats.append(table_row)
all_stats.extend(batch_stats)
progress += len(batch_stats)
fields = [{
"field": "data.progress",
"payload": int(progress * 100 / sample_count)
}, {
"field": "data.table.data",
"payload": batch_stats,
"append": True
}]
api.task.set_fields(task_id, fields)
task_progress.iters_done_report(len(batch_stats))
# average nonzero class area per image
with np.errstate(divide='ignore'):
avg_nonzero_area = np.divide(sum_class_area_per_image,
count_images_with_class)
avg_nonzero_count = np.divide(sum_class_count_per_image,
count_images_with_class)
avg_nonzero_area = np.where(np.isnan(avg_nonzero_area), None,
avg_nonzero_area)
avg_nonzero_count = np.where(np.isnan(avg_nonzero_count), None,
avg_nonzero_count)
fig = go.Figure(data=[
go.Bar(name='Area %',
x=class_names,
y=avg_nonzero_area,
yaxis='y',
offsetgroup=1),
go.Bar(name='Count',
x=class_names,
y=avg_nonzero_count,
yaxis='y2',
offsetgroup=2)
],
layout={
'yaxis': {
'title': 'Area'
},
'yaxis2': {
'title': 'Count',
'overlaying': 'y',
'side': 'right'
}
})
# Change the bar mode
fig.update_layout(barmode='group') # , legend_orientation="h")
# images count with/without classes
images_with_count = []
images_without_count = []
images_with_count_text = []
images_without_count_text = []
for idx, class_name in enumerate(class_names):
#if class_name == "unlabeled":
# continue
with_count = count_images_with_class[
idx] #- 1 if class_name == "unlabeled" else count_images_with_class[idx]
without_count = sample_count - with_count
images_with_count.append(with_count)
images_without_count.append(without_count)
images_with_count_text.append("{} ({:.2f} %)".format(
with_count, with_count * 100 / sample_count))
images_without_count_text.append("{} ({:.2f} %)".format(
without_count, without_count * 100 / sample_count))
if len(class_names) != len(images_with_count) or len(class_names) != len(images_with_count_text) or \
len(class_names) != len(images_without_count) or len(class_names) != len(images_without_count_text):
raise RuntimeError("Class names are inconsistent with images counting")
fig_with_without_count = go.Figure(data=[
go.Bar(name='# of images that have class',
x=class_names,
y=images_with_count,
text=images_with_count_text),
go.Bar(name='# of images that do not have class',
x=class_names,
y=images_without_count,
text=images_without_count_text)
], )
fig_with_without_count.update_layout(
barmode='stack') # , legend_orientation="h")
# barchart resolution
resolution_labels = []
resolution_values = []
resolution_percent = []
for label, value in sorted(resolutions_count.items(),
key=lambda item: item[1],
reverse=True):
resolution_labels.append(label)
resolution_values.append(value)
if len(resolution_labels) > 10:
resolution_labels = resolution_labels[:10]
resolution_labels.append("other")
other_value = sum(resolution_values[10:])
resolution_values = resolution_values[:10]
resolution_values.append(other_value)
resolution_percent = [
round(v * 100 / sample_count) for v in resolution_values
]
#df_resolution = pd.DataFrame({'resolution': resolution_labels, 'count': resolution_values, 'percent': resolution_percent})
pie_resolution = go.Figure(
data=[go.Pie(labels=resolution_labels, values=resolution_values)])
#pie_resolution = px.pie(df_resolution, names='resolution', values='count')
# @TODO: hotfix - pie chart do not refreshes automatically
fig.update_layout(autosize=False, height=450)
fig_with_without_count.update_layout(autosize=False, height=450)
pie_resolution.update_layout(autosize=False, height=450)
# overview table
overviewTable = {"columns": overview_columns, "data": []}
_overview_data = []
for idx, (class_name,
class_color) in enumerate(zip(class_names, class_colors)):
row = [
idx,
color_text(class_name, class_color),
count_images_with_class[
idx], # - 1 if class_name == "unlabeled" else count_images_with_class[idx],
sum_class_count_per_image[idx],
None if avg_nonzero_area[idx] is None else round(
avg_nonzero_area[idx], 2),
None if avg_nonzero_count[idx] is None else round(
avg_nonzero_count[idx], 2)
]
_overview_data.append(row)
overviewTable["data"] = _overview_data
# save report to file *.lnk (link to report)
report_name = "{}.lnk".format(project.name)
local_path = os.path.join(my_app.data_dir, report_name)
sly.fs.ensure_base_path(local_path)
with open(local_path, "w") as text_file:
print(my_app.app_url, file=text_file)
remote_path = "/reports/classes_stats/{}/{}/{}".format(
USER_LOGIN, workspace.name, report_name)
remote_path = api.file.get_free_name(TEAM_ID, remote_path)
report_name = sly.fs.get_file_name_with_ext(remote_path)
file_info = api.file.upload(TEAM_ID, local_path, remote_path)
report_url = api.file.get_url(file_info.id)
fields = [
{
"field": "data.overviewTable",
"payload": overviewTable
},
{
"field": "data.avgAreaCount",
"payload": json.loads(fig.to_json())
},
{
"field": "data.imageWithClassCount",
"payload": json.loads(fig_with_without_count.to_json())
},
{
"field": "data.resolutionsCount",
"payload": json.loads(pie_resolution.to_json())
},
{
"field": "data.loading0",
"payload": False
},
{
"field": "data.loading1",
"payload": False
},
{
"field": "data.loading2",
"payload": False
},
{
"field": "data.loading3",
"payload": False
},
{
"field": "state.showDialog",
"payload": True
},
{
"field": "data.savePath",
"payload": remote_path
},
{
"field": "data.reportName",
"payload": report_name
},
{
"field": "data.reportUrl",
"payload": report_url
},
]
api.task.set_fields(task_id, fields)
api.task.set_output_report(task_id, file_info.id, report_name)
my_app.stop()
def video_stats(api: sly.Api, task_id, context, state, app_logger):
project_info = api.project.get_info_by_id(PROJECT_ID)
key_id_map = KeyIdMap()
if project_info is None:
raise RuntimeError("Project with ID {!r} not found".format(PROJECT_ID))
if project_info.type != str(sly.ProjectType.VIDEOS):
raise TypeError("Project type is {!r}, but have to be {!r}".format(
project_info.type, sly.ProjectType.VIDEOS))
meta_json = api.project.get_meta(project_info.id)
meta = sly.ProjectMeta.from_json(meta_json)
if len(meta.obj_classes) == 0 and CLASSES in stat_type:
app_logger.warn("Project {!r} have no classes".format(
project_info.name))
if len(meta.tag_metas) == 0 and TAGS in stat_type:
app_logger.warn("Project {!r} have no tags".format(project_info.name))
if len(meta.obj_classes) == 0 and len(meta.tag_metas) == 0:
app_logger.warn("Project {!r} have no classes and tags".format(
project_info.name))
my_app.stop()
if CLASSES in stat_type:
classes = []
counter = {}
classes_id = []
for idx, curr_class in enumerate(meta.obj_classes):
classes.append(curr_class.name)
classes_id.append(idx)
counter[curr_class.name] = 0
columns_classes = [
FIRST_STRING, CLASS_NAME, 'total_objects', 'total_figures',
'total_frames'
]
data = {
FIRST_STRING: classes_id,
CLASS_NAME: classes,
'total_objects': [0] * len(classes),
'total_figures': [0] * len(classes),
'total_frames': [0] * len(classes)
}
if TAGS in stat_type:
columns = [FIRST_STRING, TAG_COLOMN]
columns_for_values = [FIRST_STRING, TAG_COLOMN, TAG_VALUE_COLOMN]
columns_frame_tag = [FIRST_STRING,
TAG_COLOMN] # ===========frame_tags=======
columns_frame_tag_values = [
FIRST_STRING, TAG_COLOMN, TAG_VALUE_COLOMN
] # ===========frame_tags=======
columns_object_tag = [FIRST_STRING,
TAG_COLOMN] # ===========object_tags=======
columns_object_tag_values = [
FIRST_STRING, TAG_COLOMN, TAG_VALUE_COLOMN
] # ===========object_tags=======
columns.extend([TOTAL])
columns_for_values.extend([TOTAL])
columns_frame_tag.extend([TOTAL, TOTAL + COUNT_SUFFIX
]) # ===========frame_tags=======
columns_frame_tag_values.extend([TOTAL
]) # ===========frame_tags=======
columns_object_tag.extend([TOTAL]) # ===========object_tags=======
columns_object_tag_values.extend([TOTAL
]) # ===========object_tags=======
datasets_counts = []
datasets_values_counts = []
datasets_frame_tag_counts = [] # ===========frame_tags=======
datasets_frame_tag_values_counts = [] # ===========frame_tags=======
datasets_object_tag_counts = [] # ===========object_tags=======
datasets_object_tag_values_counts = [] # ===========object_tags=======
for dataset in api.dataset.get_list(PROJECT_ID):
if CLASSES in stat_type:
columns_classes.extend([
dataset.name + OBJECTS, dataset.name + FIGURES,
dataset.name + FRAMES
])
classes_counter = copy.deepcopy(counter)
figures_counter = copy.deepcopy(counter)
frames_counter = copy.deepcopy(counter)
data[dataset.name + OBJECTS] = []
data[dataset.name + FIGURES] = []
data[dataset.name + FRAMES] = []
videos = api.video.get_list(dataset.id)
progress_classes = sly.Progress("Processing video classes ...",
len(videos), app_logger)
if TAGS in stat_type:
columns.extend([dataset.name])
ds_property_tags = defaultdict(int)
columns_for_values.extend([dataset.name])
ds_property_tags_values = defaultdict(lambda: defaultdict(int))
# ===========frame_tags=========================================
columns_frame_tag.extend(
[dataset.name, dataset.name + COUNT_SUFFIX])
ds_frame_tags = defaultdict(int)
ds_frame_tags_counter = defaultdict(int)
columns_frame_tag_values.extend([dataset.name])
ds_frame_tags_values = defaultdict(lambda: defaultdict(int))
ds_frame_tags_values_counter = defaultdict(
lambda: defaultdict(int))
# ===========frame_tags=========================================
# ===========object_tags=========================================
columns_object_tag.extend([dataset.name])
ds_object_tags = defaultdict(int)
columns_object_tag_values.extend([dataset.name])
ds_object_tags_values = defaultdict(lambda: defaultdict(int))
# ===========object_tags=========================================
videos = api.video.get_list(dataset.id)
progress_tags = sly.Progress("Processing video tags ...",
len(videos), app_logger)
for batch in sly.batched(videos, batch_size=10):
for video_info in batch:
ann_info = api.video.annotation.download(video_info.id)
ann = sly.VideoAnnotation.from_json(ann_info, meta, key_id_map)
if CLASSES in stat_type:
classes_counter, figures_counter, frames_counter = items_counter(
ann, classes_counter, figures_counter, frames_counter)
progress_classes.iter_done_report()
if TAGS in stat_type:
process_video_annotation(ann, ds_property_tags)
process_video_annotation_tags_values(
ann, ds_property_tags_values)
process_video_ann_frame_tags(
ann, ds_frame_tags,
ds_frame_tags_counter) # ===========frame_tags=======
process_video_ann_frame_tags_vals(
ann,
ds_frame_tags_values) # ===========frame_tags=======
process_video_ann_object_tags(
ann, ds_object_tags) # ===========object_tags=======
process_video_ann_object_tags_vals(
ann,
ds_object_tags_values) # ===========object_tags=======
progress_tags.iter_done_report()
if CLASSES in stat_type:
data = data_counter(data, dataset, classes, classes_counter,
figures_counter, frames_counter)
if TAGS in stat_type:
datasets_counts.append((dataset.name, ds_property_tags))
datasets_values_counts.append(
(dataset.name, ds_property_tags_values))
datasets_frame_tag_counts.append(
(dataset.name, ds_frame_tags)) # ===========frame_tags=======
datasets_frame_tag_values_counts.append(
(dataset.name,
ds_frame_tags_values)) # ===========frame_tags=======
datasets_object_tag_counts.append(
(dataset.name,
ds_object_tags)) # ===========object_tags=======
datasets_object_tag_values_counts.append(
(dataset.name,
ds_object_tags_values)) # ===========object_tags=======
if CLASSES in stat_type:
classes.append(TOTAL)
data[FIRST_STRING].append(len(data[FIRST_STRING]))
for key, val in data.items():
if key == CLASS_NAME or key == FIRST_STRING:
continue
data[key].append(sum(val))
df_classes = pd.DataFrame(data, columns=columns_classes, index=classes)
print(df_classes)
if TAGS in stat_type:
# =========property_tags===============================================================
df = get_pd_tag_stat(meta, datasets_counts, columns)
print('Total video tags stats')
print(df)
# =========property_tags_values=========================================================
df_values = get_pd_tag_values_stat(datasets_values_counts,
columns_for_values)
print('Total video tags values stats')
print(df_values)
# =========frame_tag=====================================================================
data_frame_tags = []
for idx, tag_meta in enumerate(meta.tag_metas):
name = tag_meta.name
row_frame_tags = [idx, name]
row_frame_tags.extend([0, 0])
for ds_name, ds_frame_tags in datasets_frame_tag_counts:
row_frame_tags.extend(
[ds_frame_tags[name], ds_frame_tags_counter[name]])
row_frame_tags[2] += ds_frame_tags[name]
row_frame_tags[3] += ds_frame_tags_counter[name]
data_frame_tags.append(row_frame_tags)
df_frame_tags = pd.DataFrame(data_frame_tags,
columns=columns_frame_tag)
total_row = list(df_frame_tags.sum(axis=0))
total_row[0] = len(df_frame_tags)
total_row[1] = TOTAL
df_frame_tags.loc[len(df_frame_tags)] = total_row
print('Total frame tags stats')
print(df_frame_tags)
# =========frame_tags_values=============================================================
df_frame_tags_values = get_pd_tag_values_stat(
datasets_frame_tag_values_counts, columns_frame_tag_values)
print('Total frame tags values stats')
print(df_frame_tags_values)
# ==========object_tag================================================================
df_object_tags = get_pd_tag_stat(meta, datasets_object_tag_counts,
columns_object_tag)
print('Total object tags stats')
print(df_object_tags)
# =========object_tags_values=========================================================
df_object_values = get_pd_tag_values_stat(
datasets_object_tag_values_counts, columns_object_tag_values)
print('Total object tags values stats')
print(df_object_values)
report_name = "{}_{}.lnk".format(PROJECT_ID, project_info.name)
local_path = os.path.join(my_app.data_dir, report_name)
sly.fs.ensure_base_path(local_path)
with open(local_path, "w") as text_file:
print(my_app.app_url, file=text_file)
remote_path = "/reports/video_stat/{}".format(report_name)
remote_path = api.file.get_free_name(TEAM_ID, remote_path)
report_name = sly.fs.get_file_name_with_ext(remote_path)
file_info = api.file.upload(TEAM_ID, local_path, remote_path)
report_url = api.file.get_url(file_info.id)
fields = [
{
"field": "data.loading",
"payload": False
},
{
"field": "data.classesTable",
"payload": json.loads(df_classes.to_json(orient="split"))
},
{
"field": "data.tagsTable",
"payload": json.loads(df.to_json(orient="split"))
},
{
"field": "data.savePath",
"payload": remote_path
},
{
"field": "data.reportName",
"payload": report_name
},
{
"field": "data.reportUrl",
"payload": report_url
},
]
api.task.set_fields(task_id, fields)
api.task.set_output_report(task_id, file_info.id, report_name)
my_app.stop()
def export_coco(api: sly.Api, task_id, context, state, app_logger):
datasets = [ds for ds in api.dataset.get_list(g.project_id)]
for dataset in datasets:
coco_dataset_dir = os.path.join(g.coco_base_dir, dataset.name)
mkdir(os.path.join(coco_dataset_dir))
ann_dir = os.path.join(g.coco_base_dir, 'annotations')
mkdir(ann_dir)
images = api.image.get_list(dataset.id)
for batch in sly.batched(images):
image_ids = [image_info.id for image_info in batch]
image_names = [image_info.name for image_info in batch]
image_paths = [
os.path.join(coco_dataset_dir, image_info.name)
for image_info in batch
]
api.image.download_paths(dataset.id, image_ids, image_paths)
ann_infos = api.annotation.download_batch(dataset.id, image_ids)
anns = [
sly.Annotation.from_json(x.annotation, g.meta)
for x in ann_infos
]
meta = convert_geometry.prepare_meta(g.meta)
new_anns = [
convert_geometry.convert_annotation(ann, meta) for ann in anns
]
data = dict(
info=dict(
description=None,
url=None,
version=1.0,
year=dataset.created_at[:4],
contributor=g.user.name,
date_created=dataset.created_at,
),
licenses=[dict(
url=None,
id=0,
name=None,
)],
images=[
# license, url, file_name, height, width, date_captured, id
],
type="instances",
annotations=[
# segmentation, area, iscrowd, image_id, bbox, category_id, id
],
categories=[
# supercategory, id, name
],
)
for image_info, ann in zip(batch, new_anns):
data["images"].append(
dict(
license=None,
url=image_info.
full_storage_url, # coco_url, flickr_url
filename=image_info.name,
height=image_info.height,
width=image_info.width,
date_captured=image_info.created_at,
id=image_info.id,
))
for label in ann.labels:
segmentation = label.geometry.to_json(
)["points"]["exterior"]
segmentation = [
coord for sublist in segmentation for coord in sublist
]
bbox = label.geometry.to_bbox().to_json(
)["points"]["exterior"]
bbox = [coord for sublist in bbox for coord in sublist]
x, y, max_x, max_y = bbox
width = max_x - x
height = max_y - y
bbox = (x, y, width, height)
data["annotations"].append(
dict(
segmentation=[segmentation],
area=label.geometry.area, # wrong?
iscrowd=0,
image_id=image_info.id,
bbox=bbox,
category_id=None,
id=None, # label.id?
))
data["categories"].append(
dict(supercategory=None,
id=None,
name=label.obj_class.name))
dump_json_file(data,
os.path.join(ann_dir, f"instances_{dataset.name}.json"))
g.my_app.stop()
dst_project = api.project.create(workspace.id,
dst_project_name,
change_name_if_conflict=True)
if dst_project.name != dst_project_name:
sly.logger.warn(
"Project with name={!r} already exists. Project is saved with autogenerated name {!r}"
.format(dst_project_name, dst_project.name))
api.project.update_meta(dst_project.id, meta.to_json())
progress = sly.Progress("Splitting images",
api.project.get_images_count(src_project.id))
for src_dataset in api.dataset.get_list(src_project.id):
dst_dataset = api.dataset.create(dst_project.id, src_dataset.name)
images = api.image.get_list(src_dataset.id)
for batch in sly.batched(images, batch_size=BATCH_SIZE):
image_ids = [image_info.id for image_info in batch] # debug [281152]
image_names = [image_info.name for image_info in batch]
ann_infos = api.annotation.download_batch(src_dataset.id, image_ids)
images = api.image.download_nps(src_dataset.id, image_ids)
for ann_info, image, image_name in zip(ann_infos, images, image_names):
window_index = 0
ann = sly.Annotation.from_json(ann_info.annotation, meta)
crop_names = []
crop_images = []
crop_anns = []
slider = SlidingWindowsFuzzy([window_height, window_width],
[overlap_y, overlap_x],
SW_BORDER_STRATEGY)
def download_project_objects(api: sly.Api, task_id, context, state,
app_logger):
try:
if not dir_exists(g.project_dir):
mkdir(g.project_dir)
project_meta_path = os.path.join(g.project_dir, "meta.json")
g.project_meta = convert_object_tags(g.project_meta)
project_meta_json = g.project_meta.to_json()
dump_json_file(project_meta_json, project_meta_path)
datasets = api.dataset.get_list(g.project_id)
for dataset in datasets:
ds_dir = os.path.join(g.project_dir, dataset.name)
img_dir = os.path.join(ds_dir, "img")
ann_dir = os.path.join(ds_dir, "ann")
mkdir(ds_dir)
mkdir(img_dir)
mkdir(ann_dir)
images_infos = api.image.get_list(dataset.id)
download_progress = get_progress_cb(
progress_index, "Download project",
g.project_info.items_count * 2)
for batch in sly.batched(images_infos):
image_ids = [image_info.id for image_info in batch]
image_names = [image_info.name for image_info in batch]
ann_infos = api.annotation.download_batch(
dataset.id, image_ids, progress_cb=download_progress)
image_nps = api.image.download_nps(
dataset.id, image_ids, progress_cb=download_progress)
anns = [
sly.Annotation.from_json(ann_info.annotation,
g.project_meta)
for ann_info in ann_infos
]
selected_classes = get_selected_classes_from_ui(
state["classesSelected"])
crops = crop_and_resize_objects(image_nps, anns, state,
selected_classes,
image_names)
crop_nps, crop_anns, crop_names = unpack_crops(
crops, image_names)
crop_anns = copy_tags(crop_anns)
write_images(crop_nps, crop_names, img_dir)
dump_anns(crop_anns, crop_names, ann_dir)
reset_progress(progress_index)
global project_fs
project_fs = sly.Project(g.project_dir, sly.OpenMode.READ)
g.images_infos = create_img_infos(project_fs)
except Exception as e:
reset_progress(progress_index)
raise e
items_count = g.project_stats["objects"]["total"]["objectsInDataset"]
train_percent = 80
train_count = int(items_count / 100 * train_percent)
random_split = {
"count": {
"total": items_count,
"train": train_count,
"val": items_count - train_count
},
"percent": {
"total": 100,
"train": train_percent,
"val": 100 - train_percent
},
"shareImagesBetweenSplits": False,
"sliderDisabled": False,
}
fields = [
{
"field": "data.done1",
"payload": True
},
{
"field": "state.collapsed2",
"payload": False
},
{
"field": "state.disabled2",
"payload": False
},
{
"field": "state.activeStep",
"payload": 2
},
{
"field": "data.totalImagesCount",
"payload": items_count
},
{
"field": "state.randomSplit",
"payload": random_split
},
]
g.api.app.set_fields(g.task_id, fields)
def do(**kwargs):
api = sly.Api.from_env()
src_project = api.project.get_info_by_id(PROJECT_ID)
if src_project.type != str(sly.ProjectType.IMAGES):
raise Exception(
"Project {!r} has type {!r}. App works only with type {!r}".format(
src_project.name, src_project.type, sly.ProjectType.IMAGES))
src_project_meta_json = api.project.get_meta(src_project.id)
src_project_meta = sly.ProjectMeta.from_json(src_project_meta_json)
# check that project has anyshape classes
find_anyshape = False
new_classes_lst = []
for cls in src_project_meta.obj_classes:
if cls.geometry_type == sly.AnyGeometry:
find_anyshape = True
continue
new_classes_lst.append(cls.clone())
dst_classes = sly.ObjClassCollection(new_classes_lst)
if find_anyshape is False:
raise Exception(
"Project {!r} doesn't have classes with shape \"Any\"".format(
src_project.name))
# create destination project
dst_name = src_project.name if _SUFFIX in src_project.name else src_project.name + _SUFFIX
dst_project = api.project.create(WORKSPACE_ID,
dst_name,
description=_SUFFIX,
change_name_if_conflict=True)
sly.logger.info('Destination project is created.',
extra={
'project_id': dst_project.id,
'project_name': dst_project.name
})
dst_project_meta = src_project_meta.clone(obj_classes=dst_classes)
api.project.update_meta(dst_project.id, dst_project_meta.to_json())
def convert_annotation(src_ann, dst_project_meta):
new_labels = []
for idx, lbl in enumerate(src_ann.labels):
lbl: sly.Label
if lbl.obj_class.geometry_type == sly.AnyGeometry:
actual_geometry = type(lbl.geometry)
new_class_name = "{}_{}".format(
lbl.obj_class.name, actual_geometry.geometry_name())
new_class = dst_project_meta.get_obj_class(new_class_name)
if new_class is None:
new_class = sly.ObjClass(name=new_class_name,
geometry_type=actual_geometry,
color=sly.color.random_rgb())
dst_project_meta = dst_project_meta.add_obj_class(
new_class)
api.project.update_meta(dst_project.id,
dst_project_meta.to_json())
new_labels.append(lbl.clone(obj_class=new_class))
else:
new_labels.append(lbl)
return src_ann.clone(labels=new_labels), dst_project_meta
for ds_info in api.dataset.get_list(src_project.id):
ds_progress = sly.Progress('Dataset: {!r}'.format(ds_info.name),
total_cnt=ds_info.images_count)
dst_dataset = api.dataset.create(dst_project.id, ds_info.name)
img_infos_all = api.image.get_list(ds_info.id)
for img_infos in sly.batched(img_infos_all):
img_names, img_ids, img_metas = zip(*((x.name, x.id, x.meta)
for x in img_infos))
ann_infos = api.annotation.download_batch(ds_info.id, img_ids)
anns = [
sly.Annotation.from_json(x.annotation, src_project_meta)
for x in ann_infos
]
new_anns = []
for ann in anns:
new_ann, dst_project_meta = convert_annotation(
ann, dst_project_meta)
new_anns.append(new_ann)
new_img_infos = api.image.upload_ids(dst_dataset.id,
img_names,
img_ids,
metas=img_metas)
new_img_ids = [x.id for x in new_img_infos]
api.annotation.upload_anns(new_img_ids, new_anns)
ds_progress.iters_done_report(len(img_infos))
api.task.set_output_project(task_id, dst_project.id, dst_project.name)
my_app.stop()
def main():
global PROJECT, RES_PROJECT, RES_PROJECT_NAME
PROJECT = api.project.get_info_by_id(PROJECT_ID)
if RES_PROJECT_NAME == "":
RES_PROJECT_NAME = PROJECT.name
read_and_validate_project_meta()
read_csv_and_create_index()
RES_PROJECT = api.project.create(WORKSPACE_ID,
RES_PROJECT_NAME,
change_name_if_conflict=True)
my_app.logger.info("Result Project is created (name={!r}; id={})".format(
RES_PROJECT.name, RES_PROJECT.id))
if ASSIGN_AS == "tags":
add_tags_to_meta()
api.project.update_meta(RES_PROJECT.id, RES_META.to_json())
progress = sly.Progress("Processing",
PROJECT.images_count,
ext_logger=my_app.logger)
for dataset in api.dataset.get_list(PROJECT.id):
res_dataset = api.dataset.create(RES_PROJECT.id, dataset.name)
ds_images = api.image.get_list(dataset.id)
for batch in sly.batched(ds_images):
image_ids = [image_info.id for image_info in batch]
image_names = [image_info.name for image_info in batch]
image_metas = [image_info.meta for image_info in batch]
ann_infos = api.annotation.download_batch(dataset.id, image_ids)
anns = [
sly.Annotation.from_json(ann_info.annotation, META)
for ann_info in ann_infos
]
original_ids = []
res_image_names = []
res_anns = []
res_metas = []
for image_id, image_name, image_meta, ann in zip(
image_ids, image_names, image_metas, anns):
tag: sly.Tag = ann.img_tags.get(IMAGE_TAG_NAME)
if tag is None:
my_app.logger.warn(
"Image {!r} in dataset {!r} doesn't have tag {!r}. Image is skipped"
.format(image_name, dataset.name, IMAGE_TAG_NAME))
progress.iter_done_report()
continue
csv_row = CSV_INDEX.get(str(tag.value), None)
if csv_row is None:
my_app.logger.warn(
"Match not found (id={}, name={!r}, dataset={!r}, tag_value={!r}). Image is skipped"
.format(image_id, image_name, dataset.name,
str(tag.value)))
progress.iter_done_report()
continue
res_ann = ann.clone()
res_meta = image_meta.copy()
if ASSIGN_AS == "tags":
res_ann = assign_csv_row_as_tags(image_id, image_name,
res_ann, csv_row)
else: # metadata
res_meta = assign_csv_row_as_metadata(
image_id, image_name, image_meta, csv_row)
original_ids.append(image_id)
res_image_names.append(image_name)
res_anns.append(res_ann)
res_metas.append(res_meta)
res_image_infos = api.image.upload_ids(res_dataset.id,
res_image_names,
original_ids,
metas=res_metas)
res_image_ids = [image_info.id for image_info in res_image_infos]
api.annotation.upload_anns(res_image_ids, res_anns)
progress.iters_done_report(len(res_image_ids))
api.task.set_output_project(my_app.task_id, RES_PROJECT.id,
RES_PROJECT.name)
def add_images_to_project():
sly.fs.ensure_base_path(sly.TaskPaths.RESULTS_DIR)
task_config = load_json_file(TaskPaths.TASK_CONFIG_PATH)
task_id = task_config['task_id']
append_to_existing_project = task_config['append_to_existing_project']
server_address = task_config['server_address']
token = task_config['api_token']
convert_options = task_config.get('options', {})
normalize_exif = convert_options.get('normalize_exif', True)
remove_alpha_channel = convert_options.get('remove_alpha_channel', True)
need_download = normalize_exif or remove_alpha_channel
api = sly.Api(server_address, token, retry_count=5)
task_info = api.task.get_info_by_id(task_id)
api.add_additional_field('taskId', task_id)
api.add_header('x-task-id', str(task_id))
workspace_id = task_info["workspaceId"]
project_name = task_config.get('project_name', None)
if project_name is None:
project_name = task_config["res_names"]["project"]
files_list = api.task.get_import_files_list(task_id)
if len(files_list) == 0:
raise RuntimeError("There are no import files")
project_info = None
if append_to_existing_project is True:
project_info = api.project.get_info_by_name(
workspace_id,
project_name,
expected_type=sly.ProjectType.IMAGES,
raise_error=True)
else:
project_info = api.project.create(workspace_id,
project_name,
type=sly.ProjectType.IMAGES,
change_name_if_conflict=True)
dataset_to_item = defaultdict(dict)
for dataset in api.dataset.get_list(project_info.id):
images = api.image.get_list(dataset.id)
for image_info in images:
dataset_to_item[dataset.name][image_info.name] = None
for file_info in files_list:
original_path = file_info["filename"]
try:
sly.image.validate_ext(original_path)
item_hash = file_info["hash"]
ds_name = get_dataset_name(original_path)
item_name = sly.fs.get_file_name_with_ext(original_path)
if item_name in dataset_to_item[ds_name]:
temp_name = sly.fs.get_file_name(original_path)
temp_ext = sly.fs.get_file_ext(original_path)
new_item_name = "{}_{}{}".format(temp_name, sly.rand_str(5),
temp_ext)
sly.logger.warning(
"Name {!r} already exists in dataset {!r}: renamed to {!r}"
.format(item_name, ds_name, new_item_name))
item_name = new_item_name
dataset_to_item[ds_name][item_name] = item_hash
except Exception as e:
sly.logger.warning(
"File skipped {!r}: error occurred during processing {!r}".
format(original_path, str(e)))
for ds_name, ds_items in dataset_to_item.items():
ds_info = api.dataset.get_or_create(project_info.id, ds_name)
names = [] # list(ds_items.keys())
hashes = [] #list(ds_items.values())
for name, hash in ds_items.items():
if hash is None:
#existing image => skip
continue
else:
names.append(name)
hashes.append(hash)
paths = [
os.path.join(sly.TaskPaths.RESULTS_DIR,
h.replace("/", "a") + sly.image.DEFAULT_IMG_EXT)
for h in hashes
]
progress = sly.Progress('Dataset: {!r}'.format(ds_name), len(ds_items))
for batch_names, batch_hashes, batch_paths in zip(
sly.batched(names, 10), sly.batched(hashes, 10),
sly.batched(paths, 10)):
if need_download is True:
res_batch_names = []
res_batch_paths = []
api.image.download_paths_by_hashes(batch_hashes, batch_paths)
for name, path in zip(batch_names, batch_paths):
try:
img = sly.image.read(path, remove_alpha_channel)
sly.image.write(path, img, remove_alpha_channel)
res_batch_names.append(name)
res_batch_paths.append(path)
except Exception as e:
sly.logger.warning("Skip image {!r}: {}".format(
name, str(e)),
extra={'file_path': path})
api.image.upload_paths(ds_info.id, res_batch_names,
res_batch_paths)
for path in res_batch_paths:
sly.fs.silent_remove(path)
#sly.fs.clean_dir(sly.TaskPaths.RESULTS_DIR)
progress.iters_done_report(len(batch_names))
else:
api.image.upload_hashes(ds_info.id,
batch_names,
batch_hashes,
progress_cb=progress.iters_done_report)
if project_info is not None:
sly.logger.info('PROJECT_CREATED',
extra={
'event_type': sly.EventType.PROJECT_CREATED,
'project_id': project_info.id
})
else:
temp_str = "Project"
if append_to_existing_project is True:
temp_str = "Dataset"
raise RuntimeError("{} wasn't created: 0 files were added")
pass
def from_sly_to_pascal(api: sly.Api, task_id, context, state, app_logger):
global PASCAL_CONTOUR_THICKNESS, TRAIN_VAL_SPLIT_COEF
project_info = api.project.get_info_by_id(PROJECT_ID)
meta_json = api.project.get_meta(PROJECT_ID)
meta = sly.ProjectMeta.from_json(meta_json)
app_logger.info("Palette has been created")
full_archive_name = str(
project_info.id) + '_' + project_info.name + ARCHIVE_NAME_ENDING
full_result_dir_name = str(
project_info.id) + '_' + project_info.name + RESULT_DIR_NAME_ENDING
result_archive = os.path.join(my_app.data_dir, full_archive_name)
result_dir = os.path.join(my_app.data_dir, full_result_dir_name)
result_subdir = os.path.join(result_dir, RESULT_SUBDIR_NAME)
result_ann_dir = os.path.join(result_subdir, ann_dir_name)
result_images_dir = os.path.join(result_subdir, images_dir_name)
result_class_dir_name = os.path.join(result_subdir, ann_class_dir_name)
result_obj_dir = os.path.join(result_subdir, ann_obj_dir_name)
result_imgsets_dir = os.path.join(result_subdir, trainval_sets_dir_name)
sly.fs.mkdir(result_ann_dir)
sly.fs.mkdir(result_imgsets_dir)
sly.fs.mkdir(result_images_dir)
sly.fs.mkdir(result_class_dir_name)
sly.fs.mkdir(result_obj_dir)
app_logger.info("Pascal VOC directories have been created")
images_stats = []
classes_colors = {}
datasets = api.dataset.get_list(PROJECT_ID)
dataset_names = ['trainval', 'val', 'train']
progress = sly.Progress('Preparing images for export',
api.project.get_images_count(PROJECT_ID),
app_logger)
for dataset in datasets:
if dataset.name in dataset_names:
is_trainval = 1
else:
is_trainval = 0
images = api.image.get_list(dataset.id)
for batch in sly.batched(images):
image_ids = [image_info.id for image_info in batch]
image_paths = [
os.path.join(result_images_dir, image_info.name)
for image_info in batch
]
api.image.download_paths(dataset.id, image_ids, image_paths)
ann_infos = api.annotation.download_batch(dataset.id, image_ids)
for image_info, ann_info in zip(batch, ann_infos):
img_title, img_ext = os.path.splitext(image_info.name)
cur_img_filename = image_info.name
if is_trainval == 1:
cur_img_stats = {
'classes': set(),
'dataset': dataset.name,
'name': img_title
}
images_stats.append(cur_img_stats)
else:
cur_img_stats = {
'classes': set(),
'dataset': None,
'name': img_title
}
images_stats.append(cur_img_stats)
if img_ext not in VALID_IMG_EXT:
orig_image_path = os.path.join(result_images_dir,
cur_img_filename)
jpg_image = img_title + ".jpg"
jpg_image_path = os.path.join(result_images_dir, jpg_image)
im = sly.image.read(orig_image_path)
sly.image.write(jpg_image_path, im)
sly.fs.silent_remove(orig_image_path)
ann = sly.Annotation.from_json(ann_info.annotation, meta)
tag = find_first_tag(ann.img_tags, SPLIT_TAGS)
if tag is not None:
cur_img_stats['dataset'] = tag.meta.name
valid_labels = []
for label in ann.labels:
if type(label.geometry) in SUPPORTED_GEOMETRY_TYPES:
valid_labels.append(label)
else:
app_logger.warn(
f"Label has unsupported geometry type ({type(label.geometry)}) and will be skipped."
)
ann = ann.clone(labels=valid_labels)
ann_to_xml(project_info, image_info, cur_img_filename,
result_ann_dir, ann)
for label in ann.labels:
cur_img_stats['classes'].add(label.obj_class.name)
classes_colors[label.obj_class.name] = tuple(
label.obj_class.color)
fake_contour_th = 0
if PASCAL_CONTOUR_THICKNESS != 0:
fake_contour_th = 2 * PASCAL_CONTOUR_THICKNESS + 1
from_ann_to_instance_mask(
ann,
os.path.join(result_class_dir_name,
img_title + pascal_ann_ext), fake_contour_th)
from_ann_to_class_mask(
ann,
os.path.join(result_obj_dir, img_title + pascal_ann_ext),
fake_contour_th)
progress.iter_done_report()
classes_colors = OrderedDict((sorted(classes_colors.items(),
key=lambda t: t[0])))
with open(os.path.join(result_subdir, "colors.txt"), "w") as cc:
if PASCAL_CONTOUR_THICKNESS != 0:
cc.write(
f"neutral {pascal_contour_color[0]} {pascal_contour_color[1]} {pascal_contour_color[2]}\n"
)
for k in classes_colors.keys():
if k == 'neutral':
continue
cc.write(
f"{k} {classes_colors[k][0]} {classes_colors[k][1]} {classes_colors[k][2]}\n"
)
imgs_to_split = [i for i in images_stats if i['dataset'] is None]
train_len = int(len(imgs_to_split) * TRAIN_VAL_SPLIT_COEF)
for img_stat in imgs_to_split[:train_len]:
img_stat['dataset'] = TRAIN_TAG_NAME
for img_stat in imgs_to_split[train_len:]:
img_stat['dataset'] = VAL_TAG_NAME
write_segm_set(is_trainval, images_stats, result_imgsets_dir)
write_main_set(is_trainval, images_stats, meta, result_imgsets_dir)
sly.fs.archive_directory(result_dir, result_archive)
app_logger.info("Result directory is archived")
upload_progress = []
remote_archive_path = "/ApplicationsData/Export-to-Pascal-VOC/{}/{}".format(
task_id, full_archive_name)
def _print_progress(monitor, upload_progress):
if len(upload_progress) == 0:
upload_progress.append(
sly.Progress(message="Upload {!r}".format(full_archive_name),
total_cnt=monitor.len,
ext_logger=app_logger,
is_size=True))
upload_progress[0].set_current_value(monitor.bytes_read)
file_info = api.file.upload(TEAM_ID, result_archive, remote_archive_path,
lambda m: _print_progress(m, upload_progress))
app_logger.info("Uploaded to Team-Files: {!r}".format(
file_info.full_storage_url))
api.task.set_output_archive(task_id,
file_info.id,
full_archive_name,
file_url=file_info.full_storage_url)
my_app.stop()
sly.io.json.dump_json_file(meta_json, os.path.join(dest_dir, 'meta.json'))
total_images = 0
src_dataset_infos = (
[api.dataset.get_info_by_id(ds_id) for ds_id in src_dataset_ids] if (src_dataset_ids is not None)
else api.dataset.get_list(project.id))
for dataset in src_dataset_infos:
ann_dir = os.path.join(dest_dir, dataset.name, 'ann')
sly.fs.mkdir(ann_dir)
images = api.image.get_list(dataset.id)
ds_progress = sly.Progress(
'Downloading annotations for: {!r}/{!r}'.format(src_project_name, dataset.name),
total_cnt=len(images))
for batch in sly.batched(images):
image_ids = [image_info.id for image_info in batch]
image_names = [image_info.name for image_info in batch]
#download annotations in json format
ann_infos = api.annotation.download_batch(dataset.id, image_ids)
ann_jsons = [ann_info.annotation for ann_info in ann_infos]
for image_name, ann_info in zip(image_names, ann_infos):
sly.io.json.dump_json_file(ann_info.annotation, os.path.join(ann_dir, image_name + '.json'))
ds_progress.iters_done_report(len(batch))
total_images += len(batch)
sly.logger.info('Project {!r} has been successfully downloaded'.format(src_project_name))
sly.logger.info('Total number of images: {!r}'.format(total_images))
def upload_and_reset(dataset_id, names, images, anns, progress):
if len(names) > 0:
new_image_infos = api.image.upload_nps(dataset_id, names, images)
new_image_ids = [img_info.id for img_info in new_image_infos]
api.annotation.upload_anns(new_image_ids, anns)
progress.iters_done_report(len(names))
del names[:]
del images[:]
del anns[:]
for dataset in api.dataset.get_list(project.id):
dst_dataset = api.dataset.create(dst_project.id, dataset.name)
videos = api.video.get_list(dataset.id)
for batch in sly.batched(videos):
for video_info in batch:
name = sly.fs.get_file_name(video_info.name)
ann_info = api.video.annotation.download(video_info.id)
ann = sly.VideoAnnotation.from_json(ann_info, meta, key_id_map)
progress = sly.Progress("Video: {!r}".format(video_info.name),
len(ann.frames))
image_names = []
frame_images = []
dst_anns = []
for frame in ann.frames:
image_names.append('{}_frame_{:05d}.png'.format(
name, frame.index))
frame_images.append(
api.video.frame.download_np(video_info.id, frame.index))
