def merge_classes(cur_meta: sly.ProjectMeta, res_meta: sly.ProjectMeta, ann: sly.Annotation):
existing_names = set([obj_class.name for obj_class in res_meta.obj_classes])
mapping = {}
for obj_class in cur_meta.obj_classes:
obj_class: sly.ObjClass
if obj_class.name in mapping:
continue
dest_class = res_meta.get_obj_class(obj_class.name)
if dest_class is None:
res_meta = res_meta.add_obj_class(obj_class)
dest_class = obj_class
elif obj_class != dest_class:
new_name = generate_free_name(existing_names, obj_class.name)
dest_class = obj_class.clone(name=new_name)
res_meta = res_meta.add_obj_class(dest_class)
mapping[obj_class.name] = dest_class
new_labels = []
for label in ann.labels:
if label.obj_class.name not in mapping:
new_labels.append(label)
else:
new_labels.append(label.clone(obj_class=mapping[label.obj_class.name]))
new_ann = ann.clone(labels=new_labels)
return (res_meta, res_meta, new_ann)
def copy_batch(self,
dst_dataset_id,
ids,
change_name_if_conflict=False,
with_annotations=False):
'''
Copy images with given ids in destination dataset
:param dst_dataset_id: int
:param ids: list of integers (raise error if type of ids not listand if images ids have to be from the same dataset)
:param change_name_if_conflict: bool (if False and images with the same names already exist in destination dataset raise error)
:param with_annotations: bool (if True - annotations will be copy to dataset too)
:return: list of images
'''
if type(ids) is not list:
raise RuntimeError(
"ids parameter has type {!r}. but has to be of type {!r}".
format(type(ids), list))
if len(ids) == 0:
return
existing_images = self.get_list(dst_dataset_id)
existing_names = {image.name for image in existing_images}
ids_info = self.get_info_by_id_batch(ids)
temp_ds_ids = {info.dataset_id for info in ids_info}
if len(temp_ds_ids) > 1:
raise RuntimeError("Images ids have to be from the same dataset")
if change_name_if_conflict:
new_names = [
generate_free_name(existing_names, info.name, with_ext=True)
for info in ids_info
]
else:
new_names = [info.name for info in ids_info]
names_intersection = existing_names.intersection(set(new_names))
if len(names_intersection) != 0:
raise RuntimeError(
'Images with the same names already exist in destination dataset. '
'Please, use argument \"change_name_if_conflict=True\" to automatically resolve '
'names intersection')
new_images = self.upload_ids(dst_dataset_id, new_names, ids)
new_ids = [new_image.id for new_image in new_images]
if with_annotations:
src_project_id = self._api.dataset.get_info_by_id(
ids_info[0].dataset_id).project_id
dst_project_id = self._api.dataset.get_info_by_id(
dst_dataset_id).project_id
self._api.project.merge_metas(src_project_id, dst_project_id)
self._api.annotation.copy_batch(ids, new_ids)
return new_images
def copy_batch(self,
dst_dataset_id,
ids,
change_name_if_conflict=False,
with_annotations=False):
if len(ids) == 0:
return
existing_images = self.get_list(dst_dataset_id)
existing_names = {image.name for image in existing_images}
ids_info = self.get_info_by_id_batch(ids)
if change_name_if_conflict:
new_names = [
generate_free_name(existing_names, info.name)
for info in ids_info
]
else:
new_names = [info.name for info in ids_info]
names_intersection = existing_names.intersection(set(new_names))
if len(names_intersection) != 0:
raise RuntimeError(
'Images with the same names already exist in destination dataset. '
'Please, use argument \"change_name_if_conflict=True\" to automatically resolve '
'names intersection')
new_images = self.upload_ids(dst_dataset_id, new_names, ids)
new_ids = [new_image.id for new_image in new_images]
if with_annotations:
src_project_id = self._api.dataset.get_info_by_id(
ids_info[0].dataset_id).project_id
dst_project_id = self._api.dataset.get_info_by_id(
dst_dataset_id).project_id
self._api.project.merge_metas(src_project_id, dst_project_id)
self._api.annotation.copy_batch(ids, new_ids)
return new_images
def merge_projects(api: sly.Api, task_id, context, state, app_logger):
dst_project_id = state["dstProjectId"]
dst_project_name = state["dstProjectName"]
dst_selected_dataset = state["selectedDatasetName"]
dst_dataset_name = state["dstDatasetName"]
dst_project = None
if state["dstProjectMode"] == "existingProject":
dst_project = api.project.get_info_by_id(dst_project_id)
dst_meta_json = api.project.get_meta(dst_project_id)
dst_meta = sly.ProjectMeta.from_json(dst_meta_json)
try:
dst_meta = dst_meta.merge(src_meta)
api.project.update_meta(dst_project.id, dst_meta.to_json())
app_logger.info(
f"Destination Project: name: '{dst_project.name}', id: '{dst_project.id}'."
)
except Exception as e:
app.show_modal_window(
"Error during merge, source project meta has conflics with destination project meta. "
"Please check shapes of classes / types of tags with the same names or select another destination. ",
level="error")
fields = [
{
"field": "data.processing",
"payload": False
},
{
"field": "state.selectedDatasetName",
"payload": None
},
]
api.task.set_fields(task_id, fields)
return
dst_dataset = None
if state["dstDatasetMode"] == "existingDataset":
dst_dataset = api.dataset.get_info_by_name(dst_project.id,
dst_selected_dataset)
app_logger.info(
f"Destination Dataset: name'{dst_dataset.name}', id:'{dst_dataset.id}'."
)
elif state["dstDatasetMode"] == "newDataset":
dst_dataset = api.dataset.create(dst_project.id,
dst_dataset_name,
change_name_if_conflict=True)
app_logger.info(
f"Destination Dataset: name'{dst_dataset.name}', id:'{dst_dataset.id}' has been created."
)
elif state["dstProjectMode"] == "newProject":
dst_project = api.project.create(WORKSPACE_ID,
dst_project_name,
type=src_project.type,
change_name_if_conflict=True)
api.project.update_meta(dst_project.id, src_meta.to_json())
dst_meta = src_meta.clone()
dst_dataset = api.dataset.create(dst_project.id, dst_dataset_name)
app_logger.info(
f"Destination Project: name '{dst_project.name}', id:'{dst_project.id}' has been created."
)
app_logger.info(
f"Destination Dataset: name '{dst_dataset.name}', id:'{dst_dataset.id}' has been created."
)
app_logger.info("Merging info",
extra={
"project name": src_project.name,
"project id": src_project.id,
"datasets to merge": state['selectedDatasets']
})
existing_names = []
if src_project.type == str(sly.ProjectType.IMAGES):
existing_names = _get_names(api.image.get_list, dst_dataset.id)
elif src_project.type == str(sly.ProjectType.VIDEOS):
existing_names = _get_names(api.video.get_list, dst_dataset.id)
ignored_items = 0
progress_items_cb = ui.get_progress_cb(api, task_id, "Items Merged",
total_items)
for dataset_name in state["selectedDatasets"]:
dataset = src_datasets_by_name[dataset_name.lstrip('/')]
if src_project.type == str(sly.ProjectType.IMAGES):
images = api.image.get_list(dataset.id)
app_logger.info(
f"Merging images and annotations from '{dataset.name}' dataset"
)
for batch in sly.batched(images):
batch_ids = [image_info.id for image_info in batch]
batch_anns = api.annotation.download_batch(
dataset.id, batch_ids)
ids = []
names = []
anns_jsons = []
for img_info, ann_info in zip(batch, batch_anns):
new_name = generate_free_name(existing_names,
img_info.name, True)
if state[
"nameConflicts"] == "ignore" and new_name != img_info.name:
ignored_items += 1
app_logger.info(
f"Image with name: '{new_name}' already exists in dataset: '{dataset.name}' and will be ignored."
)
progress_items_cb(len(batch))
continue
ids.append(img_info.id)
names.append(new_name)
existing_names.append(new_name)
anns_jsons.append(ann_info.annotation)
if len(ids) > 0:
dst_images = api.image.upload_ids(dst_dataset.id, names,
ids)
dst_ids = [dst_info.id for dst_info in dst_images]
api.annotation.upload_jsons(dst_ids, anns_jsons)
progress_items_cb(len(batch))
elif src_project.type == str(sly.ProjectType.VIDEOS):
key_id_map = KeyIdMap()
videos = api.video.get_list(dataset.id)
app_logger.info(
f"Merging videos and annotations from '{dataset.name}' dataset"
)
for video_info in videos:
if video_info.name in existing_names and state[
"nameConflicts"] == "ignore":
ignored_items += 1
app_logger.info(
f"Video with name: '{video_info.name}' already exists in dataset: '{dataset.name}' and will be ignored."
)
progress_items_cb(1)
continue
res_name = generate_free_name(existing_names,
video_info.name,
with_ext=True)
dst_video = api.video.upload_hash(dst_dataset.id, res_name,
video_info.hash)
ann_info = api.video.annotation.download(video_info.id)
ann = sly.VideoAnnotation.from_json(ann_info, dst_meta,
key_id_map)
api.video.annotation.append(dst_video.id, ann)
existing_names.append(res_name)
progress_items_cb(1)
dialog_message = f"{len(state['selectedDatasets'])} datasets from project '{src_project.name}' " \
f"have been successfully merged to the dataset: '{dst_dataset.name}' in project '{dst_project.name}'."
if ignored_items > 0:
dialog_message += f" Items ignored: {ignored_items}"
app.show_modal_window(dialog_message, level="info")
fields = [{
"field": "data.processing",
"payload": False
}, {
"field": "data.finished",
"payload": True
}]
api.app.set_fields(task_id, fields)