def merge_metas(self, src_project_id, dst_project_id):
'''
Add metadata from given progect to given destination project
:param src_project_id: int
:param dst_project_id: int
:return: merged project metainformation
'''
if src_project_id == dst_project_id:
return self.get_meta(src_project_id)
src_meta = ProjectMeta.from_json(self.get_meta(src_project_id))
dst_meta = ProjectMeta.from_json(self.get_meta(dst_project_id))
new_dst_meta = src_meta.merge(dst_meta)
new_dst_meta_json = new_dst_meta.to_json()
self.update_meta(dst_project_id, new_dst_meta.to_json())
return new_dst_meta_json
def _in_project_meta_from_msg(in_msg):
pr_meta_json = in_msg.get('meta')
return ProjectMeta.from_json(pr_meta_json) if pr_meta_json is not None else None
def download_video_project(api,
project_id,
dest_dir,
dataset_ids=None,
download_videos=True,
log_progress=False):
'''
Download project with given id in destination directory
:param api: Api class object
:param project_id: int
:param dest_dir: str
:param dataset_ids: list of integers
:param download_videos: bool
:param log_progress: bool
'''
LOG_BATCH_SIZE = 1
key_id_map = KeyIdMap()
project_fs = VideoProject(dest_dir, OpenMode.CREATE)
meta = ProjectMeta.from_json(api.project.get_meta(project_id))
project_fs.set_meta(meta)
datasets_infos = []
if dataset_ids is not None:
for ds_id in dataset_ids:
datasets_infos.append(api.dataset.get_info_by_id(ds_id))
else:
datasets_infos = api.dataset.get_list(project_id)
for dataset in datasets_infos:
dataset_fs = project_fs.create_dataset(dataset.name)
videos = api.video.get_list(dataset.id)
ds_progress = None
if log_progress:
ds_progress = Progress('Downloading dataset: {!r}'.format(
dataset.name),
total_cnt=len(videos))
for batch in batched(videos, batch_size=LOG_BATCH_SIZE):
video_ids = [video_info.id for video_info in batch]
video_names = [video_info.name for video_info in batch]
ann_jsons = api.video.annotation.download_bulk(
dataset.id, video_ids)
for video_id, video_name, ann_json in zip(video_ids, video_names,
ann_jsons):
if video_name != ann_json[ApiField.VIDEO_NAME]:
raise RuntimeError(
"Error in api.video.annotation.download_batch: broken order"
)
video_file_path = dataset_fs.generate_item_path(video_name)
if download_videos is True:
api.video.download_path(video_id, video_file_path)
else:
touch(video_file_path)
dataset_fs.add_item_file(video_name,
video_file_path,
ann=VideoAnnotation.from_json(
ann_json, project_fs.meta,
key_id_map),
_validate_item=False)
ds_progress.iters_done_report(len(batch))
project_fs.set_key_id_map(key_id_map)
def download_pointcloud_episode_project(api,
project_id,
dest_dir,
dataset_ids=None,
download_pcd=True,
download_realated_images=True,
download_annotations=True,
log_progress=False,
batch_size=10):
key_id_map = KeyIdMap()
project_fs = PointcloudEpisodeProject(dest_dir, OpenMode.CREATE)
meta = ProjectMeta.from_json(api.project.get_meta(project_id))
project_fs.set_meta(meta)
datasets_infos = []
if dataset_ids is not None:
for ds_id in dataset_ids:
datasets_infos.append(api.dataset.get_info_by_id(ds_id))
else:
datasets_infos = api.dataset.get_list(project_id)
for dataset in datasets_infos:
dataset_fs = project_fs.create_dataset(dataset.name)
pointclouds = api.pointcloud_episode.get_list(dataset.id)
if download_annotations:
# Download annotation to project_path/dataset_path/annotation.json
ann_json = api.pointcloud_episode.annotation.download(dataset.id)
annotation = dataset_fs.annotation_class.from_json(
ann_json, meta, key_id_map)
dataset_fs.set_ann(annotation)
# frames --> pointcloud mapping to project_path/dataset_path/frame_pointcloud_map.json
frame_name_map = api.pointcloud_episode.get_frame_name_map(
dataset.id)
frame_pointcloud_map_path = dataset_fs.get_frame_pointcloud_map_path(
)
dump_json_file(frame_name_map, frame_pointcloud_map_path)
# Download data
if log_progress:
ds_progress = Progress('Downloading dataset: {!r}'.format(
dataset.name),
total_cnt=len(pointclouds))
for batch in batched(pointclouds, batch_size=batch_size):
pointcloud_ids = [pointcloud_info.id for pointcloud_info in batch]
pointcloud_names = [
pointcloud_info.name for pointcloud_info in batch
]
for pointcloud_id, pointcloud_name in zip(pointcloud_ids,
pointcloud_names):
pointcloud_file_path = dataset_fs.generate_item_path(
pointcloud_name)
if download_pcd is True:
api.pointcloud_episode.download_path(
pointcloud_id, pointcloud_file_path)
else:
touch(pointcloud_file_path)
if download_realated_images:
related_images_path = dataset_fs.get_related_images_path(
pointcloud_name)
related_images = api.pointcloud_episode.get_list_related_images(
pointcloud_id)
for rimage_info in related_images:
name = rimage_info[ApiField.NAME]
rimage_id = rimage_info[ApiField.ID]
path_img = os.path.join(related_images_path, name)
path_json = os.path.join(related_images_path,
name + ".json")
api.pointcloud_episode.download_related_image(
rimage_id, path_img)
dump_json_file(rimage_info, path_json)
dataset_fs.add_item_file(pointcloud_name,
pointcloud_file_path,
_validate_item=False)
if log_progress:
ds_progress.iters_done_report(len(batch))
project_fs.set_key_id_map(key_id_map)
def download_pointcloud_project(api,
project_id,
dest_dir,
dataset_ids=None,
download_items=True,
log_progress=False):
LOG_BATCH_SIZE = 1
key_id_map = KeyIdMap()
project_fs = PointcloudProject(dest_dir, OpenMode.CREATE)
meta = ProjectMeta.from_json(api.project.get_meta(project_id))
project_fs.set_meta(meta)
datasets_infos = []
if dataset_ids is not None:
for ds_id in dataset_ids:
datasets_infos.append(api.dataset.get_info_by_id(ds_id))
else:
datasets_infos = api.dataset.get_list(project_id)
for dataset in datasets_infos:
dataset_fs = project_fs.create_dataset(dataset.name)
pointclouds = api.pointcloud.get_list(dataset.id)
ds_progress = None
if log_progress:
ds_progress = Progress('Downloading dataset: {!r}'.format(
dataset.name),
total_cnt=len(pointclouds))
for batch in batched(pointclouds, batch_size=LOG_BATCH_SIZE):
pointcloud_ids = [pointcloud_info.id for pointcloud_info in batch]
pointcloud_names = [
pointcloud_info.name for pointcloud_info in batch
]
ann_jsons = api.pointcloud.annotation.download_bulk(
dataset.id, pointcloud_ids)
for pointcloud_id, pointcloud_name, ann_json in zip(
pointcloud_ids, pointcloud_names, ann_jsons):
if pointcloud_name != ann_json[ApiField.NAME]:
raise RuntimeError(
"Error in api.video.annotation.download_batch: broken order"
)
pointcloud_file_path = dataset_fs.generate_item_path(
pointcloud_name)
if download_items is True:
api.pointcloud.download_path(pointcloud_id,
pointcloud_file_path)
related_images_path = dataset_fs.get_related_images_path(
pointcloud_name)
related_images = api.pointcloud.get_list_related_images(
pointcloud_id)
for rimage_info in related_images:
name = rimage_info[ApiField.NAME]
if not has_valid_ext(name):
new_name = get_file_name(
name) # to fix cases like .png.json
if has_valid_ext(new_name):
name = new_name
rimage_info[ApiField.NAME] = name
else:
raise RuntimeError(
'Something wrong with photo context filenames.\
Please, contact support')
rimage_id = rimage_info[ApiField.ID]
path_img = os.path.join(related_images_path, name)
path_json = os.path.join(related_images_path,
name + ".json")
api.pointcloud.download_related_image(
rimage_id, path_img)
dump_json_file(rimage_info, path_json)
else:
touch(pointcloud_file_path)
dataset_fs.add_item_file(pointcloud_name,
pointcloud_file_path,
ann=PointcloudAnnotation.from_json(
ann_json, project_fs.meta,
key_id_map),
_validate_item=False)
ds_progress.iters_done_report(len(batch))
project_fs.set_key_id_map(key_id_map)