def test_upload_fusion_segment_with_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name, is_fusion=True)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment = FusionSegment("segment1")
segment.sensors.add(Sensor.loads(LIDAR_DATA))
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
frame = Frame()
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
frame[LIDAR_DATA["name"]] = data
segment.append(frame)
dataset_client.upload_segment(segment)
segment1 = FusionSegment(name="segment1", client=dataset_client)
assert len(segment1) == 10
assert segment1[0][LIDAR_DATA["name"]].path == "hello0.txt"
assert (
segment1[0][LIDAR_DATA["name"]].path
== segment[0][LIDAR_DATA["name"]].target_remote_path
)
assert segment1[0][LIDAR_DATA["name"]].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def _load_frame_and_sensor(
segment: FusionSegment,
current_frame_token: str,
annotation_info: Dict[str, Any],
subset_path: str,
is_test: bool,
) -> None:
frame = Frame()
for sensor_frame in annotation_info["frame_data"][current_frame_token]:
calibrated_sensor_info = annotation_info["calibrated_sensors"][
sensor_frame["calibrated_sensor_token"]
]
common_sensor = annotation_info["sensor"][calibrated_sensor_info["sensor_token"]]
sensor_name = common_sensor["channel"]
sensor_type = common_sensor["modality"]
if sensor_name not in segment.sensors:
segment.sensors.add(get_sensor(sensor_type, sensor_name, calibrated_sensor_info))
data = Data(
os.path.join(subset_path, sensor_frame["filename"]),
timestamp=sensor_frame["timestamp"] / 10**6,
)
if not is_test and sensor_type == "lidar":
data.label.box3d = _get_labels(
current_frame_token,
annotation_info["ego_poses"][sensor_frame["ego_pose_token"]],
segment.sensors[sensor_name].extrinsics,
annotation_info,
)
frame[sensor_name] = data
segment.append(frame)
def _load_frame_and_sensor(
segment: FusionSegment,
annotation_info: Dict[str, Any],
subset_path: str,
is_test: bool,
) -> None:
frame = Frame()
for data_frames in annotation_info["frame_data"].values():
for data_frame in data_frames:
calibrated_sensor_info = annotation_info["calibrated_sensors"][
data_frame["calibrated_sensor_token"]]
common_sensor = annotation_info["sensor"][
calibrated_sensor_info["sensor_token"]]
sensor_name = common_sensor["channel"]
if sensor_name not in segment.sensors:
segment.sensors.add(
get_sensor("camera", sensor_name, calibrated_sensor_info))
data = Data(
os.path.join(subset_path, data_frame["filename"]),
timestamp=data_frame["timestamp"] / 10**6,
)
if not is_test:
data.label.box2d, data.label.rle = _get_labels(
data_frame["token"], annotation_info)
frame[sensor_name] = data
segment.append(frame)
def test_upload_segment(self, mocker):
self.fusion_dataset_client._status.checkout(draft_number=1)
segment_test = FusionSegment(name="test1")
for i in range(5):
temp_frame = Frame()
temp_ulid = from_timestamp(10 * i + 10)
temp_frame.frame_id = temp_ulid
if i % 2 == 0:
temp_frame["camera"] = Data(f"{i}.png")
else:
temp_frame["lidar"] = Data(f"{i}.png")
segment_test.append(temp_frame)
segment_client = FusionSegmentClient(
name="test1", data_client=self.fusion_dataset_client)
upload_segment = mocker.patch(
f"{dataset.__name__}.FusionDatasetClient._upload_segment",
return_value=segment_client)
assert self.fusion_dataset_client.upload_segment(
segment_test).name == "test1"
args, keywords = upload_segment.call_args
assert args[0] == segment_test
assert keywords["jobs"] == 1
assert not keywords["skip_uploaded_files"]
upload_segment.assert_called_once()
def test_upload_fusion_segment_without_file(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name, is_fusion=True)
dataset_client.create_draft("draft-1")
segment = FusionSegment("segment1")
dataset_client.upload_segment(segment)
segment1 = FusionSegment(name="segment1", client=dataset_client)
assert len(segment1) == 0
assert not segment1.sensors
gas_client.delete_dataset(dataset_name)
def test_import_cloud_files_to_fusiondataset(self, accesskey, url, config_name):
gas_client = GAS(access_key=accesskey, url=url)
try:
cloud_client = gas_client.get_cloud_client(config_name)
except ResourceNotExistError:
pytest.skip(f"skip this case because there's no {config_name} config")
auth_data = cloud_client.list_auth_data("tests")[:5]
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name, True, config_name=config_name)
dataset = FusionDataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
lidar = Lidar("LIDAR")
segment.sensors.add(lidar)
for data in auth_data:
data.label.classification = Classification("cat", attributes={"color": "red"})
frame = Frame()
frame["LIDAR"] = data
segment.append(frame)
dataset_client = gas_client.upload_dataset(dataset, jobs=5)
dataset_client.commit("import data")
segment1 = FusionSegment("Segment1", client=dataset_client)
assert len(segment1) == len(segment)
assert segment1[0]["LIDAR"].path == segment[0]["LIDAR"].path.split("/")[-1]
assert segment1[0]["LIDAR"].label.classification.category == "cat"
assert segment1[0]["LIDAR"].label.classification.attributes["color"] == "red"
assert len(auth_data) == len(segment)
gas_client.delete_dataset(dataset_name)
def test_copy_fusion_segment(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name, is_fusion=True)
dataset = FusionDataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
segment.sensors.add(Sensor.loads(LIDAR_DATA))
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
frame = Frame()
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
frame[LIDAR_DATA["name"]] = data
segment.append(frame)
dataset_client = gas_client.upload_dataset(dataset)
segment_client = dataset_client.copy_segment("Segment1", "Segment2")
assert segment_client.name == "Segment2"
with pytest.raises(InvalidParamsError):
dataset_client.copy_segment("Segment1", "Segment3", strategy="push")
segment2 = FusionSegment("Segment2", client=dataset_client)
assert segment2[0][LIDAR_DATA["name"]].path == "hello0.txt"
assert (
segment2[0][LIDAR_DATA["name"]].path
== segment[0][LIDAR_DATA["name"]].target_remote_path
)
assert segment2[0][LIDAR_DATA["name"]].label
gas_client.delete_dataset(dataset_name)
def _get_segment(root_path: str, subset: str,
metadata_path: str) -> FusionSegment:
is_test = subset.endswith("test")
annotation_info = _get_annotation_info(os.path.join(metadata_path, subset),
is_test)
segment = FusionSegment(subset)
_load_frame_and_sensor(segment, annotation_info, root_path, is_test)
return segment
def _generate_segments(root_path: str, subset: str) -> Iterator[FusionSegment]:
subset_path = os.path.join(root_path, subset)
is_test = subset.endswith("test")
annotation_info = _get_annotation_info(os.path.join(subset_path, subset), is_test)
for scene in annotation_info["scenes"]:
segment = FusionSegment(f"{subset}-{scene['name']}")
segment.description = scene["description"]
current_frame_token = scene["first_sample_token"]
while current_frame_token:
_load_frame_and_sensor(
segment,
current_frame_token,
annotation_info,
subset_path,
is_test,
)
current_frame_token = annotation_info["samples"][current_frame_token]["next"]
yield segment
def _generate_segments(
self,
offset: int = 0,
limit: int = 128) -> Generator[FusionSegment, None, int]:
response = self._list_segments(offset, limit)
for item in response["segments"]:
segment = FusionSegment._from_client( # pylint: disable=protected-access
FusionSegmentClient(item["name"], self))
segment.description = item["description"]
yield segment
return response["totalCount"] # type: ignore[no-any-return]
def test_cache_fusion_dataset(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name,
is_fusion=True)
dataset_client.create_draft("draft-1")
segment = FusionSegment("Segment1")
segment.sensors = Sensors.loads(_SENSORS_DATA)
paths = {"Lidar1": tmp_path / "lidar", "Camera1": tmp_path / "camera"}
for path in paths.values():
path.mkdir()
for i in range(_SEGMENT_LENGTH):
frame = Frame()
for sensor_data in _SENSORS_DATA:
sensor_name = sensor_data["name"]
data_path = paths[sensor_name] / f"{sensor_name}{i}.txt"
data_path.write_text("CONTENT")
frame[sensor_name] = Data(local_path=str(data_path))
segment.append(frame)
dataset_client.upload_segment(segment)
dataset_client.commit("commit-1")
cache_path = tmp_path / "cache_test"
dataset_client.enable_cache(str(cache_path))
segment1 = FusionSegment(name="Segment1", client=dataset_client)
for frame in segment1:
for data in frame.values():
data.open()
segment_cache_path = (cache_path / dataset_client.dataset_id /
dataset_client.status.commit_id / "Segment1")
correct_files = set(
segment_cache_path / f'{sensor_data["name"]}{i}.txt'
for i in range(_SEGMENT_LENGTH) for sensor_data in _SENSORS_DATA)
assert set(segment_cache_path.glob("*.txt")) == correct_files
gas_client.delete_dataset(dataset_name)
def test__upload_segment(self, mocker):
segment_test = FusionSegment(name="test1")
ulids = []
done_frames = []
for i in range(5):
temp_frame = Frame()
temp_ulid = from_timestamp(10 * i + 10)
temp_frame.frame_id = temp_ulid
if i % 2 == 0:
temp_frame["camera"] = Data(f"{i}.png")
done_frames.append(temp_frame)
else:
temp_frame["lidar"] = Data(f"{i}.png")
ulids.append(temp_ulid)
segment_test.append(temp_frame)
segment_client = FusionSegmentClient(
name="test1", data_client=self.fusion_dataset_client)
get_or_create_segment = mocker.patch(
f"{dataset.__name__}.FusionDatasetClient.get_or_create_segment",
return_value=segment_client,
)
list_frames = mocker.patch(
f"{segment.__name__}.FusionSegmentClient.list_frames",
return_value=done_frames,
)
multithread_upload = mocker.patch(
f"{dataset.__name__}.multithread_upload")
with Tqdm(5, disable=False) as pbar:
self.fusion_dataset_client._upload_segment(
segment_test, jobs=8, skip_uploaded_files=True, pbar=pbar)
get_or_create_segment.assert_called_once_with(segment_test.name)
list_frames.assert_called_once_with()
args, keywords = multithread_upload.call_args
for index, values in enumerate(args[1]):
data, sensor_name, frame_id = values
assert data.path == f"{index * 2 + 1}.png"
assert sensor_name == "lidar"
assert frame_id == ulids[index * 2 + 1].str
assert keywords[
"callback"] == segment_client._synchronize_upload_info
assert keywords["jobs"] == 8
assert keywords["pbar"] == pbar
multithread_upload.assert_called_once()
with Tqdm(5, disable=False) as pbar:
self.fusion_dataset_client._upload_segment(
segment_test, jobs=8, skip_uploaded_files=False, pbar=pbar)
get_or_create_segment.assert_called_with(segment_test.name)
list_frames.assert_called_with()
args, keywords = multithread_upload.call_args
for index, values in enumerate(args[1]):
data, sensor_name, frame_id = values
assert data.path == f"{index}.png"
if index % 2 == 0:
assert sensor_name == "camera"
else:
assert sensor_name == "lidar"
assert frame_id == ulids[index].str
assert keywords[
"callback"] == segment_client._synchronize_upload_info
assert keywords["jobs"] == 8
assert keywords["pbar"] == pbar