def test_upload_fusion_segment_with_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name, is_fusion=True)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment = FusionSegment("segment1")
segment.sensors.add(Sensor.loads(LIDAR_DATA))
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
frame = Frame()
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
frame[LIDAR_DATA["name"]] = data
segment.append(frame)
dataset_client.upload_segment(segment)
segment1 = FusionSegment(name="segment1", client=dataset_client)
assert len(segment1) == 10
assert segment1[0][LIDAR_DATA["name"]].path == "hello0.txt"
assert (
segment1[0][LIDAR_DATA["name"]].path
== segment[0][LIDAR_DATA["name"]].target_remote_path
)
assert segment1[0][LIDAR_DATA["name"]].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def _load_frame_and_sensor(
segment: FusionSegment,
annotation_info: Dict[str, Any],
subset_path: str,
is_test: bool,
) -> None:
frame = Frame()
for data_frames in annotation_info["frame_data"].values():
for data_frame in data_frames:
calibrated_sensor_info = annotation_info["calibrated_sensors"][
data_frame["calibrated_sensor_token"]]
common_sensor = annotation_info["sensor"][
calibrated_sensor_info["sensor_token"]]
sensor_name = common_sensor["channel"]
if sensor_name not in segment.sensors:
segment.sensors.add(
get_sensor("camera", sensor_name, calibrated_sensor_info))
data = Data(
os.path.join(subset_path, data_frame["filename"]),
timestamp=data_frame["timestamp"] / 10**6,
)
if not is_test:
data.label.box2d, data.label.rle = _get_labels(
data_frame["token"], annotation_info)
frame[sensor_name] = data
segment.append(frame)
def _load_frame_and_sensor(
segment: FusionSegment,
current_frame_token: str,
annotation_info: Dict[str, Any],
subset_path: str,
is_test: bool,
) -> None:
frame = Frame()
for sensor_frame in annotation_info["frame_data"][current_frame_token]:
calibrated_sensor_info = annotation_info["calibrated_sensors"][
sensor_frame["calibrated_sensor_token"]
]
common_sensor = annotation_info["sensor"][calibrated_sensor_info["sensor_token"]]
sensor_name = common_sensor["channel"]
sensor_type = common_sensor["modality"]
if sensor_name not in segment.sensors:
segment.sensors.add(get_sensor(sensor_type, sensor_name, calibrated_sensor_info))
data = Data(
os.path.join(subset_path, sensor_frame["filename"]),
timestamp=sensor_frame["timestamp"] / 10**6,
)
if not is_test and sensor_type == "lidar":
data.label.box3d = _get_labels(
current_frame_token,
annotation_info["ego_poses"][sensor_frame["ego_pose_token"]],
segment.sensors[sensor_name].extrinsics,
annotation_info,
)
frame[sensor_name] = data
segment.append(frame)
def test_upload_segment(self, mocker):
self.fusion_dataset_client._status.checkout(draft_number=1)
segment_test = FusionSegment(name="test1")
for i in range(5):
temp_frame = Frame()
temp_ulid = from_timestamp(10 * i + 10)
temp_frame.frame_id = temp_ulid
if i % 2 == 0:
temp_frame["camera"] = Data(f"{i}.png")
else:
temp_frame["lidar"] = Data(f"{i}.png")
segment_test.append(temp_frame)
segment_client = FusionSegmentClient(
name="test1", data_client=self.fusion_dataset_client)
upload_segment = mocker.patch(
f"{dataset.__name__}.FusionDatasetClient._upload_segment",
return_value=segment_client)
assert self.fusion_dataset_client.upload_segment(
segment_test).name == "test1"
args, keywords = upload_segment.call_args
assert args[0] == segment_test
assert keywords["jobs"] == 1
assert not keywords["skip_uploaded_files"]
upload_segment.assert_called_once()
def test_cache_fusion_dataset(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name,
is_fusion=True)
dataset_client.create_draft("draft-1")
segment = FusionSegment("Segment1")
segment.sensors = Sensors.loads(_SENSORS_DATA)
paths = {"Lidar1": tmp_path / "lidar", "Camera1": tmp_path / "camera"}
for path in paths.values():
path.mkdir()
for i in range(_SEGMENT_LENGTH):
frame = Frame()
for sensor_data in _SENSORS_DATA:
sensor_name = sensor_data["name"]
data_path = paths[sensor_name] / f"{sensor_name}{i}.txt"
data_path.write_text("CONTENT")
frame[sensor_name] = Data(local_path=str(data_path))
segment.append(frame)
dataset_client.upload_segment(segment)
dataset_client.commit("commit-1")
cache_path = tmp_path / "cache_test"
dataset_client.enable_cache(str(cache_path))
segment1 = FusionSegment(name="Segment1", client=dataset_client)
for frame in segment1:
for data in frame.values():
data.open()
segment_cache_path = (cache_path / dataset_client.dataset_id /
dataset_client.status.commit_id / "Segment1")
correct_files = set(
segment_cache_path / f'{sensor_data["name"]}{i}.txt'
for i in range(_SEGMENT_LENGTH) for sensor_data in _SENSORS_DATA)
assert set(segment_cache_path.glob("*.txt")) == correct_files
gas_client.delete_dataset(dataset_name)
def test__upload_segment(self, mocker):
segment_test = FusionSegment(name="test1")
ulids = []
done_frames = []
for i in range(5):
temp_frame = Frame()
temp_ulid = from_timestamp(10 * i + 10)
temp_frame.frame_id = temp_ulid
if i % 2 == 0:
temp_frame["camera"] = Data(f"{i}.png")
done_frames.append(temp_frame)
else:
temp_frame["lidar"] = Data(f"{i}.png")
ulids.append(temp_ulid)
segment_test.append(temp_frame)
segment_client = FusionSegmentClient(
name="test1", data_client=self.fusion_dataset_client)
get_or_create_segment = mocker.patch(
f"{dataset.__name__}.FusionDatasetClient.get_or_create_segment",
return_value=segment_client,
)
list_frames = mocker.patch(
f"{segment.__name__}.FusionSegmentClient.list_frames",
return_value=done_frames,
)
multithread_upload = mocker.patch(
f"{dataset.__name__}.multithread_upload")
with Tqdm(5, disable=False) as pbar:
self.fusion_dataset_client._upload_segment(
segment_test, jobs=8, skip_uploaded_files=True, pbar=pbar)
get_or_create_segment.assert_called_once_with(segment_test.name)
list_frames.assert_called_once_with()
args, keywords = multithread_upload.call_args
for index, values in enumerate(args[1]):
data, sensor_name, frame_id = values
assert data.path == f"{index * 2 + 1}.png"
assert sensor_name == "lidar"
assert frame_id == ulids[index * 2 + 1].str
assert keywords[
"callback"] == segment_client._synchronize_upload_info
assert keywords["jobs"] == 8
assert keywords["pbar"] == pbar
multithread_upload.assert_called_once()
with Tqdm(5, disable=False) as pbar:
self.fusion_dataset_client._upload_segment(
segment_test, jobs=8, skip_uploaded_files=False, pbar=pbar)
get_or_create_segment.assert_called_with(segment_test.name)
list_frames.assert_called_with()
args, keywords = multithread_upload.call_args
for index, values in enumerate(args[1]):
data, sensor_name, frame_id = values
assert data.path == f"{index}.png"
if index % 2 == 0:
assert sensor_name == "camera"
else:
assert sensor_name == "lidar"
assert frame_id == ulids[index].str
assert keywords[
"callback"] == segment_client._synchronize_upload_info
assert keywords["jobs"] == 8
assert keywords["pbar"] == pbar