def test_upload_fusion_segment_with_label(self, accesskey, url, tmp_path): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_dataset_name() dataset_client = gas_client.create_dataset(dataset_name, is_fusion=True) dataset_client.create_draft("draft-1") dataset_client.upload_catalog(Catalog.loads(CATALOG)) segment = FusionSegment("segment1") segment.sensors.add(Sensor.loads(LIDAR_DATA)) path = tmp_path / "sub" path.mkdir() for i in range(10): frame = Frame() local_path = path / f"hello{i}.txt" local_path.write_text("CONTENT") data = Data(local_path=str(local_path)) data.label = Label.loads(LABEL) frame[LIDAR_DATA["name"]] = data segment.append(frame) dataset_client.upload_segment(segment) segment1 = FusionSegment(name="segment1", client=dataset_client) assert len(segment1) == 10 assert segment1[0][LIDAR_DATA["name"]].path == "hello0.txt" assert ( segment1[0][LIDAR_DATA["name"]].path == segment[0][LIDAR_DATA["name"]].target_remote_path ) assert segment1[0][LIDAR_DATA["name"]].label # todo: match the input and output label gas_client.delete_dataset(dataset_name)
def _load_frame_and_sensor( segment: FusionSegment, annotation_info: Dict[str, Any], subset_path: str, is_test: bool, ) -> None: frame = Frame() for data_frames in annotation_info["frame_data"].values(): for data_frame in data_frames: calibrated_sensor_info = annotation_info["calibrated_sensors"][ data_frame["calibrated_sensor_token"]] common_sensor = annotation_info["sensor"][ calibrated_sensor_info["sensor_token"]] sensor_name = common_sensor["channel"] if sensor_name not in segment.sensors: segment.sensors.add( get_sensor("camera", sensor_name, calibrated_sensor_info)) data = Data( os.path.join(subset_path, data_frame["filename"]), timestamp=data_frame["timestamp"] / 10**6, ) if not is_test: data.label.box2d, data.label.rle = _get_labels( data_frame["token"], annotation_info) frame[sensor_name] = data segment.append(frame)
def _load_frame_and_sensor( segment: FusionSegment, current_frame_token: str, annotation_info: Dict[str, Any], subset_path: str, is_test: bool, ) -> None: frame = Frame() for sensor_frame in annotation_info["frame_data"][current_frame_token]: calibrated_sensor_info = annotation_info["calibrated_sensors"][ sensor_frame["calibrated_sensor_token"] ] common_sensor = annotation_info["sensor"][calibrated_sensor_info["sensor_token"]] sensor_name = common_sensor["channel"] sensor_type = common_sensor["modality"] if sensor_name not in segment.sensors: segment.sensors.add(get_sensor(sensor_type, sensor_name, calibrated_sensor_info)) data = Data( os.path.join(subset_path, sensor_frame["filename"]), timestamp=sensor_frame["timestamp"] / 10**6, ) if not is_test and sensor_type == "lidar": data.label.box3d = _get_labels( current_frame_token, annotation_info["ego_poses"][sensor_frame["ego_pose_token"]], segment.sensors[sensor_name].extrinsics, annotation_info, ) frame[sensor_name] = data segment.append(frame)
def test_upload_segment(self, mocker): self.fusion_dataset_client._status.checkout(draft_number=1) segment_test = FusionSegment(name="test1") for i in range(5): temp_frame = Frame() temp_ulid = from_timestamp(10 * i + 10) temp_frame.frame_id = temp_ulid if i % 2 == 0: temp_frame["camera"] = Data(f"{i}.png") else: temp_frame["lidar"] = Data(f"{i}.png") segment_test.append(temp_frame) segment_client = FusionSegmentClient( name="test1", data_client=self.fusion_dataset_client) upload_segment = mocker.patch( f"{dataset.__name__}.FusionDatasetClient._upload_segment", return_value=segment_client) assert self.fusion_dataset_client.upload_segment( segment_test).name == "test1" args, keywords = upload_segment.call_args assert args[0] == segment_test assert keywords["jobs"] == 1 assert not keywords["skip_uploaded_files"] upload_segment.assert_called_once()
def test_cache_fusion_dataset(self, accesskey, url, tmp_path): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_dataset_name() dataset_client = gas_client.create_dataset(dataset_name, is_fusion=True) dataset_client.create_draft("draft-1") segment = FusionSegment("Segment1") segment.sensors = Sensors.loads(_SENSORS_DATA) paths = {"Lidar1": tmp_path / "lidar", "Camera1": tmp_path / "camera"} for path in paths.values(): path.mkdir() for i in range(_SEGMENT_LENGTH): frame = Frame() for sensor_data in _SENSORS_DATA: sensor_name = sensor_data["name"] data_path = paths[sensor_name] / f"{sensor_name}{i}.txt" data_path.write_text("CONTENT") frame[sensor_name] = Data(local_path=str(data_path)) segment.append(frame) dataset_client.upload_segment(segment) dataset_client.commit("commit-1") cache_path = tmp_path / "cache_test" dataset_client.enable_cache(str(cache_path)) segment1 = FusionSegment(name="Segment1", client=dataset_client) for frame in segment1: for data in frame.values(): data.open() segment_cache_path = (cache_path / dataset_client.dataset_id / dataset_client.status.commit_id / "Segment1") correct_files = set( segment_cache_path / f'{sensor_data["name"]}{i}.txt' for i in range(_SEGMENT_LENGTH) for sensor_data in _SENSORS_DATA) assert set(segment_cache_path.glob("*.txt")) == correct_files gas_client.delete_dataset(dataset_name)
def test__upload_segment(self, mocker): segment_test = FusionSegment(name="test1") ulids = [] done_frames = [] for i in range(5): temp_frame = Frame() temp_ulid = from_timestamp(10 * i + 10) temp_frame.frame_id = temp_ulid if i % 2 == 0: temp_frame["camera"] = Data(f"{i}.png") done_frames.append(temp_frame) else: temp_frame["lidar"] = Data(f"{i}.png") ulids.append(temp_ulid) segment_test.append(temp_frame) segment_client = FusionSegmentClient( name="test1", data_client=self.fusion_dataset_client) get_or_create_segment = mocker.patch( f"{dataset.__name__}.FusionDatasetClient.get_or_create_segment", return_value=segment_client, ) list_frames = mocker.patch( f"{segment.__name__}.FusionSegmentClient.list_frames", return_value=done_frames, ) multithread_upload = mocker.patch( f"{dataset.__name__}.multithread_upload") with Tqdm(5, disable=False) as pbar: self.fusion_dataset_client._upload_segment( segment_test, jobs=8, skip_uploaded_files=True, pbar=pbar) get_or_create_segment.assert_called_once_with(segment_test.name) list_frames.assert_called_once_with() args, keywords = multithread_upload.call_args for index, values in enumerate(args[1]): data, sensor_name, frame_id = values assert data.path == f"{index * 2 + 1}.png" assert sensor_name == "lidar" assert frame_id == ulids[index * 2 + 1].str assert keywords[ "callback"] == segment_client._synchronize_upload_info assert keywords["jobs"] == 8 assert keywords["pbar"] == pbar multithread_upload.assert_called_once() with Tqdm(5, disable=False) as pbar: self.fusion_dataset_client._upload_segment( segment_test, jobs=8, skip_uploaded_files=False, pbar=pbar) get_or_create_segment.assert_called_with(segment_test.name) list_frames.assert_called_with() args, keywords = multithread_upload.call_args for index, values in enumerate(args[1]): data, sensor_name, frame_id = values assert data.path == f"{index}.png" if index % 2 == 0: assert sensor_name == "camera" else: assert sensor_name == "lidar" assert frame_id == ulids[index].str assert keywords[ "callback"] == segment_client._synchronize_upload_info assert keywords["jobs"] == 8 assert keywords["pbar"] == pbar