def test_upload_frame_with_label(self, accesskey, url, tmp_path): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_random_dataset_name() dataset_client = gas_client.create_dataset(dataset_name, is_fusion=True) dataset_client.create_draft("draft-1") dataset_client.upload_catalog(Catalog.loads(CATALOG)) segment_client = dataset_client.get_or_create_segment("segment1") segment_client.upload_sensor(Sensor.loads(LIDAR_DATA)) path = tmp_path / "sub" path.mkdir() for i in range(5): frame = Frame() local_path = path / f"hello{i}.txt" local_path.write_text("CONTENT") data = Data(local_path=str(local_path)) data.label = Label.loads(LABEL) frame[LIDAR_DATA["name"]] = data segment_client.upload_frame(frame, timestamp=i) frames = list(segment_client.list_frames()) assert len(frames) == 5 assert frames[0][LIDAR_DATA["name"]].path == "hello0.txt" assert frames[0][LIDAR_DATA["name"]].label # todo: match the input and output label gas_client.delete_dataset(dataset_name)
def test_replace_label(self, accesskey, url, tmp_path): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_random_dataset_name() dataset_client = gas_client.create_dataset(dataset_name) dataset_client.create_draft("draft-1") dataset_client.upload_catalog(Catalog.loads(CATALOG)) segment_client = dataset_client.get_or_create_segment("segment1") path = tmp_path / "sub" path.mkdir() local_path = path / "hello0.txt" local_path.write_text("CONTENT") data = Data(local_path=str(local_path)) segment_client.upload_file(data.path, data.target_remote_path) data.label = Label.loads(LABEL) segment_client.upload_label(data) # Replace labels data.label = Label.loads(NEW_LABEL) segment_client.upload_label(data) data = list(segment_client.list_data()) assert data[0].path == "hello0.txt" assert data[0].label # todo: match the input and output label gas_client.delete_dataset(dataset_name)
def test_delete_data(self, accesskey, url, tmp_path): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_random_dataset_name() dataset_client = gas_client.create_dataset(dataset_name) dataset_client.create_draft("draft-1") dataset_client.upload_catalog(Catalog.loads(CATALOG)) segment_client = dataset_client.get_or_create_segment("segment1") path = tmp_path / "sub" path.mkdir() for i in range(10): local_path = path / f"hello{i}.txt" local_path.write_text("CONTENT") data = Data(local_path=str(local_path)) data.label = Label.loads(LABEL) segment_client.upload_data(data) segment_client.delete_data("hello0.txt") data_paths = list(segment_client.list_data_paths()) assert "hello0.txt" not in data_paths segment_client.delete_data(segment_client.list_data_paths()) data = list(segment_client.list_data()) assert len(data) == 0 gas_client.delete_dataset(dataset_name)
def test_list_data_paths(self, accesskey, url, tmp_path): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_random_dataset_name() dataset_client = gas_client.create_dataset(dataset_name) dataset_client.create_draft("draft-1") segment_client = dataset_client.get_or_create_segment("segment1") path = tmp_path / "sub" path.mkdir() for i in range(5): local_path = path / f"hello{i}.txt" local_path.write_text("CONTENT") segment_client.upload_file(local_path=str(local_path)) # Add other files in reverse order for i in reversed(range(5)): local_path = path / f"goodbye{i}.txt" local_path.write_text("CONTENT") segment_client.upload_file(local_path=str(local_path)) data_paths = list(segment_client.list_data_paths()) assert data_paths[0] == "goodbye0.txt" assert data_paths[5] == "hello0.txt" gas_client.delete_dataset(dataset_name)
def test_upload_label(self, accesskey, url, tmp_path): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_random_dataset_name() dataset_client = gas_client.create_dataset(dataset_name) dataset_client.create_draft("draft-1") dataset_client.upload_catalog(Catalog.loads(CATALOG)) segment_client = dataset_client.get_or_create_segment("segment1") path = tmp_path / "sub" path.mkdir() local_path = path / "hello0.txt" local_path.write_text("CONTENT") data = Data(local_path=str(local_path)) data.label = Label.loads(LABEL) # If not uploading file, uploading label is not allowed with pytest.raises(GASResponseError): segment_client.upload_label(data) # Uploading files segment_client.upload_file(data.path, data.target_remote_path) data.label = Label.loads(WRONG_LABEL) # Uploading wrong label is not allowed with pytest.raises(GASResponseError): segment_client.upload_label(data) data.label = Label.loads(LABEL) segment_client.upload_label(data) data = list(segment_client.list_data()) assert data[0].path == "hello0.txt" assert data[0].label # todo: match the input and output label gas_client.delete_dataset(dataset_name)
def test_replace_file(self, accesskey, url, tmp_path): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_random_dataset_name() dataset_client = gas_client.create_dataset(dataset_name) dataset_client.create_draft("draft-1") segment_client = dataset_client.get_or_create_segment("segment1") path = tmp_path / "sub" path.mkdir() for i in range(5): local_path = path / f"hello{i}.txt" local_path.write_text("CONTENT") segment_client.upload_file(local_path=str(local_path)) # Replace files for i in range(5): local_path = path / f"hello{i}.txt" local_path.write_text("ADD CONTENT") segment_client.upload_file(local_path=str(local_path)) data = list(segment_client.list_data()) assert data[0].path == "hello0.txt" assert data[0].open().read() == b"ADD CONTENT" assert not data[0].label gas_client.delete_dataset(dataset_name)
def test_upload_frame_with_order(self, accesskey, url, tmp_path): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_random_dataset_name() dataset_client = gas_client.create_dataset(dataset_name, is_fusion=True) dataset_client.create_draft("draft-1") segment_client = dataset_client.get_or_create_segment("segment1") segment_client.upload_sensor(Sensor.loads(LIDAR_DATA)) path = tmp_path / "sub" path.mkdir() # If noe setting frame id in frame, set timestamp(order) when uploading for i in reversed(range(5)): frame = Frame() local_path = path / f"hello{i}.txt" local_path.write_text("CONTENT") data = Data(local_path=str(local_path)) frame[LIDAR_DATA["name"]] = data segment_client.upload_frame(frame, timestamp=i) # Set frame id in frame for i in range(5, 10): frame = Frame(frame_id=str(ulid.from_timestamp(i))) local_path = path / f"goodbye{i}.txt" local_path.write_text("CONTENT") data = Data(local_path=str(local_path)) frame[LIDAR_DATA["name"]] = data segment_client.upload_frame(frame) # Both setting frame id in frame and set timestamp(order) when uploading are not allowed i = 10 frame = Frame(frame_id=str(ulid.from_timestamp(i))) local_path = path / f"goodbye{i}.txt" local_path.write_text("CONTENT") data = Data(local_path=str(local_path)) frame[LIDAR_DATA["name"]] = data with pytest.raises(TypeError): segment_client.upload_frame(frame, timestamp=i) # Neither setting frame id in frame nor set timestamp(order) when uploading is not allowed frame = Frame() local_path = path / f"goodbye{i}.txt" local_path.write_text("CONTENT") data = Data(local_path=str(local_path)) frame[LIDAR_DATA["name"]] = data with pytest.raises(TypeError): segment_client.upload_frame(frame) frames = list(segment_client.list_frames()) assert len(frames) == 10 assert frames[0][LIDAR_DATA["name"]].path == "hello0.txt" assert frames[5][LIDAR_DATA["name"]].path == "goodbye5.txt" assert not frames[0][LIDAR_DATA["name"]].label # todo: match the input and output label gas_client.delete_dataset(dataset_name)
class TestVersionControlClient: gas_client = GAS("Accesskey-********************************") dataset_client = DatasetClient( "test_dataset", "12345", gas_client, status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID), alias="", is_public=DEFAULT_IS_PUBLIC, ) def test_squash_and_merge(self, mocker): post_data = { "title": "squash_merge-1", "sourceBranchName": "branch-1", "targetBranchName": "branch-2", "strategy": "abort", } response_data = {"draftNumber": 2} open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(data=response_data), ) draft_number = self.dataset_client.squash_and_merge( "squash_merge-1", source_branch_name="branch-1", target_branch_name="branch-2", strategy="abort", ) open_api_do.assert_called_once_with("POST", "squashAndMerge", self.dataset_client.dataset_id, json=post_data) assert draft_number == 2
class TestJobMixin: gas_client = GAS("Accesskey-********************************") dataset_client = DatasetClient( "test_dataset", "12345", gas_client, status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID), alias="", is_public=False, ) def test__create_job(self, mocker, mock_create_job): post_data = { "title": "test->main(abort)", "jobType": "squashAndMerge", "arguments": {"title": "draft-1"}, "description": "12", } open_api_do, response_data = mock_create_job(mocker) response_data.update( title=post_data["title"], arguments=post_data["arguments"], status="QUEUING", description=post_data["description"], ) assert response_data == self.dataset_client.squash_and_merge._create_job( post_data["title"], post_data["arguments"], post_data["description"], ) open_api_do.assert_called_once_with( "POST", "jobs", self.dataset_client.dataset_id, json=post_data ) def test__get_job(self, mocker, mock_get_job): job_id = "123" job_type = "squashAndMerge" open_api_do, response_data = mock_get_job(mocker) assert response_data == self.dataset_client.squash_and_merge._get_job(job_id) open_api_do.assert_called_once_with( "GET", f"jobs/{job_id}", self.dataset_client.dataset_id, params={"jobType": job_type} ) def test__list_jobs(self, mocker, mock_list_jobs): params = { "jobType": "squashAndMerge", "status": None, "offset": 0, "limit": 128, } open_api_do, response_data = mock_list_jobs(mocker) assert response_data == self.dataset_client.squash_and_merge._list_jobs( params["status"], params["offset"], params["limit"] ) open_api_do.assert_called_once_with( "GET", "jobs", self.dataset_client.dataset_id, params=params )
def test_sensor(self, accesskey, url): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_random_dataset_name() dataset_client = gas_client.create_dataset(dataset_name, is_fusion=True) dataset_client.create_draft("draft-1") segment_client = dataset_client.get_or_create_segment("segment1") segment_client.upload_sensor(Sensor.loads(LIDAR_DATA)) sensors = list(segment_client.list_sensors()) assert sensors # todo: match the input and output sensors segment_client.delete_sensor(LIDAR_DATA["name"]) sensors = list(segment_client.list_sensors()) assert len(sensors) == 0 gas_client.delete_dataset(dataset_name)
def test_upload_label_without_catalog(self, accesskey, url, tmp_path): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_random_dataset_name() dataset_client = gas_client.create_dataset(dataset_name) dataset_client.create_draft("draft-1") segment_client = dataset_client.get_or_create_segment("segment1") path = tmp_path / "sub" path.mkdir() local_path = path / "hello0.txt" local_path.write_text("CONTENT") data = Data(local_path=str(local_path)) segment_client.upload_file(data.path, data.target_remote_path) # If not uploading catalog, uploading label is not allowed data.label = Label.loads(LABEL) with pytest.raises(GASResponseError): segment_client.upload_label(data) gas_client.delete_dataset(dataset_name)
class TestVersionControlMixin: gas_client = GAS("Accesskey-********************************") dataset_client = DatasetClient( "test_dataset", "12345", gas_client, status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID), alias="", is_public=False, )
def test_upload_data_without_label(self, accesskey, url, tmp_path): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_random_dataset_name() dataset_client = gas_client.create_dataset(dataset_name) dataset_client.create_draft("draft-1") segment_client = dataset_client.get_or_create_segment("segment1") path = tmp_path / "sub" path.mkdir() for i in range(5): local_path = path / f"hello{i}.txt" local_path.write_text("CONTENT") segment_client.upload_data(Data(local_path=str(local_path))) data = list(segment_client.list_data()) assert data[0].path == "hello0.txt" assert data[0].open().read() == b"CONTENT" assert not data[0].label # todo: match the input and output label gas_client.delete_dataset(dataset_name)
def test_upload_frame_without_sensor(self, accesskey, url, tmp_path): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_random_dataset_name() dataset_client = gas_client.create_dataset(dataset_name, is_fusion=True) dataset_client.create_draft("draft-1") segment_client = dataset_client.get_or_create_segment("segment1") path = tmp_path / "sub" path.mkdir() frame = Frame() local_path = path / "hello0.txt" local_path.write_text("CONTENT") data = Data(local_path=str(local_path)) frame[LIDAR_DATA["name"]] = data # If not uploading sensor, uploading frame is not allowed with pytest.raises(GASResponseError): segment_client.upload_frame(frame, timestamp=0) gas_client.delete_dataset(dataset_name)
class TestSquashAndMergeJob: gas_client = GAS("Accesskey-********************************") dataset_client = DatasetClient( "test_dataset", "12345", gas_client, status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID), alias="", is_public=True, ) squash_and_merge_job1 = SquashAndMergeJob( client=dataset_client.squash_and_merge._client, dataset_id=dataset_client.dataset_id, job_updater=dataset_client.squash_and_merge._get_job, draft_getter=dataset_client.get_draft, title="test->main(abort)", job_id="234", arguments={"title": "draft-1"}, created_at=1, started_at=None, finished_at=None, status="QUEUEING", error_message="", result=None, description="12", ) squash_and_merge_job2 = SquashAndMergeJob( client=dataset_client.squash_and_merge._client, dataset_id=dataset_client.dataset_id, job_updater=dataset_client.squash_and_merge._get_job, draft_getter=dataset_client.get_draft, title="test->main(abort)", job_id="123", arguments={"title": "draft-1"}, created_at=1, started_at=2, finished_at=3, status="SUCCESS", error_message="", result={"draftNumber": 1}, description="12", ) def test_result(self, mocker, mock_list_drafts): list_drafts, drafts_list = mock_list_drafts(mocker, "main") assert self.squash_and_merge_job1.result is None draft_numbers = (item.number for item in drafts_list) assert self.squash_and_merge_job2.result.number in draft_numbers
class TestJob: gas_client = GAS("Accesskey-********************************") dataset_client = DatasetClient( "test_dataset", "12345", gas_client, status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID), alias="", is_public=True, ) job = Job( client=dataset_client.squash_and_merge._client, dataset_id=dataset_client.dataset_id, job_updater=dataset_client.squash_and_merge._get_job, title="test->main(abort)", job_id="123", arguments={"title": "draft-1"}, created_at=1, started_at=2, finished_at=3, status="SUCCESS", error_message="", result={"draftNumber": 3}, description="12", ) def test_update(self, mocker, mock_get_job): open_api_do, [_, final_responce] = mock_get_job(mocker, until_complete=True) self.job.update(until_complete=True) assert self.job.started_at == final_responce["startedAt"] assert self.job.finished_at == final_responce["finishedAt"] assert self.job.status == final_responce["status"] assert self.job.error_message == final_responce["errorMessage"] assert self.job._result == final_responce.get("result") open_api_do.assert_called_with("GET", f"jobs/{self.job.job_id}", self.dataset_client.dataset_id)
class TestDatasetClientBase: gas_client = GAS("Accesskey-********************************") dataset_client = DatasetClient( "test_dataset", "12345", gas_client, status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID), alias="", is_public=False, ) source_dataset_client = DatasetClient( "source_dataset", "544321", gas_client, status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID), alias="", is_public=False, ) def test__create_segment(self, mocker): post_data = {"name": "train"} post_data.update(self.dataset_client._status.get_status_info()) open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(), ) self.dataset_client._create_segment("train") open_api_do.assert_called_once_with("POST", "segments", self.dataset_client.dataset_id, json=post_data) def test__list_segment(self, mocker): params = self.dataset_client._status.get_status_info() offset, limit = 0, 128 params["offset"] = offset params["limit"] = limit open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(), ) self.dataset_client._list_segments() open_api_do.assert_called_once_with("GET", "segments", self.dataset_client._dataset_id, params=params) def test__generate_segment_names(self, mocker): params = self.dataset_client._status.get_status_info() offset, limit = 0, 128 params["offset"] = offset params["limit"] = limit response_data = { "offset": 0, "recordSize": 2, "totalCount": 2, "segments": [ { "name": "test", "description": "" }, { "name": "train", "description": "" }, ], } open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(data=response_data), ) assert list(self.dataset_client._generate_segment_names()) == [ segment["name"] for segment in response_data["segments"] ] open_api_do.assert_called_once_with("GET", "segments", self.dataset_client._dataset_id, params=params) def test__copy_segment(self, mocker): self.dataset_client._status.checkout(draft_number=1) source_name, target_name = "default", "train" with pytest.raises(InvalidParamsError): self.dataset_client._copy_segment(source_name, target_name, source_client=None, strategy="move") with pytest.raises(ValueError): self.dataset_client._copy_segment(source_name, source_name, source_client=None) open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(), ) source = {"segmentName": source_name} source["id"] = self.source_dataset_client.dataset_id source.update(self.source_dataset_client.status.get_status_info()) post_data = { "strategy": "abort", "source": source, "segmentName": target_name, } post_data.update(self.dataset_client._status.get_status_info()) self.dataset_client._copy_segment( source_name, target_name, source_client=self.source_dataset_client) open_api_do.assert_called_once_with("POST", "segments?copy", self.dataset_client._dataset_id, json=post_data) def test__move_segment(self, mocker): self.dataset_client._status.checkout(draft_number=1) source_name, target_name = "default", "train" with pytest.raises(InvalidParamsError): self.dataset_client._move_segment(source_name, target_name, strategy="move") open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(), ) post_data = { "strategy": "abort", "source": { "segmentName": source_name }, "segmentName": target_name, } post_data.update(self.dataset_client._status.get_status_info()) self.dataset_client._move_segment(source_name, target_name) open_api_do.assert_called_once_with("POST", "segments?move", self.dataset_client._dataset_id, json=post_data) def test_enable_cache( self, mocker, mock_list_segments, mock_list_data_details, mock_get_total_size, tmp_path, caplog, ): self.dataset_client._status.checkout(draft_number=1) with pytest.raises(StatusError): self.dataset_client.enable_cache() self.dataset_client._status.checkout(commit_id="commit-1") _, response_data = mock_get_total_size(mocker, True) _, _, free_storage = shutil.disk_usage(tempfile.gettempdir()) self.dataset_client.enable_cache() log_message = ( "bytes left on device, " f'less than the dataset size {response_data["totalSize"]} bytes.\n ' "Please be aware that there is not enough space to cache the entire dataset." ) assert log_message in caplog.text mock_get_total_size(mocker) cache_path = tmp_path / "cache_data" dataset_cache_path = cache_path / self.dataset_client.dataset_id self.dataset_client.enable_cache(cache_path) assert self.dataset_client.cache_enabled == True assert self.dataset_client._cache_path == str(dataset_cache_path) _, list_segments = mock_list_segments(mocker) segment_name = list_segments["segments"][0]["name"] segment = self.dataset_client.get_segment(segment_name) mock_list_data_details(mocker) segment_data = segment.list_data() urlopen = mocker.patch( f"{file.__name__}.RemoteFileMixin._urlopen", return_value=mock_response(read=lambda *args: bytes(1)), ) segment_cache_path = (dataset_cache_path / self.dataset_client.status.commit_id / segment_name) # Traverse the segment for the first time. for data in segment_data: assert data.cache_path == str(segment_cache_path / data.path) data.open() local_open = mocker.patch( f"builtins.open", return_value=bytes(1), ) segment_length = len(segment_data) # Traverse the segment for 3 times using cached data. epoch = 3 for i in range(epoch): for data in segment_data: assert data.cache_path == str(segment_cache_path / data.path) data.open() # local_open.assert_called_once() assert local_open.call_count == segment_length * epoch assert urlopen.call_count == segment_length self.dataset_client._status.checkout(draft_number=1) assert self.dataset_client.cache_enabled == False assert self.dataset_client._cache_path == str(dataset_cache_path) def test_update_notes(self, mocker): self.dataset_client._status.checkout(draft_number=1) patch_data = {"binPointCloudFields": None} patch_data.update(self.dataset_client._status.get_status_info()) open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(), ) self.dataset_client.update_notes(bin_point_cloud_fields=None) open_api_do.assert_called_once_with("PATCH", "notes", self.dataset_client._dataset_id, json=patch_data) patch_data = { "isContinuous": True, "binPointCloudFields": ["X", "Y", "Z", "Intensity", "Ring"], } patch_data.update(self.dataset_client._status.get_status_info()) open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(), ) self.dataset_client.update_notes( is_continuous=True, bin_point_cloud_fields=["X", "Y", "Z", "Intensity", "Ring"]) open_api_do.assert_called_once_with("PATCH", "notes", self.dataset_client._dataset_id, json=patch_data) def test_get_notes(self, mocker): params = self.dataset_client._status.get_status_info() response_data = { "isContinuous": True, "binPointCloudFields": ["X", "Y", "Z", "Intensity", "Ring"], } open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(data=response_data), ) assert self.dataset_client.get_notes() == Notes.loads(response_data) open_api_do.assert_called_once_with("GET", "notes", self.dataset_client._dataset_id, params=params) def test_list_segment(self, mocker): response_data = { "offset": 0, "recordSize": 2, "totalCount": 2, "segments": [ { "name": "test", "description": "" }, { "name": "train", "description": "" }, ], } mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(data=response_data), ) assert list(self.dataset_client.list_segment_names()) == [ segment["name"] for segment in response_data["segments"] ] def test_get_catelog(self, mocker): params = self.dataset_client._status.get_status_info() response_data = { "catalog": { "CLASSIFICATION": { "categories": [{ "name": "cat" }] } } } open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(data=response_data), ) assert self.dataset_client.get_catalog() == Catalog.loads( response_data["catalog"]) open_api_do.assert_called_once_with("GET", "labels/catalogs", self.dataset_client._dataset_id, params=params) def test_upload_catalog(self, mocker): self.dataset_client._status.checkout(draft_number=1) open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(), ) catalog = Catalog.loads( {"CLASSIFICATION": { "categories": [{ "name": "cat" }] }}) self.dataset_client.upload_catalog(catalog) put_data = {"catalog": catalog.dumps()} put_data.update(self.dataset_client._status.get_status_info()) open_api_do.assert_called_once_with("PUT", "labels/catalogs", self.dataset_client._dataset_id, json=put_data) def test_delete_segment(self, mocker): self.dataset_client._status.checkout(draft_number=1) delete_data = {"segmentName": "train"} delete_data.update(self.dataset_client._status.get_status_info()) open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(), ) self.dataset_client.delete_segment("train") open_api_do.assert_called_once_with("DELETE", "segments", self.dataset_client._dataset_id, json=delete_data) def test_get_label_statistics(self, mocker): params = self.dataset_client._status.get_status_info() response_data = { "labelStatistics": { "BOX2D": { "quantity": 10, "categories": [{ "name": "vehicles.bike", "quantity": 10, "attributes": [{ "name": "trafficLightColor", "enum": ["none", "red", "yellow"], "quantities": [5, 3, 2], }], }], } } } open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(data=response_data), ) statistics1 = self.dataset_client.get_label_statistics() open_api_do.assert_called_once_with("GET", "labels/statistics", self.dataset_client.dataset_id, params=params) assert statistics1 == Statistics(response_data["labelStatistics"]) def test_get_total_size(self, mocker, mock_get_total_size): self.dataset_client._status.checkout(commit_id="commit-1") params = {"commit": self.dataset_client._status.commit_id} get_total_size, response_data = mock_get_total_size(mocker) total_size = self.dataset_client.get_total_size() get_total_size.assert_called_once_with("GET", "total-size", self.dataset_client.dataset_id, params=params) assert total_size == response_data["totalSize"]
class TestGAS: gas_client = GAS("Accesskey-********************************") def test_generate_auth_storage_configs(self, mocker): params = {"name": "cloud_config", "offset": 0, "limit": 128} response_data = { "configs": [ { "name": "cloud_config", "type": "azure", "filePath": "tensorbay", "isGravitiStorage": True, }, ], "offset": 0, "recordSize": 1, "totalCount": 1, } open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(data=response_data), ) assert list( self.gas_client._generate_auth_storage_configs( "cloud_config")) == [ StorageConfig.loads(config) for config in response_data["configs"] ] open_api_do.assert_called_once_with("GET", "storage-configs", "", params=params) @pytest.mark.parametrize("is_fusion", [True, False]) @pytest.mark.parametrize("is_public", [True, False]) def test_get_dataset_with_any_type(self, mocker, is_fusion, is_public): response_data = { "name": "test", "type": int(is_fusion), "defaultBranch": DEFAULT_BRANCH, "commitId": "4", "updateTime": 1622693494, "owner": "", "id": "123456", "alias": "alias", "isPublic": is_public, } get_dataset = mocker.patch( f"{gas.__name__}.GAS._get_dataset", return_value=response_data, ) dataset_client = self.gas_client._get_dataset_with_any_type( response_data["name"]) dataset_type = FusionDatasetClient if is_fusion else DatasetClient assert isinstance(dataset_client, dataset_type) assert dataset_client._name == response_data["name"] assert dataset_client._dataset_id == response_data["id"] assert dataset_client._status.commit_id == response_data["commitId"] assert dataset_client._status.branch_name == response_data[ "defaultBranch"] assert dataset_client._alias == response_data["alias"] assert dataset_client._is_public == response_data["isPublic"] get_dataset.assert_called_once_with(response_data["name"]) def test__get_dataset(self, mocker): with pytest.raises(ResourceNotExistError): self.gas_client._get_dataset("") mocker.patch( f"{gas.__name__}.GAS._list_datasets", return_value={"datasets": []}, ) with pytest.raises(ResourceNotExistError): self.gas_client._get_dataset("test") list_dataset_data = { "datasets": [{ "id": "123456", "name": "test", "type": 0, "HEAD": { "commitId": "456123" }, "updateTime": 1622530298, "owner": "", "defaultBranch": DEFAULT_BRANCH, }], "offset": 0, "recordSize": 1, "totalCount": 1, } get_dataset_data = { "name": "test", "type": 0, "defaultBranch": DEFAULT_BRANCH, "commitId": "4", "updateTime": 1622693494, "owner": "", "id": "123456", "isPublic": False, } mocker.patch( f"{gas.__name__}.GAS._list_datasets", return_value=list_dataset_data, ) open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(data=get_dataset_data), ) assert self.gas_client._get_dataset( get_dataset_data["name"]) == get_dataset_data open_api_do.assert_called_once_with( "GET", "", list_dataset_data["datasets"][0]["id"]) def test__list_datasets(self, mocker): params = { "name": "test", "offset": 0, "limit": 128, } response_data = { "datasets": [{ "id": "123456", "name": "test", "type": 0, "defaultBranch": DEFAULT_BRANCH, "updateTime": 1622530298, "owner": "", }], "offset": 0, "recordSize": 1, "totalCount": 1, } open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(data=response_data), ) assert self.gas_client._list_datasets(params["name"]) == response_data open_api_do.assert_called_once_with("GET", "", params=params) def test_get_user(self, mocker): response_data = { "nickname": "user_name", "email": "user_email", "mobile": "11111111111", "description": "", "team": { "name": "TensorbaySDKTest", "email": None, "description": "" }, } dump_data = { "nickname": "user_name", "email": "user_email", "mobile": "11111111111", "team": { "name": "TensorbaySDKTest", }, } open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(data=response_data), ) assert self.gas_client.get_user().dumps() == dump_data open_api_do.assert_called_once_with("GET", "users") def test_get_auth_storage_config(self, mocker): with pytest.raises(TypeError): self.gas_client.get_auth_storage_config("") mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(data={ "configs": [], "totalCount": 0 }), ) with pytest.raises(ResourceNotExistError): self.gas_client.get_auth_storage_config("cloud_config") response_data = { "configs": [ { "name": "cloud_config", "type": "azure", "filePath": "tensorbay", "isGravitiStorage": True, }, ], "offset": 0, "recordSize": 1, "totalCount": 1, } mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(data=response_data), ) assert self.gas_client.get_auth_storage_config( response_data["configs"][0]["name"]) == StorageConfig.loads( response_data["configs"][0]) def test_list_auth_storage_configs(self, mocker): response_data = { "configs": [ { "name": "cloud_config", "type": "azure", "filePath": "tensorbay", "isGravitiStorage": True, }, ], "offset": 0, "recordSize": 1, "totalCount": 1, } mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(data=response_data), ) configs = self.gas_client.list_auth_storage_configs() assert list(configs) == [ StorageConfig.loads(config) for config in response_data["configs"] ] def test_delete_storage_config(self, mocker): open_api_do = mocker.patch(f"{gas.__name__}.Client.open_api_do") self.gas_client.delete_storage_config("test-config") open_api_do.assert_called_once_with("DELETE", "storage-configs/test-config") def test_create_oss_storage_config(self, mocker): open_api_do = mocker.patch(f"{gas.__name__}.Client.open_api_do") self.gas_client.create_oss_storage_config( "oss_config", "tests", endpoint="oss-cn-qingdao.aliyuncs.com", accesskey_id="accesskeyId", accesskey_secret="accesskeySecret", bucket_name="bucketName", ) post_data = { "name": "oss_config", "filePath": "tests", "endpoint": "oss-cn-qingdao.aliyuncs.com", "accesskeyId": "accesskeyId", "accesskeySecret": "accesskeySecret", "bucketName": "bucketName", } open_api_do("POST", "storage-configs/oss", json=post_data) def test_create_s3_storage_config(self, mocker): open_api_do = mocker.patch(f"{gas.__name__}.Client.open_api_do") self.gas_client.create_s3_storage_config( "s3_config", "tests", endpoint="s3.cn-northwest-1.amazonaws.com", accesskey_id="accesskeyId", accesskey_secret="accesskeySecret", bucket_name="bucketName", ) post_data = { "name": "s3_config", "filePath": "tests", "endpoint": "s3.cn-northwest-1.amazonaws.com", "accesskeyId": "accesskeyId", "accesskeySecret": "accesskeySecret", "bucketName": "bucketName", } open_api_do("POST", "storage-configs/s3", json=post_data) def test_create_azure_storage_config(self, mocker): open_api_do = mocker.patch(f"{gas.__name__}.Client.open_api_do") self.gas_client.create_azure_storage_config( "azure_config", "tests", account_type="China", account_name="accountName", account_key="accountKey", container_name="containerName", ) post_data = { "name": "s3_config", "filePath": "tests", "accesskeyId": "accountName", "accesskeySecret": "accountKey", "containerName": "containerName", "accountType": "China", } open_api_do("POST", "storage-configs/azure", json=post_data) def test_create_local_storage_config(self, mocker): open_api_do = mocker.patch(f"{gas.__name__}.Client.open_api_do") self.gas_client.create_local_storage_config( "local_config", "tests", "http://192.168.0.1:9000", ) post_data = { "name": "local_config", "filePath": "tests", "endpoint": "http://192.168.0.1:9000", } open_api_do("POST", "storage-configs/local", json=post_data) def test_get_cloud_client(self, mocker): config_name = "cloud_train" response_data = { "configs": [ { "name": "cloud_train", "type": "azure", "filePath": "tensorbay", "isGravitiStorage": True, }, ], "offset": 0, "recordSize": 1, "totalCount": 1, } mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(data=response_data), ) cloud_client = self.gas_client.get_cloud_client(config_name) cloud_client_1 = CloudClient(config_name, self.gas_client._client) assert cloud_client._name == cloud_client_1._name assert cloud_client._client == cloud_client_1._client @pytest.mark.parametrize("is_fusion", [True, False]) @pytest.mark.parametrize("is_public", [True, False]) def test_create_dataset(self, mocker, is_fusion, is_public): params = { "name": "test", "type": int(is_fusion), "configName": "config", "alias": "alias", "isPublic": is_public, } response_data = {"id": "12345"} open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(data=response_data), ) dataset_client = self.gas_client.create_dataset("test", is_fusion, config_name="config", alias="alias", is_public=is_public) dataset_type = FusionDatasetClient if is_fusion else DatasetClient assert isinstance(dataset_client, dataset_type) assert dataset_client._name == params["name"] assert dataset_client._dataset_id == response_data["id"] assert dataset_client.status.branch_name == DEFAULT_BRANCH assert dataset_client.status.commit_id == ROOT_COMMIT_ID assert dataset_client._alias == params["alias"] assert dataset_client._is_public == is_public open_api_do.assert_called_once_with("POST", "", json=params) @pytest.mark.parametrize("is_fusion", [True, False]) @pytest.mark.parametrize("is_public", [True, False]) def test_get_dataset(self, mocker, is_fusion, is_public): mocker.patch( f"{gas.__name__}.GAS._get_dataset", return_value={ "id": "123456", "type": 1, "commitId": "4", "defaultBranch": DEFAULT_BRANCH, "alias": "alias", "isPublic": is_public, }, ) with pytest.raises(DatasetTypeError): self.gas_client.get_dataset("test") response_data = { "id": "123456", "type": int(is_fusion), "commitId": "4", "defaultBranch": DEFAULT_BRANCH, "alias": "alias", "isPublic": is_public, } dataset_name = "test" get_dataset = mocker.patch( f"{gas.__name__}.GAS._get_dataset", return_value=response_data, ) dataset_client = self.gas_client.get_dataset(dataset_name, is_fusion) dataset_type = FusionDatasetClient if is_fusion else DatasetClient assert isinstance(dataset_client, dataset_type) assert dataset_client._name == dataset_name assert dataset_client._dataset_id == response_data["id"] assert dataset_client._status.commit_id == response_data["commitId"] assert dataset_client._status.branch_name == response_data[ "defaultBranch"] assert dataset_client._alias == response_data["alias"] assert dataset_client._is_public == response_data["isPublic"] get_dataset.assert_called_once_with(dataset_name) def test_list_dataset_names(self, mocker): params = { "offset": 0, "limit": 128, } response_data = { "datasets": [{ "id": "123456", "name": "test", "type": 0, "defaultBranch": DEFAULT_BRANCH, "updateTime": 1622530298, "owner": "", }], "offset": 0, "recordSize": 1, "totalCount": 1, } open_api_do = mocker.patch( f"{gas.__name__}.Client.open_api_do", return_value=mock_response(data=response_data), ) datasets = self.gas_client.list_dataset_names() assert list(datasets) == [ data["name"] for data in response_data["datasets"] ] open_api_do.assert_called_once_with("GET", "", params=params) def test_update_dataset(self, mocker): response_data = { "id": "123456", "type": 1, "commitId": "4", "defaultBranch": DEFAULT_BRANCH, } open_api_do = mocker.patch(f"{gas.__name__}.Client.open_api_do") mocker.patch( f"{gas.__name__}.GAS._get_dataset", return_value=response_data, ) patch_data = {"alias": "new_alias", "isPublic": True} self.gas_client.update_dataset("test", alias="new_alias", is_public=True) open_api_do.assert_called_once_with("PATCH", "", response_data["id"], json=patch_data) def test_rename_dataset(self, mocker): response_data = { "id": "123456", "type": 1, "commitId": "4", "defaultBranch": DEFAULT_BRANCH, } open_api_do = mocker.patch(f"{gas.__name__}.Client.open_api_do") mocker.patch( f"{gas.__name__}.GAS._get_dataset", return_value=response_data, ) patch_data = {"name": "new_test"} self.gas_client.rename_dataset("test", new_name="new_test") open_api_do.assert_called_once_with("PATCH", "", response_data["id"], json=patch_data) def test_upload_dataset(self, mocker): dataset = Dataset("test") dataset.load_catalog( os.path.join( os.path.dirname(__file__), "..", "..", "opendataset", "HeadPoseImage", "catalog.json", )) for i in range(5): dataset.create_segment(str(i)) # upload the dataset in main branch containing no draft get_dataset = mocker.patch( f"{gas.__name__}.GAS.get_dataset", return_value=DatasetClient( "test", "12345", self.gas_client, status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID), alias="", is_public=False, ), ) checkout = mocker.patch(f"{gas.__name__}.DatasetClient.checkout") list_drafts = mocker.patch(f"{gas.__name__}.DatasetClient.list_drafts", return_value=[]) create_draft = mocker.patch( f"{gas.__name__}.DatasetClient.create_draft") upload_catalog = mocker.patch( f"{gas.__name__}.DatasetClient.upload_catalog") update_notes = mocker.patch( f"{gas.__name__}.DatasetClient.update_notes") _upload_segment = mocker.patch( f"{gas.__name__}.DatasetClient._upload_segment") self.gas_client.upload_dataset(dataset) assert not checkout.called get_dataset.assert_called_once_with(dataset.name, False) list_drafts.assert_called_once_with(branch_name=DEFAULT_BRANCH) create_draft.assert_called_once_with( 'Draft autogenerated by "GAS.upload_dataset"', branch_name=DEFAULT_BRANCH) upload_catalog.assert_called_once_with(dataset.catalog) update_notes.assert_called_once_with(**dataset.notes) assert _upload_segment.call_count == 5 # upload the dataset in main branch containing a draft list_drafts = mocker.patch( f"{gas.__name__}.DatasetClient.list_drafts", return_value=[ Draft( 1, "title", DEFAULT_BRANCH, "OPEN", "4c564ea07f4e47679ec8c63d238bb3a1", User("test", 1636967807), 1637223060, ) ], ) checkout = mocker.patch(f"{gas.__name__}.DatasetClient.checkout") self.gas_client.upload_dataset(dataset) list_drafts.assert_called_once_with(branch_name=DEFAULT_BRANCH) checkout.assert_called_once_with(draft_number=1) # upload the dataset in dev branch containing no draft list_drafts = mocker.patch( f"{gas.__name__}.DatasetClient.list_drafts", return_value=[], ) checkout = mocker.patch(f"{gas.__name__}.DatasetClient.checkout") create_draft = mocker.patch( f"{gas.__name__}.DatasetClient.create_draft") self.gas_client.upload_dataset(dataset, branch_name="dev") assert not checkout.called list_drafts.assert_called_once_with(branch_name="dev") create_draft.assert_called_once_with( 'Draft autogenerated by "GAS.upload_dataset"', branch_name="dev") # upload the dataset in dev branch containing a draft list_drafts = mocker.patch( f"{gas.__name__}.DatasetClient.list_drafts", return_value=[ Draft( 1, "title", "dev", "OPEN", "4c564ea07f4e47679ec8c63d238bb3a1", User("test", 1636967807), 1637223060, ) ], ) checkout = mocker.patch(f"{gas.__name__}.DatasetClient.checkout") self.gas_client.upload_dataset(dataset, branch_name="dev") list_drafts.assert_called_once_with(branch_name="dev") checkout.assert_called_once_with(draft_number=1) def test_delete_dataset(self, mocker): response_data = {"id": "123456", "type": 1, "commitId": "4"} open_api_do = mocker.patch(f"{gas.__name__}.Client.open_api_do") mocker.patch( f"{gas.__name__}.GAS._get_dataset", return_value=response_data, ) self.gas_client.delete_dataset("test") open_api_do.assert_called_once_with("DELETE", "", response_data["id"])
class TestFusionDatasetClient(TestDatasetClientBase): gas_client = GAS("Accesskey-********************************") fusion_dataset_client = FusionDatasetClient( "test_dataset", "12345", gas_client, status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID), alias="", is_public=False, ) source_dataset_client = FusionDatasetClient( "source_dataset", "544321", gas_client, status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID), alias="", is_public=False, ) def test__extract_all_data(self): source_frames = [] ulids = [] for i in range(5): temp_frame = Frame() if i % 2 == 0: temp_frame["camera"] = Data(f"{i}.png") else: temp_frame["lidar"] = Data(f"{i}.png") temp_ulid = from_timestamp(10 * i + 10) temp_frame.frame_id = temp_ulid source_frames.append((temp_frame, temp_ulid)) ulids.append(temp_ulid) with Tqdm(5, disable=False) as pbar: for index, values in enumerate( self.fusion_dataset_client._extract_all_data( source_frames, pbar)): data, sensor_name, frame_id = values assert data.path == f"{index}.png" if index % 2 == 0: assert sensor_name == "camera" else: assert sensor_name == "lidar" assert frame_id == ulids[index].str def test__extract_unuploaded_data(self): source_frames = [] ulids = [] done_frames = {} for i in range(5): temp_frame = Frame() temp_ulid = from_timestamp(10 * i + 10) if i % 2 == 0: temp_frame["camera"] = Data(f"{i}.png") done_frames[temp_ulid.timestamp().timestamp] = temp_frame else: temp_frame["lidar"] = Data(f"{i}.png") ulids.append(temp_ulid) temp_frame.frame_id = temp_ulid source_frames.append((temp_frame, temp_ulid)) with Tqdm(5, disable=False) as pbar: for index, values in enumerate( self.fusion_dataset_client._extract_unuploaded_data( source_frames, pbar, done_frames=done_frames)): data, sensor_name, frame_id = values assert data.path == f"{index * 2 + 1}.png" assert sensor_name == "lidar" assert frame_id == ulids[index].str def test__generate_segments(self, mocker): params = self.dataset_client._status.get_status_info() offset, limit = 0, 128 params["offset"] = offset params["limit"] = limit response_data = { "offset": 0, "recordSize": 2, "totalCount": 2, "segments": [ { "name": "test", "description": "" }, { "name": "train", "description": "" }, ], } list_segments = mocker.patch( f"{dataset.__name__}.FusionDatasetClient._list_segments", return_value=response_data, ) segment_generator = ReturnGenerator( self.fusion_dataset_client._generate_segments()) assert [segment.name for segment in segment_generator ] == [item["name"] for item in response_data["segments"]] list_segments.assert_called_once_with(offset, limit) assert segment_generator.value == response_data["totalCount"] def test__upload_segment(self, mocker): segment_test = FusionSegment(name="test1") ulids = [] done_frames = [] for i in range(5): temp_frame = Frame() temp_ulid = from_timestamp(10 * i + 10) temp_frame.frame_id = temp_ulid if i % 2 == 0: temp_frame["camera"] = Data(f"{i}.png") done_frames.append(temp_frame) else: temp_frame["lidar"] = Data(f"{i}.png") ulids.append(temp_ulid) segment_test.append(temp_frame) segment_client = FusionSegmentClient( name="test1", data_client=self.fusion_dataset_client) get_or_create_segment = mocker.patch( f"{dataset.__name__}.FusionDatasetClient.get_or_create_segment", return_value=segment_client, ) list_frames = mocker.patch( f"{segment.__name__}.FusionSegmentClient.list_frames", return_value=done_frames, ) multithread_upload = mocker.patch( f"{dataset.__name__}.multithread_upload") with Tqdm(5, disable=False) as pbar: self.fusion_dataset_client._upload_segment( segment_test, jobs=8, skip_uploaded_files=True, pbar=pbar) get_or_create_segment.assert_called_once_with(segment_test.name) list_frames.assert_called_once_with() args, keywords = multithread_upload.call_args for index, values in enumerate(args[1]): data, sensor_name, frame_id = values assert data.path == f"{index * 2 + 1}.png" assert sensor_name == "lidar" assert frame_id == ulids[index * 2 + 1].str assert keywords[ "callback"] == segment_client._synchronize_upload_info assert keywords["jobs"] == 8 assert keywords["pbar"] == pbar multithread_upload.assert_called_once() with Tqdm(5, disable=False) as pbar: self.fusion_dataset_client._upload_segment( segment_test, jobs=8, skip_uploaded_files=False, pbar=pbar) get_or_create_segment.assert_called_with(segment_test.name) list_frames.assert_called_with() args, keywords = multithread_upload.call_args for index, values in enumerate(args[1]): data, sensor_name, frame_id = values assert data.path == f"{index}.png" if index % 2 == 0: assert sensor_name == "camera" else: assert sensor_name == "lidar" assert frame_id == ulids[index].str assert keywords[ "callback"] == segment_client._synchronize_upload_info assert keywords["jobs"] == 8 assert keywords["pbar"] == pbar def test_get_or_create_segment(self, mocker): self.fusion_dataset_client._status.checkout(draft_number=1) list_segment = mocker.patch( f"{dataset.__name__}.FusionDatasetClient.list_segment_names", return_value=["test1"]) create_segment = mocker.patch( f"{dataset.__name__}.FusionDatasetClient._create_segment", ) segment_client_1 = self.fusion_dataset_client.get_or_create_segment( "test1") list_segment.assert_called_once_with() assert segment_client_1.name == "test1" segment_client_2 = self.fusion_dataset_client.get_or_create_segment( "test2") list_segment.assert_called_with() create_segment.assert_called_once_with("test2") assert segment_client_2.name == "test2" def test_create_segment(self, mocker): self.fusion_dataset_client._status.checkout(draft_number=1) list_segment = mocker.patch( f"{dataset.__name__}.FusionDatasetClient.list_segment_names", return_value=["test1"]) create_segment = mocker.patch( f"{dataset.__name__}.FusionDatasetClient._create_segment", ) with pytest.raises(NameConflictError): self.fusion_dataset_client.create_segment("test1") list_segment.assert_called_once_with() segment_client = self.fusion_dataset_client.create_segment("test2") list_segment.assert_called_with() create_segment.assert_called_once_with("test2") assert segment_client.name == "test2" def test_copy_segment(self, mocker): self.fusion_dataset_client._status.checkout(draft_number=1) source_name, target_name = "default", "train" copy_segment = mocker.patch( f"{dataset.__name__}.FusionDatasetClient._copy_segment") strategy = "abort" assert self.fusion_dataset_client.copy_segment( source_name).name == source_name copy_segment.assert_called_once_with(source_name, source_name, source_client=None, strategy=strategy) strategy = "override" assert (self.fusion_dataset_client.copy_segment( source_name, target_name, strategy=strategy).name == target_name) copy_segment.assert_called_with(source_name, target_name, source_client=None, strategy=strategy) def test_move_segment(self, mocker): self.fusion_dataset_client._status.checkout(draft_number=1) source_name, target_name = "default", "train" move_segment = mocker.patch( f"{dataset.__name__}.FusionDatasetClient._move_segment") strategy = "abort" assert self.fusion_dataset_client.move_segment( source_name, target_name).name == target_name move_segment.assert_called_once_with(source_name, target_name, strategy=strategy) def test_get_segment(self, mocker): list_segment = mocker.patch( f"{dataset.__name__}.FusionDatasetClient.list_segment_names", return_value=["test1"]) with pytest.raises(ResourceNotExistError): self.fusion_dataset_client.get_segment("test2") list_segment.assert_called_once_with() assert self.fusion_dataset_client.get_segment("test1").name == "test1" list_segment.assert_called_with() def test_upload_segment(self, mocker): self.fusion_dataset_client._status.checkout(draft_number=1) segment_test = FusionSegment(name="test1") for i in range(5): temp_frame = Frame() temp_ulid = from_timestamp(10 * i + 10) temp_frame.frame_id = temp_ulid if i % 2 == 0: temp_frame["camera"] = Data(f"{i}.png") else: temp_frame["lidar"] = Data(f"{i}.png") segment_test.append(temp_frame) segment_client = FusionSegmentClient( name="test1", data_client=self.fusion_dataset_client) upload_segment = mocker.patch( f"{dataset.__name__}.FusionDatasetClient._upload_segment", return_value=segment_client) assert self.fusion_dataset_client.upload_segment( segment_test).name == "test1" args, keywords = upload_segment.call_args assert args[0] == segment_test assert keywords["jobs"] == 1 assert not keywords["skip_uploaded_files"] upload_segment.assert_called_once()