def test_upload_dataset_to_given_draft(self, accesskey, url, tmp_path): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_random_dataset_name() dataset_client_1 = gas_client.create_dataset(dataset_name) draft_number = dataset_client_1.create_draft("test") dataset = Dataset(name=dataset_name) segment = dataset.create_segment("Segment1") path = tmp_path / "sub" path.mkdir() for i in range(10): local_path = path / f"hello{i}.txt" local_path.write_text("CONTENT") segment.append(Data(local_path=str(local_path))) dataset_client_2 = gas_client.upload_dataset(dataset, draft_number=draft_number) segment1 = Segment("Segment1", client=dataset_client_2) assert len(segment1) == 10 assert segment1[0].path == "hello0.txt" assert not segment1[0].label with pytest.raises(GASResponseError): gas_client.upload_dataset(dataset, draft_number=draft_number + 1) gas_client.delete_dataset(dataset_name)
def init_dataset_client(accesskey, url, tmp_path_factory): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_dataset_name() gas_client.create_dataset(dataset_name, is_fusion=True) dataset = FusionDataset(name=dataset_name) dataset._catalog = Catalog.loads(CATALOG) path = tmp_path_factory.mktemp("sub") os.makedirs(path, exist_ok=True) for segment_name in SEGMENTS_NAME: segment = dataset.create_segment(segment_name) frame = Frame() for camera_name, label in LABEL.items(): camera = Camera(camera_name) translation = Vector3D(1, 2, 3) camera.set_extrinsics(translation=translation) camera.set_extrinsics(translation=translation) camera.set_camera_matrix(fx=1.1, fy=1.1, cx=1.1, cy=1.1) camera.set_distortion_coefficients(p1=1.2, p2=1.2, k1=1.2, k2=1.2) segment.sensors.add(camera) local_path = path / f"{segment_name}_{camera_name}.txt" local_path.write_text(f"CONTENT_{segment_name}_{camera_name}") data = Data(local_path=str(local_path)) data.label = Label.loads(label) frame[camera_name] = data segment.append(frame) dataset_client = gas_client.upload_dataset(dataset) dataset_client.commit("commit-1") yield dataset_client gas_client.delete_dataset(dataset_name)
def test_upload_dataset_with_label(self, accesskey, url, tmp_path): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_random_dataset_name() gas_client.create_dataset(dataset_name) dataset = Dataset(name=dataset_name) segment = dataset.create_segment("Segment1") # When uploading label, upload catalog first. dataset._catalog = Catalog.loads(CATALOG) path = tmp_path / "sub" path.mkdir() for i in range(10): local_path = path / f"hello{i}.txt" local_path.write_text("CONTENT") data = Data(local_path=str(local_path)) data.label = Label.loads(LABEL) segment.append(data) dataset_client = gas_client.upload_dataset(dataset) assert dataset_client.get_catalog() segment1 = Segment("Segment1", client=dataset_client) assert len(segment1) == 10 assert segment1[0].path == "hello0.txt" assert segment1[0].label gas_client.delete_dataset(dataset_name)
def init_dataset_client(accesskey, url, tmp_path_factory): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_dataset_name() dataset_client = gas_client.create_dataset(dataset_name) dataset_client.create_draft("draft-1") dataset_client.commit("commit-1") dataset_client.create_branch("dev") dataset = Dataset(name=dataset_name) segment = dataset.create_segment("Segment1") dataset._catalog = Catalog.loads(CATALOG) path = tmp_path_factory.mktemp("sub") os.makedirs(path, exist_ok=True) for i in range(10): local_path = path / f"hello{i}.txt" local_path.write_text(f"CONTENT_{i}") data = Data(local_path=str(local_path)) data.label = Label.loads(LABEL_2) segment.append(data) dataset_client = gas_client.upload_dataset(dataset, branch_name="dev") dataset_client.commit("commit-2") dataset_client.checkout(DEFAULT_BRANCH) dataset = Dataset(name=dataset_name) segment = dataset.create_segment("Segment1") dataset._catalog = Catalog.loads(CATALOG) path = tmp_path_factory.mktemp("sub") os.makedirs(path, exist_ok=True) for i in range(4): local_path = path / f"hello{i}.txt" local_path.write_text(f"CONTENT_{i}") data = Data(local_path=str(local_path)) data.label = Label.loads(LABEL_1) segment.append(data) dataset_client = gas_client.upload_dataset(dataset, branch_name=DEFAULT_BRANCH) dataset_client.commit("commit-3") yield dataset_client gas_client.delete_dataset(dataset_name)
def test_upload_dataset_only_with_file(self, accesskey, url, tmp_path): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_random_dataset_name() gas_client.create_dataset(dataset_name) dataset = Dataset(name=dataset_name) segment = dataset.create_segment("Segment1") path = tmp_path / "sub" path.mkdir() for i in range(10): local_path = path / f"hello{i}.txt" local_path.write_text("CONTENT") segment.append(Data(local_path=str(local_path))) dataset_client = gas_client.upload_dataset(dataset) assert not dataset_client.get_catalog() segment1 = Segment("Segment1", client=dataset_client) assert len(segment1) == 10 assert segment1[0].path == "hello0.txt" assert not segment1[0].label gas_client.delete_dataset(dataset_name)