def test_upload_dataset_to_given_draft(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client_1 = gas_client.create_dataset(dataset_name)
draft_number = dataset_client_1.create_draft("test")
dataset = Dataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
segment.append(Data(local_path=str(local_path)))
dataset_client_2 = gas_client.upload_dataset(dataset,
draft_number=draft_number)
segment1 = Segment("Segment1", client=dataset_client_2)
assert len(segment1) == 10
assert segment1[0].path == "hello0.txt"
assert not segment1[0].label
with pytest.raises(GASResponseError):
gas_client.upload_dataset(dataset, draft_number=draft_number + 1)
gas_client.delete_dataset(dataset_name)
def init_dataset_client(accesskey, url, tmp_path_factory):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name, is_fusion=True)
dataset = FusionDataset(name=dataset_name)
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path_factory.mktemp("sub")
os.makedirs(path, exist_ok=True)
for segment_name in SEGMENTS_NAME:
segment = dataset.create_segment(segment_name)
frame = Frame()
for camera_name, label in LABEL.items():
camera = Camera(camera_name)
translation = Vector3D(1, 2, 3)
camera.set_extrinsics(translation=translation)
camera.set_extrinsics(translation=translation)
camera.set_camera_matrix(fx=1.1, fy=1.1, cx=1.1, cy=1.1)
camera.set_distortion_coefficients(p1=1.2, p2=1.2, k1=1.2, k2=1.2)
segment.sensors.add(camera)
local_path = path / f"{segment_name}_{camera_name}.txt"
local_path.write_text(f"CONTENT_{segment_name}_{camera_name}")
data = Data(local_path=str(local_path))
data.label = Label.loads(label)
frame[camera_name] = data
segment.append(frame)
dataset_client = gas_client.upload_dataset(dataset)
dataset_client.commit("commit-1")
yield dataset_client
gas_client.delete_dataset(dataset_name)
def test_upload_dataset_with_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
gas_client.create_dataset(dataset_name)
dataset = Dataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
# When uploading label, upload catalog first.
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment.append(data)
dataset_client = gas_client.upload_dataset(dataset)
assert dataset_client.get_catalog()
segment1 = Segment("Segment1", client=dataset_client)
assert len(segment1) == 10
assert segment1[0].path == "hello0.txt"
assert segment1[0].label
gas_client.delete_dataset(dataset_name)
def init_dataset_client(accesskey, url, tmp_path_factory):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.commit("commit-1")
dataset_client.create_branch("dev")
dataset = Dataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path_factory.mktemp("sub")
os.makedirs(path, exist_ok=True)
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text(f"CONTENT_{i}")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL_2)
segment.append(data)
dataset_client = gas_client.upload_dataset(dataset, branch_name="dev")
dataset_client.commit("commit-2")
dataset_client.checkout(DEFAULT_BRANCH)
dataset = Dataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path_factory.mktemp("sub")
os.makedirs(path, exist_ok=True)
for i in range(4):
local_path = path / f"hello{i}.txt"
local_path.write_text(f"CONTENT_{i}")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL_1)
segment.append(data)
dataset_client = gas_client.upload_dataset(dataset,
branch_name=DEFAULT_BRANCH)
dataset_client.commit("commit-3")
yield dataset_client
gas_client.delete_dataset(dataset_name)
def test_upload_dataset_only_with_file(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
gas_client.create_dataset(dataset_name)
dataset = Dataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
segment.append(Data(local_path=str(local_path)))
dataset_client = gas_client.upload_dataset(dataset)
assert not dataset_client.get_catalog()
segment1 = Segment("Segment1", client=dataset_client)
assert len(segment1) == 10
assert segment1[0].path == "hello0.txt"
assert not segment1[0].label
gas_client.delete_dataset(dataset_name)
