def test_move_data_skip(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name)
dataset = Dataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text(f"CONTENT_{i}")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment.append(data)
dataset_client = gas_client.upload_dataset(dataset)
segment_client = dataset_client.get_segment("Segment1")
segment_client.move_data("hello0.txt", "hello1.txt", strategy="skip")
segment_moved = Segment("Segment1", client=dataset_client)
assert segment_moved[0].path == "hello1.txt"
assert segment_moved[0].open().read() == b"CONTENT_1"
gas_client.delete_dataset(dataset_name)
def test_import_cloud_files_to_fusiondataset(self, accesskey, url, config_name):
gas_client = GAS(access_key=accesskey, url=url)
try:
cloud_client = gas_client.get_cloud_client(config_name)
except ResourceNotExistError:
pytest.skip(f"skip this case because there's no {config_name} config")
auth_data = cloud_client.list_auth_data("tests")[:5]
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name, True, config_name=config_name)
dataset = FusionDataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
lidar = Lidar("LIDAR")
segment.sensors.add(lidar)
for data in auth_data:
data.label.classification = Classification("cat", attributes={"color": "red"})
frame = Frame()
frame["LIDAR"] = data
segment.append(frame)
dataset_client = gas_client.upload_dataset(dataset, jobs=5)
dataset_client.commit("import data")
segment1 = FusionSegment("Segment1", client=dataset_client)
assert len(segment1) == len(segment)
assert segment1[0]["LIDAR"].path == segment[0]["LIDAR"].path.split("/")[-1]
assert segment1[0]["LIDAR"].label.classification.category == "cat"
assert segment1[0]["LIDAR"].label.classification.attributes["color"] == "red"
assert len(auth_data) == len(segment)
gas_client.delete_dataset(dataset_name)
def test_get_commit(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.commit("commit-1", "commit-1-description")
commit_1_id = dataset_client.status.commit_id
dataset_client.create_draft("draft-2")
dataset_client.commit("commit-2")
commit_2_id = dataset_client.status.commit_id
# Get top commit
commit = dataset_client.get_commit(commit_2_id)
assert commit.commit_id == commit_2_id
assert commit.parent_commit_id == commit_1_id
assert commit.title == "commit-2"
assert commit.description == ""
assert commit.committer.name
assert commit.committer.date
# Get one commit before the top one
commit = dataset_client.get_commit(commit_1_id)
assert commit.commit_id == commit_1_id
assert commit.parent_commit_id == ROOT_COMMIT_ID
assert commit.title == "commit-1"
assert commit.description == "commit-1-description"
assert commit.committer.name
assert commit.committer.date
gas_client.delete_dataset(dataset_name)
def test_get_tag(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.commit("commit-1", "test", tag="V1")
commit_1_id = dataset_client.status.commit_id
tag = dataset_client.get_tag("V1")
assert tag.name == "V1"
assert tag.commit_id == commit_1_id
assert tag.parent_commit_id == ROOT_COMMIT_ID
assert tag.title == "commit-1"
assert tag.description == "test"
assert tag.committer.name
assert tag.committer.date
# Can not get a non-exist tag
with pytest.raises(ResourceNotExistError):
dataset_client.get_tag("V2")
dataset_client.create_draft("draft-2")
dataset_client.commit("commit-2", "test", tag="V2")
commit_2_id = dataset_client.status.commit_id
tag = dataset_client.get_tag("V2")
assert tag.name == "V2"
assert tag.commit_id == commit_2_id
assert tag.parent_commit_id == commit_1_id
assert tag.title == "commit-2"
assert tag.description == "test"
assert tag.committer.name
assert tag.committer.date
gas_client.delete_dataset(dataset_name)
def test_create_and_upload_dataset_with_config(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
try:
gas_client.get_auth_storage_config(name=_LOCAL_CONFIG_NAME)
except ResourceNotExistError:
pytest.skip(f"skip this case because there's no {_LOCAL_CONFIG_NAME} config")
gas_client.create_dataset(dataset_name, config_name=_LOCAL_CONFIG_NAME)
dataset = Dataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
# When uploading label, upload catalog first.
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path / "sub"
path.mkdir()
for i in range(5):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment.append(data)
dataset_client = gas_client.upload_dataset(dataset)
assert dataset_client.get_catalog()
segment1 = Segment("Segment1", client=dataset_client)
assert len(segment1) == 5
for i in range(5):
assert segment1[i].path == f"hello{i}.txt"
assert segment1[i].label
gas_client.delete_dataset(dataset_name)
def test_overwrite_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(BOX2D_CATALOG))
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
local_path = path / "hello0.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
segment_client.upload_file(data.path, data.target_remote_path)
data.label = Label.loads(LABEL)
segment_client.upload_label(data)
# Replace labels
data.label = Label.loads(NEW_LABEL)
segment_client.upload_label(data)
data = segment_client.list_data()
assert data[0].path == "hello0.txt"
assert data[0].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_delete_frame(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name, is_fusion=True)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(BOX2D_CATALOG))
segment_client = dataset_client.get_or_create_segment("segment1")
segment_client.upload_sensor(Sensor.loads(LIDAR_DATA))
path = tmp_path / "sub"
path.mkdir()
for i in range(5):
frame = Frame()
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
frame[LIDAR_DATA["name"]] = data
segment_client.upload_frame(frame, timestamp=i)
frame_1_id = segment_client.list_frames()[0].frame_id
segment_client.delete_frame(frame_1_id)
frame_ids = [frame.frame_id for frame in segment_client.list_frames()]
assert frame_1_id not in frame_ids
gas_client.delete_dataset(dataset_name)
def test_move_segment(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name)
dataset = Dataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment.append(data)
dataset_client = gas_client.upload_dataset(dataset)
segment_client = dataset_client.move_segment("Segment1", "Segment2")
assert segment_client.name == "Segment2"
with pytest.raises(InvalidParamsError):
dataset_client.move_segment("Segment1", "Segment3", strategy="push")
segment2 = Segment("Segment2", client=dataset_client)
assert segment2[0].path == "hello0.txt"
assert segment2[0].path == segment[0].target_remote_path
assert segment2[0].label
gas_client.delete_dataset(dataset_name)
def test_overwrite_file(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(5):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
segment_client.upload_file(local_path=str(local_path))
# Replace files
for i in range(5):
local_path = path / f"hello{i}.txt"
local_path.write_text("ADD CONTENT")
segment_client.upload_file(local_path=str(local_path))
data = segment_client.list_data()
assert data[0].path == "hello0.txt"
assert data[0].open().read() == b"ADD CONTENT"
assert not data[0].label
gas_client.delete_dataset(dataset_name)
def test_copy_data(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name)
dataset = Dataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment.append(data)
dataset_client = gas_client.upload_dataset(dataset)
segment_client = dataset_client.get_segment("Segment1")
segment_client.copy_data("hello0.txt", "goodbye0.txt")
segment_client.copy_data("hello1.txt", "hello10.txt")
with pytest.raises(InvalidParamsError):
segment_client.copy_data("hello2.txt", "see_you.txt", strategy="push")
segment2 = Segment("Segment1", client=dataset_client)
assert segment2[0].path == "goodbye0.txt"
assert segment2[3].path == "hello10.txt"
assert segment2[1].label
gas_client.delete_dataset(dataset_name)
def test_copy_data_between_datasets(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name_1 = get_dataset_name()
gas_client.create_dataset(dataset_name_1)
dataset_1 = Dataset(name=dataset_name_1)
segment_1 = dataset_1.create_segment("Segment1")
dataset_1._catalog = Catalog.loads(CATALOG)
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment_1.append(data)
dataset_client_1 = gas_client.upload_dataset(dataset_1)
dataset_client_1.commit("upload data")
segment_client_1 = dataset_client_1.get_segment("Segment1")
dataset_name_2 = dataset_name_1 + "_2"
dataset_client_2 = gas_client.create_dataset(dataset_name_2)
dataset_client_2.create_draft("draft_2")
dataset_client_2.create_segment("Segment1")
segment_client_2 = dataset_client_2.get_segment("Segment1")
segment_client_2.copy_data("hello0.txt", "hello0.txt", source_client=segment_client_1)
segment2 = Segment("Segment1", client=dataset_client_2)
assert segment2[0].path == "hello0.txt"
assert segment2[0].label
gas_client.delete_dataset(dataset_name_1)
gas_client.delete_dataset(dataset_name_2)
def test_copy_segment_skip(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name)
dataset = Dataset(name=dataset_name)
segment1 = dataset.create_segment("Segment1")
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment1.append(data)
segment2 = dataset.create_segment("Segment2")
for i in range(10, 20):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment2.append(data)
dataset_client = gas_client.upload_dataset(dataset)
dataset_client.copy_segment("Segment1", "Segment2", strategy="skip")
segment_copied = Segment("Segment2", client=dataset_client)
assert segment_copied[0].path == "hello10.txt"
assert segment_copied[0].path == segment2[0].target_remote_path
assert segment_copied[0].label
gas_client.delete_dataset(dataset_name)
def test_copy_segment_abort(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name)
dataset = Dataset(name=dataset_name)
segment1 = dataset.create_segment("Segment1")
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment1.append(data)
segment2 = dataset.create_segment("Segment2")
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment2.append(data)
dataset_client = gas_client.upload_dataset(dataset)
with pytest.raises(InternalServerError):
dataset_client.copy_segment("Segment1", "Segment2")
gas_client.delete_dataset(dataset_name)
def test_copy_fusion_segment(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name, is_fusion=True)
dataset = FusionDataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
segment.sensors.add(Sensor.loads(LIDAR_DATA))
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
frame = Frame()
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
frame[LIDAR_DATA["name"]] = data
segment.append(frame)
dataset_client = gas_client.upload_dataset(dataset)
segment_client = dataset_client.copy_segment("Segment1", "Segment2")
assert segment_client.name == "Segment2"
with pytest.raises(InvalidParamsError):
dataset_client.copy_segment("Segment1", "Segment3", strategy="push")
segment2 = FusionSegment("Segment2", client=dataset_client)
assert segment2[0][LIDAR_DATA["name"]].path == "hello0.txt"
assert (
segment2[0][LIDAR_DATA["name"]].path
== segment[0][LIDAR_DATA["name"]].target_remote_path
)
assert segment2[0][LIDAR_DATA["name"]].label
gas_client.delete_dataset(dataset_name)
def init_dataset_client(accesskey, url, tmp_path_factory):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name, is_fusion=True)
dataset = FusionDataset(name=dataset_name)
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path_factory.mktemp("sub")
os.makedirs(path, exist_ok=True)
for segment_name in SEGMENTS_NAME:
segment = dataset.create_segment(segment_name)
frame = Frame()
for camera_name, label in LABEL.items():
camera = Camera(camera_name)
translation = Vector3D(1, 2, 3)
camera.set_extrinsics(translation=translation)
camera.set_extrinsics(translation=translation)
camera.set_camera_matrix(fx=1.1, fy=1.1, cx=1.1, cy=1.1)
camera.set_distortion_coefficients(p1=1.2, p2=1.2, k1=1.2, k2=1.2)
segment.sensors.add(camera)
local_path = path / f"{segment_name}_{camera_name}.txt"
local_path.write_text(f"CONTENT_{segment_name}_{camera_name}")
data = Data(local_path=str(local_path))
data.label = Label.loads(label)
frame[camera_name] = data
segment.append(frame)
dataset_client = gas_client.upload_dataset(dataset)
dataset_client.commit("commit-1")
yield dataset_client
gas_client.delete_dataset(dataset_name)
def test_get_branch(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.commit("commit-1")
commit_1_id = dataset_client.status.commit_id
dataset_client.create_draft("draft-2")
dataset_client.commit("commit-2", "test")
commit_2_id = dataset_client.status.commit_id
branch = dataset_client.get_branch(DEFAULT_BRANCH)
assert branch.name == DEFAULT_BRANCH
assert branch.commit_id == commit_2_id
assert branch.parent_commit_id == commit_1_id
assert branch.title == "commit-2"
assert branch.description == "test"
assert branch.committer.name
assert branch.committer.date
with pytest.raises(ResourceNotExistError):
dataset_client.get_branch("main1")
dataset_client.create_branch("T123")
branch = dataset_client.get_branch("T123")
assert branch.name == "T123"
assert branch.commit_id == commit_2_id
assert branch.parent_commit_id == commit_1_id
assert branch.title == "commit-2"
assert branch.description == "test"
assert branch.committer.name
assert branch.committer.date
gas_client.delete_dataset(dataset_name)
def test_upload_dataset_only_with_file(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name)
dataset = Dataset(name=dataset_name)
dataset.notes.is_continuous = True
segment = dataset.create_segment("Segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
segment.append(Data(local_path=str(local_path)))
dataset_client = gas_client.upload_dataset(dataset)
assert dataset_client.status.branch_name == DEFAULT_BRANCH
assert dataset_client.status.draft_number
assert not dataset_client.status.commit_id
assert dataset_client.get_notes().is_continuous is True
assert not dataset_client.get_catalog()
segment1 = Segment("Segment1", client=dataset_client)
assert len(segment1) == 10
assert segment1[0].path == "hello0.txt"
assert not segment1[0].label
gas_client.delete_dataset(dataset_name)
def test_upload_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
local_path = path / "hello0.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
# If not uploading file, uploading label is not allowed
with pytest.raises(ResourceNotExistError):
segment_client.upload_label(data)
# Uploading files
segment_client.upload_file(data.path, data.target_remote_path)
data.label = Label.loads(LABEL)
segment_client.upload_label(data)
data = segment_client.list_data()
assert data[0].path == "hello0.txt"
assert data[0].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_upload_dataset_after_commit(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name)
dataset = Dataset(name=dataset_name)
dataset._catalog = Catalog.loads(CATALOG)
dataset.notes.is_continuous = True
segment = dataset.create_segment("Segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment.append(data)
dataset_client = gas_client.upload_dataset(dataset)
dataset_client.commit("test")
dataset_remote = Dataset(name=dataset_name, gas=gas_client)
assert dataset_remote.notes.is_continuous == dataset.notes.is_continuous
assert dataset_remote.catalog == dataset.catalog
segment_remote = dataset_remote[0]
assert len(segment_remote) == len(segment)
for remote_data, data in zip(segment_remote, segment):
assert remote_data.path == data.target_remote_path
assert remote_data.label == data.label
gas_client.delete_dataset(dataset_name)
def test_upload_fusion_segment_with_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name, is_fusion=True)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment = FusionSegment("segment1")
segment.sensors.add(Sensor.loads(LIDAR_DATA))
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
frame = Frame()
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
frame[LIDAR_DATA["name"]] = data
segment.append(frame)
dataset_client.upload_segment(segment)
segment1 = FusionSegment(name="segment1", client=dataset_client)
assert len(segment1) == 10
assert segment1[0][LIDAR_DATA["name"]].path == "hello0.txt"
assert (
segment1[0][LIDAR_DATA["name"]].path
== segment[0][LIDAR_DATA["name"]].target_remote_path
)
assert segment1[0][LIDAR_DATA["name"]].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_add_file(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(5):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
segment_client.upload_file(local_path=str(local_path))
# Add files
for i in range(5):
local_path = path / f"goodbye{i}.txt"
local_path.write_text("CONTENT")
segment_client.upload_file(local_path=str(local_path))
data = segment_client.list_data()
assert data[0].path == "goodbye0.txt"
assert data[5].path == "hello0.txt"
gas_client.delete_dataset(dataset_name)
def test_upload_frame_with_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name, is_fusion=True)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment_client = dataset_client.get_or_create_segment("segment1")
segment_client.upload_sensor(Sensor.loads(LIDAR_DATA))
path = tmp_path / "sub"
path.mkdir()
for i in range(5):
frame = Frame()
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
frame[LIDAR_DATA["name"]] = data
segment_client.upload_frame(frame, timestamp=i)
frames = segment_client.list_frames()
assert len(frames) == 5
assert frames[0][LIDAR_DATA["name"]].path == "hello0.txt"
assert frames[0][LIDAR_DATA["name"]].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_upload_dataset_with_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name)
dataset = Dataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
# When uploading label, upload catalog first.
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment.append(data)
dataset_client = gas_client.upload_dataset(dataset)
assert dataset_client.get_catalog()
segment1 = Segment("Segment1", client=dataset_client)
assert len(segment1) == 10
assert segment1[0].path == "hello0.txt"
assert segment1[0].label
gas_client.delete_dataset(dataset_name)
def test_upload_segment_with_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment = Segment("segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment.append(data)
dataset_client.upload_segment(segment)
segment1 = Segment(name="segment1", client=dataset_client)
assert len(segment1) == 10
assert segment1[0].path == "hello0.txt"
assert segment1[0].path == segment[0].target_remote_path
assert segment1[0].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_upload_dataset_with_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name)
dataset = Dataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
# When uploading label, upload catalog first.
dataset._catalog = Catalog.loads(CATALOG_CONTENTS)
path = tmp_path / "sub"
path.mkdir()
local_path = path / "hello.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment.append(data)
dataset_client = gas_client.upload_dataset(dataset)
dataset_client.commit("upload dataset with label")
dataset = Dataset(dataset_name, gas_client)
assert dataset.catalog == Catalog.loads(CATALOG_CONTENTS)
assert dataset[0][0].label == Label.loads(LABEL)
statistics1 = dataset_client.get_label_statistics()
assert statistics1 == Statistics(STATISTICS)
total_size = dataset_client.get_total_size()
assert total_size == TOTALSIZE
gas_client.delete_dataset(dataset_name)
def test_checkout(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.commit("commit-1", tag="V1")
commit_1_id = dataset_client.status.commit_id
commit_1_tag = "V1"
dataset_client.create_draft("draft-2")
dataset_client.commit("commit-2")
commit_2_id = dataset_client.status.commit_id
dev_branch = "dev"
dataset_client.create_branch(dev_branch)
dataset_client.create_draft("draft-3")
draft_3_number = dataset_client.status.draft_number
# Neither revision nor draft number is given
with pytest.raises(TypeError):
dataset_client.checkout()
# Both revision and draft number are given
with pytest.raises(TypeError):
dataset_client.checkout(revision=commit_1_id, draft_number=3)
# Checkout to a commit(branch name will be None)
dataset_client.checkout(revision=commit_1_id)
assert dataset_client.status.branch_name is None
assert dataset_client.status.commit_id == commit_1_id
# Checkout to a tag(branch name will be None)
dataset_client.checkout(revision=commit_1_tag)
assert dataset_client.status.branch_name is None
assert dataset_client.status.commit_id == commit_1_id
# Checkout to a branch: DEFAULT_BRANCH(branch name will be DEFAULT_BRANCH)
dataset_client.checkout(revision=DEFAULT_BRANCH)
assert dataset_client.status.branch_name == DEFAULT_BRANCH
assert dataset_client.status.commit_id == commit_2_id
# Checkout to anther branch
dataset_client.checkout(revision=dev_branch)
assert dataset_client.status.branch_name == dev_branch
assert dataset_client.status.commit_id == commit_2_id
# Checkout to a draft(branch name will be the relevant one)
dataset_client.checkout(draft_number=draft_3_number)
assert dataset_client.status.branch_name == dev_branch
assert dataset_client.status.draft_number == draft_3_number
# The revision does not exist.
with pytest.raises(ResourceNotExistError):
dataset_client.checkout(revision="123")
# The draft does not exist.
with pytest.raises(ResourceNotExistError):
dataset_client.checkout(draft_number=4)
gas_client.delete_dataset(dataset_name)
def test_list_dataset_names(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name)
datasets = gas_client.list_dataset_names()
assert dataset_name in datasets
gas_client.delete_dataset(dataset_name)
def test_create_dataset(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
assert dataset_client.dataset_id is not None
gas_client.get_dataset(dataset_name)
gas_client.delete_dataset(dataset_name)
def test_delete_dataset(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name_1 = get_dataset_name()
gas_client.create_dataset(dataset_name_1)
# Delete new dataset
gas_client.delete_dataset(dataset_name_1)
with pytest.raises(ResourceNotExistError):
gas_client.get_dataset(dataset_name_1)
# Delete dataset including commit
dataset_with_commit_name = get_dataset_name()
dataset_client_2 = gas_client.create_dataset(dataset_with_commit_name)
dataset_client_2.create_draft("v_test")
dataset_client_2.commit("Test", tag="V1")
gas_client.delete_dataset(dataset_with_commit_name)
with pytest.raises(ResourceNotExistError):
gas_client.get_dataset(dataset_with_commit_name)
def test_upload_frame_with_order(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name, is_fusion=True)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
segment_client.upload_sensor(Sensor.loads(LIDAR_DATA))
path = tmp_path / "sub"
path.mkdir()
# If noe setting frame id in frame, set timestamp(order) when uploading
for i in reversed(range(5)):
frame = Frame()
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
segment_client.upload_frame(frame, timestamp=i)
# Set frame id in frame
for i in range(5, 10):
frame = Frame(frame_id=ulid.from_timestamp(i))
local_path = path / f"goodbye{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
segment_client.upload_frame(frame)
# Both setting frame id in frame and set timestamp(order) when uploading are not allowed
i = 10
frame = Frame(frame_id=ulid.from_timestamp(i))
local_path = path / f"goodbye{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
with pytest.raises(FrameError):
segment_client.upload_frame(frame, timestamp=i)
# Neither setting frame id in frame nor set timestamp(order) when uploading is not allowed
frame = Frame()
local_path = path / f"goodbye{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
with pytest.raises(FrameError):
segment_client.upload_frame(frame)
frames = segment_client.list_frames()
assert len(frames) == 10
assert frames[0][LIDAR_DATA["name"]].path == "hello0.txt"
assert frames[5][LIDAR_DATA["name"]].path == "goodbye5.txt"
assert not frames[0][LIDAR_DATA["name"]].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
