def test_upload_dataset_with_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
gas_client.create_dataset(dataset_name)
dataset = Dataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
# When uploading label, upload catalog first.
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment.append(data)
dataset_client = gas_client.upload_dataset(dataset)
assert dataset_client.get_catalog()
segment1 = Segment("Segment1", client=dataset_client)
assert len(segment1) == 10
assert segment1[0].path == "hello0.txt"
assert segment1[0].label
gas_client.delete_dataset(dataset_name)
def test_upload_dataset_to_given_draft(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client_1 = gas_client.create_dataset(dataset_name)
draft_number = dataset_client_1.create_draft("test")
dataset = Dataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
segment.append(Data(local_path=str(local_path)))
dataset_client_2 = gas_client.upload_dataset(dataset,
draft_number=draft_number)
segment1 = Segment("Segment1", client=dataset_client_2)
assert len(segment1) == 10
assert segment1[0].path == "hello0.txt"
assert not segment1[0].label
with pytest.raises(GASResponseError):
gas_client.upload_dataset(dataset, draft_number=draft_number + 1)
gas_client.delete_dataset(dataset_name)
def init_dataset_client(accesskey, url, tmp_path_factory):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name, is_fusion=True)
dataset = FusionDataset(name=dataset_name)
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path_factory.mktemp("sub")
os.makedirs(path, exist_ok=True)
for segment_name in SEGMENTS_NAME:
segment = dataset.create_segment(segment_name)
frame = Frame()
for camera_name, label in LABEL.items():
camera = Camera(camera_name)
translation = Vector3D(1, 2, 3)
camera.set_extrinsics(translation=translation)
camera.set_extrinsics(translation=translation)
camera.set_camera_matrix(fx=1.1, fy=1.1, cx=1.1, cy=1.1)
camera.set_distortion_coefficients(p1=1.2, p2=1.2, k1=1.2, k2=1.2)
segment.sensors.add(camera)
local_path = path / f"{segment_name}_{camera_name}.txt"
local_path.write_text(f"CONTENT_{segment_name}_{camera_name}")
data = Data(local_path=str(local_path))
data.label = Label.loads(label)
frame[camera_name] = data
segment.append(frame)
dataset_client = gas_client.upload_dataset(dataset)
dataset_client.commit("commit-1")
yield dataset_client
gas_client.delete_dataset(dataset_name)
def get_gas(self, access_key: Optional[str] = None, url: Optional[str] = None) -> GAS:
"""Load an object of :class:`~tensorbay.client.gas.GAS`.
Read accessKey and URL from the appointed profile_name and login gas.
Arguments:
access_key: The accessKey of gas.
url: The login URL.
Returns:
Gas client logged in with accessKey and URL.
"""
self._set_request_config()
if access_key is None:
access_key, url = self.access_key, self.url
if not access_key and not url:
access_key, url = self.read_profile()
if not access_key:
error(
"AccessKey should be appointed. Please visit and login to the "
"TensorBay website to generate your AccessKey"
)
return GAS(access_key, url) # type:ignore[arg-type]
def test_create_draft(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
draft_1_number = dataset_client.create_draft(
"draft-1", "description", branch_name=DEFAULT_BRANCH)
assert draft_1_number == 1
dataset_client.get_draft(draft_1_number)
gas_client.delete_dataset(dataset_name)
def test_list_dataset_names(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
gas_client.create_dataset(dataset_name)
datasets = list(gas_client.list_dataset_names())
assert dataset_name in datasets
dataset_name_1 = get_random_dataset_name()
assert dataset_name_1 not in datasets
gas_client.delete_dataset(dataset_name)
def test_create_dataset(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
assert dataset_client.status.commit_id is None
assert dataset_client.status.draft_number is None
assert not dataset_client.status.is_draft
assert dataset_client.name == dataset_name
assert dataset_client.dataset_id is not None
gas_client.get_dataset(dataset_name)
gas_client.delete_dataset(dataset_name)
def test_rename_dataset(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("v_test")
dataset_client.commit(message="Test", tag="V1")
new_dataset_name = get_random_dataset_name()
gas_client.rename_dataset(name=dataset_name, new_name=new_dataset_name)
with pytest.raises(GASDatasetError):
gas_client.get_dataset(dataset_name)
gas_client.get_dataset(new_dataset_name)
gas_client.delete_dataset(new_dataset_name)
def test_update_draft(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.update_draft(1,
title="draft-updated",
description="description-updated")
draft = dataset_client.get_draft(1)
assert draft.title == "draft-updated"
assert draft.description == "description-updated"
gas_client.delete_dataset(dataset_name)
def test_create_dataset_with_region(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
regions = ("beijing", "hangzhou", "shanghai")
for region in regions:
dataset_name = get_random_dataset_name()
gas_client.create_dataset(dataset_name, region=region)
gas_client.get_dataset(dataset_name)
gas_client.delete_dataset(dataset_name)
region = "guangzhou"
dataset_name = get_random_dataset_name()
with pytest.raises(GASResponseError):
gas_client.create_dataset(dataset_name, region=region)
def test_delete_dataset(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name_1 = get_random_dataset_name()
gas_client.create_dataset(dataset_name_1)
gas_client.delete_dataset(dataset_name_1)
with pytest.raises(GASDatasetError):
gas_client.get_dataset(dataset_name_1)
dataset_name_2 = get_random_dataset_name()
dataset_client_2 = gas_client.create_dataset(dataset_name_2)
dataset_client_2.create_draft("v_test")
dataset_client_2.commit(message="Test", tag="V1")
gas_client.delete_dataset(dataset_name_2)
with pytest.raises(GASDatasetError):
gas_client.get_dataset(dataset_name_2)
def test_create_dataset(self, accesskey, url, dataset_client, job):
job.update()
job.create_dataset("search_dataset")
gas_client = GAS(access_key=accesskey, url=url)
search_dataset = gas_client.get_dataset("search_dataset", True)
search_dataset_segment = search_dataset.get_or_create_segment("test")
assert (search_dataset_segment.get_sensors() ==
dataset_client.get_segment("test").get_sensors())
search_dataset_frames = search_dataset_segment.list_frames()
assert len(search_dataset_frames) == 1
search_dataset_data = search_dataset_frames[0]["CAM_BACK_RIGHT"]
assert search_dataset_data.path == "test_CAM_BACK_RIGHT.txt"
assert search_dataset_data.label == Label.loads(LABEL_1)
def test_create_draft(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
draft_number_1 = dataset_client.create_draft("draft-1")
assert draft_number_1 == 1
assert dataset_client.status.is_draft
assert dataset_client.status.draft_number == draft_number_1
assert dataset_client.status.commit_id is None
with pytest.raises(StatusError):
dataset_client.create_draft("draft-2")
draft_number = get_draft_number_by_title(dataset_client.list_drafts(), "draft-1")
assert draft_number_1 == draft_number
gas_client.delete_dataset(dataset_name)
def test_list_drafts(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1", "description-1")
dataset_client.commit("commit-1")
dataset_client.create_draft("draft-2", "description-2")
dataset_client.checkout(revision=DEFAULT_BRANCH)
dataset_client.close_draft(2)
dataset_client.create_draft("draft-3", "description-3")
dataset_client.checkout(revision=DEFAULT_BRANCH)
dataset_client.create_branch("dev")
dataset_client.create_draft("draft-4",
"description-4",
branch_name="dev")
# After committing, the draft will be deleted
drafts_get_open = dataset_client.list_drafts(
branch_name=DEFAULT_BRANCH)
assert len(drafts_get_open) == 1
assert drafts_get_open[0].number == 3
drafts_get_closed = dataset_client.list_drafts(
status="CLOSED", branch_name=DEFAULT_BRANCH)
assert len(drafts_get_closed) == 1
assert drafts_get_closed[0].number == 2
drafts_get_commited = dataset_client.list_drafts(
status="COMMITTED", branch_name=DEFAULT_BRANCH)
assert len(drafts_get_commited) == 1
assert drafts_get_commited[0].number == 1
drafts_get_all = dataset_client.list_drafts(status="ALL",
branch_name=DEFAULT_BRANCH)
assert len(drafts_get_all) == 3
drafts_get_dev_open = dataset_client.list_drafts(branch_name="dev")
assert len(drafts_get_dev_open) == 1
assert drafts_get_dev_open[0].number == 4
drafts_get = dataset_client.list_drafts()
assert len(drafts_get) == 2
assert drafts_get[0].number == 3
assert drafts_get[1].number == 4
gas_client.delete_dataset(dataset_name)
def test_close_draft(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
# Closing the current draft is not allowed
with pytest.raises(StatusError):
dataset_client.close_draft(1)
dataset_client.checkout(revision=DEFAULT_BRANCH)
dataset_client.close_draft(1)
with pytest.raises(ResourceNotExistError):
dataset_client.get_draft(1)
gas_client.delete_dataset(dataset_name)
def test_get_dataset_to_latest_commit(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("v_test_1")
dataset_client.commit(message="Test", tag="V1")
dataset_client.create_draft("v_test_2")
dataset_client.commit(message="Test", tag="V2")
v2_commit_id = dataset_client.status.commit_id
dataset_client = gas_client.get_dataset(dataset_name)
assert dataset_client.status.commit_id == v2_commit_id
assert dataset_client.status.draft_number is None
assert dataset_client.name == dataset_name
assert dataset_client.dataset_id is not None
gas_client.delete_dataset(dataset_name)
def test_get_draft(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
draft_1_number = dataset_client.create_draft("draft-1",
"description-1")
draft_get = dataset_client.get_draft(draft_1_number)
assert draft_get.number == 1
assert draft_get.title == "draft-1"
assert draft_get.branch_name == DEFAULT_BRANCH
assert draft_get.description == "description-1"
assert draft_get.status == "OPEN"
with pytest.raises(ResourceNotExistError):
dataset_client.get_draft(2)
gas_client.delete_dataset(dataset_name)
def test_create_draft(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
draft_1_number = dataset_client.create_draft(
"draft-1", "description", branch_name=DEFAULT_BRANCH)
assert draft_1_number == 1
# Creating more than 1 draft on one branch and one accesskey is not allowed
with pytest.raises(InvalidParamsError):
dataset_client.create_draft("draft-2",
"description",
branch_name=DEFAULT_BRANCH)
dataset_client.get_draft(draft_1_number)
gas_client.delete_dataset(dataset_name)
def test_list_drafts(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.commit("commit-draft-1")
draft_number_2 = dataset_client.create_draft("draft-2")
# After committing, the draft will be deleted
with pytest.raises(TypeError):
get_draft_number_by_title(dataset_client.list_drafts(), "draft-1")
drafts = dataset_client.list_drafts()
assert len(drafts) == 1
assert drafts[0] == Draft(draft_number_2, "draft-2", DEFAULT_BRANCH)
with pytest.raises(TypeError):
get_draft_number_by_title(dataset_client.list_drafts(), "draft-3")
gas_client.delete_dataset(dataset_name)
def init_dataset_client(accesskey, url, tmp_path_factory):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.commit("commit-1")
dataset_client.create_branch("dev")
dataset = Dataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path_factory.mktemp("sub")
os.makedirs(path, exist_ok=True)
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text(f"CONTENT_{i}")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL_2)
segment.append(data)
dataset_client = gas_client.upload_dataset(dataset, branch_name="dev")
dataset_client.commit("commit-2")
dataset_client.checkout(DEFAULT_BRANCH)
dataset = Dataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path_factory.mktemp("sub")
os.makedirs(path, exist_ok=True)
for i in range(4):
local_path = path / f"hello{i}.txt"
local_path.write_text(f"CONTENT_{i}")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL_1)
segment.append(data)
dataset_client = gas_client.upload_dataset(dataset,
branch_name=DEFAULT_BRANCH)
dataset_client.commit("commit-3")
yield dataset_client
gas_client.delete_dataset(dataset_name)
def test_commit_draft(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.commit("commit-1")
dataset_client.create_draft("draft-2")
dataset_client.commit("commit-2", tag="V1")
dataset_client.create_draft("draft-3")
with pytest.raises(ResponseError):
dataset_client.commit("commit-3", tag="V1")
dataset_client.commit("commit-3", tag="V2")
assert not dataset_client.status.is_draft
assert dataset_client.status.draft_number is None
assert dataset_client.status.commit_id is not None
# After committing, the draft will be deleted
with pytest.raises(TypeError):
get_draft_number_by_title(dataset_client.list_drafts(), "draft-3")
gas_client.delete_dataset(dataset_name)
def test_commit_draft(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.commit("commit-1")
dataset_client.create_draft("draft-2")
dataset_client.commit("commit-2", tag="V2")
# Committing the draft with the duplicated tag is not allowed
dataset_client.create_draft("draft-3")
with pytest.raises(ResponseError):
dataset_client.commit("commit-3", tag="V2")
dataset_client.commit("commit-3", tag="V3")
# After committing, the draft will be deleted
with pytest.raises(ResourceNotExistError):
dataset_client.get_draft(3)
gas_client.delete_dataset(dataset_name)
def test_upload_dataset_only_with_file(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
gas_client.create_dataset(dataset_name)
dataset = Dataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
segment.append(Data(local_path=str(local_path)))
dataset_client = gas_client.upload_dataset(dataset)
assert not dataset_client.get_catalog()
segment1 = Segment("Segment1", client=dataset_client)
assert len(segment1) == 10
assert segment1[0].path == "hello0.txt"
assert not segment1[0].label
gas_client.delete_dataset(dataset_name)
def test_create_draft_on_other_branch(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.commit("commit-1")
dataset_client.create_branch("dev")
# Creating the draft on branch "dev"
draft_2_number = dataset_client.create_draft("draft-2",
"description-2",
branch_name="dev")
assert draft_2_number == 2
draft_get = dataset_client.get_draft(draft_2_number)
assert draft_get.branch_name == "dev"
# Creating the draft on DEFAULT BRANCH
draft_3_number = dataset_client.create_draft(
"draft-3", "description-3", branch_name=DEFAULT_BRANCH)
assert draft_3_number == 3
draft_get = dataset_client.get_draft(draft_3_number)
assert draft_get.branch_name == DEFAULT_BRANCH
gas_client.delete_dataset(dataset_name)