def test_create_dataset(self, accesskey, url): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_dataset_name() dataset_client = gas_client.create_dataset(dataset_name) assert dataset_client.dataset_id is not None gas_client.get_dataset(dataset_name) gas_client.delete_dataset(dataset_name)
def test_create_dataset_with_config(self, accesskey, url, config_name): gas_client = GAS(access_key=accesskey, url=url) try: gas_client.get_cloud_client(config_name) except ResourceNotExistError: pytest.skip(f"skip this case because there's no {config_name} config") dataset_name = get_dataset_name() gas_client.create_dataset(dataset_name, config_name=config_name) gas_client.get_dataset(dataset_name) gas_client.delete_dataset(dataset_name)
def test_rename_dataset(self, accesskey, url): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_dataset_name() dataset_client = gas_client.create_dataset(dataset_name) dataset_client.create_draft("v_test") dataset_client.commit(message="Test", tag="V1") new_dataset_name = f"{get_dataset_name()}new" gas_client.rename_dataset(name=dataset_name, new_name=new_dataset_name) with pytest.raises(ResourceNotExistError): gas_client.get_dataset(dataset_name) gas_client.get_dataset(new_dataset_name) gas_client.delete_dataset(new_dataset_name)
def test_create_dataset(self, accesskey, url): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_dataset_name() dataset_client = gas_client.create_dataset(dataset_name) assert dataset_client.status.commit_id == ROOT_COMMIT_ID assert dataset_client.status.draft_number is None assert not dataset_client.status.is_draft assert dataset_client.status.branch_name == DEFAULT_BRANCH assert dataset_client.name == dataset_name assert dataset_client.dataset_id is not None gas_client.get_dataset(dataset_name) gas_client.delete_dataset(dataset_name)
def test_create_dataset_with_region(self, accesskey, url): gas_client = GAS(access_key=accesskey, url=url) regions = ("beijing", "hangzhou", "shanghai") for region in regions: dataset_name = get_dataset_name() gas_client.create_dataset(dataset_name, region=region) gas_client.get_dataset(dataset_name) gas_client.delete_dataset(dataset_name) region = "guangzhou" dataset_name = get_dataset_name() with pytest.raises(ResponseError): gas_client.create_dataset(dataset_name, region=region)
def test_rename_dataset(self, accesskey, url): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_dataset_name() dataset_client = gas_client.create_dataset(dataset_name) new_dataset_name = f"{get_dataset_name()}new" gas_client.rename_dataset(name=dataset_name, new_name=new_dataset_name) with pytest.raises(ResourceNotExistError): gas_client.get_dataset(dataset_name) dataset_client_get = gas_client.get_dataset(new_dataset_name) assert dataset_client_get.status.commit_id == dataset_client.status.commit_id assert dataset_client_get.status.branch_name == dataset_client.status.branch_name assert dataset_client_get.dataset_id == dataset_client.dataset_id gas_client.delete_dataset(new_dataset_name)
def test_delete_dataset(self, accesskey, url): gas_client = GAS(access_key=accesskey, url=url) dataset_name_1 = get_dataset_name() gas_client.create_dataset(dataset_name_1) gas_client.delete_dataset(dataset_name_1) with pytest.raises(ResourceNotExistError): gas_client.get_dataset(dataset_name_1) dataset_name_2 = get_dataset_name() dataset_client_2 = gas_client.create_dataset(dataset_name_2) dataset_client_2.create_draft("v_test") dataset_client_2.commit(message="Test", tag="V1") gas_client.delete_dataset(dataset_name_2) with pytest.raises(ResourceNotExistError): gas_client.get_dataset(dataset_name_2)
def test_delete_dataset(self, accesskey, url): gas_client = GAS(access_key=accesskey, url=url) dataset_name_1 = get_dataset_name() gas_client.create_dataset(dataset_name_1) # Delete new dataset gas_client.delete_dataset(dataset_name_1) with pytest.raises(ResourceNotExistError): gas_client.get_dataset(dataset_name_1) # Delete dataset including commit dataset_with_commit_name = get_dataset_name() dataset_client_2 = gas_client.create_dataset(dataset_with_commit_name) dataset_client_2.create_draft("v_test") dataset_client_2.commit("Test", tag="V1") gas_client.delete_dataset(dataset_with_commit_name) with pytest.raises(ResourceNotExistError): gas_client.get_dataset(dataset_with_commit_name)
def test_create_public_dataset(self, accesskey, url): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_dataset_name() dataset_client = gas_client.create_dataset(dataset_name, is_public=True) assert dataset_client.dataset_id is not None dataset_client_get = gas_client.get_dataset(dataset_name) assert dataset_client_get.is_public == True gas_client.delete_dataset(dataset_name)
def test_create_dataset_with_alias(self, accesskey, url): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_dataset_name() alias = f"{dataset_name}_alias" dataset_client = gas_client.create_dataset(dataset_name, alias=alias) assert dataset_client.dataset_id is not None dataset_client_get = gas_client.get_dataset(dataset_name) assert dataset_client_get.alias == alias gas_client.delete_dataset(dataset_name)
def test_update_dataset(self, accesskey, url): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_dataset_name() gas_client.create_dataset(dataset_name) new_dataset_alias = f"{get_dataset_name()}alias" gas_client.update_dataset(name=dataset_name, alias=new_dataset_alias, is_public=True) dataset_client_get = gas_client.get_dataset(dataset_name) assert dataset_client_get.alias == new_dataset_alias assert dataset_client_get.is_public is True gas_client.delete_dataset(dataset_name)
def test_get_new_dataset(self, accesskey, url): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_dataset_name() dataset_client = gas_client.create_dataset(dataset_name) dataset_client_get = gas_client.get_dataset(dataset_name) assert dataset_client_get.status.commit_id == ROOT_COMMIT_ID assert dataset_client_get.status.branch_name == DEFAULT_BRANCH assert dataset_client_get.dataset_id == dataset_client.dataset_id assert dataset_client_get.is_public == False gas_client.delete_dataset(dataset_name)
def test_get_dataset_to_latest_commit(self, accesskey, url): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_dataset_name() dataset_client = gas_client.create_dataset(dataset_name) dataset_client.create_draft("v_test_1") dataset_client.commit("v_test_1", tag="V1") dataset_client.create_draft("v_test_2") dataset_client.commit("v_test_2", tag="V2") v2_commit_id = dataset_client.status.commit_id dataset_client_get = gas_client.get_dataset(dataset_name) assert dataset_client_get.status.commit_id == v2_commit_id assert dataset_client_get.status.branch_name == DEFAULT_BRANCH assert dataset_client_get.is_public == False gas_client.delete_dataset(dataset_name)
def test_copy_data_from_commits(self, accesskey, url, tmp_path): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_dataset_name() gas_client.create_dataset(dataset_name) dataset = Dataset(name=dataset_name) segment = dataset.create_segment("Segment1") dataset._catalog = Catalog.loads(CATALOG) path = tmp_path / "sub" path.mkdir() for i in range(10): local_path = path / f"hello{i}.txt" local_path.write_text("CONTENT") data = Data(local_path=str(local_path)) data.label = Label.loads(LABEL) segment.append(data) dataset_client = gas_client.upload_dataset(dataset) dataset_client.commit("commit_1") for i in range(10, 20): local_path = path / f"hello{i}.txt" local_path.write_text("CONTENT") data = Data(local_path=str(local_path)) data.label = Label.loads(LABEL) segment.append(data) dataset_client = gas_client.upload_dataset(dataset) dataset_client.commit("commit_2") dataset_client_1 = gas_client.get_dataset(dataset_name) commit_id = dataset_client_1.list_commits()[-1].commit_id dataset_client_1.checkout(revision=commit_id) dataset_client.create_draft("draft_3") segment_client_1 = dataset_client_1.get_segment("Segment1") segment_client_2 = dataset_client.get_segment("Segment1") segment_client_2.copy_data("hello0.txt", "goodbye0.txt", source_client=segment_client_1) segment2 = Segment("Segment1", client=dataset_client) assert segment2[0].path == "goodbye0.txt" assert segment2[0].path != segment[0].target_remote_path assert segment2[0].label assert len(segment2) == 21 gas_client.delete_dataset(dataset_name)
def test_get_dataset_to_latest_commit(self, accesskey, url): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_dataset_name() dataset_client = gas_client.create_dataset(dataset_name) dataset_client.create_draft("v_test_1") dataset_client.commit(message="Test", tag="V1") dataset_client.create_draft("v_test_2") dataset_client.commit(message="Test", tag="V2") v2_commit_id = dataset_client.status.commit_id dataset_client = gas_client.get_dataset(dataset_name) assert dataset_client.status.commit_id == v2_commit_id assert dataset_client.status.draft_number is None assert dataset_client.status.branch_name == DEFAULT_BRANCH assert dataset_client.name == dataset_name assert dataset_client.dataset_id is not None gas_client.delete_dataset(dataset_name)
def test_copy_fusion_segment_from_commits(self, accesskey, url, tmp_path): gas_client = GAS(access_key=accesskey, url=url) dataset_name = get_dataset_name() gas_client.create_dataset(dataset_name, is_fusion=True) dataset = FusionDataset(name=dataset_name) segment = dataset.create_segment("Segment1") segment.sensors.add(Sensor.loads(LIDAR_DATA)) dataset._catalog = Catalog.loads(CATALOG) path = tmp_path / "sub" path.mkdir() for i in range(10): frame = Frame() local_path = path / f"hello{i}.txt" local_path.write_text("CONTENT") data = Data(local_path=str(local_path)) data.label = Label.loads(LABEL) frame[LIDAR_DATA["name"]] = data segment.append(frame) dataset_client = gas_client.upload_dataset(dataset) dataset_client.commit("commit_1") dataset_client.create_draft("draft_2") dataset_client.commit("commit_2") dataset_client.create_draft("draft_3") dataset_client_1 = gas_client.get_dataset(dataset_name, is_fusion=True) segment_client = dataset_client.copy_segment( "Segment1", "Segment2", source_client=dataset_client_1 ) assert segment_client.name == "Segment2" with pytest.raises(InvalidParamsError): dataset_client.copy_segment("Segment1", "Segment3", strategy="push") segment2 = FusionSegment("Segment2", client=dataset_client) assert segment2[0][LIDAR_DATA["name"]].path == "hello0.txt" assert ( segment2[0][LIDAR_DATA["name"]].path == segment[0][LIDAR_DATA["name"]].target_remote_path ) assert segment2[0][LIDAR_DATA["name"]].label gas_client.delete_dataset(dataset_name)
"""""" """List Dataset Names""" list(gas.list_dataset_names()) """""" """"Read Dataset Class / organize dataset""" from tensorbay.opendataset import BSTLD dataset = BSTLD("path/to/dataset/directory") """""" """Upload Dataset""" # dataset is the one you initialized in "Organize Dataset" section dataset_client = gas.upload_dataset(dataset, jobs=8, skip_uploaded_files=False) dataset_client.commit("BSTLD") """""" """Read Dataset / get dataset""" dataset_client = gas.get_dataset("BSTLD") """""" """Read Dataset / list segment names""" list(dataset_client.list_segment_names()) """""" """Read Dataset Class / get a segment""" train_segment = dataset.get_segment_by_name("train") first_segment = dataset[0] """""" """Read Dataset / get segment""" from tensorbay.dataset import Segment train_segment = Segment("train", dataset_client) """""" """Read Dataset / get data""" data = train_segment[3]
gas.create_dataset("Neolix OD") """""" """List Dataset Names""" list(gas.list_dataset_names()) """""" from tensorbay.opendataset import NeolixOD dataset = NeolixOD("path/to/dataset/directory") """Upload Dataset""" # dataset is the one you initialized in "Organize Dataset" section dataset_client = gas.upload_dataset(dataset, jobs=8, skip_uploaded_files=False) dataset_client.commit("Neolix OD") """""" """Read Dataset / get dataset""" dataset_client = gas.get_dataset("Neolix OD") """""" """Read Dataset / get segment""" from tensorbay.dataset import Segment default_segment = Segment("", dataset_client) """""" """Read Dataset / get data""" data = default_segment[0] """""" """Read Dataset / get label""" label_box3d = data.label.box3d[0] category = label_box3d.category attributes = label_box3d.attributes """""" """Delete Dataset"""
gas.create_dataset("BSTLD") """""" """List Dataset Names""" list(gas.list_dataset_names()) """""" from tensorbay.opendataset import Newsgroups20 dataset = Newsgroups20("path/to/dataset/directory") """Upload Dataset""" # dataset is the one you initialized in "Organize Dataset" section dataset_client = gas.upload_dataset(dataset, jobs=8, skip_uploaded_files=False) dataset_client.commit("20 Newsgroup") """""" """Read Dataset / get dataset""" dataset_client = gas.get_dataset("20 Newsgroup") """""" """Read Dataset / list segment names""" list(dataset_client.list_segment_names()) """""" """Read Dataset / get segment""" from tensorbay.dataset import Segment segment_20news_18828 = Segment("20news-18828", dataset_client) """""" """Read Dataset / get data""" data = segment_20news_18828[0] """""" """Read Dataset / get label""" category = data.label.classification.category """"""
"""List Dataset Names""" list(gas.list_dataset_names()) """""" from tensorbay.opendataset import LeedsSportsPose dataset = LeedsSportsPose("path/to/dataset/directory") """Upload Dataset""" # dataset is the one you initialized in "Organize Dataset" section dataset_client = gas.upload_dataset(dataset, jobs=8, skip_uploaded_files=False) dataset_client.commit("LeedsSportsPose") """""" """Read Dataset / get dataset""" dataset_client = gas.get_dataset("LeedsSportsPose") """""" """Read Dataset / get segment""" from tensorbay.dataset import Segment default_segment = Segment("", dataset_client) """""" """Read Dataset / get data""" data = default_segment[0] """""" """Read Dataset / get label""" label_keypoints2d = data.label.keypoints2d[0] x = data.label.keypoints2d[0][0].x
from tensorbay import GAS """Upload Images to the Dataset""" from tensorbay.dataset import Dataset dataset = Dataset("DatasetName") ACCESS_KEY = "Accesskey-*****" gas = GAS(ACCESS_KEY) """""" """Update dataset meta""" gas.update_dataset("DATASET_NAME", alias="alias", is_public=True) """""" """Update label / get dataset an create draft""" dataset_client = gas.get_dataset("DATASET_NAME") dataset_client.create_draft("draft-1") """""" """Update label / update catalog""" dataset_client.upload_catalog(dataset.catalog) """""" """Update label / overwrite label""" for segment in dataset: segment_client = dataset_client.get_segment(segment.name) for data in segment: segment_client.upload_label(data) """""" """Update label / commit dataset"""
gas.create_dataset("Dogs vs Cats") """""" """List Dataset Names""" list(gas.list_dataset_names()) """""" from tensorbay.opendataset import DogsVsCats dataset = DogsVsCats("path/to/dataset/directory") """Upload Dataset""" # dataset is the one you initialized in "Organize Dataset" section dataset_client = gas.upload_dataset(dataset, jobs=8, skip_uploaded_files=False) dataset_client.commit("Dogs vs Cats") """""" """Read Dataset / get dataset""" dataset_client = gas.get_dataset("Dogs vs Cats") """""" """Read Dataset / list segment names""" list(dataset_client.list_segment_names()) """""" """Read Dataset Class / get a segment""" train_segment = dataset.get_segment_by_name("train") first_segment = dataset[0] """""" """Read Dataset / get segment""" from tensorbay.dataset import Segment train_segment = Segment("train", dataset_client) """""" """Read Dataset / get data""" data = train_segment[0]
# Copyright 2021 Graviti. Licensed under MIT License. # # pylint: disable=wrong-import-position # pylint: disable=pointless-string-statement # pylint: disable=pointless-statement # pylint: disable=invalid-name # type: ignore[attr-defined] # https://github.com/python/mypy/issues/5858 """This file includes the python code of search_result.rst.""" """Obtain a SearchResult Instance""" from tensorbay import GAS # Please visit `https://gas.graviti.cn/tensorbay/developer` to get the AccessKey. gas = GAS("<YOUR_ACCESSKEY>") dataset_client = gas.get_dataset("<DATASET_NAME>") job = dataset_client.basic_search.create_job( title="search example", description="search description", conjunction="and", unit="file", filters=[ ( "category", "in", [ "human.pedestrian.adult", "human.pedestrian.child", "human.pedestrian.construction_worker", ],
# Copyright 2021 Graviti. Licensed under MIT License. # # pylint: disable=wrong-import-position # pylint: disable=wrong-import-order # pylint: disable=pointless-statement # pylint: disable=pointless-string-statement # pylint: disable=invalid-name # pylint: disable=invalid-sequence-index """This file includes the python code of move_and_copy.rst.""" """Get Dataset Client""" from tensorbay import GAS ACCESS_KEY = "Accesskey-*****" gas = GAS(ACCESS_KEY) dataset_client = gas.get_dataset("OxfordIIITPet") dataset_client.list_segment_names() # test, trainval """""" """Copy Segment""" dataset_client.create_draft("draft-1") segment_client = dataset_client.copy_segment("test", "test_1") segment_client.name # test_1 dataset_client.list_segment_names() # test, test_1, trainval dataset_client.commit("copy test segment to test_1 segment") """""" """Move Segment""" dataset_client.create_draft("draft-2") segment_client = dataset_client.move_segment("test", "test_2")
# type: ignore[arg-type] """This file includes the python code of interact with sextant application.""" """Get sextant client""" from tensorbay.apps.sextant import Sextant sextant = Sextant("<YOUR_ACCESSKEY>") """""" """List or get benchmarks""" # list all benchmarks. benchmarks = sextant.list_benchmarks() # get benchmark with given name. benchmark = sextant.get_benchmark("test_01") """""" """Create evaluation""" from tensorbay import GAS gas = GAS("<YOUR_ACCESSKEY>") dataset_client = gas.get_dataset("<DATASET_NAME>") dataset_client.checkout(revision="<branch/tag/commitId>") evaluation = benchmark.create_evaluation(dataset_client.dataset_id, dataset_client.status.commit_id) """""" """List all evaluations""" evaluations = benchmark.list_evaluations() evaluation = evaluations[0] """""" """Get evaluation result""" result = evaluation.get_result() """"""
gas.create_dataset("THCHS-30") """""" """List Dataset Names""" list(gas.list_dataset_names()) """""" from tensorbay.opendataset import THCHS30 dataset = THCHS30("path/to/dataset/directory") """Upload Dataset""" # dataset is the one you initialized in "Organize Dataset" section dataset_client = gas.upload_dataset(dataset, jobs=8, skip_uploaded_files=False) dataset_client.commit("THCHS-30") """""" """Read Dataset / get dataset""" dataset_client = gas.get_dataset("THCHS-30") """""" """Read Dataset / list segment names""" list(dataset_client.list_segment_names()) """""" """Read Dataset / get segment""" from tensorbay.dataset import Segment dev_segment = Segment("dev", dataset_client) """""" """Read Dataset / get data""" data = dev_segment[0] """""" """Read Dataset / get label""" labeled_sentence = data.label.sentence[0] sentence = labeled_sentence.sentence
# # pylint: disable=wrong-import-position # pylint: disable=pointless-string-statement # pylint: disable=invalid-name """This file includes the python code of merged_dataset.rst.""" """Create Target Dataset""" from tensorbay import GAS # Please visit `https://gas.graviti.com/tensorbay/developer` to get the AccessKey. gas = GAS("<YOUR_ACCESSKEY>") dataset_client = gas.create_dataset("mergedDataset") dataset_client.create_draft("merge dataset") """""" """Copy Segment From Pet""" pet_dataset_client = gas.get_dataset("OxfordIIITPet") dataset_client.copy_segment("train", target_name="trainval", source_client=pet_dataset_client) dataset_client.copy_segment("test", source_client=pet_dataset_client) """""" """Upload Catalog""" dataset_client.upload_catalog(pet_dataset_client.get_catalog()) """""" """Unify Category""" from tensorbay.dataset import Data segment_client = dataset_client.get_segment("train") for remote_data in segment_client.list_data(): data = Data(remote_data.path) data.label = remote_data.label
dataset = Dataset("DatasetName") # TensorBay uses "segment" to separate different parts in a dataset. segment = dataset.create_segment() segment.append(Data("0000001.jpg")) segment.append(Data("0000002.jpg")) dataset_client = gas.upload_dataset(dataset) # TensorBay provides dataset version control feature, commit the uploaded data before using it. dataset_client.commit("Initial commit") """""" """Read Images from the Dataset""" from PIL import Image from tensorbay.dataset import Segment dataset_client = gas.get_dataset("DatasetName") segment = Segment("", dataset_client) for data in segment: with data.open() as fp: image = Image.open(fp) width, height = image.size image.show() """""" """Delete the Dataset""" gas.delete_dataset("DatasetName") """"""