def test_create_dataset(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
assert dataset_client.dataset_id is not None
gas_client.get_dataset(dataset_name)
gas_client.delete_dataset(dataset_name)
def test_create_dataset_with_config(self, accesskey, url, config_name):
gas_client = GAS(access_key=accesskey, url=url)
try:
gas_client.get_cloud_client(config_name)
except ResourceNotExistError:
pytest.skip(f"skip this case because there's no {config_name} config")
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name, config_name=config_name)
gas_client.get_dataset(dataset_name)
gas_client.delete_dataset(dataset_name)
def test_rename_dataset(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("v_test")
dataset_client.commit(message="Test", tag="V1")
new_dataset_name = f"{get_dataset_name()}new"
gas_client.rename_dataset(name=dataset_name, new_name=new_dataset_name)
with pytest.raises(ResourceNotExistError):
gas_client.get_dataset(dataset_name)
gas_client.get_dataset(new_dataset_name)
gas_client.delete_dataset(new_dataset_name)
def test_create_dataset(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
assert dataset_client.status.commit_id == ROOT_COMMIT_ID
assert dataset_client.status.draft_number is None
assert not dataset_client.status.is_draft
assert dataset_client.status.branch_name == DEFAULT_BRANCH
assert dataset_client.name == dataset_name
assert dataset_client.dataset_id is not None
gas_client.get_dataset(dataset_name)
gas_client.delete_dataset(dataset_name)
def test_create_dataset_with_region(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
regions = ("beijing", "hangzhou", "shanghai")
for region in regions:
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name, region=region)
gas_client.get_dataset(dataset_name)
gas_client.delete_dataset(dataset_name)
region = "guangzhou"
dataset_name = get_dataset_name()
with pytest.raises(ResponseError):
gas_client.create_dataset(dataset_name, region=region)
def test_rename_dataset(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
new_dataset_name = f"{get_dataset_name()}new"
gas_client.rename_dataset(name=dataset_name, new_name=new_dataset_name)
with pytest.raises(ResourceNotExistError):
gas_client.get_dataset(dataset_name)
dataset_client_get = gas_client.get_dataset(new_dataset_name)
assert dataset_client_get.status.commit_id == dataset_client.status.commit_id
assert dataset_client_get.status.branch_name == dataset_client.status.branch_name
assert dataset_client_get.dataset_id == dataset_client.dataset_id
gas_client.delete_dataset(new_dataset_name)
def test_delete_dataset(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name_1 = get_dataset_name()
gas_client.create_dataset(dataset_name_1)
gas_client.delete_dataset(dataset_name_1)
with pytest.raises(ResourceNotExistError):
gas_client.get_dataset(dataset_name_1)
dataset_name_2 = get_dataset_name()
dataset_client_2 = gas_client.create_dataset(dataset_name_2)
dataset_client_2.create_draft("v_test")
dataset_client_2.commit(message="Test", tag="V1")
gas_client.delete_dataset(dataset_name_2)
with pytest.raises(ResourceNotExistError):
gas_client.get_dataset(dataset_name_2)
def test_delete_dataset(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name_1 = get_dataset_name()
gas_client.create_dataset(dataset_name_1)
# Delete new dataset
gas_client.delete_dataset(dataset_name_1)
with pytest.raises(ResourceNotExistError):
gas_client.get_dataset(dataset_name_1)
# Delete dataset including commit
dataset_with_commit_name = get_dataset_name()
dataset_client_2 = gas_client.create_dataset(dataset_with_commit_name)
dataset_client_2.create_draft("v_test")
dataset_client_2.commit("Test", tag="V1")
gas_client.delete_dataset(dataset_with_commit_name)
with pytest.raises(ResourceNotExistError):
gas_client.get_dataset(dataset_with_commit_name)
def test_create_public_dataset(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name, is_public=True)
assert dataset_client.dataset_id is not None
dataset_client_get = gas_client.get_dataset(dataset_name)
assert dataset_client_get.is_public == True
gas_client.delete_dataset(dataset_name)
def test_create_dataset_with_alias(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
alias = f"{dataset_name}_alias"
dataset_client = gas_client.create_dataset(dataset_name, alias=alias)
assert dataset_client.dataset_id is not None
dataset_client_get = gas_client.get_dataset(dataset_name)
assert dataset_client_get.alias == alias
gas_client.delete_dataset(dataset_name)
def test_update_dataset(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name)
new_dataset_alias = f"{get_dataset_name()}alias"
gas_client.update_dataset(name=dataset_name, alias=new_dataset_alias, is_public=True)
dataset_client_get = gas_client.get_dataset(dataset_name)
assert dataset_client_get.alias == new_dataset_alias
assert dataset_client_get.is_public is True
gas_client.delete_dataset(dataset_name)
def test_get_new_dataset(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client_get = gas_client.get_dataset(dataset_name)
assert dataset_client_get.status.commit_id == ROOT_COMMIT_ID
assert dataset_client_get.status.branch_name == DEFAULT_BRANCH
assert dataset_client_get.dataset_id == dataset_client.dataset_id
assert dataset_client_get.is_public == False
gas_client.delete_dataset(dataset_name)
def test_get_dataset_to_latest_commit(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("v_test_1")
dataset_client.commit("v_test_1", tag="V1")
dataset_client.create_draft("v_test_2")
dataset_client.commit("v_test_2", tag="V2")
v2_commit_id = dataset_client.status.commit_id
dataset_client_get = gas_client.get_dataset(dataset_name)
assert dataset_client_get.status.commit_id == v2_commit_id
assert dataset_client_get.status.branch_name == DEFAULT_BRANCH
assert dataset_client_get.is_public == False
gas_client.delete_dataset(dataset_name)
def test_copy_data_from_commits(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name)
dataset = Dataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment.append(data)
dataset_client = gas_client.upload_dataset(dataset)
dataset_client.commit("commit_1")
for i in range(10, 20):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment.append(data)
dataset_client = gas_client.upload_dataset(dataset)
dataset_client.commit("commit_2")
dataset_client_1 = gas_client.get_dataset(dataset_name)
commit_id = dataset_client_1.list_commits()[-1].commit_id
dataset_client_1.checkout(revision=commit_id)
dataset_client.create_draft("draft_3")
segment_client_1 = dataset_client_1.get_segment("Segment1")
segment_client_2 = dataset_client.get_segment("Segment1")
segment_client_2.copy_data("hello0.txt",
"goodbye0.txt",
source_client=segment_client_1)
segment2 = Segment("Segment1", client=dataset_client)
assert segment2[0].path == "goodbye0.txt"
assert segment2[0].path != segment[0].target_remote_path
assert segment2[0].label
assert len(segment2) == 21
gas_client.delete_dataset(dataset_name)
def test_get_dataset_to_latest_commit(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("v_test_1")
dataset_client.commit(message="Test", tag="V1")
dataset_client.create_draft("v_test_2")
dataset_client.commit(message="Test", tag="V2")
v2_commit_id = dataset_client.status.commit_id
dataset_client = gas_client.get_dataset(dataset_name)
assert dataset_client.status.commit_id == v2_commit_id
assert dataset_client.status.draft_number is None
assert dataset_client.status.branch_name == DEFAULT_BRANCH
assert dataset_client.name == dataset_name
assert dataset_client.dataset_id is not None
gas_client.delete_dataset(dataset_name)
def test_copy_fusion_segment_from_commits(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name, is_fusion=True)
dataset = FusionDataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
segment.sensors.add(Sensor.loads(LIDAR_DATA))
dataset._catalog = Catalog.loads(CATALOG)
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
frame = Frame()
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
frame[LIDAR_DATA["name"]] = data
segment.append(frame)
dataset_client = gas_client.upload_dataset(dataset)
dataset_client.commit("commit_1")
dataset_client.create_draft("draft_2")
dataset_client.commit("commit_2")
dataset_client.create_draft("draft_3")
dataset_client_1 = gas_client.get_dataset(dataset_name, is_fusion=True)
segment_client = dataset_client.copy_segment(
"Segment1", "Segment2", source_client=dataset_client_1
)
assert segment_client.name == "Segment2"
with pytest.raises(InvalidParamsError):
dataset_client.copy_segment("Segment1", "Segment3", strategy="push")
segment2 = FusionSegment("Segment2", client=dataset_client)
assert segment2[0][LIDAR_DATA["name"]].path == "hello0.txt"
assert (
segment2[0][LIDAR_DATA["name"]].path
== segment[0][LIDAR_DATA["name"]].target_remote_path
)
assert segment2[0][LIDAR_DATA["name"]].label
gas_client.delete_dataset(dataset_name)
""""""
"""List Dataset Names"""
list(gas.list_dataset_names())
""""""
""""Read Dataset Class / organize dataset"""
from tensorbay.opendataset import BSTLD
dataset = BSTLD("path/to/dataset/directory")
""""""
"""Upload Dataset"""
# dataset is the one you initialized in "Organize Dataset" section
dataset_client = gas.upload_dataset(dataset, jobs=8, skip_uploaded_files=False)
dataset_client.commit("BSTLD")
""""""
"""Read Dataset / get dataset"""
dataset_client = gas.get_dataset("BSTLD")
""""""
"""Read Dataset / list segment names"""
list(dataset_client.list_segment_names())
""""""
"""Read Dataset Class / get a segment"""
train_segment = dataset.get_segment_by_name("train")
first_segment = dataset[0]
""""""
"""Read Dataset / get segment"""
from tensorbay.dataset import Segment
train_segment = Segment("train", dataset_client)
""""""
"""Read Dataset / get data"""
data = train_segment[3]
gas.create_dataset("Neolix OD")
""""""
"""List Dataset Names"""
list(gas.list_dataset_names())
""""""
from tensorbay.opendataset import NeolixOD
dataset = NeolixOD("path/to/dataset/directory")
"""Upload Dataset"""
# dataset is the one you initialized in "Organize Dataset" section
dataset_client = gas.upload_dataset(dataset, jobs=8, skip_uploaded_files=False)
dataset_client.commit("Neolix OD")
""""""
"""Read Dataset / get dataset"""
dataset_client = gas.get_dataset("Neolix OD")
""""""
"""Read Dataset / get segment"""
from tensorbay.dataset import Segment
default_segment = Segment("", dataset_client)
""""""
"""Read Dataset / get data"""
data = default_segment[0]
""""""
"""Read Dataset / get label"""
label_box3d = data.label.box3d[0]
category = label_box3d.category
attributes = label_box3d.attributes
""""""
"""Delete Dataset"""
gas.create_dataset("BSTLD")
""""""
"""List Dataset Names"""
list(gas.list_dataset_names())
""""""
from tensorbay.opendataset import Newsgroups20
dataset = Newsgroups20("path/to/dataset/directory")
"""Upload Dataset"""
# dataset is the one you initialized in "Organize Dataset" section
dataset_client = gas.upload_dataset(dataset, jobs=8, skip_uploaded_files=False)
dataset_client.commit("20 Newsgroup")
""""""
"""Read Dataset / get dataset"""
dataset_client = gas.get_dataset("20 Newsgroup")
""""""
"""Read Dataset / list segment names"""
list(dataset_client.list_segment_names())
""""""
"""Read Dataset / get segment"""
from tensorbay.dataset import Segment
segment_20news_18828 = Segment("20news-18828", dataset_client)
""""""
"""Read Dataset / get data"""
data = segment_20news_18828[0]
""""""
"""Read Dataset / get label"""
category = data.label.classification.category
""""""
"""List Dataset Names"""
list(gas.list_dataset_names())
""""""
from tensorbay.opendataset import LeedsSportsPose
dataset = LeedsSportsPose("path/to/dataset/directory")
"""Upload Dataset"""
# dataset is the one you initialized in "Organize Dataset" section
dataset_client = gas.upload_dataset(dataset, jobs=8, skip_uploaded_files=False)
dataset_client.commit("LeedsSportsPose")
""""""
"""Read Dataset / get dataset"""
dataset_client = gas.get_dataset("LeedsSportsPose")
""""""
"""Read Dataset / get segment"""
from tensorbay.dataset import Segment
default_segment = Segment("", dataset_client)
""""""
"""Read Dataset / get data"""
data = default_segment[0]
""""""
"""Read Dataset / get label"""
label_keypoints2d = data.label.keypoints2d[0]
x = data.label.keypoints2d[0][0].x
from tensorbay import GAS
"""Upload Images to the Dataset"""
from tensorbay.dataset import Dataset
dataset = Dataset("DatasetName")
ACCESS_KEY = "Accesskey-*****"
gas = GAS(ACCESS_KEY)
""""""
"""Update dataset meta"""
gas.update_dataset("DATASET_NAME", alias="alias", is_public=True)
""""""
"""Update label / get dataset an create draft"""
dataset_client = gas.get_dataset("DATASET_NAME")
dataset_client.create_draft("draft-1")
""""""
"""Update label / update catalog"""
dataset_client.upload_catalog(dataset.catalog)
""""""
"""Update label / overwrite label"""
for segment in dataset:
segment_client = dataset_client.get_segment(segment.name)
for data in segment:
segment_client.upload_label(data)
""""""
"""Update label / commit dataset"""
gas.create_dataset("Dogs vs Cats")
""""""
"""List Dataset Names"""
list(gas.list_dataset_names())
""""""
from tensorbay.opendataset import DogsVsCats
dataset = DogsVsCats("path/to/dataset/directory")
"""Upload Dataset"""
# dataset is the one you initialized in "Organize Dataset" section
dataset_client = gas.upload_dataset(dataset, jobs=8, skip_uploaded_files=False)
dataset_client.commit("Dogs vs Cats")
""""""
"""Read Dataset / get dataset"""
dataset_client = gas.get_dataset("Dogs vs Cats")
""""""
"""Read Dataset / list segment names"""
list(dataset_client.list_segment_names())
""""""
"""Read Dataset Class / get a segment"""
train_segment = dataset.get_segment_by_name("train")
first_segment = dataset[0]
""""""
"""Read Dataset / get segment"""
from tensorbay.dataset import Segment
train_segment = Segment("train", dataset_client)
""""""
"""Read Dataset / get data"""
data = train_segment[0]
# Copyright 2021 Graviti. Licensed under MIT License.
#
# pylint: disable=wrong-import-position
# pylint: disable=pointless-string-statement
# pylint: disable=pointless-statement
# pylint: disable=invalid-name
# type: ignore[attr-defined]
# https://github.com/python/mypy/issues/5858
"""This file includes the python code of search_result.rst."""
"""Obtain a SearchResult Instance"""
from tensorbay import GAS
# Please visit `https://gas.graviti.cn/tensorbay/developer` to get the AccessKey.
gas = GAS("<YOUR_ACCESSKEY>")
dataset_client = gas.get_dataset("<DATASET_NAME>")
job = dataset_client.basic_search.create_job(
title="search example",
description="search description",
conjunction="and",
unit="file",
filters=[
(
"category",
"in",
[
"human.pedestrian.adult",
"human.pedestrian.child",
"human.pedestrian.construction_worker",
],
# Copyright 2021 Graviti. Licensed under MIT License.
#
# pylint: disable=wrong-import-position
# pylint: disable=wrong-import-order
# pylint: disable=pointless-statement
# pylint: disable=pointless-string-statement
# pylint: disable=invalid-name
# pylint: disable=invalid-sequence-index
"""This file includes the python code of move_and_copy.rst."""
"""Get Dataset Client"""
from tensorbay import GAS
ACCESS_KEY = "Accesskey-*****"
gas = GAS(ACCESS_KEY)
dataset_client = gas.get_dataset("OxfordIIITPet")
dataset_client.list_segment_names()
# test, trainval
""""""
"""Copy Segment"""
dataset_client.create_draft("draft-1")
segment_client = dataset_client.copy_segment("test", "test_1")
segment_client.name
# test_1
dataset_client.list_segment_names()
# test, test_1, trainval
dataset_client.commit("copy test segment to test_1 segment")
""""""
"""Move Segment"""
dataset_client.create_draft("draft-2")
segment_client = dataset_client.move_segment("test", "test_2")
# type: ignore[arg-type]
"""This file includes the python code of interact with sextant application."""
"""Get sextant client"""
from tensorbay.apps.sextant import Sextant
sextant = Sextant("<YOUR_ACCESSKEY>")
""""""
"""List or get benchmarks"""
# list all benchmarks.
benchmarks = sextant.list_benchmarks()
# get benchmark with given name.
benchmark = sextant.get_benchmark("test_01")
""""""
"""Create evaluation"""
from tensorbay import GAS
gas = GAS("<YOUR_ACCESSKEY>")
dataset_client = gas.get_dataset("<DATASET_NAME>")
dataset_client.checkout(revision="<branch/tag/commitId>")
evaluation = benchmark.create_evaluation(dataset_client.dataset_id,
dataset_client.status.commit_id)
""""""
"""List all evaluations"""
evaluations = benchmark.list_evaluations()
evaluation = evaluations[0]
""""""
"""Get evaluation result"""
result = evaluation.get_result()
""""""
gas.create_dataset("THCHS-30")
""""""
"""List Dataset Names"""
list(gas.list_dataset_names())
""""""
from tensorbay.opendataset import THCHS30
dataset = THCHS30("path/to/dataset/directory")
"""Upload Dataset"""
# dataset is the one you initialized in "Organize Dataset" section
dataset_client = gas.upload_dataset(dataset, jobs=8, skip_uploaded_files=False)
dataset_client.commit("THCHS-30")
""""""
"""Read Dataset / get dataset"""
dataset_client = gas.get_dataset("THCHS-30")
""""""
"""Read Dataset / list segment names"""
list(dataset_client.list_segment_names())
""""""
"""Read Dataset / get segment"""
from tensorbay.dataset import Segment
dev_segment = Segment("dev", dataset_client)
""""""
"""Read Dataset / get data"""
data = dev_segment[0]
""""""
"""Read Dataset / get label"""
labeled_sentence = data.label.sentence[0]
sentence = labeled_sentence.sentence
#
# pylint: disable=wrong-import-position
# pylint: disable=pointless-string-statement
# pylint: disable=invalid-name
"""This file includes the python code of merged_dataset.rst."""
"""Create Target Dataset"""
from tensorbay import GAS
# Please visit `https://gas.graviti.com/tensorbay/developer` to get the AccessKey.
gas = GAS("<YOUR_ACCESSKEY>")
dataset_client = gas.create_dataset("mergedDataset")
dataset_client.create_draft("merge dataset")
""""""
"""Copy Segment From Pet"""
pet_dataset_client = gas.get_dataset("OxfordIIITPet")
dataset_client.copy_segment("train",
target_name="trainval",
source_client=pet_dataset_client)
dataset_client.copy_segment("test", source_client=pet_dataset_client)
""""""
"""Upload Catalog"""
dataset_client.upload_catalog(pet_dataset_client.get_catalog())
""""""
"""Unify Category"""
from tensorbay.dataset import Data
segment_client = dataset_client.get_segment("train")
for remote_data in segment_client.list_data():
data = Data(remote_data.path)
data.label = remote_data.label
dataset = Dataset("DatasetName")
# TensorBay uses "segment" to separate different parts in a dataset.
segment = dataset.create_segment()
segment.append(Data("0000001.jpg"))
segment.append(Data("0000002.jpg"))
dataset_client = gas.upload_dataset(dataset)
# TensorBay provides dataset version control feature, commit the uploaded data before using it.
dataset_client.commit("Initial commit")
""""""
"""Read Images from the Dataset"""
from PIL import Image
from tensorbay.dataset import Segment
dataset_client = gas.get_dataset("DatasetName")
segment = Segment("", dataset_client)
for data in segment:
with data.open() as fp:
image = Image.open(fp)
width, height = image.size
image.show()
""""""
"""Delete the Dataset"""
gas.delete_dataset("DatasetName")
""""""
