#!/usr/bin/env python3
#
# Copyright 2021 Graviti. Licensed under MIT License.
#
# pylint: disable=wrong-import-position
# pylint: disable=pointless-string-statement
# pylint: disable=invalid-name
"""This file includes the python code of tag.rst."""
"""Authorize a Dataset Client Instance"""
from tensorbay import GAS
# Please visit `https://gas.graviti.cn/tensorbay/developer` to get the AccessKey.
gas = GAS("<YOUR_ACCESSKEY>")
dataset_client = gas.create_dataset("<DATASET_NAME>")
dataset_client.create_draft("draft-1")
# Add some data to the dataset.
dataset_client.commit("commit-1", tag="V1")
commit_id_1 = dataset_client.status.commit_id
dataset_client.create_draft("draft-2")
# Do some modifications to the dataset.
dataset_client.commit("commit-2", tag="V2")
commit_id_2 = dataset_client.status.commit_id
""""""
"""Create Branch"""
dataset_client.create_branch("T123")
""""""
"""Branch Name Will Be Stored"""
branch_name = dataset_client.status.branch_name
# branch_name = "T123"
#
# pylint: disable=wrong-import-position
# pylint: disable=pointless-string-statement
# pylint: disable=invalid-name
# pylint: disable=unsubscriptable-object
# flake8: noqa: F401
"""This file includes the python code of BSTLD.rst and read_dataset_class.rst."""
"""Authorize a Client Instance"""
from tensorbay import GAS
ACCESS_KEY = "Accesskey-*****"
gas = GAS(ACCESS_KEY)
""""""
"""Create Dataset"""
gas.create_dataset("BSTLD")
""""""
"""Organize Dataset / regular import"""
from tensorbay.dataset import Dataset
""""""
"""Organize dataset / import dataloader"""
from tensorbay.opendataset import BSTLD
dataset = BSTLD("path/to/dataset/directory")
""""""
"""Upload Dataset"""
dataset_client = gas.upload_dataset(dataset, jobs=8, skip_uploaded_files=True)
dataset_client.commit("initial commit")
""""""
"""Read Dataset / get dataset"""
dataset = Dataset("BSTLD", gas)
# pylint: disable=wrong-import-position
# pylint: disable=wrong-import-order
# pylint: disable=pointless-string-statement
# pylint: disable=invalid-name
# pylint: disable=unsubscriptable-object
# flake8: noqa: F401
"""This file includes the python code of LeedsSportsPose.rst."""
"""Authorize a Client Instance"""
from tensorbay import GAS
# Please visit `https://gas.graviti.cn/tensorbay/developer` to get the AccessKey.
gas = GAS("<YOUR_ACCESSKEY>")
""""""
"""Create Dataset"""
gas.create_dataset("LeedsSportsPose")
""""""
"""Organize Dataset / regular import"""
from tensorbay.dataset import Dataset
""""""
"""Organize dataset / import dataloader"""
from tensorbay.opendataset import LeedsSportsPose
dataset = LeedsSportsPose("<path/to/dataset>")
""""""
"""Upload Dataset"""
dataset_client = gas.upload_dataset(dataset, jobs=8)
dataset_client.commit("initial commit")
""""""
"""Read Dataset / get dataset"""
dataset = Dataset("LeedsSportsPose", gas)
# pylint: disable=wrong-import-order
# pylint: disable=not-callable
# pylint: disable=ungrouped-imports
# pylint: disable=import-error
# pylint: disable=pointless-string-statement
# pylint: disable=invalid-name
"""This file includes the python code of CADC.rst."""
"""Authorize a Client Instance"""
from tensorbay import GAS
from tensorbay.dataset import FusionDataset
ACCESS_KEY = "Accesskey-*****"
gas = GAS(ACCESS_KEY)
""""""
"""Create Fusion Dataset"""
gas.create_dataset("CADC", is_fusion=True)
""""""
"""List Dataset Names"""
gas.list_dataset_names()
""""""
from tensorbay.opendataset import CADC
fusion_dataset = CADC("path/to/dataset/directory")
"""Upload Fusion Dataset"""
# fusion_dataset is the one you initialized in "Organize Fusion Dataset" section
fusion_dataset_client = gas.upload_dataset(fusion_dataset, jobs=8)
fusion_dataset_client.commit("initial commit")
""""""
"""Read Fusion Dataset / get fusion dataset"""
fusion_dataset = FusionDataset("CADC", gas)
#
# pylint: disable=wrong-import-position
# pylint: disable=wrong-import-order
# pylint: disable=not-callable
# pylint: disable=ungrouped-imports
# pylint: disable=import-error
# pylint: disable=pointless-string-statement
"""This file includes the python code of getting_started_with_tensorbay.rst."""
"""Authorize a Client Instance"""
from tensorbay import GAS
gas = GAS("<YOUR_ACCESSKEY>")
""""""
"""Create a Dataset"""
gas.create_dataset("DatasetName")
""""""
"""List Dataset Names"""
dataset_names = gas.list_dataset_names()
""""""
"""Upload Images to the Dataset"""
from tensorbay.dataset import Data, Dataset
# Organize the local dataset by the "Dataset" class before uploading.
dataset = Dataset("DatasetName")
# TensorBay uses "segment" to separate different parts in a dataset.
segment = dataset.create_segment()
segment.append(Data("0000001.jpg"))
segment.append(Data("0000002.jpg"))
# pylint: disable=pointless-string-statement
# pylint: disable=invalid-name
# pylint: disable=unsubscriptable-object
# flake8: noqa: F401
"""This file includes the python code of THCHS.rst and read_dataset_class.rst."""
"""Authorize a Client Instance"""
from tensorbay import GAS
# Please visit `https://gas.graviti.cn/tensorbay/developer` to get the AccessKey.
gas = GAS("<YOUR_ACCESSKEY>")
""""""
"""Create Dataset"""
gas.create_dataset("THCHS-30")
""""""
"""Organize Dataset / regular import"""
from tensorbay.dataset import Dataset
""""""
"""Organize dataset / import dataloader"""
from tensorbay.opendataset import THCHS30
dataset = THCHS30("<path/to/dataset>")
""""""
"""Upload Dataset"""
dataset_client = gas.upload_dataset(dataset, jobs=8)
#!/usr/bin/env python3
#
# Copyright 2021 Graviti. Licensed under MIT License.
#
# pylint: disable=wrong-import-position
# pylint: disable=pointless-string-statement
# pylint: disable=invalid-name
"""This file includes the python code of merged_dataset.rst."""
"""Create Target Dataset"""
from tensorbay import GAS
gas = GAS("<YOUR_ACCESSKEY>")
dataset_client = gas.create_dataset("mergedDataset")
dataset_client.create_draft("merge dataset")
""""""
"""Copy Segment From Pet"""
pet_dataset_client = gas.get_dataset("OxfordIIITPet")
dataset_client.copy_segment("train",
target_name="trainval",
source_client=pet_dataset_client)
dataset_client.copy_segment("test", source_client=pet_dataset_client)
""""""
"""Upload Catalog"""
dataset_client.upload_catalog(pet_dataset_client.get_catalog())
""""""
"""Unify Category"""
from tensorbay.dataset import Data
segment_client = dataset_client.get_segment("train")
for remote_data in segment_client.list_data():
# pylint: disable=not-callable
# pylint: disable=ungrouped-imports
# pylint: disable=import-error
# pylint: disable=pointless-string-statement
# pylint: disable=invalid-name
# pylint: disable=unused-import
# flake8: noqa: F401
"""This file includes the python code of NeolixOD.rst."""
"""Authorize a Client Instance"""
from tensorbay import GAS
ACCESS_KEY = "Accesskey-*****"
gas = GAS(ACCESS_KEY)
""""""
"""Create Dataset"""
gas.create_dataset("NeolixOD")
""""""
"""Organize Dataset / regular import"""
from tensorbay.dataset import Data, Dataset
from tensorbay.label import LabeledBox3D
""""""
"""Organize dataset / import dataloader"""
from tensorbay.opendataset import NeolixOD
dataset = NeolixOD("path/to/dataset/directory")
""""""
"""Upload Dataset"""
dataset_client = gas.upload_dataset(dataset, jobs=8)
dataset_client.commit("initial commit")
""""""
"""Read Dataset / get dataset"""
#
# pylint: disable=wrong-import-position
# pylint: disable=pointless-string-statement
# pylint: disable=invalid-name
# pylint: disable=unsubscriptable-object
# flake8: noqa: F401
"""This files includes the python code example in dogsvscats.rst."""
"""Authorize a Client Instance"""
from tensorbay import GAS
# Please visit `https://gas.graviti.com/tensorbay/developer` to get the AccessKey.
gas = GAS("<YOUR_ACCESSKEY>")
""""""
"""Create Dataset"""
gas.create_dataset("DogsVsCats")
""""""
"""Organize Dataset / regular import"""
from tensorbay.dataset import Dataset
""""""
"""Organize dataset / import dataloader"""
from tensorbay.opendataset import DogsVsCats
dataset = DogsVsCats("<path/to/dataset>")
""""""
"""Upload Dataset"""
dataset_client = gas.upload_dataset(dataset, jobs=8)
dataset_client.commit("initial commit")
""""""
"""Read Dataset / get dataset"""
dataset = Dataset("DogsVsCats", gas)
from tensorbay.dataset import Dataset
from tensorbay.label import Classification
# Use AuthData to organize a dataset by the "Dataset" class before importing.
dataset = Dataset("<DATASET_NAME>")
# TensorBay uses "segment" to separate different parts in a dataset.
segment = dataset.create_segment()
images = cloud_client.list_auth_data("<data/images/>")
labels = cloud_client.list_auth_data("<data/labels/>")
for auth_data, label in zip(images, labels):
with label.open() as fp:
auth_data.label.classification = Classification.loads(json.load(fp))
segment.append(auth_data)
dataset_client = gas.upload_dataset(dataset, jobs=8)
""""""
"""Create local storage config"""
gas.create_local_storage_config(
name="<LOCAL_STORAGE_CONFIG>",
file_path="<path/to/dataset>",
endpoint="<external IP address of the local storage service>",
)
""""""
"""Create authorized local storage dataset"""
dataset_client = gas.create_dataset("<DATASET_NAME>",
config_name="<LOCAL_STORAGE_CONFIG>")
""""""
#
# pylint: disable=wrong-import-position
# pylint: disable=wrong-import-order
# pylint: disable=not-callable
# pylint: disable=ungrouped-imports
# pylint: disable=import-error
# pylint: disable=pointless-string-statement
# pylint: disable=invalid-name
"""This file includes the python code of diff.rst."""
"""Authorize a Dataset Client Instance"""
from tensorbay import GAS
ACCESS_KEY = "Accesskey-*****"
gas = GAS(ACCESS_KEY)
dataset_client = gas.create_dataset("DatasetName")
dataset_client.create_draft("draft-1")
# Add some data to the dataset.
dataset_client.commit("commit-1", tag="V1")
commit_id_1 = dataset_client.status.commit_id
dataset_client.create_draft("draft-2")
# Do some modifications to the dataset.
dataset_client.commit("commit-2", tag="V2")
commit_id_2 = dataset_client.status.commit_id
dataset_client.create_draft("draft-3")
draft_number_3 = dataset_client.status.draft_number
head = ""
""""""
"""Get Diff"""
# pylint: disable=invalid-name
# pylint: disable=unsubscriptable-object
# flake8: noqa: F401
# type: ignore[attr-defined]
"""This file includes the python code of VOC2012Segmentation.rst."""
"""Authorize a Client Instance"""
from tensorbay import GAS
# Please visit `https://gas.graviti.com/tensorbay/developer` to get the AccessKey.
gas = GAS("<YOUR_ACCESSKEY>")
""""""
"""Create Dataset"""
gas.create_dataset("VOC2012Segmentation")
""""""
"""Organize Dataset / regular import"""
from tensorbay.dataset import Dataset
""""""
"""Organize dataset / import dataloader"""
from tensorbay.opendataset import VOC2012Segmentation
dataset = VOC2012Segmentation("<path/to/dataset>")
""""""
"""Upload Dataset"""
dataset_client = gas.upload_dataset(dataset, jobs=8)
# pylint: disable=pointless-string-statement
# pylint: disable=invalid-name
# pylint: disable=unused-import
# flake8: noqa: F401
"""This file includes the python code of NewsGroups.rst and read_dataset_class.rst."""
"""Authorize a Client Instance"""
from tensorbay import GAS
ACCESS_KEY = "Accesskey-*****"
gas = GAS(ACCESS_KEY)
""""""
"""Create Dataset"""
gas.create_dataset("Newsgroups20")
""""""
"""Organize Dataset / regular import"""
from tensorbay.dataset import Data, Dataset
from tensorbay.label import LabeledBox2D
""""""
"""Organize dataset / import dataloader"""
from tensorbay.opendataset import Newsgroups20
dataset = Newsgroups20("path/to/dataset/directory")
""""""
"""Upload Dataset"""
# Copyright 2021 Graviti. Licensed under MIT License.
#
# pylint: disable=wrong-import-position
# pylint: disable=wrong-import-order
# pylint: disable=pointless-string-statement
# pylint: disable=not-an-iterable
# pylint: disable=invalid-name
"""This file includes the python code of getting_started_with_tensorbay.rst."""
"""Authorize a Client Instance"""
from tensorbay import GAS
gas = GAS("<YOUR_ACCESSKEY>")
""""""
"""Create a Dataset"""
gas.create_dataset("<DATASET_NAME>")
""""""
"""List Dataset Names"""
dataset_names = gas.list_dataset_names()
""""""
"""Upload Images to the Dataset"""
from tensorbay.dataset import Data, Dataset
# Organize the local dataset by the "Dataset" class before uploading.
dataset = Dataset("<DATASET_NAME>")
# TensorBay uses "segment" to separate different parts in a dataset.
segment = dataset.create_segment()
segment.append(Data("0000001.jpg"))
segment.append(Data("0000002.jpg"))
def test_upload_dataset_with_mask(self, accesskey, url, tmp_path,
mask_file):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_dataset_name()
gas_client.create_dataset(dataset_name)
dataset = Dataset(name=dataset_name)
segment = dataset.create_segment("Segment1")
# When uploading label, upload catalog first.
dataset._catalog = Catalog.loads(CATALOG_CONTENTS)
path = tmp_path / "sub"
path.mkdir()
local_path = path / "hello.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
remote_semantic_mask = SemanticMask(str(mask_file))
remote_semantic_mask.all_attributes = {
0: {
"occluded": True
},
1: {
"occluded": False
}
}
data.label.semantic_mask = remote_semantic_mask
instance_mask = InstanceMask(str(mask_file))
instance_mask.all_attributes = {
0: {
"occluded": True
},
1: {
"occluded": False
}
}
data.label.instance_mask = instance_mask
panoptic_mask = PanopticMask(str(mask_file))
panoptic_mask.all_category_ids = {100: 0, 101: 1}
data.label.panoptic_mask = panoptic_mask
segment.append(data)
dataset_client = gas_client.upload_dataset(dataset)
dataset_client.commit("upload dataset with label")
dataset = Dataset(dataset_name, gas_client)
remote_semantic_mask = dataset[0][0].label.semantic_mask
semantic_mask = RemoteSemanticMask.from_response_body(
SEMANTIC_MASK_LABEL)
assert dataset.catalog == Catalog.loads(CATALOG_CONTENTS)
assert remote_semantic_mask.path == semantic_mask.path
assert remote_semantic_mask.all_attributes == semantic_mask.all_attributes
remote_instance_mask = dataset[0][0].label.instance_mask
instance_mask = RemoteInstanceMask.from_response_body(
INSTANCE_MASK_LABEL)
assert dataset.catalog == Catalog.loads(CATALOG_CONTENTS)
assert remote_instance_mask.path == instance_mask.path
assert remote_instance_mask.all_attributes == instance_mask.all_attributes
remote_panoptic_mask = dataset[0][0].label.panoptic_mask
panoptic_mask = RemotePanopticMask.from_response_body(
PANOPTIC_MASK_LABEL)
assert dataset.catalog == Catalog.loads(CATALOG_CONTENTS)
assert remote_panoptic_mask.path == panoptic_mask.path
assert remote_panoptic_mask.all_category_ids == panoptic_mask.all_category_ids
gas_client.delete_dataset(dataset_name)
