def test_upload_frame_with_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name,
is_fusion=True)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment_client = dataset_client.get_or_create_segment("segment1")
segment_client.upload_sensor(Sensor.loads(LIDAR_DATA))
path = tmp_path / "sub"
path.mkdir()
for i in range(5):
frame = Frame()
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
frame[LIDAR_DATA["name"]] = data
segment_client.upload_frame(frame, timestamp=i)
frames = list(segment_client.list_frames())
assert len(frames) == 5
assert frames[0][LIDAR_DATA["name"]].path == "hello0.txt"
assert frames[0][LIDAR_DATA["name"]].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_replace_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
local_path = path / "hello0.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
segment_client.upload_file(data.path, data.target_remote_path)
data.label = Label.loads(LABEL)
segment_client.upload_label(data)
# Replace labels
data.label = Label.loads(NEW_LABEL)
segment_client.upload_label(data)
data = list(segment_client.list_data())
assert data[0].path == "hello0.txt"
assert data[0].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_delete_data(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment_client.upload_data(data)
segment_client.delete_data("hello0.txt")
data_paths = list(segment_client.list_data_paths())
assert "hello0.txt" not in data_paths
segment_client.delete_data(segment_client.list_data_paths())
data = list(segment_client.list_data())
assert len(data) == 0
gas_client.delete_dataset(dataset_name)
def test_list_data_paths(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(5):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
segment_client.upload_file(local_path=str(local_path))
# Add other files in reverse order
for i in reversed(range(5)):
local_path = path / f"goodbye{i}.txt"
local_path.write_text("CONTENT")
segment_client.upload_file(local_path=str(local_path))
data_paths = list(segment_client.list_data_paths())
assert data_paths[0] == "goodbye0.txt"
assert data_paths[5] == "hello0.txt"
gas_client.delete_dataset(dataset_name)
def test_upload_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
local_path = path / "hello0.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
# If not uploading file, uploading label is not allowed
with pytest.raises(GASResponseError):
segment_client.upload_label(data)
# Uploading files
segment_client.upload_file(data.path, data.target_remote_path)
data.label = Label.loads(WRONG_LABEL)
# Uploading wrong label is not allowed
with pytest.raises(GASResponseError):
segment_client.upload_label(data)
data.label = Label.loads(LABEL)
segment_client.upload_label(data)
data = list(segment_client.list_data())
assert data[0].path == "hello0.txt"
assert data[0].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_replace_file(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(5):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
segment_client.upload_file(local_path=str(local_path))
# Replace files
for i in range(5):
local_path = path / f"hello{i}.txt"
local_path.write_text("ADD CONTENT")
segment_client.upload_file(local_path=str(local_path))
data = list(segment_client.list_data())
assert data[0].path == "hello0.txt"
assert data[0].open().read() == b"ADD CONTENT"
assert not data[0].label
gas_client.delete_dataset(dataset_name)
def test_upload_frame_with_order(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name,
is_fusion=True)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
segment_client.upload_sensor(Sensor.loads(LIDAR_DATA))
path = tmp_path / "sub"
path.mkdir()
# If noe setting frame id in frame, set timestamp(order) when uploading
for i in reversed(range(5)):
frame = Frame()
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
segment_client.upload_frame(frame, timestamp=i)
# Set frame id in frame
for i in range(5, 10):
frame = Frame(frame_id=str(ulid.from_timestamp(i)))
local_path = path / f"goodbye{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
segment_client.upload_frame(frame)
# Both setting frame id in frame and set timestamp(order) when uploading are not allowed
i = 10
frame = Frame(frame_id=str(ulid.from_timestamp(i)))
local_path = path / f"goodbye{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
with pytest.raises(TypeError):
segment_client.upload_frame(frame, timestamp=i)
# Neither setting frame id in frame nor set timestamp(order) when uploading is not allowed
frame = Frame()
local_path = path / f"goodbye{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
with pytest.raises(TypeError):
segment_client.upload_frame(frame)
frames = list(segment_client.list_frames())
assert len(frames) == 10
assert frames[0][LIDAR_DATA["name"]].path == "hello0.txt"
assert frames[5][LIDAR_DATA["name"]].path == "goodbye5.txt"
assert not frames[0][LIDAR_DATA["name"]].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
class TestVersionControlClient:
gas_client = GAS("Accesskey-********************************")
dataset_client = DatasetClient(
"test_dataset",
"12345",
gas_client,
status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID),
alias="",
is_public=DEFAULT_IS_PUBLIC,
)
def test_squash_and_merge(self, mocker):
post_data = {
"title": "squash_merge-1",
"sourceBranchName": "branch-1",
"targetBranchName": "branch-2",
"strategy": "abort",
}
response_data = {"draftNumber": 2}
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(data=response_data),
)
draft_number = self.dataset_client.squash_and_merge(
"squash_merge-1",
source_branch_name="branch-1",
target_branch_name="branch-2",
strategy="abort",
)
open_api_do.assert_called_once_with("POST",
"squashAndMerge",
self.dataset_client.dataset_id,
json=post_data)
assert draft_number == 2
class TestJobMixin:
gas_client = GAS("Accesskey-********************************")
dataset_client = DatasetClient(
"test_dataset",
"12345",
gas_client,
status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID),
alias="",
is_public=False,
)
def test__create_job(self, mocker, mock_create_job):
post_data = {
"title": "test->main(abort)",
"jobType": "squashAndMerge",
"arguments": {"title": "draft-1"},
"description": "12",
}
open_api_do, response_data = mock_create_job(mocker)
response_data.update(
title=post_data["title"],
arguments=post_data["arguments"],
status="QUEUING",
description=post_data["description"],
)
assert response_data == self.dataset_client.squash_and_merge._create_job(
post_data["title"],
post_data["arguments"],
post_data["description"],
)
open_api_do.assert_called_once_with(
"POST", "jobs", self.dataset_client.dataset_id, json=post_data
)
def test__get_job(self, mocker, mock_get_job):
job_id = "123"
job_type = "squashAndMerge"
open_api_do, response_data = mock_get_job(mocker)
assert response_data == self.dataset_client.squash_and_merge._get_job(job_id)
open_api_do.assert_called_once_with(
"GET", f"jobs/{job_id}", self.dataset_client.dataset_id, params={"jobType": job_type}
)
def test__list_jobs(self, mocker, mock_list_jobs):
params = {
"jobType": "squashAndMerge",
"status": None,
"offset": 0,
"limit": 128,
}
open_api_do, response_data = mock_list_jobs(mocker)
assert response_data == self.dataset_client.squash_and_merge._list_jobs(
params["status"], params["offset"], params["limit"]
)
open_api_do.assert_called_once_with(
"GET", "jobs", self.dataset_client.dataset_id, params=params
)
def test_sensor(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name,
is_fusion=True)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
segment_client.upload_sensor(Sensor.loads(LIDAR_DATA))
sensors = list(segment_client.list_sensors())
assert sensors
# todo: match the input and output sensors
segment_client.delete_sensor(LIDAR_DATA["name"])
sensors = list(segment_client.list_sensors())
assert len(sensors) == 0
gas_client.delete_dataset(dataset_name)
def test_upload_label_without_catalog(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
local_path = path / "hello0.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
segment_client.upload_file(data.path, data.target_remote_path)
# If not uploading catalog, uploading label is not allowed
data.label = Label.loads(LABEL)
with pytest.raises(GASResponseError):
segment_client.upload_label(data)
gas_client.delete_dataset(dataset_name)
class TestVersionControlMixin:
gas_client = GAS("Accesskey-********************************")
dataset_client = DatasetClient(
"test_dataset",
"12345",
gas_client,
status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID),
alias="",
is_public=False,
)
def test_upload_data_without_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(5):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
segment_client.upload_data(Data(local_path=str(local_path)))
data = list(segment_client.list_data())
assert data[0].path == "hello0.txt"
assert data[0].open().read() == b"CONTENT"
assert not data[0].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_upload_frame_without_sensor(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name,
is_fusion=True)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
frame = Frame()
local_path = path / "hello0.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
# If not uploading sensor, uploading frame is not allowed
with pytest.raises(GASResponseError):
segment_client.upload_frame(frame, timestamp=0)
gas_client.delete_dataset(dataset_name)
class TestSquashAndMergeJob:
gas_client = GAS("Accesskey-********************************")
dataset_client = DatasetClient(
"test_dataset",
"12345",
gas_client,
status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID),
alias="",
is_public=True,
)
squash_and_merge_job1 = SquashAndMergeJob(
client=dataset_client.squash_and_merge._client,
dataset_id=dataset_client.dataset_id,
job_updater=dataset_client.squash_and_merge._get_job,
draft_getter=dataset_client.get_draft,
title="test->main(abort)",
job_id="234",
arguments={"title": "draft-1"},
created_at=1,
started_at=None,
finished_at=None,
status="QUEUEING",
error_message="",
result=None,
description="12",
)
squash_and_merge_job2 = SquashAndMergeJob(
client=dataset_client.squash_and_merge._client,
dataset_id=dataset_client.dataset_id,
job_updater=dataset_client.squash_and_merge._get_job,
draft_getter=dataset_client.get_draft,
title="test->main(abort)",
job_id="123",
arguments={"title": "draft-1"},
created_at=1,
started_at=2,
finished_at=3,
status="SUCCESS",
error_message="",
result={"draftNumber": 1},
description="12",
)
def test_result(self, mocker, mock_list_drafts):
list_drafts, drafts_list = mock_list_drafts(mocker, "main")
assert self.squash_and_merge_job1.result is None
draft_numbers = (item.number for item in drafts_list)
assert self.squash_and_merge_job2.result.number in draft_numbers
class TestJob:
gas_client = GAS("Accesskey-********************************")
dataset_client = DatasetClient(
"test_dataset",
"12345",
gas_client,
status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID),
alias="",
is_public=True,
)
job = Job(
client=dataset_client.squash_and_merge._client,
dataset_id=dataset_client.dataset_id,
job_updater=dataset_client.squash_and_merge._get_job,
title="test->main(abort)",
job_id="123",
arguments={"title": "draft-1"},
created_at=1,
started_at=2,
finished_at=3,
status="SUCCESS",
error_message="",
result={"draftNumber": 3},
description="12",
)
def test_update(self, mocker, mock_get_job):
open_api_do, [_, final_responce] = mock_get_job(mocker,
until_complete=True)
self.job.update(until_complete=True)
assert self.job.started_at == final_responce["startedAt"]
assert self.job.finished_at == final_responce["finishedAt"]
assert self.job.status == final_responce["status"]
assert self.job.error_message == final_responce["errorMessage"]
assert self.job._result == final_responce.get("result")
open_api_do.assert_called_with("GET", f"jobs/{self.job.job_id}",
self.dataset_client.dataset_id)
class TestDatasetClientBase:
gas_client = GAS("Accesskey-********************************")
dataset_client = DatasetClient(
"test_dataset",
"12345",
gas_client,
status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID),
alias="",
is_public=False,
)
source_dataset_client = DatasetClient(
"source_dataset",
"544321",
gas_client,
status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID),
alias="",
is_public=False,
)
def test__create_segment(self, mocker):
post_data = {"name": "train"}
post_data.update(self.dataset_client._status.get_status_info())
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(),
)
self.dataset_client._create_segment("train")
open_api_do.assert_called_once_with("POST",
"segments",
self.dataset_client.dataset_id,
json=post_data)
def test__list_segment(self, mocker):
params = self.dataset_client._status.get_status_info()
offset, limit = 0, 128
params["offset"] = offset
params["limit"] = limit
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(),
)
self.dataset_client._list_segments()
open_api_do.assert_called_once_with("GET",
"segments",
self.dataset_client._dataset_id,
params=params)
def test__generate_segment_names(self, mocker):
params = self.dataset_client._status.get_status_info()
offset, limit = 0, 128
params["offset"] = offset
params["limit"] = limit
response_data = {
"offset":
0,
"recordSize":
2,
"totalCount":
2,
"segments": [
{
"name": "test",
"description": ""
},
{
"name": "train",
"description": ""
},
],
}
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(data=response_data),
)
assert list(self.dataset_client._generate_segment_names()) == [
segment["name"] for segment in response_data["segments"]
]
open_api_do.assert_called_once_with("GET",
"segments",
self.dataset_client._dataset_id,
params=params)
def test__copy_segment(self, mocker):
self.dataset_client._status.checkout(draft_number=1)
source_name, target_name = "default", "train"
with pytest.raises(InvalidParamsError):
self.dataset_client._copy_segment(source_name,
target_name,
source_client=None,
strategy="move")
with pytest.raises(ValueError):
self.dataset_client._copy_segment(source_name,
source_name,
source_client=None)
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(),
)
source = {"segmentName": source_name}
source["id"] = self.source_dataset_client.dataset_id
source.update(self.source_dataset_client.status.get_status_info())
post_data = {
"strategy": "abort",
"source": source,
"segmentName": target_name,
}
post_data.update(self.dataset_client._status.get_status_info())
self.dataset_client._copy_segment(
source_name, target_name, source_client=self.source_dataset_client)
open_api_do.assert_called_once_with("POST",
"segments?copy",
self.dataset_client._dataset_id,
json=post_data)
def test__move_segment(self, mocker):
self.dataset_client._status.checkout(draft_number=1)
source_name, target_name = "default", "train"
with pytest.raises(InvalidParamsError):
self.dataset_client._move_segment(source_name,
target_name,
strategy="move")
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(),
)
post_data = {
"strategy": "abort",
"source": {
"segmentName": source_name
},
"segmentName": target_name,
}
post_data.update(self.dataset_client._status.get_status_info())
self.dataset_client._move_segment(source_name, target_name)
open_api_do.assert_called_once_with("POST",
"segments?move",
self.dataset_client._dataset_id,
json=post_data)
def test_enable_cache(
self,
mocker,
mock_list_segments,
mock_list_data_details,
mock_get_total_size,
tmp_path,
caplog,
):
self.dataset_client._status.checkout(draft_number=1)
with pytest.raises(StatusError):
self.dataset_client.enable_cache()
self.dataset_client._status.checkout(commit_id="commit-1")
_, response_data = mock_get_total_size(mocker, True)
_, _, free_storage = shutil.disk_usage(tempfile.gettempdir())
self.dataset_client.enable_cache()
log_message = (
"bytes left on device, "
f'less than the dataset size {response_data["totalSize"]} bytes.\n '
"Please be aware that there is not enough space to cache the entire dataset."
)
assert log_message in caplog.text
mock_get_total_size(mocker)
cache_path = tmp_path / "cache_data"
dataset_cache_path = cache_path / self.dataset_client.dataset_id
self.dataset_client.enable_cache(cache_path)
assert self.dataset_client.cache_enabled == True
assert self.dataset_client._cache_path == str(dataset_cache_path)
_, list_segments = mock_list_segments(mocker)
segment_name = list_segments["segments"][0]["name"]
segment = self.dataset_client.get_segment(segment_name)
mock_list_data_details(mocker)
segment_data = segment.list_data()
urlopen = mocker.patch(
f"{file.__name__}.RemoteFileMixin._urlopen",
return_value=mock_response(read=lambda *args: bytes(1)),
)
segment_cache_path = (dataset_cache_path /
self.dataset_client.status.commit_id /
segment_name)
# Traverse the segment for the first time.
for data in segment_data:
assert data.cache_path == str(segment_cache_path / data.path)
data.open()
local_open = mocker.patch(
f"builtins.open",
return_value=bytes(1),
)
segment_length = len(segment_data)
# Traverse the segment for 3 times using cached data.
epoch = 3
for i in range(epoch):
for data in segment_data:
assert data.cache_path == str(segment_cache_path / data.path)
data.open()
# local_open.assert_called_once()
assert local_open.call_count == segment_length * epoch
assert urlopen.call_count == segment_length
self.dataset_client._status.checkout(draft_number=1)
assert self.dataset_client.cache_enabled == False
assert self.dataset_client._cache_path == str(dataset_cache_path)
def test_update_notes(self, mocker):
self.dataset_client._status.checkout(draft_number=1)
patch_data = {"binPointCloudFields": None}
patch_data.update(self.dataset_client._status.get_status_info())
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(),
)
self.dataset_client.update_notes(bin_point_cloud_fields=None)
open_api_do.assert_called_once_with("PATCH",
"notes",
self.dataset_client._dataset_id,
json=patch_data)
patch_data = {
"isContinuous": True,
"binPointCloudFields": ["X", "Y", "Z", "Intensity", "Ring"],
}
patch_data.update(self.dataset_client._status.get_status_info())
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(),
)
self.dataset_client.update_notes(
is_continuous=True,
bin_point_cloud_fields=["X", "Y", "Z", "Intensity", "Ring"])
open_api_do.assert_called_once_with("PATCH",
"notes",
self.dataset_client._dataset_id,
json=patch_data)
def test_get_notes(self, mocker):
params = self.dataset_client._status.get_status_info()
response_data = {
"isContinuous": True,
"binPointCloudFields": ["X", "Y", "Z", "Intensity", "Ring"],
}
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(data=response_data),
)
assert self.dataset_client.get_notes() == Notes.loads(response_data)
open_api_do.assert_called_once_with("GET",
"notes",
self.dataset_client._dataset_id,
params=params)
def test_list_segment(self, mocker):
response_data = {
"offset":
0,
"recordSize":
2,
"totalCount":
2,
"segments": [
{
"name": "test",
"description": ""
},
{
"name": "train",
"description": ""
},
],
}
mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(data=response_data),
)
assert list(self.dataset_client.list_segment_names()) == [
segment["name"] for segment in response_data["segments"]
]
def test_get_catelog(self, mocker):
params = self.dataset_client._status.get_status_info()
response_data = {
"catalog": {
"CLASSIFICATION": {
"categories": [{
"name": "cat"
}]
}
}
}
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(data=response_data),
)
assert self.dataset_client.get_catalog() == Catalog.loads(
response_data["catalog"])
open_api_do.assert_called_once_with("GET",
"labels/catalogs",
self.dataset_client._dataset_id,
params=params)
def test_upload_catalog(self, mocker):
self.dataset_client._status.checkout(draft_number=1)
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(),
)
catalog = Catalog.loads(
{"CLASSIFICATION": {
"categories": [{
"name": "cat"
}]
}})
self.dataset_client.upload_catalog(catalog)
put_data = {"catalog": catalog.dumps()}
put_data.update(self.dataset_client._status.get_status_info())
open_api_do.assert_called_once_with("PUT",
"labels/catalogs",
self.dataset_client._dataset_id,
json=put_data)
def test_delete_segment(self, mocker):
self.dataset_client._status.checkout(draft_number=1)
delete_data = {"segmentName": "train"}
delete_data.update(self.dataset_client._status.get_status_info())
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(),
)
self.dataset_client.delete_segment("train")
open_api_do.assert_called_once_with("DELETE",
"segments",
self.dataset_client._dataset_id,
json=delete_data)
def test_get_label_statistics(self, mocker):
params = self.dataset_client._status.get_status_info()
response_data = {
"labelStatistics": {
"BOX2D": {
"quantity":
10,
"categories": [{
"name":
"vehicles.bike",
"quantity":
10,
"attributes": [{
"name": "trafficLightColor",
"enum": ["none", "red", "yellow"],
"quantities": [5, 3, 2],
}],
}],
}
}
}
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(data=response_data),
)
statistics1 = self.dataset_client.get_label_statistics()
open_api_do.assert_called_once_with("GET",
"labels/statistics",
self.dataset_client.dataset_id,
params=params)
assert statistics1 == Statistics(response_data["labelStatistics"])
def test_get_total_size(self, mocker, mock_get_total_size):
self.dataset_client._status.checkout(commit_id="commit-1")
params = {"commit": self.dataset_client._status.commit_id}
get_total_size, response_data = mock_get_total_size(mocker)
total_size = self.dataset_client.get_total_size()
get_total_size.assert_called_once_with("GET",
"total-size",
self.dataset_client.dataset_id,
params=params)
assert total_size == response_data["totalSize"]
class TestGAS:
gas_client = GAS("Accesskey-********************************")
def test_generate_auth_storage_configs(self, mocker):
params = {"name": "cloud_config", "offset": 0, "limit": 128}
response_data = {
"configs": [
{
"name": "cloud_config",
"type": "azure",
"filePath": "tensorbay",
"isGravitiStorage": True,
},
],
"offset":
0,
"recordSize":
1,
"totalCount":
1,
}
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(data=response_data),
)
assert list(
self.gas_client._generate_auth_storage_configs(
"cloud_config")) == [
StorageConfig.loads(config)
for config in response_data["configs"]
]
open_api_do.assert_called_once_with("GET",
"storage-configs",
"",
params=params)
@pytest.mark.parametrize("is_fusion", [True, False])
@pytest.mark.parametrize("is_public", [True, False])
def test_get_dataset_with_any_type(self, mocker, is_fusion, is_public):
response_data = {
"name": "test",
"type": int(is_fusion),
"defaultBranch": DEFAULT_BRANCH,
"commitId": "4",
"updateTime": 1622693494,
"owner": "",
"id": "123456",
"alias": "alias",
"isPublic": is_public,
}
get_dataset = mocker.patch(
f"{gas.__name__}.GAS._get_dataset",
return_value=response_data,
)
dataset_client = self.gas_client._get_dataset_with_any_type(
response_data["name"])
dataset_type = FusionDatasetClient if is_fusion else DatasetClient
assert isinstance(dataset_client, dataset_type)
assert dataset_client._name == response_data["name"]
assert dataset_client._dataset_id == response_data["id"]
assert dataset_client._status.commit_id == response_data["commitId"]
assert dataset_client._status.branch_name == response_data[
"defaultBranch"]
assert dataset_client._alias == response_data["alias"]
assert dataset_client._is_public == response_data["isPublic"]
get_dataset.assert_called_once_with(response_data["name"])
def test__get_dataset(self, mocker):
with pytest.raises(ResourceNotExistError):
self.gas_client._get_dataset("")
mocker.patch(
f"{gas.__name__}.GAS._list_datasets",
return_value={"datasets": []},
)
with pytest.raises(ResourceNotExistError):
self.gas_client._get_dataset("test")
list_dataset_data = {
"datasets": [{
"id": "123456",
"name": "test",
"type": 0,
"HEAD": {
"commitId": "456123"
},
"updateTime": 1622530298,
"owner": "",
"defaultBranch": DEFAULT_BRANCH,
}],
"offset":
0,
"recordSize":
1,
"totalCount":
1,
}
get_dataset_data = {
"name": "test",
"type": 0,
"defaultBranch": DEFAULT_BRANCH,
"commitId": "4",
"updateTime": 1622693494,
"owner": "",
"id": "123456",
"isPublic": False,
}
mocker.patch(
f"{gas.__name__}.GAS._list_datasets",
return_value=list_dataset_data,
)
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(data=get_dataset_data),
)
assert self.gas_client._get_dataset(
get_dataset_data["name"]) == get_dataset_data
open_api_do.assert_called_once_with(
"GET", "", list_dataset_data["datasets"][0]["id"])
def test__list_datasets(self, mocker):
params = {
"name": "test",
"offset": 0,
"limit": 128,
}
response_data = {
"datasets": [{
"id": "123456",
"name": "test",
"type": 0,
"defaultBranch": DEFAULT_BRANCH,
"updateTime": 1622530298,
"owner": "",
}],
"offset":
0,
"recordSize":
1,
"totalCount":
1,
}
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(data=response_data),
)
assert self.gas_client._list_datasets(params["name"]) == response_data
open_api_do.assert_called_once_with("GET", "", params=params)
def test_get_user(self, mocker):
response_data = {
"nickname": "user_name",
"email": "user_email",
"mobile": "11111111111",
"description": "",
"team": {
"name": "TensorbaySDKTest",
"email": None,
"description": ""
},
}
dump_data = {
"nickname": "user_name",
"email": "user_email",
"mobile": "11111111111",
"team": {
"name": "TensorbaySDKTest",
},
}
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(data=response_data),
)
assert self.gas_client.get_user().dumps() == dump_data
open_api_do.assert_called_once_with("GET", "users")
def test_get_auth_storage_config(self, mocker):
with pytest.raises(TypeError):
self.gas_client.get_auth_storage_config("")
mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(data={
"configs": [],
"totalCount": 0
}),
)
with pytest.raises(ResourceNotExistError):
self.gas_client.get_auth_storage_config("cloud_config")
response_data = {
"configs": [
{
"name": "cloud_config",
"type": "azure",
"filePath": "tensorbay",
"isGravitiStorage": True,
},
],
"offset":
0,
"recordSize":
1,
"totalCount":
1,
}
mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(data=response_data),
)
assert self.gas_client.get_auth_storage_config(
response_data["configs"][0]["name"]) == StorageConfig.loads(
response_data["configs"][0])
def test_list_auth_storage_configs(self, mocker):
response_data = {
"configs": [
{
"name": "cloud_config",
"type": "azure",
"filePath": "tensorbay",
"isGravitiStorage": True,
},
],
"offset":
0,
"recordSize":
1,
"totalCount":
1,
}
mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(data=response_data),
)
configs = self.gas_client.list_auth_storage_configs()
assert list(configs) == [
StorageConfig.loads(config) for config in response_data["configs"]
]
def test_delete_storage_config(self, mocker):
open_api_do = mocker.patch(f"{gas.__name__}.Client.open_api_do")
self.gas_client.delete_storage_config("test-config")
open_api_do.assert_called_once_with("DELETE",
"storage-configs/test-config")
def test_create_oss_storage_config(self, mocker):
open_api_do = mocker.patch(f"{gas.__name__}.Client.open_api_do")
self.gas_client.create_oss_storage_config(
"oss_config",
"tests",
endpoint="oss-cn-qingdao.aliyuncs.com",
accesskey_id="accesskeyId",
accesskey_secret="accesskeySecret",
bucket_name="bucketName",
)
post_data = {
"name": "oss_config",
"filePath": "tests",
"endpoint": "oss-cn-qingdao.aliyuncs.com",
"accesskeyId": "accesskeyId",
"accesskeySecret": "accesskeySecret",
"bucketName": "bucketName",
}
open_api_do("POST", "storage-configs/oss", json=post_data)
def test_create_s3_storage_config(self, mocker):
open_api_do = mocker.patch(f"{gas.__name__}.Client.open_api_do")
self.gas_client.create_s3_storage_config(
"s3_config",
"tests",
endpoint="s3.cn-northwest-1.amazonaws.com",
accesskey_id="accesskeyId",
accesskey_secret="accesskeySecret",
bucket_name="bucketName",
)
post_data = {
"name": "s3_config",
"filePath": "tests",
"endpoint": "s3.cn-northwest-1.amazonaws.com",
"accesskeyId": "accesskeyId",
"accesskeySecret": "accesskeySecret",
"bucketName": "bucketName",
}
open_api_do("POST", "storage-configs/s3", json=post_data)
def test_create_azure_storage_config(self, mocker):
open_api_do = mocker.patch(f"{gas.__name__}.Client.open_api_do")
self.gas_client.create_azure_storage_config(
"azure_config",
"tests",
account_type="China",
account_name="accountName",
account_key="accountKey",
container_name="containerName",
)
post_data = {
"name": "s3_config",
"filePath": "tests",
"accesskeyId": "accountName",
"accesskeySecret": "accountKey",
"containerName": "containerName",
"accountType": "China",
}
open_api_do("POST", "storage-configs/azure", json=post_data)
def test_create_local_storage_config(self, mocker):
open_api_do = mocker.patch(f"{gas.__name__}.Client.open_api_do")
self.gas_client.create_local_storage_config(
"local_config",
"tests",
"http://192.168.0.1:9000",
)
post_data = {
"name": "local_config",
"filePath": "tests",
"endpoint": "http://192.168.0.1:9000",
}
open_api_do("POST", "storage-configs/local", json=post_data)
def test_get_cloud_client(self, mocker):
config_name = "cloud_train"
response_data = {
"configs": [
{
"name": "cloud_train",
"type": "azure",
"filePath": "tensorbay",
"isGravitiStorage": True,
},
],
"offset":
0,
"recordSize":
1,
"totalCount":
1,
}
mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(data=response_data),
)
cloud_client = self.gas_client.get_cloud_client(config_name)
cloud_client_1 = CloudClient(config_name, self.gas_client._client)
assert cloud_client._name == cloud_client_1._name
assert cloud_client._client == cloud_client_1._client
@pytest.mark.parametrize("is_fusion", [True, False])
@pytest.mark.parametrize("is_public", [True, False])
def test_create_dataset(self, mocker, is_fusion, is_public):
params = {
"name": "test",
"type": int(is_fusion),
"configName": "config",
"alias": "alias",
"isPublic": is_public,
}
response_data = {"id": "12345"}
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(data=response_data),
)
dataset_client = self.gas_client.create_dataset("test",
is_fusion,
config_name="config",
alias="alias",
is_public=is_public)
dataset_type = FusionDatasetClient if is_fusion else DatasetClient
assert isinstance(dataset_client, dataset_type)
assert dataset_client._name == params["name"]
assert dataset_client._dataset_id == response_data["id"]
assert dataset_client.status.branch_name == DEFAULT_BRANCH
assert dataset_client.status.commit_id == ROOT_COMMIT_ID
assert dataset_client._alias == params["alias"]
assert dataset_client._is_public == is_public
open_api_do.assert_called_once_with("POST", "", json=params)
@pytest.mark.parametrize("is_fusion", [True, False])
@pytest.mark.parametrize("is_public", [True, False])
def test_get_dataset(self, mocker, is_fusion, is_public):
mocker.patch(
f"{gas.__name__}.GAS._get_dataset",
return_value={
"id": "123456",
"type": 1,
"commitId": "4",
"defaultBranch": DEFAULT_BRANCH,
"alias": "alias",
"isPublic": is_public,
},
)
with pytest.raises(DatasetTypeError):
self.gas_client.get_dataset("test")
response_data = {
"id": "123456",
"type": int(is_fusion),
"commitId": "4",
"defaultBranch": DEFAULT_BRANCH,
"alias": "alias",
"isPublic": is_public,
}
dataset_name = "test"
get_dataset = mocker.patch(
f"{gas.__name__}.GAS._get_dataset",
return_value=response_data,
)
dataset_client = self.gas_client.get_dataset(dataset_name, is_fusion)
dataset_type = FusionDatasetClient if is_fusion else DatasetClient
assert isinstance(dataset_client, dataset_type)
assert dataset_client._name == dataset_name
assert dataset_client._dataset_id == response_data["id"]
assert dataset_client._status.commit_id == response_data["commitId"]
assert dataset_client._status.branch_name == response_data[
"defaultBranch"]
assert dataset_client._alias == response_data["alias"]
assert dataset_client._is_public == response_data["isPublic"]
get_dataset.assert_called_once_with(dataset_name)
def test_list_dataset_names(self, mocker):
params = {
"offset": 0,
"limit": 128,
}
response_data = {
"datasets": [{
"id": "123456",
"name": "test",
"type": 0,
"defaultBranch": DEFAULT_BRANCH,
"updateTime": 1622530298,
"owner": "",
}],
"offset":
0,
"recordSize":
1,
"totalCount":
1,
}
open_api_do = mocker.patch(
f"{gas.__name__}.Client.open_api_do",
return_value=mock_response(data=response_data),
)
datasets = self.gas_client.list_dataset_names()
assert list(datasets) == [
data["name"] for data in response_data["datasets"]
]
open_api_do.assert_called_once_with("GET", "", params=params)
def test_update_dataset(self, mocker):
response_data = {
"id": "123456",
"type": 1,
"commitId": "4",
"defaultBranch": DEFAULT_BRANCH,
}
open_api_do = mocker.patch(f"{gas.__name__}.Client.open_api_do")
mocker.patch(
f"{gas.__name__}.GAS._get_dataset",
return_value=response_data,
)
patch_data = {"alias": "new_alias", "isPublic": True}
self.gas_client.update_dataset("test",
alias="new_alias",
is_public=True)
open_api_do.assert_called_once_with("PATCH",
"",
response_data["id"],
json=patch_data)
def test_rename_dataset(self, mocker):
response_data = {
"id": "123456",
"type": 1,
"commitId": "4",
"defaultBranch": DEFAULT_BRANCH,
}
open_api_do = mocker.patch(f"{gas.__name__}.Client.open_api_do")
mocker.patch(
f"{gas.__name__}.GAS._get_dataset",
return_value=response_data,
)
patch_data = {"name": "new_test"}
self.gas_client.rename_dataset("test", new_name="new_test")
open_api_do.assert_called_once_with("PATCH",
"",
response_data["id"],
json=patch_data)
def test_upload_dataset(self, mocker):
dataset = Dataset("test")
dataset.load_catalog(
os.path.join(
os.path.dirname(__file__),
"..",
"..",
"opendataset",
"HeadPoseImage",
"catalog.json",
))
for i in range(5):
dataset.create_segment(str(i))
# upload the dataset in main branch containing no draft
get_dataset = mocker.patch(
f"{gas.__name__}.GAS.get_dataset",
return_value=DatasetClient(
"test",
"12345",
self.gas_client,
status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID),
alias="",
is_public=False,
),
)
checkout = mocker.patch(f"{gas.__name__}.DatasetClient.checkout")
list_drafts = mocker.patch(f"{gas.__name__}.DatasetClient.list_drafts",
return_value=[])
create_draft = mocker.patch(
f"{gas.__name__}.DatasetClient.create_draft")
upload_catalog = mocker.patch(
f"{gas.__name__}.DatasetClient.upload_catalog")
update_notes = mocker.patch(
f"{gas.__name__}.DatasetClient.update_notes")
_upload_segment = mocker.patch(
f"{gas.__name__}.DatasetClient._upload_segment")
self.gas_client.upload_dataset(dataset)
assert not checkout.called
get_dataset.assert_called_once_with(dataset.name, False)
list_drafts.assert_called_once_with(branch_name=DEFAULT_BRANCH)
create_draft.assert_called_once_with(
'Draft autogenerated by "GAS.upload_dataset"',
branch_name=DEFAULT_BRANCH)
upload_catalog.assert_called_once_with(dataset.catalog)
update_notes.assert_called_once_with(**dataset.notes)
assert _upload_segment.call_count == 5
# upload the dataset in main branch containing a draft
list_drafts = mocker.patch(
f"{gas.__name__}.DatasetClient.list_drafts",
return_value=[
Draft(
1,
"title",
DEFAULT_BRANCH,
"OPEN",
"4c564ea07f4e47679ec8c63d238bb3a1",
User("test", 1636967807),
1637223060,
)
],
)
checkout = mocker.patch(f"{gas.__name__}.DatasetClient.checkout")
self.gas_client.upload_dataset(dataset)
list_drafts.assert_called_once_with(branch_name=DEFAULT_BRANCH)
checkout.assert_called_once_with(draft_number=1)
# upload the dataset in dev branch containing no draft
list_drafts = mocker.patch(
f"{gas.__name__}.DatasetClient.list_drafts",
return_value=[],
)
checkout = mocker.patch(f"{gas.__name__}.DatasetClient.checkout")
create_draft = mocker.patch(
f"{gas.__name__}.DatasetClient.create_draft")
self.gas_client.upload_dataset(dataset, branch_name="dev")
assert not checkout.called
list_drafts.assert_called_once_with(branch_name="dev")
create_draft.assert_called_once_with(
'Draft autogenerated by "GAS.upload_dataset"', branch_name="dev")
# upload the dataset in dev branch containing a draft
list_drafts = mocker.patch(
f"{gas.__name__}.DatasetClient.list_drafts",
return_value=[
Draft(
1,
"title",
"dev",
"OPEN",
"4c564ea07f4e47679ec8c63d238bb3a1",
User("test", 1636967807),
1637223060,
)
],
)
checkout = mocker.patch(f"{gas.__name__}.DatasetClient.checkout")
self.gas_client.upload_dataset(dataset, branch_name="dev")
list_drafts.assert_called_once_with(branch_name="dev")
checkout.assert_called_once_with(draft_number=1)
def test_delete_dataset(self, mocker):
response_data = {"id": "123456", "type": 1, "commitId": "4"}
open_api_do = mocker.patch(f"{gas.__name__}.Client.open_api_do")
mocker.patch(
f"{gas.__name__}.GAS._get_dataset",
return_value=response_data,
)
self.gas_client.delete_dataset("test")
open_api_do.assert_called_once_with("DELETE", "", response_data["id"])
class TestFusionDatasetClient(TestDatasetClientBase):
gas_client = GAS("Accesskey-********************************")
fusion_dataset_client = FusionDatasetClient(
"test_dataset",
"12345",
gas_client,
status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID),
alias="",
is_public=False,
)
source_dataset_client = FusionDatasetClient(
"source_dataset",
"544321",
gas_client,
status=Status(DEFAULT_BRANCH, commit_id=ROOT_COMMIT_ID),
alias="",
is_public=False,
)
def test__extract_all_data(self):
source_frames = []
ulids = []
for i in range(5):
temp_frame = Frame()
if i % 2 == 0:
temp_frame["camera"] = Data(f"{i}.png")
else:
temp_frame["lidar"] = Data(f"{i}.png")
temp_ulid = from_timestamp(10 * i + 10)
temp_frame.frame_id = temp_ulid
source_frames.append((temp_frame, temp_ulid))
ulids.append(temp_ulid)
with Tqdm(5, disable=False) as pbar:
for index, values in enumerate(
self.fusion_dataset_client._extract_all_data(
source_frames, pbar)):
data, sensor_name, frame_id = values
assert data.path == f"{index}.png"
if index % 2 == 0:
assert sensor_name == "camera"
else:
assert sensor_name == "lidar"
assert frame_id == ulids[index].str
def test__extract_unuploaded_data(self):
source_frames = []
ulids = []
done_frames = {}
for i in range(5):
temp_frame = Frame()
temp_ulid = from_timestamp(10 * i + 10)
if i % 2 == 0:
temp_frame["camera"] = Data(f"{i}.png")
done_frames[temp_ulid.timestamp().timestamp] = temp_frame
else:
temp_frame["lidar"] = Data(f"{i}.png")
ulids.append(temp_ulid)
temp_frame.frame_id = temp_ulid
source_frames.append((temp_frame, temp_ulid))
with Tqdm(5, disable=False) as pbar:
for index, values in enumerate(
self.fusion_dataset_client._extract_unuploaded_data(
source_frames, pbar, done_frames=done_frames)):
data, sensor_name, frame_id = values
assert data.path == f"{index * 2 + 1}.png"
assert sensor_name == "lidar"
assert frame_id == ulids[index].str
def test__generate_segments(self, mocker):
params = self.dataset_client._status.get_status_info()
offset, limit = 0, 128
params["offset"] = offset
params["limit"] = limit
response_data = {
"offset":
0,
"recordSize":
2,
"totalCount":
2,
"segments": [
{
"name": "test",
"description": ""
},
{
"name": "train",
"description": ""
},
],
}
list_segments = mocker.patch(
f"{dataset.__name__}.FusionDatasetClient._list_segments",
return_value=response_data,
)
segment_generator = ReturnGenerator(
self.fusion_dataset_client._generate_segments())
assert [segment.name for segment in segment_generator
] == [item["name"] for item in response_data["segments"]]
list_segments.assert_called_once_with(offset, limit)
assert segment_generator.value == response_data["totalCount"]
def test__upload_segment(self, mocker):
segment_test = FusionSegment(name="test1")
ulids = []
done_frames = []
for i in range(5):
temp_frame = Frame()
temp_ulid = from_timestamp(10 * i + 10)
temp_frame.frame_id = temp_ulid
if i % 2 == 0:
temp_frame["camera"] = Data(f"{i}.png")
done_frames.append(temp_frame)
else:
temp_frame["lidar"] = Data(f"{i}.png")
ulids.append(temp_ulid)
segment_test.append(temp_frame)
segment_client = FusionSegmentClient(
name="test1", data_client=self.fusion_dataset_client)
get_or_create_segment = mocker.patch(
f"{dataset.__name__}.FusionDatasetClient.get_or_create_segment",
return_value=segment_client,
)
list_frames = mocker.patch(
f"{segment.__name__}.FusionSegmentClient.list_frames",
return_value=done_frames,
)
multithread_upload = mocker.patch(
f"{dataset.__name__}.multithread_upload")
with Tqdm(5, disable=False) as pbar:
self.fusion_dataset_client._upload_segment(
segment_test, jobs=8, skip_uploaded_files=True, pbar=pbar)
get_or_create_segment.assert_called_once_with(segment_test.name)
list_frames.assert_called_once_with()
args, keywords = multithread_upload.call_args
for index, values in enumerate(args[1]):
data, sensor_name, frame_id = values
assert data.path == f"{index * 2 + 1}.png"
assert sensor_name == "lidar"
assert frame_id == ulids[index * 2 + 1].str
assert keywords[
"callback"] == segment_client._synchronize_upload_info
assert keywords["jobs"] == 8
assert keywords["pbar"] == pbar
multithread_upload.assert_called_once()
with Tqdm(5, disable=False) as pbar:
self.fusion_dataset_client._upload_segment(
segment_test, jobs=8, skip_uploaded_files=False, pbar=pbar)
get_or_create_segment.assert_called_with(segment_test.name)
list_frames.assert_called_with()
args, keywords = multithread_upload.call_args
for index, values in enumerate(args[1]):
data, sensor_name, frame_id = values
assert data.path == f"{index}.png"
if index % 2 == 0:
assert sensor_name == "camera"
else:
assert sensor_name == "lidar"
assert frame_id == ulids[index].str
assert keywords[
"callback"] == segment_client._synchronize_upload_info
assert keywords["jobs"] == 8
assert keywords["pbar"] == pbar
def test_get_or_create_segment(self, mocker):
self.fusion_dataset_client._status.checkout(draft_number=1)
list_segment = mocker.patch(
f"{dataset.__name__}.FusionDatasetClient.list_segment_names",
return_value=["test1"])
create_segment = mocker.patch(
f"{dataset.__name__}.FusionDatasetClient._create_segment", )
segment_client_1 = self.fusion_dataset_client.get_or_create_segment(
"test1")
list_segment.assert_called_once_with()
assert segment_client_1.name == "test1"
segment_client_2 = self.fusion_dataset_client.get_or_create_segment(
"test2")
list_segment.assert_called_with()
create_segment.assert_called_once_with("test2")
assert segment_client_2.name == "test2"
def test_create_segment(self, mocker):
self.fusion_dataset_client._status.checkout(draft_number=1)
list_segment = mocker.patch(
f"{dataset.__name__}.FusionDatasetClient.list_segment_names",
return_value=["test1"])
create_segment = mocker.patch(
f"{dataset.__name__}.FusionDatasetClient._create_segment", )
with pytest.raises(NameConflictError):
self.fusion_dataset_client.create_segment("test1")
list_segment.assert_called_once_with()
segment_client = self.fusion_dataset_client.create_segment("test2")
list_segment.assert_called_with()
create_segment.assert_called_once_with("test2")
assert segment_client.name == "test2"
def test_copy_segment(self, mocker):
self.fusion_dataset_client._status.checkout(draft_number=1)
source_name, target_name = "default", "train"
copy_segment = mocker.patch(
f"{dataset.__name__}.FusionDatasetClient._copy_segment")
strategy = "abort"
assert self.fusion_dataset_client.copy_segment(
source_name).name == source_name
copy_segment.assert_called_once_with(source_name,
source_name,
source_client=None,
strategy=strategy)
strategy = "override"
assert (self.fusion_dataset_client.copy_segment(
source_name, target_name, strategy=strategy).name == target_name)
copy_segment.assert_called_with(source_name,
target_name,
source_client=None,
strategy=strategy)
def test_move_segment(self, mocker):
self.fusion_dataset_client._status.checkout(draft_number=1)
source_name, target_name = "default", "train"
move_segment = mocker.patch(
f"{dataset.__name__}.FusionDatasetClient._move_segment")
strategy = "abort"
assert self.fusion_dataset_client.move_segment(
source_name, target_name).name == target_name
move_segment.assert_called_once_with(source_name,
target_name,
strategy=strategy)
def test_get_segment(self, mocker):
list_segment = mocker.patch(
f"{dataset.__name__}.FusionDatasetClient.list_segment_names",
return_value=["test1"])
with pytest.raises(ResourceNotExistError):
self.fusion_dataset_client.get_segment("test2")
list_segment.assert_called_once_with()
assert self.fusion_dataset_client.get_segment("test1").name == "test1"
list_segment.assert_called_with()
def test_upload_segment(self, mocker):
self.fusion_dataset_client._status.checkout(draft_number=1)
segment_test = FusionSegment(name="test1")
for i in range(5):
temp_frame = Frame()
temp_ulid = from_timestamp(10 * i + 10)
temp_frame.frame_id = temp_ulid
if i % 2 == 0:
temp_frame["camera"] = Data(f"{i}.png")
else:
temp_frame["lidar"] = Data(f"{i}.png")
segment_test.append(temp_frame)
segment_client = FusionSegmentClient(
name="test1", data_client=self.fusion_dataset_client)
upload_segment = mocker.patch(
f"{dataset.__name__}.FusionDatasetClient._upload_segment",
return_value=segment_client)
assert self.fusion_dataset_client.upload_segment(
segment_test).name == "test1"
args, keywords = upload_segment.call_args
assert args[0] == segment_test
assert keywords["jobs"] == 1
assert not keywords["skip_uploaded_files"]
upload_segment.assert_called_once()
