def test_delete_data(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment_client.upload_data(data)
segment_client.delete_data("hello0.txt")
data_paths = list(segment_client.list_data_paths())
assert "hello0.txt" not in data_paths
segment_client.delete_data(segment_client.list_data_paths())
data = list(segment_client.list_data())
assert len(data) == 0
gas_client.delete_dataset(dataset_name)
def test_upload_frame_with_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name,
is_fusion=True)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment_client = dataset_client.get_or_create_segment("segment1")
segment_client.upload_sensor(Sensor.loads(LIDAR_DATA))
path = tmp_path / "sub"
path.mkdir()
for i in range(5):
frame = Frame()
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
frame[LIDAR_DATA["name"]] = data
segment_client.upload_frame(frame, timestamp=i)
frames = list(segment_client.list_frames())
assert len(frames) == 5
assert frames[0][LIDAR_DATA["name"]].path == "hello0.txt"
assert frames[0][LIDAR_DATA["name"]].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_replace_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
local_path = path / "hello0.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
segment_client.upload_file(data.path, data.target_remote_path)
data.label = Label.loads(LABEL)
segment_client.upload_label(data)
# Replace labels
data.label = Label.loads(NEW_LABEL)
segment_client.upload_label(data)
data = list(segment_client.list_data())
assert data[0].path == "hello0.txt"
assert data[0].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_upload_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
local_path = path / "hello0.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
# If not uploading file, uploading label is not allowed
with pytest.raises(GASResponseError):
segment_client.upload_label(data)
# Uploading files
segment_client.upload_file(data.path, data.target_remote_path)
data.label = Label.loads(WRONG_LABEL)
# Uploading wrong label is not allowed
with pytest.raises(GASResponseError):
segment_client.upload_label(data)
data.label = Label.loads(LABEL)
segment_client.upload_label(data)
data = list(segment_client.list_data())
assert data[0].path == "hello0.txt"
assert data[0].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_upload_frame_with_order(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name,
is_fusion=True)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
segment_client.upload_sensor(Sensor.loads(LIDAR_DATA))
path = tmp_path / "sub"
path.mkdir()
# If noe setting frame id in frame, set timestamp(order) when uploading
for i in reversed(range(5)):
frame = Frame()
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
segment_client.upload_frame(frame, timestamp=i)
# Set frame id in frame
for i in range(5, 10):
frame = Frame(frame_id=str(ulid.from_timestamp(i)))
local_path = path / f"goodbye{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
segment_client.upload_frame(frame)
# Both setting frame id in frame and set timestamp(order) when uploading are not allowed
i = 10
frame = Frame(frame_id=str(ulid.from_timestamp(i)))
local_path = path / f"goodbye{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
with pytest.raises(TypeError):
segment_client.upload_frame(frame, timestamp=i)
# Neither setting frame id in frame nor set timestamp(order) when uploading is not allowed
frame = Frame()
local_path = path / f"goodbye{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
with pytest.raises(TypeError):
segment_client.upload_frame(frame)
frames = list(segment_client.list_frames())
assert len(frames) == 10
assert frames[0][LIDAR_DATA["name"]].path == "hello0.txt"
assert frames[5][LIDAR_DATA["name"]].path == "goodbye5.txt"
assert not frames[0][LIDAR_DATA["name"]].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_get_callback_body(self, tmp_path):
local_path = tmp_path / "file"
local_path.write_text("CONTENT")
timestamp = 1234
data = Data(str(local_path), timestamp=timestamp)
assert data.get_callback_body() == {
"checksum": "238a131a3e8eb98d1fc5b27d882ca40b7618fd2a",
"fileSize": 7,
"remotePath": local_path.name,
"timestamp": timestamp,
}
def test_upload_label_without_catalog(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
local_path = path / "hello0.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
segment_client.upload_file(data.path, data.target_remote_path)
# If not uploading catalog, uploading label is not allowed
data.label = Label.loads(LABEL)
with pytest.raises(GASResponseError):
segment_client.upload_label(data)
gas_client.delete_dataset(dataset_name)
def test_init(self):
local_path = "test.json"
target_remote_path = "A/test.json"
timestamp = 1614667532
data = Data(local_path,
target_remote_path=target_remote_path,
timestamp=timestamp)
assert data.path == local_path
assert data.target_remote_path == target_remote_path
assert data.timestamp == timestamp
def test_upload_data_with_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(5):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment_client.upload_data(data)
data = list(segment_client.list_data())
assert data[0].path == "hello0.txt"
assert data[0].open().read() == b"CONTENT"
assert data[0].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_upload_frame_without_sensor(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name,
is_fusion=True)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
frame = Frame()
local_path = path / "hello0.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
# If not uploading sensor, uploading frame is not allowed
with pytest.raises(GASResponseError):
segment_client.upload_frame(frame, timestamp=0)
gas_client.delete_dataset(dataset_name)
def test_get_url(self):
local_relative_path = Path(__file__).relative_to(Path.cwd())
data = Data(str(local_relative_path))
assert data.get_url() == local_relative_path.resolve().as_uri()
data = Data(__file__)
assert data.get_url() == Path(__file__).resolve().as_uri()
def test_target_remote_path(self):
target_remote_path = "test2.json"
data_object = Data("test.json")
data_object.target_remote_path = target_remote_path
assert data_object.target_remote_path == target_remote_path