def test_upload_frame_with_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name,
is_fusion=True)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment_client = dataset_client.get_or_create_segment("segment1")
segment_client.upload_sensor(Sensor.loads(LIDAR_DATA))
path = tmp_path / "sub"
path.mkdir()
for i in range(5):
frame = Frame()
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
frame[LIDAR_DATA["name"]] = data
segment_client.upload_frame(frame, timestamp=i)
frames = list(segment_client.list_frames())
assert len(frames) == 5
assert frames[0][LIDAR_DATA["name"]].path == "hello0.txt"
assert frames[0][LIDAR_DATA["name"]].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_replace_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
local_path = path / "hello0.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
segment_client.upload_file(data.path, data.target_remote_path)
data.label = Label.loads(LABEL)
segment_client.upload_label(data)
# Replace labels
data.label = Label.loads(NEW_LABEL)
segment_client.upload_label(data)
data = list(segment_client.list_data())
assert data[0].path == "hello0.txt"
assert data[0].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_delete_data(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(10):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
segment_client.upload_data(data)
segment_client.delete_data("hello0.txt")
data_paths = list(segment_client.list_data_paths())
assert "hello0.txt" not in data_paths
segment_client.delete_data(segment_client.list_data_paths())
data = list(segment_client.list_data())
assert len(data) == 0
gas_client.delete_dataset(dataset_name)
def test_upload_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
dataset_client.upload_catalog(Catalog.loads(CATALOG))
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
local_path = path / "hello0.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
data.label = Label.loads(LABEL)
# If not uploading file, uploading label is not allowed
with pytest.raises(GASResponseError):
segment_client.upload_label(data)
# Uploading files
segment_client.upload_file(data.path, data.target_remote_path)
data.label = Label.loads(WRONG_LABEL)
# Uploading wrong label is not allowed
with pytest.raises(GASResponseError):
segment_client.upload_label(data)
data.label = Label.loads(LABEL)
segment_client.upload_label(data)
data = list(segment_client.list_data())
assert data[0].path == "hello0.txt"
assert data[0].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_list_data_paths(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(5):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
segment_client.upload_file(local_path=str(local_path))
# Add other files in reverse order
for i in reversed(range(5)):
local_path = path / f"goodbye{i}.txt"
local_path.write_text("CONTENT")
segment_client.upload_file(local_path=str(local_path))
data_paths = list(segment_client.list_data_paths())
assert data_paths[0] == "goodbye0.txt"
assert data_paths[5] == "hello0.txt"
gas_client.delete_dataset(dataset_name)
def test_replace_file(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(5):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
segment_client.upload_file(local_path=str(local_path))
# Replace files
for i in range(5):
local_path = path / f"hello{i}.txt"
local_path.write_text("ADD CONTENT")
segment_client.upload_file(local_path=str(local_path))
data = list(segment_client.list_data())
assert data[0].path == "hello0.txt"
assert data[0].open().read() == b"ADD CONTENT"
assert not data[0].label
gas_client.delete_dataset(dataset_name)
def test_upload_frame_with_order(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name,
is_fusion=True)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
segment_client.upload_sensor(Sensor.loads(LIDAR_DATA))
path = tmp_path / "sub"
path.mkdir()
# If noe setting frame id in frame, set timestamp(order) when uploading
for i in reversed(range(5)):
frame = Frame()
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
segment_client.upload_frame(frame, timestamp=i)
# Set frame id in frame
for i in range(5, 10):
frame = Frame(frame_id=str(ulid.from_timestamp(i)))
local_path = path / f"goodbye{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
segment_client.upload_frame(frame)
# Both setting frame id in frame and set timestamp(order) when uploading are not allowed
i = 10
frame = Frame(frame_id=str(ulid.from_timestamp(i)))
local_path = path / f"goodbye{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
with pytest.raises(TypeError):
segment_client.upload_frame(frame, timestamp=i)
# Neither setting frame id in frame nor set timestamp(order) when uploading is not allowed
frame = Frame()
local_path = path / f"goodbye{i}.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
with pytest.raises(TypeError):
segment_client.upload_frame(frame)
frames = list(segment_client.list_frames())
assert len(frames) == 10
assert frames[0][LIDAR_DATA["name"]].path == "hello0.txt"
assert frames[5][LIDAR_DATA["name"]].path == "goodbye5.txt"
assert not frames[0][LIDAR_DATA["name"]].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_sensor(self, accesskey, url):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name,
is_fusion=True)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
segment_client.upload_sensor(Sensor.loads(LIDAR_DATA))
sensors = list(segment_client.list_sensors())
assert sensors
# todo: match the input and output sensors
segment_client.delete_sensor(LIDAR_DATA["name"])
sensors = list(segment_client.list_sensors())
assert len(sensors) == 0
gas_client.delete_dataset(dataset_name)
def test_upload_label_without_catalog(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
local_path = path / "hello0.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
segment_client.upload_file(data.path, data.target_remote_path)
# If not uploading catalog, uploading label is not allowed
data.label = Label.loads(LABEL)
with pytest.raises(GASResponseError):
segment_client.upload_label(data)
gas_client.delete_dataset(dataset_name)
def test_upload_data_without_label(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
for i in range(5):
local_path = path / f"hello{i}.txt"
local_path.write_text("CONTENT")
segment_client.upload_data(Data(local_path=str(local_path)))
data = list(segment_client.list_data())
assert data[0].path == "hello0.txt"
assert data[0].open().read() == b"CONTENT"
assert not data[0].label
# todo: match the input and output label
gas_client.delete_dataset(dataset_name)
def test_upload_frame_without_sensor(self, accesskey, url, tmp_path):
gas_client = GAS(access_key=accesskey, url=url)
dataset_name = get_random_dataset_name()
dataset_client = gas_client.create_dataset(dataset_name,
is_fusion=True)
dataset_client.create_draft("draft-1")
segment_client = dataset_client.get_or_create_segment("segment1")
path = tmp_path / "sub"
path.mkdir()
frame = Frame()
local_path = path / "hello0.txt"
local_path.write_text("CONTENT")
data = Data(local_path=str(local_path))
frame[LIDAR_DATA["name"]] = data
# If not uploading sensor, uploading frame is not allowed
with pytest.raises(GASResponseError):
segment_client.upload_frame(frame, timestamp=0)
gas_client.delete_dataset(dataset_name)