def test_get_specific_dataset_metadata(client, database, cleanup):
new_role = create_role(*owner_role)
database.session.add(new_role)
new_role = create_role(*user_role)
database.session.add(new_role)
new_user = create_user(*user1)
database.session.add(new_user)
new_user = create_user(*user2)
database.session.add(new_user)
database.session.commit()
storage = DiskObjectStore(database)
df_metadata = create_dataset(dataset)
token = jwt.encode({"id": 1}, app.config["SECRET_KEY"])
headers = {
"token": token.decode("UTF-8"),
}
result = client.get(
"/data-centric/datasets/{}".format(df_metadata["id"]),
headers=headers,
content_type="application/json",
)
assert result.status_code == 200
assert result.get_json()["id"] == df_metadata["id"]
assert result.get_json()["tags"] == df_metadata["tags"]
assert result.get_json()["name"] == df_metadata["name"]
assert result.get_json()["manifest"] == df_metadata["manifest"]
assert result.get_json()["description"] == df_metadata["description"]
def test_clear(client, database, cleanup):
disk_store = DiskObjectStore(database)
_id = UID()
storable = StorableObject(id=_id, data=tensor1)
disk_store.__setitem__(_id, storable)
bin_obj = database.session.query(BinObject).get(str(_id.value))
metadata = (database.session.query(ObjectMetadata).filter_by(
obj=str(_id.value)).one())
assert bin_obj is not None
assert th.all(th.eq(bin_obj.object, tensor1))
assert metadata is not None
assert metadata.tags == []
assert metadata.description == ""
assert metadata.read_permissions == {}
assert metadata.search_permissions == {}
retrieved = disk_store.get_object(_id)
assert th.all(th.eq(retrieved.data, tensor1))
assert retrieved.id == _id
disk_store.clear()
assert database.session.query(BinObject).count() == 0
assert database.session.query(ObjectMetadata).count() == 0
def test_delete(client, database, cleanup):
disk_store = DiskObjectStore(database)
_id = UID()
storable = StorableObject(id=_id, data=tensor1)
disk_store.__setitem__(_id, storable)
bin_obj = database.session.query(BinObject).get(str(_id.value))
metadata = (database.session.query(ObjectMetadata).filter_by(
obj=str(_id.value)).one())
assert bin_obj is not None
assert th.all(th.eq(bin_obj.object, tensor1))
assert metadata is not None
assert metadata.tags == []
assert metadata.description == ""
assert metadata.read_permissions == {}
assert metadata.search_permissions == {}
disk_store.delete(_id)
assert database.session.query(BinObject).get(str(_id.value)) is None
with pytest.raises(NoResultFound) as e_info:
database.session.query(ObjectMetadata).filter_by(
obj=str(_id.value)).one()
def test_delete_dataset(client, database, cleanup):
new_role = create_role(*owner_role)
database.session.add(new_role)
new_role = create_role(*user_role)
database.session.add(new_role)
new_user = create_user(*user1)
database.session.add(new_user)
new_user = create_user(*user2)
database.session.add(new_user)
database.session.commit()
storage = DiskObjectStore(database)
df_json1 = create_dataset(dataset)
_id = df_json1["id"]
token = jwt.encode({"id": 1}, app.config["SECRET_KEY"])
headers = {
"token": token.decode("UTF-8"),
}
assert database.session.query(DatasetGroup).filter_by(
dataset=_id).all() is not None
storable_ids = (database.session.query(
DatasetGroup.bin_object).filter_by(dataset=_id).all())
storable_ids = [x[0] for x in storable_ids]
assert database.session.query(JsonObject).get(_id) is not None
assert database.session.query(JsonObject).get(_id).binary is not None
assert (database.session.query(JsonObject).get(_id).binary["description"]
== dataset["description"])
assert database.session.query(JsonObject).get(
_id).binary["tags"] == dataset["tags"]
result = client.delete(
"/data-centric/datasets/{}".format(_id),
headers=headers,
content_type="application/json",
)
assert result.status_code == 204
for strbl_id in storable_ids:
assert db.session.query(BinObject).filter_by(
id=strbl_id).first() is None
assert db.session.query(ObjectMetadata).filter_by(
id=strbl_id).first() is None
assert database.session.query(DatasetGroup).filter_by(
dataset=_id).all() == []
assert database.session.query(JsonObject).get(_id) is None
def test_update_request(client, database, cleanup):
new_role = create_role(*owner_role)
database.session.add(new_role)
new_user = create_user(*user1)
database.session.add(new_user)
database.session.commit()
storage = DiskObjectStore(database)
dataset_json = create_dataset(dataset)
token = jwt.encode({"id": 1}, app.config["SECRET_KEY"])
headers = {
"token": token.decode("UTF-8"),
}
object_id = dataset_json["tensors"]["train"]["id"]
reason = "this is a sample reason"
request_type = "budget"
create = client.post(
"/data-centric/requests",
json={
"object_id": object_id,
"reason": reason,
"request_type": request_type,
},
headers=headers,
)
status = "accepted"
request_id = create.get_json()["id"]
client.put(
"/data-centric/requests/" + request_id,
json={"status": status},
headers=headers,
content_type="application/json",
)
result = client.get(
"/data-centric/requests/" + request_id,
headers=headers,
content_type="application/json",
)
response = result.get_json()
assert result.status_code == 200
assert response["id"] == request_id
assert response["status"] == "accepted"
def test_create_duplicate_fail(client, database, cleanup):
new_role = create_role(*owner_role)
database.session.add(new_role)
new_user = create_user(*user1)
database.session.add(new_user)
database.session.commit()
storage = DiskObjectStore(database)
dataset_json = create_dataset(dataset)
token = jwt.encode({"id": 1}, app.config["SECRET_KEY"])
headers = {
"token": token.decode("UTF-8"),
}
object_id = dataset_json["tensors"]["train"]["id"]
reason = "sample reason"
request_type = "permissions"
result1 = client.post(
"/data-centric/requests",
json={
"object_id": object_id,
"reason": reason,
"request_type": request_type,
},
headers=headers,
)
result1 = client.get("/data-centric/requests",
headers=headers,
content_type="application/json")
result2 = client.post(
"/data-centric/requests",
json={
"object_id": object_id,
"reason": reason,
"request_type": request_type,
},
headers=headers,
)
assert result1.status_code == 200
assert object_id in [el["object_id"] for el in result1.get_json()]
assert reason in [el["reason"] for el in result1.get_json()]
assert request_type in [el["request_type"] for el in result1.get_json()]
assert result2.status_code == 403
def test_get_all_datasets_metadata(client, database, cleanup):
new_role = create_role(*owner_role)
database.session.add(new_role)
new_role = create_role(*user_role)
database.session.add(new_role)
new_user = create_user(*user1)
database.session.add(new_user)
new_user = create_user(*user2)
database.session.add(new_user)
database.session.commit()
new_dataset = {
"name": "Dummy Dataset 1",
"description": "Lorem ipsum dolor",
"manifest": "Etiam vestibulum velit a tellus aliquet varius",
"tags": ["#hashtag", "#dummy"],
"created_at": "05/12/2019",
"tensors": {
"train": tensor2.copy()
},
}
storage = DiskObjectStore(database)
df_json1 = create_dataset(dataset)
df_json2 = create_dataset(new_dataset)
token = jwt.encode({"id": 1}, app.config["SECRET_KEY"])
headers = {
"token": token.decode("UTF-8"),
}
result = client.get("/data-centric/datasets",
headers=headers,
content_type="application/json")
assert result.status_code == 200
assert df_json1["id"] in [el["id"] for el in result.get_json()]
assert df_json1["description"] in [
el["description"] for el in result.get_json()
]
assert df_json1["manifest"] in [el["manifest"] for el in result.get_json()]
assert df_json2["id"] in [el["id"] for el in result.get_json()]
assert df_json2["description"] in [
el["description"] for el in result.get_json()
]
assert df_json2["manifest"] in [el["manifest"] for el in result.get_json()]
def test_keys(client, database, cleanup):
disk_store = DiskObjectStore(database)
a_id = UID()
id1 = UID()
id2 = UID()
storable1 = StorableObject(id=id1, data=tensor1)
disk_store.__setitem__(id1, storable1)
storable2 = StorableObject(id=id2, data=tensor2)
disk_store.__setitem__(id2, storable2)
keys = disk_store.keys()
assert any(id1 == k for k in keys)
assert any(id2 == k for k in keys)
assert len(keys) == 2
def test_values(client, database, cleanup):
disk_store = DiskObjectStore(database)
a_id = UID()
id1 = UID()
id2 = UID()
storable1 = StorableObject(id=id1, data=tensor1)
disk_store.__setitem__(id1, storable1)
storable2 = StorableObject(id=id2, data=tensor2)
disk_store.__setitem__(id2, storable2)
values = disk_store.values()
values_data = [v.data for v in values]
assert any(th.all(th.eq(tensor1, v)) for v in values_data)
assert any(th.all(th.eq(tensor2, v)) for v in values_data)
assert len(values_data) == 2
def test_get_objects_of_type(client, database, cleanup):
disk_store = DiskObjectStore(database)
a_id = UID()
id1 = UID()
id2 = UID()
storable1 = StorableObject(id=id1, data=tensor1)
disk_store.__setitem__(id1, storable1)
storable2 = StorableObject(id=id2, data=tensor2)
disk_store.__setitem__(id2, storable2)
selected = disk_store.get_objects_of_type(th.Tensor)
selected_data = [x.data for x in selected]
assert any(th.all(th.eq(tensor1, d_)) for d_ in selected_data)
assert any(th.all(th.eq(tensor2, d_)) for d_ in selected_data)
assert len(selected_data) == 2
def test__contains__fail(client, database, cleanup):
disk_store = DiskObjectStore(database)
_id = UID()
retrieved = disk_store.__contains__(_id)
assert not retrieved
def test_update_dataset(client, database, cleanup):
new_role = create_role(*owner_role)
database.session.add(new_role)
new_role = create_role(*user_role)
database.session.add(new_role)
new_user = create_user(*user1)
database.session.add(new_user)
new_user = create_user(*user2)
database.session.add(new_user)
database.session.commit()
new_dataset = {
"name": "Dummy Dataset 1",
"description": "Lorem ipsum dolor",
"manifest": "Etiam vestibulum velit a tellus aliquet varius",
"tags": ["#tensor", "#dummy1"],
"created_at": "19/06/1972",
"tensors": {
"train": tensor2.copy()
},
}
storage = DiskObjectStore(database)
df_json1 = create_dataset(dataset)
token = jwt.encode({"id": 1}, app.config["SECRET_KEY"])
headers = {
"token": token.decode("UTF-8"),
}
assert (database.session.query(DatasetGroup).filter_by(
dataset=df_json1["id"]).all() is not None)
assert database.session.query(JsonObject).get(df_json1["id"]) is not None
assert database.session.query(JsonObject).get(
df_json1["id"]).binary == df_json1
result = client.put(
"/data-centric/datasets/{}".format(df_json1["id"]),
data=dumps(new_dataset),
headers=headers,
content_type="application/json",
)
assert result.status_code == 200
assert result.get_json()["id"] == df_json1["id"]
assert (database.session.query(DatasetGroup).filter_by(
dataset=df_json1["id"]).all() is not None)
assert (len(
database.session.query(DatasetGroup).filter_by(
dataset=df_json1["id"]).all()) == 1)
assert database.session.query(JsonObject).get(df_json1["id"]) is not None
metadata = database.session.query(JsonObject).get(df_json1["id"])
assert metadata is not None
metadata = metadata.binary
assert metadata["description"] == new_dataset["description"]
assert metadata["manifest"] == new_dataset["manifest"]
assert metadata["created_at"] == new_dataset["created_at"]
assert metadata["tags"] == new_dataset["tags"]
assert metadata["name"] == new_dataset["name"]