def _assert_project_db_row(postgres_db: sa.engine.Engine,
project: Dict[str, Any], **kwargs):
row: Optional[Row] = postgres_db.execute(
f"SELECT * FROM projects WHERE \"uuid\"='{project['uuid']}'").fetchone(
)
expected_db_entries = {
"type": "STANDARD",
"uuid": project["uuid"],
"name": project["name"],
"description": project["description"],
"thumbnail": project["thumbnail"],
"prj_owner": None,
"workbench": project["workbench"],
"published": False,
"access_rights": {},
"dev": project["dev"],
"classifiers": project["classifiers"],
"ui": project["ui"],
"quality": project["quality"],
"creation_date": to_datetime(project["creationDate"]),
"last_change_date": to_datetime(project["lastChangeDate"]),
}
expected_db_entries.update(kwargs)
for k in expected_db_entries:
assert (row[k] == expected_db_entries[k]
), f"project column [{k}] does not correspond"
assert row["last_change_date"] >= row["creation_date"]
def assert_replaced(current_project, update_data):
def _extract(dikt, keys):
return {k: dikt[k] for k in keys}
modified = [
"lastChangeDate",
]
keep = [k for k in update_data.keys() if k not in modified]
assert _extract(current_project, keep) == _extract(update_data, keep)
k = "lastChangeDate"
assert to_datetime(update_data[k]) < to_datetime(current_project[k])
def test_time_utils():
snapshot0 = now_str()
time.sleep(0.5)
snapshot1 = now_str()
now0 = to_datetime(snapshot0)
now1 = to_datetime(snapshot1)
assert now0 < now1
# tests biyective
now_time = datetime.utcnow()
snapshot = now_time.strftime(DATETIME_FORMAT)
assert now_time == datetime.strptime(snapshot, DATETIME_FORMAT)
async def test_new_project(
client, logged_user, expected, computational_system_mock, storage_subsystem_mock,
):
# POST /v0/projects
url = client.app.router["create_projects"].url_for()
assert str(url) == API_PREFIX + "/projects"
# Pre-defined fields imposed by required properties in schema
default_project = {
"uuid": "0000000-invalid-uuid",
"name": "Minimal name",
"description": "this description should not change",
"prjOwner": "me but I will be removed anyway",
"creationDate": now_str(),
"lastChangeDate": now_str(),
"thumbnail": "",
"accessRights": {"12": "some rights"},
"workbench": {},
"tags": [],
}
resp = await client.post(url, json=default_project)
data, error = await assert_status(resp, expected)
if not error:
new_project = data
# updated fields
assert default_project["uuid"] != new_project["uuid"]
assert default_project["prjOwner"] != logged_user["email"]
assert new_project["prjOwner"] == logged_user["email"]
assert to_datetime(default_project["creationDate"]) < to_datetime(
new_project["creationDate"]
)
# invariant fields
for key in new_project.keys():
if key not in ("uuid", "prjOwner", "creationDate", "lastChangeDate"):
assert default_project[key] == new_project[key]
# TODO: validate response using OAS?
# FIXME: cannot delete user until project is deleted. See cascade or too coupled??
# i.e. removing a user, removes all its projects!!
# asyncpg.exceptions.ForeignKeyViolationError: update or delete on table "users"
# violates foreign key constraint "user_to_projects_user_id_fkey" on table "user_to_projects"
await delete_all_projects(client.app)
async def test_new_project_from_template(
client,
logged_user,
template_project,
expected,
computational_system_mock,
storage_subsystem_mock,
):
# POST /v0/projects?from_template={template_uuid}
url = (
client.app.router["create_projects"]
.url_for()
.with_query(from_template=template_project["uuid"])
)
resp = await client.post(url)
data, error = await assert_status(resp, expected)
if not error:
project = data
modified = ["prjOwner", "creationDate", "lastChangeDate", "uuid"]
# different ownership
assert project["prjOwner"] == logged_user["email"]
assert project["prjOwner"] != template_project["prjOwner"]
assert project["accessRights"] == template_project["accessRights"]
# different timestamps
assert to_datetime(template_project["creationDate"]) < to_datetime(
project["creationDate"]
)
assert to_datetime(template_project["lastChangeDate"]) < to_datetime(
project["lastChangeDate"]
)
# different uuids for project and nodes!?
assert project["uuid"] != template_project["uuid"]
# check uuid replacement
for node_name in project["workbench"]:
try:
uuidlib.UUID(node_name)
except ValueError:
pytest.fail("Invalid uuid in workbench node {}".format(node_name))
def _assert_added_project(
exp_project: Dict[str, Any],
added_project: Dict[str, Any],
exp_overrides: Dict[str, Any],
):
original_prj = deepcopy(exp_project)
added_prj = deepcopy(added_project)
# no user so the project owner has a pre-defined value
_DIFFERENT_KEYS = ["creationDate", "lastChangeDate"]
assert all(added_prj[k] != original_prj[k] for k in _DIFFERENT_KEYS)
assert to_datetime(added_prj["creationDate"]) > to_datetime(
exp_project["creationDate"])
assert to_datetime(added_prj["creationDate"]) <= to_datetime(
added_prj["lastChangeDate"])
original_prj.update(exp_overrides)
for k in _DIFFERENT_KEYS:
added_prj.pop(k)
original_prj.pop(k)
# the rest of the keys shall be the same as the original
assert added_prj == original_prj
async def _new_project(
client,
expected_response: Type[web.HTTPException],
logged_user: Dict[str, str],
primary_group: Dict[str, str],
*,
project: Optional[Dict] = None,
from_template: Optional[Dict] = None,
) -> Dict:
# POST /v0/projects
url = client.app.router["create_projects"].url_for()
assert str(url) == f"{API_PREFIX}/projects"
if from_template:
url = url.with_query(from_template=from_template["uuid"])
# Pre-defined fields imposed by required properties in schema
project_data = {}
expected_data = {}
if from_template:
# access rights are replaced
expected_data = deepcopy(from_template)
expected_data["accessRights"] = {}
if not from_template or project:
project_data = {
"uuid": "0000000-invalid-uuid",
"name": "Minimal name",
"description": "this description should not change",
"prjOwner": "me but I will be removed anyway",
"creationDate": now_str(),
"lastChangeDate": now_str(),
"thumbnail": "",
"accessRights": {},
"workbench": {},
"tags": [],
"classifiers": [],
"ui": {},
"dev": {},
"quality": {},
}
if project:
project_data.update(project)
for key in project_data:
expected_data[key] = project_data[key]
if (
key in OVERRIDABLE_DOCUMENT_KEYS
and not project_data[key]
and from_template
):
expected_data[key] = from_template[key]
resp = await client.post(url, json=project_data)
new_project, error = await assert_status(resp, expected_response)
if not error:
# has project state
assert not ProjectState(
**new_project.pop("state")
).locked.value, "Newly created projects should be unlocked"
# updated fields
assert expected_data["uuid"] != new_project["uuid"]
assert (
new_project["prjOwner"] == logged_user["email"]
) # the project owner is assigned the user id e-mail
assert to_datetime(expected_data["creationDate"]) < to_datetime(
new_project["creationDate"]
)
assert to_datetime(expected_data["lastChangeDate"]) < to_datetime(
new_project["lastChangeDate"]
)
# the access rights are set to use the logged user primary group + whatever was inside the project
expected_data["accessRights"].update(
{str(primary_group["gid"]): {"read": True, "write": True, "delete": True}}
)
assert new_project["accessRights"] == expected_data["accessRights"]
# invariant fields
modified_fields = [
"uuid",
"prjOwner",
"creationDate",
"lastChangeDate",
"accessRights",
"workbench" if from_template else None,
"ui" if from_template else None,
]
for key in new_project.keys():
if key not in modified_fields:
assert expected_data[key] == new_project[key]
return new_project
async def test_new_template_from_project(
client: TestClient,
logged_user: Dict[str, Any],
primary_group: Dict[str, str],
all_group: Dict[str, str],
user_project: Dict[str, Any],
expected: ExpectedResponse,
storage_subsystem_mock: MockedStorageSubsystem,
catalog_subsystem_mock: Callable,
project_db_cleaner: None,
):
# POST /v0/projects?as_template={project_uuid}
url = (client.app.router["create_projects"].url_for().with_query(
as_template=user_project["uuid"]))
resp = await client.post(f"{url}")
data, error = await assert_status(resp, expected.created)
if not error:
template_project = data
catalog_subsystem_mock([template_project])
templates, *_ = await _list_projects(client, web.HTTPOk,
{"type": "template"})
assert len(templates) == 1
assert templates[0] == template_project
assert template_project["name"] == user_project["name"]
assert template_project["description"] == user_project["description"]
assert template_project["prjOwner"] == logged_user["email"]
assert template_project["accessRights"] == user_project["accessRights"]
# different timestamps
assert to_datetime(user_project["creationDate"]) < to_datetime(
template_project["creationDate"])
assert to_datetime(user_project["lastChangeDate"]) < to_datetime(
template_project["lastChangeDate"])
# different uuids for project and nodes!?
assert template_project["uuid"] != user_project["uuid"]
# check uuid replacement
for node_name in template_project["workbench"]:
try:
uuidlib.UUID(node_name)
except ValueError:
pytest.fail(
"Invalid uuid in workbench node {}".format(node_name))
# do the same with a body
predefined = {
"uuid": "",
"name": "My super duper new template",
"description": "Some lines from user",
"thumbnail": "",
"prjOwner": "",
"creationDate": "2019-06-03T09:59:31.987Z",
"lastChangeDate": "2019-06-03T09:59:31.987Z",
"workbench": {},
"accessRights": {
str(all_group["gid"]): {
"read": True,
"write": False,
"delete": False
},
},
"tags": [],
"classifiers": [],
}
resp = await client.post(f"{url}", json=predefined)
data, error = await assert_status(resp, expected.created)
if not error:
template_project = data
# uses predefined
assert template_project["name"] == predefined["name"]
assert template_project["description"] == predefined["description"]
assert template_project["prjOwner"] == logged_user["email"]
# the logged in user access rights are added by default
predefined["accessRights"].update({
str(primary_group["gid"]): {
"read": True,
"write": True,
"delete": True
}
})
assert template_project["accessRights"] == predefined["accessRights"]
# different ownership
assert template_project["prjOwner"] == logged_user["email"]
assert template_project["prjOwner"] == user_project["prjOwner"]
# different timestamps
assert to_datetime(user_project["creationDate"]) < to_datetime(
template_project["creationDate"])
assert to_datetime(user_project["lastChangeDate"]) < to_datetime(
template_project["lastChangeDate"])
# different uuids for project and nodes!?
assert template_project["uuid"] != user_project["uuid"]
# check uuid replacement
for node_name in template_project["workbench"]:
try:
uuidlib.UUID(node_name)
except ValueError:
pytest.fail(
"Invalid uuid in workbench node {}".format(node_name))
async def test_patch_user_project_workbench_concurrently(
fake_project: Dict[str, Any],
postgres_db: sa.engine.Engine,
logged_user: Dict[str, Any],
primary_group: Dict[str, str],
db_api: ProjectDBAPI,
number_of_nodes: int,
):
_NUMBER_OF_NODES = number_of_nodes
BASE_UUID = UUID("ccc0839f-93b8-4387-ab16-197281060927")
node_uuids = [
str(uuid5(BASE_UUID, f"{n}")) for n in range(_NUMBER_OF_NODES)
]
# create a project with a lot of nodes
fake_project["workbench"] = {
node_uuids[n]: {
"key": "simcore/services/comp/sleepers",
"version": "1.43.5",
"label": f"I am node {n}",
}
for n in range(_NUMBER_OF_NODES)
}
expected_project = deepcopy(fake_project)
# add the project
original_project = deepcopy(fake_project)
new_project = await db_api.add_project(prj=fake_project,
user_id=logged_user["id"])
_assert_added_project(
original_project,
new_project,
exp_overrides={
"prjOwner": logged_user["email"],
"accessRights": {
str(primary_group["gid"]): {
"read": True,
"write": True,
"delete": True
}
},
},
)
_assert_project_db_row(
postgres_db,
new_project,
prj_owner=logged_user["id"],
access_rights={
str(primary_group["gid"]): {
"read": True,
"write": True,
"delete": True
}
},
)
# patch all the nodes concurrently
randomly_created_outputs = [{
"outputs": {
f"out_{k}": f"{k}"
}
for k in range(randint(1, 10))
} for n in range(_NUMBER_OF_NODES)]
for n in range(_NUMBER_OF_NODES):
expected_project["workbench"][node_uuids[n]].update(
randomly_created_outputs[n])
patched_projects: Tuple[Tuple[Dict[str, Any], Dict[
str, Any]]] = await asyncio.gather(*[
db_api.patch_user_project_workbench(
{node_uuids[n]: randomly_created_outputs[n]},
logged_user["id"],
new_project["uuid"],
) for n in range(_NUMBER_OF_NODES)
])
# NOTE: each returned project contains the project with some updated workbenches
# the ordering is uncontrolled.
# The important thing is that the final result shall contain ALL the changes
for (prj, changed_entries), node_uuid, exp_outputs in zip(
patched_projects, node_uuids, randomly_created_outputs):
assert prj["workbench"][node_uuid]["outputs"] == exp_outputs["outputs"]
assert changed_entries == {
node_uuid: {
"outputs": exp_outputs["outputs"]
}
}
# get the latest change date
latest_change_date = max(
to_datetime(prj["lastChangeDate"]) for prj, _ in patched_projects)
# check the nodes are completely patched as expected
_assert_project_db_row(
postgres_db,
expected_project,
prj_owner=logged_user["id"],
access_rights={
str(primary_group["gid"]): {
"read": True,
"write": True,
"delete": True
}
},
creation_date=to_datetime(new_project["creationDate"]),
last_change_date=latest_change_date,
)
# now concurrently remove the outputs
for n in range(_NUMBER_OF_NODES):
expected_project["workbench"][node_uuids[n]]["outputs"] = {}
patched_projects = await asyncio.gather(*[
db_api.patch_user_project_workbench(
{node_uuids[n]: {
"outputs": {}
}},
logged_user["id"],
new_project["uuid"],
) for n in range(_NUMBER_OF_NODES)
])
# get the latest change date
latest_change_date = max(
to_datetime(prj["lastChangeDate"]) for prj, _ in patched_projects)
# check the nodes are completely patched as expected
_assert_project_db_row(
postgres_db,
expected_project,
prj_owner=logged_user["id"],
access_rights={
str(primary_group["gid"]): {
"read": True,
"write": True,
"delete": True
}
},
creation_date=to_datetime(new_project["creationDate"]),
last_change_date=latest_change_date,
)
# now concurrently remove the outputs
for n in range(_NUMBER_OF_NODES):
expected_project["workbench"][node_uuids[n]]["outputs"] = {}
patched_projects = await asyncio.gather(*[
db_api.patch_user_project_workbench(
{node_uuids[n]: {
"outputs": {}
}},
logged_user["id"],
new_project["uuid"],
) for n in range(_NUMBER_OF_NODES)
])
# get the latest change date
latest_change_date = max(
to_datetime(prj["lastChangeDate"]) for prj, _ in patched_projects)
# check the nodes are completely patched as expected
_assert_project_db_row(
postgres_db,
expected_project,
prj_owner=logged_user["id"],
access_rights={
str(primary_group["gid"]): {
"read": True,
"write": True,
"delete": True
}
},
creation_date=to_datetime(new_project["creationDate"]),
last_change_date=latest_change_date,
)
async def test_new_template_from_project(
client,
logged_user,
user_project,
expected,
computational_system_mock,
storage_subsystem_mock,
):
# POST /v0/projects?as_template={user_uuid}
url = (
client.app.router["create_projects"]
.url_for()
.with_query(as_template=user_project["uuid"])
)
resp = await client.post(url)
data, error = await assert_status(resp, expected)
if not error:
template_project = data
url = client.app.router["list_projects"].url_for().with_query(type="template")
resp = await client.get(url)
templates, _ = await assert_status(resp, web.HTTPOk)
assert len(templates) == 1
assert templates[0] == template_project
assert template_project["name"] == user_project["name"]
assert template_project["description"] == user_project["description"]
assert template_project["prjOwner"] == logged_user["email"]
assert template_project["accessRights"] == user_project["accessRights"]
# different timestamps
assert to_datetime(user_project["creationDate"]) < to_datetime(
template_project["creationDate"]
)
assert to_datetime(user_project["lastChangeDate"]) < to_datetime(
template_project["lastChangeDate"]
)
# different uuids for project and nodes!?
assert template_project["uuid"] != user_project["uuid"]
# check uuid replacement
for node_name in template_project["workbench"]:
try:
uuidlib.UUID(node_name)
except ValueError:
pytest.fail("Invalid uuid in workbench node {}".format(node_name))
# do the same with a body
predefined = {
"uuid": "",
"name": "My super duper new template",
"description": "Some lines from user",
"thumbnail": "",
"prjOwner": "",
"creationDate": "2019-06-03T09:59:31.987Z",
"lastChangeDate": "2019-06-03T09:59:31.987Z",
"workbench": {},
"accessRights": {"12": "rwx"},
"tags": [],
}
resp = await client.post(url, json=predefined)
data, error = await assert_status(resp, expected)
if not error:
template_project = data
# uses predefined
assert template_project["name"] == predefined["name"]
assert template_project["description"] == predefined["description"]
assert template_project["prjOwner"] == logged_user["email"]
assert template_project["accessRights"] == predefined["accessRights"]
modified = [
"prjOwner",
"creationDate",
"lastChangeDate",
"uuid",
"accessRights",
]
# different ownership
assert template_project["prjOwner"] == logged_user["email"]
assert template_project["prjOwner"] == user_project["prjOwner"]
# different timestamps
assert to_datetime(user_project["creationDate"]) < to_datetime(
template_project["creationDate"]
)
assert to_datetime(user_project["lastChangeDate"]) < to_datetime(
template_project["lastChangeDate"]
)
# different uuids for project and nodes!?
assert template_project["uuid"] != user_project["uuid"]
# check uuid replacement
for node_name in template_project["workbench"]:
try:
uuidlib.UUID(node_name)
except ValueError:
pytest.fail("Invalid uuid in workbench node {}".format(node_name))
async def test_new_project_from_template_with_body(
client,
logged_user,
template_project,
expected,
computational_system_mock,
storage_subsystem_mock,
):
# POST /v0/projects?from_template={template_uuid}
url = (
client.app.router["create_projects"]
.url_for()
.with_query(from_template=template_project["uuid"])
)
predefined = {
"uuid": "",
"name": "Sleepers8",
"description": "Some lines from user",
"thumbnail": "",
"prjOwner": "",
"creationDate": "2019-06-03T09:59:31.987Z",
"lastChangeDate": "2019-06-03T09:59:31.987Z",
"accessRights": {"123": "some new access rights"},
"workbench": {},
"tags": [],
}
resp = await client.post(url, json=predefined)
data, error = await assert_status(resp, expected)
if not error:
project = data
# uses predefined
assert project["name"] == predefined["name"]
assert project["description"] == predefined["description"]
modified = ["prjOwner", "creationDate", "lastChangeDate", "uuid"]
# different ownership
assert project["prjOwner"] == logged_user["email"]
assert project["prjOwner"] != template_project["prjOwner"]
# different access rights
assert project["accessRights"] != template_project["accessRights"]
assert project["accessRights"] == predefined["accessRights"]
# different timestamps
assert to_datetime(template_project["creationDate"]) < to_datetime(
project["creationDate"]
)
assert to_datetime(template_project["lastChangeDate"]) < to_datetime(
project["lastChangeDate"]
)
# different uuids for project and nodes!?
assert project["uuid"] != template_project["uuid"]
# check uuid replacement
for node_name in project["workbench"]:
try:
uuidlib.UUID(node_name)
except ValueError:
pytest.fail("Invalid uuid in workbench node {}".format(node_name))
