def _assert_project_db_row(postgres_db: sa.engine.Engine, project: Dict[str, Any], **kwargs): row: Optional[Row] = postgres_db.execute( f"SELECT * FROM projects WHERE \"uuid\"='{project['uuid']}'").fetchone( ) expected_db_entries = { "type": "STANDARD", "uuid": project["uuid"], "name": project["name"], "description": project["description"], "thumbnail": project["thumbnail"], "prj_owner": None, "workbench": project["workbench"], "published": False, "access_rights": {}, "dev": project["dev"], "classifiers": project["classifiers"], "ui": project["ui"], "quality": project["quality"], "creation_date": to_datetime(project["creationDate"]), "last_change_date": to_datetime(project["lastChangeDate"]), } expected_db_entries.update(kwargs) for k in expected_db_entries: assert (row[k] == expected_db_entries[k] ), f"project column [{k}] does not correspond" assert row["last_change_date"] >= row["creation_date"]
def assert_replaced(current_project, update_data): def _extract(dikt, keys): return {k: dikt[k] for k in keys} modified = [ "lastChangeDate", ] keep = [k for k in update_data.keys() if k not in modified] assert _extract(current_project, keep) == _extract(update_data, keep) k = "lastChangeDate" assert to_datetime(update_data[k]) < to_datetime(current_project[k])
def test_time_utils(): snapshot0 = now_str() time.sleep(0.5) snapshot1 = now_str() now0 = to_datetime(snapshot0) now1 = to_datetime(snapshot1) assert now0 < now1 # tests biyective now_time = datetime.utcnow() snapshot = now_time.strftime(DATETIME_FORMAT) assert now_time == datetime.strptime(snapshot, DATETIME_FORMAT)
async def test_new_project( client, logged_user, expected, computational_system_mock, storage_subsystem_mock, ): # POST /v0/projects url = client.app.router["create_projects"].url_for() assert str(url) == API_PREFIX + "/projects" # Pre-defined fields imposed by required properties in schema default_project = { "uuid": "0000000-invalid-uuid", "name": "Minimal name", "description": "this description should not change", "prjOwner": "me but I will be removed anyway", "creationDate": now_str(), "lastChangeDate": now_str(), "thumbnail": "", "accessRights": {"12": "some rights"}, "workbench": {}, "tags": [], } resp = await client.post(url, json=default_project) data, error = await assert_status(resp, expected) if not error: new_project = data # updated fields assert default_project["uuid"] != new_project["uuid"] assert default_project["prjOwner"] != logged_user["email"] assert new_project["prjOwner"] == logged_user["email"] assert to_datetime(default_project["creationDate"]) < to_datetime( new_project["creationDate"] ) # invariant fields for key in new_project.keys(): if key not in ("uuid", "prjOwner", "creationDate", "lastChangeDate"): assert default_project[key] == new_project[key] # TODO: validate response using OAS? # FIXME: cannot delete user until project is deleted. See cascade or too coupled?? # i.e. removing a user, removes all its projects!! # asyncpg.exceptions.ForeignKeyViolationError: update or delete on table "users" # violates foreign key constraint "user_to_projects_user_id_fkey" on table "user_to_projects" await delete_all_projects(client.app)
async def test_new_project_from_template( client, logged_user, template_project, expected, computational_system_mock, storage_subsystem_mock, ): # POST /v0/projects?from_template={template_uuid} url = ( client.app.router["create_projects"] .url_for() .with_query(from_template=template_project["uuid"]) ) resp = await client.post(url) data, error = await assert_status(resp, expected) if not error: project = data modified = ["prjOwner", "creationDate", "lastChangeDate", "uuid"] # different ownership assert project["prjOwner"] == logged_user["email"] assert project["prjOwner"] != template_project["prjOwner"] assert project["accessRights"] == template_project["accessRights"] # different timestamps assert to_datetime(template_project["creationDate"]) < to_datetime( project["creationDate"] ) assert to_datetime(template_project["lastChangeDate"]) < to_datetime( project["lastChangeDate"] ) # different uuids for project and nodes!? assert project["uuid"] != template_project["uuid"] # check uuid replacement for node_name in project["workbench"]: try: uuidlib.UUID(node_name) except ValueError: pytest.fail("Invalid uuid in workbench node {}".format(node_name))
def _assert_added_project( exp_project: Dict[str, Any], added_project: Dict[str, Any], exp_overrides: Dict[str, Any], ): original_prj = deepcopy(exp_project) added_prj = deepcopy(added_project) # no user so the project owner has a pre-defined value _DIFFERENT_KEYS = ["creationDate", "lastChangeDate"] assert all(added_prj[k] != original_prj[k] for k in _DIFFERENT_KEYS) assert to_datetime(added_prj["creationDate"]) > to_datetime( exp_project["creationDate"]) assert to_datetime(added_prj["creationDate"]) <= to_datetime( added_prj["lastChangeDate"]) original_prj.update(exp_overrides) for k in _DIFFERENT_KEYS: added_prj.pop(k) original_prj.pop(k) # the rest of the keys shall be the same as the original assert added_prj == original_prj
async def _new_project( client, expected_response: Type[web.HTTPException], logged_user: Dict[str, str], primary_group: Dict[str, str], *, project: Optional[Dict] = None, from_template: Optional[Dict] = None, ) -> Dict: # POST /v0/projects url = client.app.router["create_projects"].url_for() assert str(url) == f"{API_PREFIX}/projects" if from_template: url = url.with_query(from_template=from_template["uuid"]) # Pre-defined fields imposed by required properties in schema project_data = {} expected_data = {} if from_template: # access rights are replaced expected_data = deepcopy(from_template) expected_data["accessRights"] = {} if not from_template or project: project_data = { "uuid": "0000000-invalid-uuid", "name": "Minimal name", "description": "this description should not change", "prjOwner": "me but I will be removed anyway", "creationDate": now_str(), "lastChangeDate": now_str(), "thumbnail": "", "accessRights": {}, "workbench": {}, "tags": [], "classifiers": [], "ui": {}, "dev": {}, "quality": {}, } if project: project_data.update(project) for key in project_data: expected_data[key] = project_data[key] if ( key in OVERRIDABLE_DOCUMENT_KEYS and not project_data[key] and from_template ): expected_data[key] = from_template[key] resp = await client.post(url, json=project_data) new_project, error = await assert_status(resp, expected_response) if not error: # has project state assert not ProjectState( **new_project.pop("state") ).locked.value, "Newly created projects should be unlocked" # updated fields assert expected_data["uuid"] != new_project["uuid"] assert ( new_project["prjOwner"] == logged_user["email"] ) # the project owner is assigned the user id e-mail assert to_datetime(expected_data["creationDate"]) < to_datetime( new_project["creationDate"] ) assert to_datetime(expected_data["lastChangeDate"]) < to_datetime( new_project["lastChangeDate"] ) # the access rights are set to use the logged user primary group + whatever was inside the project expected_data["accessRights"].update( {str(primary_group["gid"]): {"read": True, "write": True, "delete": True}} ) assert new_project["accessRights"] == expected_data["accessRights"] # invariant fields modified_fields = [ "uuid", "prjOwner", "creationDate", "lastChangeDate", "accessRights", "workbench" if from_template else None, "ui" if from_template else None, ] for key in new_project.keys(): if key not in modified_fields: assert expected_data[key] == new_project[key] return new_project
async def test_new_template_from_project( client: TestClient, logged_user: Dict[str, Any], primary_group: Dict[str, str], all_group: Dict[str, str], user_project: Dict[str, Any], expected: ExpectedResponse, storage_subsystem_mock: MockedStorageSubsystem, catalog_subsystem_mock: Callable, project_db_cleaner: None, ): # POST /v0/projects?as_template={project_uuid} url = (client.app.router["create_projects"].url_for().with_query( as_template=user_project["uuid"])) resp = await client.post(f"{url}") data, error = await assert_status(resp, expected.created) if not error: template_project = data catalog_subsystem_mock([template_project]) templates, *_ = await _list_projects(client, web.HTTPOk, {"type": "template"}) assert len(templates) == 1 assert templates[0] == template_project assert template_project["name"] == user_project["name"] assert template_project["description"] == user_project["description"] assert template_project["prjOwner"] == logged_user["email"] assert template_project["accessRights"] == user_project["accessRights"] # different timestamps assert to_datetime(user_project["creationDate"]) < to_datetime( template_project["creationDate"]) assert to_datetime(user_project["lastChangeDate"]) < to_datetime( template_project["lastChangeDate"]) # different uuids for project and nodes!? assert template_project["uuid"] != user_project["uuid"] # check uuid replacement for node_name in template_project["workbench"]: try: uuidlib.UUID(node_name) except ValueError: pytest.fail( "Invalid uuid in workbench node {}".format(node_name)) # do the same with a body predefined = { "uuid": "", "name": "My super duper new template", "description": "Some lines from user", "thumbnail": "", "prjOwner": "", "creationDate": "2019-06-03T09:59:31.987Z", "lastChangeDate": "2019-06-03T09:59:31.987Z", "workbench": {}, "accessRights": { str(all_group["gid"]): { "read": True, "write": False, "delete": False }, }, "tags": [], "classifiers": [], } resp = await client.post(f"{url}", json=predefined) data, error = await assert_status(resp, expected.created) if not error: template_project = data # uses predefined assert template_project["name"] == predefined["name"] assert template_project["description"] == predefined["description"] assert template_project["prjOwner"] == logged_user["email"] # the logged in user access rights are added by default predefined["accessRights"].update({ str(primary_group["gid"]): { "read": True, "write": True, "delete": True } }) assert template_project["accessRights"] == predefined["accessRights"] # different ownership assert template_project["prjOwner"] == logged_user["email"] assert template_project["prjOwner"] == user_project["prjOwner"] # different timestamps assert to_datetime(user_project["creationDate"]) < to_datetime( template_project["creationDate"]) assert to_datetime(user_project["lastChangeDate"]) < to_datetime( template_project["lastChangeDate"]) # different uuids for project and nodes!? assert template_project["uuid"] != user_project["uuid"] # check uuid replacement for node_name in template_project["workbench"]: try: uuidlib.UUID(node_name) except ValueError: pytest.fail( "Invalid uuid in workbench node {}".format(node_name))
async def test_patch_user_project_workbench_concurrently( fake_project: Dict[str, Any], postgres_db: sa.engine.Engine, logged_user: Dict[str, Any], primary_group: Dict[str, str], db_api: ProjectDBAPI, number_of_nodes: int, ): _NUMBER_OF_NODES = number_of_nodes BASE_UUID = UUID("ccc0839f-93b8-4387-ab16-197281060927") node_uuids = [ str(uuid5(BASE_UUID, f"{n}")) for n in range(_NUMBER_OF_NODES) ] # create a project with a lot of nodes fake_project["workbench"] = { node_uuids[n]: { "key": "simcore/services/comp/sleepers", "version": "1.43.5", "label": f"I am node {n}", } for n in range(_NUMBER_OF_NODES) } expected_project = deepcopy(fake_project) # add the project original_project = deepcopy(fake_project) new_project = await db_api.add_project(prj=fake_project, user_id=logged_user["id"]) _assert_added_project( original_project, new_project, exp_overrides={ "prjOwner": logged_user["email"], "accessRights": { str(primary_group["gid"]): { "read": True, "write": True, "delete": True } }, }, ) _assert_project_db_row( postgres_db, new_project, prj_owner=logged_user["id"], access_rights={ str(primary_group["gid"]): { "read": True, "write": True, "delete": True } }, ) # patch all the nodes concurrently randomly_created_outputs = [{ "outputs": { f"out_{k}": f"{k}" } for k in range(randint(1, 10)) } for n in range(_NUMBER_OF_NODES)] for n in range(_NUMBER_OF_NODES): expected_project["workbench"][node_uuids[n]].update( randomly_created_outputs[n]) patched_projects: Tuple[Tuple[Dict[str, Any], Dict[ str, Any]]] = await asyncio.gather(*[ db_api.patch_user_project_workbench( {node_uuids[n]: randomly_created_outputs[n]}, logged_user["id"], new_project["uuid"], ) for n in range(_NUMBER_OF_NODES) ]) # NOTE: each returned project contains the project with some updated workbenches # the ordering is uncontrolled. # The important thing is that the final result shall contain ALL the changes for (prj, changed_entries), node_uuid, exp_outputs in zip( patched_projects, node_uuids, randomly_created_outputs): assert prj["workbench"][node_uuid]["outputs"] == exp_outputs["outputs"] assert changed_entries == { node_uuid: { "outputs": exp_outputs["outputs"] } } # get the latest change date latest_change_date = max( to_datetime(prj["lastChangeDate"]) for prj, _ in patched_projects) # check the nodes are completely patched as expected _assert_project_db_row( postgres_db, expected_project, prj_owner=logged_user["id"], access_rights={ str(primary_group["gid"]): { "read": True, "write": True, "delete": True } }, creation_date=to_datetime(new_project["creationDate"]), last_change_date=latest_change_date, ) # now concurrently remove the outputs for n in range(_NUMBER_OF_NODES): expected_project["workbench"][node_uuids[n]]["outputs"] = {} patched_projects = await asyncio.gather(*[ db_api.patch_user_project_workbench( {node_uuids[n]: { "outputs": {} }}, logged_user["id"], new_project["uuid"], ) for n in range(_NUMBER_OF_NODES) ]) # get the latest change date latest_change_date = max( to_datetime(prj["lastChangeDate"]) for prj, _ in patched_projects) # check the nodes are completely patched as expected _assert_project_db_row( postgres_db, expected_project, prj_owner=logged_user["id"], access_rights={ str(primary_group["gid"]): { "read": True, "write": True, "delete": True } }, creation_date=to_datetime(new_project["creationDate"]), last_change_date=latest_change_date, ) # now concurrently remove the outputs for n in range(_NUMBER_OF_NODES): expected_project["workbench"][node_uuids[n]]["outputs"] = {} patched_projects = await asyncio.gather(*[ db_api.patch_user_project_workbench( {node_uuids[n]: { "outputs": {} }}, logged_user["id"], new_project["uuid"], ) for n in range(_NUMBER_OF_NODES) ]) # get the latest change date latest_change_date = max( to_datetime(prj["lastChangeDate"]) for prj, _ in patched_projects) # check the nodes are completely patched as expected _assert_project_db_row( postgres_db, expected_project, prj_owner=logged_user["id"], access_rights={ str(primary_group["gid"]): { "read": True, "write": True, "delete": True } }, creation_date=to_datetime(new_project["creationDate"]), last_change_date=latest_change_date, )
async def test_new_template_from_project( client, logged_user, user_project, expected, computational_system_mock, storage_subsystem_mock, ): # POST /v0/projects?as_template={user_uuid} url = ( client.app.router["create_projects"] .url_for() .with_query(as_template=user_project["uuid"]) ) resp = await client.post(url) data, error = await assert_status(resp, expected) if not error: template_project = data url = client.app.router["list_projects"].url_for().with_query(type="template") resp = await client.get(url) templates, _ = await assert_status(resp, web.HTTPOk) assert len(templates) == 1 assert templates[0] == template_project assert template_project["name"] == user_project["name"] assert template_project["description"] == user_project["description"] assert template_project["prjOwner"] == logged_user["email"] assert template_project["accessRights"] == user_project["accessRights"] # different timestamps assert to_datetime(user_project["creationDate"]) < to_datetime( template_project["creationDate"] ) assert to_datetime(user_project["lastChangeDate"]) < to_datetime( template_project["lastChangeDate"] ) # different uuids for project and nodes!? assert template_project["uuid"] != user_project["uuid"] # check uuid replacement for node_name in template_project["workbench"]: try: uuidlib.UUID(node_name) except ValueError: pytest.fail("Invalid uuid in workbench node {}".format(node_name)) # do the same with a body predefined = { "uuid": "", "name": "My super duper new template", "description": "Some lines from user", "thumbnail": "", "prjOwner": "", "creationDate": "2019-06-03T09:59:31.987Z", "lastChangeDate": "2019-06-03T09:59:31.987Z", "workbench": {}, "accessRights": {"12": "rwx"}, "tags": [], } resp = await client.post(url, json=predefined) data, error = await assert_status(resp, expected) if not error: template_project = data # uses predefined assert template_project["name"] == predefined["name"] assert template_project["description"] == predefined["description"] assert template_project["prjOwner"] == logged_user["email"] assert template_project["accessRights"] == predefined["accessRights"] modified = [ "prjOwner", "creationDate", "lastChangeDate", "uuid", "accessRights", ] # different ownership assert template_project["prjOwner"] == logged_user["email"] assert template_project["prjOwner"] == user_project["prjOwner"] # different timestamps assert to_datetime(user_project["creationDate"]) < to_datetime( template_project["creationDate"] ) assert to_datetime(user_project["lastChangeDate"]) < to_datetime( template_project["lastChangeDate"] ) # different uuids for project and nodes!? assert template_project["uuid"] != user_project["uuid"] # check uuid replacement for node_name in template_project["workbench"]: try: uuidlib.UUID(node_name) except ValueError: pytest.fail("Invalid uuid in workbench node {}".format(node_name))
async def test_new_project_from_template_with_body( client, logged_user, template_project, expected, computational_system_mock, storage_subsystem_mock, ): # POST /v0/projects?from_template={template_uuid} url = ( client.app.router["create_projects"] .url_for() .with_query(from_template=template_project["uuid"]) ) predefined = { "uuid": "", "name": "Sleepers8", "description": "Some lines from user", "thumbnail": "", "prjOwner": "", "creationDate": "2019-06-03T09:59:31.987Z", "lastChangeDate": "2019-06-03T09:59:31.987Z", "accessRights": {"123": "some new access rights"}, "workbench": {}, "tags": [], } resp = await client.post(url, json=predefined) data, error = await assert_status(resp, expected) if not error: project = data # uses predefined assert project["name"] == predefined["name"] assert project["description"] == predefined["description"] modified = ["prjOwner", "creationDate", "lastChangeDate", "uuid"] # different ownership assert project["prjOwner"] == logged_user["email"] assert project["prjOwner"] != template_project["prjOwner"] # different access rights assert project["accessRights"] != template_project["accessRights"] assert project["accessRights"] == predefined["accessRights"] # different timestamps assert to_datetime(template_project["creationDate"]) < to_datetime( project["creationDate"] ) assert to_datetime(template_project["lastChangeDate"]) < to_datetime( project["lastChangeDate"] ) # different uuids for project and nodes!? assert project["uuid"] != template_project["uuid"] # check uuid replacement for node_name in project["workbench"]: try: uuidlib.UUID(node_name) except ValueError: pytest.fail("Invalid uuid in workbench node {}".format(node_name))