def test_build_UserListItem_model(fake_user: User, faker: Faker): # model for response payload of List method https://google.aip.dev/132) # Typically a light version of the Get model _UserListItem = copy_model( UserGet, name="UserListItem", exclude={"display_name"}, skip_validators=True, ) assert _trim_descriptions(UserListItem.schema()) == _trim_descriptions( _UserListItem.schema()) # to build the pagination model, simply apply the Page generic assert _trim_descriptions( Page[_UserListItem].schema()) == _trim_descriptions( Page[UserListItem].schema()) # parse stored data item_user = _UserListItem.parse_obj(fake_user).dict(exclude_unset=True) page: PageDict = paginate_data( chunk=[item_user], request_url=URL(faker.url()).with_path("/users"), total=100, limit=1, offset=0, ) page_users = Page[_UserListItem].parse_obj(page) print(page_users.json(indent=2, exclude_unset=True))
async def _list_meta_project_iterations_handler( request: web.Request) -> web.Response: # TODO: check access to non owned projects user_id = request[RQT_USERID_KEY] # SEE https://github.com/ITISFoundation/osparc-simcore/issues/2735 # parse and validate request ---- q = parse_query_parameters(request) meta_project_uuid = q.project_uuid meta_project_commit_id = q.ref_id url_for = create_url_for_function(request) vc_repo = VersionControlForMetaModeling(request) # core function ---- iterations_range = await _get_project_iterations_range( vc_repo, meta_project_uuid, meta_project_commit_id, offset=q.offset, limit=q.limit, ) if iterations_range.total_count == 0: raise web.HTTPNotFound( reason= f"No iterations found for project {meta_project_uuid=}/{meta_project_commit_id=}" ) assert len(iterations_range.items) <= q.limit # nosec # parse and validate response ---- page_items = [ ProjectIterationItem.create( meta_project_uuid, meta_project_commit_id, item.iteration_index, item.project_id, url_for, ) for item in iterations_range.items ] page = Page[ProjectIterationItem].parse_obj( paginate_data( chunk=page_items, request_url=request.url, total=iterations_range.total_count, limit=q.limit, offset=q.offset, )) return web.Response( text=page.json(**RESPONSE_MODEL_POLICY), content_type="application/json", )
async def list_projects(request: web.Request): # TODO: implement all query parameters as # in https://www.ibm.com/support/knowledgecenter/en/SSCRJU_3.2.0/com.ibm.swg.im.infosphere.streams.rest.api.doc/doc/restapis-queryparms-list.html from servicelib.aiohttp.rest_utils import extract_and_validate user_id, product_name = request[RQT_USERID_KEY], request[RQ_PRODUCT_KEY] _, query, _ = await extract_and_validate(request) project_type = ProjectTypeAPI(query["type"]) offset = query["offset"] limit = query["limit"] show_hidden = query["show_hidden"] db: ProjectDBAPI = request.config_dict[APP_PROJECT_DBAPI] async def set_all_project_states(projects: List[Dict[str, Any]], project_types: List[bool]): await logged_gather( *[ projects_api.add_project_states_for_user( user_id=user_id, project=prj, is_template=prj_type == ProjectTypeDB.TEMPLATE, app=request.app, ) for prj, prj_type in zip(projects, project_types) ], reraise=True, max_concurrency=100, ) user_available_services: List[ Dict] = await catalog.get_services_for_user_in_product( request.app, user_id, product_name, only_key_versions=True) projects, project_types, total_number_projects = await db.load_projects( user_id=user_id, filter_by_project_type=ProjectTypeAPI.to_project_type_db(project_type), filter_by_services=user_available_services, offset=offset, limit=limit, include_hidden=show_hidden, ) await set_all_project_states(projects, project_types) page = Page[ProjectDict].parse_obj( paginate_data( chunk=projects, request_url=request.url, total=total_number_projects, limit=limit, offset=offset, )) return page.dict(**RESPONSE_MODEL_POLICY)
async def _list_checkpoints_handler(request: web.Request): url_for = create_url_for_function(request) vc_repo = VersionControlRepository(request) _project_uuid = request.match_info["project_uuid"] _limit = int(request.query.get("limit", DEFAULT_NUMBER_OF_ITEMS_PER_PAGE)) _offset = int(request.query.get("offset", 0)) checkpoints: List[Checkpoint] checkpoints, total = await list_checkpoints_safe( vc_repo, project_uuid=_project_uuid, # type: ignore offset=_offset, limit=_limit, ) # parse and validate checkpoints_list = [ CheckpointApiModel.parse_obj({ "url": url_for( f"{__name__}._get_checkpoint_handler", project_uuid=_project_uuid, ref_id=checkpoint.id, ), **checkpoint.dict(), }) for checkpoint in checkpoints ] page = Page[CheckpointApiModel].parse_obj( paginate_data( chunk=checkpoints_list, request_url=request.url, total=total, limit=_limit, offset=_offset, )) return web.Response( text=page.json(**RESPONSE_MODEL_POLICY), content_type="application/json", )
async def _list_repos_handler(request: web.Request): # FIXME: check access to non owned projects user_id = request[RQT_USERID_KEY] url_for = create_url_for_function(request) vc_repo = VersionControlRepository(request) _limit = int(request.query.get("limit", DEFAULT_NUMBER_OF_ITEMS_PER_PAGE)) _offset = int(request.query.get("offset", 0)) repos_rows, total_number_of_repos = await list_repos_safe(vc_repo, offset=_offset, limit=_limit) assert len(repos_rows) <= _limit # nosec # parse and validate repos_list = [ RepoApiModel.parse_obj({ "url": url_for( f"{__name__}._list_checkpoints_handler", project_uuid=row.project_uuid, ), **dict(row.items()), }) for row in repos_rows ] page = Page[RepoApiModel].parse_obj( paginate_data( chunk=repos_list, request_url=request.url, total=total_number_of_repos, limit=_limit, offset=_offset, )) return web.Response( text=page.json(**RESPONSE_MODEL_POLICY), content_type="application/json", )
async def _list_meta_project_iterations_results_handler( request: web.Request, ) -> web.Response: # parse and validate request ---- q = parse_query_parameters(request) meta_project_uuid = q.project_uuid meta_project_commit_id = q.ref_id url_for = create_url_for_function(request) vc_repo = VersionControlForMetaModeling(request) # core function ---- iterations_range = await _get_project_iterations_range( vc_repo, meta_project_uuid, meta_project_commit_id, offset=q.offset, limit=q.limit, ) if iterations_range.total_count == 0: raise web.HTTPNotFound( reason= f"No iterations found for projects/{meta_project_uuid}/checkpoint/{meta_project_commit_id}" ) assert len(iterations_range.items) <= q.limit # nosec # get every project from the database and extract results _prj_data = {} for item in iterations_range.items: # TODO: fetch ALL project iterations at once. Otherwise they will have different results # TODO: if raises? prj = await vc_repo.get_project(f"{item.project_id}", include=["workbench"]) _prj_data[item.project_id] = prj["workbench"] def _get_project_results(project_id) -> ExtractedResults: # TODO: if raises? results = extract_project_results(_prj_data[project_id]) return results # parse and validate response ---- page_items = [ ProjectIterationResultItem.create( meta_project_uuid, meta_project_commit_id, item.iteration_index, item.project_id, _get_project_results(item.project_id), url_for, ) for item in iterations_range.items ] page = Page[ProjectIterationResultItem].parse_obj( paginate_data( chunk=page_items, request_url=request.url, total=iterations_range.total_count, limit=q.limit, offset=q.offset, )) return web.Response( text=page.json(**RESPONSE_MODEL_POLICY), content_type="application/json", )
async def _list_meta_project_iterations_handler( request: web.Request) -> web.Response: # TODO: check access to non owned projects user_id = request[RQT_USERID_KEY] # SEE https://github.com/ITISFoundation/osparc-simcore/issues/2735 # parse and validate request ---- url_for = create_url_for_function(request) vc_repo = VersionControlForMetaModeling(request) _project_uuid = ProjectID(request.match_info["project_uuid"]) _ref_id = request.match_info["ref_id"] _limit = int(request.query.get("limit", DEFAULT_NUMBER_OF_ITEMS_PER_PAGE)) _offset = int(request.query.get("offset", 0)) try: commit_id = CommitID(_ref_id) except ValueError as err: # e.g. HEAD raise NotImplementedError( "cannot convert ref (e.g. HEAD) -> commit id") from err # core function ---- iterations = await _get_project_iterations_range(vc_repo, _project_uuid, commit_id, offset=_offset, limit=_limit) if iterations.total_count == 0: raise web.HTTPNotFound( reason= f"No iterations found for project {_project_uuid=}/{commit_id=}") assert len(iterations.items) <= _limit # nosec # parse and validate response ---- page_items = [ ProjectIterationAsItem( name= f"projects/{_project_uuid}/checkpoint/{commit_id}/iterations/{iter_id}", parent=ParentMetaProjectRef(project_id=_project_uuid, ref_id=commit_id), workcopy_project_id=wcp_id, workcopy_project_url=url_for( "get_project", project_id=wcp_id, ), url=url_for( f"{__name__}._list_meta_project_iterations_handler", project_uuid=_project_uuid, ref_id=commit_id, ), ) for wcp_id, iter_id in iterations.items ] page = Page[ProjectIterationAsItem].parse_obj( paginate_data( chunk=page_items, request_url=request.url, total=iterations.total_count, limit=_limit, offset=_offset, )) return web.Response( text=page.json(**RESPONSE_MODEL_POLICY), content_type="application/json", )
def test_paginating_data(base_url): # create random data total_number_of_items = 29 limit = 9 data_chunk = list(range(limit)) request_url = URL(f"{base_url}?some=1&random=4&query=true") number_of_chunks = total_number_of_items // limit + 1 last_chunk_size = total_number_of_items % limit last_chunk_offset = (number_of_chunks - 1) * len(data_chunk) # first "call" offset = 0 data_obj: PageDict = paginate_data( data_chunk, total=total_number_of_items, limit=limit, offset=offset, request_url=request_url, ) assert data_obj model_instance = Page[int].parse_obj(data_obj) assert model_instance assert model_instance.meta == PageMetaInfoLimitOffset( total=total_number_of_items, count=len(data_chunk), limit=limit, offset=offset) assert model_instance.links == PageLinks( self=str( URL(base_url).with_query( f"some=1&random=4&query=true&offset={offset}&limit={limit}")), first=str( URL(base_url).with_query( f"some=1&random=4&query=true&offset=0&limit={limit}")), prev=None, next=str( URL(base_url).with_query( f"some=1&random=4&query=true&offset={offset+limit}&limit={limit}" )), last=str( URL(base_url).with_query( f"some=1&random=4&query=true&offset={last_chunk_offset}&limit={limit}" )), ) # next "call"s for _ in range(1, number_of_chunks - 1): offset += len(data_chunk) assert model_instance.links.next is not None data_obj: PageDict = paginate_data( data_chunk, request_url=URL(model_instance.links.next), total=total_number_of_items, limit=limit, offset=offset, ) model_instance = Page[int].parse_obj(data_obj) assert model_instance assert model_instance.meta == PageMetaInfoLimitOffset( total=total_number_of_items, count=len(data_chunk), limit=limit, offset=offset, ) assert model_instance.links == PageLinks( self=str( URL(base_url).with_query( f"some=1&random=4&query=true&offset={offset}&limit={limit}" )), first=str( URL(base_url).with_query( f"some=1&random=4&query=true&offset=0&limit={limit}")), prev=str( URL(base_url).with_query( f"some=1&random=4&query=true&offset={offset-limit}&limit={limit}" )), next=str( URL(base_url).with_query( f"some=1&random=4&query=true&offset={offset+limit}&limit={limit}" )), last=str( URL(base_url).with_query( f"some=1&random=4&query=true&offset={last_chunk_offset}&limit={limit}" )), ) # last "call" # offset += len(data_chunk) data_chunk = data_chunk[:last_chunk_size] assert offset == last_chunk_offset assert model_instance.links.next is not None data_obj: PageDict = paginate_data( data_chunk, request_url=URL(model_instance.links.next), total=total_number_of_items, limit=limit, offset=offset, ) assert data_obj model_instance = Page[int].parse_obj(data_obj) assert model_instance assert model_instance.meta == PageMetaInfoLimitOffset( total=total_number_of_items, count=len(data_chunk), limit=limit, offset=offset, ) assert model_instance.links == PageLinks( self=str( URL(base_url).with_query( f"some=1&random=4&query=true&offset={offset}&limit={limit}")), first=str( URL(base_url).with_query( f"some=1&random=4&query=true&offset=0&limit={limit}")), prev=str( URL(base_url).with_query( f"some=1&random=4&query=true&offset={last_chunk_offset - limit}&limit={limit}" )), next=None, last=str( URL(base_url).with_query( f"some=1&random=4&query=true&offset={last_chunk_offset}&limit={limit}" )), )