def test_get_jobs_private_service_public_process_unauthorized_in_query(self): path = get_path_kvp(jobs_short_uri, service=self.service_private.name, process=self.process_public.identifier) resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 401 assert resp.content_type == CONTENT_TYPE_APP_JSON
def test_get_jobs_valid_grouping_by_service(self): path = get_path_kvp(jobs_short_uri, detail="false", groups="service") resp = self.app.get(path, headers=self.json_headers) self.check_basic_jobs_grouped_info(resp, groups="service") # ensure that group categories are distinct for i, grouped_jobs in enumerate(resp.json["groups"]): categories = grouped_jobs["category"] for j, grp_jobs in enumerate(resp.json["groups"]): compared = grp_jobs["category"] if i == j: continue assert categories != compared # validate groups with expected jobs counts and ids (nb: only public jobs are returned) if categories["service"] == self.service_public.name: assert len(grouped_jobs["jobs"]) == 3 assert set(grouped_jobs["jobs"]) == {self.job_info[1].id, self.job_info[5].id, self.job_info[6].id} elif categories["service"] == self.service_private.name: assert len(grouped_jobs["jobs"]) == 2 assert set(grouped_jobs["jobs"]) == {self.job_info[7].id, self.job_info[8].id} elif categories["service"] is None: assert len(grouped_jobs["jobs"]) == 2 assert set(grouped_jobs["jobs"]) == {self.job_info[0].id, self.job_info[2].id} else: pytest.fail("Unknown job grouping 'service' value not expected.")
def test_get_jobs_process_in_query_detail(self): path = get_path_kvp(jobs_short_uri, process=self.job_info[0].process, detail="true") resp = self.app.get(path, headers=self.json_headers) self.check_basic_jobs_info(resp) job_ids = [j["jobID"] for j in resp.json["jobs"]] assert self.job_info[0].id in job_ids, self.message_with_jobs_mapping("expected in") assert self.job_info[1].id not in job_ids, self.message_with_jobs_mapping("expected not in")
def test_get_jobs_by_encrypted_email(self): """Verifies that literal email can be used as search criterion although not saved in plain text within db.""" email = "*****@*****.**" body = { "inputs": [{"id": "test_input", "data": "test"}], "outputs": [{"id": "test_output", "transmissionMode": EXECUTE_TRANSMISSION_MODE_REFERENCE}], "mode": EXECUTE_MODE_ASYNC, "response": EXECUTE_RESPONSE_DOCUMENT, "notification_email": email } with contextlib.ExitStack() as stack: for runner in mocked_process_job_runner(): stack.enter_context(runner) path = "/processes/{}/jobs".format(self.process_public.identifier) resp = self.app.post_json(path, params=body, headers=self.json_headers) assert resp.status_code == 201 assert resp.content_type == CONTENT_TYPE_APP_JSON job_id = resp.json["jobID"] # verify the email is not in plain text job = self.job_store.fetch_by_id(job_id) assert job.notification_email != email and job.notification_email is not None assert int(job.notification_email, 16) != 0 # email should be encrypted with hex string path = get_path_kvp(jobs_short_uri, detail="true", notification_email=email) resp = self.app.get(path, headers=self.json_headers) assert resp.status_code == 200 assert resp.content_type == CONTENT_TYPE_APP_JSON assert resp.json["total"] == 1, "Should match exactly 1 email with specified literal string as query param." assert resp.json["jobs"][0]["jobID"] == job_id
def test_get_jobs_detail_paged(self): for detail in ("true", 1, "True", "yes"): path = get_path_kvp(jobs_short_uri, detail=detail) resp = self.app.get(path, headers=self.json_headers) self.check_basic_jobs_info(resp) for job in resp.json["jobs"]: self.check_job_format(job)
def _parse_for_app_req(method, url, **req_kwargs): """ WebTest application employs ``params`` instead of ``data``/``json``. Actual query parameters must be pre-appended to ``url``. """ method = method.lower() url = req_kwargs.pop("base_url", url) body = req_kwargs.pop("data", None) query = req_kwargs.pop("query", None) params = req_kwargs.pop("params", {}) if query: url += ("" if query.startswith("?") else "?") + query elif params: if isinstance(params, str): url += ("" if params.startswith("?") else "?") + params else: url = get_path_kvp(url, **params) req_kwargs["params"] = body # remove unsupported parameters that cannot be passed down to TestApp for key in [ "timeout", "cert", "auth", "ssl_verify", "verify", "language" ]: req_kwargs.pop(key, None) req = getattr(app, method) return url, req, req_kwargs
def test_get_jobs_detail_grouped(self): for detail in ("true", 1, "True", "yes"): groups = ["process", "service"] path = get_path_kvp(jobs_short_uri, detail=detail, groups=groups) resp = self.app.get(path, headers=self.json_headers) self.check_basic_jobs_grouped_info(resp, groups=groups) for grouped_jobs in resp.json["groups"]: for job in grouped_jobs["jobs"]: self.check_job_format(job)
def test_get_jobs_normal_grouped(self): for detail in ("false", 0, "False", "no"): groups = ["process", "service"] path = get_path_kvp(jobs_short_uri, detail=detail, groups=groups) resp = self.app.get(path, headers=self.json_headers) self.check_basic_jobs_grouped_info(resp, groups=groups) for grouped_jobs in resp.json["groups"]: for job in grouped_jobs["jobs"]: assert isinstance(job, str)
def test_get_jobs_normal_paged(self): resp = self.app.get(jobs_short_uri, headers=self.json_headers) self.check_basic_jobs_info(resp) for job_id in resp.json["jobs"]: assert isinstance(job_id, str) for detail in ("false", 0, "False", "no", "None", "null", None, ""): path = get_path_kvp(jobs_short_uri, detail=detail) resp = self.app.get(path, headers=self.json_headers) self.check_basic_jobs_info(resp) for job_id in resp.json["jobs"]: assert isinstance(job_id, str)
def test_get_jobs_public_service_no_processes(self): """ NOTE: it is up to the remote service to hide private processes if the process is invisible, no job should have been executed nor can be fetched """ path = get_path_kvp(jobs_short_uri, service=self.service_public.name, process=self.process_private.identifier) with contextlib.ExitStack() as stack: for job in self.get_job_remote_service_mock([]): # process invisible (not returned by remote) stack.enter_context(job) resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 404 assert resp.content_type == CONTENT_TYPE_APP_JSON
def test_get_jobs_public_service_private_process_unauthorized_in_query(self): """ NOTE: it is up to the remote service to hide private processes if the process is visible, the a job can be executed and it is automatically considered public """ path = get_path_kvp(jobs_short_uri, service=self.service_public.name, process=self.process_private.identifier) with contextlib.ExitStack() as stack: for runner in self.get_job_remote_service_mock([self.process_private]): # process visible on remote stack.enter_context(runner) resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 200 assert resp.content_type == CONTENT_TYPE_APP_JSON
def _parse_for_app_req(method, url, **req_kwargs): """ Obtain request details with adjustments to support specific handling for :class:`webTest.TestApp`. WebTest application employs ``params`` instead of ``data``/``json``. Actual query parameters must be pre-appended to ``url``. """ method = method.lower() url = req_kwargs.pop("base_url", url) body = req_kwargs.pop("data", None) _json = req_kwargs.pop("json", None) query = req_kwargs.pop("query", None) params = req_kwargs.pop("params", {}) if query: url += ("" if query.startswith("?") else "?") + query elif params: if isinstance(params, str): url += ("" if params.startswith("?") else "?") + params else: url = get_path_kvp(url, **params) req_kwargs["params"] = content = body or _json or {} # remove unsupported parameters that cannot be passed down to TestApp for key in [ "timeout", "cert", "auth", "ssl_verify", "verify", "language", "stream" ]: req_kwargs.pop(key, None) cookies = req_kwargs.pop("cookies", None) if cookies: cookies = dict(cookies) # in case list of tuples for name, value in cookies.items(): app.set_cookie(name, value) # although headers for JSON content can be set, some methods are not working (eg: PUT) # obtain the corresponding '<method>_json' function to have the proper behaviour headers = req_kwargs.get("headers", {}) or {} if ((get_header("Content-Type", headers) == CONTENT_TYPE_APP_JSON or isinstance(content, (dict, list))) and hasattr(app, method + "_json")): method = method + "_json" if isinstance(content, str): req_kwargs["params"] = json.loads(req_kwargs["params"]) req = getattr(app, method) return url, req, req_kwargs
def submit_local_job(request): # type: (PyramidRequest) -> AnyViewResponse """ Execute a process registered locally. Execution location and method is according to deployed Application Package. """ process = get_process(request=request) ctype = clean_mime_type_format(get_header("content-type", request.headers, default=None), strip_parameters=True) if ctype in ContentType.ANY_XML: # Send the XML request to the WPS endpoint which knows how to parse it properly. # Execution will end up in the same 'submit_job_handler' function as other branch for JSON. service = get_pywps_service() wps_params = {"version": "1.0.0", "request": "Execute", "service": "WPS", "identifier": process.id} request.path_info = get_wps_path(request) request.query_string = get_path_kvp("", **wps_params)[1:] location = request.application_url + request.path_info + request.query_string LOGGER.warning("Route redirection [%s] -> [%s] for WPS-XML support.", request.url, location) http_request = extend_instance(request, WerkzeugRequest) http_request.shallow = False return service.call(http_request) return submit_job(request, process, tags=["wps-rest"])
def test_get_jobs_public_with_access_and_request_user(self): """Verifies that corresponding processes are returned when proper access/user-id are respected.""" uri_direct_jobs = jobs_short_uri uri_process_jobs = process_jobs_uri.format(process_id=self.process_public.identifier) uri_provider_jobs = jobs_full_uri.format( provider_id=self.service_public.name, process_id=self.process_public.identifier) admin_public_jobs = list(filter(lambda j: VISIBILITY_PUBLIC in j.access, self.job_info)) admin_private_jobs = list(filter(lambda j: VISIBILITY_PRIVATE in j.access, self.job_info)) editor1_all_jobs = list(filter(lambda j: j.user_id == self.user_editor1_id, self.job_info)) editor1_public_jobs = list(filter(lambda j: VISIBILITY_PUBLIC in j.access, editor1_all_jobs)) editor1_private_jobs = list(filter(lambda j: VISIBILITY_PRIVATE in j.access, editor1_all_jobs)) public_jobs = list(filter(lambda j: VISIBILITY_PUBLIC in j.access, self.job_info)) def filter_process(jobs): # type: (Iterable[Job]) -> List[Job] return list(filter(lambda j: j.process == self.process_public.identifier, jobs)) def filter_service(jobs): # type: (Iterable[Job]) -> List[Job] return list(filter(lambda j: j.service == self.service_public.name, jobs)) # test variations of [paths, query, user-id, expected-job-ids] path_jobs_user_req_tests = [ # pylint: disable=C0301,line-too-long,C0326,bad-whitespace # URI ACCESS USER EXPECTED JOBS (uri_direct_jobs, None, None, public_jobs), # noqa: E241,E501 (uri_direct_jobs, None, self.user_editor1_id, editor1_all_jobs), # noqa: E241,E501 (uri_direct_jobs, None, self.user_admin_id, self.job_info), # noqa: E241,E501 (uri_direct_jobs, VISIBILITY_PRIVATE, None, public_jobs), # noqa: E241,E501 (uri_direct_jobs, VISIBILITY_PRIVATE, self.user_editor1_id, editor1_private_jobs), # noqa: E241,E501 (uri_direct_jobs, VISIBILITY_PRIVATE, self.user_admin_id, admin_private_jobs), # noqa: E241,E501 (uri_direct_jobs, VISIBILITY_PUBLIC, None, public_jobs), # noqa: E241,E501 (uri_direct_jobs, VISIBILITY_PUBLIC, self.user_editor1_id, editor1_public_jobs), # noqa: E241,E501 (uri_direct_jobs, VISIBILITY_PUBLIC, self.user_admin_id, admin_public_jobs), # noqa: E241,E501 # --- (uri_process_jobs, None, None, filter_process(public_jobs)), # noqa: E241,E501 (uri_process_jobs, None, self.user_editor1_id, filter_process(editor1_all_jobs)), # noqa: E241,E501 (uri_process_jobs, None, self.user_admin_id, filter_process(self.job_info)), # noqa: E241,E501 (uri_process_jobs, VISIBILITY_PRIVATE, None, filter_process(public_jobs)), # noqa: E241,E501 (uri_process_jobs, VISIBILITY_PRIVATE, self.user_editor1_id, filter_process(editor1_private_jobs)), # noqa: E241,E501 (uri_process_jobs, VISIBILITY_PRIVATE, self.user_admin_id, filter_process(admin_private_jobs)), # noqa: E241,E501 (uri_process_jobs, VISIBILITY_PUBLIC, None, filter_process(public_jobs)), # noqa: E241,E501 (uri_process_jobs, VISIBILITY_PUBLIC, self.user_editor1_id, filter_process(editor1_public_jobs)), # noqa: E241,E501 (uri_process_jobs, VISIBILITY_PUBLIC, self.user_admin_id, filter_process(self.job_info)), # noqa: E241,E501 # --- (uri_provider_jobs, None, None, filter_service(public_jobs)), # noqa: E241,E501 (uri_provider_jobs, None, self.user_editor1_id, filter_service(editor1_all_jobs)), # noqa: E241,E501 (uri_provider_jobs, None, self.user_admin_id, filter_service(self.job_info)), # noqa: E241,E501 (uri_provider_jobs, VISIBILITY_PRIVATE, None, filter_service(public_jobs)), # noqa: E241,E501 (uri_provider_jobs, VISIBILITY_PRIVATE, self.user_editor1_id, filter_service(editor1_private_jobs)), # noqa: E241,E501 (uri_provider_jobs, VISIBILITY_PRIVATE, self.user_admin_id, filter_service(admin_private_jobs)), # noqa: E241,E501 (uri_provider_jobs, VISIBILITY_PUBLIC, None, filter_service(public_jobs)), # noqa: E241,E501 (uri_provider_jobs, VISIBILITY_PUBLIC, self.user_editor1_id, filter_service(editor1_public_jobs)), # noqa: E241,E501 (uri_provider_jobs, VISIBILITY_PUBLIC, self.user_admin_id, filter_service(self.job_info)), # noqa: E241,E501 ] # type: List[Tuple[str, str, Union[None, int], List[Job]]] for i, (path, access, user_id, expected_jobs) in enumerate(path_jobs_user_req_tests): with contextlib.ExitStack() as stack: for patch in self.get_job_request_auth_mock(user_id): stack.enter_context(patch) for patch in self.get_job_remote_service_mock([self.process_public]): stack.enter_context(patch) test = get_path_kvp(path, access=access) if access else path resp = self.app.get(test, headers=self.json_headers) self.check_basic_jobs_info(resp) job_ids = [job.id for job in expected_jobs] job_match = all(job in job_ids for job in resp.json["jobs"]) test_values = dict(path=path, access=access, user_id=user_id) assert job_match, self.message_with_jobs_diffs(resp.json["jobs"], job_ids, test_values, index=i)
def test_get_path_kvp(): res = get_path_kvp("http://localhost", test1="value1", test2=["sub1", "sub2"]) assert res == "http://localhost?test1=value1&test2=sub1,sub2"
def test_get_jobs_process_in_query_normal(self): path = get_path_kvp(jobs_short_uri, process=self.job_info[0].process) resp = self.app.get(path, headers=self.json_headers) self.check_basic_jobs_info(resp) assert self.job_info[0].id in resp.json["jobs"], self.message_with_jobs_mapping("expected in") assert self.job_info[1].id not in resp.json["jobs"], self.message_with_jobs_mapping("expected not in")
def get_process_list_links(request, paging, total, provider=None): # type: (Request, Dict[str, int], Optional[int], Optional[Service]) -> List[JSON] """ Obtains a list of all relevant links for the corresponding :term:`Process` listing defined by query parameters. :raises IndexError: if the paging values are out of bounds compared to available total :term:`Process`. """ # reapply queries that must be given to obtain the same result in case of subsequent requests (sort, limits, etc.) kvp_params = { param: value for param, value in request.params.items() if param != "page" } base_url = get_weaver_url(request) links = [] if provider: proc_path = sd.provider_processes_service.path.format( provider_id=provider.id) links.extend(provider.links(request, self_link="provider")) else: proc_path = sd.processes_service.path proc_url = base_url + proc_path links.extend([ { "href": proc_url, "rel": "collection", "type": CONTENT_TYPE_APP_JSON, "title": "Process listing (no filtering queries applied)." }, { "href": proc_url, "rel": "search", "type": CONTENT_TYPE_APP_JSON, "title": "Generic query endpoint to list processes." }, { "href": proc_url + "?detail=false", "rel": "preview", "type": CONTENT_TYPE_APP_JSON, "title": "Process listing summary (identifiers and count only)." }, { "href": proc_url, "rel": "http://www.opengis.net/def/rel/ogc/1.0/processes", "type": CONTENT_TYPE_APP_JSON, "title": "List of registered local processes." }, { "href": get_path_kvp(proc_url, **request.params), "rel": "self", "type": CONTENT_TYPE_APP_JSON, "title": "Current process listing." }, ]) if provider: prov_url = proc_url.rsplit("/", 1)[0] links.append({ "href": prov_url, "rel": "up", "type": CONTENT_TYPE_APP_JSON, "title": "Provider description." }) else: links.append({ "href": base_url, "rel": "up", "type": CONTENT_TYPE_APP_JSON, "title": "API entrypoint." }) cur_page = paging.get("page", None) per_page = paging.get("limit", None) if all(isinstance(num, int) for num in [cur_page, per_page, total]): max_page = max(math.ceil(total / per_page) - 1, 0) if cur_page < 0 or cur_page > max_page: raise IndexError( f"Page index {cur_page} is out of range from [0,{max_page}].") links.extend([ { "href": get_path_kvp(proc_url, page=cur_page, **kvp_params), "rel": "current", "type": CONTENT_TYPE_APP_JSON, "title": "Current page of processes query listing." }, { "href": get_path_kvp(proc_url, page=0, **kvp_params), "rel": "first", "type": CONTENT_TYPE_APP_JSON, "title": "First page of processes query listing." }, { "href": get_path_kvp(proc_url, page=max_page, **kvp_params), "rel": "last", "type": CONTENT_TYPE_APP_JSON, "title": "Last page of processes query listing." }, ]) if cur_page > 0: links.append({ "href": get_path_kvp(proc_url, page=cur_page - 1, **kvp_params), "rel": "prev", "type": CONTENT_TYPE_APP_JSON, "title": "Previous page of processes query listing." }) if cur_page < max_page: links.append({ "href": get_path_kvp(proc_url, page=cur_page + 1, **kvp_params), "rel": "next", "type": CONTENT_TYPE_APP_JSON, "title": "Next page of processes query listing." }) return links
def test_get_jobs_private_process_not_returned_in_query(self): path = get_path_kvp(jobs_short_uri, process=self.process_private.identifier) resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 401 assert resp.content_type == CONTENT_TYPE_APP_JSON
def get_job_list_links(job_total, filters, request): # type: (int, Dict[str, AnyValueType], Request) -> List[JSON] """ Obtains a list of all relevant links for the corresponding job listing defined by query parameter filters. :raises IndexError: if the paging values are out of bounds compared to available total :term:`Job` matching search. """ base_url = get_weaver_url(request) # reapply queries that must be given to obtain the same result in case of subsequent requests (sort, limits, etc.) kvp_params = { param: value for param, value in request.params.items() if param != "page" } # patch datetime that have some extra character manipulation (reapply '+' auto-converted to ' ' by params parser) if "datetime" in kvp_params: kvp_params["datetime"] = kvp_params["datetime"].replace(" ", "+") alt_kvp = deepcopy(kvp_params) # request job uses general endpoint, obtain the full path if any service/process was given as alternate location if request.path.startswith(sd.jobs_service.path): job_path = base_url + sd.jobs_service.path alt_path = None parent_url = None # cannot generate full path apply for 'service' by itself if filters["process"] and filters["service"]: alt_path = base_url + sd.provider_jobs_service.path.format( provider_id=filters["service"], process_id=filters["process"]) parent_url = alt_path.rsplit("/", 1)[0] elif filters["process"]: alt_path = base_url + sd.process_jobs_service.path.format( process_id=filters["process"]) parent_url = alt_path.rsplit("/", 1)[0] for param in ["service", "provider", "process"]: alt_kvp.pop(param, None) # path is whichever specific service/process endpoint, jobs are pre-filtered by them # transform sub-endpoints into matching query parameters and use generic path as alternate location else: job_path = base_url + request.path alt_path = base_url + sd.jobs_service.path alt_kvp["process"] = filters["process"] if filters["service"]: alt_kvp["provider"] = filters["service"] parent_url = job_path.rsplit("/", 1)[0] cur_page = filters["page"] per_page = filters["limit"] max_page = max(math.ceil(job_total / per_page) - 1, 0) if cur_page < 0 or cur_page > max_page: raise IndexError( f"Page index {cur_page} is out of range from [0,{max_page}].") alt_links = [] if alt_path: alt_links = [{ "href": get_path_kvp(alt_path, page=cur_page, **alt_kvp), "rel": "alternate", "type": CONTENT_TYPE_APP_JSON, "title": "Alternate endpoint with equivalent set of filtered jobs." }] links = alt_links + [ { "href": job_path, "rel": "collection", "type": CONTENT_TYPE_APP_JSON, "title": "Complete job listing (no filtering queries applied)." }, { "href": base_url + sd.jobs_service.path, "rel": "search", "type": CONTENT_TYPE_APP_JSON, "title": "Generic query endpoint to search for jobs." }, { "href": job_path + "?detail=false", "rel": "preview", "type": CONTENT_TYPE_APP_JSON, "title": "Job listing summary (UUID and count only)." }, { "href": job_path, "rel": "http://www.opengis.net/def/rel/ogc/1.0/job-list", "type": CONTENT_TYPE_APP_JSON, "title": "List of registered jobs." }, { "href": get_path_kvp(job_path, page=cur_page, **kvp_params), "rel": "current", "type": CONTENT_TYPE_APP_JSON, "title": "Current page of job query listing." }, { "href": get_path_kvp(job_path, page=0, **kvp_params), "rel": "first", "type": CONTENT_TYPE_APP_JSON, "title": "First page of job query listing." }, { "href": get_path_kvp(job_path, page=max_page, **kvp_params), "rel": "last", "type": CONTENT_TYPE_APP_JSON, "title": "Last page of job query listing." }, ] if cur_page > 0: links.append({ "href": get_path_kvp(job_path, page=cur_page - 1, **kvp_params), "rel": "prev", "type": CONTENT_TYPE_APP_JSON, "title": "Previous page of job query listing." }) if cur_page < max_page: links.append({ "href": get_path_kvp(job_path, page=cur_page + 1, **kvp_params), "rel": "next", "type": CONTENT_TYPE_APP_JSON, "title": "Next page of job query listing." }) if parent_url: links.append({ "href": parent_url, "rel": "up", "type": CONTENT_TYPE_APP_JSON, "title": "Parent collection for which listed jobs apply." }) return links
def test_get_jobs_service_and_process_unknown_in_query(self): path = get_path_kvp(jobs_short_uri, service="unknown-service-id", process="unknown-process-id") resp = self.app.get(path, headers=self.json_headers, expect_errors=True) assert resp.status_code == 404 assert resp.content_type == CONTENT_TYPE_APP_JSON