def run_execute_inputs_schema_variant(self, inputs_param, preload=False, expect_success=True, mock_exec=True):
if isinstance(inputs_param, str):
if preload:
inputs_param = self.load_resource_file(inputs_param)
else:
inputs_param = self.get_resource_file(inputs_param)
with contextlib.ExitStack() as stack_exec:
# use pass-through function because don't care about execution result here, only the parsing of I/O
if mock_exec:
mock_exec_func = lambda *_, **__: None # noqa
else:
mock_exec_func = None
for mock_exec_proc in mocked_execute_process(func_execute_process=mock_exec_func):
stack_exec.enter_context(mock_exec_proc)
result = mocked_sub_requests(self.app, self.client.execute, self.test_process, inputs=inputs_param)
if expect_success:
assert result.success, result.text
assert "jobID" in result.body
assert "processID" in result.body
assert "status" in result.body
assert "location" in result.body
assert result.body["processID"] == self.test_process
assert result.body["status"] == STATUS_ACCEPTED
assert result.body["location"] == result.headers["Location"]
assert "undefined" not in result.message
else:
assert not result.success, result.text
return result
def test_execute_allowed_demo(self):
template = "service=wps&request=execute&version=1.0.0&identifier={}&datainputs=name=tux"
params = template.format(HelloWPS.identifier)
url = self.make_url(params)
with contextlib.ExitStack() as stack_exec:
for mock_exec in mocked_execute_process():
stack_exec.enter_context(mock_exec)
resp = self.app.get(url)
assert resp.status_code == 200 # FIXME: replace by 202 Accepted (?) https://github.com/crim-ca/weaver/issues/14
assert resp.content_type in CONTENT_TYPE_ANY_XML
resp.mustcontain("<wps:ExecuteResponse")
resp.mustcontain("<wps:ProcessAccepted")
resp.mustcontain("PyWPS Process {}".format(HelloWPS.identifier))
def test_execute_deployed_with_visibility_allowed(self):
headers = {"Accept": CONTENT_TYPE_APP_XML}
params_template = "service=wps&request=execute&version=1.0.0&identifier={}&datainputs=test_input=test"
url = self.make_url(
params_template.format(self.process_public.identifier))
with contextlib.ExitStack() as stack_exec:
for mock_exec in mocked_execute_process():
stack_exec.enter_context(mock_exec)
resp = self.app.get(url, headers=headers)
assert resp.status_code == 200 # FIXME: replace by 202 Accepted (?) https://github.com/crim-ca/weaver/issues/14
assert resp.content_type in CONTENT_TYPE_ANY_XML
resp.mustcontain("<wps:ExecuteResponse")
resp.mustcontain("<wps:ProcessAccepted")
resp.mustcontain("PyWPS Process {}".format(
self.process_public.identifier))
def test_execute_process_transmission_mode_value_not_supported(self):
execute_data = self.get_process_execute_template(
fully_qualified_name(self))
execute_data["outputs"][0][
"transmissionMode"] = EXECUTE_TRANSMISSION_MODE_VALUE
uri = "/processes/{}/jobs".format(self.process_public.identifier)
with contextlib.ExitStack() as stack_exec:
for mock_exec in mocked_execute_process():
stack_exec.enter_context(mock_exec)
resp = self.app.post_json(uri,
params=execute_data,
headers=self.json_headers,
expect_errors=True)
assert resp.status_code == 501
assert resp.content_type == CONTENT_TYPE_APP_JSON
def test_execute_wps_rest_resp_json(self):
"""
Test validates that basic Docker application runs successfully, fetching the reference as needed.
The job execution is launched using the WPS-REST endpoint for this test.
Both the request body and response content are JSON.
.. seealso::
- :meth:`test_execute_wps_kvp_get_resp_xml`
- :meth:`test_execute_wps_kvp_get_resp_json`
- :meth:`test_execute_wps_xml_post_resp_xml`
- :meth:`test_execute_wps_xml_post_resp_json`
"""
test_content = "Test file in Docker - WPS-REST job endpoint"
with contextlib.ExitStack() as stack_exec:
# setup
dir_name = tempfile.gettempdir()
tmp_path = tempfile.NamedTemporaryFile(dir=dir_name, mode="w", suffix=".txt")
tmp_file = stack_exec.enter_context(tmp_path) # noqa
tmp_file.write(test_content)
tmp_file.seek(0)
exec_body = {
"mode": EXECUTE_MODE_ASYNC,
"response": EXECUTE_RESPONSE_DOCUMENT,
"inputs": [
{"id": "file", "href": tmp_file.name},
],
"outputs": [
{"id": self.out_key, "transmissionMode": EXECUTE_TRANSMISSION_MODE_REFERENCE},
]
}
for mock_exec in mocked_execute_process():
stack_exec.enter_context(mock_exec)
# execute
proc_url = "/processes/{}/jobs".format(self.process_id)
resp = mocked_sub_requests(self.app, "post_json", proc_url,
data=exec_body, headers=self.json_headers, only_local=True)
assert resp.status_code in [200, 201], "Failed with: [{}]\nReason:\n{}".format(resp.status_code, resp.json)
status_url = resp.json["location"]
job_id = resp.json["jobID"]
# job monitoring
result = self.monitor_job(status_url)
self.validate_outputs(job_id, result, test_content)
def test_execute_deployed_with_visibility_denied(self):
headers = {"Accept": CONTENT_TYPE_APP_XML}
params_template = "service=wps&request=execute&version=1.0.0&identifier={}&datainputs=test_input=test"
url = self.make_url(
params_template.format(self.process_private.identifier))
with contextlib.ExitStack() as stack_exec:
for mock_exec in mocked_execute_process():
stack_exec.enter_context(mock_exec)
resp = self.app.get(url, headers=headers, expect_errors=True)
assert resp.status_code == 403
assert resp.content_type in CONTENT_TYPE_ANY_XML, "Error Response: {}".format(
resp.text)
resp.mustcontain(
"<Exception exceptionCode=\"AccessForbidden\" locator=\"service\">"
)
err_desc = "Process with ID '{}' is not accessible.".format(
self.process_private.identifier)
resp.mustcontain("<ExceptionText>{}</ExceptionText>".format(err_desc))
def test_execute_manual_monitor(self):
with contextlib.ExitStack() as stack_exec:
for mock_exec_proc in mocked_execute_process():
stack_exec.enter_context(mock_exec_proc)
lines = mocked_sub_requests(
self.app, run_command,
[
# "weaver",
"execute",
self.url,
"-p", self.test_process,
"-I", "message='TEST MESSAGE!'"
],
trim=False,
entrypoint=weaver_cli,
only_local=True,
)
# ignore indents of fields from formatted JSON content
assert any(f"\"processID\": \"{self.test_process}\"" in line for line in lines)
assert any("\"jobID\": \"" in line for line in lines)
assert any("\"location\": \"" in line for line in lines)
job_loc = [line for line in lines if "location" in line][0]
job_ref = [line for line in job_loc.split("\"") if line][-1]
job_id = job_ref.split("/")[-1]
lines = mocked_sub_requests(
self.app, run_command,
[
# "weaver",
"monitor",
"-j", job_ref,
"-T", 10,
"-W", 1,
],
trim=False,
entrypoint=weaver_cli,
only_local=True,
)
assert any(f"\"jobID\": \"{job_id}\"" in line for line in lines)
assert any(f"\"status\": \"{STATUS_SUCCEEDED}\"" in line for line in lines)
assert any(f"\"href\": \"{job_ref}/results\"" in line for line in lines)
assert any("\"rel\": \"http://www.opengis.net/def/rel/ogc/1.0/results\"" in line for line in lines)
def test_execute_docker_embedded_python_script(self):
test_proc = "test-docker-python-script"
cwl = load_file(os.path.join(WEAVER_ROOT_DIR, "docs/examples/docker-python-script-report.cwl"))
body = {
"processDescription": {
"process": {
"id": test_proc
}
},
"executionUnit": [{"unit": cwl}],
"deploymentProfileName": "http://www.opengis.net/profiles/eoc/dockerizedApplication"
}
self.deploy_process(body)
with contextlib.ExitStack() as stack:
for mock_exec in mocked_execute_process():
stack.enter_context(mock_exec)
path = f"/processes/{test_proc}/execution"
cost = 2.45
amount = 3
body = {
"mode": EXECUTE_MODE_ASYNC,
"response": EXECUTE_RESPONSE_DOCUMENT,
"inputs": [
{"id": "amount", "value": amount},
{"id": "cost", "value": cost}
],
"outputs": [
{"id": "quote", "transmissionMode": EXECUTE_TRANSMISSION_MODE_REFERENCE},
]
}
resp = mocked_sub_requests(self.app, "POST", path, json=body, headers=self.json_headers, only_local=True)
status_url = resp.headers["Location"]
results = self.monitor_job(status_url)
assert results["quote"]["href"].startswith("http")
stack.enter_context(mocked_wps_output(self.settings))
tmpdir = stack.enter_context(tempfile.TemporaryDirectory())
report_file = fetch_file(results["quote"]["href"], tmpdir, self.settings)
report_data = load_file(report_file, text=True)
assert report_data == f"Order Total: {amount * cost:0.2f}$\n"
def test_execute_auto_monitor(self):
with contextlib.ExitStack() as stack_exec:
for mock_exec_proc in mocked_execute_process():
stack_exec.enter_context(mock_exec_proc)
lines = mocked_sub_requests(
self.app, run_command,
[
# "weaver",
"execute",
self.url,
"-p", self.test_process,
"-I", "message='TEST MESSAGE!'",
"-M",
"-T", 10,
"-W", 1
],
trim=False,
entrypoint=weaver_cli,
only_local=True,
)
assert any("\"jobID\": \"" in line for line in lines) # don't care value, self-handled
assert any(f"\"status\": \"{STATUS_SUCCEEDED}\"" in line for line in lines)
assert any("\"rel\": \"http://www.opengis.net/def/rel/ogc/1.0/results\"" in line for line in lines)
def test_execute_inputs_capture(self):
"""
Verify that specified inputs are captured for a limited number of 1 item per ``-I`` option.
"""
with contextlib.ExitStack() as stack_exec:
for mock_exec_proc in mocked_execute_process():
stack_exec.enter_context(mock_exec_proc)
lines = mocked_sub_requests(
self.app, run_command,
[
# "weaver",
"execute",
"-p", self.test_process,
"-I", "message='TEST MESSAGE!'", # if -I not capture as indented, URL after would be combined in it
self.url,
"-M",
"-T", 10,
"-W", 1,
],
trim=False,
entrypoint=weaver_cli,
only_local=True,
)
assert any(f"\"status\": \"{STATUS_SUCCEEDED}\"" in line for line in lines)
def wps_execute(self, version, accept):
wps_url = get_wps_url(self.settings)
if version == "1.0.0":
test_content = "Test file in Docker - WPS KVP"
wps_method = "GET"
elif version == "2.0.0":
test_content = "Test file in Docker - WPS XML"
wps_method = "POST"
else:
raise ValueError("Invalid WPS version: {}".format(version))
test_content += " {} request - Accept {}".format(wps_method, accept.split("/")[-1].upper())
with contextlib.ExitStack() as stack_exec:
# setup
dir_name = tempfile.gettempdir()
tmp_path = tempfile.NamedTemporaryFile(dir=dir_name, mode="w", suffix=".txt")
tmp_file = stack_exec.enter_context(tmp_path) # noqa
tmp_file.write(test_content)
tmp_file.seek(0)
for mock_exec in mocked_execute_process():
stack_exec.enter_context(mock_exec)
# execute
if version == "1.0.0":
wps_inputs = ["file={}@mimeType={}".format(tmp_file.name, CONTENT_TYPE_TEXT_PLAIN)]
wps_params = {
"service": "WPS",
"request": "Execute",
"version": version,
"identifier": self.process_id,
"DataInputs": wps_inputs,
}
wps_headers = {"Accept": accept}
wps_data = None
else:
wps_inputs = [("file", ComplexDataInput(tmp_file.name, mimeType=CONTENT_TYPE_TEXT_PLAIN))]
wps_outputs = [(self.out_key, True)] # as reference
wps_exec = WPSExecution(version=version, url=wps_url)
wps_req = wps_exec.buildRequest(self.process_id, wps_inputs, wps_outputs)
wps_data = lxml.etree.tostring(wps_req)
wps_headers = {"Accept": accept, "Content-Type": CONTENT_TYPE_APP_XML}
wps_params = None
resp = mocked_sub_requests(self.app, wps_method, wps_url,
params=wps_params, data=wps_data, headers=wps_headers, only_local=True)
assert resp.status_code in [200, 201], \
"Failed with: [{}]\nTest: [{}]\nReason:\n{}".format(resp.status_code, test_content, resp.text)
# parse response status
if accept == CONTENT_TYPE_APP_XML:
assert resp.content_type in CONTENT_TYPE_ANY_XML, test_content
xml = lxml.etree.fromstring(str2bytes(resp.text))
status_url = xml.get("statusLocation")
job_id = status_url.split("/")[-1]
elif accept == CONTENT_TYPE_APP_JSON:
assert resp.content_type == CONTENT_TYPE_APP_JSON, test_content
status_url = resp.json["location"]
job_id = resp.json["jobID"]
assert status_url
assert job_id
# job monitoring
result = self.monitor_job(status_url)
self.validate_outputs(job_id, result, test_content)
def test_jsonarray2netcdf_execute(self):
dirname = tempfile.gettempdir()
nc_data = "Hello NetCDF!"
with contextlib.ExitStack() as stack_exec:
tmp_ncdf = tempfile.NamedTemporaryFile(dir=dirname, mode="w", suffix=".nc")
tmp_json = tempfile.NamedTemporaryFile(dir=dirname, mode="w", suffix=".json")
tmp_ncdf = stack_exec.enter_context(tmp_ncdf) # noqa
tmp_json = stack_exec.enter_context(tmp_json) # noqa
tmp_ncdf.write(nc_data)
tmp_ncdf.seek(0)
tmp_json.write(json.dumps(["file://{}".format(os.path.join(dirname, tmp_ncdf.name))]))
tmp_json.seek(0)
data = {
"mode": EXECUTE_MODE_ASYNC,
"response": EXECUTE_RESPONSE_DOCUMENT,
"inputs": [{"id": "input", "href": os.path.join(dirname, tmp_json.name)}],
"outputs": [{"id": "output", "transmissionMode": EXECUTE_TRANSMISSION_MODE_REFERENCE}],
}
for mock_exec in mocked_execute_process():
stack_exec.enter_context(mock_exec)
path = "/processes/jsonarray2netcdf/jobs"
resp = mocked_sub_requests(self.app, "post_json", path,
data=data, headers=self.json_headers, only_local=True)
assert resp.status_code == 201, "Error: {}".format(resp.json)
assert resp.content_type in CONTENT_TYPE_APP_JSON
job_url = resp.json["location"]
results = self.monitor_job(job_url)
# first validate format of OGC-API results
assert "output" in results, "Expected result ID 'output' in response body"
assert isinstance(results["output"], dict), "Container of result ID 'output' should be a dict"
assert "href" in results["output"]
assert "format" in results["output"]
fmt = results["output"]["format"] # type: JSON
assert isinstance(fmt, dict), "Result format should be provided with content details"
assert "mediaType" in fmt
assert isinstance(fmt["mediaType"], str), "Result format Content-Type should be a single string definition"
assert fmt["mediaType"] == CONTENT_TYPE_APP_NETCDF, "Result 'output' format expected to be NetCDF file"
nc_path = results["output"]["href"]
assert isinstance(nc_path, str) and len(nc_path)
settings = get_settings_from_testapp(self.app)
wps_out = "{}{}".format(settings.get("weaver.url"), settings.get("weaver.wps_output_path"))
nc_real_path = nc_path.replace(wps_out, settings.get("weaver.wps_output_dir"))
assert nc_path.startswith(wps_out)
assert os.path.split(nc_real_path)[-1] == os.path.split(nc_path)[-1]
assert os.path.isfile(nc_real_path)
with open(nc_real_path, "r") as f:
assert f.read() == nc_data
# if everything was valid for results, validate equivalent but differently formatted outputs response
output_url = job_url + "/outputs"
resp = self.app.get(output_url, headers=self.json_headers)
assert resp.status_code == 200, "Error job outputs:\n{}".format(resp.json)
outputs = resp.json
assert outputs["outputs"][0]["id"] == "output"
nc_path = outputs["outputs"][0]["href"]
assert isinstance(nc_path, str) and len(nc_path)
assert nc_path.startswith(wps_out)
assert os.path.split(nc_real_path)[-1] == os.path.split(nc_path)[-1]
def test_jsonarray2netcdf_execute(self):
dirname = tempfile.gettempdir()
nc_data = "Hello NetCDF!"
with contextlib.ExitStack() as stack_exec:
tmp_ncdf = tempfile.NamedTemporaryFile(dir=dirname,
mode="w",
suffix=".nc")
tmp_json = tempfile.NamedTemporaryFile(dir=dirname,
mode="w",
suffix=".json")
tmp_ncdf = stack_exec.enter_context(tmp_ncdf) # noqa
tmp_json = stack_exec.enter_context(tmp_json) # noqa
tmp_ncdf.write(nc_data)
tmp_ncdf.seek(0)
tmp_json.write(
json.dumps(
["file://{}".format(os.path.join(dirname,
tmp_ncdf.name))]))
tmp_json.seek(0)
data = {
"mode":
"async",
"response":
"document",
"inputs": [{
"id": "input",
"href": os.path.join(dirname, tmp_json.name)
}],
"outputs": [{
"id":
"output",
"transmissionMode":
EXECUTE_TRANSMISSION_MODE_REFERENCE
}],
}
for mock_exec in mocked_execute_process():
stack_exec.enter_context(mock_exec)
path = "/processes/jsonarray2netcdf/jobs"
resp = mocked_sub_requests(self.app,
"post_json",
path,
data=data,
headers=self.json_headers,
only_local=True)
assert resp.status_code == 201, "Error: {}".format(resp.json)
assert resp.content_type in CONTENT_TYPE_APP_JSON
job_url = resp.json["location"]
results = self.monitor_job(job_url)
assert results["outputs"][0]["id"] == "output"
nc_path = results["outputs"][0]["href"]
assert isinstance(nc_path, str) and len(nc_path)
settings = get_settings_from_testapp(self.app)
wps_out = "{}{}".format(settings.get("weaver.url"),
settings.get("weaver.wps_output_path"))
nc_real_path = nc_path.replace(wps_out,
settings.get("weaver.wps_output_dir"))
assert nc_path.startswith(wps_out)
assert os.path.split(nc_real_path)[-1] == os.path.split(nc_path)[-1]
assert os.path.isfile(nc_real_path)
with open(nc_real_path, "r") as f:
assert f.read() == nc_data
def wps_execute(self, version, accept):
wps_url = get_wps_url(self.settings)
if version == "1.0.0":
test_content = "Test file in Docker - WPS KVP"
wps_method = "GET"
elif version == "2.0.0":
test_content = "Test file in Docker - WPS XML"
wps_method = "POST"
else:
raise ValueError("Invalid WPS version: {}".format(version))
test_content += " {} request - Accept {}".format(wps_method, accept.split("/")[-1].upper())
with contextlib.ExitStack() as stack_exec:
# setup
dir_name = tempfile.gettempdir()
tmp_file = stack_exec.enter_context(tempfile.NamedTemporaryFile(dir=dir_name, mode="w", suffix=".txt"))
tmp_file.write(test_content)
tmp_file.seek(0)
for mock_exec in mocked_execute_process():
stack_exec.enter_context(mock_exec)
# execute
if version == "1.0.0":
wps_inputs = ["file={}@mimeType={}".format(tmp_file.name, CONTENT_TYPE_TEXT_PLAIN)]
wps_params = {
"service": "WPS",
"request": "Execute",
"version": version,
"identifier": self.process_id,
"DataInputs": wps_inputs,
}
wps_headers = {"Accept": accept}
wps_data = None
else:
wps_inputs = [("file", ComplexDataInput(tmp_file.name, mimeType=CONTENT_TYPE_TEXT_PLAIN))]
wps_outputs = [(self.out_key, True)] # as reference
wps_exec = WPSExecution(version=version, url=wps_url)
wps_req = wps_exec.buildRequest(self.process_id, wps_inputs, wps_outputs)
wps_data = xml_util.tostring(wps_req)
wps_headers = {"Accept": accept, "Content-Type": CONTENT_TYPE_APP_XML}
wps_params = None
resp = mocked_sub_requests(self.app, wps_method, wps_url,
params=wps_params, data=wps_data, headers=wps_headers, only_local=True)
assert resp.status_code in [200, 201], (
"Failed with: [{}]\nTest: [{}]\nReason:\n{}".format(resp.status_code, test_content, resp.text)
)
# parse response status
if accept == CONTENT_TYPE_APP_XML:
assert resp.content_type in CONTENT_TYPE_ANY_XML, test_content
xml_body = xml_util.fromstring(str2bytes(resp.text))
status_url = xml_body.get("statusLocation")
job_id = status_url.split("/")[-1].split(".")[0]
elif accept == CONTENT_TYPE_APP_JSON:
assert resp.content_type == CONTENT_TYPE_APP_JSON, test_content
status_url = resp.json["location"]
job_id = resp.json["jobID"]
assert status_url
assert job_id
if accept == CONTENT_TYPE_APP_XML:
wps_out_url = self.settings["weaver.wps_output_url"]
weaver_url = self.settings["weaver.url"]
assert status_url == f"{wps_out_url}/{job_id}.xml", "Status URL should be XML file for WPS-1 request"
# remap to employ JSON monitor method (could be done with XML parsing otherwise)
status_url = f"{weaver_url}/jobs/{job_id}"
# job monitoring
results = self.monitor_job(status_url)
outputs = self.get_outputs(status_url)
# validate XML status is updated accordingly
wps_xml_status = os.path.join(self.settings["weaver.wps_output_dir"], job_id + ".xml")
assert os.path.isfile(wps_xml_status)
with open(wps_xml_status, "r") as status_file:
assert "ProcessSucceeded" in status_file.read()
self.validate_outputs(job_id, results, outputs, test_content)
def test_register_describe_execute_ncdump(self, mock_responses):
# type: (RequestsMock) -> None
"""
Test the full workflow from remote WPS-1 provider registration, process description, execution and fetch result.
The complete execution and definitions (XML responses) of the "remote" WPS are mocked.
Requests and response negotiation between Weaver and that "remote" WPS are effectively executed and validated.
Validation is accomplished against the same process and mocked server from corresponding test deployment
server in order to detect early any breaking feature. Responses XML bodies employed to simulate the mocked
server are pre-generated from real request calls to the actual service that was running on a live platform.
.. seealso::
- Reference notebook testing the same process on a live server:
https://github.com/Ouranosinc/pavics-sdi/blob/master/docs/source/notebook-components/weaver_example.ipynb
- Evaluate format of submitted Execute body (see `#340 <https://github.com/crim-ca/weaver/issues/340>`_).
"""
self.service_store.clear_services()
# register the provider
remote_provider_name = "test-wps-remote-provider-hummingbird"
path = "/providers"
data = {
"id": remote_provider_name,
"url": resources.TEST_REMOTE_SERVER_URL
}
resp = self.app.post_json(path, params=data, headers=self.json_headers)
assert resp.status_code == 201
# validate service capabilities
path = "/providers/{}".format(remote_provider_name)
resp = self.app.get(path, headers=self.json_headers)
assert resp.status_code == 200
body = resp.json
assert "id" in body and body["id"] == remote_provider_name
assert "hummingbird" in body["title"].lower()
assert body["type"] == PROCESS_WPS_REMOTE
# validate processes capabilities
path = "/providers/{}/processes".format(remote_provider_name)
resp = self.app.get(path, headers=self.json_headers)
body = resp.json
assert resp.status_code == 200
assert "processes" in body and len(
body["processes"]) == 14 # in TEST_HUMMINGBIRD_GETCAP_WPS1_XML
processes = {process["id"]: process for process in body["processes"]}
assert "ncdump" in processes
assert processes["ncdump"]["version"] == "4.4.1.1"
assert processes["ncdump"]["metadata"][0]["rel"] == "birdhouse"
assert processes["ncdump"]["metadata"][1]["rel"] == "user-guide"
# keyword 'Hummingbird' in this case is from GetCapabilities ProviderName
# keyword of the service name within Weaver is also provided, which can be different than provider
expect_keywords = [
PROCESS_WPS_REMOTE, "Hummingbird", remote_provider_name
]
assert all(key in processes["ncdump"]["keywords"]
for key in expect_keywords)
proc_desc_url = processes["ncdump"]["processDescriptionURL"]
proc_wps1_url = processes["ncdump"]["processEndpointWPS1"]
proc_exec_url = processes["ncdump"]["executeEndpoint"]
assert proc_wps1_url.startswith(resources.TEST_REMOTE_SERVER_URL)
assert proc_desc_url == self.app_url + path + "/ncdump"
assert proc_exec_url == self.app_url + path + "/ncdump/jobs"
# validate process description
resp = self.app.get(proc_desc_url, headers=self.json_headers)
assert resp.status_code == 200
body = resp.json
assert "inputs" in body and len(body["inputs"]) == 2
assert all(iid in body["inputs"]
for iid in ["dataset", "dataset_opendap"])
assert body["inputs"]["dataset"]["minOccurs"] == 0
assert body["inputs"]["dataset"]["maxOccurs"] == 100
assert "formats" in body["inputs"]["dataset"]
assert len(body["inputs"]["dataset"]["formats"]) == 1
assert body["inputs"]["dataset"]["formats"][0]["default"] is True
assert "literalDataDomains" not in body["inputs"]["dataset"]
assert body["inputs"]["dataset"]["formats"][0][
"mediaType"] == CONTENT_TYPE_APP_NETCDF
assert body["inputs"]["dataset_opendap"]["minOccurs"] == 0
assert body["inputs"]["dataset_opendap"]["maxOccurs"] == 100
assert "formats" not in body["inputs"]["dataset_opendap"]
assert "literalDataDomains" in body["inputs"]["dataset_opendap"]
assert len(
body["inputs"]["dataset_opendap"]["literalDataDomains"]) == 1
assert body["inputs"]["dataset_opendap"]["literalDataDomains"][0][
"dataType"]["name"] == "string"
assert body["inputs"]["dataset_opendap"]["literalDataDomains"][0][
"valueDefinition"]["anyValue"] is True
assert body["inputs"]["dataset_opendap"]["literalDataDomains"][0][
"default"] is True
assert "outputs" in body and len(body["outputs"]) == 1
assert "output" in body["outputs"]
assert "formats" in body["outputs"]["output"]
assert len(body["outputs"]["output"]["formats"]) == 1
assert body["outputs"]["output"]["formats"][0]["default"] is True
assert body["outputs"]["output"]["formats"][0][
"mediaType"] == CONTENT_TYPE_TEXT_PLAIN
assert "literalDataDomains" not in body["outputs"]["output"]
assert body["processDescriptionURL"] == proc_desc_url
assert body["processEndpointWPS1"] == proc_wps1_url
assert body["executeEndpoint"] == proc_exec_url
job_exec_url = proc_exec_url.replace(
"/execution", "/jobs") # both are aliases, any could be returned
ogc_exec_url = proc_exec_url.replace("/jobs", "/execution")
links = {
link["rel"].rsplit("/")[-1]: link["href"]
for link in body["links"]
}
assert links["execute"] in [job_exec_url, ogc_exec_url]
assert links["process-meta"] == proc_desc_url
# WPS-1 URL also includes relevant query parameters to obtain a valid response directly from remote service
assert links["process-desc"] == proc_wps1_url
assert links["service-desc"].startswith(
resources.TEST_REMOTE_SERVER_URL)
assert "DescribeProcess" in links["process-desc"]
assert "GetCapabilities" in links["service-desc"]
assert EXECUTE_CONTROL_OPTION_ASYNC in body["jobControlOptions"]
assert EXECUTE_TRANSMISSION_MODE_REFERENCE in body[
"outputTransmission"]
# validate execution submission
# (don't actually execute because server is mocked, only validate parsing of I/O and job creation)
# first setup all expected contents and files
exec_file = "http://localhost.com/dont/care.nc"
exec_body = {
"mode":
EXECUTE_MODE_ASYNC,
"response":
EXECUTE_RESPONSE_DOCUMENT,
"inputs": [{
"id": "dataset",
"href": exec_file
}],
"outputs": [{
"id": "output",
"transmissionMode": EXECUTE_TRANSMISSION_MODE_REFERENCE
}]
}
status_url = resources.TEST_REMOTE_SERVER_URL + "/status.xml"
output_url = resources.TEST_REMOTE_SERVER_URL + "/output.txt"
with open(resources.TEST_HUMMINGBIRD_STATUS_WPS1_XML) as status_file:
status = status_file.read().format(
TEST_SERVER_URL=resources.TEST_REMOTE_SERVER_URL,
LOCATION_XML=status_url,
OUTPUT_FILE=output_url,
)
xml_headers = {"Content-Type": CONTENT_TYPE_TEXT_XML}
ncdump_data = "Fake NetCDF Data"
with contextlib.ExitStack() as stack_exec:
# mock direct execution bypassing celery
for mock_exec in mocked_execute_process():
stack_exec.enter_context(mock_exec)
# mock responses expected by "remote" WPS-1 Execute request and relevant documents
mock_responses.add(
"GET",
exec_file,
body=ncdump_data,
headers={"Content-Type": CONTENT_TYPE_APP_NETCDF})
mock_responses.add("POST",
resources.TEST_REMOTE_SERVER_URL,
body=status,
headers=xml_headers)
mock_responses.add("GET",
status_url,
body=status,
headers=xml_headers)
mock_responses.add(
"GET",
output_url,
body=ncdump_data,
headers={"Content-Type": CONTENT_TYPE_TEXT_PLAIN})
# add reference to specific provider execute class to validate it was called
# (whole procedure must run even though a lot of parts are mocked)
real_wps1_process_execute = Wps1Process.execute
handle_wps1_process_execute = stack_exec.enter_context(
mock.patch.object(Wps1Process,
"execute",
side_effect=real_wps1_process_execute,
autospec=True)) # type: MockPatch
# launch job execution and validate
resp = mocked_sub_requests(self.app,
"post_json",
proc_exec_url,
timeout=5,
data=exec_body,
headers=self.json_headers,
only_local=True)
assert resp.status_code in [
200, 201
], "Failed with: [{}]\nReason:\n{}".format(resp.status_code,
resp.json)
assert handle_wps1_process_execute.called, "WPS-1 handler should have been called by CWL runner context"
status_url = resp.json["location"]
job_id = resp.json["jobID"]
assert status_url == proc_exec_url + "/" + job_id
results = self.monitor_job(status_url)
output_url = "{}/{}/{}".format(
self.settings["weaver.wps_output_url"], job_id, "output.txt")
output_path = "{}/{}/{}".format(
self.settings["weaver.wps_output_dir"], job_id, "output.txt")
assert results["output"]["format"][
"mediaType"] == CONTENT_TYPE_TEXT_PLAIN
assert results["output"]["href"] == output_url
with open(output_path) as out_file:
data = out_file.read()
assert data == ncdump_data
