def test_deploy_with_undeploy(self):
test_id = f"{self.test_process_prefix}-deploy-undeploy-flag"
result = mocked_sub_requests(self.app, self.client.deploy, test_id, self.test_payload)
assert result.success
result = mocked_sub_requests(self.app, self.client.deploy, test_id, self.test_payload, undeploy=True)
assert result.success
assert "undefined" not in result.message
def test_execute_manual_monitor_status_and_download_results(self):
"""
Test a typical case of :term:`Job` execution, result retrieval and download, but with manual monitoring.
Manual monitoring can be valid in cases where a *very* long :term:`Job` must be executed, and the user does
not intend to wait after it. This avoids leaving some shell/notebook/etc. open of a long time and provide a
massive ``timeout`` value. Instead, the user can simply re-call :meth:`WeaverClient.monitor` at a later time
to resume monitoring. Other situation can be if the connection was dropped or script runner crashed, and the
want to pick up monitoring again.
.. note::
The :meth:`WeaverClient.execute` is accomplished synchronously during this test because of the mock.
The :meth:`WeaverClient.monitor` step can therefore only return ``success``/``failed`` directly
without any intermediate and asynchronous pooling of ``running`` status.
The first status result from :meth:`WeaverClient.execute` is ``accept`` because this is the
default status that is generated by the HTTP response from the :term:`Job` creation.
Any following GET status will directly return the final :term:`Job` result.
"""
result = self.run_execute_inputs_schema_variant("Execute_Echo_cwl_schema.yml", mock_exec=False)
job_id = result.body["jobID"]
result = mocked_sub_requests(self.app, self.client.monitor, job_id, timeout=1, interval=1)
assert result.success, result.text
assert "undefined" not in result.message
assert result.body.get("status") == STATUS_SUCCEEDED
links = result.body.get("links")
assert isinstance(links, list)
assert len(list(filter(lambda _link: _link["rel"].endswith("results"), links))) == 1
# first test to get job results details, but not downloading yet
result = mocked_sub_requests(self.app, self.client.results, job_id)
assert result.success, result.text
assert "undefined" not in result.message
outputs_body = result.body
assert isinstance(outputs_body, dict) and len(outputs_body) == 1
output = outputs_body.get("output") # single of this process
assert isinstance(output, dict) and "href" in output, "Output named 'output' should be a 'File' reference."
output_href = output.get("href")
assert isinstance(output_href, str) and output_href.startswith(self.settings["weaver.wps_output_url"])
# test download feature
with contextlib.ExitStack() as stack:
server_mock = stack.enter_context(mocked_wps_output(self.settings))
target_dir = stack.enter_context(tempfile.TemporaryDirectory())
result = mocked_sub_requests(self.app, self.client.results,
job_id, download=True, out_dir=target_dir, # 'client.results' parameters
only_local=True) # mock parameter (avoid download HTTP redirect to TestApp)
assert result.success, result.text
assert "undefined" not in result.message
assert result.body != outputs_body, "Download operation should modify the original outputs body."
output = result.body.get("output", {})
assert output.get("href") == output_href
output_path = output.get("path") # inserted by download
assert isinstance(output_path, str) and output_path.startswith(target_dir)
output_name = output_href.split(job_id)[-1][1:] # everything after jobID, and without the first '/'
output_file = os.path.join(target_dir, output_name)
assert output_path == output_file
assert os.path.isfile(output_file) and not os.path.islink(output_file)
assert len(server_mock.calls) == 1 # list of (PreparedRequest, Response)
assert server_mock.calls[0][0].url == output_href
def test_vault_encrypted_decrypted_contents(self):
"""
Validate that Vault file gets encrypted on upload, that description still works, and is decrypted on download.
"""
# upload file
data = {"fake": "data"}
text = json.dumps(data)
with tempfile.NamedTemporaryFile(suffix=".json",
mode="w+") as tmp_file:
tmp_file.write(text)
tmp_file.seek(0)
tmp_name = os.path.split(tmp_file.name)[-1]
lines = mocked_sub_requests(
self.app,
run_command,
[
# weaver
"upload",
"-u",
self.url,
"-f",
tmp_file.name,
],
trim=False,
entrypoint=weaver_cli,
only_local=True,
)
result = json.loads("\n".join(lines))
assert "file_id" in result
# check encrypted contents in vault
vault_dir = self.settings.get("weaver.vault_dir")
vault_id = result["file_id"]
vault_path = os.path.join(vault_dir, vault_id, tmp_name)
with open(vault_path, mode="r", encoding="utf-8") as vault_fd:
vault_data = vault_fd.read()
assert vault_data != data
assert "{" not in vault_data
assert vault_data.endswith("=")
# check details
vault_token = result["access_token"]
vault_url = f"/vault/{vault_id}"
resp = mocked_sub_requests(self.app,
"HEAD",
vault_url,
headers={"X-Auth-Vault": vault_token})
assert resp.status_code == 200
assert resp.headers["Content-Type"].startswith(ContentType.APP_JSON)
# check download decrypted
resp = mocked_sub_requests(self.app,
"GET",
vault_url,
headers={"X-Auth-Vault": vault_token})
assert resp.status_code == 200
assert resp.text == text
def test_undeploy(self):
# deploy a new process to leave the test one available
other_payload = copy.deepcopy(self.test_payload)
other_process = self.test_process + "-other"
self.deploy_process(other_payload, process_id=other_process)
result = mocked_sub_requests(self.app, self.client.undeploy, other_process)
assert result.success
assert result.body.get("undeploymentDone", None) is True
assert "undefined" not in result.message
path = f"/processes/{other_process}"
resp = mocked_sub_requests(self.app, "get", path, expect_errors=True)
assert resp.status_code == 404
def deploy_process(cls, payload):
# type: (JSON) -> JSON
"""
Deploys a process with :paramref:`payload`.
:returns: resulting tuple of ``(process-description, package)`` JSON responses.
"""
resp = mocked_sub_requests(cls.app,
"post_json",
"/processes",
data=payload,
headers=cls.json_headers)
# TODO: status should be 201 when properly modified to match API conformance
assert resp.status_code == 200, "Expected successful deployment.\nError:\n{}".format(
resp.text)
path = resp.json["processSummary"]["processDescriptionURL"]
body = {"value": VISIBILITY_PUBLIC}
resp = cls.app.put_json("{}/visibility".format(path),
params=body,
headers=cls.json_headers)
assert resp.status_code == 200, "Expected successful visibility.\nError:\n{}".format(
resp.text)
info = []
for pkg_url in [path, "{}/package".format(path)]:
resp = cls.app.get(pkg_url, headers=cls.json_headers)
assert resp.status_code == 200
info.append(deepcopy(resp.json))
return info
def run_execute_inputs_schema_variant(self, inputs_param, preload=False, expect_success=True, mock_exec=True):
if isinstance(inputs_param, str):
if preload:
inputs_param = self.load_resource_file(inputs_param)
else:
inputs_param = self.get_resource_file(inputs_param)
with contextlib.ExitStack() as stack_exec:
# use pass-through function because don't care about execution result here, only the parsing of I/O
if mock_exec:
mock_exec_func = lambda *_, **__: None # noqa
else:
mock_exec_func = None
for mock_exec_proc in mocked_execute_process(func_execute_process=mock_exec_func):
stack_exec.enter_context(mock_exec_proc)
result = mocked_sub_requests(self.app, self.client.execute, self.test_process, inputs=inputs_param)
if expect_success:
assert result.success, result.text
assert "jobID" in result.body
assert "processID" in result.body
assert "status" in result.body
assert "location" in result.body
assert result.body["processID"] == self.test_process
assert result.body["status"] == STATUS_ACCEPTED
assert result.body["location"] == result.headers["Location"]
assert "undefined" not in result.message
else:
assert not result.success, result.text
return result
def deploy_process(cls, payload, process_id=None, describe_schema=PROCESS_SCHEMA_OGC):
# type: (JSON, Optional[str], str) -> JSON
"""
Deploys a process with :paramref:`payload`.
:returns: resulting tuple of ``(process-description, package)`` JSON responses.
"""
if process_id:
payload["processDescription"]["process"]["id"] = process_id # type: ignore
exec_list = payload.get("executionUnit", [])
if len(exec_list):
# test-only feature:
# substitute 'href' starting by 'tests/' by the corresponding file in test resources
# this allows clean separation of deploy payload from CWL to allow reuse and test CWL locally beforehand
exec_href = exec_list[0].get("href", "")
if exec_href.startswith("tests/"):
exec_unit = load_file(os.path.join(WEAVER_ROOT_DIR, exec_href))
exec_list[0]["unit"] = exec_unit
exec_list[0].pop("href")
resp = mocked_sub_requests(cls.app, "post_json", "/processes", data=payload, headers=cls.json_headers)
assert resp.status_code == 201, "Expected successful deployment.\nError:\n{}".format(resp.text)
path = resp.json["processSummary"]["processDescriptionURL"]
body = {"value": VISIBILITY_PUBLIC}
resp = cls.app.put_json("{}/visibility".format(path), params=body, headers=cls.json_headers)
assert resp.status_code == 200, "Expected successful visibility.\nError:\n{}".format(resp.text)
info = []
for info_path in ["{}?schema={}".format(path, describe_schema), "{}/package".format(path)]:
resp = cls.app.get(info_path, headers=cls.json_headers)
assert resp.status_code == 200
info.append(deepcopy(resp.json))
return info
def test_deploy_payload_file_cwl_embedded(self):
test_id = f"{self.test_process_prefix}-deploy-file-no-cwl"
payload = self.load_resource_file("DeployProcess_Echo.yml")
package = self.get_resource_file("echo.cwl")
payload["executionUnit"][0] = {"href": package}
with tempfile.NamedTemporaryFile(mode="w", suffix=".cwl") as body_file:
json.dump(payload, body_file)
body_file.flush()
body_file.seek(0)
lines = mocked_sub_requests(
self.app, run_command,
[
# weaver
"deploy",
"-p", test_id,
"-b", body_file.name,
self.url
],
trim=False,
entrypoint=weaver_cli,
only_local=True,
)
assert any(f"\"id\": \"{test_id}\"" in line for line in lines)
assert any("\"deploymentDone\": true" in line for line in lines)
def test_dismiss(self):
for status in [STATUS_ACCEPTED, STATUS_FAILED, STATUS_RUNNING, STATUS_SUCCEEDED]:
job = self.job_store.save_job(task_id="12345678-1111-2222-3333-111122223333", process=self.test_process)
job.status = status
job = self.job_store.update_job(job)
result = mocked_sub_requests(self.app, self.client.dismiss, str(job.id))
assert result.success
assert "undefined" not in result.message
def test_execute_manual_monitor(self):
with contextlib.ExitStack() as stack_exec:
for mock_exec_proc in mocked_execute_process():
stack_exec.enter_context(mock_exec_proc)
lines = mocked_sub_requests(
self.app, run_command,
[
# "weaver",
"execute",
self.url,
"-p", self.test_process,
"-I", "message='TEST MESSAGE!'"
],
trim=False,
entrypoint=weaver_cli,
only_local=True,
)
# ignore indents of fields from formatted JSON content
assert any(f"\"processID\": \"{self.test_process}\"" in line for line in lines)
assert any("\"jobID\": \"" in line for line in lines)
assert any("\"location\": \"" in line for line in lines)
job_loc = [line for line in lines if "location" in line][0]
job_ref = [line for line in job_loc.split("\"") if line][-1]
job_id = job_ref.split("/")[-1]
lines = mocked_sub_requests(
self.app, run_command,
[
# "weaver",
"monitor",
"-j", job_ref,
"-T", 10,
"-W", 1,
],
trim=False,
entrypoint=weaver_cli,
only_local=True,
)
assert any(f"\"jobID\": \"{job_id}\"" in line for line in lines)
assert any(f"\"status\": \"{STATUS_SUCCEEDED}\"" in line for line in lines)
assert any(f"\"href\": \"{job_ref}/results\"" in line for line in lines)
assert any("\"rel\": \"http://www.opengis.net/def/rel/ogc/1.0/results\"" in line for line in lines)
def test_quote_atomic_process(self, mocked_estimate):
with contextlib.ExitStack() as stack_quote:
for mock_quote in mocked_execute_celery(
celery_task=
"weaver.quotation.estimation.process_quote_estimator"):
stack_quote.enter_context(mock_quote)
data = {
"inputs": {
"message": "test quote"
},
"outputs": {
"output": {
"transmissionMode": ExecuteTransmissionMode.VALUE
}
}
}
path = sd.process_quotes_service.path.format(process_id="Echo")
resp = mocked_sub_requests(self.app,
"POST",
path,
json=data,
headers=self.json_headers,
only_local=True)
assert resp.status_code == 202 # 'Accepted' async task, but already finished by skipping celery
assert mocked_estimate.called
body = resp.json
assert body["status"] == QuoteStatus.SUBMITTED
path = sd.process_quote_service.path.format(process_id="Echo",
quote_id=body["id"])
resp = mocked_sub_requests(self.app,
"GET",
path,
headers=self.json_headers,
only_local=True)
assert resp.status_code == 200
body = resp.json
assert body["status"] == QuoteStatus.COMPLETED
assert isinstance(body["price"], float) and body["price"] > 0
assert isinstance(body["currency"],
str) and body["currency"] == "CAD"
assert isinstance(body["estimatedSeconds"],
int) and body["estimatedSeconds"] > 0
def test_jsonarray2netcdf_execute_async_output_by_reference_dontcare_response_document(
self):
"""
Jobs submitted with ``response=document`` are not impacted by ``transmissionMode``.
The results schema should always be returned when document is requested.
.. seealso::
https://docs.ogc.org/is/18-062r2/18-062r2.html#req_core_process-execute-sync-document
"""
with contextlib.ExitStack() as stack_exec:
body, nc_data = self.setup_inputs(stack_exec)
body.update({
"response":
ExecuteResponse.
DOCUMENT, # by value/reference don't care because of this
"outputs": [{
"id":
"output",
"transmissionMode":
ExecuteTransmissionMode.REFERENCE
}],
})
for mock_exec in mocked_execute_celery():
stack_exec.enter_context(mock_exec)
path = "/processes/jsonarray2netcdf/jobs"
resp = mocked_sub_requests(self.app,
"post_json",
path,
data=body,
headers=self.json_headers,
only_local=True)
assert resp.content_type in ContentType.APP_JSON
assert resp.status_code == 201, f"Error: {resp.json}"
job_url = resp.json["location"]
self.monitor_job(
job_url, return_status=True) # don't fetch results automatically
resp = self.app.get(f"{job_url}/results", headers=self.json_headers)
assert resp.status_code == 200, f"Error: {resp.text}"
assert resp.content_type == ContentType.APP_JSON
result_links = [
hdr for hdr in resp.headers if hdr[0].lower() == "link"
]
assert len(result_links) == 0
results = resp.json
# even though results are requested by Link reference,
# Weaver still offers them with document on outputs endpoint
output_url = job_url + "/outputs"
resp = self.app.get(output_url, headers=self.json_headers)
assert resp.status_code == 200, f"Error job outputs:\n{resp.text}"
outputs = resp.json
self.validate_results(results, outputs, nc_data, result_links)
def test_quote_bad_inputs(self):
path = sd.process_quotes_service.path.format(process_id="Echo")
data = {"inputs": [1, 2, 3]}
resp = mocked_sub_requests(self.app,
"POST",
path,
json=data,
headers=self.json_headers,
only_local=True)
assert resp.status_code == 400
def test_jsonarray2netcdf_execute_async_output_by_value_response_raw(self):
"""
Jobs submitted with ``response=raw`` and single output as ``transmissionMode=value`` must return its raw data.
.. seealso::
https://docs.ogc.org/is/18-062r2/18-062r2.html#req_core_process-execute-sync-raw-value-one
"""
with contextlib.ExitStack() as stack_exec:
body, nc_data = self.setup_inputs(stack_exec)
body.update({
"response":
ExecuteResponse.RAW, # by value/reference important here
# NOTE: quantity of outputs important as well
# since single output, content-type is directly that output (otherwise should be multipart)
"outputs": [{
"id": "output",
"transmissionMode": ExecuteTransmissionMode.VALUE
}], # data dump
})
for mock_exec in mocked_execute_celery():
stack_exec.enter_context(mock_exec)
path = "/processes/jsonarray2netcdf/jobs"
resp = mocked_sub_requests(self.app,
"post_json",
path,
data=body,
headers=self.json_headers,
only_local=True)
assert resp.content_type in ContentType.APP_JSON
assert resp.status_code == 201, f"Error: {resp.json}"
job_url = resp.json["location"]
self.monitor_job(
job_url, return_status=True) # don't fetch results automatically
resp = self.app.get(f"{job_url}/results", headers=self.json_headers)
assert resp.status_code < 400, f"Error: {resp.text}"
assert resp.status_code == 200, "Body should contain literal raw data dump"
assert resp.content_type in ContentType.APP_NETCDF, "raw result by value should be directly the content-type"
assert resp.text == nc_data, "raw result by value should be directly the data content"
assert resp.headers
result_links = [
hdr for hdr in resp.headers if hdr[0].lower() == "link"
]
assert len(result_links) == 0
# even though results are requested by raw data,
# Weaver still offers them with document on outputs endpoint
output_url = job_url + "/outputs"
resp = self.app.get(output_url, headers=self.json_headers)
assert resp.status_code == 200, f"Error job outputs:\n{resp.text}"
outputs = resp.json
self.validate_results(None, outputs, nc_data, result_links)
def test_jsonarray2netcdf_execute_async_output_by_reference_response_raw(
self):
"""
Jobs submitted with ``response=raw`` and single output as ``transmissionMode=reference`` must a link.
Contents should be empty, and the reference should be provided with HTTP ``Link`` header.
.. seealso::
https://docs.ogc.org/is/18-062r2/18-062r2.html#req_core_process-execute-sync-raw-ref
"""
with contextlib.ExitStack() as stack_exec:
body, nc_data = self.setup_inputs(stack_exec)
body.update({
"response":
ExecuteResponse.RAW, # by value/reference important here
"outputs": [{
"id":
"output",
"transmissionMode":
ExecuteTransmissionMode.REFERENCE
}], # Link header
})
for mock_exec in mocked_execute_celery():
stack_exec.enter_context(mock_exec)
path = "/processes/jsonarray2netcdf/jobs"
resp = mocked_sub_requests(self.app,
"post_json",
path,
data=body,
headers=self.json_headers,
only_local=True)
assert resp.content_type in ContentType.APP_JSON
assert resp.status_code == 201, f"Error: {resp.json}"
job_url = resp.json["location"]
self.monitor_job(
job_url, return_status=True) # don't fetch results automatically
resp = self.app.get(f"{job_url}/results", headers=self.json_headers)
assert resp.status_code < 400, f"Error: {resp.text}"
assert resp.status_code == 204, "Body should be empty since all outputs requested by reference (Link header)"
assert resp.content_type is None
assert resp.headers
result_links = [hdr for hdr in resp.headers if hdr[0] == "Link"]
# even though results are requested by Link reference,
# Weaver still offers them with document on outputs endpoint
resp = self.app.get(f"{job_url}/outputs", headers=self.json_headers)
assert resp.status_code == 200, f"Error job outputs:\n{resp.json}"
outputs = resp.json
self.validate_results(None, outputs, nc_data, result_links)
def test_quote_workflow_process(self, mocked_estimate):
with contextlib.ExitStack() as stack_quote:
for mock_quote in mocked_execute_celery(
celery_task=
"weaver.quotation.estimation.process_quote_estimator"):
stack_quote.enter_context(mock_quote)
path = os.path.join(APP_PKG_ROOT, "WorkflowChainStrings",
"execute.json")
with open(path, mode="r", encoding="utf-8") as exec_file:
data = yaml.safe_load(exec_file)
path = sd.process_quotes_service.path.format(
process_id="WorkflowChainStrings")
resp = mocked_sub_requests(self.app,
"POST",
path,
json=data,
headers=self.json_headers,
only_local=True)
assert resp.status_code == 202 # 'Accepted' async task, but already finished by skipping celery
assert mocked_estimate.called
body = resp.json
assert body["status"] == QuoteStatus.SUBMITTED
path = sd.process_quote_service.path.format(process_id="Echo",
quote_id=body["id"])
resp = mocked_sub_requests(self.app,
"GET",
path,
headers=self.json_headers,
only_local=True)
assert resp.status_code == 200
body = resp.json
assert body["status"] == QuoteStatus.COMPLETED
assert isinstance(body["price"], float) and body["price"] > 0
assert isinstance(body["currency"],
str) and body["currency"] == "CAD"
assert isinstance(body["estimatedSeconds"],
int) and body["estimatedSeconds"] > 0
def test_deploy_payload_body_cwl_embedded(self):
test_id = f"{self.test_process_prefix}-deploy-body-no-cwl"
payload = self.load_resource_file("DeployProcess_Echo.yml")
package = self.load_resource_file("echo.cwl")
payload["executionUnit"][0] = {"unit": package}
result = mocked_sub_requests(self.app, self.client.deploy, test_id, payload)
assert result.success
assert "processSummary" in result.body
assert result.body["processSummary"]["id"] == test_id
assert "deploymentDone" in result.body
assert result.body["deploymentDone"] is True
assert "undefined" not in result.message
def test_deploy_payload_inject_cwl_file(self):
test_id = f"{self.test_process_prefix}-deploy-body-with-cwl-file"
payload = self.load_resource_file("DeployProcess_Echo.yml")
package = self.get_resource_file("echo.cwl")
payload.pop("executionUnit", None)
result = mocked_sub_requests(self.app, self.client.deploy, test_id, payload, package)
assert result.success
assert "processSummary" in result.body
assert result.body["processSummary"]["id"] == test_id
assert "deploymentDone" in result.body
assert result.body["deploymentDone"] is True
assert "undefined" not in result.message
def process_listing_op(self, operation):
result = mocked_sub_requests(self.app, operation)
assert result.success
assert "processes" in result.body
assert set(result.body["processes"]) == {
# builtin
"file2string_array",
"file_index_selector",
"jsonarray2netcdf",
"metalink2netcdf",
# test process
self.test_process,
}
assert "undefined" not in result.message
def test_execute_wps_rest_resp_json(self):
"""
Test validates that basic Docker application runs successfully, fetching the reference as needed.
The job execution is launched using the WPS-REST endpoint for this test.
Both the request body and response content are JSON.
.. seealso::
- :meth:`test_execute_wps_kvp_get_resp_xml`
- :meth:`test_execute_wps_kvp_get_resp_json`
- :meth:`test_execute_wps_xml_post_resp_xml`
- :meth:`test_execute_wps_xml_post_resp_json`
"""
test_content = "Test file in Docker - WPS-REST job endpoint"
with contextlib.ExitStack() as stack_exec:
# setup
dir_name = tempfile.gettempdir()
tmp_path = tempfile.NamedTemporaryFile(dir=dir_name, mode="w", suffix=".txt")
tmp_file = stack_exec.enter_context(tmp_path) # noqa
tmp_file.write(test_content)
tmp_file.seek(0)
exec_body = {
"mode": EXECUTE_MODE_ASYNC,
"response": EXECUTE_RESPONSE_DOCUMENT,
"inputs": [
{"id": "file", "href": tmp_file.name},
],
"outputs": [
{"id": self.out_key, "transmissionMode": EXECUTE_TRANSMISSION_MODE_REFERENCE},
]
}
for mock_exec in mocked_execute_process():
stack_exec.enter_context(mock_exec)
# execute
proc_url = "/processes/{}/jobs".format(self.process_id)
resp = mocked_sub_requests(self.app, "post_json", proc_url,
data=exec_body, headers=self.json_headers, only_local=True)
assert resp.status_code in [200, 201], "Failed with: [{}]\nReason:\n{}".format(resp.status_code, resp.json)
status_url = resp.json["location"]
job_id = resp.json["jobID"]
# job monitoring
result = self.monitor_job(status_url)
self.validate_outputs(job_id, result, test_content)
def test_log_options_any_level(self):
"""
Logging parameters should be allowed at main parser level or under any operation subparser.
"""
for options in [
["--verbose", "describe", self.url, "-p", self.test_process],
["describe", self.url, "--verbose", "-p", self.test_process],
["describe", self.url, "-p", self.test_process, "--verbose"],
]:
lines = mocked_sub_requests(
self.app, run_command,
options,
trim=False,
entrypoint=weaver_cli,
only_local=True,
)
assert any(f"\"id\": \"{self.test_process}\"" in line for line in lines)
def test_deploy_payload_file_cwl_embedded(self):
test_id = f"{self.test_process_prefix}-deploy-file-no-cwl"
payload = self.load_resource_file("DeployProcess_Echo.yml")
package = self.get_resource_file("echo.cwl")
payload["executionUnit"][0] = {"href": package}
with tempfile.NamedTemporaryFile(mode="w", suffix=".cwl") as body_file:
json.dump(payload, body_file)
body_file.flush()
body_file.seek(0)
result = mocked_sub_requests(self.app, self.client.deploy, test_id, body_file.name)
assert result.success
assert "processSummary" in result.body
assert result.body["processSummary"]["id"] == test_id
assert "deploymentDone" in result.body
assert result.body["deploymentDone"] is True
assert "undefined" not in result.message
def test_describe(self):
# prints formatted JSON ProcessDescription over many lines
lines = mocked_sub_requests(
self.app, run_command,
[
# "weaver",
"describe",
self.url,
"-p", self.test_process,
],
trim=False,
entrypoint=weaver_cli,
only_local=True,
)
# ignore indents of fields from formatted JSON content
assert any(f"\"id\": \"{self.test_process}\"" in line for line in lines)
assert any("\"inputs\": {" in line for line in lines)
assert any("\"outputs\": {" in line for line in lines)
def test_deploy_no_process_id_option(self):
payload = self.get_resource_file("DeployProcess_Echo.yml")
package = self.get_resource_file("echo.cwl")
lines = mocked_sub_requests(
self.app, run_command,
[
# weaver
"deploy",
"--body", payload, # no --process/--id, but available through --body
"--cwl", package,
self.url
],
trim=False,
entrypoint=weaver_cli,
only_local=True,
)
assert any("\"id\": \"Echo\"" in line for line in lines)
assert any("\"deploymentDone\": true" in line for line in lines)
def test_describe(self):
result = mocked_sub_requests(self.app, self.client.describe, self.test_process)
assert result.success
# see deployment file for details that are expected here
assert result.body["id"] == self.test_process
assert result.body["version"] == "1.0"
assert result.body["keywords"] == ["test", "application"] # app is added by Weaver since not CWL Workflow
assert "message" in result.body["inputs"]
assert result.body["inputs"]["message"]["title"] == "message"
assert result.body["inputs"]["message"]["description"] == "Message to echo."
assert result.body["inputs"]["message"]["minOccurs"] == 1
assert result.body["inputs"]["message"]["maxOccurs"] == 1
assert result.body["inputs"]["message"]["literalDataDomains"][0]["dataType"]["name"] == "string"
assert "output" in result.body["outputs"]
assert result.body["outputs"]["output"]["title"] == "output"
assert result.body["outputs"]["output"]["description"] == "Output file with echo message."
assert result.body["outputs"]["output"]["formats"] == [{"default": True, "mediaType": CONTENT_TYPE_TEXT_PLAIN}]
assert "undefined" not in result.message, "CLI should not have confused process description as response detail."
assert "description" not in result.body, "CLI should not have overridden the process description field."
def test_execute_docker_embedded_python_script(self):
test_proc = "test-docker-python-script"
cwl = load_file(os.path.join(WEAVER_ROOT_DIR, "docs/examples/docker-python-script-report.cwl"))
body = {
"processDescription": {
"process": {
"id": test_proc
}
},
"executionUnit": [{"unit": cwl}],
"deploymentProfileName": "http://www.opengis.net/profiles/eoc/dockerizedApplication"
}
self.deploy_process(body)
with contextlib.ExitStack() as stack:
for mock_exec in mocked_execute_process():
stack.enter_context(mock_exec)
path = f"/processes/{test_proc}/execution"
cost = 2.45
amount = 3
body = {
"mode": EXECUTE_MODE_ASYNC,
"response": EXECUTE_RESPONSE_DOCUMENT,
"inputs": [
{"id": "amount", "value": amount},
{"id": "cost", "value": cost}
],
"outputs": [
{"id": "quote", "transmissionMode": EXECUTE_TRANSMISSION_MODE_REFERENCE},
]
}
resp = mocked_sub_requests(self.app, "POST", path, json=body, headers=self.json_headers, only_local=True)
status_url = resp.headers["Location"]
results = self.monitor_job(status_url)
assert results["quote"]["href"].startswith("http")
stack.enter_context(mocked_wps_output(self.settings))
tmpdir = stack.enter_context(tempfile.TemporaryDirectory())
report_file = fetch_file(results["quote"]["href"], tmpdir, self.settings)
report_data = load_file(report_file, text=True)
assert report_data == f"Order Total: {amount * cost:0.2f}$\n"
def test_execute_invalid_format(self):
bad_input_value = "'this is my malformed message'" # missing '<id>=' portion
lines = mocked_sub_requests(
self.app, run_command,
[
# "weaver",
"execute",
self.url,
"-p", self.test_process,
"-I", bad_input_value,
"-M",
"-T", 10,
"-W", 1
],
trim=False,
entrypoint=weaver_cli,
expect_error=True,
only_local=True,
)
assert any(bad_input_value in line for line in lines)
def test_deploy_payload_body_cwl_embedded(self):
test_id = f"{self.test_process_prefix}-deploy-body-no-cwl"
payload = self.load_resource_file("DeployProcess_Echo.yml")
package = self.load_resource_file("echo.cwl")
payload["executionUnit"][0] = {"unit": package}
lines = mocked_sub_requests(
self.app, run_command,
[
# weaver
"deploy",
"-p", test_id,
"-b", json.dumps(payload), # literal JSON string accepted for CLI
self.url
],
trim=False,
entrypoint=weaver_cli,
only_local=True,
)
assert any(f"\"id\": \"{test_id}\"" in line for line in lines)
assert any("\"deploymentDone\": true" in line for line in lines)
def test_jsonarray2netcdf_execute_async(self):
with contextlib.ExitStack() as stack_exec:
body, nc_data = self.setup_inputs(stack_exec)
body.update({
"mode":
ExecuteMode.ASYNC,
"response":
ExecuteResponse.DOCUMENT,
"outputs": [{
"id": "output",
"transmissionMode": ExecuteTransmissionMode.VALUE
}],
})
for mock_exec in mocked_execute_celery():
stack_exec.enter_context(mock_exec)
path = "/processes/jsonarray2netcdf/jobs"
resp = mocked_sub_requests(self.app,
"post_json",
path,
data=body,
headers=self.json_headers,
only_local=True)
assert resp.status_code == 201, f"Error: {resp.json}"
assert resp.content_type in ContentType.APP_JSON
# following details not available yet in async, but are in sync
assert "created" not in resp.json
assert "finished" not in resp.json
assert "duration" not in resp.json
assert "progress" not in resp.json
job_url = resp.json["location"]
results = self.monitor_job(job_url)
output_url = job_url + "/outputs"
resp = self.app.get(output_url, headers=self.json_headers)
assert resp.status_code == 200, f"Error job outputs:\n{resp.json}"
outputs = resp.json
self.validate_results(results, outputs, nc_data, None)
def test_deploy_payload_inject_cwl_file(self):
test_id = f"{self.test_process_prefix}-deploy-body-with-cwl-file"
payload = self.load_resource_file("DeployProcess_Echo.yml")
package = self.get_resource_file("echo.cwl")
payload.pop("executionUnit", None)
lines = mocked_sub_requests(
self.app, run_command,
[
# weaver
"deploy",
"-p", test_id,
"--body", json.dumps(payload), # literal JSON string accepted for CLI
"--cwl", package,
self.url
],
trim=False,
entrypoint=weaver_cli,
only_local=True,
)
assert any(f"\"id\": \"{test_id}\"" in line for line in lines)
assert any("\"deploymentDone\": true" in line for line in lines)
