async def handle_trafo_revision_execution_request(
# pylint: disable=redefined-builtin
exec_by_id: ExecByIdInput,
) -> ExecutionResponseFrontendDto:
"""Execute a transformation revision.
The transformation will be loaded from the DB and executed with the wiring sent in the request
body.
The test wiring will not be updated.
"""
if exec_by_id.job_id is None:
exec_by_id.job_id = uuid4()
try:
return await execute_transformation_revision(exec_by_id)
except TrafoExecutionNotFoundError as e:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=str(e)) from e
except TrafoExecutionRuntimeConnectionError as e:
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) from e
except TrafoExecutionResultValidationError as e:
raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=str(e)) from e
async def test_execute_for_transformation_revision(async_test_client,
clean_test_db_engine):
patched_session = sessionmaker(clean_test_db_engine)
with mock.patch(
"hetdesrun.persistence.dbservice.nesting.Session",
patched_session,
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
patched_session,
):
tr_component_1 = TransformationRevision(**tr_json_component_1)
tr_component_1.content = update_code(tr_component_1)
store_single_transformation_revision(tr_component_1)
tr_workflow_2 = TransformationRevision(**tr_json_workflow_2_update)
store_single_transformation_revision(tr_workflow_2)
update_or_create_nesting(tr_workflow_2)
exec_by_id_input = ExecByIdInput(
id=tr_workflow_2.id,
wiring=tr_workflow_2.test_wiring,
job_id=UUID("1270547c-b224-461d-9387-e9d9d465bbe1"),
)
async with async_test_client as ac:
response = await ac.post(
"/api/transformations/execute",
json=json.loads(exec_by_id_input.json()),
)
assert response.status_code == 200
resp_data = response.json()
assert "output_types_by_output_name" in resp_data
assert "job_id" in resp_data
assert UUID(resp_data["job_id"]) == UUID(
"1270547c-b224-461d-9387-e9d9d465bbe1")
async def execute_component_revision(
# pylint: disable=redefined-builtin
id: UUID,
wiring_dto: WiringFrontendDto,
run_pure_plot_operators: bool = False,
job_id: Optional[UUID] = None,
) -> ExecutionResponseFrontendDto:
"""Execute a transformation revision of type component.
This endpoint is deprecated and will be removed soon,
use POST /api/transformations/execute instead which uses a new model for the payload.
"""
if job_id is None:
job_id = uuid4()
exec_by_id = ExecByIdInput(
id=id,
wiring=wiring_dto.to_workflow_wiring(),
run_pure_plot_operators=run_pure_plot_operators,
job_id=job_id,
)
return await handle_trafo_revision_execution_request(exec_by_id)
async def consume_execution_trigger_message(
kafka_ctx: KafkaWorkerContext,
) -> None:
"""Executes transformation revisions as requested by Kafka messages to the respective topic"""
async for msg in kafka_ctx.consumer:
try:
logger.debug("Consumed msg: %s", str(msg))
logger.info(
(
"Consumer %s is with partition assignment %s is starting"
" to consume message from Kafka."
),
kafka_ctx.consumer_id,
str(kafka_ctx.consumer.assignment()),
)
try:
exec_by_id_input = ExecByIdInput.parse_raw(msg.value.decode("utf8"))
except ValidationError as validate_exec_by_id_input_error:
try:
exec_latest_by_group_id_input = ExecLatestByGroupIdInput.parse_raw(
msg.value.decode("utf8")
)
except ValidationError as validate_exec_latest_by_group_id_input_error:
msg = (
f"Kafka consumer {kafka_ctx.consumer_id} failed to parse message"
f" payload for execution.\n"
f"Validation Error assuming ExecByIdInput was\n"
f"{str(validate_exec_by_id_input_error)}\n"
f"Validation Error assuming ExecLatestByGroupIdInput was\n"
f"{str(validate_exec_latest_by_group_id_input_error)}\n"
f"Aborting."
)
kafka_ctx.last_unhandled_exception = (
validate_exec_latest_by_group_id_input_error
)
logger.error(msg)
continue
try:
latest_id = get_latest_revision_id(
exec_latest_by_group_id_input.revision_group_id
)
except DBNotFoundError as e:
msg = (
f"Kafka consumer {kafka_ctx.consumer_id} failed to receive"
f" id of latest revision of revision group "
f"{exec_latest_by_group_id_input.revision_group_id} from datatbase.\n"
f"Aborting."
)
kafka_ctx.last_unhandled_exception = e
logger.error(msg)
continue
exec_by_id_input = ExecByIdInput(
id=latest_id,
wiring=exec_latest_by_group_id_input.wiring,
run_pure_plot_operators=exec_latest_by_group_id_input.run_pure_plot_operators,
job_id=exec_latest_by_group_id_input.job_id,
)
logger.info(
"Start execution of trafo rev %s with job id %s from Kafka consumer %s",
str(exec_by_id_input.id),
str(exec_by_id_input.job_id),
kafka_ctx.consumer_id,
)
try:
exec_result = await execute_transformation_revision(exec_by_id_input)
except TrafoExecutionError as e:
msg = (
f"Kafka consumer failed to execute trafo rev {exec_by_id_input.id}"
f" for job {exec_by_id_input.job_id}. Error Message: {str(e)}. Aborting."
)
kafka_ctx.last_unhandled_exception = e
logger.error(msg)
continue
logger.info(
"Kafka consumer %s finished execution for job %s with result status %s. Error: %s",
kafka_ctx.consumer_id,
str(exec_by_id_input.job_id),
str(exec_result.result),
str(exec_result.error),
)
logger.debug("Kafka consumer execution result: \n%s", str(exec_result))
await producer_send_result_msg(kafka_ctx, exec_result)
except Exception as e: # pylint: disable=broad-except
kafka_ctx.last_unhandled_exception = e
logger.error(
"Unexpected Error during Kafka execution: %s. Aborting.", str(e)
)
continue
async def test_execute_for_nested_workflow(async_test_client,
clean_test_db_engine):
patched_session = sessionmaker(clean_test_db_engine)
with mock.patch(
"hetdesrun.persistence.dbservice.nesting.Session",
patched_session,
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
patched_session,
):
async with async_test_client as ac:
json_files = [
"./transformations/components/connectors/pass-through-integer_100_57eea09f-d28e-89af-4e81-2027697a3f0f.json",
"./transformations/components/connectors/pass-through-series_100_bfa27afc-dea8-b8aa-4b15-94402f0739b6.json",
"./transformations/components/connectors/pass-through-string_100_2b1b474f-ddf5-1f4d-fec4-17ef9122112b.json",
"./transformations/components/remaining-useful-life/univariate-linear-rul-regression_100_8d61a267-3a71-51cd-2817-48c320469d6b.json",
"./transformations/components/visualization/univariate-linear-rul-regression-result-plot_100_9c3f88ce-1311-241e-18b7-acf7d3f5a051.json",
"./transformations/components/arithmetic/consecutive-differences_100_ce801dcb-8ce1-14ad-029d-a14796dcac92.json",
"./transformations/components/basic/filter_100_18260aab-bdd6-af5c-cac1-7bafde85188f.json",
"./transformations/components/basic/greater-or-equal_100_f759e4c0-1468-0f2e-9740-41302b860193.json",
"./transformations/components/basic/last-datetime-index_100_c8e3bc64-b214-6486-31db-92a8888d8991.json",
"./transformations/components/basic/restrict-to-time-interval_100_bf469c0a-d17c-ca6f-59ac-9838b2ff67ac.json",
"./transformations/components/connectors/pass-through-float_100_2f511674-f766-748d-2de3-ad5e62e10a1a.json",
"./transformations/components/visualization/single-timeseries-plot_100_8fba9b51-a0f1-6c6c-a6d4-e224103b819c.json",
"./transformations/workflows/examples/data-from-last-positive-step_100_2cbb87e7-ea99-4404-abe1-be550f22763f.json",
"./transformations/workflows/examples/univariate-linear-rul-regression-example_100_806df1b9-2fc8-4463-943f-3d258c569663.json",
"./transformations/workflows/examples/linear-rul-from-last-positive-step_100_3d504361-e351-4d52-8734-391aa47e8f24.json",
]
for file in json_files:
tr_json = load_json(file)
response = await ac.put(
posix_urljoin(
get_config().hd_backend_api_url,
"transformations",
tr_json["id"],
) + "?allow_overwrite_released=True",
json=tr_json,
)
component_id = UUID("57eea09f-d28e-89af-4e81-2027697a3f0f")
updated_component = read_single_transformation_revision(
component_id)
updated_component.deprecate()
response = await ac.put(
"/api/transformations/" + str(component_id),
json=json.loads(updated_component.json(by_alias=True)),
)
# linear rul from last positive step
workflow_id = UUID("3d504361-e351-4d52-8734-391aa47e8f24")
tr_workflow = read_single_transformation_revision(workflow_id)
exec_by_id_input = ExecByIdInput(
id=workflow_id, wiring=tr_workflow.test_wiring)
response = await ac.post(
"/api/transformations/execute",
json=json.loads(exec_by_id_input.json()),
)
assert response.status_code == 200
assert "output_types_by_output_name" in response.json()
assert response.json()["result"] == "ok"
assert (abs(response.json()["output_results_by_output_name"]
["intercept"] - 2.88) < 0.01)
assert (response.json()["output_results_by_output_name"]
["before_step_detect"] == {})
assert (response.json()["output_results_by_output_name"]
["rul_regression_result_plot"] == {})
async def test_execute_for_separate_runtime_container(async_test_client,
clean_test_db_engine):
patched_session = sessionmaker(clean_test_db_engine)
with mock.patch(
"hetdesrun.persistence.dbservice.nesting.Session",
patched_session,
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
patched_session,
):
with mock.patch(
"hetdesrun.webservice.config.runtime_config",
is_runtime_service=False,
):
resp_mock = mock.Mock()
resp_mock.status_code = 200
resp_mock.json = mock.Mock(
return_value={
"output_results_by_output_name": {
"wf_output": 100
},
"output_types_by_output_name": {
"wf_output": "INT"
},
"result": "ok",
"job_id": "1270547c-b224-461d-9387-e9d9d465bbe1",
})
with mock.patch(
"hetdesrun.backend.execution.httpx.AsyncClient.post",
return_value=resp_mock,
) as mocked_post:
tr_component_1 = TransformationRevision(
**tr_json_component_1)
tr_component_1.content = update_code(tr_component_1)
store_single_transformation_revision(tr_component_1)
tr_workflow_2 = TransformationRevision(
**tr_json_workflow_2_update)
store_single_transformation_revision(tr_workflow_2)
update_or_create_nesting(tr_workflow_2)
exec_by_id_input = ExecByIdInput(
id=tr_workflow_2.id,
wiring=tr_workflow_2.test_wiring,
job_id=UUID("1270547c-b224-461d-9387-e9d9d465bbe1"),
)
async with async_test_client as ac:
response = await ac.post(
"/api/transformations/execute",
json=json.loads(exec_by_id_input.json()),
)
assert response.status_code == 200
resp_data = response.json()
assert "output_types_by_output_name" in resp_data
assert "job_id" in resp_data
assert UUID(resp_data["job_id"]) == UUID(
"1270547c-b224-461d-9387-e9d9d465bbe1")
mocked_post.assert_called_once()