async def test_put_component_transformation_without_update_code(
async_test_client, clean_test_db_engine):
patched_session = sessionmaker(clean_test_db_engine)
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
patched_session,
):
path = "./tests/data/components/alerts-from-score_100_38f168ef-cb06-d89c-79b3-0cd823f32e9d.json"
example_component_tr_json = load_json(path)
async with async_test_client as ac:
response = await ac.put(
posix_urljoin("/api/transformations/",
example_component_tr_json["id"]) +
"?update_component_code=False",
json=example_component_tr_json,
)
component_tr_in_db = read_single_transformation_revision(
example_component_tr_json["id"])
assert response.status_code == 201
assert "COMPONENT_INFO" not in response.json()["content"]
assert "COMPONENT_INFO" not in component_tr_in_db.content
assert "register" in response.json()["content"]
assert "register" in component_tr_in_db.content
async def test_execute_for_component_without_hetdesrun_imports(
async_test_client, clean_test_db_engine
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
path = (
"./tests/data/components/"
"alerts-from-score_100_38f168ef-cb06-d89c-79b3-0cd823f32e9d"
".json"
)
component_tr_json = load_json(path)
wiring_json = {
"id": "38f168ef-cb06-d89c-79b3-0cd823f32e9d",
"name": "STANDARD-WIRING",
"inputWirings": [
{
"id": "8c249f92-4b81-457e-9371-24204d6b373b",
"workflowInputName": "scores",
"adapterId": "direct_provisioning",
"filters": {
"value": (
"{\n"
' "2020-01-03T08:20:03.000Z": 18.7,\n'
' "2020-01-01T01:15:27.000Z": 42.2,\n'
' "2020-01-03T08:20:04.000Z": 25.9\n'
"}"
)
},
},
{
"id": "0f0f97f7-1f5d-4f5d-be11-7c7b78d02129",
"workflowInputName": "threshold",
"adapterId": "direct_provisioning",
"filters": {"value": "30"},
},
],
"outputWirings": [],
}
tr = TransformationRevision(**component_tr_json)
tr.content = update_code(tr)
assert "COMPONENT_INFO" in tr.content
store_single_transformation_revision(tr)
async with async_test_client as ac:
response = await ac.post(
"/api/components/" + component_tr_json["id"] + "/execute",
json=wiring_json,
)
assert response.status_code == 200
assert "output_types_by_output_name" in response.json()
async def test_execute_for_full_workflow_dto(async_test_client,
clean_test_db_engine):
patched_session = sessionmaker(clean_test_db_engine)
with mock.patch(
"hetdesrun.persistence.dbservice.nesting.Session",
patched_session,
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
patched_session,
):
async with async_test_client as ac:
json_files = [
"./transformations/components/connectors/pass-through-integer_100_57eea09f-d28e-89af-4e81-2027697a3f0f.json",
"./transformations/components/connectors/pass-through-series_100_bfa27afc-dea8-b8aa-4b15-94402f0739b6.json",
"./transformations/components/connectors/pass-through-string_100_2b1b474f-ddf5-1f4d-fec4-17ef9122112b.json",
"./transformations/components/remaining-useful-life/univariate-linear-rul-regression_100_8d61a267-3a71-51cd-2817-48c320469d6b.json",
"./transformations/components/visualization/univariate-linear-rul-regression-result-plot_100_9c3f88ce-1311-241e-18b7-acf7d3f5a051.json",
"./transformations/components/arithmetic/consecutive-differences_100_ce801dcb-8ce1-14ad-029d-a14796dcac92.json",
"./transformations/components/basic/filter_100_18260aab-bdd6-af5c-cac1-7bafde85188f.json",
"./transformations/components/basic/greater-or-equal_100_f759e4c0-1468-0f2e-9740-41302b860193.json",
"./transformations/components/basic/last-datetime-index_100_c8e3bc64-b214-6486-31db-92a8888d8991.json",
"./transformations/components/basic/restrict-to-time-interval_100_bf469c0a-d17c-ca6f-59ac-9838b2ff67ac.json",
"./transformations/components/connectors/pass-through-float_100_2f511674-f766-748d-2de3-ad5e62e10a1a.json",
"./transformations/components/visualization/single-timeseries-plot_100_8fba9b51-a0f1-6c6c-a6d4-e224103b819c.json",
"./transformations/workflows/examples/data-from-last-positive-step_100_2cbb87e7-ea99-4404-abe1-be550f22763f.json",
"./transformations/workflows/examples/univariate-linear-rul-regression-example_100_806df1b9-2fc8-4463-943f-3d258c569663.json",
"./transformations/workflows/examples/linear-rul-from-last-positive-step_100_3d504361-e351-4d52-8734-391aa47e8f24.json",
]
for file in json_files:
tr_json = load_json(file)
response = await ac.put(
posix_urljoin(
get_config().hd_backend_api_url,
"transformations",
tr_json["id"],
) + "?allow_overwrite_released=True",
json=tr_json,
)
workflow_id = UUID("3d504361-e351-4d52-8734-391aa47e8f24")
tr_workflow = read_single_transformation_revision(workflow_id)
wiring_dto = WiringFrontendDto.from_wiring(
tr_workflow.test_wiring, workflow_id)
response = await ac.post(
"/api/workflows/" + str(workflow_id) + "/execute",
json=json.loads(wiring_dto.json(by_alias=True)),
)
assert response.status_code == 200
assert "output_types_by_output_name" in response.json()
async def test_put_workflow_transformation(async_test_client,
clean_test_db_engine):
patched_session = sessionmaker(clean_test_db_engine)
with mock.patch(
"hetdesrun.persistence.dbservice.nesting.Session",
patched_session,
):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
patched_session,
):
example_workflow_tr_json = load_json(
"./transformations/workflows/examples/data-from-last-positive-step_100_2cbb87e7-ea99-4404-abe1-be550f22763f.json"
)
async with async_test_client as ac:
response = await ac.put(
posix_urljoin("/api/transformations/",
example_workflow_tr_json["id"]),
json=example_workflow_tr_json,
)
assert response.status_code == 201
from uuid import uuid4
import pytest
from pydantic import ValidationError
from hetdesrun.backend.execution import nested_nodes
from hetdesrun.backend.models.component import ComponentRevisionFrontendDto
from hetdesrun.exportimport.importing import load_json
from hetdesrun.models.wiring import WorkflowWiring
from hetdesrun.persistence.models.io import IOInterface
from hetdesrun.persistence.models.transformation import TransformationRevision
from hetdesrun.persistence.models.workflow import WorkflowContent
from hetdesrun.utils import State, Type, get_uuid_from_seed
tr_json_valid_released_example = load_json(
"./transformations/workflows/examples/iso-forest-example_100_67c14cf2-cd4e-410e-9aca-6664273ccc3f.json"
)
valid_component_dto_dict = {
"category":
"Arithmetic",
"code":
"",
"description":
"Calculates the modulo to some given b",
"groupId":
"ebb5b2d1-7c25-94dd-ca81-6a9e5b21bc2f",
"id":
"ebb5b2d1-7c25-94dd-ca81-6a9e5b21bc2f",
"inputs": [
{
def gen_execution_input_from_single_component(
component_json_path,
direct_provisioning_data_dict=None,
wf_wiring=None):
"""Wraps a single component into a workflow and generates the execution input json
input data is provided directly
"""
if (direct_provisioning_data_dict is None) == (wf_wiring is None):
raise ValueError(
"Excatly one of direct_provisioning_data_dict or wf_wiring must be provided"
)
tr_component_json = load_json(component_json_path)
code = tr_component_json["content"]
# Build up execution input Json
code_module_uuid = str(get_uuid_from_seed("code_module_uuid"))
component_uuid = str(get_uuid_from_seed("component_uuid"))
comp_inputs = [
ComponentInput(id=str(uuid4()),
name=inp["name"],
type=inp["data_type"])
for inp in tr_component_json["io_interface"]["inputs"]
]
comp_outputs = [
ComponentOutput(id=str(uuid4()),
name=outp["name"],
type=outp["data_type"])
for outp in tr_component_json["io_interface"]["outputs"]
]
component_node_id = "component_node_id"
return WorkflowExecutionInput(
code_modules=[CodeModule(code=code, uuid=code_module_uuid)],
components=[
ComponentRevision(
uuid=component_uuid,
name=tr_component_json["name"],
code_module_uuid=code_module_uuid,
function_name="main",
inputs=comp_inputs,
outputs=comp_outputs,
)
],
workflow=WorkflowNode(
id="root_node",
sub_nodes=[
ComponentNode(component_uuid=component_uuid,
id=component_node_id)
],
connections=[],
inputs=[
WorkflowInput(
id=str(
get_uuid_from_seed(
str(comp_input.id) + "_as_wf_input")),
id_of_sub_node=component_node_id,
name=comp_input.name,
name_in_subnode=comp_input.name,
type=comp_input.type,
) for comp_input in comp_inputs
],
outputs=[
WorkflowOutput(
id=str(
get_uuid_from_seed(
str(comp_output.id) + "_as_wf_output")),
id_of_sub_node=component_node_id,
name=comp_output.name,
name_in_subnode=comp_output.name,
type=comp_output.type,
) for comp_output in comp_outputs
],
name="root node",
),
configuration=ConfigurationInput(engine="plain",
run_pure_plot_operators=True),
workflow_wiring=WorkflowWiring(
input_wirings=[
InputWiring(
workflow_input_name=comp_input.name,
adapter_id=1,
filters={
"value": direct_provisioning_data_dict[comp_input.name]
},
) for comp_input in comp_inputs
],
output_wirings=[
OutputWiring(
workflow_output_name=comp_output.name,
adapter_id=1,
) for comp_output in comp_outputs
],
) if wf_wiring is None else wf_wiring,
)