def test_deleting(clean_test_db_engine):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
tr_draft_uuid = get_uuid_from_seed("draft")
tr_draft_object = TransformationRevision(
id=tr_draft_uuid,
revision_group_id=tr_draft_uuid,
name="Test",
description="Test description",
version_tag="1.0.0",
category="Test category",
state=State.DRAFT,
type=Type.COMPONENT,
content="code",
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
tr_released_uuid = get_uuid_from_seed("released")
tr_released_object = TransformationRevision(
id=tr_released_uuid,
revision_group_id=tr_released_uuid,
name="Test",
description="Test description",
version_tag="1.0.0",
category="Test category",
released_timestamp="2021-12-24 00:00",
state=State.RELEASED,
type=Type.COMPONENT,
content="code",
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
store_single_transformation_revision(tr_draft_object)
store_single_transformation_revision(tr_released_object)
delete_single_transformation_revision(tr_draft_uuid)
with pytest.raises(DBNotFoundError):
read_single_transformation_revision(tr_draft_uuid)
with pytest.raises(DBBadRequestError):
delete_single_transformation_revision(tr_released_uuid)
def to_wiring(self) -> WorkflowWiring:
return WorkflowWiring(
input_wirings=[iw.to_input_wiring() for iw in self.input_wirings],
output_wirings=[
ow.to_output_wiring() for ow in self.output_wirings
],
)
def to_transformation_revision(self,
documentation: str = ""
) -> TransformationRevision:
return TransformationRevision(
id=self.id,
revision_group_id=self.group_id,
name=self.name,
description=self.description,
category=self.category,
version_tag=self.tag,
released_timestamp=datetime.now()
if self.state == State.RELEASED else None,
disabled_timestamp=datetime.now()
if self.state == State.DISABLED else None,
state=self.state,
type=self.type,
documentation=documentation,
io_interface=IOInterface(
inputs=[input.to_io() for input in self.inputs],
outputs=[output.to_io() for output in self.outputs],
),
content=self.code,
test_wiring=self.wirings[0].to_wiring()
if len(self.wirings) > 0 else WorkflowWiring(),
)
def test_updating(clean_test_db_engine):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
tr_uuid = get_uuid_from_seed("test_updating")
tr_object = TransformationRevision(
id=tr_uuid,
revision_group_id=tr_uuid,
name="Test",
description="Test description",
version_tag="1.0.0",
category="Test category",
state=State.DRAFT,
type=Type.COMPONENT,
content="code",
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
store_single_transformation_revision(tr_object)
tr_object.name = "Test Update"
update_or_create_single_transformation_revision(tr_object)
received_tr_object = read_single_transformation_revision(tr_uuid)
assert tr_object == received_tr_object
def test_tr_validator_content_type_correct():
id = get_uuid_from_seed("test")
combi = namedtuple("combi", "type content")
incorrect_combis = (
combi(type=Type.WORKFLOW, content="test"),
combi(type=Type.COMPONENT, content=WorkflowContent()),
)
correct_combis = (
combi(type=Type.WORKFLOW, content=WorkflowContent()),
combi(type=Type.COMPONENT, content="test"),
)
for combi in incorrect_combis:
with pytest.raises(ValidationError):
TransformationRevision(
id=id,
revision_group_id=id,
name="Test",
description="Test description",
version_tag="1.0.0",
category="Test category",
state=State.DRAFT,
type=combi.type,
content=combi.content,
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
for combi in correct_combis:
# no validation errors
TransformationRevision(
id=id,
revision_group_id=id,
name="Test",
description="Test description",
version_tag="1.0.0",
category="Test category",
state=State.DRAFT,
type=combi.type,
content=combi.content,
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
def test_multiple_select(clean_test_db_engine):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
tr_template_id = get_uuid_from_seed("object_template")
tr_object_template = TransformationRevision(
id=tr_template_id,
revision_group_id=tr_template_id,
name="Test",
description="Test description",
version_tag="1.0.0",
category="Test category",
state=State.DRAFT,
type=Type.COMPONENT,
content="code",
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
tr_uuid_1 = get_uuid_from_seed("test_multiple_select_1")
tr_object_1 = tr_object_template.copy()
tr_object_1.id = tr_uuid_1
tr_object_1.revision_group_id = tr_uuid_1
store_single_transformation_revision(tr_object_1)
tr_uuid_2 = get_uuid_from_seed("test_multiple_select_2")
tr_object_2 = tr_object_1.copy()
tr_object_2.id = tr_uuid_2
tr_object_2.version_tag = "1.0.1"
store_single_transformation_revision(tr_object_2)
tr_object_3 = tr_object_template.copy()
tr_uuid_3 = get_uuid_from_seed("test_multiple_select_3")
tr_object_3.id = tr_uuid_3
tr_object_3.revision_group_id = tr_uuid_3
tr_object_3.release()
store_single_transformation_revision(tr_object_3)
results = select_multiple_transformation_revisions()
assert len(results) == 3
results = select_multiple_transformation_revisions(state=State.DRAFT)
assert len(results) == 2
results = select_multiple_transformation_revisions(state=State.RELEASED)
assert len(results) == 1
results = select_multiple_transformation_revisions(revision_group_id=tr_uuid_1)
assert len(results) == 2
results = select_multiple_transformation_revisions(type=Type.COMPONENT)
assert len(results) == 3
results = select_multiple_transformation_revisions(type=Type.WORKFLOW)
assert len(results) == 0
def to_workflow_wiring(self) -> WorkflowWiring:
return WorkflowWiring(
input_wirings=[
wiring.to_input_wiring() for wiring in self.input_wirings
],
output_wirings=[
wiring.to_output_wiring() for wiring in self.output_wirings
],
)
def component_creator(identifier: str) -> TransformationRevision:
tr_component = TransformationRevision(
id=get_uuid_from_seed("component " + identifier),
revision_group_id=get_uuid_from_seed("group of component " +
identifier),
name=identifier,
type="COMPONENT",
state="DRAFT",
version_tag="1.0.0",
io_interface=IOInterface(),
content="code",
test_wiring=WorkflowWiring(),
)
return tr_component
def workflow_creator(identifier: str) -> TransformationRevision:
tr_workflow = TransformationRevision(
id=get_uuid_from_seed("workflow " + identifier),
revision_group_id=get_uuid_from_seed("group of workflow " +
identifier),
name=identifier,
type="WORKFLOW",
state="DRAFT",
version_tag="1.0.0",
io_interface=IOInterface(),
content=WorkflowContent(),
test_wiring=WorkflowWiring(),
)
return tr_workflow
def test_tr_nonemptyvalidstr_regex_validator_fancy_characters():
id = get_uuid_from_seed("test")
TransformationRevision(
id=id,
revision_group_id=id,
name="bößä",
description="中文, español, Çok teşekkürler",
version_tag="(-_-) / =.= & +_+",
category="ไทย",
state=State.DRAFT,
type=Type.COMPONENT,
content="test",
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
def test_tr_validstr_regex_validator_empty():
id = get_uuid_from_seed("test")
TransformationRevision(
id=id,
revision_group_id=id,
name="Test",
description="",
version_tag="1.0.0",
category="Test category",
state=State.DRAFT,
type=Type.COMPONENT,
content="test",
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
def test_tr_shortnonemptyvalidstr_validator_max_characters():
id = get_uuid_from_seed("test")
with pytest.raises(ValidationError):
TransformationRevision(
id=id,
revision_group_id=id,
name="Name",
description="Test description",
version_tag="1.0.0.0.0.0.0.0.0.0.0",
category="Test category",
state=State.DRAFT,
type=Type.COMPONENT,
content="test",
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
def test_get_latest_revision_id(clean_test_db_engine):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
tr_template_id = get_uuid_from_seed("object_template")
tr_object_template = TransformationRevision(
id=get_uuid_from_seed("test_get_latest_revision_0"),
revision_group_id=tr_template_id,
name="Test",
description="Test description",
version_tag="1.0.0",
category="Test category",
state=State.DRAFT,
type=Type.COMPONENT,
content="code",
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
tr_object_1 = tr_object_template.copy()
tr_object_1.id = get_uuid_from_seed("test_get_latest_revision_1")
tr_object_1.version_tag = "1.0.1"
tr_object_1.release()
store_single_transformation_revision(tr_object_1)
tr_object_2 = tr_object_template.copy()
tr_object_2.id = get_uuid_from_seed("test_get_latest_revision_2")
tr_object_2.version_tag = "1.0.2"
tr_object_2.release()
store_single_transformation_revision(tr_object_2)
assert get_latest_revision_id(tr_template_id) == get_uuid_from_seed(
"test_get_latest_revision_2"
)
async def test_wiring_with_generic_rest_input(
input_json_with_wiring_with_input, async_test_client):
async with async_test_client as client:
json_with_wiring = deepcopy(input_json_with_wiring_with_input)
json_with_wiring["workflow_wiring"]["input_wirings"] = [{
"workflow_input_name":
"val_inp",
"adapter_id":
"gen_rest_adapter_test_id",
"ref_id":
"thing_node_id",
"ref_id_type":
"THINGNODE",
"ref_key":
"number",
"type":
"metadata(int)",
}]
ww = WorkflowWiring(**json_with_wiring["workflow_wiring"])
resp_mock = mock.Mock()
resp_mock.status_code = 200
resp_mock.json = mock.Mock(return_value={
"key": "number",
"value": 32,
"dataType": "int"
})
with mock.patch(
"hetdesrun.adapters.generic_rest.load_metadata.get_generic_rest_adapter_base_url",
return_value="https://hetida.de",
):
with mock.patch(
"hetdesrun.adapters.generic_rest.load_metadata.httpx.AsyncClient.get",
return_value=resp_mock,
) as mocked_async_client_get:
status_code, output = await run_workflow_with_client(
json_with_wiring, client)
assert status_code == 200
node_results = output["node_results"]
assert "32.0" in node_results # intermediate result
assert "64.0" in node_results
# now add sending metadata from the only output
json_with_wiring["workflow_wiring"]["output_wirings"] = [{
"workflow_output_name":
"z",
"adapter_id":
"gen_rest_adapter_test_id",
"ref_id":
"thing_node_id",
"ref_id_type":
"THINGNODE",
"ref_key":
"limit",
"type":
"metadata(float)",
}]
with mock.patch(
"hetdesrun.adapters.generic_rest.send_metadata.get_generic_rest_adapter_base_url",
return_value="https://hetida.de",
):
response = mock.Mock()
response.status_code = 200
send_metadata_post_mock = mock.AsyncMock(
return_value=response)
with mock.patch(
"hetdesrun.adapters.generic_rest.send_metadata.post_json_with_open_client",
new=send_metadata_post_mock,
):
status_code, output = await run_workflow_with_client(
json_with_wiring, client)
# what gets into the post request sent from send_metadata:
func_name, args, kwargs = send_metadata_post_mock.mock_calls[
0]
assert kwargs["json_payload"] == ({
"key": "limit",
"value": 64.0,
"dataType": "float"
})
assert (
kwargs["url"] ==
"https://hetida.de/thingNodes/thing_node_id/metadata/limit"
)
def gen_execution_input_from_single_component(
component_json_path, direct_provisioning_data_dict=None, wf_wiring=None
):
"""Wraps a single component into a workflow and generates the execution input json
input data is provided directly
"""
if (direct_provisioning_data_dict is None) == (wf_wiring is None):
raise ValueError(
"Excatly one of direct_provisioning_data_dict or wf_wiring must be provided"
)
# Load component stuff
(
base_name,
path_to_component_json,
component_doc_file,
component_code_file,
) = file_pathes_from_component_json(component_json_path)
info, doc, code = load_data(
path_to_component_json, component_doc_file, component_code_file
)
# Build up execution input Json
code_module_uuid = str(get_uuid_from_seed("code_module_uuid"))
component_uuid = str(get_uuid_from_seed("component_uuid"))
comp_inputs = [
ComponentInput(id=str(uuid4()), name=inp["name"], type=inp["type"])
for inp in info["inputs"]
]
comp_outputs = [
ComponentOutput(id=str(uuid4()), name=outp["name"], type=outp["type"])
for outp in info["outputs"]
]
component_node_id = "component_node_id"
return WorkflowExecutionInput(
code_modules=[CodeModule(code=code, uuid=code_module_uuid)],
components=[
ComponentRevision(
uuid=component_uuid,
name=info["name"],
code_module_uuid=code_module_uuid,
function_name="main",
inputs=comp_inputs,
outputs=comp_outputs,
)
],
workflow=WorkflowNode(
id="root_node",
sub_nodes=[
ComponentNode(component_uuid=component_uuid, id=component_node_id)
],
connections=[],
inputs=[
WorkflowInput(
id=str(get_uuid_from_seed(str(comp_input.id) + "_as_wf_input")),
id_of_sub_node=component_node_id,
name=comp_input.name,
name_in_subnode=comp_input.name,
type=comp_input.type,
)
for comp_input in comp_inputs
],
outputs=[
WorkflowOutput(
id=str(get_uuid_from_seed(str(comp_output.id) + "_as_wf_output")),
id_of_sub_node=component_node_id,
name=comp_output.name,
name_in_subnode=comp_output.name,
type=comp_output.type,
)
for comp_output in comp_outputs
],
name="root node",
),
configuration=ConfigurationInput(engine="plain", run_pure_plot_operators=True),
workflow_wiring=WorkflowWiring(
input_wirings=[
InputWiring(
workflow_input_name=comp_input.name,
adapter_id="direct_provisioning",
filters={"value": direct_provisioning_data_dict[comp_input.name]},
)
for comp_input in comp_inputs
],
output_wirings=[
OutputWiring(
workflow_output_name=comp_output.name,
adapter_id="direct_provisioning",
)
for comp_output in comp_outputs
],
)
if wf_wiring is None
else wf_wiring,
)
return engine
component_tr_1 = TransformationRevision(
id=get_uuid_from_seed("component 1"),
revision_group_id=get_uuid_from_seed("group of component 1"),
name="component 0",
description="description of component 0",
category="category",
type=Type.COMPONENT,
state=State.DRAFT,
version_tag="1.0.0",
io_interface=IOInterface(),
documentation="documentation",
content="code",
test_wiring=WorkflowWiring(),
)
@pytest.mark.asyncio
async def test_get_documentation(async_test_client, clean_test_db_engine):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
store_single_transformation_revision(component_tr_1)
async with async_test_client as ac:
response = await ac.get("/api/documentations/" +
str(get_uuid_from_seed("component 1")))
assert response.status_code == 200
def transformation_revision_from_python_code(code: str, path: str) -> Any:
"""Get the TransformationRevision as a json file from just the Python code of some component
This uses information from the register decorator and docstrings.
Note: This needs to import the code module, which may have arbitrary side effects and security
implications.
"""
try:
main_func = import_func_from_code(code, "main")
except ComponentCodeImportError as e:
logging.error(
"Could not load function from %s\n"
"due to error during import of component code:\n%s",
path,
str(e),
)
module_path = module_path_from_code(code)
mod = importlib.import_module(module_path)
mod_docstring = mod.__doc__ or ""
mod_docstring_lines = mod_docstring.splitlines()
if hasattr(main_func, "registered_metadata"):
logger.info("Get component info from registered metadata")
component_name = main_func.registered_metadata[
"name"] or ( # type: ignore
"Unnamed Component")
component_description = main_func.registered_metadata[
"description"] or ( # type: ignore
"No description provided")
component_category = main_func.registered_metadata[
"category"] or ( # type: ignore
"Other")
component_id = main_func.registered_metadata["id"] or ( # type: ignore
get_uuid_from_seed(str(component_name)))
component_group_id = main_func.registered_metadata[
"revision_group_id"] or ( # type: ignore
get_uuid_from_seed(str(component_name)))
component_tag = main_func.registered_metadata["version_tag"] or (
"1.0.0") # type: ignore
component_inputs = main_func.registered_metadata[
"inputs"] # type: ignore
component_outputs = main_func.registered_metadata[
"outputs"] # type: ignore
component_state = main_func.registered_metadata[
"state"] or "RELEASED" # type: ignore
component_released_timestamp = main_func.registered_metadata[ # type: ignore
"released_timestamp"] or (datetime.now(timezone.utc).isoformat() if
component_state == "RELEASED" else None)
component_disabled_timestamp = main_func.registered_metadata[ # type: ignore
"disabled_timestamp"] or (datetime.now(timezone.utc).isoformat() if
component_state == "DISABLED" else None)
elif hasattr(mod, "COMPONENT_INFO"):
logger.info("Get component info from dictionary in code")
info_dict = mod.COMPONENT_INFO
component_inputs = info_dict.get("inputs", {})
component_outputs = info_dict.get("outputs", {})
component_name = info_dict.get("name", "Unnamed Component")
component_description = info_dict.get("description",
"No description provided")
component_category = info_dict.get("category", "Other")
component_tag = info_dict.get("version_tag", "1.0.0")
component_id = info_dict.get("id",
get_uuid_from_seed(str(component_name)))
component_group_id = info_dict.get(
"revision_group_id", get_uuid_from_seed(str(component_name)))
component_state = info_dict.get("state", "RELEASED")
component_released_timestamp = info_dict.get(
"released_timestamp",
datetime.now(timezone.utc).isoformat()
if component_state == "RELEASED" else None,
)
component_disabled_timestamp = info_dict.get(
"released_timestamp",
datetime.now(timezone.utc).isoformat()
if component_state == "DISABLED" else None,
)
else:
raise ComponentCodeImportError
component_documentation = "\n".join(mod_docstring_lines[2:])
transformation_revision = TransformationRevision(
id=component_id,
revision_group_id=component_group_id,
name=component_name,
description=component_description,
category=component_category,
version_tag=component_tag,
type=Type.COMPONENT,
state=component_state,
released_timestamp=component_released_timestamp,
disabled_timestamp=component_disabled_timestamp,
documentation=component_documentation,
io_interface=IOInterface(
inputs=[
IO(
id=get_uuid_from_seed("component_input_" + input_name),
name=input_name,
data_type=input_data_type,
) for input_name, input_data_type in component_inputs.items()
],
outputs=[
IO(
id=get_uuid_from_seed("component_output_" + output_name),
name=output_name,
data_type=output_data_type,
)
for output_name, output_data_type in component_outputs.items()
],
),
content=code,
test_wiring=WorkflowWiring(),
)
tr_json = json.loads(transformation_revision.json())
return tr_json
