def test_deleting(clean_test_db_engine): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", sessionmaker(clean_test_db_engine), ): tr_draft_uuid = get_uuid_from_seed("draft") tr_draft_object = TransformationRevision( id=tr_draft_uuid, revision_group_id=tr_draft_uuid, name="Test", description="Test description", version_tag="1.0.0", category="Test category", state=State.DRAFT, type=Type.COMPONENT, content="code", io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", ) tr_released_uuid = get_uuid_from_seed("released") tr_released_object = TransformationRevision( id=tr_released_uuid, revision_group_id=tr_released_uuid, name="Test", description="Test description", version_tag="1.0.0", category="Test category", released_timestamp="2021-12-24 00:00", state=State.RELEASED, type=Type.COMPONENT, content="code", io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", ) store_single_transformation_revision(tr_draft_object) store_single_transformation_revision(tr_released_object) delete_single_transformation_revision(tr_draft_uuid) with pytest.raises(DBNotFoundError): read_single_transformation_revision(tr_draft_uuid) with pytest.raises(DBBadRequestError): delete_single_transformation_revision(tr_released_uuid)
def to_wiring(self) -> WorkflowWiring: return WorkflowWiring( input_wirings=[iw.to_input_wiring() for iw in self.input_wirings], output_wirings=[ ow.to_output_wiring() for ow in self.output_wirings ], )
def to_transformation_revision(self, documentation: str = "" ) -> TransformationRevision: return TransformationRevision( id=self.id, revision_group_id=self.group_id, name=self.name, description=self.description, category=self.category, version_tag=self.tag, released_timestamp=datetime.now() if self.state == State.RELEASED else None, disabled_timestamp=datetime.now() if self.state == State.DISABLED else None, state=self.state, type=self.type, documentation=documentation, io_interface=IOInterface( inputs=[input.to_io() for input in self.inputs], outputs=[output.to_io() for output in self.outputs], ), content=self.code, test_wiring=self.wirings[0].to_wiring() if len(self.wirings) > 0 else WorkflowWiring(), )
def test_updating(clean_test_db_engine): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", sessionmaker(clean_test_db_engine), ): tr_uuid = get_uuid_from_seed("test_updating") tr_object = TransformationRevision( id=tr_uuid, revision_group_id=tr_uuid, name="Test", description="Test description", version_tag="1.0.0", category="Test category", state=State.DRAFT, type=Type.COMPONENT, content="code", io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", ) store_single_transformation_revision(tr_object) tr_object.name = "Test Update" update_or_create_single_transformation_revision(tr_object) received_tr_object = read_single_transformation_revision(tr_uuid) assert tr_object == received_tr_object
def test_tr_validator_content_type_correct(): id = get_uuid_from_seed("test") combi = namedtuple("combi", "type content") incorrect_combis = ( combi(type=Type.WORKFLOW, content="test"), combi(type=Type.COMPONENT, content=WorkflowContent()), ) correct_combis = ( combi(type=Type.WORKFLOW, content=WorkflowContent()), combi(type=Type.COMPONENT, content="test"), ) for combi in incorrect_combis: with pytest.raises(ValidationError): TransformationRevision( id=id, revision_group_id=id, name="Test", description="Test description", version_tag="1.0.0", category="Test category", state=State.DRAFT, type=combi.type, content=combi.content, io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", ) for combi in correct_combis: # no validation errors TransformationRevision( id=id, revision_group_id=id, name="Test", description="Test description", version_tag="1.0.0", category="Test category", state=State.DRAFT, type=combi.type, content=combi.content, io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", )
def test_multiple_select(clean_test_db_engine): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", sessionmaker(clean_test_db_engine), ): tr_template_id = get_uuid_from_seed("object_template") tr_object_template = TransformationRevision( id=tr_template_id, revision_group_id=tr_template_id, name="Test", description="Test description", version_tag="1.0.0", category="Test category", state=State.DRAFT, type=Type.COMPONENT, content="code", io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", ) tr_uuid_1 = get_uuid_from_seed("test_multiple_select_1") tr_object_1 = tr_object_template.copy() tr_object_1.id = tr_uuid_1 tr_object_1.revision_group_id = tr_uuid_1 store_single_transformation_revision(tr_object_1) tr_uuid_2 = get_uuid_from_seed("test_multiple_select_2") tr_object_2 = tr_object_1.copy() tr_object_2.id = tr_uuid_2 tr_object_2.version_tag = "1.0.1" store_single_transformation_revision(tr_object_2) tr_object_3 = tr_object_template.copy() tr_uuid_3 = get_uuid_from_seed("test_multiple_select_3") tr_object_3.id = tr_uuid_3 tr_object_3.revision_group_id = tr_uuid_3 tr_object_3.release() store_single_transformation_revision(tr_object_3) results = select_multiple_transformation_revisions() assert len(results) == 3 results = select_multiple_transformation_revisions(state=State.DRAFT) assert len(results) == 2 results = select_multiple_transformation_revisions(state=State.RELEASED) assert len(results) == 1 results = select_multiple_transformation_revisions(revision_group_id=tr_uuid_1) assert len(results) == 2 results = select_multiple_transformation_revisions(type=Type.COMPONENT) assert len(results) == 3 results = select_multiple_transformation_revisions(type=Type.WORKFLOW) assert len(results) == 0
def to_workflow_wiring(self) -> WorkflowWiring: return WorkflowWiring( input_wirings=[ wiring.to_input_wiring() for wiring in self.input_wirings ], output_wirings=[ wiring.to_output_wiring() for wiring in self.output_wirings ], )
def component_creator(identifier: str) -> TransformationRevision: tr_component = TransformationRevision( id=get_uuid_from_seed("component " + identifier), revision_group_id=get_uuid_from_seed("group of component " + identifier), name=identifier, type="COMPONENT", state="DRAFT", version_tag="1.0.0", io_interface=IOInterface(), content="code", test_wiring=WorkflowWiring(), ) return tr_component
def workflow_creator(identifier: str) -> TransformationRevision: tr_workflow = TransformationRevision( id=get_uuid_from_seed("workflow " + identifier), revision_group_id=get_uuid_from_seed("group of workflow " + identifier), name=identifier, type="WORKFLOW", state="DRAFT", version_tag="1.0.0", io_interface=IOInterface(), content=WorkflowContent(), test_wiring=WorkflowWiring(), ) return tr_workflow
def test_tr_nonemptyvalidstr_regex_validator_fancy_characters(): id = get_uuid_from_seed("test") TransformationRevision( id=id, revision_group_id=id, name="bößä", description="中文, español, Çok teşekkürler", version_tag="(-_-) / =.= & +_+", category="ไทย", state=State.DRAFT, type=Type.COMPONENT, content="test", io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", )
def test_tr_validstr_regex_validator_empty(): id = get_uuid_from_seed("test") TransformationRevision( id=id, revision_group_id=id, name="Test", description="", version_tag="1.0.0", category="Test category", state=State.DRAFT, type=Type.COMPONENT, content="test", io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", )
def test_tr_shortnonemptyvalidstr_validator_max_characters(): id = get_uuid_from_seed("test") with pytest.raises(ValidationError): TransformationRevision( id=id, revision_group_id=id, name="Name", description="Test description", version_tag="1.0.0.0.0.0.0.0.0.0.0", category="Test category", state=State.DRAFT, type=Type.COMPONENT, content="test", io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", )
def test_get_latest_revision_id(clean_test_db_engine): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", sessionmaker(clean_test_db_engine), ): tr_template_id = get_uuid_from_seed("object_template") tr_object_template = TransformationRevision( id=get_uuid_from_seed("test_get_latest_revision_0"), revision_group_id=tr_template_id, name="Test", description="Test description", version_tag="1.0.0", category="Test category", state=State.DRAFT, type=Type.COMPONENT, content="code", io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", ) tr_object_1 = tr_object_template.copy() tr_object_1.id = get_uuid_from_seed("test_get_latest_revision_1") tr_object_1.version_tag = "1.0.1" tr_object_1.release() store_single_transformation_revision(tr_object_1) tr_object_2 = tr_object_template.copy() tr_object_2.id = get_uuid_from_seed("test_get_latest_revision_2") tr_object_2.version_tag = "1.0.2" tr_object_2.release() store_single_transformation_revision(tr_object_2) assert get_latest_revision_id(tr_template_id) == get_uuid_from_seed( "test_get_latest_revision_2" )
async def test_wiring_with_generic_rest_input( input_json_with_wiring_with_input, async_test_client): async with async_test_client as client: json_with_wiring = deepcopy(input_json_with_wiring_with_input) json_with_wiring["workflow_wiring"]["input_wirings"] = [{ "workflow_input_name": "val_inp", "adapter_id": "gen_rest_adapter_test_id", "ref_id": "thing_node_id", "ref_id_type": "THINGNODE", "ref_key": "number", "type": "metadata(int)", }] ww = WorkflowWiring(**json_with_wiring["workflow_wiring"]) resp_mock = mock.Mock() resp_mock.status_code = 200 resp_mock.json = mock.Mock(return_value={ "key": "number", "value": 32, "dataType": "int" }) with mock.patch( "hetdesrun.adapters.generic_rest.load_metadata.get_generic_rest_adapter_base_url", return_value="https://hetida.de", ): with mock.patch( "hetdesrun.adapters.generic_rest.load_metadata.httpx.AsyncClient.get", return_value=resp_mock, ) as mocked_async_client_get: status_code, output = await run_workflow_with_client( json_with_wiring, client) assert status_code == 200 node_results = output["node_results"] assert "32.0" in node_results # intermediate result assert "64.0" in node_results # now add sending metadata from the only output json_with_wiring["workflow_wiring"]["output_wirings"] = [{ "workflow_output_name": "z", "adapter_id": "gen_rest_adapter_test_id", "ref_id": "thing_node_id", "ref_id_type": "THINGNODE", "ref_key": "limit", "type": "metadata(float)", }] with mock.patch( "hetdesrun.adapters.generic_rest.send_metadata.get_generic_rest_adapter_base_url", return_value="https://hetida.de", ): response = mock.Mock() response.status_code = 200 send_metadata_post_mock = mock.AsyncMock( return_value=response) with mock.patch( "hetdesrun.adapters.generic_rest.send_metadata.post_json_with_open_client", new=send_metadata_post_mock, ): status_code, output = await run_workflow_with_client( json_with_wiring, client) # what gets into the post request sent from send_metadata: func_name, args, kwargs = send_metadata_post_mock.mock_calls[ 0] assert kwargs["json_payload"] == ({ "key": "limit", "value": 64.0, "dataType": "float" }) assert ( kwargs["url"] == "https://hetida.de/thingNodes/thing_node_id/metadata/limit" )
def gen_execution_input_from_single_component( component_json_path, direct_provisioning_data_dict=None, wf_wiring=None ): """Wraps a single component into a workflow and generates the execution input json input data is provided directly """ if (direct_provisioning_data_dict is None) == (wf_wiring is None): raise ValueError( "Excatly one of direct_provisioning_data_dict or wf_wiring must be provided" ) # Load component stuff ( base_name, path_to_component_json, component_doc_file, component_code_file, ) = file_pathes_from_component_json(component_json_path) info, doc, code = load_data( path_to_component_json, component_doc_file, component_code_file ) # Build up execution input Json code_module_uuid = str(get_uuid_from_seed("code_module_uuid")) component_uuid = str(get_uuid_from_seed("component_uuid")) comp_inputs = [ ComponentInput(id=str(uuid4()), name=inp["name"], type=inp["type"]) for inp in info["inputs"] ] comp_outputs = [ ComponentOutput(id=str(uuid4()), name=outp["name"], type=outp["type"]) for outp in info["outputs"] ] component_node_id = "component_node_id" return WorkflowExecutionInput( code_modules=[CodeModule(code=code, uuid=code_module_uuid)], components=[ ComponentRevision( uuid=component_uuid, name=info["name"], code_module_uuid=code_module_uuid, function_name="main", inputs=comp_inputs, outputs=comp_outputs, ) ], workflow=WorkflowNode( id="root_node", sub_nodes=[ ComponentNode(component_uuid=component_uuid, id=component_node_id) ], connections=[], inputs=[ WorkflowInput( id=str(get_uuid_from_seed(str(comp_input.id) + "_as_wf_input")), id_of_sub_node=component_node_id, name=comp_input.name, name_in_subnode=comp_input.name, type=comp_input.type, ) for comp_input in comp_inputs ], outputs=[ WorkflowOutput( id=str(get_uuid_from_seed(str(comp_output.id) + "_as_wf_output")), id_of_sub_node=component_node_id, name=comp_output.name, name_in_subnode=comp_output.name, type=comp_output.type, ) for comp_output in comp_outputs ], name="root node", ), configuration=ConfigurationInput(engine="plain", run_pure_plot_operators=True), workflow_wiring=WorkflowWiring( input_wirings=[ InputWiring( workflow_input_name=comp_input.name, adapter_id="direct_provisioning", filters={"value": direct_provisioning_data_dict[comp_input.name]}, ) for comp_input in comp_inputs ], output_wirings=[ OutputWiring( workflow_output_name=comp_output.name, adapter_id="direct_provisioning", ) for comp_output in comp_outputs ], ) if wf_wiring is None else wf_wiring, )
return engine component_tr_1 = TransformationRevision( id=get_uuid_from_seed("component 1"), revision_group_id=get_uuid_from_seed("group of component 1"), name="component 0", description="description of component 0", category="category", type=Type.COMPONENT, state=State.DRAFT, version_tag="1.0.0", io_interface=IOInterface(), documentation="documentation", content="code", test_wiring=WorkflowWiring(), ) @pytest.mark.asyncio async def test_get_documentation(async_test_client, clean_test_db_engine): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", sessionmaker(clean_test_db_engine), ): store_single_transformation_revision(component_tr_1) async with async_test_client as ac: response = await ac.get("/api/documentations/" + str(get_uuid_from_seed("component 1"))) assert response.status_code == 200
def transformation_revision_from_python_code(code: str, path: str) -> Any: """Get the TransformationRevision as a json file from just the Python code of some component This uses information from the register decorator and docstrings. Note: This needs to import the code module, which may have arbitrary side effects and security implications. """ try: main_func = import_func_from_code(code, "main") except ComponentCodeImportError as e: logging.error( "Could not load function from %s\n" "due to error during import of component code:\n%s", path, str(e), ) module_path = module_path_from_code(code) mod = importlib.import_module(module_path) mod_docstring = mod.__doc__ or "" mod_docstring_lines = mod_docstring.splitlines() if hasattr(main_func, "registered_metadata"): logger.info("Get component info from registered metadata") component_name = main_func.registered_metadata[ "name"] or ( # type: ignore "Unnamed Component") component_description = main_func.registered_metadata[ "description"] or ( # type: ignore "No description provided") component_category = main_func.registered_metadata[ "category"] or ( # type: ignore "Other") component_id = main_func.registered_metadata["id"] or ( # type: ignore get_uuid_from_seed(str(component_name))) component_group_id = main_func.registered_metadata[ "revision_group_id"] or ( # type: ignore get_uuid_from_seed(str(component_name))) component_tag = main_func.registered_metadata["version_tag"] or ( "1.0.0") # type: ignore component_inputs = main_func.registered_metadata[ "inputs"] # type: ignore component_outputs = main_func.registered_metadata[ "outputs"] # type: ignore component_state = main_func.registered_metadata[ "state"] or "RELEASED" # type: ignore component_released_timestamp = main_func.registered_metadata[ # type: ignore "released_timestamp"] or (datetime.now(timezone.utc).isoformat() if component_state == "RELEASED" else None) component_disabled_timestamp = main_func.registered_metadata[ # type: ignore "disabled_timestamp"] or (datetime.now(timezone.utc).isoformat() if component_state == "DISABLED" else None) elif hasattr(mod, "COMPONENT_INFO"): logger.info("Get component info from dictionary in code") info_dict = mod.COMPONENT_INFO component_inputs = info_dict.get("inputs", {}) component_outputs = info_dict.get("outputs", {}) component_name = info_dict.get("name", "Unnamed Component") component_description = info_dict.get("description", "No description provided") component_category = info_dict.get("category", "Other") component_tag = info_dict.get("version_tag", "1.0.0") component_id = info_dict.get("id", get_uuid_from_seed(str(component_name))) component_group_id = info_dict.get( "revision_group_id", get_uuid_from_seed(str(component_name))) component_state = info_dict.get("state", "RELEASED") component_released_timestamp = info_dict.get( "released_timestamp", datetime.now(timezone.utc).isoformat() if component_state == "RELEASED" else None, ) component_disabled_timestamp = info_dict.get( "released_timestamp", datetime.now(timezone.utc).isoformat() if component_state == "DISABLED" else None, ) else: raise ComponentCodeImportError component_documentation = "\n".join(mod_docstring_lines[2:]) transformation_revision = TransformationRevision( id=component_id, revision_group_id=component_group_id, name=component_name, description=component_description, category=component_category, version_tag=component_tag, type=Type.COMPONENT, state=component_state, released_timestamp=component_released_timestamp, disabled_timestamp=component_disabled_timestamp, documentation=component_documentation, io_interface=IOInterface( inputs=[ IO( id=get_uuid_from_seed("component_input_" + input_name), name=input_name, data_type=input_data_type, ) for input_name, input_data_type in component_inputs.items() ], outputs=[ IO( id=get_uuid_from_seed("component_output_" + output_name), name=output_name, data_type=output_data_type, ) for output_name, output_data_type in component_outputs.items() ], ), content=code, test_wiring=WorkflowWiring(), ) tr_json = json.loads(transformation_revision.json()) return tr_json