def test_deleting(clean_test_db_engine): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", sessionmaker(clean_test_db_engine), ): tr_draft_uuid = get_uuid_from_seed("draft") tr_draft_object = TransformationRevision( id=tr_draft_uuid, revision_group_id=tr_draft_uuid, name="Test", description="Test description", version_tag="1.0.0", category="Test category", state=State.DRAFT, type=Type.COMPONENT, content="code", io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", ) tr_released_uuid = get_uuid_from_seed("released") tr_released_object = TransformationRevision( id=tr_released_uuid, revision_group_id=tr_released_uuid, name="Test", description="Test description", version_tag="1.0.0", category="Test category", released_timestamp="2021-12-24 00:00", state=State.RELEASED, type=Type.COMPONENT, content="code", io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", ) store_single_transformation_revision(tr_draft_object) store_single_transformation_revision(tr_released_object) delete_single_transformation_revision(tr_draft_uuid) with pytest.raises(DBNotFoundError): read_single_transformation_revision(tr_draft_uuid) with pytest.raises(DBBadRequestError): delete_single_transformation_revision(tr_released_uuid)
def test_updating(clean_test_db_engine): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", sessionmaker(clean_test_db_engine), ): tr_uuid = get_uuid_from_seed("test_updating") tr_object = TransformationRevision( id=tr_uuid, revision_group_id=tr_uuid, name="Test", description="Test description", version_tag="1.0.0", category="Test category", state=State.DRAFT, type=Type.COMPONENT, content="code", io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", ) store_single_transformation_revision(tr_object) tr_object.name = "Test Update" update_or_create_single_transformation_revision(tr_object) received_tr_object = read_single_transformation_revision(tr_uuid) assert tr_object == received_tr_object
def to_transformation_revision(self, documentation: str = "" ) -> TransformationRevision: return TransformationRevision( id=self.id, revision_group_id=self.group_id, name=self.name, description=self.description, category=self.category, version_tag=self.tag, released_timestamp=datetime.now() if self.state == State.RELEASED else None, disabled_timestamp=datetime.now() if self.state == State.DISABLED else None, state=self.state, type=self.type, documentation=documentation, io_interface=IOInterface( inputs=[input.to_io() for input in self.inputs], outputs=[output.to_io() for output in self.outputs], ), content=self.code, test_wiring=self.wirings[0].to_wiring() if len(self.wirings) > 0 else WorkflowWiring(), )
def test_function_header_multiple_inputs(): component = TransformationRevision( io_interface=IOInterface( inputs=[ IO(name="x", data_type=DataType.Float), IO(name="okay", data_type=DataType.Boolean), ], outputs=[IO(name="output", data_type=DataType.Float)], ), name="Test Component", description="A test component", category="Tests", id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565", revision_group_id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565", version_tag="1.0.0", state="DRAFT", type="COMPONENT", content="", test_wiring=[], ) func_header = generate_function_header(component) assert "main(*, x, okay)" in func_header assert (""" "inputs": { "x": "FLOAT", "okay": "BOOLEAN", }, """ in func_header) assert (""" "outputs": { "output": "FLOAT", }, """ in func_header) assert '"version_tag": "1.0.0"' in func_header
def test_tr_validator_content_type_correct(): id = get_uuid_from_seed("test") combi = namedtuple("combi", "type content") incorrect_combis = ( combi(type=Type.WORKFLOW, content="test"), combi(type=Type.COMPONENT, content=WorkflowContent()), ) correct_combis = ( combi(type=Type.WORKFLOW, content=WorkflowContent()), combi(type=Type.COMPONENT, content="test"), ) for combi in incorrect_combis: with pytest.raises(ValidationError): TransformationRevision( id=id, revision_group_id=id, name="Test", description="Test description", version_tag="1.0.0", category="Test category", state=State.DRAFT, type=combi.type, content=combi.content, io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", ) for combi in correct_combis: # no validation errors TransformationRevision( id=id, revision_group_id=id, name="Test", description="Test description", version_tag="1.0.0", category="Test category", state=State.DRAFT, type=combi.type, content=combi.content, io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", )
def test_update_code(): component = TransformationRevision( io_interface=IOInterface(inputs=[], outputs=[]), name="Test Component", description="A test component", category="Tests", id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565", revision_group_id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565", version_tag="1.0.0", state="RELEASED", type="COMPONENT", released_timestamp="2019-12-01T12:00:00+00:00", content=example_code, test_wiring=[], ) updated_component = TransformationRevision( io_interface=IOInterface(inputs=[], outputs=[]), name="Test Component", description="A test component", category="Tests", id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565", revision_group_id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565", version_tag="1.0.1", state="DRAFT", type="COMPONENT", content=example_code, test_wiring=[], ) new_code = update_code(updated_component) assert """return {"z": x+y}""" in new_code assert "c6eff22c-21c4-43c6-9ae1-b2bdfb944565" in new_code assert "1.0.0" not in new_code # test input without both start/stop markers component.content = "" new_code = update_code(component) assert "pass" in new_code # test input without only stop marker component.content = "# ***** DO NOT EDIT LINES BELOW *****" new_code = update_code(component) # test with async def in function header component.content = example_code_async new_code = update_code(component) assert "async def" in new_code
def test_multiple_select(clean_test_db_engine): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", sessionmaker(clean_test_db_engine), ): tr_template_id = get_uuid_from_seed("object_template") tr_object_template = TransformationRevision( id=tr_template_id, revision_group_id=tr_template_id, name="Test", description="Test description", version_tag="1.0.0", category="Test category", state=State.DRAFT, type=Type.COMPONENT, content="code", io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", ) tr_uuid_1 = get_uuid_from_seed("test_multiple_select_1") tr_object_1 = tr_object_template.copy() tr_object_1.id = tr_uuid_1 tr_object_1.revision_group_id = tr_uuid_1 store_single_transformation_revision(tr_object_1) tr_uuid_2 = get_uuid_from_seed("test_multiple_select_2") tr_object_2 = tr_object_1.copy() tr_object_2.id = tr_uuid_2 tr_object_2.version_tag = "1.0.1" store_single_transformation_revision(tr_object_2) tr_object_3 = tr_object_template.copy() tr_uuid_3 = get_uuid_from_seed("test_multiple_select_3") tr_object_3.id = tr_uuid_3 tr_object_3.revision_group_id = tr_uuid_3 tr_object_3.release() store_single_transformation_revision(tr_object_3) results = select_multiple_transformation_revisions() assert len(results) == 3 results = select_multiple_transformation_revisions(state=State.DRAFT) assert len(results) == 2 results = select_multiple_transformation_revisions(state=State.RELEASED) assert len(results) == 1 results = select_multiple_transformation_revisions(revision_group_id=tr_uuid_1) assert len(results) == 2 results = select_multiple_transformation_revisions(type=Type.COMPONENT) assert len(results) == 3 results = select_multiple_transformation_revisions(type=Type.WORKFLOW) assert len(results) == 0
def wrap_component_in_tr_workflow(self) -> "TransformationRevision": operator = self.to_operator() wf_inputs = [] wf_outputs = [] links = [] for input_connector in operator.inputs: wf_input = IOConnector.from_connector(input_connector, operator.id, operator.name) wf_inputs.append(wf_input) link = Link( start=Vertex(operator=None, connector=wf_input.to_connector()), end=Vertex(operator=operator.id, connector=input_connector), ) links.append(link) for output_connector in operator.outputs: wf_output = IOConnector.from_connector(output_connector, operator.id, operator.name) wf_outputs.append(wf_output) link = Link( start=Vertex(operator=operator.id, connector=output_connector), end=Vertex(operator=None, connector=wf_output.to_connector()), ) links.append(link) return TransformationRevision( id=uuid4(), revision_group_id=uuid4(), name=self.name, category=self.category, version_tag=self.version_tag, released_timestamp=self.released_timestamp, disabled_timestamp=self.disabled_timestamp, state=self.state, type=Type.WORKFLOW, content=WorkflowContent( inputs=wf_inputs, outputs=wf_outputs, operators=[operator], links=links, ), io_interface=IOInterface( inputs=[ input_connector.to_io() for input_connector in wf_inputs ], outputs=[ output_connector.to_io() for output_connector in wf_outputs ], ), test_wiring=self.test_wiring, )
def io_interface_fits_to_content(cls, io_interface: IOInterface, values: dict) -> IOInterface: if values["type"] is not Type.WORKFLOW: return io_interface try: workflow_content = values["content"] assert isinstance(workflow_content, WorkflowContent) # hint for mypy except KeyError as e: raise ValueError( "Cannot fit io_interface to content if attribute 'content' is missing" ) from e io_interface.inputs = [ input.to_io() for input in workflow_content.inputs ] io_interface.outputs = [ output.to_io() for output in workflow_content.outputs ] return io_interface
def workflow_creator(identifier: str) -> TransformationRevision: tr_workflow = TransformationRevision( id=get_uuid_from_seed("workflow " + identifier), revision_group_id=get_uuid_from_seed("group of workflow " + identifier), name=identifier, type="WORKFLOW", state="DRAFT", version_tag="1.0.0", io_interface=IOInterface(), content=WorkflowContent(), test_wiring=WorkflowWiring(), ) return tr_workflow
def component_creator(identifier: str) -> TransformationRevision: tr_component = TransformationRevision( id=get_uuid_from_seed("component " + identifier), revision_group_id=get_uuid_from_seed("group of component " + identifier), name=identifier, type="COMPONENT", state="DRAFT", version_tag="1.0.0", io_interface=IOInterface(), content="code", test_wiring=WorkflowWiring(), ) return tr_component
def test_tr_nonemptyvalidstr_regex_validator_fancy_characters(): id = get_uuid_from_seed("test") TransformationRevision( id=id, revision_group_id=id, name="bößä", description="中文, español, Çok teşekkürler", version_tag="(-_-) / =.= & +_+", category="ไทย", state=State.DRAFT, type=Type.COMPONENT, content="test", io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", )
def test_tr_validstr_regex_validator_empty(): id = get_uuid_from_seed("test") TransformationRevision( id=id, revision_group_id=id, name="Test", description="", version_tag="1.0.0", category="Test category", state=State.DRAFT, type=Type.COMPONENT, content="test", io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", )
def test_tr_shortnonemptyvalidstr_validator_max_characters(): id = get_uuid_from_seed("test") with pytest.raises(ValidationError): TransformationRevision( id=id, revision_group_id=id, name="Name", description="Test description", version_tag="1.0.0.0.0.0.0.0.0.0.0", category="Test category", state=State.DRAFT, type=Type.COMPONENT, content="test", io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", )
def test_function_header_no_params(): component = TransformationRevision( io_interface=IOInterface(inputs=[], outputs=[]), name="Test Component", description="A test component", category="Tests", id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565", revision_group_id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565", version_tag="1.0.0", state="DRAFT", type="COMPONENT", content="", test_wiring=[], ) func_header = generate_function_header(component) assert "main()" in func_header assert '"inputs": {' + "}" in func_header assert '"outputs": {' + "}" in func_header assert '"id": "c6eff22c-21c4-43c6-9ae1-b2bdfb944565"' in func_header
def test_get_latest_revision_id(clean_test_db_engine): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", sessionmaker(clean_test_db_engine), ): tr_template_id = get_uuid_from_seed("object_template") tr_object_template = TransformationRevision( id=get_uuid_from_seed("test_get_latest_revision_0"), revision_group_id=tr_template_id, name="Test", description="Test description", version_tag="1.0.0", category="Test category", state=State.DRAFT, type=Type.COMPONENT, content="code", io_interface=IOInterface(), test_wiring=WorkflowWiring(), documentation="", ) tr_object_1 = tr_object_template.copy() tr_object_1.id = get_uuid_from_seed("test_get_latest_revision_1") tr_object_1.version_tag = "1.0.1" tr_object_1.release() store_single_transformation_revision(tr_object_1) tr_object_2 = tr_object_template.copy() tr_object_2.id = get_uuid_from_seed("test_get_latest_revision_2") tr_object_2.version_tag = "1.0.2" tr_object_2.release() store_single_transformation_revision(tr_object_2) assert get_latest_revision_id(tr_template_id) == get_uuid_from_seed( "test_get_latest_revision_2" )
engine = get_db_engine() Base.metadata.drop_all(engine) Base.metadata.create_all(engine) return engine component_tr_1 = TransformationRevision( id=get_uuid_from_seed("component 1"), revision_group_id=get_uuid_from_seed("group of component 1"), name="component 0", description="description of component 0", category="category", type=Type.COMPONENT, state=State.DRAFT, version_tag="1.0.0", io_interface=IOInterface(), documentation="documentation", content="code", test_wiring=WorkflowWiring(), ) @pytest.mark.asyncio async def test_get_documentation(async_test_client, clean_test_db_engine): with mock.patch( "hetdesrun.persistence.dbservice.revision.Session", sessionmaker(clean_test_db_engine), ): store_single_transformation_revision(component_tr_1) async with async_test_client as ac:
def transformation_revision_from_python_code(code: str, path: str) -> Any: """Get the TransformationRevision as a json file from just the Python code of some component This uses information from the register decorator and docstrings. Note: This needs to import the code module, which may have arbitrary side effects and security implications. """ try: main_func = import_func_from_code(code, "main") except ComponentCodeImportError as e: logging.error( "Could not load function from %s\n" "due to error during import of component code:\n%s", path, str(e), ) module_path = module_path_from_code(code) mod = importlib.import_module(module_path) mod_docstring = mod.__doc__ or "" mod_docstring_lines = mod_docstring.splitlines() if hasattr(main_func, "registered_metadata"): logger.info("Get component info from registered metadata") component_name = main_func.registered_metadata[ "name"] or ( # type: ignore "Unnamed Component") component_description = main_func.registered_metadata[ "description"] or ( # type: ignore "No description provided") component_category = main_func.registered_metadata[ "category"] or ( # type: ignore "Other") component_id = main_func.registered_metadata["id"] or ( # type: ignore get_uuid_from_seed(str(component_name))) component_group_id = main_func.registered_metadata[ "revision_group_id"] or ( # type: ignore get_uuid_from_seed(str(component_name))) component_tag = main_func.registered_metadata["version_tag"] or ( "1.0.0") # type: ignore component_inputs = main_func.registered_metadata[ "inputs"] # type: ignore component_outputs = main_func.registered_metadata[ "outputs"] # type: ignore component_state = main_func.registered_metadata[ "state"] or "RELEASED" # type: ignore component_released_timestamp = main_func.registered_metadata[ # type: ignore "released_timestamp"] or (datetime.now(timezone.utc).isoformat() if component_state == "RELEASED" else None) component_disabled_timestamp = main_func.registered_metadata[ # type: ignore "disabled_timestamp"] or (datetime.now(timezone.utc).isoformat() if component_state == "DISABLED" else None) elif hasattr(mod, "COMPONENT_INFO"): logger.info("Get component info from dictionary in code") info_dict = mod.COMPONENT_INFO component_inputs = info_dict.get("inputs", {}) component_outputs = info_dict.get("outputs", {}) component_name = info_dict.get("name", "Unnamed Component") component_description = info_dict.get("description", "No description provided") component_category = info_dict.get("category", "Other") component_tag = info_dict.get("version_tag", "1.0.0") component_id = info_dict.get("id", get_uuid_from_seed(str(component_name))) component_group_id = info_dict.get( "revision_group_id", get_uuid_from_seed(str(component_name))) component_state = info_dict.get("state", "RELEASED") component_released_timestamp = info_dict.get( "released_timestamp", datetime.now(timezone.utc).isoformat() if component_state == "RELEASED" else None, ) component_disabled_timestamp = info_dict.get( "released_timestamp", datetime.now(timezone.utc).isoformat() if component_state == "DISABLED" else None, ) else: raise ComponentCodeImportError component_documentation = "\n".join(mod_docstring_lines[2:]) transformation_revision = TransformationRevision( id=component_id, revision_group_id=component_group_id, name=component_name, description=component_description, category=component_category, version_tag=component_tag, type=Type.COMPONENT, state=component_state, released_timestamp=component_released_timestamp, disabled_timestamp=component_disabled_timestamp, documentation=component_documentation, io_interface=IOInterface( inputs=[ IO( id=get_uuid_from_seed("component_input_" + input_name), name=input_name, data_type=input_data_type, ) for input_name, input_data_type in component_inputs.items() ], outputs=[ IO( id=get_uuid_from_seed("component_output_" + output_name), name=output_name, data_type=output_data_type, ) for output_name, output_data_type in component_outputs.items() ], ), content=code, test_wiring=WorkflowWiring(), ) tr_json = json.loads(transformation_revision.json()) return tr_json