def test_deleting(clean_test_db_engine):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
tr_draft_uuid = get_uuid_from_seed("draft")
tr_draft_object = TransformationRevision(
id=tr_draft_uuid,
revision_group_id=tr_draft_uuid,
name="Test",
description="Test description",
version_tag="1.0.0",
category="Test category",
state=State.DRAFT,
type=Type.COMPONENT,
content="code",
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
tr_released_uuid = get_uuid_from_seed("released")
tr_released_object = TransformationRevision(
id=tr_released_uuid,
revision_group_id=tr_released_uuid,
name="Test",
description="Test description",
version_tag="1.0.0",
category="Test category",
released_timestamp="2021-12-24 00:00",
state=State.RELEASED,
type=Type.COMPONENT,
content="code",
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
store_single_transformation_revision(tr_draft_object)
store_single_transformation_revision(tr_released_object)
delete_single_transformation_revision(tr_draft_uuid)
with pytest.raises(DBNotFoundError):
read_single_transformation_revision(tr_draft_uuid)
with pytest.raises(DBBadRequestError):
delete_single_transformation_revision(tr_released_uuid)
def test_updating(clean_test_db_engine):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
tr_uuid = get_uuid_from_seed("test_updating")
tr_object = TransformationRevision(
id=tr_uuid,
revision_group_id=tr_uuid,
name="Test",
description="Test description",
version_tag="1.0.0",
category="Test category",
state=State.DRAFT,
type=Type.COMPONENT,
content="code",
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
store_single_transformation_revision(tr_object)
tr_object.name = "Test Update"
update_or_create_single_transformation_revision(tr_object)
received_tr_object = read_single_transformation_revision(tr_uuid)
assert tr_object == received_tr_object
def to_transformation_revision(self,
documentation: str = ""
) -> TransformationRevision:
return TransformationRevision(
id=self.id,
revision_group_id=self.group_id,
name=self.name,
description=self.description,
category=self.category,
version_tag=self.tag,
released_timestamp=datetime.now()
if self.state == State.RELEASED else None,
disabled_timestamp=datetime.now()
if self.state == State.DISABLED else None,
state=self.state,
type=self.type,
documentation=documentation,
io_interface=IOInterface(
inputs=[input.to_io() for input in self.inputs],
outputs=[output.to_io() for output in self.outputs],
),
content=self.code,
test_wiring=self.wirings[0].to_wiring()
if len(self.wirings) > 0 else WorkflowWiring(),
)
def test_function_header_multiple_inputs():
component = TransformationRevision(
io_interface=IOInterface(
inputs=[
IO(name="x", data_type=DataType.Float),
IO(name="okay", data_type=DataType.Boolean),
],
outputs=[IO(name="output", data_type=DataType.Float)],
),
name="Test Component",
description="A test component",
category="Tests",
id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565",
revision_group_id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565",
version_tag="1.0.0",
state="DRAFT",
type="COMPONENT",
content="",
test_wiring=[],
)
func_header = generate_function_header(component)
assert "main(*, x, okay)" in func_header
assert ("""
"inputs": {
"x": "FLOAT",
"okay": "BOOLEAN",
},
""" in func_header)
assert ("""
"outputs": {
"output": "FLOAT",
},
""" in func_header)
assert '"version_tag": "1.0.0"' in func_header
def test_tr_validator_content_type_correct():
id = get_uuid_from_seed("test")
combi = namedtuple("combi", "type content")
incorrect_combis = (
combi(type=Type.WORKFLOW, content="test"),
combi(type=Type.COMPONENT, content=WorkflowContent()),
)
correct_combis = (
combi(type=Type.WORKFLOW, content=WorkflowContent()),
combi(type=Type.COMPONENT, content="test"),
)
for combi in incorrect_combis:
with pytest.raises(ValidationError):
TransformationRevision(
id=id,
revision_group_id=id,
name="Test",
description="Test description",
version_tag="1.0.0",
category="Test category",
state=State.DRAFT,
type=combi.type,
content=combi.content,
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
for combi in correct_combis:
# no validation errors
TransformationRevision(
id=id,
revision_group_id=id,
name="Test",
description="Test description",
version_tag="1.0.0",
category="Test category",
state=State.DRAFT,
type=combi.type,
content=combi.content,
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
def test_update_code():
component = TransformationRevision(
io_interface=IOInterface(inputs=[], outputs=[]),
name="Test Component",
description="A test component",
category="Tests",
id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565",
revision_group_id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565",
version_tag="1.0.0",
state="RELEASED",
type="COMPONENT",
released_timestamp="2019-12-01T12:00:00+00:00",
content=example_code,
test_wiring=[],
)
updated_component = TransformationRevision(
io_interface=IOInterface(inputs=[], outputs=[]),
name="Test Component",
description="A test component",
category="Tests",
id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565",
revision_group_id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565",
version_tag="1.0.1",
state="DRAFT",
type="COMPONENT",
content=example_code,
test_wiring=[],
)
new_code = update_code(updated_component)
assert """return {"z": x+y}""" in new_code
assert "c6eff22c-21c4-43c6-9ae1-b2bdfb944565" in new_code
assert "1.0.0" not in new_code
# test input without both start/stop markers
component.content = ""
new_code = update_code(component)
assert "pass" in new_code
# test input without only stop marker
component.content = "# ***** DO NOT EDIT LINES BELOW *****"
new_code = update_code(component)
# test with async def in function header
component.content = example_code_async
new_code = update_code(component)
assert "async def" in new_code
def test_multiple_select(clean_test_db_engine):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
tr_template_id = get_uuid_from_seed("object_template")
tr_object_template = TransformationRevision(
id=tr_template_id,
revision_group_id=tr_template_id,
name="Test",
description="Test description",
version_tag="1.0.0",
category="Test category",
state=State.DRAFT,
type=Type.COMPONENT,
content="code",
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
tr_uuid_1 = get_uuid_from_seed("test_multiple_select_1")
tr_object_1 = tr_object_template.copy()
tr_object_1.id = tr_uuid_1
tr_object_1.revision_group_id = tr_uuid_1
store_single_transformation_revision(tr_object_1)
tr_uuid_2 = get_uuid_from_seed("test_multiple_select_2")
tr_object_2 = tr_object_1.copy()
tr_object_2.id = tr_uuid_2
tr_object_2.version_tag = "1.0.1"
store_single_transformation_revision(tr_object_2)
tr_object_3 = tr_object_template.copy()
tr_uuid_3 = get_uuid_from_seed("test_multiple_select_3")
tr_object_3.id = tr_uuid_3
tr_object_3.revision_group_id = tr_uuid_3
tr_object_3.release()
store_single_transformation_revision(tr_object_3)
results = select_multiple_transformation_revisions()
assert len(results) == 3
results = select_multiple_transformation_revisions(state=State.DRAFT)
assert len(results) == 2
results = select_multiple_transformation_revisions(state=State.RELEASED)
assert len(results) == 1
results = select_multiple_transformation_revisions(revision_group_id=tr_uuid_1)
assert len(results) == 2
results = select_multiple_transformation_revisions(type=Type.COMPONENT)
assert len(results) == 3
results = select_multiple_transformation_revisions(type=Type.WORKFLOW)
assert len(results) == 0
def wrap_component_in_tr_workflow(self) -> "TransformationRevision":
operator = self.to_operator()
wf_inputs = []
wf_outputs = []
links = []
for input_connector in operator.inputs:
wf_input = IOConnector.from_connector(input_connector, operator.id,
operator.name)
wf_inputs.append(wf_input)
link = Link(
start=Vertex(operator=None, connector=wf_input.to_connector()),
end=Vertex(operator=operator.id, connector=input_connector),
)
links.append(link)
for output_connector in operator.outputs:
wf_output = IOConnector.from_connector(output_connector,
operator.id, operator.name)
wf_outputs.append(wf_output)
link = Link(
start=Vertex(operator=operator.id, connector=output_connector),
end=Vertex(operator=None, connector=wf_output.to_connector()),
)
links.append(link)
return TransformationRevision(
id=uuid4(),
revision_group_id=uuid4(),
name=self.name,
category=self.category,
version_tag=self.version_tag,
released_timestamp=self.released_timestamp,
disabled_timestamp=self.disabled_timestamp,
state=self.state,
type=Type.WORKFLOW,
content=WorkflowContent(
inputs=wf_inputs,
outputs=wf_outputs,
operators=[operator],
links=links,
),
io_interface=IOInterface(
inputs=[
input_connector.to_io() for input_connector in wf_inputs
],
outputs=[
output_connector.to_io() for output_connector in wf_outputs
],
),
test_wiring=self.test_wiring,
)
def io_interface_fits_to_content(cls, io_interface: IOInterface,
values: dict) -> IOInterface:
if values["type"] is not Type.WORKFLOW:
return io_interface
try:
workflow_content = values["content"]
assert isinstance(workflow_content,
WorkflowContent) # hint for mypy
except KeyError as e:
raise ValueError(
"Cannot fit io_interface to content if attribute 'content' is missing"
) from e
io_interface.inputs = [
input.to_io() for input in workflow_content.inputs
]
io_interface.outputs = [
output.to_io() for output in workflow_content.outputs
]
return io_interface
def workflow_creator(identifier: str) -> TransformationRevision:
tr_workflow = TransformationRevision(
id=get_uuid_from_seed("workflow " + identifier),
revision_group_id=get_uuid_from_seed("group of workflow " +
identifier),
name=identifier,
type="WORKFLOW",
state="DRAFT",
version_tag="1.0.0",
io_interface=IOInterface(),
content=WorkflowContent(),
test_wiring=WorkflowWiring(),
)
return tr_workflow
def component_creator(identifier: str) -> TransformationRevision:
tr_component = TransformationRevision(
id=get_uuid_from_seed("component " + identifier),
revision_group_id=get_uuid_from_seed("group of component " +
identifier),
name=identifier,
type="COMPONENT",
state="DRAFT",
version_tag="1.0.0",
io_interface=IOInterface(),
content="code",
test_wiring=WorkflowWiring(),
)
return tr_component
def test_tr_nonemptyvalidstr_regex_validator_fancy_characters():
id = get_uuid_from_seed("test")
TransformationRevision(
id=id,
revision_group_id=id,
name="bößä",
description="中文, español, Çok teşekkürler",
version_tag="(-_-) / =.= & +_+",
category="ไทย",
state=State.DRAFT,
type=Type.COMPONENT,
content="test",
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
def test_tr_validstr_regex_validator_empty():
id = get_uuid_from_seed("test")
TransformationRevision(
id=id,
revision_group_id=id,
name="Test",
description="",
version_tag="1.0.0",
category="Test category",
state=State.DRAFT,
type=Type.COMPONENT,
content="test",
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
def test_tr_shortnonemptyvalidstr_validator_max_characters():
id = get_uuid_from_seed("test")
with pytest.raises(ValidationError):
TransformationRevision(
id=id,
revision_group_id=id,
name="Name",
description="Test description",
version_tag="1.0.0.0.0.0.0.0.0.0.0",
category="Test category",
state=State.DRAFT,
type=Type.COMPONENT,
content="test",
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
def test_function_header_no_params():
component = TransformationRevision(
io_interface=IOInterface(inputs=[], outputs=[]),
name="Test Component",
description="A test component",
category="Tests",
id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565",
revision_group_id="c6eff22c-21c4-43c6-9ae1-b2bdfb944565",
version_tag="1.0.0",
state="DRAFT",
type="COMPONENT",
content="",
test_wiring=[],
)
func_header = generate_function_header(component)
assert "main()" in func_header
assert '"inputs": {' + "}" in func_header
assert '"outputs": {' + "}" in func_header
assert '"id": "c6eff22c-21c4-43c6-9ae1-b2bdfb944565"' in func_header
def test_get_latest_revision_id(clean_test_db_engine):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
tr_template_id = get_uuid_from_seed("object_template")
tr_object_template = TransformationRevision(
id=get_uuid_from_seed("test_get_latest_revision_0"),
revision_group_id=tr_template_id,
name="Test",
description="Test description",
version_tag="1.0.0",
category="Test category",
state=State.DRAFT,
type=Type.COMPONENT,
content="code",
io_interface=IOInterface(),
test_wiring=WorkflowWiring(),
documentation="",
)
tr_object_1 = tr_object_template.copy()
tr_object_1.id = get_uuid_from_seed("test_get_latest_revision_1")
tr_object_1.version_tag = "1.0.1"
tr_object_1.release()
store_single_transformation_revision(tr_object_1)
tr_object_2 = tr_object_template.copy()
tr_object_2.id = get_uuid_from_seed("test_get_latest_revision_2")
tr_object_2.version_tag = "1.0.2"
tr_object_2.release()
store_single_transformation_revision(tr_object_2)
assert get_latest_revision_id(tr_template_id) == get_uuid_from_seed(
"test_get_latest_revision_2"
)
engine = get_db_engine()
Base.metadata.drop_all(engine)
Base.metadata.create_all(engine)
return engine
component_tr_1 = TransformationRevision(
id=get_uuid_from_seed("component 1"),
revision_group_id=get_uuid_from_seed("group of component 1"),
name="component 0",
description="description of component 0",
category="category",
type=Type.COMPONENT,
state=State.DRAFT,
version_tag="1.0.0",
io_interface=IOInterface(),
documentation="documentation",
content="code",
test_wiring=WorkflowWiring(),
)
@pytest.mark.asyncio
async def test_get_documentation(async_test_client, clean_test_db_engine):
with mock.patch(
"hetdesrun.persistence.dbservice.revision.Session",
sessionmaker(clean_test_db_engine),
):
store_single_transformation_revision(component_tr_1)
async with async_test_client as ac:
def transformation_revision_from_python_code(code: str, path: str) -> Any:
"""Get the TransformationRevision as a json file from just the Python code of some component
This uses information from the register decorator and docstrings.
Note: This needs to import the code module, which may have arbitrary side effects and security
implications.
"""
try:
main_func = import_func_from_code(code, "main")
except ComponentCodeImportError as e:
logging.error(
"Could not load function from %s\n"
"due to error during import of component code:\n%s",
path,
str(e),
)
module_path = module_path_from_code(code)
mod = importlib.import_module(module_path)
mod_docstring = mod.__doc__ or ""
mod_docstring_lines = mod_docstring.splitlines()
if hasattr(main_func, "registered_metadata"):
logger.info("Get component info from registered metadata")
component_name = main_func.registered_metadata[
"name"] or ( # type: ignore
"Unnamed Component")
component_description = main_func.registered_metadata[
"description"] or ( # type: ignore
"No description provided")
component_category = main_func.registered_metadata[
"category"] or ( # type: ignore
"Other")
component_id = main_func.registered_metadata["id"] or ( # type: ignore
get_uuid_from_seed(str(component_name)))
component_group_id = main_func.registered_metadata[
"revision_group_id"] or ( # type: ignore
get_uuid_from_seed(str(component_name)))
component_tag = main_func.registered_metadata["version_tag"] or (
"1.0.0") # type: ignore
component_inputs = main_func.registered_metadata[
"inputs"] # type: ignore
component_outputs = main_func.registered_metadata[
"outputs"] # type: ignore
component_state = main_func.registered_metadata[
"state"] or "RELEASED" # type: ignore
component_released_timestamp = main_func.registered_metadata[ # type: ignore
"released_timestamp"] or (datetime.now(timezone.utc).isoformat() if
component_state == "RELEASED" else None)
component_disabled_timestamp = main_func.registered_metadata[ # type: ignore
"disabled_timestamp"] or (datetime.now(timezone.utc).isoformat() if
component_state == "DISABLED" else None)
elif hasattr(mod, "COMPONENT_INFO"):
logger.info("Get component info from dictionary in code")
info_dict = mod.COMPONENT_INFO
component_inputs = info_dict.get("inputs", {})
component_outputs = info_dict.get("outputs", {})
component_name = info_dict.get("name", "Unnamed Component")
component_description = info_dict.get("description",
"No description provided")
component_category = info_dict.get("category", "Other")
component_tag = info_dict.get("version_tag", "1.0.0")
component_id = info_dict.get("id",
get_uuid_from_seed(str(component_name)))
component_group_id = info_dict.get(
"revision_group_id", get_uuid_from_seed(str(component_name)))
component_state = info_dict.get("state", "RELEASED")
component_released_timestamp = info_dict.get(
"released_timestamp",
datetime.now(timezone.utc).isoformat()
if component_state == "RELEASED" else None,
)
component_disabled_timestamp = info_dict.get(
"released_timestamp",
datetime.now(timezone.utc).isoformat()
if component_state == "DISABLED" else None,
)
else:
raise ComponentCodeImportError
component_documentation = "\n".join(mod_docstring_lines[2:])
transformation_revision = TransformationRevision(
id=component_id,
revision_group_id=component_group_id,
name=component_name,
description=component_description,
category=component_category,
version_tag=component_tag,
type=Type.COMPONENT,
state=component_state,
released_timestamp=component_released_timestamp,
disabled_timestamp=component_disabled_timestamp,
documentation=component_documentation,
io_interface=IOInterface(
inputs=[
IO(
id=get_uuid_from_seed("component_input_" + input_name),
name=input_name,
data_type=input_data_type,
) for input_name, input_data_type in component_inputs.items()
],
outputs=[
IO(
id=get_uuid_from_seed("component_output_" + output_name),
name=output_name,
data_type=output_data_type,
)
for output_name, output_data_type in component_outputs.items()
],
),
content=code,
test_wiring=WorkflowWiring(),
)
tr_json = json.loads(transformation_revision.json())
return tr_json
