def test_notebooks_field_circular_dependency(monkeypatch):
"""
Test that NotebooksField raises a ValidationError if notebook
specifications have circular dependencies.
"""
monkeypatch.setattr("pathlib.Path.exists", lambda self: True)
notebooks = {
"notebook1": {
"filename": "NOTEBOOK1.ipynb",
"parameters": {
"param": "notebook2"
},
},
"notebook2": {
"filename": "NOTEBOOK2.ipynb",
"parameters": {
"param": "notebook1"
},
},
}
notebooks_field = NotebooksField()
# Can't set context directly on a Field - must be set on the parent Schema
notebooks_field._bind_to_schema(
"notebooks", Schema(context={"inputs_dir": "DUMMY_INPUTS_DIR"}))
with pytest.raises(ValidationError) as exc_info:
deserialised_notebooks = notebooks_field.deserialize(notebooks)
assert ("Notebook specifications contain circular dependencies."
in exc_info.value.messages)
def test_notebooks_field_invalid_keys(monkeypatch, key, message):
"""
Test that NotebooksField raises a ValidationError if a notebook key is not
a string, or has a disallowed value.
"""
monkeypatch.setattr("pathlib.Path.exists", lambda self: True)
notebooks = {key: {"filename": "NOTEBOOK1.ipynb"}}
notebooks_field = NotebooksField()
# Can't set context directly on a Field - must be set on the parent Schema
notebooks_field._bind_to_schema(
"notebooks", Schema(context={"inputs_dir": "DUMMY_INPUTS_DIR"}))
with pytest.raises(ValidationError) as exc_info:
deserialised_notebooks = notebooks_field.deserialize(notebooks)
assert message in exc_info.value.messages[key]["key"]
def test_notebooks_field_init_does_not_accept_values():
"""
Test that NotebooksField.__init__ does not accept a 'values' argument.
"""
with pytest.raises(
TypeError,
match="The Notebooks field does not accept a 'values' argument."):
field = NotebooksField(values=str)
class WorkflowSchema(Schema):
"""
Schema for a notebook-based workflow specification.
Fields
------
name : str
Name of the prefect flow.
notebooks : dict
Dictionary of notebook task specifications.
"""
name = fields.String(required=True)
notebooks = NotebooksField(required=True)
storage_path = fields.String(required=False)
@validates_schema(pass_many=True)
def check_for_duplicate_names(self, data, many, **kwargs):
"""
If this schema is used with 'many=True', raise a ValidationError if any workflow names are duplicated.
"""
if many:
errors = {}
names = set()
for i, workflow in enumerate(data):
if workflow["name"] in names:
errors[i] = {"name": [f"Duplicate workflow name."]}
else:
names.add(workflow["name"])
if errors:
raise ValidationError(errors)
else:
pass
@post_load(pass_many=True)
def make_and_store_workflows(self, data, many, **kwargs) -> storage.Local:
"""
Create a prefect flow for each of the provided workflow specifications,
and return as a prefect 'Local' storage object.
"""
if not many:
data = [data]
storage_path = data[0].get("storage_path")
workflow_storage = storage.Local(directory=storage_path)
for workflow_spec in data:
workflow = make_notebooks_workflow(**workflow_spec)
workflow_storage.add_flow(workflow)
return workflow_storage
def test_notebooks_field_deserialise(monkeypatch):
"""
Test that NotebooksField deserialises a dict of notebook specifications as
an OrderedDict.
"""
monkeypatch.setattr("pathlib.Path.exists", lambda self: True)
notebooks = {
"notebook1": {
"filename": "NOTEBOOK1.ipynb"
},
"notebook2": {
"filename": "NOTEBOOK2.ipynb",
"parameters": {
"other_notebook": "notebook1"
},
},
}
notebooks_field = NotebooksField()
# Can't set context directly on a Field - must be set on the parent Schema
notebooks_field._bind_to_schema(
"notebooks", Schema(context={"inputs_dir": "DUMMY_INPUTS_DIR"}))
deserialised_notebooks = notebooks_field.deserialize(notebooks)
assert isinstance(deserialised_notebooks, OrderedDict)
assert deserialised_notebooks == notebooks