def test_workflow_config_schema_missing_and_unexpected_parameter_names():
"""
Test that WorkflowConfigSchema raises a ValidationError if the names of the
provided parameters are not valid for the named workflow.
"""
# Create storage object to pass in context
dummy_workflow = Flow(name="DUMMY_WORKFLOW")
for parameter in ["reference_date", "date_ranges", "DUMMY_PARAM"]:
dummy_workflow.add_task(Parameter(parameter))
workflow_storage = storage.Memory()
workflow_storage.add_flow(dummy_workflow)
input_dict = dict(
workflow_name="DUMMY_WORKFLOW", parameters={"EXTRA_PARAM": "DUMMY_VALUE"}
)
with pytest.raises(ValidationError) as exc_info:
workflow_config = WorkflowConfigSchema(
context={"workflow_storage": workflow_storage}
).load(input_dict)
assert (
"Missing required parameters {'DUMMY_PARAM'} for workflow 'DUMMY_WORKFLOW'."
in exc_info.value.messages["parameters"]
)
assert (
"Unexpected parameters provided for workflow 'DUMMY_WORKFLOW': {'EXTRA_PARAM'}."
in exc_info.value.messages["parameters"]
)
def test_workflow_config_schema():
"""
Test that WorkflowConfigSchema loads input data into a WorkflowConfig namedtuple.
"""
# Create storage object to pass in context
dummy_workflow = Flow(name="DUMMY_WORKFLOW")
for parameter in ["reference_date", "date_ranges", "DUMMY_PARAM"]:
dummy_workflow.add_task(Parameter(parameter))
workflow_storage = storage.Memory()
workflow_storage.add_flow(dummy_workflow)
input_dict = dict(
workflow_name="DUMMY_WORKFLOW",
parameters={"DUMMY_PARAM": "DUMMY_VALUE"},
earliest_date=datetime.date(2016, 1, 1),
date_stencil=[-1, 0],
)
workflow_config = WorkflowConfigSchema(
context={"workflow_storage": workflow_storage}
).load(input_dict)
assert isinstance(workflow_config, WorkflowConfig)
assert workflow_config.workflow_name == input_dict["workflow_name"]
assert workflow_config.parameters == input_dict["parameters"]
assert workflow_config.earliest_date == input_dict["earliest_date"]
assert workflow_config.date_stencil == DateStencil(input_dict["date_stencil"])
def test_memory_roundtrip():
s = storage.Memory()
s.add_flow(prefect.Flow("test"))
serialized = MemorySchema().dump(s)
assert serialized == {"__version__": prefect.__version__}
deserialized = MemorySchema().load(serialized)
assert deserialized.flows == dict()
def make_and_store_workflows(self, data, many, **kwargs) -> storage.Memory:
"""
Create a prefect flow for each of the provided workflow specifications,
and return as a prefect 'Memory' storage object.
"""
workflow_storage = storage.Memory()
if not many:
data = [data]
for workflow_spec in data:
workflow = make_notebooks_workflow(**workflow_spec)
workflow_storage.add_flow(workflow)
return workflow_storage
def test_workflow_config_schema_invalid_parameter_names(key):
"""
Test that WorkflowConfigSchema raises a ValidationError if the 'parameters'
dict keys contain 'reference_date' or 'date_ranges'.
"""
# Create storage object to pass in context
dummy_workflow = Flow(name="DUMMY_WORKFLOW")
for parameter in ["reference_date", "date_ranges"]:
dummy_workflow.add_task(Parameter(parameter))
workflow_storage = storage.Memory()
workflow_storage.add_flow(dummy_workflow)
input_dict = dict(workflow_name="DUMMY_WORKFLOW", parameters={key: "DUMMY_VALUE"})
with pytest.raises(ValidationError) as exc_info:
workflow_config = WorkflowConfigSchema(
context={"workflow_storage": workflow_storage}
).load(input_dict)
assert "Invalid input." in exc_info.value.messages["parameters"][key]["key"]
def test_workflow_config_schema_defaults():
"""
Test that WorkflowConfigSchema loads input data if 'parameters',
'earliest_date' and 'date_stencil' are not specified.
"""
# Create storage object to pass in context
dummy_workflow = Flow(name="DUMMY_WORKFLOW")
for parameter in ["reference_date", "date_ranges"]:
dummy_workflow.add_task(Parameter(parameter))
workflow_storage = storage.Memory()
workflow_storage.add_flow(dummy_workflow)
input_dict = dict(workflow_name="DUMMY_WORKFLOW")
workflow_config = WorkflowConfigSchema(
context={"workflow_storage": workflow_storage}
).load(input_dict)
assert isinstance(workflow_config, WorkflowConfig)
assert workflow_config == WorkflowConfig(workflow_name="DUMMY_WORKFLOW")
def test_workflow_config_schema_invalid_earliest_date():
"""
Test that WorkflowConfigSchema raises a ValidationError if the
'earliest_date' field is not a date.
"""
# Create storage object to pass in context
dummy_workflow = Flow(name="DUMMY_WORKFLOW")
for parameter in ["reference_date", "date_ranges"]:
dummy_workflow.add_task(Parameter(parameter))
workflow_storage = storage.Memory()
workflow_storage.add_flow(dummy_workflow)
input_dict = dict(workflow_name="DUMMY_WORKFLOW", earliest_date=datetime.time(11))
with pytest.raises(ValidationError) as exc_info:
workflow_config = WorkflowConfigSchema(
context={"workflow_storage": workflow_storage}
).load(input_dict)
assert "Not a valid date." in exc_info.value.messages["earliest_date"]
def test_workflow_config_schema_workflow_not_found():
"""
Test that WorkflowConfigSchema raises a ValidationError if the named
workflow does not exist.
"""
# Create storage object to pass in context
dummy_workflow = Flow(name="DUMMY_WORKFLOW")
for parameter in ["reference_date", "date_ranges"]:
dummy_workflow.add_task(Parameter(parameter))
workflow_storage = storage.Memory()
workflow_storage.add_flow(dummy_workflow)
with pytest.raises(ValidationError) as exc_info:
workflow_config = WorkflowConfigSchema(
context={"workflow_storage": workflow_storage}
).load({"workflow_name": "NONEXISTENT_WORKFLOW"})
assert (
"Workflow does not exist in this storage."
in exc_info.value.messages["workflow_name"]
)
def test_workflow_config_schema_workflow_does_not_accept_automatic_parameters(
missing_parameter,
):
"""
Test that WorkflowConfigSchema raises a ValidationError if the named
workflow does not accept parameters 'reference_date' and 'date_ranges'.
"""
# Create storage object to pass in context
dummy_workflow = Flow(name="DUMMY_WORKFLOW")
for parameter in {"reference_date", "date_ranges"} - {missing_parameter}:
dummy_workflow.add_task(Parameter(parameter))
workflow_storage = storage.Memory()
workflow_storage.add_flow(dummy_workflow)
with pytest.raises(ValidationError) as exc_info:
workflow_config = WorkflowConfigSchema(
context={"workflow_storage": workflow_storage}
).load({"workflow_name": "DUMMY_WORKFLOW"})
assert (
f"Workflow does not accept parameters {{'{missing_parameter}'}}."
in exc_info.value.messages["workflow_name"]
)
def test_memory_serialize():
s = storage.Memory()
serialized = MemorySchema().dump(s)
assert serialized == {"__version__": prefect.__version__}
