async def test_export(self):
self.required_plugins("shouldi")
stdout = io.StringIO()
# Use shouldi's dataflow for tests
with relative_chdir("..", "..", "examples", "shouldi"):
with unittest.mock.patch("sys.stdout.buffer.write") as write:
await Develop.cli("export", "shouldi.cli:DATAFLOW")
DataFlow._fromdict(**json.loads(write.call_args[0][0]))
async def test_run(self):
stdout = io.BytesIO()
with unittest.mock.patch("sys.stdout.buffer.write", new=stdout.write):
await Export(
export="tests.test_df:DATAFLOW", not_linked=False
).run()
exported = json.loads(stdout.getvalue())
DataFlow._fromdict(**exported)
async def test_dataflow_run_cli_example(self):
# Write out override dataflow
created = self.mktempfile() + ".yaml"
with open(created, "w") as fileobj:
with contextlib.redirect_stdout(fileobj):
await CLI.cli(
"dataflow",
"create",
"dffml.mapping.create",
"print_output",
"-configloader",
"yaml",
)
# Load the generated dataflow
async with ConfigLoaders() as cfgl:
_, exported = await cfgl.load_file(created)
dataflow = DataFlow._fromdict(**exported)
# Modify the dataflow
dataflow.flow["print_output"].inputs["data"] = [{
"dffml.mapping.create":
"mapping"
}]
# Write back modified dataflow
async with BaseConfigLoader.load("yaml").withconfig(
{}) as configloader:
async with configloader() as loader:
with open(created, "wb") as fileobj:
fileobj.write(await
loader.dumpb(dataflow.export(linked=True)))
# Run the dataflow
with contextlib.redirect_stdout(self.stdout):
await CLI.cli(
"dataflow",
"run",
"records",
"all",
"-no-echo",
"-record-def",
"value",
"-inputs",
"hello=key",
"-dataflow",
created,
"-sources",
"m=memory",
"-source-records",
"world",
"user",
)
self.assertEqual(self.stdout.getvalue(),
"{'hello': 'world'}\n{'hello': 'user'}\n")
async def __aenter__(self) -> "DataFlowSourceContext":
self.sctx = await self.parent.source().__aenter__()
if isinstance(self.parent.config.dataflow, str):
dataflow_path = pathlib.Path(self.parent.config.dataflow)
config_type = dataflow_path.suffix.replace(".", "")
config_cls = BaseConfigLoader.load(config_type)
async with config_cls.withconfig({}) as configloader:
async with configloader() as loader:
exported = await loader.loadb(dataflow_path.read_bytes())
self.parent.config.dataflow = DataFlow._fromdict(**exported)
self.octx = await self.parent.orchestrator(self.parent.config.dataflow
).__aenter__()
return self
def test_resolve_missing_input_output_definition(self):
exported = DataFlow.auto(add).export(linked=True)
del exported["definitions"]["result"]
with self.assertRaisesRegex(DefinitionMissing, "add.outputs.*result"):
DataFlow._fromdict(**exported)
def test_resolve_missing_condition_definition(self):
exported = DataFlow.auto(add).export(linked=True)
del exported["definitions"]["is_add"]
with self.assertRaisesRegex(DefinitionMissing,
"add.conditions.*is_add"):
DataFlow._fromdict(**exported)
def _fromdict(cls, **kwargs):
kwargs["dataflow"] = DataFlow._fromdict(**kwargs["dataflow"])
return cls(**kwargs)