class Test_baseclass:
# noinspection PyPep8Naming
def tearDown(self):
pass
def setup(self):
self.db = FlowDatabase("test")
self.db.collection.delete_many({})
self.db.add_node({"name": "a", "dependencies": []})
self.db.start_flow()
self.flow = SampleFlow("test-flow.py")
def test_runmethod(self):
result = self.flow._run("a")
assert result["name"] == "a"
def test_database_insertion(self):
result = self.flow._run("a")
dbresult = self.db.get_node("a")
assert dbresult.result["name"] == "a"
assert dbresult.result["result"]["everything"] == "ok"
class Test_flowdb:
def tearDown(self):
pass
def setup(self):
self.db = FlowDatabase("test")
self.db.collection.delete_many({})
def test_add_node(self):
test_node = Node("test test")
self.db.add_node(test_node.toDict())
num_nodes = self.db.collection.count_documents({})
assert num_nodes == 1
def test_add_edge(self):
node_1 = Node("testsource")
node_2 = Node("testdest")
self.db.add_node(node_1.toDict())
self.db.add_node(node_2.toDict())
self.db.add_edge(node_1.name, node_2.name)
deps = self.db.collection.count_documents(
{"dependencies.0": {
"$exists": True
}})
assert deps == 1
def test_set_node_status(self):
node_name = "status_test"
status = "testing"
node_1 = Node(node_name)
self.db.add_node(node_1.toDict())
inserted_node = self.db.get_node(node_name)
print(inserted_node.status)
self.db.set_node_status(node_name, status)
reset_node = self.db.get_node(node_name)
assert reset_node.status == status
def test_start_flow(self):
self.db.collection.delete_many({})
node_1 = Node("node1")
node_2 = Node("node2")
node_3 = Node("node3")
node_1.add_dependency(node_2)
for node in [node_1, node_2, node_3]:
self.db.add_node(node.toDict())
self.db.start_flow()
new_collection = self.db.collection
print(new_collection)
new_nodes = self.db.list_nodes()
for node in new_nodes:
print(node.status)
assert node.status == "pending"
self.db = FlowDatabase("test")
def test_remove(self):
self.db.collection.delete_many({})
node_1 = Node("testsource")
node_2 = Node("testdest")
self.db.add_node(node_1.toDict())
self.db.add_node(node_2.toDict())
self.db.add_edge(node_1.name, node_2.name)
deps = self.db.collection.count_documents(
{"dependencies.0": {
"$exists": True
}})
assert deps == 1
self.db.remove_edge(node_1.name, node_2.name)
deps = self.db.collection.count_documents(
{"dependencies.0": {
"$exists": True
}})
assert deps == 0
self.db.remove_node(node_1.name)
nodes = self.db.collection.count_documents({"name": node_1.name})
assert nodes == 0
def test_flowstring_parser(self):
self.db.collection.delete_many({})
flowstring = "( pytesta; pytestb ) ; ( pytestc; ( pytestd || pyteste ) ) || pytestf"
parse_string_to_workflow(flowstring, "test")
nodes = self.db.list_nodes()
names = [node.name for node in nodes]
for node in nodes:
print(node)
for flow_string_name in [
"pytesta", "pytestb", "pytestc", "pytestd", "pyteste",
"pytestf"
]:
assert flow_string_name in names
node_a = [node for node in nodes if node.name == "pytesta"][0]
assert len(node_a.dependencies) == 0
node_f = [node for node in nodes if node.name == "pytestf"][0]
assert len(node_f.dependencies) == 0
node_b = [node for node in nodes if node.name == "pytestb"][0]
assert len(node_b.dependencies) == 1
assert "pytesta" in node_b.dependencies
def do_flow(self, args, arguments):
"""
::
Usage:
flow list [--flow=NAME] [--output=FORMAT]
flow add [--flowname=FLOWNAME] --flowfile=FILENAME
flow run [--flowname=FLOWNAME] [--flowfile=FILENAME]
flow node add NODENAME [--flowname=FLOWNAME]
flow edge add FROM TO [--flowname=FLOWNAME]
flow node delete NODENAME
flow edge delete FROM TO
flow edge invert FROM TO
flow visualize start
flow visualize stop
flow refresh
This command manages and executes workflows
The default workflow is just named "workflow" but you can specify multiple
Arguments:
NAME the name of the workflow
FILENAME a file name
NODENAME the name of the node
FROM the edge source (a node name)
TO the edge destination (a node name)
NODE the name of the node
Options:
--flow=NAME the name or the flow
--file specify the file
--log specify the log file
--flowname=FLOWNAME the name or the workflow
--output=OUTPUT the output format [default: table]
"""
arguments.FLOWNAME = arguments["--flowname"] or "workflow"
arguments.FLOWFILE = arguments[
"--flowfile"] or f"{arguments.FLOWNAME}-flow.py"
arguments.output = arguments["--output"]
VERBOSE(arguments, verbose=0)
if arguments["add"] and arguments.edge:
db = FlowDatabase(arguments.FLOWNAME)
db.add_edge(arguments.FROM, arguments.TO)
elif arguments["add"]:
print("adding a node")
if arguments.NODENAME:
node = Node(arguments.NODENAME)
node.workflow = arguments.FLOWNAME
try:
db = FlowDatabase(arguments.FLOWNAME)
db.add_node(node.toDict())
except Exception as e:
print("error executing", e)
elif arguments["--flowfile"]:
filename = arguments["--flowfile"]
print("load from file", filename)
parse_yaml_to_workflow(filename)
elif arguments["list"]:
arguments.flow = arguments["--flow"] or "workflow"
db = CmDatabase()
name = arguments["--flow"]
if name is not None:
flows = [name]
else:
candidates = db.collections()
flows = []
for flow in candidates:
if flow.endswith("-flow"):
flows.append(flow)
entries = []
for name in flows:
nodes = db.find(collection=f"{name}-flow")
for node in nodes:
node["dependencies"] = ", ".join(node["dependencies"])
entries = entries + nodes
order = ["name", "workflow", "dependencies", "cm.modified"]
header = ["Name", "Workflow", "Dependencies", "Modified"]
print(
Printer.flatwrite(nodes,
order=order,
header=header,
output=arguments.output))
elif arguments._run:
runner = FlowRunner(arguments.FLOWNAME, arguments.FLOWFILE)
runner.start_flow()
elif arguments.visualize:
if arguments["start"]:
manager.start()
print(
"The visualization servive started at http://127.0.0.1:8080/flow/"
)
elif arguments["stop"]:
manager.stop()
elif arguments["delete"] and arguments.edge:
db = FlowDatabase(arguments.FLOWNAME)
db.remove_edge(arguments.FROM, arguments.TO)
elif arguments["delete"] and arguments.node:
db = FlowDatabase(arguments.FLOWNAME)
db.remove_node(arguments.NODENAME)
elif arguments["invert"] and arguments.edge:
db = FlowDatabase(arguments.FLOWNAME)
db.remove_edge(arguments.TO, arguments.FROM)
db.add_edge(arguments.FROM, arguments.TO)
elif arguments.refresh:
raise NotImplementedError