def __init__(self, flowname, filename=None):
self.filename = filename or f"{flowname}-flow.py"
self.flowname = flowname
if self.flowname is None:
Console.error("The floname is not defined")
sys.exit(1)
self.db = FlowDatabase(flowname)
self.running_jobs = []
class FlowRunner(object):
def __init__(self, flowname, filename=None):
self.filename = filename or f"{flowname}-flow.py"
self.flowname = flowname
if self.flowname is None:
Console.error("The floname is not defined")
sys.exit(1)
self.db = FlowDatabase(flowname)
self.running_jobs = []
def start_available_nodes(self):
available_nodes = self.db.find_root_nodes()
for node in available_nodes:
print("starting a new node", node)
self.start_node(node)
def start_flow(self):
self.running = True
self.db.start_flow()
self.start_available_nodes()
while (self.running):
self.check_on_running_processes()
self.running = len(self.running_jobs) > 0
def start_node(self, node):
self.db.set_node_status(node.name, "running")
print("running command", node.get_command())
process = subprocess.Popen(node.get_command(), stdout=subprocess.PIPE)
self.running_jobs.append({"handle": process, "node": node})
def resolve_node(self, node, status):
resolution = "finished" if status == 0 else "error"
self.db.set_node_status(node.name, resolution)
#self.db.add_node_result(node.name, output)
if status == 0:
self.db.resolve_node_dependencies(node.name)
#easiest way to remove object, but will be slow for large workflows. to improve later
self.running_jobs = [
job for job in self.running_jobs if job["node"].name != node.name
]
def check_on_running_processes(self):
for process in self.running_jobs:
process_handle = process["handle"]
status = process_handle.poll()
if status is None:
continue
else:
#printed_output = process_handle.communicate()[0]
#print(printed_output)
#output = json.loads(printed_output)
self.resolve_node(process["node"], status)
self.start_available_nodes()
time.sleep(3)
def visualize(self):
url = "http://127.0.0.1:8080/flow/monitor/" + self.flowname + "-flow"
webbrowser.open(url)
def test_start_flow(self):
self.db.collection.delete_many({})
node_1 = Node("node1")
node_2 = Node("node2")
node_3 = Node("node3")
node_1.add_dependency(node_2)
for node in [node_1, node_2, node_3]:
self.db.add_node(node.toDict())
self.db.start_flow()
new_collection = self.db.collection
print(new_collection)
new_nodes = self.db.list_nodes()
for node in new_nodes:
print(node.status)
assert node.status == "pending"
self.db = FlowDatabase("test")
class Test_baseclass:
# noinspection PyPep8Naming
def tearDown(self):
pass
def setup(self):
self.db = FlowDatabase("test")
self.db.collection.delete_many({})
self.db.add_node({"name": "a", "dependencies": []})
self.db.start_flow()
self.flow = SampleFlow("test-flow.py")
def test_runmethod(self):
result = self.flow._run("a")
assert result["name"] == "a"
def test_database_insertion(self):
result = self.flow._run("a")
dbresult = self.db.get_node("a")
assert dbresult.result["name"] == "a"
assert dbresult.result["result"]["everything"] == "ok"
def setup(self):
self.db = FlowDatabase("test")
self.db.collection.delete_many({})
class Test_flowdb:
def tearDown(self):
pass
def setup(self):
self.db = FlowDatabase("test")
self.db.collection.delete_many({})
def test_add_node(self):
test_node = Node("test test")
self.db.add_node(test_node.toDict())
num_nodes = self.db.collection.count_documents({})
assert num_nodes == 1
def test_add_edge(self):
node_1 = Node("testsource")
node_2 = Node("testdest")
self.db.add_node(node_1.toDict())
self.db.add_node(node_2.toDict())
self.db.add_edge(node_1.name, node_2.name)
deps = self.db.collection.count_documents(
{"dependencies.0": {
"$exists": True
}})
assert deps == 1
def test_set_node_status(self):
node_name = "status_test"
status = "testing"
node_1 = Node(node_name)
self.db.add_node(node_1.toDict())
inserted_node = self.db.get_node(node_name)
print(inserted_node.status)
self.db.set_node_status(node_name, status)
reset_node = self.db.get_node(node_name)
assert reset_node.status == status
def test_start_flow(self):
self.db.collection.delete_many({})
node_1 = Node("node1")
node_2 = Node("node2")
node_3 = Node("node3")
node_1.add_dependency(node_2)
for node in [node_1, node_2, node_3]:
self.db.add_node(node.toDict())
self.db.start_flow()
new_collection = self.db.collection
print(new_collection)
new_nodes = self.db.list_nodes()
for node in new_nodes:
print(node.status)
assert node.status == "pending"
self.db = FlowDatabase("test")
def test_remove(self):
self.db.collection.delete_many({})
node_1 = Node("testsource")
node_2 = Node("testdest")
self.db.add_node(node_1.toDict())
self.db.add_node(node_2.toDict())
self.db.add_edge(node_1.name, node_2.name)
deps = self.db.collection.count_documents(
{"dependencies.0": {
"$exists": True
}})
assert deps == 1
self.db.remove_edge(node_1.name, node_2.name)
deps = self.db.collection.count_documents(
{"dependencies.0": {
"$exists": True
}})
assert deps == 0
self.db.remove_node(node_1.name)
nodes = self.db.collection.count_documents({"name": node_1.name})
assert nodes == 0
def test_flowstring_parser(self):
self.db.collection.delete_many({})
flowstring = "( pytesta; pytestb ) ; ( pytestc; ( pytestd || pyteste ) ) || pytestf"
parse_string_to_workflow(flowstring, "test")
nodes = self.db.list_nodes()
names = [node.name for node in nodes]
for node in nodes:
print(node)
for flow_string_name in [
"pytesta", "pytestb", "pytestc", "pytestd", "pyteste",
"pytestf"
]:
assert flow_string_name in names
node_a = [node for node in nodes if node.name == "pytesta"][0]
assert len(node_a.dependencies) == 0
node_f = [node for node in nodes if node.name == "pytestf"][0]
assert len(node_f.dependencies) == 0
node_b = [node for node in nodes if node.name == "pytestb"][0]
assert len(node_b.dependencies) == 1
assert "pytesta" in node_b.dependencies
def save_result_to_db(self, nodeName, result):
print("saving result to", self.flowname, result)
db = FlowDatabase(self.flowname, True)
db.add_node_result(nodeName, result)
def setup(self):
self.db = FlowDatabase("test")
self.db.collection.delete_many({})
self.db.add_node({"name": "a", "dependencies": []})
self.db.start_flow()
self.flow = SampleFlow("test-flow.py")
def do_flow(self, args, arguments):
"""
::
Usage:
flow list [--flow=NAME] [--output=FORMAT]
flow add [--flowname=FLOWNAME] --flowfile=FILENAME
flow run [--flowname=FLOWNAME] [--flowfile=FILENAME]
flow node add NODENAME [--flowname=FLOWNAME]
flow edge add FROM TO [--flowname=FLOWNAME]
flow node delete NODENAME
flow edge delete FROM TO
flow edge invert FROM TO
flow visualize start
flow visualize stop
flow refresh
This command manages and executes workflows
The default workflow is just named "workflow" but you can specify multiple
Arguments:
NAME the name of the workflow
FILENAME a file name
NODENAME the name of the node
FROM the edge source (a node name)
TO the edge destination (a node name)
NODE the name of the node
Options:
--flow=NAME the name or the flow
--file specify the file
--log specify the log file
--flowname=FLOWNAME the name or the workflow
--output=OUTPUT the output format [default: table]
"""
arguments.FLOWNAME = arguments["--flowname"] or "workflow"
arguments.FLOWFILE = arguments[
"--flowfile"] or f"{arguments.FLOWNAME}-flow.py"
arguments.output = arguments["--output"]
VERBOSE(arguments, verbose=0)
if arguments["add"] and arguments.edge:
db = FlowDatabase(arguments.FLOWNAME)
db.add_edge(arguments.FROM, arguments.TO)
elif arguments["add"]:
print("adding a node")
if arguments.NODENAME:
node = Node(arguments.NODENAME)
node.workflow = arguments.FLOWNAME
try:
db = FlowDatabase(arguments.FLOWNAME)
db.add_node(node.toDict())
except Exception as e:
print("error executing", e)
elif arguments["--flowfile"]:
filename = arguments["--flowfile"]
print("load from file", filename)
parse_yaml_to_workflow(filename)
elif arguments["list"]:
arguments.flow = arguments["--flow"] or "workflow"
db = CmDatabase()
name = arguments["--flow"]
if name is not None:
flows = [name]
else:
candidates = db.collections()
flows = []
for flow in candidates:
if flow.endswith("-flow"):
flows.append(flow)
entries = []
for name in flows:
nodes = db.find(collection=f"{name}-flow")
for node in nodes:
node["dependencies"] = ", ".join(node["dependencies"])
entries = entries + nodes
order = ["name", "workflow", "dependencies", "cm.modified"]
header = ["Name", "Workflow", "Dependencies", "Modified"]
print(
Printer.flatwrite(nodes,
order=order,
header=header,
output=arguments.output))
elif arguments._run:
runner = FlowRunner(arguments.FLOWNAME, arguments.FLOWFILE)
runner.start_flow()
elif arguments.visualize:
if arguments["start"]:
manager.start()
print(
"The visualization servive started at http://127.0.0.1:8080/flow/"
)
elif arguments["stop"]:
manager.stop()
elif arguments["delete"] and arguments.edge:
db = FlowDatabase(arguments.FLOWNAME)
db.remove_edge(arguments.FROM, arguments.TO)
elif arguments["delete"] and arguments.node:
db = FlowDatabase(arguments.FLOWNAME)
db.remove_node(arguments.NODENAME)
elif arguments["invert"] and arguments.edge:
db = FlowDatabase(arguments.FLOWNAME)
db.remove_edge(arguments.TO, arguments.FROM)
db.add_edge(arguments.FROM, arguments.TO)
elif arguments.refresh:
raise NotImplementedError