def __init__(self, flowname, filename=None): self.filename = filename or f"{flowname}-flow.py" self.flowname = flowname if self.flowname is None: Console.error("The floname is not defined") sys.exit(1) self.db = FlowDatabase(flowname) self.running_jobs = []
class FlowRunner(object): def __init__(self, flowname, filename=None): self.filename = filename or f"{flowname}-flow.py" self.flowname = flowname if self.flowname is None: Console.error("The floname is not defined") sys.exit(1) self.db = FlowDatabase(flowname) self.running_jobs = [] def start_available_nodes(self): available_nodes = self.db.find_root_nodes() for node in available_nodes: print("starting a new node", node) self.start_node(node) def start_flow(self): self.running = True self.db.start_flow() self.start_available_nodes() while (self.running): self.check_on_running_processes() self.running = len(self.running_jobs) > 0 def start_node(self, node): self.db.set_node_status(node.name, "running") print("running command", node.get_command()) process = subprocess.Popen(node.get_command(), stdout=subprocess.PIPE) self.running_jobs.append({"handle": process, "node": node}) def resolve_node(self, node, status): resolution = "finished" if status == 0 else "error" self.db.set_node_status(node.name, resolution) #self.db.add_node_result(node.name, output) if status == 0: self.db.resolve_node_dependencies(node.name) #easiest way to remove object, but will be slow for large workflows. to improve later self.running_jobs = [ job for job in self.running_jobs if job["node"].name != node.name ] def check_on_running_processes(self): for process in self.running_jobs: process_handle = process["handle"] status = process_handle.poll() if status is None: continue else: #printed_output = process_handle.communicate()[0] #print(printed_output) #output = json.loads(printed_output) self.resolve_node(process["node"], status) self.start_available_nodes() time.sleep(3) def visualize(self): url = "http://127.0.0.1:8080/flow/monitor/" + self.flowname + "-flow" webbrowser.open(url)
def test_start_flow(self): self.db.collection.delete_many({}) node_1 = Node("node1") node_2 = Node("node2") node_3 = Node("node3") node_1.add_dependency(node_2) for node in [node_1, node_2, node_3]: self.db.add_node(node.toDict()) self.db.start_flow() new_collection = self.db.collection print(new_collection) new_nodes = self.db.list_nodes() for node in new_nodes: print(node.status) assert node.status == "pending" self.db = FlowDatabase("test")
class Test_baseclass: # noinspection PyPep8Naming def tearDown(self): pass def setup(self): self.db = FlowDatabase("test") self.db.collection.delete_many({}) self.db.add_node({"name": "a", "dependencies": []}) self.db.start_flow() self.flow = SampleFlow("test-flow.py") def test_runmethod(self): result = self.flow._run("a") assert result["name"] == "a" def test_database_insertion(self): result = self.flow._run("a") dbresult = self.db.get_node("a") assert dbresult.result["name"] == "a" assert dbresult.result["result"]["everything"] == "ok"
def setup(self): self.db = FlowDatabase("test") self.db.collection.delete_many({})
class Test_flowdb: def tearDown(self): pass def setup(self): self.db = FlowDatabase("test") self.db.collection.delete_many({}) def test_add_node(self): test_node = Node("test test") self.db.add_node(test_node.toDict()) num_nodes = self.db.collection.count_documents({}) assert num_nodes == 1 def test_add_edge(self): node_1 = Node("testsource") node_2 = Node("testdest") self.db.add_node(node_1.toDict()) self.db.add_node(node_2.toDict()) self.db.add_edge(node_1.name, node_2.name) deps = self.db.collection.count_documents( {"dependencies.0": { "$exists": True }}) assert deps == 1 def test_set_node_status(self): node_name = "status_test" status = "testing" node_1 = Node(node_name) self.db.add_node(node_1.toDict()) inserted_node = self.db.get_node(node_name) print(inserted_node.status) self.db.set_node_status(node_name, status) reset_node = self.db.get_node(node_name) assert reset_node.status == status def test_start_flow(self): self.db.collection.delete_many({}) node_1 = Node("node1") node_2 = Node("node2") node_3 = Node("node3") node_1.add_dependency(node_2) for node in [node_1, node_2, node_3]: self.db.add_node(node.toDict()) self.db.start_flow() new_collection = self.db.collection print(new_collection) new_nodes = self.db.list_nodes() for node in new_nodes: print(node.status) assert node.status == "pending" self.db = FlowDatabase("test") def test_remove(self): self.db.collection.delete_many({}) node_1 = Node("testsource") node_2 = Node("testdest") self.db.add_node(node_1.toDict()) self.db.add_node(node_2.toDict()) self.db.add_edge(node_1.name, node_2.name) deps = self.db.collection.count_documents( {"dependencies.0": { "$exists": True }}) assert deps == 1 self.db.remove_edge(node_1.name, node_2.name) deps = self.db.collection.count_documents( {"dependencies.0": { "$exists": True }}) assert deps == 0 self.db.remove_node(node_1.name) nodes = self.db.collection.count_documents({"name": node_1.name}) assert nodes == 0 def test_flowstring_parser(self): self.db.collection.delete_many({}) flowstring = "( pytesta; pytestb ) ; ( pytestc; ( pytestd || pyteste ) ) || pytestf" parse_string_to_workflow(flowstring, "test") nodes = self.db.list_nodes() names = [node.name for node in nodes] for node in nodes: print(node) for flow_string_name in [ "pytesta", "pytestb", "pytestc", "pytestd", "pyteste", "pytestf" ]: assert flow_string_name in names node_a = [node for node in nodes if node.name == "pytesta"][0] assert len(node_a.dependencies) == 0 node_f = [node for node in nodes if node.name == "pytestf"][0] assert len(node_f.dependencies) == 0 node_b = [node for node in nodes if node.name == "pytestb"][0] assert len(node_b.dependencies) == 1 assert "pytesta" in node_b.dependencies
def save_result_to_db(self, nodeName, result): print("saving result to", self.flowname, result) db = FlowDatabase(self.flowname, True) db.add_node_result(nodeName, result)
def setup(self): self.db = FlowDatabase("test") self.db.collection.delete_many({}) self.db.add_node({"name": "a", "dependencies": []}) self.db.start_flow() self.flow = SampleFlow("test-flow.py")
def do_flow(self, args, arguments): """ :: Usage: flow list [--flow=NAME] [--output=FORMAT] flow add [--flowname=FLOWNAME] --flowfile=FILENAME flow run [--flowname=FLOWNAME] [--flowfile=FILENAME] flow node add NODENAME [--flowname=FLOWNAME] flow edge add FROM TO [--flowname=FLOWNAME] flow node delete NODENAME flow edge delete FROM TO flow edge invert FROM TO flow visualize start flow visualize stop flow refresh This command manages and executes workflows The default workflow is just named "workflow" but you can specify multiple Arguments: NAME the name of the workflow FILENAME a file name NODENAME the name of the node FROM the edge source (a node name) TO the edge destination (a node name) NODE the name of the node Options: --flow=NAME the name or the flow --file specify the file --log specify the log file --flowname=FLOWNAME the name or the workflow --output=OUTPUT the output format [default: table] """ arguments.FLOWNAME = arguments["--flowname"] or "workflow" arguments.FLOWFILE = arguments[ "--flowfile"] or f"{arguments.FLOWNAME}-flow.py" arguments.output = arguments["--output"] VERBOSE(arguments, verbose=0) if arguments["add"] and arguments.edge: db = FlowDatabase(arguments.FLOWNAME) db.add_edge(arguments.FROM, arguments.TO) elif arguments["add"]: print("adding a node") if arguments.NODENAME: node = Node(arguments.NODENAME) node.workflow = arguments.FLOWNAME try: db = FlowDatabase(arguments.FLOWNAME) db.add_node(node.toDict()) except Exception as e: print("error executing", e) elif arguments["--flowfile"]: filename = arguments["--flowfile"] print("load from file", filename) parse_yaml_to_workflow(filename) elif arguments["list"]: arguments.flow = arguments["--flow"] or "workflow" db = CmDatabase() name = arguments["--flow"] if name is not None: flows = [name] else: candidates = db.collections() flows = [] for flow in candidates: if flow.endswith("-flow"): flows.append(flow) entries = [] for name in flows: nodes = db.find(collection=f"{name}-flow") for node in nodes: node["dependencies"] = ", ".join(node["dependencies"]) entries = entries + nodes order = ["name", "workflow", "dependencies", "cm.modified"] header = ["Name", "Workflow", "Dependencies", "Modified"] print( Printer.flatwrite(nodes, order=order, header=header, output=arguments.output)) elif arguments._run: runner = FlowRunner(arguments.FLOWNAME, arguments.FLOWFILE) runner.start_flow() elif arguments.visualize: if arguments["start"]: manager.start() print( "The visualization servive started at http://127.0.0.1:8080/flow/" ) elif arguments["stop"]: manager.stop() elif arguments["delete"] and arguments.edge: db = FlowDatabase(arguments.FLOWNAME) db.remove_edge(arguments.FROM, arguments.TO) elif arguments["delete"] and arguments.node: db = FlowDatabase(arguments.FLOWNAME) db.remove_node(arguments.NODENAME) elif arguments["invert"] and arguments.edge: db = FlowDatabase(arguments.FLOWNAME) db.remove_edge(arguments.TO, arguments.FROM) db.add_edge(arguments.FROM, arguments.TO) elif arguments.refresh: raise NotImplementedError