def json2workflow(json):
model = Workflow(spec={'x': 'y'})
model.uuid = json['uuid']
model.spec = json['spec']
model.inputs = json['inputs']
model.dependencies = json['dependencies']
model.topsort = json['topsort']
model.running = json['running']
model.failed = json['failed']
model.done = json['done']
return model
async def executeWorkflow(request):
""" Gather the workflow contents from the request. """
workflow_spec = request.json['workflow']
logger.debug(f"Received workflow execution request.")
""" Build an async executor passing the workflow and its arguments. """
executor = AsyncioExecutor(
workflow=Workflow(spec=workflow_spec, inputs=request.json['args']))
""" Execute the workflow coroutine asynchronously and return results when available. """
return json(await executor.execute())
def execute_remote (workflow="mq2.ros", host="localhost", port=8080, args={}, library_path=["."]):
""" Execute the workflow remotely via a web API. """
workflow_spec = Workflow.get_workflow (workflow, library_path)
return requests.post (
url = f"{host}:{port}/api/executeWorkflow",
json = {
"workflow" : workflow_spec,
"args" : args
})
def main ():
arg_parser = argparse.ArgumentParser(
description='Ros Workflow CLI',
formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=60))
arg_parser.add_argument('-a', '--api', help="Execute via API instead of locally.", action="store_true")
arg_parser.add_argument('-w', '--workflow', help="Workflow to execute.", default="mq2.ros")
arg_parser.add_argument('-s', '--server', help="Hostname of api server", default="localhost")
arg_parser.add_argument('-p', '--port', help="Port of the server", default="80")
arg_parser.add_argument('-i', '--arg', help="Add an argument expressed as key=val", action='append', default=[])
arg_parser.add_argument('-o', '--out', help="Output the workflow result graph to a file. Use 'stdout' to print to terminal.")
arg_parser.add_argument('-l', '--lib_path', help="A directory containing workflow modules.", action='append', default=["."])
arg_parser.add_argument('-n', '--ndex_id', help="Publish the graph to NDEx")
arg_parser.add_argument('--validate', help="Validate inputs and outputs", action="store_true")
args = arg_parser.parse_args ()
setup_logging ()
#start_task_queue ()
""" Parse input arguments. """
wf_args = { k : v for k, v in [ arg.split("=") for arg in args.arg ] }
response = None
if args.api:
""" Invoke a remote API endpoint. """
response = execute_rmote (workflow=args.workflow,
host=args.server,
port=args.port,
args=wf_args)
else:
""" Execute the workflow in process. """
executor = CeleryDAGExecutor (
spec=Workflow.get_workflow (workflow=args.workflow,
inputs=wf_args,
library_path=args.lib_path))
response = executor.execute ()
if args.ndex_id:
jsonpath_query = parse ("$.[*].result_list.[*].[*].result_graph")
graph = [ match.value for match in jsonpath_query.find (response) ]
logger.debug (f"{args.ndex_id} => {json.dumps(graph, indent=2)}")
ndex = NDEx ()
ndex.publish (args.ndex_id, graph)
if args.out:
if args.out == "stdout":
logger.debug (f"{graph_text}")
else:
with open(args.out, "w") as stream:
stream.write (json.dumps(response, indent=2))
def run(self, workflow, args={}, library_path=["."]):
""" Execute the workflow remotely via a web API. """
logger.debug(
f"execute remote: {workflow} libpath: {library_path} args: {args} at {self.url}"
)
""" Construct a workflow object, parse the workflow, and resolve imports - all locally. """
workflow = Workflow(spec=workflow,
inputs=args,
libpath=library_path,
local_db_connection=False)
""" Execute the workflow remotely and return both the workflow object and the response we got. """
return WorkflowResult(workflow=workflow,
result=requests.post(
url=f"{self.url}/api/executeWorkflow",
json={
"workflow": workflow.spec,
"args": args
}).json())
def run_local_flow(path, inputs):
workflow_path = os.path.join(os.path.dirname(__file__), path)
workflow = Workflow(spec=workflow_path, inputs=inputs)
execute_workflow(workflow)
def main():
arg_parser = argparse.ArgumentParser(
description='Ros Workflow CLI',
formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter(
prog, max_help_position=60))
arg_parser.add_argument('-a',
'--api',
help="URL of the remote Ros server to use.",
action="store_true")
arg_parser.add_argument('-w',
'--workflow',
help="Workflow to execute.",
default="workflow_one.ros")
arg_parser.add_argument('-s',
'--server',
help="Hostname of api server",
default="http://localhost:5002")
arg_parser.add_argument('-i',
'--arg',
help="Add an argument expressed as key=val",
action='append',
default=[])
arg_parser.add_argument(
'-o',
'--out',
help=
"Output the workflow result graph to a file. Use 'stdout' to print to terminal."
)
arg_parser.add_argument('-l',
'--libpath',
help="A directory containing workflow modules.",
action='append',
default=["."])
arg_parser.add_argument(
'-n',
'--ndex',
help=
"Name of the graph to publish to NDEx. Requires valid ~/.ndex credential file."
)
arg_parser.add_argument('--validate',
help="Validate inputs and outputs",
action="store_true")
args = arg_parser.parse_args()
LoggingUtil.setup_logging()
""" Parse input arguments. """
wf_args = {k: v for k, v in [arg.split("=") for arg in args.arg]}
response = None
if args.api:
""" Use the Ros client to run a workflow remotely. """
client = Client(url=args.server)
ros_result = client.run(workflow=args.workflow,
args=wf_args,
library_path=args.libpath)
response = ros_result.result
else:
""" Execute locally via python async. """
executor = AsyncioExecutor(workflow=Workflow.get_workflow(
workflow=args.workflow, inputs=wf_args, library_path=args.libpath))
tasks = [asyncio.ensure_future(executor.execute())]
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))
response = tasks[0].result()
if args.ndex:
""" Output to NDEx. """
jsonkit = JSONKit()
graph = jsonkit.select("$.[*][*].result_list.[*][*].result_graph",
response)
logger.debug(
f"Publishing result as NDEx graph({args.ndex})=> {json.dumps(graph, indent=2)}"
)
NDEx()._publish(args.ndex, graph)
if args.out:
""" Write to a file, possibly standard ouput. """
if args.out == "stdout":
print(f"{json.dumps(response, indent=2)}")
else:
with open(args.out, "w") as stream:
json.dump(response, stream, indent=2)