def main(dataarg, workflow, initfiles, controller, ctrlopt, toplevel, verbosity, loginterval, updateinterval, schemadir, backend, dataopt, wflowopt, backendopt, strategy, modelsetup, modelopt, metadir, parameter, validate, visualize, cache, plugins, accept_metadir): logging.basicConfig(level=getattr(logging, verbosity), format=LOGFORMAT) from packtivity.plugins import enable_plugins if plugins: enable_plugins(plugins.split(',')) initdata = utils.getinit_data(initfiles, parameter) dataopts = utils.options_from_eqdelimstring(dataopt) backendopts = utils.options_from_eqdelimstring(backendopt) modelopts = utils.options_from_eqdelimstring(modelopt) ctrlopts = utils.options_from_eqdelimstring(ctrlopt) wflowopts = utils.options_from_eqdelimstring(wflowopt) backend = utils.setupbackend_fromstring(backend, backendopts) rc = RC_FAILED try: steering_api.run_workflow( workflow=workflow, toplevel=toplevel, validate=validate, schemadir=schemadir, initdata=initdata, wflowopts=wflowopts, controller=controller, ctrlopts=ctrlopts, backend=backend, cache=cache, dataarg=dataarg, dataopts=dataopts, metadir=metadir, accept_metadir=accept_metadir, modelsetup=modelsetup, modelopts=modelopts, updateinterval=updateinterval, loginterval=loginterval, visualize=visualize, strategy=strategy, ) rc = RC_SUCCEEDED except: log.exception('workflow failed') if rc != RC_SUCCEEDED: exc = click.exceptions.ClickException( click.style("Workflow failed", fg='red')) exc.exit_code = rc raise exc
def add( offset, groupname, dataarg, dataopts, workflow, toplevel, parameter, metadir, accept_metadir, controller, ctrlopt, modelsetup, modelopt, backend, local, verbosity, ): parameter = utils.options_from_eqdelimstring(parameter) dataarg = utils.coerce_data_arg(dataarg) dataopts = utils.options_from_eqdelimstring(dataopts) handle_common_options(verbosity) ys = handle_connection_options( metadir, accept_metadir, controller, ctrlopt, modelsetup, modelopt, backend, local, ) controller = ys.controller stages = [] if parameter: inspec = utils.init_stage_spec(parameter, False, [], "init") stages.append(inspec) stages = (stages + workflow_loader.workflow( workflow, toplevel=toplevel, validate=True)["stages"]) controller.add_rules(stages, dataarg, offset=offset, groupname=groupname, dataopts=dataopts)
def handle_connection_options(metadir, accept_metadir, controller, ctrlopt, modelsetup, modelopt, backend, local): ctrlopts = utils.options_from_eqdelimstring(ctrlopt) modelopts = utils.options_from_eqdelimstring(modelopt) if modelsetup == 'filebacked': modelsetup = 'filebacked:{}/yadage_state.json'.format(metadir) ys = manualutils.connect(metadir=metadir, accept_metadir=accept_metadir, ctrlstring=controller, ctrlopts=ctrlopts, modelsetup=modelsetup, modelopts=modelopts, backendstring=backend if local else None) ys.controller.sync_backend() return ys
def handle_connection_options(metadir, accept_metadir, controller, ctrlopt, modelsetup, modelopt, backend, local): ctrlopts = utils.options_from_eqdelimstring(ctrlopt) modelopts = utils.options_from_eqdelimstring(modelopt) if modelsetup == 'filebacked': modelsetup = 'filebacked:{}/yadage_state.json'.format(metadir) ys = manualutils.connect( metadir = metadir, accept_metadir = accept_metadir, ctrlstring = controller, ctrlopts = ctrlopts, modelsetup = modelsetup, modelopts = modelopts, backendstring = backend if local else None ) ys.controller.sync_backend() return ys
def serve(statetype, backend, modelopt, ip ,port): from yadage.utils import options_from_eqdelimstring logging.basicConfig(level = logging.INFO) stateopts = options_from_eqdelimstring(modelopt) backendopts = {} init_app(app, statetype, stateopts, backend, backendopts) sio.start_background_task(lambda: watch_state(statetype)) pywsgi.WSGIServer(('0.0.0.0', port), app, handler_class = WebSocketHandler, ).serve_forever()
def add(offset, groupname, dataarg, dataopts, workflow, toplevel, parameter, metadir, accept_metadir, controller, ctrlopt, modelsetup, modelopt, backend, local, verbosity ): parameter = utils.options_from_eqdelimstring(parameter) dataopts = utils.options_from_eqdelimstring(dataopts) handle_common_options(verbosity) ys = handle_connection_options(metadir, accept_metadir, controller, ctrlopt, modelsetup, modelopt, backend, local) controller = ys.controller stages = [] if parameter: inspec = utils.init_stage_spec(parameter, False, [], 'init') stages.append(inspec) stages = stages + workflow_loader.workflow( workflow, toplevel=toplevel, validate=True )['stages'] controller.add_rules(stages, dataarg, offset = offset, groupname = groupname, dataopts = dataopts)
def init(workdir, workflow, initfiles, modelsetup, dataopt, metadir, toplevel, parameter): initdata = utils.getinit_data(initfiles, parameter) dataopts = utils.options_from_eqdelimstring(dataopt) ys = YadageSteering.create( dataarg=workdir, dataopts=dataopts, workflow=workflow, toplevel=toplevel, initdata=initdata, metadir=metadir, modelsetup=modelsetup, ) assert ys
def init(dataarg, workflow, initfiles, modelsetup, dataopt, metadir, toplevel, parameter): if os.path.exists('input.yml') and not initfiles: initfiles = ('input.yml',) initdata = utils.getinit_data(initfiles, parameter) dataopts = utils.options_from_eqdelimstring(dataopt) ys = YadageSteering.create( dataarg = dataarg, dataopts = dataopts, workflow = workflow, toplevel = toplevel, initdata = initdata, metadir = metadir, modelsetup = modelsetup, ) assert ys
def init(dataarg, workflow, initfiles, modelsetup, dataopt, metadir, toplevel, parameter): if os.path.exists("input.yml") and not initfiles: initfiles = ("input.yml", ) initdata = utils.getinit_data(initfiles, parameter) dataarg = utils.coerce_data_arg(dataarg) dataopts = utils.options_from_eqdelimstring(dataopt) ys = YadageSteering.create( dataarg=dataarg, dataopts=dataopts, workflow=workflow, toplevel=toplevel, initdata=initdata, metadir=metadir, modelsetup=modelsetup, ) assert ys
def step( track, strategy, nsteps, update_interval, metadir, accept_metadir, controller, ctrlopt, modelsetup, modelopt, backend, local, strategyopt, verbosity, ): handle_common_options(verbosity) ys = handle_connection_options( metadir, accept_metadir, controller, ctrlopt, modelsetup, modelopt, backend, local, ) strategyopts = utils.options_from_eqdelimstring(strategyopt) execute_steering( ys, updateinterval=update_interval, default_trackers=track, strategy=strategy, strategyopts=strategyopts, )
def main( dataarg, workflow, initfiles, controller, ctrlopt, toplevel, verbosity, loginterval, updateinterval, schemadir, backend, dataopt, wflowopt, backendopt, strategy, strategyopt, modelsetup, modelopt, metadir, parameter, validate, visualize, cache, plugins, accept_metadir, ): if os.path.exists("input.yml") and not initfiles: initfiles = ("input.yml", ) logging.basicConfig(level=getattr(logging, verbosity), format=LOGFORMAT) from packtivity.plugins import enable_plugins if plugins: enable_plugins(plugins.split(",")) initdata = utils.getinit_data(initfiles, parameter) dataarg = utils.coerce_data_arg(dataarg) dataopts = utils.options_from_eqdelimstring(dataopt) backendopts = utils.options_from_eqdelimstring(backendopt) modelopts = utils.options_from_eqdelimstring(modelopt) ctrlopts = utils.options_from_eqdelimstring(ctrlopt) wflowopts = utils.options_from_eqdelimstring(wflowopt) strategyopts = utils.options_from_eqdelimstring(strategyopt) backend = utils.setupbackend_fromstring(backend, backendopts) rc = RC_FAILED try: steering_api.run_workflow( workflow=workflow, toplevel=toplevel, validate=validate, schemadir=schemadir, initdata=initdata, wflowopts=wflowopts, controller=controller, ctrlopts=ctrlopts, backend=backend, cache=cache, dataarg=dataarg, dataopts=dataopts, metadir=metadir, accept_metadir=accept_metadir, modelsetup=modelsetup, modelopts=modelopts, updateinterval=updateinterval, loginterval=loginterval, visualize=visualize, strategy=strategy, strategyopts=strategyopts, ) rc = RC_SUCCEEDED except: if rc != RC_SUCCEEDED: if logging.root.level < logging.INFO: log.exception("workflow failed") exc = click.exceptions.ClickException( click.style("Workflow failed: {}".format(sys.exc_info()[1]), fg="red")) exc.exit_code = rc raise exc