def main(dataarg, workflow, initfiles, controller, ctrlopt, toplevel,
verbosity, loginterval, updateinterval, schemadir, backend, dataopt,
wflowopt, backendopt, strategy, modelsetup, modelopt, metadir,
parameter, validate, visualize, cache, plugins, accept_metadir):
logging.basicConfig(level=getattr(logging, verbosity), format=LOGFORMAT)
from packtivity.plugins import enable_plugins
if plugins:
enable_plugins(plugins.split(','))
initdata = utils.getinit_data(initfiles, parameter)
dataopts = utils.options_from_eqdelimstring(dataopt)
backendopts = utils.options_from_eqdelimstring(backendopt)
modelopts = utils.options_from_eqdelimstring(modelopt)
ctrlopts = utils.options_from_eqdelimstring(ctrlopt)
wflowopts = utils.options_from_eqdelimstring(wflowopt)
backend = utils.setupbackend_fromstring(backend, backendopts)
rc = RC_FAILED
try:
steering_api.run_workflow(
workflow=workflow,
toplevel=toplevel,
validate=validate,
schemadir=schemadir,
initdata=initdata,
wflowopts=wflowopts,
controller=controller,
ctrlopts=ctrlopts,
backend=backend,
cache=cache,
dataarg=dataarg,
dataopts=dataopts,
metadir=metadir,
accept_metadir=accept_metadir,
modelsetup=modelsetup,
modelopts=modelopts,
updateinterval=updateinterval,
loginterval=loginterval,
visualize=visualize,
strategy=strategy,
)
rc = RC_SUCCEEDED
except:
log.exception('workflow failed')
if rc != RC_SUCCEEDED:
exc = click.exceptions.ClickException(
click.style("Workflow failed", fg='red'))
exc.exit_code = rc
raise exc
def add(
offset,
groupname,
dataarg,
dataopts,
workflow,
toplevel,
parameter,
metadir,
accept_metadir,
controller,
ctrlopt,
modelsetup,
modelopt,
backend,
local,
verbosity,
):
parameter = utils.options_from_eqdelimstring(parameter)
dataarg = utils.coerce_data_arg(dataarg)
dataopts = utils.options_from_eqdelimstring(dataopts)
handle_common_options(verbosity)
ys = handle_connection_options(
metadir,
accept_metadir,
controller,
ctrlopt,
modelsetup,
modelopt,
backend,
local,
)
controller = ys.controller
stages = []
if parameter:
inspec = utils.init_stage_spec(parameter, False, [], "init")
stages.append(inspec)
stages = (stages + workflow_loader.workflow(
workflow, toplevel=toplevel, validate=True)["stages"])
controller.add_rules(stages,
dataarg,
offset=offset,
groupname=groupname,
dataopts=dataopts)
def handle_connection_options(metadir, accept_metadir, controller, ctrlopt,
modelsetup, modelopt, backend, local):
ctrlopts = utils.options_from_eqdelimstring(ctrlopt)
modelopts = utils.options_from_eqdelimstring(modelopt)
if modelsetup == 'filebacked':
modelsetup = 'filebacked:{}/yadage_state.json'.format(metadir)
ys = manualutils.connect(metadir=metadir,
accept_metadir=accept_metadir,
ctrlstring=controller,
ctrlopts=ctrlopts,
modelsetup=modelsetup,
modelopts=modelopts,
backendstring=backend if local else None)
ys.controller.sync_backend()
return ys
def handle_connection_options(metadir, accept_metadir, controller, ctrlopt, modelsetup, modelopt, backend, local):
ctrlopts = utils.options_from_eqdelimstring(ctrlopt)
modelopts = utils.options_from_eqdelimstring(modelopt)
if modelsetup == 'filebacked':
modelsetup = 'filebacked:{}/yadage_state.json'.format(metadir)
ys = manualutils.connect(
metadir = metadir,
accept_metadir = accept_metadir,
ctrlstring = controller,
ctrlopts = ctrlopts,
modelsetup = modelsetup,
modelopts = modelopts,
backendstring = backend if local else None
)
ys.controller.sync_backend()
return ys
def serve(statetype, backend, modelopt, ip ,port):
from yadage.utils import options_from_eqdelimstring
logging.basicConfig(level = logging.INFO)
stateopts = options_from_eqdelimstring(modelopt)
backendopts = {}
init_app(app, statetype, stateopts, backend, backendopts)
sio.start_background_task(lambda: watch_state(statetype))
pywsgi.WSGIServer(('0.0.0.0', port), app,
handler_class = WebSocketHandler,
).serve_forever()
def add(offset, groupname, dataarg, dataopts, workflow, toplevel, parameter,
metadir, accept_metadir, controller, ctrlopt, modelsetup, modelopt, backend, local,
verbosity
):
parameter = utils.options_from_eqdelimstring(parameter)
dataopts = utils.options_from_eqdelimstring(dataopts)
handle_common_options(verbosity)
ys = handle_connection_options(metadir, accept_metadir, controller, ctrlopt, modelsetup, modelopt, backend, local)
controller = ys.controller
stages = []
if parameter:
inspec = utils.init_stage_spec(parameter, False, [], 'init')
stages.append(inspec)
stages = stages + workflow_loader.workflow(
workflow,
toplevel=toplevel,
validate=True
)['stages']
controller.add_rules(stages, dataarg, offset = offset, groupname = groupname, dataopts = dataopts)
def init(workdir, workflow, initfiles, modelsetup, dataopt, metadir, toplevel,
parameter):
initdata = utils.getinit_data(initfiles, parameter)
dataopts = utils.options_from_eqdelimstring(dataopt)
ys = YadageSteering.create(
dataarg=workdir,
dataopts=dataopts,
workflow=workflow,
toplevel=toplevel,
initdata=initdata,
metadir=metadir,
modelsetup=modelsetup,
)
assert ys
def init(dataarg, workflow, initfiles, modelsetup, dataopt, metadir, toplevel, parameter):
if os.path.exists('input.yml') and not initfiles:
initfiles = ('input.yml',)
initdata = utils.getinit_data(initfiles, parameter)
dataopts = utils.options_from_eqdelimstring(dataopt)
ys = YadageSteering.create(
dataarg = dataarg,
dataopts = dataopts,
workflow = workflow,
toplevel = toplevel,
initdata = initdata,
metadir = metadir,
modelsetup = modelsetup,
)
assert ys
def init(dataarg, workflow, initfiles, modelsetup, dataopt, metadir, toplevel,
parameter):
if os.path.exists("input.yml") and not initfiles:
initfiles = ("input.yml", )
initdata = utils.getinit_data(initfiles, parameter)
dataarg = utils.coerce_data_arg(dataarg)
dataopts = utils.options_from_eqdelimstring(dataopt)
ys = YadageSteering.create(
dataarg=dataarg,
dataopts=dataopts,
workflow=workflow,
toplevel=toplevel,
initdata=initdata,
metadir=metadir,
modelsetup=modelsetup,
)
assert ys
def step(
track,
strategy,
nsteps,
update_interval,
metadir,
accept_metadir,
controller,
ctrlopt,
modelsetup,
modelopt,
backend,
local,
strategyopt,
verbosity,
):
handle_common_options(verbosity)
ys = handle_connection_options(
metadir,
accept_metadir,
controller,
ctrlopt,
modelsetup,
modelopt,
backend,
local,
)
strategyopts = utils.options_from_eqdelimstring(strategyopt)
execute_steering(
ys,
updateinterval=update_interval,
default_trackers=track,
strategy=strategy,
strategyopts=strategyopts,
)
def main(
dataarg,
workflow,
initfiles,
controller,
ctrlopt,
toplevel,
verbosity,
loginterval,
updateinterval,
schemadir,
backend,
dataopt,
wflowopt,
backendopt,
strategy,
strategyopt,
modelsetup,
modelopt,
metadir,
parameter,
validate,
visualize,
cache,
plugins,
accept_metadir,
):
if os.path.exists("input.yml") and not initfiles:
initfiles = ("input.yml", )
logging.basicConfig(level=getattr(logging, verbosity), format=LOGFORMAT)
from packtivity.plugins import enable_plugins
if plugins:
enable_plugins(plugins.split(","))
initdata = utils.getinit_data(initfiles, parameter)
dataarg = utils.coerce_data_arg(dataarg)
dataopts = utils.options_from_eqdelimstring(dataopt)
backendopts = utils.options_from_eqdelimstring(backendopt)
modelopts = utils.options_from_eqdelimstring(modelopt)
ctrlopts = utils.options_from_eqdelimstring(ctrlopt)
wflowopts = utils.options_from_eqdelimstring(wflowopt)
strategyopts = utils.options_from_eqdelimstring(strategyopt)
backend = utils.setupbackend_fromstring(backend, backendopts)
rc = RC_FAILED
try:
steering_api.run_workflow(
workflow=workflow,
toplevel=toplevel,
validate=validate,
schemadir=schemadir,
initdata=initdata,
wflowopts=wflowopts,
controller=controller,
ctrlopts=ctrlopts,
backend=backend,
cache=cache,
dataarg=dataarg,
dataopts=dataopts,
metadir=metadir,
accept_metadir=accept_metadir,
modelsetup=modelsetup,
modelopts=modelopts,
updateinterval=updateinterval,
loginterval=loginterval,
visualize=visualize,
strategy=strategy,
strategyopts=strategyopts,
)
rc = RC_SUCCEEDED
except:
if rc != RC_SUCCEEDED:
if logging.root.level < logging.INFO:
log.exception("workflow failed")
exc = click.exceptions.ClickException(
click.style("Workflow failed: {}".format(sys.exc_info()[1]),
fg="red"))
exc.exit_code = rc
raise exc