def yadage_load(workflow_file, toplevel=".", **kwargs):
"""Validate and return yadage workflow specification.
:param workflow_file: A specification file compliant with
`yadage` workflow specification.
:type workflow_file: string
:param toplevel: URL/path for the workflow file
:type toplevel: string
:returns: A dictionary which represents the valid `yadage` workflow.
"""
schema_name = "yadage/workflow-schema"
schemadir = None
specopts = {
"toplevel": toplevel,
"schema_name": schema_name,
"schemadir": schemadir,
"load_as_ref": False,
}
validopts = {
"schema_name": schema_name,
"schemadir": schemadir,
}
try:
yadageschemas.load(spec=workflow_file,
specopts=specopts,
validopts=validopts,
validate=True)
except ValidationError as e:
e.message = str(e)
raise e
def load_packtivity(spec,
toplevel=os.getcwd(),
schemasource=yadageschemas.schemadir,
validate=True):
# in case that spec is a json reference string, we will treat it as such
# if it's just a filename, this should not affect it...
spec, specopts = (
spec,
{
"toplevel": toplevel,
"schemadir": schemasource,
"schema_name": "packtivity/packtivity-schema",
"load_as_ref": True,
},
)
validopts = {
"schemadir": schemasource,
"schema_name": "packtivity/packtivity-schema",
}
spec = yadageschemas.load(spec,
specopts,
validate=validate,
validopts=validopts)
return spec
def workflow(source,
toplevel,
schema_name='yadage/workflow-schema',
schemadir=None,
validate=True):
'''
load (and validate) workflow from source
param source: URI fragment to the source
param toplevel: base URI to resolve references from
schemadir
'''
dialect, spec = ('raw_with_defaults',
source) if ':' not in source else source.split(':', 1)
specopts = {
'toplevel': toplevel,
'schema_name': schema_name,
'schemadir': schemadir,
'load_as_ref': False
}
validopts = {
'schema_name': schema_name,
'schemadir': schemadir,
}
data = yadageschemas.load(spec,
specopts,
validate=True,
validopts=validopts,
dialect=dialect)
return data
def yadage_load(workflow_file, toplevel='.'):
"""Validate and return yadage workflow specification.
:param workflow_file: A specification file compliant with
`yadage` workflow specification.
:type workflow_file: string
:param toplevel: URL/path for the workflow file
:type toplevel: string
:returns: A dictionary which represents the valid `yadage` workflow.
"""
schema_name = 'yadage/workflow-schema'
schemadir = None
specopts = {
'toplevel': toplevel,
'schema_name': schema_name,
'schemadir': schemadir,
'load_as_ref': False,
}
validopts = {
'schema_name': schema_name,
'schemadir': schemadir,
}
return yadageschemas.load(spec=workflow_file, specopts=specopts,
validopts=validopts, validate=True)
def workflow(
source,
toplevel,
schema_name="yadage/workflow-schema",
schemadir=None,
validate=True,
):
"""
load (and validate) workflow from source
param source: URI fragment to the source
param toplevel: base URI to resolve references from
schemadir
"""
dialect, spec = (
("raw_with_defaults", source) if ":" not in source else source.split(":", 1)
)
specopts = {
"toplevel": toplevel,
"schema_name": schema_name,
"schemadir": schemadir,
"load_as_ref": False,
}
validopts = {"schema_name": schema_name, "schemadir": schemadir}
data = yadageschemas.load(
spec, specopts, validate=True, validopts=validopts, dialect=dialect
)
return data
def yadage_load(workflow_file, toplevel='.'):
"""Validate and return yadage workflow specification.
:param workflow_file: A specification file compliant with
`yadage` workflow specification.
:returns: A dictionary which represents the valid `yadage` workflow.
"""
return yadageschemas.load(workflow_file, toplevel=toplevel,
schema_name='yadage/workflow-schema',
schemadir=None, validate=True)
def load_pack(spec, toplevel, schemasource, validate):
#in case that spec is a json reference string, we will treat it as such
#if it's just a filename, this should not affect it...
spec = yadageschemas.load({'$ref': spec},
toplevel,
'packtivity/packtivity-schema',
schemadir=schemasource,
validate=validate,
initialload=False)
return spec
def validate_entry(data):
toplevel = data["spec"]["toplevel"]
workflow = data["spec"]["workflow"]
try:
yadageschemas.load(
workflow,
specopts={
"toplevel": toplevel,
"load_as_ref": False,
"schema_name": "yadage/workflow-schema",
"schemadir": yadageschemas.schemadir,
},
validopts={
"schemadir": yadageschemas.schemadir,
"schema_name": "yadage/workflow-schema",
},
)
return True
except:
pass
return False
def test_load_jsondump():
spec, specopts = 'workflow.yml', {
'toplevel': 'tests/testspecs/local-helloworld-jsonref',
'schema_name': 'yadage/workflow-schema',
'schemadir': yadageschemas.schemadir,
'load_as_ref': False,
}
validopts = {
'schema_name': 'yadage/workflow-schema',
'schemadir': yadageschemas.schemadir,
}
data = yadageschemas.load(spec,specopts,validopts = validopts)
json.dumps(data)
def test_load_default_unwrap():
spec, specopts = 'workflow.yml', {
'toplevel': 'tests/testspecs/mapreduce',
'schema_name': 'yadage/workflow-schema',
'schemadir': yadageschemas.schemadir,
'load_as_ref': False,
}
validopts = {
'schema_name': 'yadage/workflow-schema',
'schemadir': yadageschemas.schemadir,
}
data = yadageschemas.load(spec,specopts,validopts = validopts)
json.dumps(data)
def main(workflow, toplevel, schemadir, stdout, dialect,plugins,load_as_ref):
if plugins:
for p in plugins.split(','):
import importlib
importlib.import_module(p)
rc = 3
if not toplevel:
toplevel = os.getcwd()
if not schemadir:
schemadir = yadageschemas.schemadir
try:
spec, specopts = workflow, {
'toplevel': toplevel,
'load_as_ref': load_as_ref,
'schemadir': schemadir,
'schema_name': 'yadage/workflow-schema'
}
validopts = {
'schemadir': schemadir,
'schema_name': 'yadage/workflow-schema'
}
data = yadageschemas.load(spec, specopts, validate = True, validopts = validopts, dialect = dialect)
if stdout:
print(json.dumps(data, cls=WithJsonRefEncoder))
else:
click.secho('workflow validates against schema', fg='green')
rc = 0
except jsonschema.exceptions.ValidationError as e:
rc = 1
log.exception(e)
click.secho('workflow does not validate against schema', fg='red')
except:
rc = 2
log.exception('hm')
click.secho('this is not even wrong (non-ValidationError exception)', fg='red')
if rc != 0:
exc = click.exceptions.ClickException(
click.style("validation failed", fg='red'))
exc.exit_code = rc
raise exc
def load_packtivity(spec,toplevel = os.getcwd() ,schemasource = yadageschemas.schemadir, validate = True):
#in case that spec is a json reference string, we will treat it as such
#if it's just a filename, this should not affect it...
spec, specopts = spec, {
'toplevel': toplevel,
'schemadir': schemasource,
'schema_name': 'packtivity/packtivity-schema',
'load_as_ref': True
}
validopts = {
'schemadir': schemasource,
'schema_name': 'packtivity/packtivity-schema',
}
spec = yadageschemas.load(
spec, specopts,
validate = validate,
validopts = validopts
)
return spec
def workflow(source, toplevel, schema_name='yadage/workflow-schema', schemadir=None, validate=True):
'''
load (and validate) workflow from source
param source: URI fragment to the source
param toplevel: base URI to resolve references from
schemadir
'''
dialect, spec = ('raw_with_defaults', source) if ':' not in source else source.split(':',1)
specopts = {
'toplevel': toplevel,
'schema_name': schema_name,
'schemadir': schemadir,
'load_as_ref': False
}
validopts = {
'schema_name': schema_name,
'schemadir': schemadir,
}
data = yadageschemas.load(
spec, specopts, validate = True, validopts = validopts, dialect = dialect
)
return data
def run_yadage_workflow(
workflow_uuid,
workflow_workspace,
workflow_file,
workflow_parameters=None,
operational_options={},
):
"""Run a ``yadage`` workflow."""
log.info("getting socket..")
workflow_workspace = "{0}/{1}".format(SHARED_VOLUME_PATH,
workflow_workspace)
# use some shared object between tasks.
os.environ["workflow_uuid"] = workflow_uuid
os.environ["workflow_workspace"] = workflow_workspace
os.umask(REANA_WORKFLOW_UMASK)
cap_backend = setupbackend_fromstring("fromenv")
workflow_file_abs_path = os.path.join(workflow_workspace, workflow_file)
publisher = REANAWorkflowStatusPublisher()
try:
if not os.path.exists(workflow_file_abs_path):
message = f"Workflow file {workflow_file} does not exist"
raise Exception(message)
else:
schema_name = "yadage/workflow-schema"
schemadir = None
specopts = {
"toplevel": operational_options["toplevel"],
"schema_name": schema_name,
"schemadir": schemadir,
"load_as_ref": False,
}
validopts = {
"schema_name": schema_name,
"schemadir": schemadir,
}
workflow_json = yadageschemas.load(
spec=workflow_file,
specopts=specopts,
validopts=validopts,
validate=True,
)
workflow_kwargs = dict(workflow_json=workflow_json)
dataopts = {"initdir": operational_options["initdir"]}
check_connection_to_job_controller()
with steering_ctx(
dataarg=workflow_workspace,
dataopts=dataopts,
initdata=workflow_parameters if workflow_parameters else {},
visualize=True,
updateinterval=5,
loginterval=5,
backend=cap_backend,
**workflow_kwargs,
) as ys:
log.info("running workflow on context: {0}".format(locals()))
publisher.publish_workflow_status(workflow_uuid, 1)
ys.adage_argument(
additional_trackers=[REANATracker(identifier=workflow_uuid)])
publisher.publish_workflow_status(workflow_uuid, 2)
log.info("Workflow {workflow_uuid} finished. Files available "
"at {workflow_workspace}.".format(
workflow_uuid=workflow_uuid,
workflow_workspace=workflow_workspace))
except Exception as e:
log.error("Workflow failed: {0}".format(e), exc_info=True)
if publisher:
publisher.publish_workflow_status(
workflow_uuid, 3, logs="workflow failed: {0}".format(e))
else:
log.error(
"Workflow {workflow_uuid} failed but status "
"could not be published.".format(workflow_uuid=workflow_uuid))
def run_yadage_workflow(workflow_uuid,
workflow_workspace,
workflow_json=None,
workflow_file=None,
workflow_parameters=None):
"""Run a ``yadage`` workflow."""
log.info('getting socket..')
workflow_workspace = '{0}/{1}'.format(SHARED_VOLUME_PATH,
workflow_workspace)
# use some shared object between tasks.
os.environ["workflow_uuid"] = workflow_uuid
os.environ["workflow_workspace"] = workflow_workspace
os.umask(REANA_WORKFLOW_UMASK)
cap_backend = setupbackend_fromstring('fromenv')
toplevel = os.getcwd()
workflow = None
if workflow_json:
# When `yadage` is launched using an already validated workflow file.
workflow_kwargs = dict(workflow_json=workflow_json)
elif workflow:
# When `yadage` resolves the workflow file from a remote repository:
# i.e. github:reanahub/reana-demo-root6-roofit/workflow.yaml
workflow_kwargs = dict(workflow=workflow, toplevel=toplevel)
elif workflow_file:
workflow_file_abs_path = os.path.join(workflow_workspace,
workflow_file)
if os.path.exists(workflow_file_abs_path):
schema_name = 'yadage/workflow-schema'
schemadir = None
specopts = {
'toplevel': workflow_workspace,
'schema_name': schema_name,
'schemadir': schemadir,
'load_as_ref': False,
}
validopts = {
'schema_name': schema_name,
'schemadir': schemadir,
}
workflow_json = yadageschemas.load(spec=workflow_file,
specopts=specopts,
validopts=validopts,
validate=True)
workflow_kwargs = dict(workflow_json=workflow_json)
dataopts = {'initdir': workflow_workspace}
try:
check_connection_to_job_controller()
publisher = REANAWorkflowStatusPublisher()
with steering_ctx(
dataarg=workflow_workspace,
dataopts=dataopts,
initdata=workflow_parameters if workflow_parameters else {},
visualize=True,
updateinterval=5,
loginterval=5,
backend=cap_backend,
**workflow_kwargs) as ys:
log.info('running workflow on context: {0}'.format(locals()))
publisher.publish_workflow_status(workflow_uuid, 1)
ys.adage_argument(
additional_trackers=[REANATracker(identifier=workflow_uuid)])
publisher.publish_workflow_status(workflow_uuid, 2)
log.info('Workflow {workflow_uuid} finished. Files available '
'at {workflow_workspace}.'.format(
workflow_uuid=workflow_uuid,
workflow_workspace=workflow_workspace))
except Exception as e:
log.info('workflow failed: {0}'.format(e), exc_info=True)
if publisher:
publisher.publish_workflow_status(
workflow_uuid, 3, logs='workflow failed: {0}'.format(e))
else:
log.error(
'Workflow {workflow_uuid} failed but status '
'could not be published.'.format(workflow_uuid=workflow_uuid))
'/Users/lukas/Code/yadagedev/typedleafs/objectstore/public0002.dat',
'checksum': 'okcool'
},
'message': 'this is a message to the world:'
}
input_objects = []
inputdata = json.loads(json.dumps(inputdata),
cls=TypedLeafDecoder,
type_identifiers=['$type'],
type_loader=typeloader,
obj_store=input_objects)
print inputdata
raise RuntimeError('..ok..')
logging.basicConfig(level=logging.INFO)
pack = yadageschemas.load('exposing_transform.yml', os.getcwd(),
'packtivity/packtivity-schema')
proc = pack['process']
script = proc['script'].format(**inputdata)
env = pack['environment']
mounts = []
for obj in input_objects:
mounts.append('{}:{}'.format(obj.public_path, obj.local_path))
if obj.local_path.startswith('/outputs'):
open(obj.public_path, 'w').close() #touch file
dockercmd = 'docker run --rm -i '
for mount in mounts:
dockercmd += '-v {} '.format(mount)
dockercmd += '{}:{} {}'.format(env['image'], env['imagetag'],
