def yadage_load(workflow_file, toplevel=".", **kwargs): """Validate and return yadage workflow specification. :param workflow_file: A specification file compliant with `yadage` workflow specification. :type workflow_file: string :param toplevel: URL/path for the workflow file :type toplevel: string :returns: A dictionary which represents the valid `yadage` workflow. """ schema_name = "yadage/workflow-schema" schemadir = None specopts = { "toplevel": toplevel, "schema_name": schema_name, "schemadir": schemadir, "load_as_ref": False, } validopts = { "schema_name": schema_name, "schemadir": schemadir, } try: yadageschemas.load(spec=workflow_file, specopts=specopts, validopts=validopts, validate=True) except ValidationError as e: e.message = str(e) raise e
def load_packtivity(spec, toplevel=os.getcwd(), schemasource=yadageschemas.schemadir, validate=True): # in case that spec is a json reference string, we will treat it as such # if it's just a filename, this should not affect it... spec, specopts = ( spec, { "toplevel": toplevel, "schemadir": schemasource, "schema_name": "packtivity/packtivity-schema", "load_as_ref": True, }, ) validopts = { "schemadir": schemasource, "schema_name": "packtivity/packtivity-schema", } spec = yadageschemas.load(spec, specopts, validate=validate, validopts=validopts) return spec
def workflow(source, toplevel, schema_name='yadage/workflow-schema', schemadir=None, validate=True): ''' load (and validate) workflow from source param source: URI fragment to the source param toplevel: base URI to resolve references from schemadir ''' dialect, spec = ('raw_with_defaults', source) if ':' not in source else source.split(':', 1) specopts = { 'toplevel': toplevel, 'schema_name': schema_name, 'schemadir': schemadir, 'load_as_ref': False } validopts = { 'schema_name': schema_name, 'schemadir': schemadir, } data = yadageschemas.load(spec, specopts, validate=True, validopts=validopts, dialect=dialect) return data
def yadage_load(workflow_file, toplevel='.'): """Validate and return yadage workflow specification. :param workflow_file: A specification file compliant with `yadage` workflow specification. :type workflow_file: string :param toplevel: URL/path for the workflow file :type toplevel: string :returns: A dictionary which represents the valid `yadage` workflow. """ schema_name = 'yadage/workflow-schema' schemadir = None specopts = { 'toplevel': toplevel, 'schema_name': schema_name, 'schemadir': schemadir, 'load_as_ref': False, } validopts = { 'schema_name': schema_name, 'schemadir': schemadir, } return yadageschemas.load(spec=workflow_file, specopts=specopts, validopts=validopts, validate=True)
def workflow( source, toplevel, schema_name="yadage/workflow-schema", schemadir=None, validate=True, ): """ load (and validate) workflow from source param source: URI fragment to the source param toplevel: base URI to resolve references from schemadir """ dialect, spec = ( ("raw_with_defaults", source) if ":" not in source else source.split(":", 1) ) specopts = { "toplevel": toplevel, "schema_name": schema_name, "schemadir": schemadir, "load_as_ref": False, } validopts = {"schema_name": schema_name, "schemadir": schemadir} data = yadageschemas.load( spec, specopts, validate=True, validopts=validopts, dialect=dialect ) return data
def yadage_load(workflow_file, toplevel='.'): """Validate and return yadage workflow specification. :param workflow_file: A specification file compliant with `yadage` workflow specification. :returns: A dictionary which represents the valid `yadage` workflow. """ return yadageschemas.load(workflow_file, toplevel=toplevel, schema_name='yadage/workflow-schema', schemadir=None, validate=True)
def load_pack(spec, toplevel, schemasource, validate): #in case that spec is a json reference string, we will treat it as such #if it's just a filename, this should not affect it... spec = yadageschemas.load({'$ref': spec}, toplevel, 'packtivity/packtivity-schema', schemadir=schemasource, validate=validate, initialload=False) return spec
def validate_entry(data): toplevel = data["spec"]["toplevel"] workflow = data["spec"]["workflow"] try: yadageschemas.load( workflow, specopts={ "toplevel": toplevel, "load_as_ref": False, "schema_name": "yadage/workflow-schema", "schemadir": yadageschemas.schemadir, }, validopts={ "schemadir": yadageschemas.schemadir, "schema_name": "yadage/workflow-schema", }, ) return True except: pass return False
def test_load_jsondump(): spec, specopts = 'workflow.yml', { 'toplevel': 'tests/testspecs/local-helloworld-jsonref', 'schema_name': 'yadage/workflow-schema', 'schemadir': yadageschemas.schemadir, 'load_as_ref': False, } validopts = { 'schema_name': 'yadage/workflow-schema', 'schemadir': yadageschemas.schemadir, } data = yadageschemas.load(spec,specopts,validopts = validopts) json.dumps(data)
def test_load_default_unwrap(): spec, specopts = 'workflow.yml', { 'toplevel': 'tests/testspecs/mapreduce', 'schema_name': 'yadage/workflow-schema', 'schemadir': yadageschemas.schemadir, 'load_as_ref': False, } validopts = { 'schema_name': 'yadage/workflow-schema', 'schemadir': yadageschemas.schemadir, } data = yadageschemas.load(spec,specopts,validopts = validopts) json.dumps(data)
def main(workflow, toplevel, schemadir, stdout, dialect,plugins,load_as_ref): if plugins: for p in plugins.split(','): import importlib importlib.import_module(p) rc = 3 if not toplevel: toplevel = os.getcwd() if not schemadir: schemadir = yadageschemas.schemadir try: spec, specopts = workflow, { 'toplevel': toplevel, 'load_as_ref': load_as_ref, 'schemadir': schemadir, 'schema_name': 'yadage/workflow-schema' } validopts = { 'schemadir': schemadir, 'schema_name': 'yadage/workflow-schema' } data = yadageschemas.load(spec, specopts, validate = True, validopts = validopts, dialect = dialect) if stdout: print(json.dumps(data, cls=WithJsonRefEncoder)) else: click.secho('workflow validates against schema', fg='green') rc = 0 except jsonschema.exceptions.ValidationError as e: rc = 1 log.exception(e) click.secho('workflow does not validate against schema', fg='red') except: rc = 2 log.exception('hm') click.secho('this is not even wrong (non-ValidationError exception)', fg='red') if rc != 0: exc = click.exceptions.ClickException( click.style("validation failed", fg='red')) exc.exit_code = rc raise exc
def load_packtivity(spec,toplevel = os.getcwd() ,schemasource = yadageschemas.schemadir, validate = True): #in case that spec is a json reference string, we will treat it as such #if it's just a filename, this should not affect it... spec, specopts = spec, { 'toplevel': toplevel, 'schemadir': schemasource, 'schema_name': 'packtivity/packtivity-schema', 'load_as_ref': True } validopts = { 'schemadir': schemasource, 'schema_name': 'packtivity/packtivity-schema', } spec = yadageschemas.load( spec, specopts, validate = validate, validopts = validopts ) return spec
def workflow(source, toplevel, schema_name='yadage/workflow-schema', schemadir=None, validate=True): ''' load (and validate) workflow from source param source: URI fragment to the source param toplevel: base URI to resolve references from schemadir ''' dialect, spec = ('raw_with_defaults', source) if ':' not in source else source.split(':',1) specopts = { 'toplevel': toplevel, 'schema_name': schema_name, 'schemadir': schemadir, 'load_as_ref': False } validopts = { 'schema_name': schema_name, 'schemadir': schemadir, } data = yadageschemas.load( spec, specopts, validate = True, validopts = validopts, dialect = dialect ) return data
def run_yadage_workflow( workflow_uuid, workflow_workspace, workflow_file, workflow_parameters=None, operational_options={}, ): """Run a ``yadage`` workflow.""" log.info("getting socket..") workflow_workspace = "{0}/{1}".format(SHARED_VOLUME_PATH, workflow_workspace) # use some shared object between tasks. os.environ["workflow_uuid"] = workflow_uuid os.environ["workflow_workspace"] = workflow_workspace os.umask(REANA_WORKFLOW_UMASK) cap_backend = setupbackend_fromstring("fromenv") workflow_file_abs_path = os.path.join(workflow_workspace, workflow_file) publisher = REANAWorkflowStatusPublisher() try: if not os.path.exists(workflow_file_abs_path): message = f"Workflow file {workflow_file} does not exist" raise Exception(message) else: schema_name = "yadage/workflow-schema" schemadir = None specopts = { "toplevel": operational_options["toplevel"], "schema_name": schema_name, "schemadir": schemadir, "load_as_ref": False, } validopts = { "schema_name": schema_name, "schemadir": schemadir, } workflow_json = yadageschemas.load( spec=workflow_file, specopts=specopts, validopts=validopts, validate=True, ) workflow_kwargs = dict(workflow_json=workflow_json) dataopts = {"initdir": operational_options["initdir"]} check_connection_to_job_controller() with steering_ctx( dataarg=workflow_workspace, dataopts=dataopts, initdata=workflow_parameters if workflow_parameters else {}, visualize=True, updateinterval=5, loginterval=5, backend=cap_backend, **workflow_kwargs, ) as ys: log.info("running workflow on context: {0}".format(locals())) publisher.publish_workflow_status(workflow_uuid, 1) ys.adage_argument( additional_trackers=[REANATracker(identifier=workflow_uuid)]) publisher.publish_workflow_status(workflow_uuid, 2) log.info("Workflow {workflow_uuid} finished. Files available " "at {workflow_workspace}.".format( workflow_uuid=workflow_uuid, workflow_workspace=workflow_workspace)) except Exception as e: log.error("Workflow failed: {0}".format(e), exc_info=True) if publisher: publisher.publish_workflow_status( workflow_uuid, 3, logs="workflow failed: {0}".format(e)) else: log.error( "Workflow {workflow_uuid} failed but status " "could not be published.".format(workflow_uuid=workflow_uuid))
def run_yadage_workflow(workflow_uuid, workflow_workspace, workflow_json=None, workflow_file=None, workflow_parameters=None): """Run a ``yadage`` workflow.""" log.info('getting socket..') workflow_workspace = '{0}/{1}'.format(SHARED_VOLUME_PATH, workflow_workspace) # use some shared object between tasks. os.environ["workflow_uuid"] = workflow_uuid os.environ["workflow_workspace"] = workflow_workspace os.umask(REANA_WORKFLOW_UMASK) cap_backend = setupbackend_fromstring('fromenv') toplevel = os.getcwd() workflow = None if workflow_json: # When `yadage` is launched using an already validated workflow file. workflow_kwargs = dict(workflow_json=workflow_json) elif workflow: # When `yadage` resolves the workflow file from a remote repository: # i.e. github:reanahub/reana-demo-root6-roofit/workflow.yaml workflow_kwargs = dict(workflow=workflow, toplevel=toplevel) elif workflow_file: workflow_file_abs_path = os.path.join(workflow_workspace, workflow_file) if os.path.exists(workflow_file_abs_path): schema_name = 'yadage/workflow-schema' schemadir = None specopts = { 'toplevel': workflow_workspace, 'schema_name': schema_name, 'schemadir': schemadir, 'load_as_ref': False, } validopts = { 'schema_name': schema_name, 'schemadir': schemadir, } workflow_json = yadageschemas.load(spec=workflow_file, specopts=specopts, validopts=validopts, validate=True) workflow_kwargs = dict(workflow_json=workflow_json) dataopts = {'initdir': workflow_workspace} try: check_connection_to_job_controller() publisher = REANAWorkflowStatusPublisher() with steering_ctx( dataarg=workflow_workspace, dataopts=dataopts, initdata=workflow_parameters if workflow_parameters else {}, visualize=True, updateinterval=5, loginterval=5, backend=cap_backend, **workflow_kwargs) as ys: log.info('running workflow on context: {0}'.format(locals())) publisher.publish_workflow_status(workflow_uuid, 1) ys.adage_argument( additional_trackers=[REANATracker(identifier=workflow_uuid)]) publisher.publish_workflow_status(workflow_uuid, 2) log.info('Workflow {workflow_uuid} finished. Files available ' 'at {workflow_workspace}.'.format( workflow_uuid=workflow_uuid, workflow_workspace=workflow_workspace)) except Exception as e: log.info('workflow failed: {0}'.format(e), exc_info=True) if publisher: publisher.publish_workflow_status( workflow_uuid, 3, logs='workflow failed: {0}'.format(e)) else: log.error( 'Workflow {workflow_uuid} failed but status ' 'could not be published.'.format(workflow_uuid=workflow_uuid))
'/Users/lukas/Code/yadagedev/typedleafs/objectstore/public0002.dat', 'checksum': 'okcool' }, 'message': 'this is a message to the world:' } input_objects = [] inputdata = json.loads(json.dumps(inputdata), cls=TypedLeafDecoder, type_identifiers=['$type'], type_loader=typeloader, obj_store=input_objects) print inputdata raise RuntimeError('..ok..') logging.basicConfig(level=logging.INFO) pack = yadageschemas.load('exposing_transform.yml', os.getcwd(), 'packtivity/packtivity-schema') proc = pack['process'] script = proc['script'].format(**inputdata) env = pack['environment'] mounts = [] for obj in input_objects: mounts.append('{}:{}'.format(obj.public_path, obj.local_path)) if obj.local_path.startswith('/outputs'): open(obj.public_path, 'w').close() #touch file dockercmd = 'docker run --rm -i ' for mount in mounts: dockercmd += '-v {} '.format(mount) dockercmd += '{}:{} {}'.format(env['image'], env['imagetag'],