def test_parse_yaml(self, as_function=False): r"""Test successfully reading & parsing yaml.""" if self.nfiles == 0: pass elif self.nfiles == 1: yamlfile.parse_yaml(self.files[0], as_function=as_function) else: yamlfile.parse_yaml(self.files, as_function=as_function)
def test_parse_yaml(self, parse_kwargs, nfiles, files): r"""Test successfully reading & parsing yaml.""" if nfiles == 0: pass elif nfiles == 1: yamlfile.parse_yaml(files[0], **parse_kwargs) else: yamlfile.parse_yaml(files, **parse_kwargs)
def test_parse_yaml(self): r"""Test successfully reading & parsing yaml.""" if self.nfiles == 0: pass elif self.nfiles == 1: yamlfile.parse_yaml(self.files[0]) else: yamlfile.parse_yaml(self.files)
def validate_yaml(): r"""Validate a set of or or more YAMLs defining an integration.""" from yggdrasil import yamlfile parser = argparse.ArgumentParser( description='Validate a set of YAML specification files for an integration.') parser.add_argument('yamlfile', nargs='+', help='One or more YAML specification files.') args = parser.parse_args() yamlfile.parse_yaml(args.yamlfile) logger.info("Validation succesful.")
def __init__(self, modelYmls, namespace=None, host=None, rank=0, ygg_debug_level=None, rmq_debug_level=None, ygg_debug_prefix=None, connection_task_method='thread', as_function=False, production_run=False): super(YggRunner, self).__init__('runner') if namespace is None: namespace = ygg_cfg.get('rmq', 'namespace', False) if not namespace: # pragma: debug raise Exception('rmq:namespace not set in config file') self.namespace = namespace self.host = host self.rank = rank self.connection_task_method = connection_task_method self.modeldrivers = {} self.connectiondrivers = {} self.interrupt_time = 0 self._old_handlers = {} self.production_run = production_run self.error_flag = False self.as_function = as_function self.debug("Running in %s with path %s namespace %s rank %d", os.getcwd(), sys.path, namespace, rank) # Update environment based on config cfg_environment() # Parse yamls self.drivers = yamlfile.parse_yaml(modelYmls, as_function=as_function) self.connectiondrivers = self.drivers['connection'] self.modeldrivers = self.drivers['model']
def validate_model_submission(fname): r"""Validate a YAML file according to the standards for submission to the yggdrasil model repository. Args: fname (str): YAML file to validate or directory in which to check each of the YAML files. """ from yggdrasil import yamlfile, runner if isinstance(fname, list): for x in fname: validate_model_submission(x) return elif os.path.isdir(fname): files = sorted( glob.glob(os.path.join(fname, '*.yml')) + glob.glob(os.path.join(fname, '*.yaml'))) for x in files: validate_model_submission(x) return # 1-2. YAML syntax and schema yml = yamlfile.parse_yaml(fname, model_submission=True) # 3a. LICENSE repo_dir = yml['models'][0]['working_dir'] patterns = ['LICENSE', 'LICENSE.*'] for x in patterns: if ((glob.glob(os.path.join(repo_dir, x.upper())) or glob.glob(os.path.join(repo_dir, x.lower())))): break else: raise RuntimeError("Model repository does not contain a LICENSE file.") # 4. Run & validate runner.run(fname, validate=True)
def __init__(self, modelYmls, namespace, host=None, rank=0, ygg_debug_level=None, rmq_debug_level=None, ygg_debug_prefix=None): super(YggRunner, self).__init__('runner') self.namespace = namespace self.host = host self.rank = rank self.modeldrivers = {} self.inputdrivers = {} self.outputdrivers = {} self.serverdrivers = {} self.interrupt_time = 0 self._inputchannels = {} self._outputchannels = {} self._old_handlers = {} self.error_flag = False # Setup logging # if ygg_debug_prefix is None: # ygg_debug_prefix = namespace # setup_ygg_logging(ygg_debug_prefix, level=ygg_debug_level) # Update environment based on config cfg_environment() # Parse yamls drivers = yamlfile.parse_yaml(modelYmls) self.inputdrivers = drivers['input'] self.outputdrivers = drivers['output'] self.modeldrivers = drivers['model'] for x in self.outputdrivers.values(): self._outputchannels[x['args']] = x for x in self.inputdrivers.values(): self._inputchannels[x['args']] = x
def __init__(self, name, yamls): self.runner = None self.name = name self.yamls = yamls self.languages = [] for x in yamlfile.parse_yaml(yamls)['model'].values(): drv = import_component('model', x['driver'], without_schema=True) if drv.language not in self.languages: self.languages.append(drv.language) super(DemoRun, self).__init__()
def test_run(self, run_name, yamls, check_required_languages): r"""Run the integration.""" languages = [] for x in yamlfile.parse_yaml(list(yamls))['model'].values(): drv = import_component('model', x['driver'], without_schema=True) if drv.language not in languages: languages.append(drv.language) check_required_languages(languages) # Run r = runner.get_runner(yamls, namespace=run_name, production_run=True) r.run() assert (not r.error_flag) del r
def __init__(self, modelYmls, namespace=None, host=None, rank=0, ygg_debug_level=None, rmq_debug_level=None, ygg_debug_prefix=None): super(YggRunner, self).__init__('runner') if namespace is None: namespace = ygg_cfg.get('rmq', 'namespace', False) if not namespace: # pragma: debug raise Exception('rmq:namespace not set in config file') self.namespace = namespace self.host = host self.rank = rank self.modeldrivers = {} self.inputdrivers = {} self.outputdrivers = {} self.serverdrivers = {} self.interrupt_time = 0 self._inputchannels = {} self._outputchannels = {} self._old_handlers = {} self.error_flag = False self.debug("Running in %s with path %s namespace %s rank %d", os.getcwd(), sys.path, namespace, rank) # Update environment based on config cfg_environment() # Parse yamls drivers = yamlfile.parse_yaml(modelYmls) self.inputdrivers = drivers['input'] self.outputdrivers = drivers['output'] self.modeldrivers = drivers['model'] for x in self.outputdrivers.values(): self._outputchannels[x['args']] = x for x in self.inputdrivers.values(): self._inputchannels[x['args']] = x
def __init__(self, modelYmls, namespace=None, host=None, rank=0, ygg_debug_level=None, rmq_debug_level=None, ygg_debug_prefix=None, connection_task_method='thread', as_service=False, complete_partial=False, partial_commtype=None, production_run=False, mpi_tag_start=None, yaml_param=None, validate=False): self.mpi_comm = None name = 'runner' if MPI is not None: comm = MPI.COMM_WORLD if comm.Get_size() > 1: self.mpi_comm = comm rank = comm.Get_rank() name += str(rank) super(YggRunner, self).__init__(name) if namespace is None: namespace = ygg_cfg.get('rmq', 'namespace', False) if not namespace: # pragma: debug raise Exception('rmq:namespace not set in config file') if as_service: complete_partial = True self.namespace = namespace self.host = host self.rank = rank self.connection_task_method = connection_task_method self.base_dup = {} self.modelcopies = {} self.modeldrivers = {} self.connectiondrivers = {} self.interrupt_time = 0 self._old_handlers = {} self.production_run = production_run self.error_flag = False self.complete_partial = complete_partial self.partial_commtype = partial_commtype self.validate = validate self.debug("Running in %s with path %s namespace %s rank %d", os.getcwd(), sys.path, namespace, rank) # Update environment based on config cfg_environment() # Parse yamls self.mpi_tag_start = mpi_tag_start if self.mpi_comm and (self.rank > 0): pass else: self.drivers = yamlfile.parse_yaml( modelYmls, complete_partial=complete_partial, partial_commtype=partial_commtype, yaml_param=yaml_param) self.connectiondrivers = self.drivers['connection'] self.modeldrivers = self.drivers['model'] for x in self.modeldrivers.values(): if x['driver'] == 'DummyModelDriver': x['runner'] = self if as_service: for io in x['output_drivers']: for comm in io['inputs']: comm['for_service'] = True for io in x['input_drivers']: for comm in io['outputs']: comm['for_service'] = True
def validate_yaml(): r"""Validate a set of or or more YAMLs defining an integration.""" files = sys.argv[1:] yamlfile.parse_yaml(files) logging.info("Validation succesful.")
def test_parse_yaml(self, parse_kwargs, error, nfiles, files): r"""Test error reading & parsing yaml.""" if (error is None) or (nfiles == 0): return with pytest.raises(error): yamlfile.parse_yaml(files, parse_kwargs)