def get_job_file(job_file: str = ''): """ Takes a path to a YAML file that describes a job to execute and parses the YAML into a DICT. Supports single environment-variables in the form of ${env_var_here:default_value_here}. If `env_var_here` is not defined in the current environment, the default_value_here is used :param job_file: :return: """ log = logging.getLogger(__name__) log.debug("Fetching job file: `{}`".format(job_file)) try: ## # This is a very clever use of add_implicit_resolver from the elastic/curator project. # Any string that is not native YAML, but looks like the regex ${*} will be treated as # an env-var lookup. # See: https://pyyaml.org/wiki/PyYAMLDocumentation the Constructors, represents, resolvers section ## env_var_matcher = re.compile(r"\${([^}^{]+)\}") log.debug("env_var_matcher:{}".format(env_var_matcher)) yaml.add_implicit_resolver("!env", env_var_matcher) # We're required to pass a function that yaml will run when it encounters an !env tag or ${} syntax def single_constructor(loader, node): # get the string that triggered the custom constructor value = loader.construct_scalar(node) # Get the string *inside* of the ${} that we'll now need to pull from env_vars proto = env_var_matcher.match(value).group(1) log.debug("Resolving '{}' via env vars...".format(proto)) default = None # If there's a : in the ${thing:default} then we split on : and whatever is to the left is the env var to use # and whatever is on the right is the default to use if len(proto.split(':')) > 1: envvar, default = proto.split(':') log.debug("user supplied default:{}".format(default)) else: envvar = proto log.debug("envvar:{} default:{}".format(envvar, default)) log.info( "Attempting to resolve {envvar} from environment variables, falling back to: {default}" .format(envvar=envvar, default=default)) _r = os.environ[envvar] if envvar in os.environ else default log.debug("_R:{}".format(_r)) return os.environ[envvar] if envvar in os.environ else default yaml.add_constructor('!env', single_constructor) # When parsing untrusted YAML, it's possible to make Python execute code. This is by design, but to # prevent warning messages, and to disable support for the dangerous YAML, we specify the loader type # See: https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation ## return yaml.load(utils.read_file(job_file), Loader=yaml.FullLoader) except yaml.scanner.ScannerError as err: # If the file could be opened, but wasn't valid YAML... _e = "The job_file:{} could not be parsed as YAML. Err:{}".format( job_file, err) log.fatal(_e) raise JobFileError(_e)
def _configure_pyyaml(): # Configure PyYaml to implicitly resolve environment variables of form '$ENV_VAR'. env_var_pattern = re.compile(r'\$([A-Z_]*)') yaml.add_implicit_resolver('!envvar', env_var_pattern, Loader=Config.yaml_loader) def _env_var_constructor(loader, node): """ PyYaml constructor for resolving env vars. """ value = loader.construct_scalar(node) env_vars = env_var_pattern.findall(value) for ev in env_vars: try: ev_val = os.environ[ev] except KeyError: _logger.exception( "Couldn't parse env var '%s' as it hasn't been set", ev) raise value = re.sub(env_var_pattern, ev_val, value, count=1) return value yaml.add_constructor('!envvar', _env_var_constructor, Loader=Config.yaml_loader)
def get_yaml(path): """ Read the file identified by `path` and import its YAML contents. :arg path: The path to a YAML configuration file. :rtype: dict """ # Set the stage here to parse single scalar value environment vars from # the YAML file being read single = re.compile( r'^\$\{(.*)\}$' ) yaml.add_implicit_resolver ( "!single", single ) def single_constructor(loader,node): value = loader.construct_scalar(node) proto = single.match(value).group(1) default = None if len(proto.split(':')) > 1: envvar, default = proto.split(':') else: envvar = proto return os.environ[envvar] if envvar in os.environ else default yaml.add_constructor('!single', single_constructor) raw = read_file(path) try: cfg = yaml.load(raw) except yaml.scanner.ScannerError as e: raise ConfigurationError( 'Unable to parse YAML file. Error: {0}'.format(e)) return cfg
def setup_yaml_parser(): yaml.add_constructor('!env_var', env_var_constructor, yaml.UnsafeLoader) yaml.add_constructor('!raw_env_var', partial(env_var_constructor, raw=True), yaml.UnsafeLoader) yaml.add_implicit_resolver('!env_var', IMPLICIT_ENV_VAR_MATCHER, Loader=yaml.UnsafeLoader)
def parse_basic_config(config_file, resolve_env=True): """ Configures custom yaml loading behavior and parses config file """ if resolve_env: env_pattern = re.compile(r"\$\{(.*?)\}(.*)", re.VERBOSE) def env_var_constructor(loader, node): """ Converts ${VAR}/* from config file to 'os.environ[VAR] + *' Modified from: https://www.programcreek.com/python/example/61563/yaml.add_implicit_resolver """ value = loader.construct_scalar(node) env_var, remainder = env_pattern.match(value).groups() if env_var not in os.environ: raise ValueError( "config requires envirnonment variable {} which is not set" .format(env_var)) return os.environ[env_var] + remainder yaml.add_implicit_resolver("!env", env_pattern, Loader=yaml.SafeLoader) yaml.add_constructor('!env', env_var_constructor, Loader=yaml.SafeLoader) with open(os.path.expanduser(config_file), 'r') as config: return yaml.load(config, Loader=yaml.SafeLoader)
def load_config_file(self, config_paths): """Load a yaml config file from path.""" config_path = "" for possible_path in config_paths: if not os.path.isfile(possible_path): _LOGGER.debug("Config file " + possible_path + " not found") else: config_path = possible_path break if not config_path: _LOGGER.info("No configuration files found.") config_path = self.create_default_config(DEFAULT_CONFIG_PATH) env_var_pattern = re.compile(r'^\$([A-Z_]*)$') yaml.add_implicit_resolver("!envvar", env_var_pattern) def envvar_constructor(loader, node): """Yaml parser for env vars.""" value = loader.construct_scalar(node) [env_var] = env_var_pattern.match(value).groups() return os.environ[env_var] yaml.add_constructor('!envvar', envvar_constructor) try: with open(config_path, 'r') as stream: _LOGGER.info("Loaded config from %s", config_path) return yaml.load(stream) except yaml.YAMLError as error: self.opsdroid.critical(error, 1) except FileNotFoundError as error: self.opsdroid.critical(str(error), 1)
def load_config(path): """validates, loads and configures the yaml document at the specified path :param path: the path to the file :return: the parsed yaml document :raises SchemaError: if the yaml document does not validate """ validator = Core(source_file=path, schema_data=config_schema) validator.validate(raise_exception=True) pattern = re.compile(r'^(.*)<%= ENV\[\'(.*)\'\] %>(.*)$') yaml.add_implicit_resolver('!env_regex', pattern) def env_regex(loader, node): value = loader.construct_scalar(node) front, variable_name, back = pattern.match(value).groups() return str(front) + os.environ[variable_name] + str(back) yaml.add_constructor('!env_regex', env_regex) with open(path, 'r') as stream: doc = yaml.load(stream) return doc
def read_config(filename: str): if os.path.exists(filename): with open(filename) as f: data = f.read() else: data = filename path_matcher = re.compile(r'\$\{(.*?)\}') def repl(match): args = match.group(1).rsplit(':-', 1) + [''] name = args[0] default = args[1] return os.getenv(name, default) def path_constructor(loader, node): val = path_matcher.sub(repl, node.value) try: return int(val) except: pass try: return float(val) except: pass return val yaml.add_implicit_resolver('!path', path_matcher) yaml.add_constructor('!path', path_constructor) config = yaml.load(data, Loader=yaml.Loader) logging.debug(json.dumps(config, indent=4)) return config
def read(self, filepath): stream = False # Setup environment variables replacement def env_var_single_replace(match): return os.environ[match.group(1)] if match.group( 1) in os.environ else match.group() def env_var_multi_replacer(loader, node): value = loader.construct_scalar(node) return re.sub(ENV_VAR_REPLACER_PATTERN, env_var_single_replace, value) yaml.add_implicit_resolver("!envvarreplacer", ENV_VAR_MATCHER_PATTERN) yaml.add_constructor('!envvarreplacer', env_var_multi_replacer) # Read file try: stream = open(filepath, "r") self.config = yaml.load(stream) # Handle an empty configuration file if not self.config: self.config = {} finally: if stream: stream.close()
def parse_tune_config(config_file): """ Configures custom yaml loading behavior and parses config file """ import ray.tune as tune search_pattern = re.compile(r".*search\/(.*?)\((.*?)\)", re.VERBOSE) def search_constructor(loader, node): value = loader.construct_scalar(node) search_type, args = search_pattern.match(value).groups() if search_type == 'grid': return tune.grid_search(json.loads(args)) raise NotImplementedError("search {} is not implemented".format(search_type)) yaml.add_implicit_resolver("!custom_search", search_pattern, Loader=yaml.SafeLoader) yaml.add_constructor('!custom_search', search_constructor, Loader=yaml.SafeLoader) env_pattern = re.compile(r"\$\{(.*?)\}(.*)", re.VERBOSE) def env_var_constructor(loader, node): """ Converts ${VAR}/* from config file to 'os.environ[VAR] + *' Modified from: https://www.programcreek.com/python/example/61563/yaml.add_implicit_resolver """ value = loader.construct_scalar(node) env_var, remainder = env_pattern.match(value).groups() if env_var not in os.environ: raise ValueError("config requires envirnonment variable {} which is not set".format(env_var)) return os.environ[env_var] + remainder yaml.add_implicit_resolver("!env", env_pattern, Loader=yaml.SafeLoader) yaml.add_constructor('!env', env_var_constructor, Loader=yaml.SafeLoader) with open(config_file) as config: return yaml.load(config, Loader=yaml.SafeLoader)
def _initialize(): """ Notes ----- Taken (with minor changes) from `Pylearn2`_. .. _Pylearn2: \ http://github.com/lisa-lab/pylearn2/blob/master/pylearn2/config/yaml_parse.py """ global IS_INITIALIZED yaml.add_multi_constructor("!obj:", _multi_constructor_obj, Loader=yaml.SafeLoader) yaml.add_multi_constructor("!import:", _multi_constructor_import, Loader=yaml.SafeLoader) yaml.add_constructor("!import", _constructor_import, Loader=yaml.SafeLoader) yaml.add_constructor("!float", _constructor_float, Loader=yaml.SafeLoader) pattern = re.compile(SCIENTIFIC_NOTATION_REGEXP) yaml.add_implicit_resolver("!float", pattern) IS_INITIALIZED = True
def get_yaml(path): """ Read the file identified by `path` and import its YAML contents. :arg path: The path to a YAML configuration file. :rtype: dict """ # Set the stage here to parse single scalar value environment vars from # the YAML file being read single = re.compile(r'^\$\{(.*)\}$') yaml.add_implicit_resolver("!single", single) def single_constructor(loader, node): value = loader.construct_scalar(node) proto = single.match(value).group(1) default = None if len(proto.split(':')) > 1: envvar, default = proto.split(':') else: envvar = proto return os.environ[envvar] if envvar in os.environ else default yaml.add_constructor('!single', single_constructor) raw = read_file(path) try: cfg = yaml.load(raw) except yaml.scanner.ScannerError as e: raise ConfigurationError( 'Unable to parse YAML file. Error: {0}'.format(e)) return cfg
def add_custom_type(typ: Type['T'], tag: str, representer: Representer, constructor: Constructor, pattern: Optional[Pattern]) -> None: yaml.add_representer(typ, representer) yaml.add_constructor(tag, constructor) if pattern is not None: yaml.add_implicit_resolver(tag, pattern)
def loadYaml(config_path, keys): """ load yaml configuration file for specific keys. """ # Parse environment variables # see: https://stackoverflow.com/questions/26712003/pyyaml-parsing-of-the-environment-variable-in-the-yaml-configuration-file # define the regex pattern that the parser will use to 'implicitely' tag your node pattern = re.compile(r'^\<%= ENV\[\'(.*)\'\] %\>(.*)$') # now define a custom tag ( say pathex ) and associate the regex pattern we defined yaml.add_implicit_resolver("!pathex", pattern) # at this point the parser will associate '!pathex' tag whenever the node matches the pattern # you need to now define a constructor that the parser will invoke # you can do whatever you want with the node value def pathex_constructor(loader, node): value = loader.construct_scalar(node) envVar, remainingPath = pattern.match(value).groups() return os.environ[envVar] + remainingPath # 'register' the constructor so that the parser will invoke 'pathex_constructor' for each node '!pathex' yaml.add_constructor('!pathex', pathex_constructor) # load the configs configs = [] with open(config_path, 'r') as stream: try: cfg = yaml.load(stream) [configs.append(cfg[key]) for key in keys] except yaml.YAMLError as exc: print(exc) if len(configs) == 1: return configs[0] else: return tuple(configs)
def load_config_file(cls, config_paths): """Load a yaml config file from path. Args: config_paths: List of paths to configuration.yaml files Returns: dict: Dict containing config fields """ config_path = "" for possible_path in config_paths: if not os.path.isfile(possible_path): _LOGGER.debug(_("Config file %s not found."), possible_path) else: config_path = possible_path break if not config_path: try: move_config_to_appdir(PRE_0_12_0_ROOT_PATH, DEFAULT_ROOT_PATH) except FileNotFoundError: _LOGGER.info( _("No configuration files found. " "Creating %s"), DEFAULT_CONFIG_PATH, ) config_path = cls.create_default_config(DEFAULT_CONFIG_PATH) env_var_pattern = re.compile(r"^\$([A-Z_]*)$") yaml.add_implicit_resolver("!envvar", env_var_pattern) def envvar_constructor(loader, node): """Yaml parser for env vars.""" value = loader.construct_scalar(node) [env_var] = env_var_pattern.match(value).groups() return os.environ[env_var] def include_constructor(loader, node): """Add a yaml file to be loaded inside another.""" main_yaml_path = os.path.split(stream.name)[0] included_yaml = os.path.join(main_yaml_path, loader.construct_scalar(node)) with open(included_yaml, "r") as included: return yaml.safe_load(included) yaml.add_constructor("!envvar", envvar_constructor) yaml.add_constructor("!include", include_constructor) try: with open(config_path, "r") as stream: _LOGGER.info(_("Loaded config from %s."), config_path) return yaml.load(stream, Loader=yaml.SafeLoader) except yaml.YAMLError as error: _LOGGER.critical(error) sys.exit(1) except FileNotFoundError as error: _LOGGER.critical(error) sys.exit(1)
def main(argv): config_file = argv[1] global env env = Env() env.read_env(_get_env_path(config_file)) yaml.add_implicit_resolver('!envvar', envvar_matcher, Loader=yaml.FullLoader) yaml.add_constructor('!envvar', envvar_constructor, Loader=yaml.FullLoader) config = yaml.load(open(config_file), Loader=yaml.FullLoader) if argv[2] == "upload": return upload(config, argv) ffrom = datetime.date(*map(int, argv[2].split("-"))) tto = datetime.date(*map(int, argv[3].split("-"))) extra_args = argv[4:] skip_metrics = "--skip-metrics" in extra_args js_metrics = codecs.open(config["js_metrics"], "w", encoding="utf-8") js_metrics.write("metrics = [\n") for metric in get_metrics(config): metric_class = plugins.get_metric_class(metric["type"]) if not metric_class: print("Error no existe la métrica de tipo %s" % metric["type"], file=sys.stderr) sys.exit(3) metric_obj = metric_class(config, metric, metric["name"]) if not skip_metrics: print("Generating metric %s" % metric["name"]) data = metric_obj.generate(ffrom, tto) save_data(data, metric["name"], config) js_metrics.write(metric_obj.js_line()) js_metrics.write("];\n") js_metrics.write("\n") js_metrics.write("\n") js_metrics.write("pills = [\n") for pill in config["pills"]: js_metrics.write("jQuery.parseJSON('%s'),\n" % json.dumps(pill)) js_metrics.write("];\n") js_metrics.write("\n") print("Saving js_metrics file") js_metrics.write("periodChoices = [\n") for period_choice in config["period_choices"]: js_metrics.write(' {value: %(value)s, name: "%(name)s"},\n' % period_choice) js_metrics.write("];\n")
def register_temp_stow_dir(cls): yaml_pattern = re.compile(r'^\<%= temp_stow_dir %\>$') cls.stow_root_dir = tempfile.mkdtemp(prefix='stow_root') yaml.add_implicit_resolver('!temp_stow_dir', yaml_pattern) def temp_stow_dir(loader, node): return cls.stow_root_dir yaml.add_constructor('!temp_stow_dir', temp_stow_dir)
def load_config(config_file): with open(config_file, 'r') as stream: try: yaml.add_implicit_resolver('!path', path_matcher, None, yaml.SafeLoader) yaml.add_constructor('!path', path_constructor, yaml.SafeLoader) return yaml.safe_load(stream) except yaml.YAMLError as exc: print(exc)
def decorator(f): if tag is None or f is tag: tag_ = '!{}'.format(f.__name__) else: tag_ = tag yaml.add_constructor(tag_, f) if pattern is not None: yaml.add_implicit_resolver(tag_, re.compile(pattern)) return f
def parse_data_file(filename): """Parses yaml file and returns it as a Data class.""" yaml.add_constructor(u"!vec", pos_constructor, Loader=yaml.SafeLoader) yaml.add_implicit_resolver(u"!vec", re.compile(r"[0-9.]+, [0-9.]+"), Loader=yaml.SafeLoader) with open(filename) as contents: yaml_data = yaml.safe_load(contents) return yaml_data
def setup_yaml_parser(): yaml.add_constructor('!env_var', env_var_constructor, yaml.UnsafeLoader) yaml.add_constructor( '!raw_env_var', partial(env_var_constructor, raw=True), yaml.UnsafeLoader ) yaml.add_implicit_resolver( '!env_var', IMPLICIT_ENV_VAR_MATCHER, Loader=yaml.UnsafeLoader )
def setup_yaml_parser(): """ Add environment variable parsing logic to YAML module config. This and dependent functions were adopted from https://github.com/zhouxiaoxiang/oriole/blob/master/oriole/yml.py """ var = re.compile(r".*\$\{.*\}.*", re.VERBOSE) yaml.add_constructor('!env_var', _env_var_constructor) yaml.add_implicit_resolver('!env_var', var)
def load_file(file_name: str, cmd_args: dict = None): yaml.add_implicit_resolver('!env_var', _value_matcher, None, yaml.FullLoader) yaml.add_constructor('!env_var', _var_evaluator, yaml.FullLoader) global _sys_arg_map _sys_arg_map = cmd_args with open(file_name, 'r') as yaml_file: conf_data = yaml.load(yaml_file, Loader=yaml.FullLoader) return conf_data
def __init__(self, project_dir, mode): self.project_dir = project_dir self.mode = mode self.path_matcher = re.compile(r'\$\{([^}^{]+)\}') yaml.add_implicit_resolver('!path', self.path_matcher, None, yaml.SafeLoader) yaml.add_constructor('!path', self.path_constructor, yaml.SafeLoader) with open(os.path.join(self.project_dir, 'config', 'config_%s.yaml' % (self.mode)), 'r') as ymlfile: self.config = yaml.safe_load(ymlfile)
def _add_env_loader(self): # pattern matcher for environment variables env_pattern = re.compile(r'^\<%= ENV\[\'(.*)\'\] %\>(.*)$') def env_constructor(loader, node): """PyYAML constructor for environment variables""" value = loader.construct_scalar(node) var, remaining = env_pattern.match(value).groups() return os.environ.get(var) + remaining yaml.add_implicit_resolver("!env", env_pattern) yaml.add_constructor("!env", env_constructor)
def load_yaml(yaml_file, vault_secret=None): """ Load a YAML file into a python dictionary. The YAML file can be fully encrypted by Ansible-Vault or can contain multiple inline Ansible-Vault encrypted values. Ansible Vault encryption is ideal to store passwords or encrypt the entire file with sensitive data if required. """ vault = VaultLib() if vault_secret: secret_file = get_file_vault_secret(filename=vault_secret, loader=DataLoader()) secret_file.load() vault.secrets = [('default', secret_file)] # YAML ENV VAR # name: !ENV env_var('FOO')/bar yaml.add_implicit_resolver("!ENV", ENV_VAR_PATTERN) yaml.add_constructor('!ENV', env_var_constructor) data = None if os.path.isfile(yaml_file): with open(yaml_file, 'r') as stream: # Render environment variables using jinja templates contents = stream.read() template = Template(contents) stream = StringIO(template.render(env_var=os.environ)) try: if is_encrypted_file(stream): file_data = stream.read() data = yaml.load(vault.decrypt(file_data, None)) else: file_data = stream.read() data = yaml.load(file_data, Loader=yaml.Loader) ''' Commenting code below for posterity. We are not using ansible functionality but yaml load should follow the same code path regardless of the file encryption state. ''' #loader = AnsibleLoader(stream, None, vault.secrets) #try: # data = loader.get_single_data() #except Exception as exc: # raise Exception(f'Error when loading YAML config at {yaml_file} {exc}') from exc #finally: # loader.dispose() except yaml.YAMLError as exc: raise Exception(f'Error when loading YAML config at {yaml_file} {exc}') from exc else: LOGGER.debug('No file at %s', yaml_file) return data
def parse_config(config_file): """Parse the YAML config""" pattern = re.compile(r'^\<%= ENV\[\'(.*)\'\] %\>(.*)$') yaml.add_implicit_resolver("!env", pattern) def env_constructor(loader, node): """Constructor for environment variables""" value = loader.construct_scalar(node) env_var, remaining_path = pattern.match(value).groups() return os.environ[env_var] + remaining_path yaml.add_constructor('!env', env_constructor) with open(config_file) as config: return yaml.load(config)
def _setup_yaml_with_canonical_dict(): """https://stackoverflow.com/a/8661021""" yaml.add_representer( OrderedDict, lambda self, data: self.represent_mapping( "tag:yaml.org,2002:map", data.items() ), Dumper=yaml.SafeDumper, ) def repr_ordered_dict(self, data): return self.represent_mapping("tag:yaml.org,2002:map", data.items()) yaml.add_representer(dict, repr_ordered_dict, Dumper=yaml.SafeDumper) def repr_dict(self, data): return self.represent_mapping( "tag:yaml.org,2002:map", sorted(data.items(), key=lambda t: t[0]) ) yaml.add_representer(dict, repr_dict, Dumper=yaml.SafeDumper) # https://stackoverflow.com/a/45004464 def repr_str(dumper, data): if "\n" in data: return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") return dumper.represent_str(data) yaml.add_representer(str, repr_str, Dumper=yaml.SafeDumper) def repr_tuple(dumper, data): return dumper.represent_sequence("tag:yaml.org,2002:seq", list(data)) yaml.add_representer(tuple, repr_tuple, Dumper=yaml.SafeDumper) # loader = yaml.SafeLoader yaml.add_implicit_resolver( "tag:yaml.org,2002:float", re.compile( """^(?: [-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)? |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+) |\\.[0-9_]+(?:[eE][-+][0-9]+)? |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]* |[-+]?\\.(?:inf|Inf|INF) |\\.(?:nan|NaN|NAN))$""", re.X, ), list("-+0123456789."), )
def _read_yaml_config(self, config_file): # Fills in env var env_regex = re.compile(r'\$\{(.*)\}') def env_constructor(loader, node): match = env_regex.match(node.value) env_var = match.group()[2:-1] return os.getenv(env_var) + node.value[match.end():] yaml.add_implicit_resolver('!env', env_regex, None, yaml.SafeLoader) yaml.add_constructor('!env', env_constructor, yaml.SafeLoader) with open(config_file) as data_file: loaded_config = yaml.safe_load(data_file) return loaded_config
def load_config_file(cls, config_paths): """Load a yaml config file from path.""" config_path = "" for possible_path in config_paths: if not os.path.isfile(possible_path): _LOGGER.debug(_("Config file %s not found."), possible_path) else: config_path = possible_path break if not config_path: try: move_config_to_appdir(PRE_0_12_0_ROOT_PATH, DEFAULT_ROOT_PATH) except FileNotFoundError: _LOGGER.info(_("No configuration files found. " "Creating %s"), DEFAULT_CONFIG_PATH) config_path = cls.create_default_config(DEFAULT_CONFIG_PATH) env_var_pattern = re.compile(r'^\$([A-Z_]*)$') yaml.add_implicit_resolver("!envvar", env_var_pattern) def envvar_constructor(loader, node): """Yaml parser for env vars.""" value = loader.construct_scalar(node) [env_var] = env_var_pattern.match(value).groups() return os.environ[env_var] def include_constructor(loader, node): """Add a yaml file to be loaded inside another.""" main_yaml_path = os.path.split(stream.name)[0] included_yaml = os.path.join(main_yaml_path, loader.construct_scalar(node)) with open(included_yaml, 'r') as included: return yaml.safe_load(included) yaml.add_constructor('!envvar', envvar_constructor) yaml.add_constructor('!include', include_constructor) try: with open(config_path, 'r') as stream: _LOGGER.info(_("Loaded config from %s."), config_path) return yaml.load(stream, Loader=yaml.SafeLoader) except yaml.YAMLError as error: _LOGGER.critical(error) sys.exit(1) except FileNotFoundError as error: _LOGGER.critical(error) sys.exit(1)
def __require_tag(self, tag_name): tag_cls = self._registry.get(tag_name) if tag_cls.registered: return True tag_instance = tag_cls() yaml.add_implicit_resolver(tag_instance.yaml_tag(), tag_instance.yaml_pattern()) yaml.add_constructor(tag_instance.yaml_tag(), tag_instance.from_yaml) yaml.add_representer(tag_cls, tag_instance.to_yaml) tag_cls.registered = True return True
def load_yaml(path): # configure the yaml parser to allow grabbing OS environment variables in the config. # TODO(plombardi:) Improve so that the default argument is optional pattern = re.compile(r'^(.*)<%= ENV\[\'(.*)\',\'(.*)\'\] %>(.*)$') yaml.add_implicit_resolver('!env_regex', pattern) def env_regex(loader, doc_node): value = loader.construct_scalar(doc_node) front, variable_name, default, back = pattern.match(value).groups() return str(front) + os.getenv(variable_name, default) + str(back) yaml.add_constructor('!env_regex', env_regex) with open(path, 'r') as stream: return yaml.load(stream)
def _parse(data: str): path_matcher = re.compile(r'.*(\$\{([^}^{]+)\}).*') def path_constructor(_, node): value = node.value match = path_matcher.match(value) env_var = match.groups()[1] if env_var not in os.environ: raise ffd.ConfigurationError(f'Environment variable {env_var} is used in config, but is not set') return str(value).replace(f'${{{env_var}}}', os.environ.get(env_var)) yaml.add_implicit_resolver('!path', path_matcher, None, yaml.SafeLoader) yaml.add_constructor('!path', path_constructor, yaml.SafeLoader) return yaml.safe_load(data)
def replace_environment_variables(): """Enable yaml loader to process the environment variables in the yaml.""" import re import os env_var_pattern = re.compile(r"^(.*)\$\{(.*)\}(.*)$") yaml.add_implicit_resolver('!env_var', env_var_pattern) def env_var_constructor(loader, node): """Process environment variables found in the YAML.""" value = loader.construct_scalar(node) prefix, env_var, postfix = env_var_pattern.match(value).groups() return prefix + os.environ[env_var] + postfix yaml.add_constructor(u'!env_var', env_var_constructor)
def __init__(self): config_path = os.path.abspath( os.path.join(os.path.dirname(__file__), 'config.yml')) config_file = open(config_path) self.env_matcher = re.compile(r'(.*)\${([^}^{]+)}(.*)') yaml.add_implicit_resolver('!path', self.env_matcher, None, yaml.SafeLoader) yaml.add_constructor('!path', self.config_constructor, yaml.SafeLoader) self.parsed_config_file = yaml.load(config_file, Loader=yaml.SafeLoader) self.deployments = self.parsed_config_file['scaler'] self.hosts = self.parsed_config_file['server']['hosts'] self.username = self.parsed_config_file['server']['username'] self.password = self.parsed_config_file['server']['password']
def init_yaml_env_vars(): """Initializes the yaml parser to be able to set the value of fields from environment variables :rtype: None """ env_rx = re.compile(r'\$\{[^}]+\}') yaml.add_implicit_resolver('!envvar', env_rx) def envvar_constructor(loader, node): value = loader.construct_scalar(node) value = os.path.expandvars(value) return value yaml.add_constructor('!envvar', envvar_constructor)
def load_config(path): PROJECT_ROOT = pathlib.Path(__file__).parent.parent var_pattern = re.compile(r'^(.*)?\$\{(.*)\}(.*)$') yaml.add_implicit_resolver("!vars", var_pattern) VARS = {'PROJECT_ROOT': PROJECT_ROOT} def _yaml_vars_constructor(loader, node): value = loader.construct_scalar(node) prefix, var_name, postfix = var_pattern.match(value).groups() return f'{prefix}{VARS.get(var_name)}{postfix}' yaml.add_constructor('!vars', _yaml_vars_constructor) with open(f'{PROJECT_ROOT}/{path}', 'r') as f: return yaml.load(f)
def parse_config(filename): pattern = re.compile('^"?\\$\\{([^}^{]+)\\}"?$') def _path_constructor(loader, node): value = node.value match = pattern.match(value) env_var = match.group().strip('"${}') return os.environ.get(env_var) + value[match.end():] yaml.add_implicit_resolver('env', pattern, None, yaml.SafeLoader) yaml.add_constructor('env', _path_constructor, yaml.SafeLoader) with open(filename, "r") as f: cfg = yaml.load(f, Loader=yaml.SafeLoader) conf = Config(cfg) return conf
def parse_config(filename): """ parses the YAML config file and expands any environment variables """ pattern = re.compile(r'^\<%= ENV\[\'(.*)\'\] %\>(.*)$') yaml.add_implicit_resolver("!pathex", pattern) def pathex_constructor(loader, node): value = loader.construct_scalar(node) envVar, remainingPath = pattern.match(value).groups() return os.environ[envVar] + remainingPath yaml.add_constructor('!pathex', pathex_constructor) with open(filename) as f: return( yaml.load(f) )
def _readConfig(): nameConfig = utils.getConfig() fileOut = 'configurations/{0}.yaml'.format(nameConfig) yaml.add_implicit_resolver ( "!pathex", pattern ) yaml.add_constructor('!pathex', pathex_constructor) with open('configurations/default.yaml') as stream: R = yaml.load(stream) with open(fileOut, 'r') as stream: Rb = yaml.load(stream) R = dict(mergedicts(R,Rb)) return R
def initialize(): """ Initialize the configuration system by installing YAML handlers. Automatically done on first call to load() specified in this file. """ global is_initialized # Add the custom multi-constructor yaml.add_multi_constructor('!obj:', multi_constructor_obj) yaml.add_multi_constructor('!pkl:', multi_constructor_pkl) yaml.add_multi_constructor('!import:', multi_constructor_import) yaml.add_constructor('!import', constructor_import) yaml.add_constructor("!float", constructor_float) pattern = re.compile(SCIENTIFIC_NOTATION_REGEXP) yaml.add_implicit_resolver('!float', pattern) is_initialized = True
def initialize(): """ Initialize the configuration system by installing YAML handlers. Automatically done on first call to load() specified in this file. """ global is_initialized # Add the custom multi-constructor yaml.add_multi_constructor("!obj:", multi_constructor_obj) yaml.add_multi_constructor("!pkl:", multi_constructor_pkl) yaml.add_multi_constructor("!import:", multi_constructor_import) yaml.add_constructor("!import", constructor_import) yaml.add_implicit_resolver("!import", re.compile(r"(?:[a-zA-Z_][\w_]+\.)+[a-zA-Z_][\w_]+")) yaml.add_constructor("!float", constructor_float) yaml.add_implicit_resolver("!float", re.compile(r" [-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?")) is_initialized = True
def load_config(self, path, environments, fill_with_defaults=False): """Will load default.yaml and <environment>.yaml at given path. The environment config will override the default values. :param path: directory where to find your config files. If the last character is not a slash (/) it will be appended. Example: resources/ :param environments: list of environment configs to load. File name pattern: <environment>.yaml. Example: develop.yaml. Latter configs will override previous ones. :param fill_with_defaults: use 'defaults' keyword in config file to fill up following config entrys. :return: your config as dictionary. """ yaml.add_implicit_resolver("!environ", self.__environ_pattern) yaml.add_constructor('!environ', self.__get_from_environment) yaml.add_implicit_resolver("!vault", self.__vault_pattern) yaml.add_constructor('!vault', self.__get_from_vault) if not path.endswith('/'): path += '/' if type(environments) != list: environments = [environments] config = {} try: for env in environments: with open(path + env + '.yaml', 'r') as configFile: env_config = yaml.load(configFile.read()) or {} config.update(env_config) if fill_with_defaults: if 'defaults' in config: defaults = config['defaults'] for target in defaults: for index, item in enumerate(config[target]): tmp = defaults[target].copy() tmp.update(config[target][index]) config[target][index] = tmp return config except exceptions.VaultError as error: raise ConfigLoaderError("Could not read vault secrets [" + error.__class__.__name__ + "]") except yaml.YAMLError as error: raise ConfigLoaderError("Configuration files malformed [" + error.__class__.__name__ + "]") except json.decoder.JSONDecodeError as error: raise ConfigLoaderError("Vault response was not json [" + error.__class__.__name__ + "]") except Exception as error: raise ConfigLoaderError("WTF? [" + error.__class__.__name__ + "]")
def initialize(): """ Initialize the configuration system by installing YAML handlers. Automatically done on first call to load() specified in this file. """ global is_initialized # Add the custom multi-constructor yaml.add_multi_constructor('!obj:', multi_constructor) yaml.add_multi_constructor('!pkl:', multi_constructor_pkl) yaml.add_multi_constructor('!import:', multi_constructor_import) def import_constructor(loader, node): value = loader.construct_scalar(node) return try_to_import(value) yaml.add_constructor('!import', import_constructor) yaml.add_implicit_resolver( '!import', re.compile(r'(?:[a-zA-Z_][\w_]+\.)+[a-zA-Z_][\w_]+') ) is_initialized = True
def uuid_add_implicit_resolver(Loader=ConfigLoader, Dumper=ConfigDumper): """ Attach an implicit pattern resolver for UUID objects. Tests: >>> class TestDumper(yaml.SafeDumper): pass >>> class TestLoader(yaml.SafeLoader): pass >>> TUUID = 'cc3702ca-699a-4aa6-8226-4c938f294d9b' >>> IN = {'test': uuid.UUID(TUUID)} >>> OUT = '{test: cc3702ca-699a-4aa6-8226-4c938f294d9b}' >>> yaml.add_representer(uuid.UUID, uuid_representer, Dumper=TestDumper) >>> yaml.add_constructor('!uuid', uuid_constructor, Loader=TestLoader) >>> uuid_add_implicit_resolver(Loader=TestLoader, Dumper=TestDumper) >>> yaml.dump(IN, Dumper=TestDumper) 'test: cc3702ca-699a-4aa6-8226-4c938f294d9b\\n' >>> yaml.load(OUT, Loader=TestLoader) {'test': UUID('cc3702ca-699a-4aa6-8226-4c938f294d9b')} """ uuid_regex = r'^\b[a-f0-9]{8}-\b[a-f0-9]{4}-\b[a-f0-9]{4}-\b[a-f0-9]{4}-\b[a-f0-9]{12}$' pattern = re.compile(uuid_regex) yaml.add_implicit_resolver('!uuid', pattern, Loader=Loader, Dumper=Dumper)
"Output variables in YAML as ?name." return dumper.represent_scalar(u"!var", u"?%s" % var.name()) yaml.add_representer(Variable, variable_representer) def variable_constructor(loader, node): "Recognize variables written as ?name in YAML." value = loader.construct_scalar(node) name = value[1:] return Variable(name) yaml.add_constructor(u"!var", variable_constructor) yaml.add_implicit_resolver(u"!var", re.compile(r"^\?\w+$")) def _copy_and_bind(feature, bindings, memo=None): """ Make a deep copy of a feature structure, preserving reentrance using the C{memo} dictionary. Meanwhile, variables are replaced by their bound values, if these values are already known, and variables with unknown values are given placeholder bindings. """ if memo is None: memo = {} if id(feature) in memo: return memo[id(feature)] if isinstance(feature, Variable) and bindings is not None: if not bindings.has_key(feature.name()):
def setup_yaml_parser(): yaml.add_constructor('!env_var', _env_var_constructor) yaml.add_implicit_resolver('!env_var', IMPLICIT_ENV_VAR_MATCHER)
import re import yaml JINJA = yaml.add_implicit_resolver( u'tag:yaml.org,2002:jinja', re.compile('{{.*?}}') )
import os.path import yaml import dsl import dsl.typespec from dsl.yaql_expression import YaqlExpression import system.yaql_functions def yaql_constructor(loader, node): value = loader.construct_scalar(node) return YaqlExpression(value) yaml.add_constructor(u'!yaql', yaql_constructor) yaml.add_implicit_resolver(u'!yaql', YaqlExpression) class ClassLoader(dsl.MuranoClassLoader): def __init__(self, base_path): self._base_path = base_path super(ClassLoader, self).__init__() def load_definition(self, name): path = os.path.join(self._base_path, name, 'manifest.yaml') if not os.path.exists(path): return None with open(path) as stream: return yaml.load(stream) def create_root_context(self): context = super(ClassLoader, self).create_root_context()
# Copyright (c) 2010 A. Frederick Dudley. All rights reserved. # """This code converts game objects to and from yaml streams""" import re import yaml from binary_tactics.grid import Loc from stores.store import convert_dict def loc_representer(dumper, data): return dumper.represent_scalar(u'!loc', u'(%2s,%2s)' % data) yaml.add_representer(Loc, loc_representer) def loc_constructor(loader, node): value = loader.construct_scalar(node) try: x, y = map(int, value.strip('()').split(',')) except ValueError: #This isn't so bad. x = y = None return Loc(x,y) yaml.add_constructor(u'!loc', loc_constructor) loc_pat = re.compile(r'^\(\s*\d+\s*,\s*\d+\s*\)\s*$') none_pat = re.compile(r'^\(\s*None\s*,\s*None\s*\)\s*$') yaml.add_implicit_resolver(u'!loc', loc_pat) yaml.add_implicit_resolver(u'!loc', none_pat) def load(filename): return convert_dict(yaml.load(open(filename, 'r')))
from muranoapi.packages import exceptions as pkg_exc LOG = logging.getLogger(__name__) class YaqlYamlLoader(yaml.Loader): pass def yaql_constructor(loader, node): value = loader.construct_scalar(node) return yaql_expression.YaqlExpression(value) yaml.add_constructor(u"!yaql", yaql_constructor, YaqlYamlLoader) yaml.add_implicit_resolver(u"!yaql", yaql_expression.YaqlExpression, Loader=YaqlYamlLoader) class PackageLoader(six.with_metaclass(abc.ABCMeta)): @abc.abstractmethod def get_package(self, name): pass @abc.abstractmethod def get_package_by_class(self, name): pass class ApiPackageLoader(PackageLoader): def __init__(self, token_id, tenant_id): self._cache_directory = self._get_cache_directory()
import yaml, os, re #define the regex pattern that the parser will use to 'implicitely' tag your node pattern = re.compile( r'^\<%= ENV\[\'(.*)\'\] %\>(.*)$' ) #now define a custom tag ( say pathex ) and associate the regex pattern we defined yaml.add_implicit_resolver ( "!pathex", pattern ) #at this point the parser will associate '!pathex' tag whenever the node matches the pattern #you need to now define a constructor that the parser will invoke #you can do whatever you want with the node value def pathex_constructor(loader,node): value = loader.construct_scalar(node) envVar, remainingPath = pattern.match(value).groups() return os.environ[envVar] + remainingPath #'register' the constructor so that the parser will invoke 'pathex_constructor' for each node '!pathex' yaml.add_constructor('!pathex', pathex_constructor) #that is it data = """ version: 1 disable_existing_loggers: False formatters: precise: format: "%(name)-15s # %(levelname)-8s # %(asctime)s # [Line: %(lineno)-3d]: %(message)s" datefmt: "%Y-%m-%d %H:%M:%S" handlers: file:
argv = sys.argv if len(argv) == 1: _fullusage() try: command = argv[1] if command in ["get", "dump"]: _rosparam_cmd_get_dump(command, argv) elif command in ["set", "load"]: _rosparam_cmd_set_load(command, argv) elif command in ["delete"]: _rosparam_cmd_delete(argv) elif command == "list": _rosparam_cmd_list(argv) else: _fullusage() except ROSParamException, e: print >> sys.stderr, "ERROR: " + str(e) sys.exit(1) # YAML configuration. Doxygen does not like these being higher up in the code yaml.add_constructor(u"!radians", construct_angle_radians) yaml.add_constructor(u"!degrees", construct_angle_degrees) # allow both !degrees 180, !radians 2*pi pattern = re.compile(r"^deg\([^\)]*\)$") yaml.add_implicit_resolver(u"!degrees", pattern, first="deg(") pattern = re.compile(r"^rad\([^\)]*\)$") yaml.add_implicit_resolver(u"!radians", pattern, first="rad(")
argv = sys.argv if len(argv) == 1: _fullusage() try: command = argv[1] if command in ['get', 'dump']: _rosparam_cmd_get_dump(command, argv) elif command in ['set', 'load']: _rosparam_cmd_set_load(command, argv) elif command in ['delete']: _rosparam_cmd_delete(argv) elif command == 'list': _rosparam_cmd_list(argv) else: _fullusage() except RosParamException as e: print("ERROR: "+str(e), file=sys.stderr) sys.exit(1) # YAML configuration. Doxygen does not like these being higher up in the code yaml.add_constructor(u'!radians', construct_angle_radians) yaml.add_constructor(u'!degrees', construct_angle_degrees) # allow both !degrees 180, !radians 2*pi pattern = re.compile(r'^deg\([^\)]*\)$') yaml.add_implicit_resolver(u'!degrees', pattern, first="deg(") pattern = re.compile(r'^rad\([^\)]*\)$') yaml.add_implicit_resolver(u'!radians', pattern, first="rad(")
return el value = loader.construct_scalar(node) # remove the ( ) from the string tup_elements = [ele.strip() for ele in value[1:-1].split(',')] # remove the last element if the tuple was written as (x,b,) if tup_elements[-1] == '': tup_elements.pop(-1) tup = tuple(map(parse_tup_el, tup_elements)) return tup # !tuple is my own tag name, I think you could choose anything you want yaml.add_constructor(u'!tuple', _yml_tuple_constructor) # this is to spot the strings written as tuple in the yaml yaml.add_implicit_resolver(u'!tuple', re.compile(r"\((.*?,.*?)\)")) def parse_yml_file(fname: str) -> Dict: """parse_yml_file reads the given file and parses its content as YML. Parameters ---------- fname : str fname is the of the file to parse. Returns ------- content : Dict The content of the file as dictionary. """
yaml.add_constructor(u'!include', include_constructor) def range_constructor(loader, node): data = loader.construct_scalar(node) match = range_regex.match(data) if match: start, stop = map(int, match.groups()) return range(start, stop + 1) else: raise ValueError('Invalid range: %s' % data) range_regex = re.compile(r'^\s*(\d+)\s*\.\.\s*(\d+)\s*$') yaml.add_constructor(u'!range', range_constructor) yaml.add_implicit_resolver(u'!range', range_regex) def refer_constructor(loader, node): def getRefer(content, keys): content = content.replace('!refer ', '') ref = yaml.load(content) for k in keys: ref = ref.get(k) return ref data = loader.construct_scalar(node) keys = data.split('.') offset = loader.stream.tell() loader.stream.seek(0) content = loader.stream.read()
return os.environ[var] def connect(db_config): return pymysql.connect( charset=db_config['charset'], cursorclass=pymysql.cursors.DictCursor, db=db_config['database'], host=db_config['host'], password=db_config['password'], port=3306, user=db_config['username'] ) if __name__ == "__main__": pattern = re.compile(r'^<%= ENV\[\'(.*)\'\] %>$') yaml.add_implicit_resolver('!env_regex', pattern) def env_regex(loader, node): value = loader.construct_scalar(node) var = pattern.match(value).groups()[0] print "VAR is %s" % var return os.environ[var] yaml.add_constructor('!env_regex', env_regex) passed_args = sys.argv with open(passed_args[2], 'r') as f: config = yaml.load(f) # only using debug=True for the nice auto-reload on change feature environ = passed_args[1]
def __str__(self): if self.comment: return '"%s" <%s@%s>' % (self.comment, self.user, self.domain) else: return '<%s@%s>' % (self.user, self.domain) def __repr__(self): return "%s('%s')" % (self.__class__.__name__, self.__str__()) def __eq__(self, other): if isinstance(other, EmailAddress): if (other.user == self.user and other.domain == self.domain): return True return False def __ne__(self, other): if isinstance(other, EmailAddress): if (other.user != self.user and other.domain != self.domain): return True return False # Make sure that when we emit stuff taht the email addresses show up properly # without extra annoying tags. yaml.add_implicit_resolver(EmailAddress.yaml_tag, EMAIL_RE)
if re.match('^[\s\w\d.:]*$', expr): return False try: yaql.parse(expr) return True except yaql.exceptions.YaqlGrammarException: return False except yaql.exceptions.YaqlLexicalException: return False def evaluate(self, data=None, context=None): return self._parsed_expression.evaluate(data=data, context=context) class YaqlYamlLoader(yaml.Loader): pass # workaround for PyYAML bug: http://pyyaml.org/ticket/221 resolvers = {} for k, v in yaml.Loader.yaml_implicit_resolvers.items(): resolvers[k] = v[:] YaqlYamlLoader.yaml_implicit_resolvers = resolvers def yaql_constructor(loader, node): value = loader.construct_scalar(node) return YaqlExpression(value) yaml.add_constructor(u'!yaql', yaql_constructor, YaqlYamlLoader) yaml.add_implicit_resolver(u'!yaql', YaqlExpression, Loader=YaqlYamlLoader)
is always off, and there is no blank line at the end. """ return yaml.dump(data, default_flow_style=False).strip() def variable_representer(dumper, var): "Output variables in YAML as ?name." return dumper.represent_scalar(u'!var', u'?%s' % var.name()) yaml.add_representer(Variable, variable_representer) def variable_constructor(loader, node): "Recognize variables written as ?name in YAML." value = loader.construct_scalar(node) name = value[1:] return Variable(name) yaml.add_constructor(u'!var', variable_constructor) yaml.add_implicit_resolver(u'!var', re.compile(r'^\?\w+$')) def _copy_and_bind(feature, bindings, memo=None): """ Make a deep copy of a feature structure, preserving reentrance using the C{memo} dictionary. Meanwhile, variables are replaced by their bound values, if these values are already known, and variables with unknown values are given placeholder bindings. """ if memo is None: memo = {} if id(feature) in memo: return memo[id(feature)] if isinstance(feature, Variable) and bindings is not None: if not bindings.has_key(feature.name()): bindings[feature.name()] = feature.copy() result = _copy_and_bind(bindings[feature.name()], None, memo) else: