def get_job_file(job_file: str = ''):
"""
Takes a path to a YAML file that describes a job to execute and parses the YAML into a DICT.
Supports single environment-variables in the form of ${env_var_here:default_value_here}.
If `env_var_here` is not defined in the current environment, the default_value_here is used
:param job_file:
:return:
"""
log = logging.getLogger(__name__)
log.debug("Fetching job file: `{}`".format(job_file))
try:
##
# This is a very clever use of add_implicit_resolver from the elastic/curator project.
# Any string that is not native YAML, but looks like the regex ${*} will be treated as
# an env-var lookup.
# See: https://pyyaml.org/wiki/PyYAMLDocumentation the Constructors, represents, resolvers section
##
env_var_matcher = re.compile(r"\${([^}^{]+)\}")
log.debug("env_var_matcher:{}".format(env_var_matcher))
yaml.add_implicit_resolver("!env", env_var_matcher)
# We're required to pass a function that yaml will run when it encounters an !env tag or ${} syntax
def single_constructor(loader, node):
# get the string that triggered the custom constructor
value = loader.construct_scalar(node)
# Get the string *inside* of the ${} that we'll now need to pull from env_vars
proto = env_var_matcher.match(value).group(1)
log.debug("Resolving '{}' via env vars...".format(proto))
default = None
# If there's a : in the ${thing:default} then we split on : and whatever is to the left is the env var to use
# and whatever is on the right is the default to use
if len(proto.split(':')) > 1:
envvar, default = proto.split(':')
log.debug("user supplied default:{}".format(default))
else:
envvar = proto
log.debug("envvar:{} default:{}".format(envvar, default))
log.info(
"Attempting to resolve {envvar} from environment variables, falling back to: {default}"
.format(envvar=envvar, default=default))
_r = os.environ[envvar] if envvar in os.environ else default
log.debug("_R:{}".format(_r))
return os.environ[envvar] if envvar in os.environ else default
yaml.add_constructor('!env', single_constructor)
# When parsing untrusted YAML, it's possible to make Python execute code. This is by design, but to
# prevent warning messages, and to disable support for the dangerous YAML, we specify the loader type
# See: https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation
##
return yaml.load(utils.read_file(job_file), Loader=yaml.FullLoader)
except yaml.scanner.ScannerError as err:
# If the file could be opened, but wasn't valid YAML...
_e = "The job_file:{} could not be parsed as YAML. Err:{}".format(
job_file, err)
log.fatal(_e)
raise JobFileError(_e)
def _configure_pyyaml():
# Configure PyYaml to implicitly resolve environment variables of form '$ENV_VAR'.
env_var_pattern = re.compile(r'\$([A-Z_]*)')
yaml.add_implicit_resolver('!envvar',
env_var_pattern,
Loader=Config.yaml_loader)
def _env_var_constructor(loader, node):
"""
PyYaml constructor for resolving env vars.
"""
value = loader.construct_scalar(node)
env_vars = env_var_pattern.findall(value)
for ev in env_vars:
try:
ev_val = os.environ[ev]
except KeyError:
_logger.exception(
"Couldn't parse env var '%s' as it hasn't been set",
ev)
raise
value = re.sub(env_var_pattern, ev_val, value, count=1)
return value
yaml.add_constructor('!envvar',
_env_var_constructor,
Loader=Config.yaml_loader)
def get_yaml(path):
"""
Read the file identified by `path` and import its YAML contents.
:arg path: The path to a YAML configuration file.
:rtype: dict
"""
# Set the stage here to parse single scalar value environment vars from
# the YAML file being read
single = re.compile( r'^\$\{(.*)\}$' )
yaml.add_implicit_resolver ( "!single", single )
def single_constructor(loader,node):
value = loader.construct_scalar(node)
proto = single.match(value).group(1)
default = None
if len(proto.split(':')) > 1:
envvar, default = proto.split(':')
else:
envvar = proto
return os.environ[envvar] if envvar in os.environ else default
yaml.add_constructor('!single', single_constructor)
raw = read_file(path)
try:
cfg = yaml.load(raw)
except yaml.scanner.ScannerError as e:
raise ConfigurationError(
'Unable to parse YAML file. Error: {0}'.format(e))
return cfg
def setup_yaml_parser():
yaml.add_constructor('!env_var', env_var_constructor, yaml.UnsafeLoader)
yaml.add_constructor('!raw_env_var', partial(env_var_constructor,
raw=True), yaml.UnsafeLoader)
yaml.add_implicit_resolver('!env_var',
IMPLICIT_ENV_VAR_MATCHER,
Loader=yaml.UnsafeLoader)
def parse_basic_config(config_file, resolve_env=True):
"""
Configures custom yaml loading behavior and parses config file
"""
if resolve_env:
env_pattern = re.compile(r"\$\{(.*?)\}(.*)", re.VERBOSE)
def env_var_constructor(loader, node):
"""
Converts ${VAR}/* from config file to 'os.environ[VAR] + *'
Modified from: https://www.programcreek.com/python/example/61563/yaml.add_implicit_resolver
"""
value = loader.construct_scalar(node)
env_var, remainder = env_pattern.match(value).groups()
if env_var not in os.environ:
raise ValueError(
"config requires envirnonment variable {} which is not set"
.format(env_var))
return os.environ[env_var] + remainder
yaml.add_implicit_resolver("!env", env_pattern, Loader=yaml.SafeLoader)
yaml.add_constructor('!env',
env_var_constructor,
Loader=yaml.SafeLoader)
with open(os.path.expanduser(config_file), 'r') as config:
return yaml.load(config, Loader=yaml.SafeLoader)
def load_config_file(self, config_paths):
"""Load a yaml config file from path."""
config_path = ""
for possible_path in config_paths:
if not os.path.isfile(possible_path):
_LOGGER.debug("Config file " + possible_path + " not found")
else:
config_path = possible_path
break
if not config_path:
_LOGGER.info("No configuration files found.")
config_path = self.create_default_config(DEFAULT_CONFIG_PATH)
env_var_pattern = re.compile(r'^\$([A-Z_]*)$')
yaml.add_implicit_resolver("!envvar", env_var_pattern)
def envvar_constructor(loader, node):
"""Yaml parser for env vars."""
value = loader.construct_scalar(node)
[env_var] = env_var_pattern.match(value).groups()
return os.environ[env_var]
yaml.add_constructor('!envvar', envvar_constructor)
try:
with open(config_path, 'r') as stream:
_LOGGER.info("Loaded config from %s", config_path)
return yaml.load(stream)
except yaml.YAMLError as error:
self.opsdroid.critical(error, 1)
except FileNotFoundError as error:
self.opsdroid.critical(str(error), 1)
def load_config(path):
"""validates, loads and configures the yaml document at the specified path
:param path: the path to the file
:return: the parsed yaml document
:raises SchemaError: if the yaml document does not validate
"""
validator = Core(source_file=path, schema_data=config_schema)
validator.validate(raise_exception=True)
pattern = re.compile(r'^(.*)<%= ENV\[\'(.*)\'\] %>(.*)$')
yaml.add_implicit_resolver('!env_regex', pattern)
def env_regex(loader, node):
value = loader.construct_scalar(node)
front, variable_name, back = pattern.match(value).groups()
return str(front) + os.environ[variable_name] + str(back)
yaml.add_constructor('!env_regex', env_regex)
with open(path, 'r') as stream:
doc = yaml.load(stream)
return doc
def read_config(filename: str):
if os.path.exists(filename):
with open(filename) as f:
data = f.read()
else:
data = filename
path_matcher = re.compile(r'\$\{(.*?)\}')
def repl(match):
args = match.group(1).rsplit(':-', 1) + ['']
name = args[0]
default = args[1]
return os.getenv(name, default)
def path_constructor(loader, node):
val = path_matcher.sub(repl, node.value)
try:
return int(val)
except:
pass
try:
return float(val)
except:
pass
return val
yaml.add_implicit_resolver('!path', path_matcher)
yaml.add_constructor('!path', path_constructor)
config = yaml.load(data, Loader=yaml.Loader)
logging.debug(json.dumps(config, indent=4))
return config
def read(self, filepath):
stream = False
# Setup environment variables replacement
def env_var_single_replace(match):
return os.environ[match.group(1)] if match.group(
1) in os.environ else match.group()
def env_var_multi_replacer(loader, node):
value = loader.construct_scalar(node)
return re.sub(ENV_VAR_REPLACER_PATTERN, env_var_single_replace,
value)
yaml.add_implicit_resolver("!envvarreplacer", ENV_VAR_MATCHER_PATTERN)
yaml.add_constructor('!envvarreplacer', env_var_multi_replacer)
# Read file
try:
stream = open(filepath, "r")
self.config = yaml.load(stream)
# Handle an empty configuration file
if not self.config:
self.config = {}
finally:
if stream:
stream.close()
def parse_tune_config(config_file):
"""
Configures custom yaml loading behavior and parses config file
"""
import ray.tune as tune
search_pattern = re.compile(r".*search\/(.*?)\((.*?)\)", re.VERBOSE)
def search_constructor(loader, node):
value = loader.construct_scalar(node)
search_type, args = search_pattern.match(value).groups()
if search_type == 'grid':
return tune.grid_search(json.loads(args))
raise NotImplementedError("search {} is not implemented".format(search_type))
yaml.add_implicit_resolver("!custom_search", search_pattern, Loader=yaml.SafeLoader)
yaml.add_constructor('!custom_search', search_constructor, Loader=yaml.SafeLoader)
env_pattern = re.compile(r"\$\{(.*?)\}(.*)", re.VERBOSE)
def env_var_constructor(loader, node):
"""
Converts ${VAR}/* from config file to 'os.environ[VAR] + *'
Modified from: https://www.programcreek.com/python/example/61563/yaml.add_implicit_resolver
"""
value = loader.construct_scalar(node)
env_var, remainder = env_pattern.match(value).groups()
if env_var not in os.environ:
raise ValueError("config requires envirnonment variable {} which is not set".format(env_var))
return os.environ[env_var] + remainder
yaml.add_implicit_resolver("!env", env_pattern, Loader=yaml.SafeLoader)
yaml.add_constructor('!env', env_var_constructor, Loader=yaml.SafeLoader)
with open(config_file) as config:
return yaml.load(config, Loader=yaml.SafeLoader)
def _initialize():
"""
Notes
-----
Taken (with minor changes) from `Pylearn2`_.
.. _Pylearn2: \
http://github.com/lisa-lab/pylearn2/blob/master/pylearn2/config/yaml_parse.py
"""
global IS_INITIALIZED
yaml.add_multi_constructor("!obj:",
_multi_constructor_obj,
Loader=yaml.SafeLoader)
yaml.add_multi_constructor("!import:",
_multi_constructor_import,
Loader=yaml.SafeLoader)
yaml.add_constructor("!import",
_constructor_import,
Loader=yaml.SafeLoader)
yaml.add_constructor("!float", _constructor_float, Loader=yaml.SafeLoader)
pattern = re.compile(SCIENTIFIC_NOTATION_REGEXP)
yaml.add_implicit_resolver("!float", pattern)
IS_INITIALIZED = True
def get_yaml(path):
"""
Read the file identified by `path` and import its YAML contents.
:arg path: The path to a YAML configuration file.
:rtype: dict
"""
# Set the stage here to parse single scalar value environment vars from
# the YAML file being read
single = re.compile(r'^\$\{(.*)\}$')
yaml.add_implicit_resolver("!single", single)
def single_constructor(loader, node):
value = loader.construct_scalar(node)
proto = single.match(value).group(1)
default = None
if len(proto.split(':')) > 1:
envvar, default = proto.split(':')
else:
envvar = proto
return os.environ[envvar] if envvar in os.environ else default
yaml.add_constructor('!single', single_constructor)
raw = read_file(path)
try:
cfg = yaml.load(raw)
except yaml.scanner.ScannerError as e:
raise ConfigurationError(
'Unable to parse YAML file. Error: {0}'.format(e))
return cfg
def add_custom_type(typ: Type['T'], tag: str, representer: Representer,
constructor: Constructor,
pattern: Optional[Pattern]) -> None:
yaml.add_representer(typ, representer)
yaml.add_constructor(tag, constructor)
if pattern is not None:
yaml.add_implicit_resolver(tag, pattern)
def loadYaml(config_path, keys):
""" load yaml configuration file for specific keys.
"""
# Parse environment variables
# see: https://stackoverflow.com/questions/26712003/pyyaml-parsing-of-the-environment-variable-in-the-yaml-configuration-file
# define the regex pattern that the parser will use to 'implicitely' tag your node
pattern = re.compile(r'^\<%= ENV\[\'(.*)\'\] %\>(.*)$')
# now define a custom tag ( say pathex ) and associate the regex pattern we defined
yaml.add_implicit_resolver("!pathex", pattern)
# at this point the parser will associate '!pathex' tag whenever the node matches the pattern
# you need to now define a constructor that the parser will invoke
# you can do whatever you want with the node value
def pathex_constructor(loader, node):
value = loader.construct_scalar(node)
envVar, remainingPath = pattern.match(value).groups()
return os.environ[envVar] + remainingPath
# 'register' the constructor so that the parser will invoke 'pathex_constructor' for each node '!pathex'
yaml.add_constructor('!pathex', pathex_constructor)
# load the configs
configs = []
with open(config_path, 'r') as stream:
try:
cfg = yaml.load(stream)
[configs.append(cfg[key]) for key in keys]
except yaml.YAMLError as exc:
print(exc)
if len(configs) == 1:
return configs[0]
else:
return tuple(configs)
def load_config_file(cls, config_paths):
"""Load a yaml config file from path.
Args:
config_paths: List of paths to configuration.yaml files
Returns:
dict: Dict containing config fields
"""
config_path = ""
for possible_path in config_paths:
if not os.path.isfile(possible_path):
_LOGGER.debug(_("Config file %s not found."), possible_path)
else:
config_path = possible_path
break
if not config_path:
try:
move_config_to_appdir(PRE_0_12_0_ROOT_PATH, DEFAULT_ROOT_PATH)
except FileNotFoundError:
_LOGGER.info(
_("No configuration files found. "
"Creating %s"),
DEFAULT_CONFIG_PATH,
)
config_path = cls.create_default_config(DEFAULT_CONFIG_PATH)
env_var_pattern = re.compile(r"^\$([A-Z_]*)$")
yaml.add_implicit_resolver("!envvar", env_var_pattern)
def envvar_constructor(loader, node):
"""Yaml parser for env vars."""
value = loader.construct_scalar(node)
[env_var] = env_var_pattern.match(value).groups()
return os.environ[env_var]
def include_constructor(loader, node):
"""Add a yaml file to be loaded inside another."""
main_yaml_path = os.path.split(stream.name)[0]
included_yaml = os.path.join(main_yaml_path,
loader.construct_scalar(node))
with open(included_yaml, "r") as included:
return yaml.safe_load(included)
yaml.add_constructor("!envvar", envvar_constructor)
yaml.add_constructor("!include", include_constructor)
try:
with open(config_path, "r") as stream:
_LOGGER.info(_("Loaded config from %s."), config_path)
return yaml.load(stream, Loader=yaml.SafeLoader)
except yaml.YAMLError as error:
_LOGGER.critical(error)
sys.exit(1)
except FileNotFoundError as error:
_LOGGER.critical(error)
sys.exit(1)
def main(argv):
config_file = argv[1]
global env
env = Env()
env.read_env(_get_env_path(config_file))
yaml.add_implicit_resolver('!envvar',
envvar_matcher,
Loader=yaml.FullLoader)
yaml.add_constructor('!envvar', envvar_constructor, Loader=yaml.FullLoader)
config = yaml.load(open(config_file), Loader=yaml.FullLoader)
if argv[2] == "upload":
return upload(config, argv)
ffrom = datetime.date(*map(int, argv[2].split("-")))
tto = datetime.date(*map(int, argv[3].split("-")))
extra_args = argv[4:]
skip_metrics = "--skip-metrics" in extra_args
js_metrics = codecs.open(config["js_metrics"], "w", encoding="utf-8")
js_metrics.write("metrics = [\n")
for metric in get_metrics(config):
metric_class = plugins.get_metric_class(metric["type"])
if not metric_class:
print("Error no existe la métrica de tipo %s" % metric["type"],
file=sys.stderr)
sys.exit(3)
metric_obj = metric_class(config, metric, metric["name"])
if not skip_metrics:
print("Generating metric %s" % metric["name"])
data = metric_obj.generate(ffrom, tto)
save_data(data, metric["name"], config)
js_metrics.write(metric_obj.js_line())
js_metrics.write("];\n")
js_metrics.write("\n")
js_metrics.write("\n")
js_metrics.write("pills = [\n")
for pill in config["pills"]:
js_metrics.write("jQuery.parseJSON('%s'),\n" % json.dumps(pill))
js_metrics.write("];\n")
js_metrics.write("\n")
print("Saving js_metrics file")
js_metrics.write("periodChoices = [\n")
for period_choice in config["period_choices"]:
js_metrics.write(' {value: %(value)s, name: "%(name)s"},\n' %
period_choice)
js_metrics.write("];\n")
def register_temp_stow_dir(cls):
yaml_pattern = re.compile(r'^\<%= temp_stow_dir %\>$')
cls.stow_root_dir = tempfile.mkdtemp(prefix='stow_root')
yaml.add_implicit_resolver('!temp_stow_dir', yaml_pattern)
def temp_stow_dir(loader, node):
return cls.stow_root_dir
yaml.add_constructor('!temp_stow_dir', temp_stow_dir)
def load_config(config_file):
with open(config_file, 'r') as stream:
try:
yaml.add_implicit_resolver('!path', path_matcher, None,
yaml.SafeLoader)
yaml.add_constructor('!path', path_constructor, yaml.SafeLoader)
return yaml.safe_load(stream)
except yaml.YAMLError as exc:
print(exc)
def decorator(f):
if tag is None or f is tag:
tag_ = '!{}'.format(f.__name__)
else:
tag_ = tag
yaml.add_constructor(tag_, f)
if pattern is not None:
yaml.add_implicit_resolver(tag_, re.compile(pattern))
return f
def parse_data_file(filename):
"""Parses yaml file and returns it as a Data class."""
yaml.add_constructor(u"!vec", pos_constructor, Loader=yaml.SafeLoader)
yaml.add_implicit_resolver(u"!vec", re.compile(r"[0-9.]+, [0-9.]+"),
Loader=yaml.SafeLoader)
with open(filename) as contents:
yaml_data = yaml.safe_load(contents)
return yaml_data
def setup_yaml_parser():
yaml.add_constructor('!env_var', env_var_constructor, yaml.UnsafeLoader)
yaml.add_constructor(
'!raw_env_var',
partial(env_var_constructor, raw=True),
yaml.UnsafeLoader
)
yaml.add_implicit_resolver(
'!env_var', IMPLICIT_ENV_VAR_MATCHER, Loader=yaml.UnsafeLoader
)
def setup_yaml_parser():
"""
Add environment variable parsing logic to YAML module config.
This and dependent functions were adopted from
https://github.com/zhouxiaoxiang/oriole/blob/master/oriole/yml.py
"""
var = re.compile(r".*\$\{.*\}.*", re.VERBOSE)
yaml.add_constructor('!env_var', _env_var_constructor)
yaml.add_implicit_resolver('!env_var', var)
def load_file(file_name: str, cmd_args: dict = None):
yaml.add_implicit_resolver('!env_var', _value_matcher, None,
yaml.FullLoader)
yaml.add_constructor('!env_var', _var_evaluator, yaml.FullLoader)
global _sys_arg_map
_sys_arg_map = cmd_args
with open(file_name, 'r') as yaml_file:
conf_data = yaml.load(yaml_file, Loader=yaml.FullLoader)
return conf_data
def __init__(self, project_dir, mode):
self.project_dir = project_dir
self.mode = mode
self.path_matcher = re.compile(r'\$\{([^}^{]+)\}')
yaml.add_implicit_resolver('!path', self.path_matcher, None, yaml.SafeLoader)
yaml.add_constructor('!path', self.path_constructor, yaml.SafeLoader)
with open(os.path.join(self.project_dir, 'config', 'config_%s.yaml' % (self.mode)), 'r') as ymlfile:
self.config = yaml.safe_load(ymlfile)
def _add_env_loader(self):
# pattern matcher for environment variables
env_pattern = re.compile(r'^\<%= ENV\[\'(.*)\'\] %\>(.*)$')
def env_constructor(loader, node):
"""PyYAML constructor for environment variables"""
value = loader.construct_scalar(node)
var, remaining = env_pattern.match(value).groups()
return os.environ.get(var) + remaining
yaml.add_implicit_resolver("!env", env_pattern)
yaml.add_constructor("!env", env_constructor)
def load_yaml(yaml_file, vault_secret=None):
"""
Load a YAML file into a python dictionary.
The YAML file can be fully encrypted by Ansible-Vault or can contain
multiple inline Ansible-Vault encrypted values. Ansible Vault
encryption is ideal to store passwords or encrypt the entire file
with sensitive data if required.
"""
vault = VaultLib()
if vault_secret:
secret_file = get_file_vault_secret(filename=vault_secret, loader=DataLoader())
secret_file.load()
vault.secrets = [('default', secret_file)]
# YAML ENV VAR
# name: !ENV env_var('FOO')/bar
yaml.add_implicit_resolver("!ENV", ENV_VAR_PATTERN)
yaml.add_constructor('!ENV', env_var_constructor)
data = None
if os.path.isfile(yaml_file):
with open(yaml_file, 'r') as stream:
# Render environment variables using jinja templates
contents = stream.read()
template = Template(contents)
stream = StringIO(template.render(env_var=os.environ))
try:
if is_encrypted_file(stream):
file_data = stream.read()
data = yaml.load(vault.decrypt(file_data, None))
else:
file_data = stream.read()
data = yaml.load(file_data, Loader=yaml.Loader)
'''
Commenting code below for posterity. We are not using ansible functionality but yaml load should
follow the same code path regardless of the file encryption state.
'''
#loader = AnsibleLoader(stream, None, vault.secrets)
#try:
# data = loader.get_single_data()
#except Exception as exc:
# raise Exception(f'Error when loading YAML config at {yaml_file} {exc}') from exc
#finally:
# loader.dispose()
except yaml.YAMLError as exc:
raise Exception(f'Error when loading YAML config at {yaml_file} {exc}') from exc
else:
LOGGER.debug('No file at %s', yaml_file)
return data
def parse_config(config_file):
"""Parse the YAML config"""
pattern = re.compile(r'^\<%= ENV\[\'(.*)\'\] %\>(.*)$')
yaml.add_implicit_resolver("!env", pattern)
def env_constructor(loader, node):
"""Constructor for environment variables"""
value = loader.construct_scalar(node)
env_var, remaining_path = pattern.match(value).groups()
return os.environ[env_var] + remaining_path
yaml.add_constructor('!env', env_constructor)
with open(config_file) as config:
return yaml.load(config)
def _setup_yaml_with_canonical_dict():
"""https://stackoverflow.com/a/8661021"""
yaml.add_representer(
OrderedDict,
lambda self, data: self.represent_mapping(
"tag:yaml.org,2002:map", data.items()
),
Dumper=yaml.SafeDumper,
)
def repr_ordered_dict(self, data):
return self.represent_mapping("tag:yaml.org,2002:map", data.items())
yaml.add_representer(dict, repr_ordered_dict, Dumper=yaml.SafeDumper)
def repr_dict(self, data):
return self.represent_mapping(
"tag:yaml.org,2002:map", sorted(data.items(), key=lambda t: t[0])
)
yaml.add_representer(dict, repr_dict, Dumper=yaml.SafeDumper)
# https://stackoverflow.com/a/45004464
def repr_str(dumper, data):
if "\n" in data:
return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
return dumper.represent_str(data)
yaml.add_representer(str, repr_str, Dumper=yaml.SafeDumper)
def repr_tuple(dumper, data):
return dumper.represent_sequence("tag:yaml.org,2002:seq", list(data))
yaml.add_representer(tuple, repr_tuple, Dumper=yaml.SafeDumper)
# loader = yaml.SafeLoader
yaml.add_implicit_resolver(
"tag:yaml.org,2002:float",
re.compile(
"""^(?:
[-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)?
|[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
|\\.[0-9_]+(?:[eE][-+][0-9]+)?
|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]*
|[-+]?\\.(?:inf|Inf|INF)
|\\.(?:nan|NaN|NAN))$""",
re.X,
),
list("-+0123456789."),
)
def _read_yaml_config(self, config_file):
# Fills in env var
env_regex = re.compile(r'\$\{(.*)\}')
def env_constructor(loader, node):
match = env_regex.match(node.value)
env_var = match.group()[2:-1]
return os.getenv(env_var) + node.value[match.end():]
yaml.add_implicit_resolver('!env', env_regex, None, yaml.SafeLoader)
yaml.add_constructor('!env', env_constructor, yaml.SafeLoader)
with open(config_file) as data_file:
loaded_config = yaml.safe_load(data_file)
return loaded_config
def load_config_file(cls, config_paths):
"""Load a yaml config file from path."""
config_path = ""
for possible_path in config_paths:
if not os.path.isfile(possible_path):
_LOGGER.debug(_("Config file %s not found."), possible_path)
else:
config_path = possible_path
break
if not config_path:
try:
move_config_to_appdir(PRE_0_12_0_ROOT_PATH, DEFAULT_ROOT_PATH)
except FileNotFoundError:
_LOGGER.info(_("No configuration files found. "
"Creating %s"), DEFAULT_CONFIG_PATH)
config_path = cls.create_default_config(DEFAULT_CONFIG_PATH)
env_var_pattern = re.compile(r'^\$([A-Z_]*)$')
yaml.add_implicit_resolver("!envvar", env_var_pattern)
def envvar_constructor(loader, node):
"""Yaml parser for env vars."""
value = loader.construct_scalar(node)
[env_var] = env_var_pattern.match(value).groups()
return os.environ[env_var]
def include_constructor(loader, node):
"""Add a yaml file to be loaded inside another."""
main_yaml_path = os.path.split(stream.name)[0]
included_yaml = os.path.join(main_yaml_path,
loader.construct_scalar(node))
with open(included_yaml, 'r') as included:
return yaml.safe_load(included)
yaml.add_constructor('!envvar', envvar_constructor)
yaml.add_constructor('!include', include_constructor)
try:
with open(config_path, 'r') as stream:
_LOGGER.info(_("Loaded config from %s."), config_path)
return yaml.load(stream, Loader=yaml.SafeLoader)
except yaml.YAMLError as error:
_LOGGER.critical(error)
sys.exit(1)
except FileNotFoundError as error:
_LOGGER.critical(error)
sys.exit(1)
def __require_tag(self, tag_name):
tag_cls = self._registry.get(tag_name)
if tag_cls.registered:
return True
tag_instance = tag_cls()
yaml.add_implicit_resolver(tag_instance.yaml_tag(),
tag_instance.yaml_pattern())
yaml.add_constructor(tag_instance.yaml_tag(), tag_instance.from_yaml)
yaml.add_representer(tag_cls, tag_instance.to_yaml)
tag_cls.registered = True
return True
def load_yaml(path):
# configure the yaml parser to allow grabbing OS environment variables in the config.
# TODO(plombardi:) Improve so that the default argument is optional
pattern = re.compile(r'^(.*)<%= ENV\[\'(.*)\',\'(.*)\'\] %>(.*)$')
yaml.add_implicit_resolver('!env_regex', pattern)
def env_regex(loader, doc_node):
value = loader.construct_scalar(doc_node)
front, variable_name, default, back = pattern.match(value).groups()
return str(front) + os.getenv(variable_name, default) + str(back)
yaml.add_constructor('!env_regex', env_regex)
with open(path, 'r') as stream:
return yaml.load(stream)
def _parse(data: str):
path_matcher = re.compile(r'.*(\$\{([^}^{]+)\}).*')
def path_constructor(_, node):
value = node.value
match = path_matcher.match(value)
env_var = match.groups()[1]
if env_var not in os.environ:
raise ffd.ConfigurationError(f'Environment variable {env_var} is used in config, but is not set')
return str(value).replace(f'${{{env_var}}}', os.environ.get(env_var))
yaml.add_implicit_resolver('!path', path_matcher, None, yaml.SafeLoader)
yaml.add_constructor('!path', path_constructor, yaml.SafeLoader)
return yaml.safe_load(data)
def replace_environment_variables():
"""Enable yaml loader to process the environment variables in the yaml."""
import re
import os
env_var_pattern = re.compile(r"^(.*)\$\{(.*)\}(.*)$")
yaml.add_implicit_resolver('!env_var', env_var_pattern)
def env_var_constructor(loader, node):
"""Process environment variables found in the YAML."""
value = loader.construct_scalar(node)
prefix, env_var, postfix = env_var_pattern.match(value).groups()
return prefix + os.environ[env_var] + postfix
yaml.add_constructor(u'!env_var', env_var_constructor)
def __init__(self):
config_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'config.yml'))
config_file = open(config_path)
self.env_matcher = re.compile(r'(.*)\${([^}^{]+)}(.*)')
yaml.add_implicit_resolver('!path', self.env_matcher, None,
yaml.SafeLoader)
yaml.add_constructor('!path', self.config_constructor, yaml.SafeLoader)
self.parsed_config_file = yaml.load(config_file,
Loader=yaml.SafeLoader)
self.deployments = self.parsed_config_file['scaler']
self.hosts = self.parsed_config_file['server']['hosts']
self.username = self.parsed_config_file['server']['username']
self.password = self.parsed_config_file['server']['password']
def init_yaml_env_vars():
"""Initializes the yaml parser to be able to set
the value of fields from environment variables
:rtype: None
"""
env_rx = re.compile(r'\$\{[^}]+\}')
yaml.add_implicit_resolver('!envvar', env_rx)
def envvar_constructor(loader, node):
value = loader.construct_scalar(node)
value = os.path.expandvars(value)
return value
yaml.add_constructor('!envvar', envvar_constructor)
def load_config(path):
PROJECT_ROOT = pathlib.Path(__file__).parent.parent
var_pattern = re.compile(r'^(.*)?\$\{(.*)\}(.*)$')
yaml.add_implicit_resolver("!vars", var_pattern)
VARS = {'PROJECT_ROOT': PROJECT_ROOT}
def _yaml_vars_constructor(loader, node):
value = loader.construct_scalar(node)
prefix, var_name, postfix = var_pattern.match(value).groups()
return f'{prefix}{VARS.get(var_name)}{postfix}'
yaml.add_constructor('!vars', _yaml_vars_constructor)
with open(f'{PROJECT_ROOT}/{path}', 'r') as f:
return yaml.load(f)
def parse_config(filename):
pattern = re.compile('^"?\\$\\{([^}^{]+)\\}"?$')
def _path_constructor(loader, node):
value = node.value
match = pattern.match(value)
env_var = match.group().strip('"${}')
return os.environ.get(env_var) + value[match.end():]
yaml.add_implicit_resolver('env', pattern, None, yaml.SafeLoader)
yaml.add_constructor('env', _path_constructor, yaml.SafeLoader)
with open(filename, "r") as f:
cfg = yaml.load(f, Loader=yaml.SafeLoader)
conf = Config(cfg)
return conf
def parse_config(filename):
""" parses the YAML config file and expands any environment variables """
pattern = re.compile(r'^\<%= ENV\[\'(.*)\'\] %\>(.*)$')
yaml.add_implicit_resolver("!pathex", pattern)
def pathex_constructor(loader, node):
value = loader.construct_scalar(node)
envVar, remainingPath = pattern.match(value).groups()
return os.environ[envVar] + remainingPath
yaml.add_constructor('!pathex', pathex_constructor)
with open(filename) as f:
return(
yaml.load(f)
)
def _readConfig():
nameConfig = utils.getConfig()
fileOut = 'configurations/{0}.yaml'.format(nameConfig)
yaml.add_implicit_resolver ( "!pathex", pattern )
yaml.add_constructor('!pathex', pathex_constructor)
with open('configurations/default.yaml') as stream:
R = yaml.load(stream)
with open(fileOut, 'r') as stream:
Rb = yaml.load(stream)
R = dict(mergedicts(R,Rb))
return R
def initialize():
"""
Initialize the configuration system by installing YAML handlers.
Automatically done on first call to load() specified in this file.
"""
global is_initialized
# Add the custom multi-constructor
yaml.add_multi_constructor('!obj:', multi_constructor_obj)
yaml.add_multi_constructor('!pkl:', multi_constructor_pkl)
yaml.add_multi_constructor('!import:', multi_constructor_import)
yaml.add_constructor('!import', constructor_import)
yaml.add_constructor("!float", constructor_float)
pattern = re.compile(SCIENTIFIC_NOTATION_REGEXP)
yaml.add_implicit_resolver('!float', pattern)
is_initialized = True
def initialize():
"""
Initialize the configuration system by installing YAML handlers.
Automatically done on first call to load() specified in this file.
"""
global is_initialized
# Add the custom multi-constructor
yaml.add_multi_constructor("!obj:", multi_constructor_obj)
yaml.add_multi_constructor("!pkl:", multi_constructor_pkl)
yaml.add_multi_constructor("!import:", multi_constructor_import)
yaml.add_constructor("!import", constructor_import)
yaml.add_implicit_resolver("!import", re.compile(r"(?:[a-zA-Z_][\w_]+\.)+[a-zA-Z_][\w_]+"))
yaml.add_constructor("!float", constructor_float)
yaml.add_implicit_resolver("!float", re.compile(r" [-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?"))
is_initialized = True
def load_config(self, path, environments, fill_with_defaults=False):
"""Will load default.yaml and <environment>.yaml at given path.
The environment config will override the default values.
:param path: directory where to find your config files. If the last character is not a slash (/) it will be appended. Example: resources/
:param environments: list of environment configs to load. File name pattern: <environment>.yaml. Example: develop.yaml. Latter configs will override previous ones.
:param fill_with_defaults: use 'defaults' keyword in config file to fill up following config entrys.
:return: your config as dictionary.
"""
yaml.add_implicit_resolver("!environ", self.__environ_pattern)
yaml.add_constructor('!environ', self.__get_from_environment)
yaml.add_implicit_resolver("!vault", self.__vault_pattern)
yaml.add_constructor('!vault', self.__get_from_vault)
if not path.endswith('/'):
path += '/'
if type(environments) != list:
environments = [environments]
config = {}
try:
for env in environments:
with open(path + env + '.yaml', 'r') as configFile:
env_config = yaml.load(configFile.read()) or {}
config.update(env_config)
if fill_with_defaults:
if 'defaults' in config:
defaults = config['defaults']
for target in defaults:
for index, item in enumerate(config[target]):
tmp = defaults[target].copy()
tmp.update(config[target][index])
config[target][index] = tmp
return config
except exceptions.VaultError as error:
raise ConfigLoaderError("Could not read vault secrets [" + error.__class__.__name__ + "]")
except yaml.YAMLError as error:
raise ConfigLoaderError("Configuration files malformed [" + error.__class__.__name__ + "]")
except json.decoder.JSONDecodeError as error:
raise ConfigLoaderError("Vault response was not json [" + error.__class__.__name__ + "]")
except Exception as error:
raise ConfigLoaderError("WTF? [" + error.__class__.__name__ + "]")
def initialize():
"""
Initialize the configuration system by installing YAML handlers.
Automatically done on first call to load() specified in this file.
"""
global is_initialized
# Add the custom multi-constructor
yaml.add_multi_constructor('!obj:', multi_constructor)
yaml.add_multi_constructor('!pkl:', multi_constructor_pkl)
yaml.add_multi_constructor('!import:', multi_constructor_import)
def import_constructor(loader, node):
value = loader.construct_scalar(node)
return try_to_import(value)
yaml.add_constructor('!import', import_constructor)
yaml.add_implicit_resolver(
'!import',
re.compile(r'(?:[a-zA-Z_][\w_]+\.)+[a-zA-Z_][\w_]+')
)
is_initialized = True
def uuid_add_implicit_resolver(Loader=ConfigLoader, Dumper=ConfigDumper):
"""
Attach an implicit pattern resolver for UUID objects.
Tests:
>>> class TestDumper(yaml.SafeDumper): pass
>>> class TestLoader(yaml.SafeLoader): pass
>>> TUUID = 'cc3702ca-699a-4aa6-8226-4c938f294d9b'
>>> IN = {'test': uuid.UUID(TUUID)}
>>> OUT = '{test: cc3702ca-699a-4aa6-8226-4c938f294d9b}'
>>> yaml.add_representer(uuid.UUID, uuid_representer, Dumper=TestDumper)
>>> yaml.add_constructor('!uuid', uuid_constructor, Loader=TestLoader)
>>> uuid_add_implicit_resolver(Loader=TestLoader, Dumper=TestDumper)
>>> yaml.dump(IN, Dumper=TestDumper)
'test: cc3702ca-699a-4aa6-8226-4c938f294d9b\\n'
>>> yaml.load(OUT, Loader=TestLoader)
{'test': UUID('cc3702ca-699a-4aa6-8226-4c938f294d9b')}
"""
uuid_regex = r'^\b[a-f0-9]{8}-\b[a-f0-9]{4}-\b[a-f0-9]{4}-\b[a-f0-9]{4}-\b[a-f0-9]{12}$'
pattern = re.compile(uuid_regex)
yaml.add_implicit_resolver('!uuid', pattern, Loader=Loader, Dumper=Dumper)
"Output variables in YAML as ?name."
return dumper.represent_scalar(u"!var", u"?%s" % var.name())
yaml.add_representer(Variable, variable_representer)
def variable_constructor(loader, node):
"Recognize variables written as ?name in YAML."
value = loader.construct_scalar(node)
name = value[1:]
return Variable(name)
yaml.add_constructor(u"!var", variable_constructor)
yaml.add_implicit_resolver(u"!var", re.compile(r"^\?\w+$"))
def _copy_and_bind(feature, bindings, memo=None):
"""
Make a deep copy of a feature structure, preserving reentrance using the
C{memo} dictionary. Meanwhile, variables are replaced by their bound
values, if these values are already known, and variables with unknown
values are given placeholder bindings.
"""
if memo is None:
memo = {}
if id(feature) in memo:
return memo[id(feature)]
if isinstance(feature, Variable) and bindings is not None:
if not bindings.has_key(feature.name()):
def setup_yaml_parser():
yaml.add_constructor('!env_var', _env_var_constructor)
yaml.add_implicit_resolver('!env_var', IMPLICIT_ENV_VAR_MATCHER)
import re
import yaml
JINJA = yaml.add_implicit_resolver(
u'tag:yaml.org,2002:jinja',
re.compile('{{.*?}}')
)
import os.path
import yaml
import dsl
import dsl.typespec
from dsl.yaql_expression import YaqlExpression
import system.yaql_functions
def yaql_constructor(loader, node):
value = loader.construct_scalar(node)
return YaqlExpression(value)
yaml.add_constructor(u'!yaql', yaql_constructor)
yaml.add_implicit_resolver(u'!yaql', YaqlExpression)
class ClassLoader(dsl.MuranoClassLoader):
def __init__(self, base_path):
self._base_path = base_path
super(ClassLoader, self).__init__()
def load_definition(self, name):
path = os.path.join(self._base_path, name, 'manifest.yaml')
if not os.path.exists(path):
return None
with open(path) as stream:
return yaml.load(stream)
def create_root_context(self):
context = super(ClassLoader, self).create_root_context()
# Copyright (c) 2010 A. Frederick Dudley. All rights reserved.
#
"""This code converts game objects to and from yaml streams"""
import re
import yaml
from binary_tactics.grid import Loc
from stores.store import convert_dict
def loc_representer(dumper, data):
return dumper.represent_scalar(u'!loc', u'(%2s,%2s)' % data)
yaml.add_representer(Loc, loc_representer)
def loc_constructor(loader, node):
value = loader.construct_scalar(node)
try:
x, y = map(int, value.strip('()').split(','))
except ValueError: #This isn't so bad.
x = y = None
return Loc(x,y)
yaml.add_constructor(u'!loc', loc_constructor)
loc_pat = re.compile(r'^\(\s*\d+\s*,\s*\d+\s*\)\s*$')
none_pat = re.compile(r'^\(\s*None\s*,\s*None\s*\)\s*$')
yaml.add_implicit_resolver(u'!loc', loc_pat)
yaml.add_implicit_resolver(u'!loc', none_pat)
def load(filename):
return convert_dict(yaml.load(open(filename, 'r')))
from muranoapi.packages import exceptions as pkg_exc
LOG = logging.getLogger(__name__)
class YaqlYamlLoader(yaml.Loader):
pass
def yaql_constructor(loader, node):
value = loader.construct_scalar(node)
return yaql_expression.YaqlExpression(value)
yaml.add_constructor(u"!yaql", yaql_constructor, YaqlYamlLoader)
yaml.add_implicit_resolver(u"!yaql", yaql_expression.YaqlExpression, Loader=YaqlYamlLoader)
class PackageLoader(six.with_metaclass(abc.ABCMeta)):
@abc.abstractmethod
def get_package(self, name):
pass
@abc.abstractmethod
def get_package_by_class(self, name):
pass
class ApiPackageLoader(PackageLoader):
def __init__(self, token_id, tenant_id):
self._cache_directory = self._get_cache_directory()
import yaml, os, re
#define the regex pattern that the parser will use to 'implicitely' tag your node
pattern = re.compile( r'^\<%= ENV\[\'(.*)\'\] %\>(.*)$' )
#now define a custom tag ( say pathex ) and associate the regex pattern we defined
yaml.add_implicit_resolver ( "!pathex", pattern )
#at this point the parser will associate '!pathex' tag whenever the node matches the pattern
#you need to now define a constructor that the parser will invoke
#you can do whatever you want with the node value
def pathex_constructor(loader,node):
value = loader.construct_scalar(node)
envVar, remainingPath = pattern.match(value).groups()
return os.environ[envVar] + remainingPath
#'register' the constructor so that the parser will invoke 'pathex_constructor' for each node '!pathex'
yaml.add_constructor('!pathex', pathex_constructor)
#that is it
data = """
version: 1
disable_existing_loggers: False
formatters:
precise:
format: "%(name)-15s # %(levelname)-8s # %(asctime)s # [Line: %(lineno)-3d]: %(message)s"
datefmt: "%Y-%m-%d %H:%M:%S"
handlers:
file:
argv = sys.argv
if len(argv) == 1:
_fullusage()
try:
command = argv[1]
if command in ["get", "dump"]:
_rosparam_cmd_get_dump(command, argv)
elif command in ["set", "load"]:
_rosparam_cmd_set_load(command, argv)
elif command in ["delete"]:
_rosparam_cmd_delete(argv)
elif command == "list":
_rosparam_cmd_list(argv)
else:
_fullusage()
except ROSParamException, e:
print >> sys.stderr, "ERROR: " + str(e)
sys.exit(1)
# YAML configuration. Doxygen does not like these being higher up in the code
yaml.add_constructor(u"!radians", construct_angle_radians)
yaml.add_constructor(u"!degrees", construct_angle_degrees)
# allow both !degrees 180, !radians 2*pi
pattern = re.compile(r"^deg\([^\)]*\)$")
yaml.add_implicit_resolver(u"!degrees", pattern, first="deg(")
pattern = re.compile(r"^rad\([^\)]*\)$")
yaml.add_implicit_resolver(u"!radians", pattern, first="rad(")
argv = sys.argv
if len(argv) == 1:
_fullusage()
try:
command = argv[1]
if command in ['get', 'dump']:
_rosparam_cmd_get_dump(command, argv)
elif command in ['set', 'load']:
_rosparam_cmd_set_load(command, argv)
elif command in ['delete']:
_rosparam_cmd_delete(argv)
elif command == 'list':
_rosparam_cmd_list(argv)
else:
_fullusage()
except RosParamException as e:
print("ERROR: "+str(e), file=sys.stderr)
sys.exit(1)
# YAML configuration. Doxygen does not like these being higher up in the code
yaml.add_constructor(u'!radians', construct_angle_radians)
yaml.add_constructor(u'!degrees', construct_angle_degrees)
# allow both !degrees 180, !radians 2*pi
pattern = re.compile(r'^deg\([^\)]*\)$')
yaml.add_implicit_resolver(u'!degrees', pattern, first="deg(")
pattern = re.compile(r'^rad\([^\)]*\)$')
yaml.add_implicit_resolver(u'!radians', pattern, first="rad(")
return el
value = loader.construct_scalar(node)
# remove the ( ) from the string
tup_elements = [ele.strip() for ele in value[1:-1].split(',')]
# remove the last element if the tuple was written as (x,b,)
if tup_elements[-1] == '':
tup_elements.pop(-1)
tup = tuple(map(parse_tup_el, tup_elements))
return tup
# !tuple is my own tag name, I think you could choose anything you want
yaml.add_constructor(u'!tuple', _yml_tuple_constructor)
# this is to spot the strings written as tuple in the yaml
yaml.add_implicit_resolver(u'!tuple', re.compile(r"\((.*?,.*?)\)"))
def parse_yml_file(fname: str) -> Dict:
"""parse_yml_file reads the given file and parses its content as YML.
Parameters
----------
fname : str
fname is the of the file to parse.
Returns
-------
content : Dict
The content of the file as dictionary.
"""
yaml.add_constructor(u'!include', include_constructor)
def range_constructor(loader, node):
data = loader.construct_scalar(node)
match = range_regex.match(data)
if match:
start, stop = map(int, match.groups())
return range(start, stop + 1)
else:
raise ValueError('Invalid range: %s' % data)
range_regex = re.compile(r'^\s*(\d+)\s*\.\.\s*(\d+)\s*$')
yaml.add_constructor(u'!range', range_constructor)
yaml.add_implicit_resolver(u'!range', range_regex)
def refer_constructor(loader, node):
def getRefer(content, keys):
content = content.replace('!refer ', '')
ref = yaml.load(content)
for k in keys:
ref = ref.get(k)
return ref
data = loader.construct_scalar(node)
keys = data.split('.')
offset = loader.stream.tell()
loader.stream.seek(0)
content = loader.stream.read()
return os.environ[var]
def connect(db_config):
return pymysql.connect(
charset=db_config['charset'],
cursorclass=pymysql.cursors.DictCursor,
db=db_config['database'],
host=db_config['host'],
password=db_config['password'],
port=3306,
user=db_config['username']
)
if __name__ == "__main__":
pattern = re.compile(r'^<%= ENV\[\'(.*)\'\] %>$')
yaml.add_implicit_resolver('!env_regex', pattern)
def env_regex(loader, node):
value = loader.construct_scalar(node)
var = pattern.match(value).groups()[0]
print "VAR is %s" % var
return os.environ[var]
yaml.add_constructor('!env_regex', env_regex)
passed_args = sys.argv
with open(passed_args[2], 'r') as f:
config = yaml.load(f)
# only using debug=True for the nice auto-reload on change feature
environ = passed_args[1]
def __str__(self):
if self.comment:
return '"%s" <%s@%s>' % (self.comment,
self.user,
self.domain)
else:
return '<%s@%s>' % (self.user, self.domain)
def __repr__(self):
return "%s('%s')" % (self.__class__.__name__,
self.__str__())
def __eq__(self, other):
if isinstance(other, EmailAddress):
if (other.user == self.user and
other.domain == self.domain):
return True
return False
def __ne__(self, other):
if isinstance(other, EmailAddress):
if (other.user != self.user and
other.domain != self.domain):
return True
return False
# Make sure that when we emit stuff taht the email addresses show up properly
# without extra annoying tags.
yaml.add_implicit_resolver(EmailAddress.yaml_tag, EMAIL_RE)
if re.match('^[\s\w\d.:]*$', expr):
return False
try:
yaql.parse(expr)
return True
except yaql.exceptions.YaqlGrammarException:
return False
except yaql.exceptions.YaqlLexicalException:
return False
def evaluate(self, data=None, context=None):
return self._parsed_expression.evaluate(data=data, context=context)
class YaqlYamlLoader(yaml.Loader):
pass
# workaround for PyYAML bug: http://pyyaml.org/ticket/221
resolvers = {}
for k, v in yaml.Loader.yaml_implicit_resolvers.items():
resolvers[k] = v[:]
YaqlYamlLoader.yaml_implicit_resolvers = resolvers
def yaql_constructor(loader, node):
value = loader.construct_scalar(node)
return YaqlExpression(value)
yaml.add_constructor(u'!yaql', yaql_constructor, YaqlYamlLoader)
yaml.add_implicit_resolver(u'!yaql', YaqlExpression, Loader=YaqlYamlLoader)
is always off, and there is no blank line at the end.
"""
return yaml.dump(data, default_flow_style=False).strip()
def variable_representer(dumper, var):
"Output variables in YAML as ?name."
return dumper.represent_scalar(u'!var', u'?%s' % var.name())
yaml.add_representer(Variable, variable_representer)
def variable_constructor(loader, node):
"Recognize variables written as ?name in YAML."
value = loader.construct_scalar(node)
name = value[1:]
return Variable(name)
yaml.add_constructor(u'!var', variable_constructor)
yaml.add_implicit_resolver(u'!var', re.compile(r'^\?\w+$'))
def _copy_and_bind(feature, bindings, memo=None):
"""
Make a deep copy of a feature structure, preserving reentrance using the
C{memo} dictionary. Meanwhile, variables are replaced by their bound
values, if these values are already known, and variables with unknown
values are given placeholder bindings.
"""
if memo is None: memo = {}
if id(feature) in memo: return memo[id(feature)]
if isinstance(feature, Variable) and bindings is not None:
if not bindings.has_key(feature.name()):
bindings[feature.name()] = feature.copy()
result = _copy_and_bind(bindings[feature.name()], None, memo)
else:
