def substitute_config_variable(template_str,
config_variables_dict,
dollar_sign_escape_string: str = r"\$"):
"""
This method takes a string, and if it contains a pattern ${SOME_VARIABLE} or $SOME_VARIABLE,
returns a string where the pattern is replaced with the value of SOME_VARIABLE,
otherwise returns the string unchanged. These patterns are case sensitive. There can be multiple
patterns in a string, e.g. all 3 will be substituted in the following:
$SOME_VARIABLE${some_OTHER_variable}$another_variable
If the environment variable SOME_VARIABLE is set, the method uses its value for substitution.
If it is not set, the value of SOME_VARIABLE is looked up in the config variables store (file).
If it is not found there, the input string is returned as is.
If the value to substitute is not a string, it is returned as-is.
If the value to substitute begins with dollar_sign_escape_string it is not substituted.
:param template_str: a string that might or might not be of the form ${SOME_VARIABLE}
or $SOME_VARIABLE
:param config_variables_dict: a dictionary of config variables. It is loaded from the
config variables store (by default, "uncommitted/config_variables.yml file)
:param dollar_sign_escape_string: a string that will be used in place of a `$` when substitution
is not desired.
:return: a string with values substituted, or the same object if template_str is not a string.
"""
if template_str is None:
return template_str
# 1. Make substitutions for non-escaped patterns
try:
match = re.finditer(
r"(?<!\\)\$\{(.*?)\}|(?<!\\)\$([_a-zA-Z][_a-zA-Z0-9]*)",
template_str)
except TypeError:
# If the value is not a string (e.g., a boolean), we should return it as is
return template_str
for m in match:
# Match either the first group e.g. ${Variable} or the second e.g. $Variable
config_variable_name = m.group(1) or m.group(2)
config_variable_value = config_variables_dict.get(config_variable_name)
if config_variable_value is not None:
if not isinstance(config_variable_value, str):
return config_variable_value
template_str = template_str.replace(m.group(),
config_variable_value)
else:
raise MissingConfigVariableError(
f"""\n\nUnable to find a match for config substitution variable: `{config_variable_name}`.
Please add this missing variable to your `uncommitted/config_variables.yml` file or your environment variables.
See https://great-expectations.readthedocs.io/en/latest/reference/data_context_reference.html#managing-environment-and-secrets""",
missing_config_variable=config_variable_name,
)
# 2. Replace the "$"'s that had been escaped
return template_str.replace(dollar_sign_escape_string, "$")
def substitute_config_variable(template_str, config_variables_dict):
"""
This method takes a string, and if it contains a pattern ${SOME_VARIABLE} or $SOME_VARIABLE,
returns a string where the pattern is replaced with the value of SOME_VARIABLE,
otherwise returns the string unchanged.
If the environment variable SOME_VARIABLE is set, the method uses its value for substitution.
If it is not set, the value of SOME_VARIABLE is looked up in the config variables store (file).
If it is not found there, the input string is returned as is.
:param template_str: a string that might or might not be of the form ${SOME_VARIABLE}
or $SOME_VARIABLE
:param config_variables_dict: a dictionary of config variables. It is loaded from the
config variables store (by default, "uncommitted/config_variables.yml file)
:return:
"""
if template_str is None:
return template_str
try:
match = re.search(r"\$\{(.*?)\}", template_str) or re.search(
r"\$([_a-z][_a-z0-9]*)", template_str)
except TypeError:
# If the value is not a string (e.g., a boolean), we should return it as is
return template_str
if match:
config_variable_value = config_variables_dict.get(match.group(1))
try:
inner_match = re.search(
r"\$\{(.*?)\}", config_variable_value) or re.search(
r"\$([_a-z][_a-z0-9]*)", config_variable_value)
except TypeError:
inner_match = None
if inner_match:
config_variable_value = os.getenv(inner_match.group(1))
if config_variable_value is not None:
if match.start() == 0 and match.end() == len(template_str):
return config_variable_value
else:
return (template_str[:match.start()] + config_variable_value +
template_str[match.end():])
raise MissingConfigVariableError(
f"""\n\nUnable to find a match for config substitution variable: `{match.group(1)}`.
Please add this missing variable to your `uncommitted/config_variables.yml` file or your environment variables.
See https://great-expectations.readthedocs.io/en/latest/reference/data_context_reference.html#managing-environment-and-secrets""",
missing_config_variable=match.group(1),
)
return template_str