def find_evaluation_parameter_dependencies(parameter_expression):
"""Parse a parameter expression to identify dependencies including GE URNs.
Args:
parameter_expression: the parameter to parse
Returns:
a dictionary including:
- "urns": set of strings that are valid GE URN objects
- "other": set of non-GE URN strings that are required to evaluate the parameter expression
"""
expr = EvaluationParameterParser()
dependencies = {"urns": set(), "other": set()}
# Calling get_parser clears the stack
parser = expr.get_parser()
try:
_ = parser.parseString(parameter_expression, parseAll=True)
except ParseException as err:
raise EvaluationParameterError(
f"Unable to parse evaluation parameter: {str(err)} at line {err.line}, column {err.column}"
)
except AttributeError as err:
raise EvaluationParameterError(
f"Unable to parse evaluation parameter: {str(err)}"
)
for word in expr.exprStack:
if isinstance(word, (int, float)):
continue
if not isinstance(word, str):
# If we have a function that itself is a tuple (e.g. (trunc, 1))
continue
if word in expr.opn or word in expr.fn or word == "unary -":
# operations and functions
continue
# if this is parseable as a number, then we do not include it
try:
_ = float(word)
continue
except ValueError:
pass
try:
_ = ge_urn.parseString(word)
dependencies["urns"].add(word)
continue
except ParseException:
# This particular evaluation_parameter or operator is not a valid URN
pass
# If we got this far, it's a legitimate "other" evaluation parameter
dependencies["other"].add(word)
return dependencies
def build_evaluation_parameters(expectation_args,
evaluation_parameters=None,
interactive_evaluation=True):
"""Build a dictionary of parameters to evaluate, using the provided evaluation_parameters,
AND mutate expectation_args by removing any parameter values passed in as temporary values during
exploratory work.
"""
evaluation_args = copy.deepcopy(expectation_args)
# Iterate over arguments, and replace $PARAMETER-defined args with their
# specified parameters.
for key, value in evaluation_args.items():
if isinstance(value, dict) and "$PARAMETER" in value:
# We do not even need to search for a value if we are not going to do interactive evaluation
if not interactive_evaluation:
continue
# First, check to see whether an argument was supplied at runtime
# If it was, use that one, but remove it from the stored config
if "$PARAMETER." + value["$PARAMETER"] in value:
evaluation_args[key] = evaluation_args[key][
"$PARAMETER." + value["$PARAMETER"]]
del expectation_args[key]["$PARAMETER." + value["$PARAMETER"]]
elif evaluation_parameters is not None:
# parse_evaluation_parameter will raise EvaluationParameterError if we cannot find a suitable value
evaluation_args[key] = parse_evaluation_parameter(
value["$PARAMETER"], evaluation_parameters)
else:
# If we haven't been able to continue already, we failed to find a parameter
raise EvaluationParameterError(
"No value found for $PARAMETER " + value["$PARAMETER"])
return evaluation_args
def parse_evaluation_parameter(parameter_expression,
evaluation_parameters=None):
"""Use the provided evaluation_parameters dict to parse a given parameter expression.
Args:
parameter_expression (str): A string, potentially containing basic arithmetic operations and functions,
and variables to be substituted
evaluation_parameters (dict): A dictionary of name-value pairs consisting of values to substitute
The parser will allow arithmetic operations +, -, /, *, as well as basic functions, including trunc() and round() to
obtain integer values when needed for certain expectations (e.g. expect_column_value_length_to_be_between).
Valid variables must begin with an alphabetic character and may contain alphanumeric characters plus '_' and '$',
EXCEPT if they begin with the string "urn:great_expectations" in which case they may also include additional
characters to support inclusion of GE URLs (see :ref:`evaluation_parameters` for more information).
"""
if evaluation_parameters is None:
evaluation_parameters = {}
# Calling get_parser clears the stack
parser = expr.get_parser()
try:
L = parser.parseString(parameter_expression, parseAll=True)
except ParseException as err:
L = [
"Parse Failure", parameter_expression,
(str(err), err.line, err.column)
]
if len(L) == 1 and L[0] not in evaluation_parameters:
# In this special case there were no operations to find, so only one value, but we don't have something to
# substitute for that value
raise EvaluationParameterError("No value found for $PARAMETER " +
str(L[0]))
elif len(L) == 1:
# In this case, we *do* have a substitution for a single type. We treat this specially because in this
# case, we allow complex type substitutions (i.e. do not coerce to string as part of parsing)
return evaluation_parameters[L[0]]
elif len(L) == 0 or L[0] != "Parse Failure":
for i, ob in enumerate(expr.exprStack):
if isinstance(ob, str) and ob in evaluation_parameters:
expr.exprStack[i] = str(evaluation_parameters[ob])
else:
err_str, err_line, err_col = L[-1]
raise EvaluationParameterError(
f"Parse Failure: {err_str}\nStatement: {err_line}\nColumn: {err_col}"
)
try:
result = expr.evaluate_stack(expr.exprStack)
except Exception as e:
exception_traceback = traceback.format_exc()
exception_message = (
f'{type(e).__name__}: "{str(e)}". Traceback: "{exception_traceback}".'
)
logger.debug(exception_message, e, exc_info=True)
raise EvaluationParameterError(
"Error while evaluating evaluation parameter expression: " +
str(e))
return result
def parse_evaluation_parameter(
parameter_expression: str,
evaluation_parameters: Optional[Dict[str, Any]] = None,
data_context: Optional[
Any] = None, # Cannot type 'DataContext' due to import cycle
) -> Any:
"""Use the provided evaluation_parameters dict to parse a given parameter expression.
Args:
parameter_expression (str): A string, potentially containing basic arithmetic operations and functions,
and variables to be substituted
evaluation_parameters (dict): A dictionary of name-value pairs consisting of values to substitute
data_context (DataContext): A data context to use to obtain metrics, if necessary
The parser will allow arithmetic operations +, -, /, *, as well as basic functions, including trunc() and round() to
obtain integer values when needed for certain expectations (e.g. expect_column_value_length_to_be_between).
Valid variables must begin with an alphabetic character and may contain alphanumeric characters plus '_' and '$',
EXCEPT if they begin with the string "urn:great_expectations" in which case they may also include additional
characters to support inclusion of GE URLs (see :ref:`evaluation_parameters` for more information).
"""
if evaluation_parameters is None:
evaluation_parameters = {}
# Calling get_parser clears the stack
parser = expr.get_parser()
try:
L = parser.parseString(parameter_expression, parseAll=True)
except ParseException as err:
L = [
"Parse Failure", parameter_expression,
(str(err), err.line, err.column)
]
# Represents a valid parser result of a single function that has no arguments
if len(L) == 1 and isinstance(L[0], tuple) and L[0][2] is False:
# Necessary to catch `now()` (which only needs to be evaluated with `expr.exprStack`)
# NOTE: 20211122 - Chetan - Any future built-ins that are zero arity functions will match this behavior
pass
elif len(L) == 1 and L[0] not in evaluation_parameters:
# In this special case there were no operations to find, so only one value, but we don't have something to
# substitute for that value
try:
res = ge_urn.parseString(L[0])
if res["urn_type"] == "stores":
store = data_context.stores.get(res["store_name"])
return store.get_query_result(res["metric_name"],
res.get("metric_kwargs", {}))
else:
logger.error(
"Unrecognized urn_type in ge_urn: must be 'stores' to use a metric store."
)
raise EvaluationParameterError(
"No value found for $PARAMETER " + str(L[0]))
except ParseException as e:
logger.debug(
f"Parse exception while parsing evaluation parameter: {str(e)}"
)
raise EvaluationParameterError("No value found for $PARAMETER " +
str(L[0]))
except AttributeError:
logger.warning(
"Unable to get store for store-type valuation parameter.")
raise EvaluationParameterError("No value found for $PARAMETER " +
str(L[0]))
elif len(L) == 1:
# In this case, we *do* have a substitution for a single type. We treat this specially because in this
# case, we allow complex type substitutions (i.e. do not coerce to string as part of parsing)
# NOTE: 20201023 - JPC - to support MetricDefinition as an evaluation parameter type, we need to handle that
# case here; is the evaluation parameter provided here in fact a metric definition?
return evaluation_parameters[L[0]]
elif len(L) == 0 or L[0] != "Parse Failure":
for i, ob in enumerate(expr.exprStack):
if isinstance(ob, str) and ob in evaluation_parameters:
expr.exprStack[i] = str(evaluation_parameters[ob])
else:
err_str, err_line, err_col = L[-1]
raise EvaluationParameterError(
f"Parse Failure: {err_str}\nStatement: {err_line}\nColumn: {err_col}"
)
try:
result = expr.evaluate_stack(expr.exprStack)
result = convert_to_json_serializable(result)
except Exception as e:
exception_traceback = traceback.format_exc()
exception_message = (
f'{type(e).__name__}: "{str(e)}". Traceback: "{exception_traceback}".'
)
logger.debug(exception_message, e, exc_info=True)
raise EvaluationParameterError(
"Error while evaluating evaluation parameter expression: " +
str(e))
return result