def find_evaluation_parameter_dependencies(parameter_expression): """Parse a parameter expression to identify dependencies including GE URNs. Args: parameter_expression: the parameter to parse Returns: a dictionary including: - "urns": set of strings that are valid GE URN objects - "other": set of non-GE URN strings that are required to evaluate the parameter expression """ expr = EvaluationParameterParser() dependencies = {"urns": set(), "other": set()} # Calling get_parser clears the stack parser = expr.get_parser() try: _ = parser.parseString(parameter_expression, parseAll=True) except ParseException as err: raise EvaluationParameterError( f"Unable to parse evaluation parameter: {str(err)} at line {err.line}, column {err.column}" ) except AttributeError as err: raise EvaluationParameterError( f"Unable to parse evaluation parameter: {str(err)}" ) for word in expr.exprStack: if isinstance(word, (int, float)): continue if not isinstance(word, str): # If we have a function that itself is a tuple (e.g. (trunc, 1)) continue if word in expr.opn or word in expr.fn or word == "unary -": # operations and functions continue # if this is parseable as a number, then we do not include it try: _ = float(word) continue except ValueError: pass try: _ = ge_urn.parseString(word) dependencies["urns"].add(word) continue except ParseException: # This particular evaluation_parameter or operator is not a valid URN pass # If we got this far, it's a legitimate "other" evaluation parameter dependencies["other"].add(word) return dependencies
def build_evaluation_parameters(expectation_args, evaluation_parameters=None, interactive_evaluation=True): """Build a dictionary of parameters to evaluate, using the provided evaluation_parameters, AND mutate expectation_args by removing any parameter values passed in as temporary values during exploratory work. """ evaluation_args = copy.deepcopy(expectation_args) # Iterate over arguments, and replace $PARAMETER-defined args with their # specified parameters. for key, value in evaluation_args.items(): if isinstance(value, dict) and "$PARAMETER" in value: # We do not even need to search for a value if we are not going to do interactive evaluation if not interactive_evaluation: continue # First, check to see whether an argument was supplied at runtime # If it was, use that one, but remove it from the stored config if "$PARAMETER." + value["$PARAMETER"] in value: evaluation_args[key] = evaluation_args[key][ "$PARAMETER." + value["$PARAMETER"]] del expectation_args[key]["$PARAMETER." + value["$PARAMETER"]] elif evaluation_parameters is not None: # parse_evaluation_parameter will raise EvaluationParameterError if we cannot find a suitable value evaluation_args[key] = parse_evaluation_parameter( value["$PARAMETER"], evaluation_parameters) else: # If we haven't been able to continue already, we failed to find a parameter raise EvaluationParameterError( "No value found for $PARAMETER " + value["$PARAMETER"]) return evaluation_args
def parse_evaluation_parameter(parameter_expression, evaluation_parameters=None): """Use the provided evaluation_parameters dict to parse a given parameter expression. Args: parameter_expression (str): A string, potentially containing basic arithmetic operations and functions, and variables to be substituted evaluation_parameters (dict): A dictionary of name-value pairs consisting of values to substitute The parser will allow arithmetic operations +, -, /, *, as well as basic functions, including trunc() and round() to obtain integer values when needed for certain expectations (e.g. expect_column_value_length_to_be_between). Valid variables must begin with an alphabetic character and may contain alphanumeric characters plus '_' and '$', EXCEPT if they begin with the string "urn:great_expectations" in which case they may also include additional characters to support inclusion of GE URLs (see :ref:`evaluation_parameters` for more information). """ if evaluation_parameters is None: evaluation_parameters = {} # Calling get_parser clears the stack parser = expr.get_parser() try: L = parser.parseString(parameter_expression, parseAll=True) except ParseException as err: L = [ "Parse Failure", parameter_expression, (str(err), err.line, err.column) ] if len(L) == 1 and L[0] not in evaluation_parameters: # In this special case there were no operations to find, so only one value, but we don't have something to # substitute for that value raise EvaluationParameterError("No value found for $PARAMETER " + str(L[0])) elif len(L) == 1: # In this case, we *do* have a substitution for a single type. We treat this specially because in this # case, we allow complex type substitutions (i.e. do not coerce to string as part of parsing) return evaluation_parameters[L[0]] elif len(L) == 0 or L[0] != "Parse Failure": for i, ob in enumerate(expr.exprStack): if isinstance(ob, str) and ob in evaluation_parameters: expr.exprStack[i] = str(evaluation_parameters[ob]) else: err_str, err_line, err_col = L[-1] raise EvaluationParameterError( f"Parse Failure: {err_str}\nStatement: {err_line}\nColumn: {err_col}" ) try: result = expr.evaluate_stack(expr.exprStack) except Exception as e: exception_traceback = traceback.format_exc() exception_message = ( f'{type(e).__name__}: "{str(e)}". Traceback: "{exception_traceback}".' ) logger.debug(exception_message, e, exc_info=True) raise EvaluationParameterError( "Error while evaluating evaluation parameter expression: " + str(e)) return result
def parse_evaluation_parameter( parameter_expression: str, evaluation_parameters: Optional[Dict[str, Any]] = None, data_context: Optional[ Any] = None, # Cannot type 'DataContext' due to import cycle ) -> Any: """Use the provided evaluation_parameters dict to parse a given parameter expression. Args: parameter_expression (str): A string, potentially containing basic arithmetic operations and functions, and variables to be substituted evaluation_parameters (dict): A dictionary of name-value pairs consisting of values to substitute data_context (DataContext): A data context to use to obtain metrics, if necessary The parser will allow arithmetic operations +, -, /, *, as well as basic functions, including trunc() and round() to obtain integer values when needed for certain expectations (e.g. expect_column_value_length_to_be_between). Valid variables must begin with an alphabetic character and may contain alphanumeric characters plus '_' and '$', EXCEPT if they begin with the string "urn:great_expectations" in which case they may also include additional characters to support inclusion of GE URLs (see :ref:`evaluation_parameters` for more information). """ if evaluation_parameters is None: evaluation_parameters = {} # Calling get_parser clears the stack parser = expr.get_parser() try: L = parser.parseString(parameter_expression, parseAll=True) except ParseException as err: L = [ "Parse Failure", parameter_expression, (str(err), err.line, err.column) ] # Represents a valid parser result of a single function that has no arguments if len(L) == 1 and isinstance(L[0], tuple) and L[0][2] is False: # Necessary to catch `now()` (which only needs to be evaluated with `expr.exprStack`) # NOTE: 20211122 - Chetan - Any future built-ins that are zero arity functions will match this behavior pass elif len(L) == 1 and L[0] not in evaluation_parameters: # In this special case there were no operations to find, so only one value, but we don't have something to # substitute for that value try: res = ge_urn.parseString(L[0]) if res["urn_type"] == "stores": store = data_context.stores.get(res["store_name"]) return store.get_query_result(res["metric_name"], res.get("metric_kwargs", {})) else: logger.error( "Unrecognized urn_type in ge_urn: must be 'stores' to use a metric store." ) raise EvaluationParameterError( "No value found for $PARAMETER " + str(L[0])) except ParseException as e: logger.debug( f"Parse exception while parsing evaluation parameter: {str(e)}" ) raise EvaluationParameterError("No value found for $PARAMETER " + str(L[0])) except AttributeError: logger.warning( "Unable to get store for store-type valuation parameter.") raise EvaluationParameterError("No value found for $PARAMETER " + str(L[0])) elif len(L) == 1: # In this case, we *do* have a substitution for a single type. We treat this specially because in this # case, we allow complex type substitutions (i.e. do not coerce to string as part of parsing) # NOTE: 20201023 - JPC - to support MetricDefinition as an evaluation parameter type, we need to handle that # case here; is the evaluation parameter provided here in fact a metric definition? return evaluation_parameters[L[0]] elif len(L) == 0 or L[0] != "Parse Failure": for i, ob in enumerate(expr.exprStack): if isinstance(ob, str) and ob in evaluation_parameters: expr.exprStack[i] = str(evaluation_parameters[ob]) else: err_str, err_line, err_col = L[-1] raise EvaluationParameterError( f"Parse Failure: {err_str}\nStatement: {err_line}\nColumn: {err_col}" ) try: result = expr.evaluate_stack(expr.exprStack) result = convert_to_json_serializable(result) except Exception as e: exception_traceback = traceback.format_exc() exception_message = ( f'{type(e).__name__}: "{str(e)}". Traceback: "{exception_traceback}".' ) logger.debug(exception_message, e, exc_info=True) raise EvaluationParameterError( "Error while evaluating evaluation parameter expression: " + str(e)) return result