def test_add_column_row_condition(spark_session): df = pd.DataFrame({"foo": [1, 2, 3, 3, None, 2, 3, 4, 5, 6]}) df = spark_session.createDataFrame( [ tuple( None if isinstance(x, (float, int)) and np.isnan(x) else x for x in record.tolist() ) for record in df.to_records(index=False) ], df.columns.tolist(), ) engine = SparkDFExecutionEngine(batch_data_dict={tuple(): df}) domain_kwargs = {"column": "foo"} new_domain_kwargs = engine.add_column_row_condition( domain_kwargs, filter_null=True, filter_nan=False ) assert new_domain_kwargs["row_condition"] == 'col("foo").notnull()' df, cd, ad = engine.get_compute_domain(new_domain_kwargs, domain_type="table") res = df.collect() assert res == [(1,), (2,), (3,), (3,), (2,), (3,), (4,), (5,), (6,)] new_domain_kwargs = engine.add_column_row_condition( domain_kwargs, filter_null=True, filter_nan=True ) assert new_domain_kwargs["row_condition"] == "NOT isnan(foo) AND foo IS NOT NULL" df, cd, ad = engine.get_compute_domain(new_domain_kwargs, domain_type="table") res = df.collect() assert res == [(1,), (2,), (3,), (3,), (2,), (3,), (4,), (5,), (6,)] new_domain_kwargs = engine.add_column_row_condition( domain_kwargs, filter_null=False, filter_nan=True ) assert new_domain_kwargs["row_condition"] == "NOT isnan(foo)" df, cd, ad = engine.get_compute_domain(new_domain_kwargs, domain_type="table") res = df.collect() assert res == [(1,), (2,), (3,), (3,), (None,), (2,), (3,), (4,), (5,), (6,)] # This time, our skip value *will* be nan df = pd.DataFrame({"foo": [1, 2, 3, 3, None, 2, 3, 4, 5, 6]}) df = spark_session.createDataFrame(df) engine = SparkDFExecutionEngine(batch_data_dict={tuple(): df}) new_domain_kwargs = engine.add_column_row_condition( domain_kwargs, filter_null=False, filter_nan=True ) assert new_domain_kwargs["row_condition"] == "NOT isnan(foo)" df, cd, ad = engine.get_compute_domain(new_domain_kwargs, domain_type="table") res = df.collect() assert res == [(1,), (2,), (3,), (3,), (2,), (3,), (4,), (5,), (6,)] new_domain_kwargs = engine.add_column_row_condition( domain_kwargs, filter_null=True, filter_nan=False ) assert new_domain_kwargs["row_condition"] == 'col("foo").notnull()' df, cd, ad = engine.get_compute_domain(new_domain_kwargs, domain_type="table") res = df.collect() expected = [(1,), (2,), (3,), (3,), (np.nan,), (2,), (3,), (4,), (5,), (6,)] # since nan != nan by default assert np.allclose(res, expected, rtol=0, atol=0, equal_nan=True)
def _spark( cls, execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: Dict, metric_value_kwargs: Dict, metrics: Dict[str, Any], runtime_configuration: Dict, ): parse_strings_as_datetimes: bool = ( metric_value_kwargs.get("parse_strings_as_datetimes") or False) if parse_strings_as_datetimes: # deprecated-v0.13.41 warnings.warn( """The parameter "parse_strings_as_datetimes" is deprecated as of v0.13.41 in \ v0.16. As part of the V3 API transition, we've moved away from input transformation. For more information, \ please see: https://greatexpectations.io/blog/why_we_dont_do_transformations_for_expectations/ """, DeprecationWarning, ) # check if column is any type that could have na (numeric types) column_name = metric_domain_kwargs["column"] table_columns = metrics["table.column_types"] column_metadata = [ col for col in table_columns if col["name"] == column_name ][0] if isinstance( column_metadata["type"], ( sparktypes.LongType, sparktypes.DoubleType, sparktypes.IntegerType, ), ): # if column is any type that could have NA values, remove them (not filtered by .isNotNull()) compute_domain_kwargs = execution_engine.add_column_row_condition( metric_domain_kwargs, filter_null=cls.filter_column_isnull, filter_nan=True, ) else: compute_domain_kwargs = metric_domain_kwargs ( df, compute_domain_kwargs, accessor_domain_kwargs, ) = execution_engine.get_compute_domain( compute_domain_kwargs, domain_type=MetricDomainTypes.COLUMN) # NOTE: 20201105 - parse_strings_as_datetimes is not supported here; # instead detect types naturally column = F.col(column_name) if isinstance(column_metadata["type"], (sparktypes.TimestampType, sparktypes.DateType)): diff = F.datediff( column, F.lag(column).over(Window.orderBy(F.lit("constant")))) else: diff = column - F.lag(column).over( Window.orderBy(F.lit("constant"))) diff = F.when(diff.isNull(), 1).otherwise(diff) # NOTE: because in spark we are implementing the window function directly, # we have to return the *unexpected* condition. # If we expect values to be *strictly* increasing then unexpected values are those # that are flat or decreasing if metric_value_kwargs["strictly"] is True: return ( F.when(diff <= 0, F.lit(True)).otherwise(F.lit(False)), compute_domain_kwargs, accessor_domain_kwargs, ) # If we expect values to be flat or increasing then unexpected values are those # that are decreasing else: return ( F.when(diff < 0, F.lit(True)).otherwise(F.lit(False)), compute_domain_kwargs, accessor_domain_kwargs, )
def _spark( cls, execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: Dict, metric_value_kwargs: Dict, metrics: Dict[str, Any], runtime_configuration: Dict, ): parse_strings_as_datetimes: bool = ( metric_value_kwargs.get("parse_strings_as_datetimes") or False) if parse_strings_as_datetimes: warnings.warn( f"""The parameter "parse_strings_as_datetimes" is no longer supported and will be deprecated in a \ future release. Please update code accordingly. Moreover, in "{cls.__name__}._spark()", types are detected naturally. """, DeprecationWarning, ) # check if column is any type that could have na (numeric types) column_name = metric_domain_kwargs["column"] table_columns = metrics["table.column_types"] column_metadata = [ col for col in table_columns if col["name"] == column_name ][0] if isinstance( column_metadata["type"], ( sparktypes.LongType, sparktypes.DoubleType, sparktypes.IntegerType, ), ): # if column is any type that could have NA values, remove them (not filtered by .isNotNull()) compute_domain_kwargs = execution_engine.add_column_row_condition( metric_domain_kwargs, filter_null=cls.filter_column_isnull, filter_nan=True, ) else: compute_domain_kwargs = metric_domain_kwargs ( df, compute_domain_kwargs, accessor_domain_kwargs, ) = execution_engine.get_compute_domain(compute_domain_kwargs, MetricDomainTypes.COLUMN) # NOTE: 20201105 - parse_strings_as_datetimes is not supported here; # instead detect types naturally column = F.col(column_name) if isinstance(column_metadata["type"], (sparktypes.TimestampType, sparktypes.DateType)): diff = F.datediff( column, F.lag(column).over(Window.orderBy(F.lit("constant")))) else: diff = column - F.lag(column).over( Window.orderBy(F.lit("constant"))) diff = F.when(diff.isNull(), -1).otherwise(diff) # NOTE: because in spark we are implementing the window function directly, # we have to return the *unexpected* condition if metric_value_kwargs["strictly"]: return ( F.when(diff >= 0, F.lit(True)).otherwise(F.lit(False)), compute_domain_kwargs, accessor_domain_kwargs, ) # If we expect values to be flat or decreasing then unexpected values are those # that are decreasing else: return ( F.when(diff > 0, F.lit(True)).otherwise(F.lit(False)), compute_domain_kwargs, accessor_domain_kwargs, )
def _spark( cls, execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: Dict, metric_value_kwargs: Dict, metrics: Dict[Tuple, Any], runtime_configuration: Dict, ): # check if column is any type that could have na (numeric types) column_name = metric_domain_kwargs["column"] table_columns = metrics["table.column_types"] column_metadata = [ col for col in table_columns if col["name"] == column_name ][0] if isinstance( column_metadata["type"], ( sparktypes.LongType, sparktypes.DoubleType, sparktypes.IntegerType, ), ): # if column is any type that could have NA values, remove them (not filtered by .isNotNull()) compute_domain_kwargs = execution_engine.add_column_row_condition( metric_domain_kwargs, filter_null=cls.filter_column_isnull, filter_nan=True, ) else: compute_domain_kwargs = metric_domain_kwargs ( df, compute_domain_kwargs, accessor_domain_kwargs, ) = execution_engine.get_compute_domain( compute_domain_kwargs, domain_type=MetricDomainTypes.COLUMN) # NOTE: 20201105 - parse_strings_as_datetimes is not supported here; # instead detect types naturally column = F.col(column_name) if isinstance(column_metadata["type"], (sparktypes.TimestampType, sparktypes.DateType)): diff = F.datediff( column, F.lag(column).over(Window.orderBy(F.lit("constant")))) else: diff = column - F.lag(column).over( Window.orderBy(F.lit("constant"))) diff = F.when(diff.isNull(), 1).otherwise(diff) # NOTE: because in spark we are implementing the window function directly, # we have to return the *unexpected* condition. # If we expect values to be *strictly* increasing then unexpected values are those # that are flat or decreasing if metric_value_kwargs["strictly"] is True: return ( F.when(diff <= 0, F.lit(True)).otherwise(F.lit(False)), compute_domain_kwargs, accessor_domain_kwargs, ) # If we expect values to be flat or increasing then unexpected values are those # that are decreasing else: return ( F.when(diff < 0, F.lit(True)).otherwise(F.lit(False)), compute_domain_kwargs, accessor_domain_kwargs, )