Python validate_schema Examples

Example #1

 def test_validate_schema_when_struct_field_is_missing(self):
     data = [("jose", 1), ("li", 2), ("luisa", 3)]
     source_df = spark.createDataFrame(data, ["name", "age"])
     required_schema = StructType([
         StructField("name", StringType(), True),
         StructField("age", LongType(), True),
     ])
     quinn.validate_schema(source_df, required_schema)

Example #2

File: test_dataframe_validator.py Project: vivekkkumar/quinn

 def it_does_nothing_when_the_schema_matches(spark):
     data = [("jose", 1), ("li", 2), ("luisa", 3)]
     source_df = spark.createDataFrame(data, ["name", "age"])
     required_schema = StructType([
         StructField("name", StringType(), True),
         StructField("age", LongType(), True),
     ])
     quinn.validate_schema(source_df, required_schema)

Example #3

 def test_validate_schema_when_struct_field_is_missing(self):
     data = [("jose", 1), ("li", 2), ("luisa", 3)]
     source_df = spark.createDataFrame(data, ["name", "age"])
     required_schema = StructType([
         StructField("name", StringType(), True),
         StructField("city", StringType(), True),
     ])
     with pytest.raises(quinn.DataFrameMissingStructFieldError) as excinfo:
         quinn.validate_schema(source_df, required_schema)
     assert excinfo.value.args[
         0] == "The [StructField(city,StringType,true)] StructFields are not included in the DataFrame with the following StructFields StructType(List(StructField(name,StringType,true),StructField(age,LongType,true)))"