def should_successfully_merge_array_types_with( schema_a: ArrayType, schema_b: ArrayType, expected_schema: ArrayType ): # given ^ # when merged_schema = merge_schemas(schema_a, schema_b) # then assert merged_schema.jsonValue() == expected_schema.jsonValue() # ...expect distinct objects assert merged_schema is not schema_a assert merged_schema is not schema_b
# -*- coding: utf-8 -*- """ Created on Sun Jun 14 10:20:19 2020 """ from pyspark.sql import SparkSession from pyspark.sql.types import DataType from pyspark.sql.types import StructType, StructField, StringType, ArrayType, IntegerType spark = SparkSession.builder.appName('SparkByExamples.com').getOrCreate() from pyspark.sql.types import ArrayType, IntegerType arrayType = ArrayType(IntegerType(), False) print(arrayType.jsonValue()) print(arrayType.simpleString()) print(arrayType.typeName()) from pyspark.sql.types import MapType, StringType, IntegerType mapType = MapType(StringType(), IntegerType()) print(mapType.keyType) print(mapType.valueType) print(mapType.valueContainsNull) data = [("James", "", "Smith", "36", "M", 3000), ("Michael", "Rose", "", "40", "M", 4000), ("Robert", "", "Williams", "42", "M", 4000), ("Maria", "Anne", "Jones", "39", "F", 4000), ("Jen", "Mary", "Brown", "", "F", -1)] schema = StructType([