def should_successfully_merge_array_types_with(
schema_a: ArrayType, schema_b: ArrayType, expected_schema: ArrayType
):
# given ^
# when
merged_schema = merge_schemas(schema_a, schema_b)
# then
assert merged_schema.jsonValue() == expected_schema.jsonValue()
# ...expect distinct objects
assert merged_schema is not schema_a
assert merged_schema is not schema_b
# -*- coding: utf-8 -*-
"""
Created on Sun Jun 14 10:20:19 2020
"""
from pyspark.sql import SparkSession
from pyspark.sql.types import DataType
from pyspark.sql.types import StructType, StructField, StringType, ArrayType, IntegerType
spark = SparkSession.builder.appName('SparkByExamples.com').getOrCreate()
from pyspark.sql.types import ArrayType, IntegerType
arrayType = ArrayType(IntegerType(), False)
print(arrayType.jsonValue())
print(arrayType.simpleString())
print(arrayType.typeName())
from pyspark.sql.types import MapType, StringType, IntegerType
mapType = MapType(StringType(), IntegerType())
print(mapType.keyType)
print(mapType.valueType)
print(mapType.valueContainsNull)
data = [("James", "", "Smith", "36", "M", 3000),
("Michael", "Rose", "", "40", "M", 4000),
("Robert", "", "Williams", "42", "M", 4000),
("Maria", "Anne", "Jones", "39", "F", 4000),
("Jen", "Mary", "Brown", "", "F", -1)]
schema = StructType([
