def test_standardize_sensordata(spark):
"""Test data transform"""
sensordata_sdf = spark.read.json("./data/MelbParkingSensorData.json",
multiLine=True)
load_id = 1
loaded_on = datetime.datetime.now()
t_sensordata_sdf = standardize.standardize_sensordata(
sensordata_sdf, load_id, loaded_on)
assert t_sensordata_sdf.count() != 0
assert t_sensordata_sdf.filter(isnull("bay_id")).count() == 0
def test_standardize_sensordata(spark):
"""Test data transform"""
# Arrange
schema = standardize.get_schema("in_sensordata_schema")
sensordata_sdf = spark.read.json("./data/MelbParkingSensorData.json", multiLine=True, schema=schema)
load_id = 1
loaded_on = datetime.datetime.now()
# Act
t_sensordata_sdf, t_sensordata_malformed_sdf = standardize.standardize_sensordata(sensordata_sdf, load_id, loaded_on) # noqa: E501
# Assert
assert t_sensordata_sdf.count() != 0
assert t_sensordata_malformed_sdf.count() == 0
assert t_sensordata_sdf.filter(isnull("bay_id")).count() == 0
# Read data
parkingbay_sdf = spark.read\
.schema(parkingbay_schema)\
.option("badRecordsPath", os.path.join(base_path, "__corrupt", "MelbParkingBayData"))\
.option("multiLine", True)\
.json(parkingbay_filepath)
sensordata_sdf = spark.read\
.schema(sensordata_schema)\
.option("badRecordsPath", os.path.join(base_path, "__corrupt", "MelbParkingSensorData"))\
.option("multiLine", True)\
.json(sensors_filepath)
# Standardize
t_parkingbay_sdf, t_parkingbay_malformed_sdf = s.standardize_parking_bay(
parkingbay_sdf, load_id, loaded_on)
t_sensordata_sdf, t_sensordata_malformed_sdf = s.standardize_sensordata(
sensordata_sdf, load_id, loaded_on)
# Insert new rows
t_parkingbay_sdf.write.mode("append").insertInto("interim.parking_bay")
t_sensordata_sdf.write.mode("append").insertInto("interim.sensor")
# Insert bad rows
t_parkingbay_malformed_sdf.write.mode("append").insertInto(
"malformed.parking_bay")
t_sensordata_malformed_sdf.write.mode("append").insertInto("malformed.sensor")
# COMMAND ----------
# MAGIC %md
# MAGIC ### Metrics