def test_dataframe_partition(spark_session):
miscProcess.global_SQLContext(spark_session)
miscProcess.initial_log_file('test_log.log')
json_file = 'test1.json'
with open(json_file) as jfile:
df_dict = json.load(jfile)
actual_partition = processDataframeConfig.partition_column(df_dict)
expected_partition = "MONTH"
assert actual_partition == expected_partition
def test_file_path(spark_session):
miscProcess.global_SQLContext(spark_session)
miscProcess.initial_log_file('test_log.log')
json_file = 'test1.json'
with open(json_file) as jfile:
df_dict = json.load(jfile)
actual_filepath = processDataframeConfig.get_source_driverFilerPath(
df_dict)
expected_filePath = "C:\\Test\\Paid_Parking.csv"
assert actual_filepath == expected_filePath
def test_column_list(spark_session):
miscProcess.global_SQLContext(spark_session)
miscProcess.initial_log_file('test_log.log')
json_file = 'test1.json'
with open(json_file) as jfile:
df_dict = json.load(jfile)
actual_list = processDataframeConfig.build_dataframe_column_list(df_dict)
expected_list = ['occupancydatetime', 'occupied_spots']
print(actual_list)
assert actual_list[0] == expected_list[0]
assert actual_list[1] == expected_list[1]
SCRIPT_NAME,
"Failed on writing to Output after {} tries: {} ".format(
max_retry_count, output))
ReturnCode = 2
return ReturnCode, rec_cnt
else:
miscProcess.log_info(
SCRIPT_NAME,
"Failed on writing to Output, re-try in {} seconds ".
format(retry_delay))
time.sleep(retry_delay)
miscProcess.log_print("Number of Records Processed: {}".format(rec_cnt))
return ReturnCode, rec_cnt
def main():
# logger = spark._jvm.org.apache.log4j
# logger.LogManager.getLogger("org").setLevel(logger.Level.ERROR)
# logger.LogManager.getLogger("akka").setLevel(logger.Level.ERROR)
executeOccupancyOperations(src_df, output, cols_list, cols_dict, partn_col,
max_retry_count, retry_delay)
if __name__ == '__main__':
log_file = 'test.log'
miscProcess.initial_log_file(logfile)
main()
#=========================================================================================================
spark = create_sparksession()
# Make the SQLContext Session available to sub-scripts
executeBlockface.global_SQLContext(spark)
executeOccupancyProcess.global_SQLContext(spark)
readEnvironmentParameters.global_SQLContext(spark)
miscProcess.global_SQLContext(spark)
#=========================================================================================================
#================ Initialize log Filename =============================================================
#=========================================================================================================
miscProcess.initial_log_file(LogFileName)
#=========================================================================================================
# PROCESS ALL PARAMETERS
STEP, STEP_DESC=(10, "Read Job Specific Parameter Files")
#=========================================================================================================
miscProcess.log_step(SCRIPT_NAME, "PERFORMING STEP {}:{} ".format(STEP, STEP_DESC))
if SparkSubmitClientMode == 'Y':
# Spark Submitted in Client Mode
job_control_file = ControlPath + JOBNAME +".cfg"
blockface_config_filename = ConfigPath +BlockfaceDataframeName.lower()+'.json'
occupancy_config_filename = ConfigPath +OccupancyDataframeName.lower()+'.json'
else: