def setUp(self): spark_session, spark_logger, config = start_spark_session( app_name="top_marketing_campaigns_df_api_test", files=["config/test/top_marketing_campaigns_config.yaml"]) self.spark_session: SparkSession = spark_session self.spark_logger: Log4jWrapper = spark_logger self.config: dict = config
def setUp(self): spark_session, spark_logger, config = start_spark_session( app_name="most_popular_channels_df_api_test", files=["config/test/most_popular_channels_config.yaml"]) self.spark_session: SparkSession = spark_session self.spark_logger: Log4jWrapper = spark_logger self.config: dict = config
def setUp(self): spark_session, spark_logger, config = start_spark_session( app_name="purchases_attribution_sql_test", files=["config/test/purchases_attribution_config.yaml"]) self.spark_session: SparkSession = spark_session self.spark_logger: Log4jWrapper = spark_logger self.config: dict = config
def test_start_spark_session_with_config(self): """ Test: Function start_spark_session() loads config from file """ spark_session, _, config = start_spark_session( "test-app", files=["config/test/dummy_config.yaml"]) self.assertIsNotNone(config) self.assertTrue("input" in config) spark_session.stop()
def test_start_spark_session(self): """ Test: Function start_spark_session() creates Spark session """ res = start_spark_session("test-app") self.assertIsNotNone(res, "") self.assertEqual(3, len(res), "") spark_session: SparkSession = res[0] self.assertTrue(isinstance(spark_session, SparkSession), "") self.assertEqual("local[*]", spark_session.sparkContext.master) logger: Log4jWrapper = res[1] self.assertTrue(isinstance(logger, Log4jWrapper), "") self.assertIsNone(res[2]) spark_session.stop()
def setUp(self): spark_session, logger, _ = start_spark_session("test-app") self.logger = logger self.spark_session = spark_session