def test__set_up_pipeline_auto_spark(self):
from baskerville.models.pipelines import KafkaPipeline
with mock.patch.object(KafkaPipeline, '__init__') as mock_pipeline:
from baskerville.models.engine import BaskervilleAnalyticsEngine
mock_pipeline.return_value = None
engine = BaskervilleAnalyticsEngine(RunType.kafka,
self.test_config)
self.assertTrue(engine.run_type == RunType.kafka)
engine.config.engine.use_spark = True
p = engine._set_up_pipeline()
mock_pipeline.assert_called_once()
self.assertTrue(isinstance(p, KafkaPipeline))
def test__set_up_pipeline_manual_raw_logs_path_spark(self):
from baskerville.models.pipelines import RawLogPipeline
with mock.patch.object(RawLogPipeline, '__init__') as mock_pipeline:
from baskerville.models.engine import BaskervilleAnalyticsEngine
mock_pipeline.return_value = None
engine = BaskervilleAnalyticsEngine(RunType.rawlog,
self.test_config)
self.assertTrue(engine.run_type == RunType.rawlog)
engine.config.engine.manual.host = None
engine.config.engine.manual.raw_logs_path = 'some_path'
engine.config.engine.manual.chunk_size = 0
engine.config.engine.use_spark = True
p = engine._set_up_pipeline()
mock_pipeline.assert_called_once()
self.assertTrue(isinstance(p, RawLogPipeline))