def test_str_fmt(self):
p = pipeline.Pipeline(id='test',
extractors=None,
transform=None,
loaders=None)
assert str(p) == 'Pipeline(test)'
async def test_load(self):
extractor = plugin.PluginBase(id='test-extractor', config={})
transformer = plugin.PluginBase(id='test-transformer', config={})
loader = plugin.PluginBase(id='test-loader', config={})
extractor.run = asynctest.CoroutineMock(side_effect='extract-record')
transformer.run = asynctest.CoroutineMock(
side_effect='transform-record')
loader.run = asynctest.CoroutineMock(return_value='load-record')
p = pipeline.Pipeline(
id='test',
extractors=[extractor],
transform=transformer,
loaders=[loader],
)
assert p.timer._start_time is None
assert loader.timer._start_time is None
await p.load({'test': 'record'})
assert p.timer._start_time is None
assert loader.timer._start_time is not None
extractor.run.assert_not_awaited()
transformer.run.assert_not_awaited()
loader.run.assert_awaited_once_with({'test': 'record'})
async def test_process(self):
extractor = plugin.PluginBase(id='test-extractor', config={})
transformer = plugin.PluginBase(id='test-transformer', config={})
loader = plugin.PluginBase(id='test-loader', config={})
async def transform_fn(*args, **kwargs):
yield 'transform-record'
extractor.run = asynctest.CoroutineMock(side_effect='extract-record')
transformer.run = asynctest.MagicMock(side_effect=transform_fn)
loader.run = asynctest.CoroutineMock(return_value='load-record')
p = pipeline.Pipeline(
id='test',
extractors=[extractor],
transform=transformer,
loaders=[loader],
)
p.load = asynctest.CoroutineMock()
assert p.timer._start_time is None
assert transformer.timer._start_time is None
await p.process({'test': 'record'})
assert p.timer._start_time is None
assert transformer.timer._start_time is not None
extractor.run.assert_not_awaited()
loader.run.assert_not_awaited()
transformer.run.assert_called_once()
p.load.assert_awaited_once_with('transform-record')
def test_str_fmt(self):
p1 = pipeline.Pipeline(id='1',
extractors=None,
transform=None,
loaders=None)
p = pipeline.Pipelines()
p['1'] = p1
assert str(p) == 'Pipelines([Pipeline(1)])'
async def test_load_no_record(self):
p = pipeline.Pipeline(
id='test',
extractors=[],
transform=None,
loaders=[],
)
assert p.timer._start_time is None
await p.load(None)
assert p.timer._start_time is None
def test_init(self):
p = pipeline.Pipeline(id='test',
extractors=['extractor'],
transform='transform',
loaders=['loader'])
assert p.id == 'test'
assert p.extractors == ['extractor']
assert p.transform == 'transform'
assert p.loaders == ['loader']
assert p.timer.histogram == metrics.pipeline_latency
assert p.timer.labels == ('test', )
async def test_run_with_pipelines(self):
p1 = pipeline.Pipeline(id='p1',
extractors=None,
transform=None,
loaders=None)
p2 = pipeline.Pipeline(id='p2',
extractors=None,
transform=None,
loaders=None)
p1.run = asynctest.CoroutineMock()
p2.run = asynctest.CoroutineMock()
mage = app.Prophetess({})
mage.pipelines.append(p1)
mage.pipelines.append(p2)
assert len(mage.pipelines) == 2
await mage.run()
p1.run.assert_called_once()
p2.run.assert_called_once()
def test_append(self):
p1 = pipeline.Pipeline(id='1',
extractors=None,
transform=None,
loaders=None)
p = pipeline.Pipelines()
assert len(p) == 0
p.append(p1)
assert len(p) == 1
assert '1' in p
async def test_close(self):
p1 = pipeline.Pipeline(id='1',
extractors=None,
transform=None,
loaders=None)
p1.close = asynctest.CoroutineMock()
p = pipeline.Pipelines()
p['1'] = p1
await p.close()
p1.close.assert_called_once()
async def test_load_error_2(self):
loader = plugin.PluginBase(id='test-loader', config={})
loader.run = asynctest.CoroutineMock(side_effect=ValueError)
p = pipeline.Pipeline(
id='test',
extractors=[],
transform=None,
loaders=[loader],
)
assert p.timer._start_time is None
assert loader.timer._start_time is None
await p.load({'test': 'record'})
assert p.timer._start_time is None
assert loader.timer._start_time is not None
loader.run.assert_awaited_once_with({'test': 'record'})
async def test_close(self):
extractor = plugin.PluginBase(id='test-extractor', config={})
transformer = plugin.PluginBase(id='test-transformer', config={})
loader = plugin.PluginBase(id='test-loader', config={})
extractor.close = asynctest.CoroutineMock()
transformer.close = asynctest.CoroutineMock()
loader.close = asynctest.CoroutineMock()
p = pipeline.Pipeline(
id='test',
extractors=[extractor],
transform=transformer,
loaders=[loader],
)
await p.close()
extractor.close.assert_awaited_once()
transformer.close.assert_awaited_once()
loader.close.assert_awaited_once()
async def test_process_with_error(self):
transformer = plugin.PluginBase(id='test-transformer', config={})
transformer.run = asynctest.MagicMock(side_effect=KeyError)
p = pipeline.Pipeline(
id='test',
extractors=[],
transform=transformer,
loaders=[],
)
p.load = asynctest.CoroutineMock()
assert p.timer._start_time is None
assert transformer.timer._start_time is None
with pytest.raises(KeyError):
await p.process({'test': 'record'})
assert p.timer._start_time is None
assert transformer.timer._start_time is not None
transformer.run.assert_called_once()
p.load.assert_not_awaited()