async def test_load(self):
extractor = plugin.PluginBase(id='test-extractor', config={})
transformer = plugin.PluginBase(id='test-transformer', config={})
loader = plugin.PluginBase(id='test-loader', config={})
extractor.run = asynctest.CoroutineMock(side_effect='extract-record')
transformer.run = asynctest.CoroutineMock(
side_effect='transform-record')
loader.run = asynctest.CoroutineMock(return_value='load-record')
p = pipeline.Pipeline(
id='test',
extractors=[extractor],
transform=transformer,
loaders=[loader],
)
assert p.timer._start_time is None
assert loader.timer._start_time is None
await p.load({'test': 'record'})
assert p.timer._start_time is None
assert loader.timer._start_time is not None
extractor.run.assert_not_awaited()
transformer.run.assert_not_awaited()
loader.run.assert_awaited_once_with({'test': 'record'})
async def test_process(self):
extractor = plugin.PluginBase(id='test-extractor', config={})
transformer = plugin.PluginBase(id='test-transformer', config={})
loader = plugin.PluginBase(id='test-loader', config={})
async def transform_fn(*args, **kwargs):
yield 'transform-record'
extractor.run = asynctest.CoroutineMock(side_effect='extract-record')
transformer.run = asynctest.MagicMock(side_effect=transform_fn)
loader.run = asynctest.CoroutineMock(return_value='load-record')
p = pipeline.Pipeline(
id='test',
extractors=[extractor],
transform=transformer,
loaders=[loader],
)
p.load = asynctest.CoroutineMock()
assert p.timer._start_time is None
assert transformer.timer._start_time is None
await p.process({'test': 'record'})
assert p.timer._start_time is None
assert transformer.timer._start_time is not None
extractor.run.assert_not_awaited()
loader.run.assert_not_awaited()
transformer.run.assert_called_once()
p.load.assert_awaited_once_with('transform-record')
def test_str_fmt(self):
p = plugin.PluginBase(
id='test-plugin',
config={},
)
assert str(p) == 'PluginBase(test-plugin)'
def test_loop_default(self):
p = plugin.PluginBase(
id='test-plugin',
config={},
)
assert p.loop == asyncio.get_event_loop()
async def test_close(self):
p = plugin.PluginBase(
id='test-plugin',
config={},
)
# this does nothing for the base class
await p.close()
def test_loop_new(self):
loop = asyncio.new_event_loop()
p = plugin.PluginBase(
id='test-plugin',
config={},
loop=loop,
)
assert p.loop == loop
async def test_close(self):
extractor = plugin.PluginBase(id='test-extractor', config={})
transformer = plugin.PluginBase(id='test-transformer', config={})
loader = plugin.PluginBase(id='test-loader', config={})
extractor.close = asynctest.CoroutineMock()
transformer.close = asynctest.CoroutineMock()
loader.close = asynctest.CoroutineMock()
p = pipeline.Pipeline(
id='test',
extractors=[extractor],
transform=transformer,
loaders=[loader],
)
await p.close()
extractor.close.assert_awaited_once()
transformer.close.assert_awaited_once()
loader.close.assert_awaited_once()
async def test_load_error_2(self):
loader = plugin.PluginBase(id='test-loader', config={})
loader.run = asynctest.CoroutineMock(side_effect=ValueError)
p = pipeline.Pipeline(
id='test',
extractors=[],
transform=None,
loaders=[loader],
)
assert p.timer._start_time is None
assert loader.timer._start_time is None
await p.load({'test': 'record'})
assert p.timer._start_time is None
assert loader.timer._start_time is not None
loader.run.assert_awaited_once_with({'test': 'record'})
async def test_process_with_error(self):
transformer = plugin.PluginBase(id='test-transformer', config={})
transformer.run = asynctest.MagicMock(side_effect=KeyError)
p = pipeline.Pipeline(
id='test',
extractors=[],
transform=transformer,
loaders=[],
)
p.load = asynctest.CoroutineMock()
assert p.timer._start_time is None
assert transformer.timer._start_time is None
with pytest.raises(KeyError):
await p.process({'test': 'record'})
assert p.timer._start_time is None
assert transformer.timer._start_time is not None
transformer.run.assert_called_once()
p.load.assert_not_awaited()