def test_fetch_wf_module(self, save_result, load_module):
result = ProcessResult(pd.DataFrame({'A': [1]}), error='hi')
async def fake_fetch(*args, **kwargs):
return result
fake_module = Mock(LoadedModule)
load_module.return_value = fake_module
fake_module.fetch.side_effect = fake_fetch
workflow = Workflow.create_and_init()
wf_module = workflow.tabs.first().wf_modules.create(
order=0,
next_update=parser.parse('Aug 28 1999 2:24PM UTC'),
update_interval=600)
now = parser.parse('Aug 28 1999 2:24:02PM UTC')
due_for_update = parser.parse('Aug 28 1999 2:34PM UTC')
with self.assertLogs(fetch.__name__, logging.DEBUG):
self.run_with_async_db(
fetch.fetch_wf_module(workflow.id, wf_module, now))
save_result.assert_called_with(workflow.id, wf_module, result)
wf_module.refresh_from_db()
self.assertEqual(wf_module.last_update_check, now)
self.assertEqual(wf_module.next_update, due_for_update)
def _test_fetch(self, fn, wf_module, save, load) -> ProcessResult:
"""
Stub out a `fetch` method for `wf_module`.
Return result.
"""
if wf_module.module_version is None:
# White-box: we aren't testing what happens in the (valid) case
# that a ModuleVersion has been deleted while in use. Pretend it's
# there.
wf_module._module_version = ModuleVersion(spec={'parameters': []})
try:
workflow_id = wf_module.workflow_id
except AttributeError: # No tab/workflow in database
workflow_id = 1
# Mock the module we load, so it calls fn() directly.
load.return_value = LoadedModule('test', '1', False, fetch_impl=fn)
load.return_value.fetch = fn
save.return_value = future_none
# Mock wf_module.save(), which we aren't testing.
wf_module.save = Mock()
with self.assertLogs(fetch.__name__, logging.DEBUG):
self.run_with_async_db(
fetch.fetch_wf_module(workflow_id, wf_module, timezone.now()))
save.assert_called_once()
self.assertEqual(save.call_args[0][0], workflow_id)
self.assertEqual(save.call_args[0][1], wf_module)
result = save.call_args[0][2]
return result
def test_crashing_module(self, load_module):
async def fake_fetch(*args, **kwargs):
raise ValueError('boo')
fake_module = Mock(LoadedModule)
load_module.return_value = fake_module
fake_module.fetch.side_effect = fake_fetch
workflow = Workflow.objects.create()
tab = workflow.tabs.create(position=0)
wf_module = tab.wf_modules.create(
order=0,
next_update=parser.parse('Aug 28 1999 2:24PM UTC'),
update_interval=600)
now = parser.parse('Aug 28 1999 2:34:02PM UTC')
due_for_update = parser.parse('Aug 28 1999 2:44PM UTC')
with self.assertLogs(fetch.__name__, level='ERROR') as cm:
# We should log the actual error
self.run_with_async_db(
fetch.fetch_wf_module(workflow.id, wf_module, now))
self.assertEqual(cm.records[0].exc_info[0], ValueError)
wf_module.refresh_from_db()
# [adamhooper, 2018-10-26] while fiddling with tests, I changed the
# behavior to record the update check even when module fetch fails.
# Previously, an exception would prevent updating last_update_check,
# and I think that must be wrong.
self.assertEqual(wf_module.last_update_check, now)
self.assertEqual(wf_module.next_update, due_for_update)
def test_fetch_wf_module_skip_missed_update(self, load_module):
workflow = Workflow.objects.create()
tab = workflow.tabs.create(position=0)
wf_module = tab.wf_modules.create(
order=0,
next_update=parser.parse('Aug 28 1999 2:24PM UTC'),
update_interval=600)
load_module.side_effect = Exception('caught') # least-code test case
now = parser.parse('Aug 28 1999 2:34:02PM UTC')
due_for_update = parser.parse('Aug 28 1999 2:44PM UTC')
with self.assertLogs(fetch.__name__):
self.run_with_async_db(
fetch.fetch_wf_module(workflow.id, wf_module, now))
wf_module.refresh_from_db()
self.assertEqual(wf_module.next_update, due_for_update)
def test_fetch_ignore_wf_module_deleted_when_updating(
self, save_result, load_module):
"""
It's okay if wf_module is gone when updating wf_module.next_update.
"""
workflow = Workflow.create_and_init()
wf_module = workflow.tabs.first().wf_modules.create(
order=0,
next_update=parser.parse('Aug 28 1999 2:24PM UTC'),
update_interval=600)
async def fake_fetch(*args, **kwargs):
return ProcessResult(pd.DataFrame({'A': [1]}))
fake_module = Mock(LoadedModule)
load_module.return_value = fake_module
fake_module.fetch.side_effect = fake_fetch
# We're testing what happens if wf_module disappears after save, before
# update. To mock that, delete after fetch, when saving result.
async def fake_save(workflow_id, wf_module, *args, **kwargs):
@database_sync_to_async
def do_delete():
# We can't just call wf_module.delete(), because that will
# change wf_module.id, which the code under test will notice.
# We want to test what happens when wf_module.id is not None
# and the value is not in the DB. Solution: look up a copy and
# delete the copy.
WfModule.objects.get(id=wf_module.id).delete()
await do_delete()
save_result.side_effect = fake_save
now = parser.parse('Aug 28 1999 2:34:02PM UTC')
# Assert fetch does not crash with
# DatabaseError: Save with update_fields did not affect any rows
with self.assertLogs(fetch.__name__, level='DEBUG'):
self.run_with_async_db(
fetch.fetch_wf_module(workflow.id, wf_module, now))
def test_fetch_poll_when_setting_next_update(self, save_result,
load_module):
"""
Handle `.auto_update_data` and `.update_interval` changing mid-fetch.
"""
workflow = Workflow.create_and_init()
wf_module = workflow.tabs.first().wf_modules.create(
order=0,
auto_update_data=True,
next_update=parser.parse('Aug 28 1999 2:24PM UTC'),
update_interval=600)
async def fake_fetch(*args, **kwargs):
return ProcessResult(pd.DataFrame({'A': [1]}))
fake_module = Mock(LoadedModule)
load_module.return_value = fake_module
fake_module.fetch.side_effect = fake_fetch
# We're testing what happens if wf_module disappears after save, before
# update. To mock that, delete after fetch, when saving result.
async def fake_save(workflow_id, wf_module, *args, **kwargs):
@database_sync_to_async
def change_wf_module_during_fetch():
WfModule.objects.filter(id=wf_module.id).update(
auto_update_data=False,
next_update=None,
)
await change_wf_module_during_fetch()
save_result.side_effect = fake_save
now = parser.parse('Aug 28 1999 2:34:02PM UTC')
with self.assertLogs(fetch.__name__, level='DEBUG'):
self.run_with_async_db(
fetch.fetch_wf_module(workflow.id, wf_module, now))
wf_module.refresh_from_db()
self.assertEqual(wf_module.auto_update_data, False)
self.assertIsNone(wf_module.next_update)
