def test_fetch_wf_module(self, save_result, load_module):
result = ProcessResult(pd.DataFrame({"A": [1]}), error="hi")
async def fake_fetch(*args, **kwargs):
return result
fake_module = Mock(LoadedModule)
load_module.return_value = fake_module
fake_module.param_schema = ParamDType.Dict({})
fake_module.migrate_params.side_effect = lambda x: x
fake_module.fetch.side_effect = fake_fetch
save_result.side_effect = async_noop
workflow = Workflow.create_and_init()
wf_module = workflow.tabs.first().wf_modules.create(
order=0,
slug="step-1",
auto_update_data=True,
next_update=parser.parse("Aug 28 1999 2:24PM UTC"),
update_interval=600,
)
wf_module._module_version = ModuleVersion(spec={"parameters": []})
now = parser.parse("Aug 28 1999 2:24:02PM UTC")
due_for_update = parser.parse("Aug 28 1999 2:34PM UTC")
self.run_with_async_db(
fetch.fetch_wf_module(workflow.id, wf_module, now))
save_result.assert_called_with(workflow.id, wf_module, result)
wf_module.refresh_from_db()
self.assertEqual(wf_module.last_update_check, now)
self.assertEqual(wf_module.next_update, due_for_update)
def test_crashing_fetch(self, load_module):
async def fake_fetch(*args, **kwargs):
raise ValueError('boo')
fake_module = Mock(LoadedModule)
load_module.return_value = fake_module
fake_module.param_schema = ParamDType.Dict({})
fake_module.migrate_params.side_effect = lambda x: x
fake_module.fetch.side_effect = fake_fetch
workflow = Workflow.objects.create()
tab = workflow.tabs.create(position=0)
wf_module = tab.wf_modules.create(
order=0,
next_update=parser.parse('Aug 28 1999 2:24PM UTC'),
update_interval=600,
module_id_name='x')
wf_module._module_version = ModuleVersion(spec={'parameters': []}),
now = parser.parse('Aug 28 1999 2:34:02PM UTC')
due_for_update = parser.parse('Aug 28 1999 2:44PM UTC')
with self.assertLogs(fetch.__name__, level='ERROR') as cm:
# We should log the actual error
self.run_with_async_db(
fetch.fetch_wf_module(workflow.id, wf_module, now))
self.assertEqual(cm.records[0].exc_info[0], ValueError)
wf_module.refresh_from_db()
# [adamhooper, 2018-10-26] while fiddling with tests, I changed the
# behavior to record the update check even when module fetch fails.
# Previously, an exception would prevent updating last_update_check,
# and I think that must be wrong.
self.assertEqual(wf_module.last_update_check, now)
self.assertEqual(wf_module.next_update, due_for_update)
def test_fetch_ignore_wf_module_deleted_when_updating(
self, save_result, load_module):
"""
It's okay if wf_module is gone when updating wf_module.next_update.
"""
workflow = Workflow.create_and_init()
wf_module = workflow.tabs.first().wf_modules.create(
order=0,
slug="step-1",
auto_update_data=True,
next_update=parser.parse("Aug 28 1999 2:24PM UTC"),
update_interval=600,
)
async def fake_fetch(*args, **kwargs):
return ProcessResult(pd.DataFrame({"A": [1]}))
fake_module = Mock(LoadedModule)
load_module.return_value = fake_module
fake_module.migrate_params.side_effect = lambda x: x
fake_module.fetch.side_effect = fake_fetch
# We're testing what happens if wf_module disappears after save, before
# update. To mock that, delete after fetch, when saving result.
async def fake_save(workflow_id, wf_module, *args, **kwargs):
@database_sync_to_async
def do_delete():
# We can't just call wf_module.delete(), because that will
# change wf_module.id, which the code under test will notice.
# We want to test what happens when wf_module.id is not None
# and the value is not in the DB. Solution: look up a copy and
# delete the copy.
WfModule.objects.get(id=wf_module.id).delete()
await do_delete()
save_result.side_effect = fake_save
now = parser.parse("Aug 28 1999 2:34:02PM UTC")
# Assert fetch does not crash with
# DatabaseError: Save with update_fields did not affect any rows
with self.assertLogs(fetch.__name__, level="DEBUG"):
self.run_with_async_db(
fetch.fetch_wf_module(workflow.id, wf_module, now))
def test_fetch_poll_when_setting_next_update(self, save_result,
load_module):
"""
Handle `.auto_update_data` and `.update_interval` changing mid-fetch.
"""
workflow = Workflow.create_and_init()
wf_module = workflow.tabs.first().wf_modules.create(
order=0,
slug="step-1",
auto_update_data=True,
next_update=parser.parse("Aug 28 1999 2:24PM UTC"),
update_interval=600,
)
wf_module._module_version = ModuleVersion(spec={"parameters": []})
async def fake_fetch(*args, **kwargs):
return ProcessResult(pd.DataFrame({"A": [1]}))
fake_module = Mock(LoadedModule)
load_module.return_value = fake_module
fake_module.param_schema = ParamDType.Dict({})
fake_module.migrate_params.side_effect = lambda x: x
fake_module.fetch.side_effect = fake_fetch
# We're testing what happens if wf_module disappears after save, before
# update. To mock that, delete after fetch, when saving result.
async def fake_save(workflow_id, wf_module, *args, **kwargs):
@database_sync_to_async
def change_wf_module_during_fetch():
WfModule.objects.filter(id=wf_module.id).update(
auto_update_data=False, next_update=None)
await change_wf_module_during_fetch()
save_result.side_effect = fake_save
now = parser.parse("Aug 28 1999 2:34:02PM UTC")
self.run_with_async_db(
fetch.fetch_wf_module(workflow.id, wf_module, now))
wf_module.refresh_from_db()
self.assertEqual(wf_module.auto_update_data, False)
self.assertIsNone(wf_module.next_update)
def test_fetch_deleted_wf_module(self, save_result):
save_result.side_effect = async_noop
workflow = Workflow.create_and_init()
wf_module = workflow.tabs.first().wf_modules.create(
order=0, slug="step-1", module_id_name="deleted_module")
now = parser.parse("Aug 28 1999 2:24:02PM UTC")
with self.assertLogs(fetch.__name__, level="INFO") as cm:
self.run_with_async_db(
fetch.fetch_wf_module(workflow.id, wf_module, now))
self.assertRegex(cm.output[0],
r"fetch\(\) deleted module 'deleted_module'")
save_result.assert_called_with(
workflow.id,
wf_module,
ProcessResult(error="Cannot fetch: module was deleted"),
)
def test_fetch_wf_module_skip_missed_update(self, load_module):
workflow = Workflow.objects.create()
tab = workflow.tabs.create(position=0)
wf_module = tab.wf_modules.create(
order=0,
next_update=parser.parse('Aug 28 1999 2:24PM UTC'),
update_interval=600)
load_module.side_effect = Exception('caught') # least-code test case
now = parser.parse('Aug 28 1999 2:34:02PM UTC')
due_for_update = parser.parse('Aug 28 1999 2:44PM UTC')
with self.assertLogs(fetch.__name__):
self.run_with_async_db(
fetch.fetch_wf_module(workflow.id, wf_module, now))
wf_module.refresh_from_db()
self.assertEqual(wf_module.next_update, due_for_update)
def _test_fetch(self, fn, migrate_params_fn, wf_module, param_schema, save,
load) -> ProcessResult:
"""
Stub out a `fetch` method for `wf_module`.
Return result.
"""
if wf_module.module_version is None:
# White-box: we aren't testing what happens in the (valid) case
# that a ModuleVersion has been deleted while in use. Pretend it's
# there.
wf_module._module_version = ModuleVersion(spec={"parameters": []})
try:
workflow_id = wf_module.workflow_id
except AttributeError: # No tab/workflow in database
workflow_id = 1
@dataclass(frozen=True)
class MockLoadedModule:
fetch: Callable
migrate_params: Callable
param_schema: ParamDType.Dict = ParamDType.Dict({})
# Mock the module we load, so it calls fn() directly.
load.return_value = MockLoadedModule(fn, migrate_params_fn)
save.return_value = future_none
self.run_with_async_db(
fetch.fetch_wf_module(workflow_id, wf_module, timezone.now()))
save.assert_called_once()
self.assertEqual(save.call_args[0][0], workflow_id)
self.assertEqual(save.call_args[0][1], wf_module)
result = save.call_args[0][2]
return result