def clear_deltas(self):
"""Become a single-Delta Workflow."""
from cjwstate.models.commands import InitWorkflowCommand
try:
from cjwstate.models import Delta
first_delta = self.deltas.get(prev_delta_id=None)
except Delta.DoesNotExist:
# Invariant failed. Defensive programming: recover.
first_delta = InitWorkflowCommand.create(self)
if not isinstance(first_delta, InitWorkflowCommand):
# Invariant failed: first delta should be InitWorkflowCommand.
# Defensive programming: recover. Delete _every_ Delta, and then
# add the one that belongs.
first_delta.delete()
first_delta = InitWorkflowCommand.create(self)
else:
self.last_delta_id = first_delta.id
self.save(update_fields=["last_delta_id"])
try:
# Select the _second_ delta.
second_delta = first_delta.next_delta
except Delta.DoesNotExist:
# We're already a 1-delta Workflow
return
second_delta.delete_with_successors()
self.delete_orphan_soft_deleted_models()
def test_email_no_delta_when_no_cached_render_result(self, email):
# No cached render result means one of two things:
#
# 1. This is a new module (in which case, why email the user?)
# 2. We cleared the render cache (in which case, better to skip emailing a few
# users than to email _every_ user that results have changed when they haven't)
workflow = Workflow.objects.create()
tab = workflow.tabs.create(position=0)
delta1 = InitWorkflowCommand.create(workflow)
create_module_zipfile(
"mod",
python_code=
'import pandas as pd\ndef render(table, params): return pd.DataFrame({"A": [1]})',
)
wf_module = tab.wf_modules.create(
order=0,
slug="step-1",
last_relevant_delta_id=delta1.id,
module_id_name="mod",
notifications=True,
)
# Make a new delta, so we need to re-render. Give it the same output.
delta2 = InitWorkflowCommand.create(workflow)
wf_module.last_relevant_delta_id = delta2.id
wf_module.save(update_fields=["last_relevant_delta_id"])
self._execute(workflow)
email.assert_not_called()
def test_email_no_delta_when_not_changed(self, email, fake_load_module):
workflow = Workflow.objects.create()
tab = workflow.tabs.create(position=0)
delta1 = InitWorkflowCommand.create(workflow)
ModuleVersion.create_or_replace_from_spec({
"id_name": "mod",
"name": "Mod",
"category": "Clean",
"parameters": []
})
wf_module = tab.wf_modules.create(
order=0,
slug="step-1",
last_relevant_delta_id=delta1.id,
module_id_name="mod",
notifications=True,
)
cache_render_result(workflow, wf_module, delta1.id,
RenderResult(arrow_table({"A": [1]})))
# Make a new delta, so we need to re-render. Give it the same output.
delta2 = InitWorkflowCommand.create(workflow)
wf_module.last_relevant_delta_id = delta2.id
wf_module.save(update_fields=["last_relevant_delta_id"])
fake_loaded_module = Mock(LoadedModule)
fake_load_module.return_value = fake_loaded_module
fake_loaded_module.migrate_params.return_value = {}
fake_loaded_module.render.return_value = RenderResult(
arrow_table({"A": [1]}))
self._execute(workflow)
email.assert_not_called()
def _duplicate(
self, name: str, owner: Optional[User], session_key: Optional[str]
) -> "Workflow":
with self.cooperative_lock():
wf = Workflow.objects.create(
name=name,
owner=owner,
original_workflow_id=self.pk,
anonymous_owner_session_key=session_key,
selected_tab_position=self.selected_tab_position,
public=False,
last_delta=None,
)
# Set wf.last_delta and wf.last_delta_id, so we can render.
# Import here to avoid circular deps
from cjwstate.models.commands import InitWorkflowCommand
InitWorkflowCommand.create(wf)
tabs = list(self.live_tabs)
for tab in tabs:
tab.duplicate_into_new_workflow(wf)
return wf
def test_email_no_delta_when_not_changed(self, email):
workflow = Workflow.objects.create()
tab = workflow.tabs.create(position=0)
delta1 = InitWorkflowCommand.create(workflow)
create_module_zipfile(
"mod",
python_code=
'import pandas as pd\ndef render(table, params): return pd.DataFrame({"A": [1]})',
)
wf_module = tab.wf_modules.create(
order=0,
slug="step-1",
last_relevant_delta_id=delta1.id,
module_id_name="mod",
notifications=True,
)
cache_render_result(workflow, wf_module, delta1.id,
RenderResult(arrow_table({"A": [1]})))
# Make a new delta, so we need to re-render. Give it the same output.
delta2 = InitWorkflowCommand.create(workflow)
wf_module.last_relevant_delta_id = delta2.id
wf_module.save(update_fields=["last_relevant_delta_id"])
self._execute(workflow)
email.assert_not_called()
def create_and_init(**kwargs):
"""Create and return a _valid_ Workflow: one with a Tab and a Delta."""
from cjwstate.models.commands import InitWorkflowCommand
with transaction.atomic():
workflow = Workflow.objects.create(**kwargs)
InitWorkflowCommand.create(workflow)
workflow.tabs.create(position=0, slug="tab-1", name="Tab 1")
return workflow
def setUp(self):
super().setUp() # log in
self.queue_render_patcher = patch.object(rabbitmq, "queue_render")
self.queue_render = self.queue_render_patcher.start()
self.queue_render.return_value = future_none
self.log_patcher = patch("server.utils.log_user_event_from_request")
self.log_patch = self.log_patcher.start()
self.factory = APIRequestFactory()
self.workflow1 = Workflow.objects.create(name="Workflow 1", owner=self.user)
self.delta = InitWorkflowCommand.create(self.workflow1)
self.tab1 = self.workflow1.tabs.create(position=0)
self.module_version1 = ModuleVersion.create_or_replace_from_spec(
{
"id_name": "module1",
"name": "Module 1",
"category": "Clean",
"parameters": [],
}
)
# Add another user, with one public and one private workflow
self.otheruser = User.objects.create(
username="******", email="*****@*****.**", password="******"
)
self.other_workflow_private = Workflow.objects.create(
name="Other workflow private", owner=self.otheruser
)
self.other_workflow_public = Workflow.objects.create(
name="Other workflow public", owner=self.otheruser, public=True
)
def test_render_unneeded_execution_so_requeue(self, mock_execute):
mock_execute.side_effect = async_err(execute.UnneededExecution)
workflow = Workflow.objects.create()
delta = InitWorkflowCommand.create(workflow)
ack = Mock(name="ack", side_effect=async_noop)
requeue = Mock(name="requeue", side_effect=async_noop)
async def inner():
with self.assertLogs("renderer", level="INFO") as cm:
await render_workflow_and_maybe_requeue(
SuccessfulRenderLocker(), workflow.id, delta.id, ack,
requeue)
self.assertRegex(
cm.output[0],
r"^INFO:renderer.render:Start execute_workflow\(\d+, \d+\)",
)
self.assertRegex(
cm.output[1],
r"^INFO:renderer.render:End execute_workflow\(\d+, \d+\)",
)
self.assertRegex(
cm.output[2],
r"^INFO:renderer.render:UnneededExecution in execute_workflow\(\d+, \d+\)$",
)
self.run_with_async_db(inner())
ack.assert_called()
requeue.assert_called_with(workflow.id, delta.id)
def test_render_unknown_error_so_crash(self, execute):
# Test what happens when our `renderer.execute` module is buggy and
# raises something it shouldn't raise.
execute.side_effect = FileNotFoundError
workflow = Workflow.objects.create()
delta = InitWorkflowCommand.create(workflow)
ack = Mock(name="ack", side_effect=async_noop)
requeue = Mock(name="requeue", side_effect=async_noop)
async def inner():
with self.assertLogs("renderer", level="INFO") as cm:
await render_workflow_and_maybe_requeue(
SuccessfulRenderLocker(), workflow.id, delta.id, ack,
requeue)
self.assertRegex(
cm.output[0],
r"^INFO:renderer.render:Start execute_workflow\(\d+, \d+\)",
)
self.assertRegex(
cm.output[1],
r"^INFO:renderer.render:End execute_workflow\(\d+, \d+\)",
)
self.assertRegex(
cm.output[2],
r"^ERROR:renderer.render:Error during render of workflow \d+\n",
)
self.assertEqual(len(cm.output), 3)
self.run_with_async_db(inner())
ack.assert_called()
requeue.assert_not_called()
def test_execute_cache_hit(self):
workflow = Workflow.objects.create()
create_module_zipfile("mod")
tab = workflow.tabs.create(position=0)
delta = InitWorkflowCommand.create(workflow)
wf_module1 = tab.wf_modules.create(
order=0,
slug="step-1",
module_id_name="mod",
last_relevant_delta_id=delta.id,
)
cache_render_result(workflow, wf_module1, delta.id,
RenderResult(arrow_table({"A": [1]})))
wf_module2 = tab.wf_modules.create(
order=1,
slug="step-2",
module_id_name="mod",
last_relevant_delta_id=delta.id,
)
cache_render_result(workflow, wf_module2, delta.id,
RenderResult(arrow_table({"B": [2]})))
with patch.object(Kernel, "render", return_value=None):
self._execute(workflow)
Kernel.render.assert_not_called()
def test_change_parameters_on_soft_deleted_tab(self):
workflow = Workflow.objects.create()
delta = InitWorkflowCommand.create(workflow)
tab = workflow.tabs.create(position=0, is_deleted=True)
ModuleVersion.create_or_replace_from_spec({
"id_name":
"loadurl",
"name":
"loadurl",
"category":
"Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
})
wf_module = tab.wf_modules.create(
order=0,
slug="step-1",
module_id_name="loadurl",
last_relevant_delta_id=delta.id,
params={"url": ""},
)
cmd = self.run_with_async_db(
commands.do(
ChangeParametersCommand,
workflow_id=workflow.id,
wf_module=wf_module,
new_values={"url": "https://example.com"},
))
self.assertIsNone(cmd)
def setUp(self):
super().setUp()
self.workflow = Workflow.objects.create()
self.delta = InitWorkflowCommand.create(self.workflow)
self.tab = self.workflow.tabs.create(position=0)
self.wf_module = self.tab.wf_modules.create(
order=0, slug="step-1", last_relevant_delta_id=self.delta.id)
def test_execute_new_revision(self):
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
delta1 = workflow.last_delta
create_module_zipfile(
"mod",
python_code=
'import pandas as pd\ndef render(table, params): return pd.DataFrame({"B": [2]})',
)
wf_module = tab.wf_modules.create(
order=0,
slug="step-1",
last_relevant_delta_id=delta1.id,
module_id_name="mod",
)
result1 = RenderResult(arrow_table({"A": [1]}))
cache_render_result(workflow, wf_module, delta1.id, result1)
delta2 = InitWorkflowCommand.create(workflow)
wf_module.last_relevant_delta_id = delta2.id
wf_module.save(update_fields=["last_relevant_delta_id"])
self._execute(workflow)
wf_module.refresh_from_db()
with open_cached_render_result(
wf_module.cached_render_result) as result:
assert_render_result_equals(result,
RenderResult(arrow_table({"B": [2]})))
def test_duplicate_copies_fresh_cache(self):
# The cache's filename depends on workflow_id and wf_module_id.
# Duplicating it would need more complex code :).
result = RenderResult(
arrow_table({"A": [1]}), [RenderError(I18nMessage("X", []), [])], {}
)
cache_render_result(self.workflow, self.wf_module, self.delta.id, result)
workflow2 = Workflow.objects.create()
tab2 = workflow2.tabs.create(position=0)
InitWorkflowCommand.create(workflow2)
dup = self.wf_module.duplicate_into_new_workflow(tab2)
dup_cached_result = dup.cached_render_result
with open_cached_render_result(dup_cached_result) as result2:
self.assertEqual(result2, result)
def _init_workflow_for_lesson(workflow, lesson):
InitWorkflowCommand.create(workflow)
# Create each wfModule of each tab
tab_dicts = lesson.initial_workflow.tabs
for position, tab_dict in enumerate(tab_dicts):
# Set selected module to last wfmodule in stack
tab = workflow.tabs.create(
position=position,
slug=f"tab-{position + 1}",
name=tab_dict["name"],
selected_wf_module_position=len(tab_dict["wfModules"]) - 1,
)
for order, wfm in enumerate(tab_dict["wfModules"]):
_add_wf_module_to_tab(wfm, order, tab, workflow.last_delta_id,
lesson)
def test_duplicate_ignores_stale_cache(self):
# The cache's filename depends on workflow_id and wf_module_id.
# Duplicating it would need more complex code :).
result = RenderResult(
arrow_table({"A": [1]}), [RenderError(I18nMessage("X", []), [])], {}
)
cache_render_result(self.workflow, self.wf_module, self.delta.id, result)
# Now simulate a new delta that hasn't been rendered
self.wf_module.last_relevant_delta_id += 1
self.wf_module.save(update_fields=["last_relevant_delta_id"])
workflow2 = Workflow.objects.create()
tab2 = workflow2.tabs.create(position=0)
InitWorkflowCommand.create(workflow2)
dup = self.wf_module.duplicate_into_new_workflow(tab2)
dup_cached_result = dup.cached_render_result
self.assertIsNone(dup_cached_result)
self.assertEqual(dup.cached_render_result_status, None)
def test_wf_module_duplicate_disable_auto_update(self):
"""
Duplicates should be lightweight by default: no auto-updating.
"""
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
wf_module = tab.wf_modules.create(
order=0,
slug="step-1",
auto_update_data=True,
next_update=timezone.now(),
update_interval=600,
)
workflow2 = Workflow.create_and_init()
InitWorkflowCommand.create(workflow2)
tab2 = workflow2.tabs.create(position=0)
wf_module2 = wf_module.duplicate_into_new_workflow(tab2)
self.assertEqual(wf_module2.auto_update_data, False)
self.assertIsNone(wf_module2.next_update)
self.assertEqual(wf_module2.update_interval, 600)
def test_email_no_delta_when_no_cached_render_result(
self, email, fake_load_module):
# No cached render result means one of two things:
#
# 1. This is a new module (in which case, why email the user?)
# 2. We cleared the render cache (in which case, better to skip emailing a few
# users than to email _every_ user that results have changed when they haven't)
workflow = Workflow.objects.create()
tab = workflow.tabs.create(position=0)
delta1 = InitWorkflowCommand.create(workflow)
ModuleVersion.create_or_replace_from_spec({
"id_name": "mod",
"name": "Mod",
"category": "Clean",
"parameters": []
})
wf_module = tab.wf_modules.create(
order=0,
slug="step-1",
last_relevant_delta_id=delta1.id,
module_id_name="mod",
notifications=True,
)
# Make a new delta, so we need to re-render. Give it the same output.
delta2 = InitWorkflowCommand.create(workflow)
wf_module.last_relevant_delta_id = delta2.id
wf_module.save(update_fields=["last_relevant_delta_id"])
fake_loaded_module = Mock(LoadedModule)
fake_load_module.return_value = fake_loaded_module
fake_loaded_module.migrate_params.return_value = {}
fake_loaded_module.render.return_value = RenderResult(
arrow_table({"A": [1]}))
self._execute(workflow)
email.assert_not_called()
def test_render_happy_path(self, execute):
execute.side_effect = async_noop
workflow = Workflow.objects.create()
delta = InitWorkflowCommand.create(workflow)
ack = Mock(name="ack", side_effect=async_noop)
requeue = Mock(name="requeue", side_effect=async_noop)
with self.assertLogs():
self.run_with_async_db(
render_workflow_and_maybe_requeue(SuccessfulRenderLocker(),
workflow.id, delta.id, ack,
requeue))
execute.assert_called_with(workflow, delta.id)
ack.assert_called()
requeue.assert_not_called()
def setUp(self):
super().setUp()
self.workflow = Workflow.objects.create()
self.tab = self.workflow.tabs.create(position=0)
self.module_zipfile = create_module_zipfile(
"loadsomething",
spec_kwargs={"parameters": [{
"id_name": "url",
"type": "string"
}]},
)
self.kernel.migrate_params.side_effect = RuntimeError(
"AddModuleCommand and tests should cache migrated params correctly"
)
self.delta = InitWorkflowCommand.create(self.workflow)
def test_execute_cache_hit(self, fake_module):
workflow = Workflow.objects.create()
tab = workflow.tabs.create(position=0)
delta = InitWorkflowCommand.create(workflow)
wf_module1 = tab.wf_modules.create(order=0,
slug="step-1",
last_relevant_delta_id=delta.id)
cache_render_result(workflow, wf_module1, delta.id,
RenderResult(arrow_table({"A": [1]})))
wf_module2 = tab.wf_modules.create(order=1,
slug="step-2",
last_relevant_delta_id=delta.id)
cache_render_result(workflow, wf_module2, delta.id,
RenderResult(arrow_table({"B": [2]})))
self._execute(workflow)
fake_module.assert_not_called()
def test_workflow_view_triggers_render_if_stale_cache(self):
step = self.tab1.wf_modules.create(
order=0,
slug="step-1",
last_relevant_delta_id=self.delta.id,
cached_render_result_delta_id=self.delta.id, # stale
)
# Cache a result
cache_render_result(self.workflow1, step, self.delta.id,
RenderResult(arrow_table({"A": ["a"]})))
# Make the cached result stale. (The view will actually send the
# stale-result metadata to the client. That's why we cached it.)
delta2 = InitWorkflowCommand.create(self.workflow1)
step.last_relevant_delta_id = delta2.id
step.save(update_fields=["last_relevant_delta_id"])
self.client.force_login(self.user)
self.client.get("/workflows/%d/" % self.workflow1.id)
self.queue_render.assert_called_with(self.workflow1.id, delta2.id)
def setUp(self):
super().setUp()
self.workflow = Workflow.objects.create()
self.tab = self.workflow.tabs.create(position=0)
self.module_version = ModuleVersion.create_or_replace_from_spec(
{
"id_name": "loadurl",
"name": "Load URL",
"category": "Clean",
"parameters": [{
"id_name": "url",
"type": "string"
}],
},
source_version_hash="1.0",
)
self.delta = InitWorkflowCommand.create(self.workflow)
def test_render_unneeded_execution_so_requeue(self, mock_execute):
mock_execute.side_effect = execute.UnneededExecution
workflow = Workflow.objects.create()
delta = InitWorkflowCommand.create(workflow)
ack = Mock(name="ack", side_effect=async_noop)
requeue = Mock(name="requeue", side_effect=async_noop)
async def inner():
with self.assertLogs("renderer", level="INFO") as cm:
await render_workflow_and_maybe_requeue(
SuccessfulRenderLocker(), workflow.id, delta.id, ack,
requeue)
self.assertEqual(
cm.output,
[(f"INFO:renderer.render:UnneededExecution in "
f"execute_workflow({workflow.id}, {delta.id})")],
)
self.run_with_async_db(inner())
ack.assert_called()
requeue.assert_called_with(workflow.id, delta.id)
def test_change_parameters_on_soft_deleted_tab(self):
workflow = Workflow.objects.create()
delta = InitWorkflowCommand.create(workflow)
tab = workflow.tabs.create(position=0, is_deleted=True)
wf_module = tab.wf_modules.create(
order=0,
slug="step-1",
module_id_name="loadurl",
last_relevant_delta_id=delta.id,
params={"url": ""},
)
cmd = self.run_with_async_db(
commands.do(
ChangeParametersCommand,
workflow_id=workflow.id,
wf_module=wf_module,
new_values={"url": "https://example.com"},
))
self.assertIsNone(cmd)
def test_render_other_renderer_rendering_so_skip(self, execute):
execute.side_effect = async_noop
workflow = Workflow.objects.create()
delta = InitWorkflowCommand.create(workflow)
ack = Mock(name="ack", side_effect=async_noop)
requeue = Mock(name="requeue", side_effect=async_noop)
async def inner():
with self.assertLogs("renderer", level="INFO") as cm:
await render_workflow_and_maybe_requeue(
FailedRenderLocker(), workflow.id, delta.id, ack, requeue)
self.assertEqual(
cm.output,
[(f"INFO:renderer.render:Workflow {workflow.id} is "
"being rendered elsewhere; ignoring")],
)
self.run_with_async_db(inner())
execute.assert_not_called()
ack.assert_called()
requeue.assert_not_called()
def test_execute_new_revision(self, fake_load_module):
workflow = Workflow.create_and_init()
tab = workflow.tabs.first()
delta1 = workflow.last_delta
ModuleVersion.create_or_replace_from_spec({
"id_name": "mod",
"name": "Mod",
"category": "Clean",
"parameters": []
})
wf_module = tab.wf_modules.create(
order=0,
slug="step-1",
last_relevant_delta_id=delta1.id,
module_id_name="mod",
)
result1 = RenderResult(arrow_table({"A": [1]}))
cache_render_result(workflow, wf_module, delta1.id, result1)
delta2 = InitWorkflowCommand.create(workflow)
wf_module.last_relevant_delta_id = delta2.id
wf_module.save(update_fields=["last_relevant_delta_id"])
result2 = RenderResult(arrow_table({"B": [2]}))
fake_module = Mock(LoadedModule)
fake_module.migrate_params.return_value = {}
fake_load_module.return_value = fake_module
fake_module.render.return_value = result2
self._execute(workflow)
wf_module.refresh_from_db()
with open_cached_render_result(
wf_module.cached_render_result) as result:
assert_render_result_equals(result, result2)
def setUp(self):
super().setUp()
self.workflow = Workflow.objects.create()
self.delta = InitWorkflowCommand.create(self.workflow)
self.tab = self.workflow.tabs.create(position=0, slug="tab-1")