def clear_deltas(self): """Become a single-Delta Workflow.""" from cjwstate.models.commands import InitWorkflowCommand try: from cjwstate.models import Delta first_delta = self.deltas.get(prev_delta_id=None) except Delta.DoesNotExist: # Invariant failed. Defensive programming: recover. first_delta = InitWorkflowCommand.create(self) if not isinstance(first_delta, InitWorkflowCommand): # Invariant failed: first delta should be InitWorkflowCommand. # Defensive programming: recover. Delete _every_ Delta, and then # add the one that belongs. first_delta.delete() first_delta = InitWorkflowCommand.create(self) else: self.last_delta_id = first_delta.id self.save(update_fields=["last_delta_id"]) try: # Select the _second_ delta. second_delta = first_delta.next_delta except Delta.DoesNotExist: # We're already a 1-delta Workflow return second_delta.delete_with_successors() self.delete_orphan_soft_deleted_models()
def test_email_no_delta_when_no_cached_render_result(self, email): # No cached render result means one of two things: # # 1. This is a new module (in which case, why email the user?) # 2. We cleared the render cache (in which case, better to skip emailing a few # users than to email _every_ user that results have changed when they haven't) workflow = Workflow.objects.create() tab = workflow.tabs.create(position=0) delta1 = InitWorkflowCommand.create(workflow) create_module_zipfile( "mod", python_code= 'import pandas as pd\ndef render(table, params): return pd.DataFrame({"A": [1]})', ) wf_module = tab.wf_modules.create( order=0, slug="step-1", last_relevant_delta_id=delta1.id, module_id_name="mod", notifications=True, ) # Make a new delta, so we need to re-render. Give it the same output. delta2 = InitWorkflowCommand.create(workflow) wf_module.last_relevant_delta_id = delta2.id wf_module.save(update_fields=["last_relevant_delta_id"]) self._execute(workflow) email.assert_not_called()
def test_email_no_delta_when_not_changed(self, email, fake_load_module): workflow = Workflow.objects.create() tab = workflow.tabs.create(position=0) delta1 = InitWorkflowCommand.create(workflow) ModuleVersion.create_or_replace_from_spec({ "id_name": "mod", "name": "Mod", "category": "Clean", "parameters": [] }) wf_module = tab.wf_modules.create( order=0, slug="step-1", last_relevant_delta_id=delta1.id, module_id_name="mod", notifications=True, ) cache_render_result(workflow, wf_module, delta1.id, RenderResult(arrow_table({"A": [1]}))) # Make a new delta, so we need to re-render. Give it the same output. delta2 = InitWorkflowCommand.create(workflow) wf_module.last_relevant_delta_id = delta2.id wf_module.save(update_fields=["last_relevant_delta_id"]) fake_loaded_module = Mock(LoadedModule) fake_load_module.return_value = fake_loaded_module fake_loaded_module.migrate_params.return_value = {} fake_loaded_module.render.return_value = RenderResult( arrow_table({"A": [1]})) self._execute(workflow) email.assert_not_called()
def _duplicate( self, name: str, owner: Optional[User], session_key: Optional[str] ) -> "Workflow": with self.cooperative_lock(): wf = Workflow.objects.create( name=name, owner=owner, original_workflow_id=self.pk, anonymous_owner_session_key=session_key, selected_tab_position=self.selected_tab_position, public=False, last_delta=None, ) # Set wf.last_delta and wf.last_delta_id, so we can render. # Import here to avoid circular deps from cjwstate.models.commands import InitWorkflowCommand InitWorkflowCommand.create(wf) tabs = list(self.live_tabs) for tab in tabs: tab.duplicate_into_new_workflow(wf) return wf
def test_email_no_delta_when_not_changed(self, email): workflow = Workflow.objects.create() tab = workflow.tabs.create(position=0) delta1 = InitWorkflowCommand.create(workflow) create_module_zipfile( "mod", python_code= 'import pandas as pd\ndef render(table, params): return pd.DataFrame({"A": [1]})', ) wf_module = tab.wf_modules.create( order=0, slug="step-1", last_relevant_delta_id=delta1.id, module_id_name="mod", notifications=True, ) cache_render_result(workflow, wf_module, delta1.id, RenderResult(arrow_table({"A": [1]}))) # Make a new delta, so we need to re-render. Give it the same output. delta2 = InitWorkflowCommand.create(workflow) wf_module.last_relevant_delta_id = delta2.id wf_module.save(update_fields=["last_relevant_delta_id"]) self._execute(workflow) email.assert_not_called()
def create_and_init(**kwargs): """Create and return a _valid_ Workflow: one with a Tab and a Delta.""" from cjwstate.models.commands import InitWorkflowCommand with transaction.atomic(): workflow = Workflow.objects.create(**kwargs) InitWorkflowCommand.create(workflow) workflow.tabs.create(position=0, slug="tab-1", name="Tab 1") return workflow
def setUp(self): super().setUp() # log in self.queue_render_patcher = patch.object(rabbitmq, "queue_render") self.queue_render = self.queue_render_patcher.start() self.queue_render.return_value = future_none self.log_patcher = patch("server.utils.log_user_event_from_request") self.log_patch = self.log_patcher.start() self.factory = APIRequestFactory() self.workflow1 = Workflow.objects.create(name="Workflow 1", owner=self.user) self.delta = InitWorkflowCommand.create(self.workflow1) self.tab1 = self.workflow1.tabs.create(position=0) self.module_version1 = ModuleVersion.create_or_replace_from_spec( { "id_name": "module1", "name": "Module 1", "category": "Clean", "parameters": [], } ) # Add another user, with one public and one private workflow self.otheruser = User.objects.create( username="******", email="*****@*****.**", password="******" ) self.other_workflow_private = Workflow.objects.create( name="Other workflow private", owner=self.otheruser ) self.other_workflow_public = Workflow.objects.create( name="Other workflow public", owner=self.otheruser, public=True )
def test_render_unneeded_execution_so_requeue(self, mock_execute): mock_execute.side_effect = async_err(execute.UnneededExecution) workflow = Workflow.objects.create() delta = InitWorkflowCommand.create(workflow) ack = Mock(name="ack", side_effect=async_noop) requeue = Mock(name="requeue", side_effect=async_noop) async def inner(): with self.assertLogs("renderer", level="INFO") as cm: await render_workflow_and_maybe_requeue( SuccessfulRenderLocker(), workflow.id, delta.id, ack, requeue) self.assertRegex( cm.output[0], r"^INFO:renderer.render:Start execute_workflow\(\d+, \d+\)", ) self.assertRegex( cm.output[1], r"^INFO:renderer.render:End execute_workflow\(\d+, \d+\)", ) self.assertRegex( cm.output[2], r"^INFO:renderer.render:UnneededExecution in execute_workflow\(\d+, \d+\)$", ) self.run_with_async_db(inner()) ack.assert_called() requeue.assert_called_with(workflow.id, delta.id)
def test_render_unknown_error_so_crash(self, execute): # Test what happens when our `renderer.execute` module is buggy and # raises something it shouldn't raise. execute.side_effect = FileNotFoundError workflow = Workflow.objects.create() delta = InitWorkflowCommand.create(workflow) ack = Mock(name="ack", side_effect=async_noop) requeue = Mock(name="requeue", side_effect=async_noop) async def inner(): with self.assertLogs("renderer", level="INFO") as cm: await render_workflow_and_maybe_requeue( SuccessfulRenderLocker(), workflow.id, delta.id, ack, requeue) self.assertRegex( cm.output[0], r"^INFO:renderer.render:Start execute_workflow\(\d+, \d+\)", ) self.assertRegex( cm.output[1], r"^INFO:renderer.render:End execute_workflow\(\d+, \d+\)", ) self.assertRegex( cm.output[2], r"^ERROR:renderer.render:Error during render of workflow \d+\n", ) self.assertEqual(len(cm.output), 3) self.run_with_async_db(inner()) ack.assert_called() requeue.assert_not_called()
def test_execute_cache_hit(self): workflow = Workflow.objects.create() create_module_zipfile("mod") tab = workflow.tabs.create(position=0) delta = InitWorkflowCommand.create(workflow) wf_module1 = tab.wf_modules.create( order=0, slug="step-1", module_id_name="mod", last_relevant_delta_id=delta.id, ) cache_render_result(workflow, wf_module1, delta.id, RenderResult(arrow_table({"A": [1]}))) wf_module2 = tab.wf_modules.create( order=1, slug="step-2", module_id_name="mod", last_relevant_delta_id=delta.id, ) cache_render_result(workflow, wf_module2, delta.id, RenderResult(arrow_table({"B": [2]}))) with patch.object(Kernel, "render", return_value=None): self._execute(workflow) Kernel.render.assert_not_called()
def test_change_parameters_on_soft_deleted_tab(self): workflow = Workflow.objects.create() delta = InitWorkflowCommand.create(workflow) tab = workflow.tabs.create(position=0, is_deleted=True) ModuleVersion.create_or_replace_from_spec({ "id_name": "loadurl", "name": "loadurl", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }) wf_module = tab.wf_modules.create( order=0, slug="step-1", module_id_name="loadurl", last_relevant_delta_id=delta.id, params={"url": ""}, ) cmd = self.run_with_async_db( commands.do( ChangeParametersCommand, workflow_id=workflow.id, wf_module=wf_module, new_values={"url": "https://example.com"}, )) self.assertIsNone(cmd)
def setUp(self): super().setUp() self.workflow = Workflow.objects.create() self.delta = InitWorkflowCommand.create(self.workflow) self.tab = self.workflow.tabs.create(position=0) self.wf_module = self.tab.wf_modules.create( order=0, slug="step-1", last_relevant_delta_id=self.delta.id)
def test_execute_new_revision(self): workflow = Workflow.create_and_init() tab = workflow.tabs.first() delta1 = workflow.last_delta create_module_zipfile( "mod", python_code= 'import pandas as pd\ndef render(table, params): return pd.DataFrame({"B": [2]})', ) wf_module = tab.wf_modules.create( order=0, slug="step-1", last_relevant_delta_id=delta1.id, module_id_name="mod", ) result1 = RenderResult(arrow_table({"A": [1]})) cache_render_result(workflow, wf_module, delta1.id, result1) delta2 = InitWorkflowCommand.create(workflow) wf_module.last_relevant_delta_id = delta2.id wf_module.save(update_fields=["last_relevant_delta_id"]) self._execute(workflow) wf_module.refresh_from_db() with open_cached_render_result( wf_module.cached_render_result) as result: assert_render_result_equals(result, RenderResult(arrow_table({"B": [2]})))
def test_duplicate_copies_fresh_cache(self): # The cache's filename depends on workflow_id and wf_module_id. # Duplicating it would need more complex code :). result = RenderResult( arrow_table({"A": [1]}), [RenderError(I18nMessage("X", []), [])], {} ) cache_render_result(self.workflow, self.wf_module, self.delta.id, result) workflow2 = Workflow.objects.create() tab2 = workflow2.tabs.create(position=0) InitWorkflowCommand.create(workflow2) dup = self.wf_module.duplicate_into_new_workflow(tab2) dup_cached_result = dup.cached_render_result with open_cached_render_result(dup_cached_result) as result2: self.assertEqual(result2, result)
def _init_workflow_for_lesson(workflow, lesson): InitWorkflowCommand.create(workflow) # Create each wfModule of each tab tab_dicts = lesson.initial_workflow.tabs for position, tab_dict in enumerate(tab_dicts): # Set selected module to last wfmodule in stack tab = workflow.tabs.create( position=position, slug=f"tab-{position + 1}", name=tab_dict["name"], selected_wf_module_position=len(tab_dict["wfModules"]) - 1, ) for order, wfm in enumerate(tab_dict["wfModules"]): _add_wf_module_to_tab(wfm, order, tab, workflow.last_delta_id, lesson)
def test_duplicate_ignores_stale_cache(self): # The cache's filename depends on workflow_id and wf_module_id. # Duplicating it would need more complex code :). result = RenderResult( arrow_table({"A": [1]}), [RenderError(I18nMessage("X", []), [])], {} ) cache_render_result(self.workflow, self.wf_module, self.delta.id, result) # Now simulate a new delta that hasn't been rendered self.wf_module.last_relevant_delta_id += 1 self.wf_module.save(update_fields=["last_relevant_delta_id"]) workflow2 = Workflow.objects.create() tab2 = workflow2.tabs.create(position=0) InitWorkflowCommand.create(workflow2) dup = self.wf_module.duplicate_into_new_workflow(tab2) dup_cached_result = dup.cached_render_result self.assertIsNone(dup_cached_result) self.assertEqual(dup.cached_render_result_status, None)
def test_wf_module_duplicate_disable_auto_update(self): """ Duplicates should be lightweight by default: no auto-updating. """ workflow = Workflow.create_and_init() tab = workflow.tabs.first() wf_module = tab.wf_modules.create( order=0, slug="step-1", auto_update_data=True, next_update=timezone.now(), update_interval=600, ) workflow2 = Workflow.create_and_init() InitWorkflowCommand.create(workflow2) tab2 = workflow2.tabs.create(position=0) wf_module2 = wf_module.duplicate_into_new_workflow(tab2) self.assertEqual(wf_module2.auto_update_data, False) self.assertIsNone(wf_module2.next_update) self.assertEqual(wf_module2.update_interval, 600)
def test_email_no_delta_when_no_cached_render_result( self, email, fake_load_module): # No cached render result means one of two things: # # 1. This is a new module (in which case, why email the user?) # 2. We cleared the render cache (in which case, better to skip emailing a few # users than to email _every_ user that results have changed when they haven't) workflow = Workflow.objects.create() tab = workflow.tabs.create(position=0) delta1 = InitWorkflowCommand.create(workflow) ModuleVersion.create_or_replace_from_spec({ "id_name": "mod", "name": "Mod", "category": "Clean", "parameters": [] }) wf_module = tab.wf_modules.create( order=0, slug="step-1", last_relevant_delta_id=delta1.id, module_id_name="mod", notifications=True, ) # Make a new delta, so we need to re-render. Give it the same output. delta2 = InitWorkflowCommand.create(workflow) wf_module.last_relevant_delta_id = delta2.id wf_module.save(update_fields=["last_relevant_delta_id"]) fake_loaded_module = Mock(LoadedModule) fake_load_module.return_value = fake_loaded_module fake_loaded_module.migrate_params.return_value = {} fake_loaded_module.render.return_value = RenderResult( arrow_table({"A": [1]})) self._execute(workflow) email.assert_not_called()
def test_render_happy_path(self, execute): execute.side_effect = async_noop workflow = Workflow.objects.create() delta = InitWorkflowCommand.create(workflow) ack = Mock(name="ack", side_effect=async_noop) requeue = Mock(name="requeue", side_effect=async_noop) with self.assertLogs(): self.run_with_async_db( render_workflow_and_maybe_requeue(SuccessfulRenderLocker(), workflow.id, delta.id, ack, requeue)) execute.assert_called_with(workflow, delta.id) ack.assert_called() requeue.assert_not_called()
def setUp(self): super().setUp() self.workflow = Workflow.objects.create() self.tab = self.workflow.tabs.create(position=0) self.module_zipfile = create_module_zipfile( "loadsomething", spec_kwargs={"parameters": [{ "id_name": "url", "type": "string" }]}, ) self.kernel.migrate_params.side_effect = RuntimeError( "AddModuleCommand and tests should cache migrated params correctly" ) self.delta = InitWorkflowCommand.create(self.workflow)
def test_execute_cache_hit(self, fake_module): workflow = Workflow.objects.create() tab = workflow.tabs.create(position=0) delta = InitWorkflowCommand.create(workflow) wf_module1 = tab.wf_modules.create(order=0, slug="step-1", last_relevant_delta_id=delta.id) cache_render_result(workflow, wf_module1, delta.id, RenderResult(arrow_table({"A": [1]}))) wf_module2 = tab.wf_modules.create(order=1, slug="step-2", last_relevant_delta_id=delta.id) cache_render_result(workflow, wf_module2, delta.id, RenderResult(arrow_table({"B": [2]}))) self._execute(workflow) fake_module.assert_not_called()
def test_workflow_view_triggers_render_if_stale_cache(self): step = self.tab1.wf_modules.create( order=0, slug="step-1", last_relevant_delta_id=self.delta.id, cached_render_result_delta_id=self.delta.id, # stale ) # Cache a result cache_render_result(self.workflow1, step, self.delta.id, RenderResult(arrow_table({"A": ["a"]}))) # Make the cached result stale. (The view will actually send the # stale-result metadata to the client. That's why we cached it.) delta2 = InitWorkflowCommand.create(self.workflow1) step.last_relevant_delta_id = delta2.id step.save(update_fields=["last_relevant_delta_id"]) self.client.force_login(self.user) self.client.get("/workflows/%d/" % self.workflow1.id) self.queue_render.assert_called_with(self.workflow1.id, delta2.id)
def setUp(self): super().setUp() self.workflow = Workflow.objects.create() self.tab = self.workflow.tabs.create(position=0) self.module_version = ModuleVersion.create_or_replace_from_spec( { "id_name": "loadurl", "name": "Load URL", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }, source_version_hash="1.0", ) self.delta = InitWorkflowCommand.create(self.workflow)
def test_render_unneeded_execution_so_requeue(self, mock_execute): mock_execute.side_effect = execute.UnneededExecution workflow = Workflow.objects.create() delta = InitWorkflowCommand.create(workflow) ack = Mock(name="ack", side_effect=async_noop) requeue = Mock(name="requeue", side_effect=async_noop) async def inner(): with self.assertLogs("renderer", level="INFO") as cm: await render_workflow_and_maybe_requeue( SuccessfulRenderLocker(), workflow.id, delta.id, ack, requeue) self.assertEqual( cm.output, [(f"INFO:renderer.render:UnneededExecution in " f"execute_workflow({workflow.id}, {delta.id})")], ) self.run_with_async_db(inner()) ack.assert_called() requeue.assert_called_with(workflow.id, delta.id)
def test_change_parameters_on_soft_deleted_tab(self): workflow = Workflow.objects.create() delta = InitWorkflowCommand.create(workflow) tab = workflow.tabs.create(position=0, is_deleted=True) wf_module = tab.wf_modules.create( order=0, slug="step-1", module_id_name="loadurl", last_relevant_delta_id=delta.id, params={"url": ""}, ) cmd = self.run_with_async_db( commands.do( ChangeParametersCommand, workflow_id=workflow.id, wf_module=wf_module, new_values={"url": "https://example.com"}, )) self.assertIsNone(cmd)
def test_render_other_renderer_rendering_so_skip(self, execute): execute.side_effect = async_noop workflow = Workflow.objects.create() delta = InitWorkflowCommand.create(workflow) ack = Mock(name="ack", side_effect=async_noop) requeue = Mock(name="requeue", side_effect=async_noop) async def inner(): with self.assertLogs("renderer", level="INFO") as cm: await render_workflow_and_maybe_requeue( FailedRenderLocker(), workflow.id, delta.id, ack, requeue) self.assertEqual( cm.output, [(f"INFO:renderer.render:Workflow {workflow.id} is " "being rendered elsewhere; ignoring")], ) self.run_with_async_db(inner()) execute.assert_not_called() ack.assert_called() requeue.assert_not_called()
def test_execute_new_revision(self, fake_load_module): workflow = Workflow.create_and_init() tab = workflow.tabs.first() delta1 = workflow.last_delta ModuleVersion.create_or_replace_from_spec({ "id_name": "mod", "name": "Mod", "category": "Clean", "parameters": [] }) wf_module = tab.wf_modules.create( order=0, slug="step-1", last_relevant_delta_id=delta1.id, module_id_name="mod", ) result1 = RenderResult(arrow_table({"A": [1]})) cache_render_result(workflow, wf_module, delta1.id, result1) delta2 = InitWorkflowCommand.create(workflow) wf_module.last_relevant_delta_id = delta2.id wf_module.save(update_fields=["last_relevant_delta_id"]) result2 = RenderResult(arrow_table({"B": [2]})) fake_module = Mock(LoadedModule) fake_module.migrate_params.return_value = {} fake_load_module.return_value = fake_module fake_module.render.return_value = result2 self._execute(workflow) wf_module.refresh_from_db() with open_cached_render_result( wf_module.cached_render_result) as result: assert_render_result_equals(result, result2)
def setUp(self): super().setUp() self.workflow = Workflow.objects.create() self.delta = InitWorkflowCommand.create(self.workflow) self.tab = self.workflow.tabs.create(position=0, slug="tab-1")