Ejemplo n.º 1
0
    def clear_deltas(self):
        """Become a single-Delta Workflow."""
        from cjwstate.models.commands import InitWorkflowCommand

        try:
            from cjwstate.models import Delta

            first_delta = self.deltas.get(prev_delta_id=None)
        except Delta.DoesNotExist:
            # Invariant failed. Defensive programming: recover.
            first_delta = InitWorkflowCommand.create(self)

        if not isinstance(first_delta, InitWorkflowCommand):
            # Invariant failed: first delta should be InitWorkflowCommand.
            # Defensive programming: recover. Delete _every_ Delta, and then
            # add the one that belongs.
            first_delta.delete()
            first_delta = InitWorkflowCommand.create(self)
        else:
            self.last_delta_id = first_delta.id
            self.save(update_fields=["last_delta_id"])

        try:
            # Select the _second_ delta.
            second_delta = first_delta.next_delta
        except Delta.DoesNotExist:
            # We're already a 1-delta Workflow
            return

        second_delta.delete_with_successors()
        self.delete_orphan_soft_deleted_models()
Ejemplo n.º 2
0
    def test_email_no_delta_when_no_cached_render_result(self, email):
        # No cached render result means one of two things:
        #
        # 1. This is a new module (in which case, why email the user?)
        # 2. We cleared the render cache (in which case, better to skip emailing a few
        #    users than to email _every_ user that results have changed when they haven't)

        workflow = Workflow.objects.create()
        tab = workflow.tabs.create(position=0)
        delta1 = InitWorkflowCommand.create(workflow)
        create_module_zipfile(
            "mod",
            python_code=
            'import pandas as pd\ndef render(table, params): return pd.DataFrame({"A": [1]})',
        )
        wf_module = tab.wf_modules.create(
            order=0,
            slug="step-1",
            last_relevant_delta_id=delta1.id,
            module_id_name="mod",
            notifications=True,
        )

        # Make a new delta, so we need to re-render. Give it the same output.
        delta2 = InitWorkflowCommand.create(workflow)
        wf_module.last_relevant_delta_id = delta2.id
        wf_module.save(update_fields=["last_relevant_delta_id"])

        self._execute(workflow)

        email.assert_not_called()
Ejemplo n.º 3
0
    def test_email_no_delta_when_not_changed(self, email, fake_load_module):
        workflow = Workflow.objects.create()
        tab = workflow.tabs.create(position=0)
        delta1 = InitWorkflowCommand.create(workflow)
        ModuleVersion.create_or_replace_from_spec({
            "id_name": "mod",
            "name": "Mod",
            "category": "Clean",
            "parameters": []
        })
        wf_module = tab.wf_modules.create(
            order=0,
            slug="step-1",
            last_relevant_delta_id=delta1.id,
            module_id_name="mod",
            notifications=True,
        )
        cache_render_result(workflow, wf_module, delta1.id,
                            RenderResult(arrow_table({"A": [1]})))

        # Make a new delta, so we need to re-render. Give it the same output.
        delta2 = InitWorkflowCommand.create(workflow)
        wf_module.last_relevant_delta_id = delta2.id
        wf_module.save(update_fields=["last_relevant_delta_id"])

        fake_loaded_module = Mock(LoadedModule)
        fake_load_module.return_value = fake_loaded_module
        fake_loaded_module.migrate_params.return_value = {}
        fake_loaded_module.render.return_value = RenderResult(
            arrow_table({"A": [1]}))

        self._execute(workflow)

        email.assert_not_called()
Ejemplo n.º 4
0
    def _duplicate(
        self, name: str, owner: Optional[User], session_key: Optional[str]
    ) -> "Workflow":
        with self.cooperative_lock():
            wf = Workflow.objects.create(
                name=name,
                owner=owner,
                original_workflow_id=self.pk,
                anonymous_owner_session_key=session_key,
                selected_tab_position=self.selected_tab_position,
                public=False,
                last_delta=None,
            )

            # Set wf.last_delta and wf.last_delta_id, so we can render.
            # Import here to avoid circular deps
            from cjwstate.models.commands import InitWorkflowCommand

            InitWorkflowCommand.create(wf)

            tabs = list(self.live_tabs)
            for tab in tabs:
                tab.duplicate_into_new_workflow(wf)

        return wf
Ejemplo n.º 5
0
    def test_email_no_delta_when_not_changed(self, email):
        workflow = Workflow.objects.create()
        tab = workflow.tabs.create(position=0)
        delta1 = InitWorkflowCommand.create(workflow)
        create_module_zipfile(
            "mod",
            python_code=
            'import pandas as pd\ndef render(table, params): return pd.DataFrame({"A": [1]})',
        )
        wf_module = tab.wf_modules.create(
            order=0,
            slug="step-1",
            last_relevant_delta_id=delta1.id,
            module_id_name="mod",
            notifications=True,
        )
        cache_render_result(workflow, wf_module, delta1.id,
                            RenderResult(arrow_table({"A": [1]})))

        # Make a new delta, so we need to re-render. Give it the same output.
        delta2 = InitWorkflowCommand.create(workflow)
        wf_module.last_relevant_delta_id = delta2.id
        wf_module.save(update_fields=["last_relevant_delta_id"])

        self._execute(workflow)

        email.assert_not_called()
Ejemplo n.º 6
0
    def create_and_init(**kwargs):
        """Create and return a _valid_ Workflow: one with a Tab and a Delta."""
        from cjwstate.models.commands import InitWorkflowCommand

        with transaction.atomic():
            workflow = Workflow.objects.create(**kwargs)
            InitWorkflowCommand.create(workflow)
            workflow.tabs.create(position=0, slug="tab-1", name="Tab 1")
            return workflow
Ejemplo n.º 7
0
    def setUp(self):
        super().setUp()  # log in

        self.queue_render_patcher = patch.object(rabbitmq, "queue_render")
        self.queue_render = self.queue_render_patcher.start()
        self.queue_render.return_value = future_none

        self.log_patcher = patch("server.utils.log_user_event_from_request")
        self.log_patch = self.log_patcher.start()

        self.factory = APIRequestFactory()
        self.workflow1 = Workflow.objects.create(name="Workflow 1", owner=self.user)
        self.delta = InitWorkflowCommand.create(self.workflow1)
        self.tab1 = self.workflow1.tabs.create(position=0)
        self.module_version1 = ModuleVersion.create_or_replace_from_spec(
            {
                "id_name": "module1",
                "name": "Module 1",
                "category": "Clean",
                "parameters": [],
            }
        )

        # Add another user, with one public and one private workflow
        self.otheruser = User.objects.create(
            username="******", email="*****@*****.**", password="******"
        )
        self.other_workflow_private = Workflow.objects.create(
            name="Other workflow private", owner=self.otheruser
        )
        self.other_workflow_public = Workflow.objects.create(
            name="Other workflow public", owner=self.otheruser, public=True
        )
Ejemplo n.º 8
0
    def test_render_unneeded_execution_so_requeue(self, mock_execute):
        mock_execute.side_effect = async_err(execute.UnneededExecution)
        workflow = Workflow.objects.create()
        delta = InitWorkflowCommand.create(workflow)
        ack = Mock(name="ack", side_effect=async_noop)
        requeue = Mock(name="requeue", side_effect=async_noop)

        async def inner():
            with self.assertLogs("renderer", level="INFO") as cm:
                await render_workflow_and_maybe_requeue(
                    SuccessfulRenderLocker(), workflow.id, delta.id, ack,
                    requeue)
                self.assertRegex(
                    cm.output[0],
                    r"^INFO:renderer.render:Start execute_workflow\(\d+, \d+\)",
                )
                self.assertRegex(
                    cm.output[1],
                    r"^INFO:renderer.render:End execute_workflow\(\d+, \d+\)",
                )
                self.assertRegex(
                    cm.output[2],
                    r"^INFO:renderer.render:UnneededExecution in execute_workflow\(\d+, \d+\)$",
                )

        self.run_with_async_db(inner())
        ack.assert_called()
        requeue.assert_called_with(workflow.id, delta.id)
Ejemplo n.º 9
0
    def test_render_unknown_error_so_crash(self, execute):
        # Test what happens when our `renderer.execute` module is buggy and
        # raises something it shouldn't raise.
        execute.side_effect = FileNotFoundError
        workflow = Workflow.objects.create()
        delta = InitWorkflowCommand.create(workflow)
        ack = Mock(name="ack", side_effect=async_noop)
        requeue = Mock(name="requeue", side_effect=async_noop)

        async def inner():
            with self.assertLogs("renderer", level="INFO") as cm:
                await render_workflow_and_maybe_requeue(
                    SuccessfulRenderLocker(), workflow.id, delta.id, ack,
                    requeue)
                self.assertRegex(
                    cm.output[0],
                    r"^INFO:renderer.render:Start execute_workflow\(\d+, \d+\)",
                )
                self.assertRegex(
                    cm.output[1],
                    r"^INFO:renderer.render:End execute_workflow\(\d+, \d+\)",
                )
                self.assertRegex(
                    cm.output[2],
                    r"^ERROR:renderer.render:Error during render of workflow \d+\n",
                )
                self.assertEqual(len(cm.output), 3)

        self.run_with_async_db(inner())
        ack.assert_called()
        requeue.assert_not_called()
Ejemplo n.º 10
0
    def test_execute_cache_hit(self):
        workflow = Workflow.objects.create()
        create_module_zipfile("mod")
        tab = workflow.tabs.create(position=0)
        delta = InitWorkflowCommand.create(workflow)
        wf_module1 = tab.wf_modules.create(
            order=0,
            slug="step-1",
            module_id_name="mod",
            last_relevant_delta_id=delta.id,
        )
        cache_render_result(workflow, wf_module1, delta.id,
                            RenderResult(arrow_table({"A": [1]})))
        wf_module2 = tab.wf_modules.create(
            order=1,
            slug="step-2",
            module_id_name="mod",
            last_relevant_delta_id=delta.id,
        )
        cache_render_result(workflow, wf_module2, delta.id,
                            RenderResult(arrow_table({"B": [2]})))

        with patch.object(Kernel, "render", return_value=None):
            self._execute(workflow)
            Kernel.render.assert_not_called()
Ejemplo n.º 11
0
    def test_change_parameters_on_soft_deleted_tab(self):
        workflow = Workflow.objects.create()
        delta = InitWorkflowCommand.create(workflow)
        tab = workflow.tabs.create(position=0, is_deleted=True)

        ModuleVersion.create_or_replace_from_spec({
            "id_name":
            "loadurl",
            "name":
            "loadurl",
            "category":
            "Clean",
            "parameters": [{
                "id_name": "url",
                "type": "string"
            }],
        })

        wf_module = tab.wf_modules.create(
            order=0,
            slug="step-1",
            module_id_name="loadurl",
            last_relevant_delta_id=delta.id,
            params={"url": ""},
        )

        cmd = self.run_with_async_db(
            commands.do(
                ChangeParametersCommand,
                workflow_id=workflow.id,
                wf_module=wf_module,
                new_values={"url": "https://example.com"},
            ))
        self.assertIsNone(cmd)
Ejemplo n.º 12
0
 def setUp(self):
     super().setUp()
     self.workflow = Workflow.objects.create()
     self.delta = InitWorkflowCommand.create(self.workflow)
     self.tab = self.workflow.tabs.create(position=0)
     self.wf_module = self.tab.wf_modules.create(
         order=0, slug="step-1", last_relevant_delta_id=self.delta.id)
Ejemplo n.º 13
0
    def test_execute_new_revision(self):
        workflow = Workflow.create_and_init()
        tab = workflow.tabs.first()
        delta1 = workflow.last_delta
        create_module_zipfile(
            "mod",
            python_code=
            'import pandas as pd\ndef render(table, params): return pd.DataFrame({"B": [2]})',
        )
        wf_module = tab.wf_modules.create(
            order=0,
            slug="step-1",
            last_relevant_delta_id=delta1.id,
            module_id_name="mod",
        )

        result1 = RenderResult(arrow_table({"A": [1]}))
        cache_render_result(workflow, wf_module, delta1.id, result1)

        delta2 = InitWorkflowCommand.create(workflow)
        wf_module.last_relevant_delta_id = delta2.id
        wf_module.save(update_fields=["last_relevant_delta_id"])

        self._execute(workflow)

        wf_module.refresh_from_db()

        with open_cached_render_result(
                wf_module.cached_render_result) as result:
            assert_render_result_equals(result,
                                        RenderResult(arrow_table({"B": [2]})))
Ejemplo n.º 14
0
    def test_duplicate_copies_fresh_cache(self):
        # The cache's filename depends on workflow_id and wf_module_id.
        # Duplicating it would need more complex code :).
        result = RenderResult(
            arrow_table({"A": [1]}), [RenderError(I18nMessage("X", []), [])], {}
        )
        cache_render_result(self.workflow, self.wf_module, self.delta.id, result)

        workflow2 = Workflow.objects.create()
        tab2 = workflow2.tabs.create(position=0)
        InitWorkflowCommand.create(workflow2)
        dup = self.wf_module.duplicate_into_new_workflow(tab2)

        dup_cached_result = dup.cached_render_result
        with open_cached_render_result(dup_cached_result) as result2:
            self.assertEqual(result2, result)
Ejemplo n.º 15
0
def _init_workflow_for_lesson(workflow, lesson):
    InitWorkflowCommand.create(workflow)

    # Create each wfModule of each tab
    tab_dicts = lesson.initial_workflow.tabs
    for position, tab_dict in enumerate(tab_dicts):
        # Set selected module to last wfmodule in stack
        tab = workflow.tabs.create(
            position=position,
            slug=f"tab-{position + 1}",
            name=tab_dict["name"],
            selected_wf_module_position=len(tab_dict["wfModules"]) - 1,
        )

        for order, wfm in enumerate(tab_dict["wfModules"]):
            _add_wf_module_to_tab(wfm, order, tab, workflow.last_delta_id,
                                  lesson)
Ejemplo n.º 16
0
    def test_duplicate_ignores_stale_cache(self):
        # The cache's filename depends on workflow_id and wf_module_id.
        # Duplicating it would need more complex code :).
        result = RenderResult(
            arrow_table({"A": [1]}), [RenderError(I18nMessage("X", []), [])], {}
        )
        cache_render_result(self.workflow, self.wf_module, self.delta.id, result)
        # Now simulate a new delta that hasn't been rendered
        self.wf_module.last_relevant_delta_id += 1
        self.wf_module.save(update_fields=["last_relevant_delta_id"])

        workflow2 = Workflow.objects.create()
        tab2 = workflow2.tabs.create(position=0)
        InitWorkflowCommand.create(workflow2)
        dup = self.wf_module.duplicate_into_new_workflow(tab2)

        dup_cached_result = dup.cached_render_result
        self.assertIsNone(dup_cached_result)
        self.assertEqual(dup.cached_render_result_status, None)
Ejemplo n.º 17
0
    def test_wf_module_duplicate_disable_auto_update(self):
        """
        Duplicates should be lightweight by default: no auto-updating.
        """
        workflow = Workflow.create_and_init()
        tab = workflow.tabs.first()
        wf_module = tab.wf_modules.create(
            order=0,
            slug="step-1",
            auto_update_data=True,
            next_update=timezone.now(),
            update_interval=600,
        )

        workflow2 = Workflow.create_and_init()
        InitWorkflowCommand.create(workflow2)
        tab2 = workflow2.tabs.create(position=0)
        wf_module2 = wf_module.duplicate_into_new_workflow(tab2)

        self.assertEqual(wf_module2.auto_update_data, False)
        self.assertIsNone(wf_module2.next_update)
        self.assertEqual(wf_module2.update_interval, 600)
Ejemplo n.º 18
0
    def test_email_no_delta_when_no_cached_render_result(
            self, email, fake_load_module):
        # No cached render result means one of two things:
        #
        # 1. This is a new module (in which case, why email the user?)
        # 2. We cleared the render cache (in which case, better to skip emailing a few
        #    users than to email _every_ user that results have changed when they haven't)

        workflow = Workflow.objects.create()
        tab = workflow.tabs.create(position=0)
        delta1 = InitWorkflowCommand.create(workflow)
        ModuleVersion.create_or_replace_from_spec({
            "id_name": "mod",
            "name": "Mod",
            "category": "Clean",
            "parameters": []
        })
        wf_module = tab.wf_modules.create(
            order=0,
            slug="step-1",
            last_relevant_delta_id=delta1.id,
            module_id_name="mod",
            notifications=True,
        )

        # Make a new delta, so we need to re-render. Give it the same output.
        delta2 = InitWorkflowCommand.create(workflow)
        wf_module.last_relevant_delta_id = delta2.id
        wf_module.save(update_fields=["last_relevant_delta_id"])

        fake_loaded_module = Mock(LoadedModule)
        fake_load_module.return_value = fake_loaded_module
        fake_loaded_module.migrate_params.return_value = {}
        fake_loaded_module.render.return_value = RenderResult(
            arrow_table({"A": [1]}))

        self._execute(workflow)

        email.assert_not_called()
Ejemplo n.º 19
0
    def test_render_happy_path(self, execute):
        execute.side_effect = async_noop
        workflow = Workflow.objects.create()
        delta = InitWorkflowCommand.create(workflow)
        ack = Mock(name="ack", side_effect=async_noop)
        requeue = Mock(name="requeue", side_effect=async_noop)

        with self.assertLogs():
            self.run_with_async_db(
                render_workflow_and_maybe_requeue(SuccessfulRenderLocker(),
                                                  workflow.id, delta.id, ack,
                                                  requeue))

        execute.assert_called_with(workflow, delta.id)
        ack.assert_called()
        requeue.assert_not_called()
Ejemplo n.º 20
0
    def setUp(self):
        super().setUp()

        self.workflow = Workflow.objects.create()
        self.tab = self.workflow.tabs.create(position=0)
        self.module_zipfile = create_module_zipfile(
            "loadsomething",
            spec_kwargs={"parameters": [{
                "id_name": "url",
                "type": "string"
            }]},
        )
        self.kernel.migrate_params.side_effect = RuntimeError(
            "AddModuleCommand and tests should cache migrated params correctly"
        )

        self.delta = InitWorkflowCommand.create(self.workflow)
Ejemplo n.º 21
0
    def test_execute_cache_hit(self, fake_module):
        workflow = Workflow.objects.create()
        tab = workflow.tabs.create(position=0)
        delta = InitWorkflowCommand.create(workflow)
        wf_module1 = tab.wf_modules.create(order=0,
                                           slug="step-1",
                                           last_relevant_delta_id=delta.id)
        cache_render_result(workflow, wf_module1, delta.id,
                            RenderResult(arrow_table({"A": [1]})))
        wf_module2 = tab.wf_modules.create(order=1,
                                           slug="step-2",
                                           last_relevant_delta_id=delta.id)
        cache_render_result(workflow, wf_module2, delta.id,
                            RenderResult(arrow_table({"B": [2]})))

        self._execute(workflow)

        fake_module.assert_not_called()
Ejemplo n.º 22
0
 def test_workflow_view_triggers_render_if_stale_cache(self):
     step = self.tab1.wf_modules.create(
         order=0,
         slug="step-1",
         last_relevant_delta_id=self.delta.id,
         cached_render_result_delta_id=self.delta.id,  # stale
     )
     # Cache a result
     cache_render_result(self.workflow1, step, self.delta.id,
                         RenderResult(arrow_table({"A": ["a"]})))
     # Make the cached result stale. (The view will actually send the
     # stale-result metadata to the client. That's why we cached it.)
     delta2 = InitWorkflowCommand.create(self.workflow1)
     step.last_relevant_delta_id = delta2.id
     step.save(update_fields=["last_relevant_delta_id"])
     self.client.force_login(self.user)
     self.client.get("/workflows/%d/" % self.workflow1.id)
     self.queue_render.assert_called_with(self.workflow1.id, delta2.id)
Ejemplo n.º 23
0
    def setUp(self):
        super().setUp()

        self.workflow = Workflow.objects.create()
        self.tab = self.workflow.tabs.create(position=0)
        self.module_version = ModuleVersion.create_or_replace_from_spec(
            {
                "id_name": "loadurl",
                "name": "Load URL",
                "category": "Clean",
                "parameters": [{
                    "id_name": "url",
                    "type": "string"
                }],
            },
            source_version_hash="1.0",
        )

        self.delta = InitWorkflowCommand.create(self.workflow)
Ejemplo n.º 24
0
    def test_render_unneeded_execution_so_requeue(self, mock_execute):
        mock_execute.side_effect = execute.UnneededExecution
        workflow = Workflow.objects.create()
        delta = InitWorkflowCommand.create(workflow)
        ack = Mock(name="ack", side_effect=async_noop)
        requeue = Mock(name="requeue", side_effect=async_noop)

        async def inner():
            with self.assertLogs("renderer", level="INFO") as cm:
                await render_workflow_and_maybe_requeue(
                    SuccessfulRenderLocker(), workflow.id, delta.id, ack,
                    requeue)
                self.assertEqual(
                    cm.output,
                    [(f"INFO:renderer.render:UnneededExecution in "
                      f"execute_workflow({workflow.id}, {delta.id})")],
                )

        self.run_with_async_db(inner())
        ack.assert_called()
        requeue.assert_called_with(workflow.id, delta.id)
    def test_change_parameters_on_soft_deleted_tab(self):
        workflow = Workflow.objects.create()
        delta = InitWorkflowCommand.create(workflow)
        tab = workflow.tabs.create(position=0, is_deleted=True)

        wf_module = tab.wf_modules.create(
            order=0,
            slug="step-1",
            module_id_name="loadurl",
            last_relevant_delta_id=delta.id,
            params={"url": ""},
        )

        cmd = self.run_with_async_db(
            commands.do(
                ChangeParametersCommand,
                workflow_id=workflow.id,
                wf_module=wf_module,
                new_values={"url": "https://example.com"},
            ))
        self.assertIsNone(cmd)
Ejemplo n.º 26
0
    def test_render_other_renderer_rendering_so_skip(self, execute):
        execute.side_effect = async_noop
        workflow = Workflow.objects.create()
        delta = InitWorkflowCommand.create(workflow)
        ack = Mock(name="ack", side_effect=async_noop)
        requeue = Mock(name="requeue", side_effect=async_noop)

        async def inner():
            with self.assertLogs("renderer", level="INFO") as cm:
                await render_workflow_and_maybe_requeue(
                    FailedRenderLocker(), workflow.id, delta.id, ack, requeue)
                self.assertEqual(
                    cm.output,
                    [(f"INFO:renderer.render:Workflow {workflow.id} is "
                      "being rendered elsewhere; ignoring")],
                )

        self.run_with_async_db(inner())

        execute.assert_not_called()
        ack.assert_called()
        requeue.assert_not_called()
Ejemplo n.º 27
0
    def test_execute_new_revision(self, fake_load_module):
        workflow = Workflow.create_and_init()
        tab = workflow.tabs.first()
        delta1 = workflow.last_delta
        ModuleVersion.create_or_replace_from_spec({
            "id_name": "mod",
            "name": "Mod",
            "category": "Clean",
            "parameters": []
        })
        wf_module = tab.wf_modules.create(
            order=0,
            slug="step-1",
            last_relevant_delta_id=delta1.id,
            module_id_name="mod",
        )

        result1 = RenderResult(arrow_table({"A": [1]}))
        cache_render_result(workflow, wf_module, delta1.id, result1)

        delta2 = InitWorkflowCommand.create(workflow)
        wf_module.last_relevant_delta_id = delta2.id
        wf_module.save(update_fields=["last_relevant_delta_id"])

        result2 = RenderResult(arrow_table({"B": [2]}))
        fake_module = Mock(LoadedModule)
        fake_module.migrate_params.return_value = {}
        fake_load_module.return_value = fake_module
        fake_module.render.return_value = result2

        self._execute(workflow)

        wf_module.refresh_from_db()

        with open_cached_render_result(
                wf_module.cached_render_result) as result:
            assert_render_result_equals(result, result2)
    def setUp(self):
        super().setUp()

        self.workflow = Workflow.objects.create()
        self.delta = InitWorkflowCommand.create(self.workflow)
        self.tab = self.workflow.tabs.create(position=0, slug="tab-1")