def test_internal_module_get_cached(self, load_module): workflow = Workflow.create_and_init() wf_module = workflow.tabs.first().wf_modules.create( order=0, module_id_name="yay", params={}, cached_migrated_params={"foo": "bar"}, cached_migrated_params_module_version="v2", ) ModuleVersion.create_or_replace_from_spec( { "id_name": "yay", "name": "Yay", "category": "Clean", "parameters_version": 2, "parameters": [{ "id_name": "foo", "type": "string" }], }, source_version_hash="internal", ) self.assertEqual(get_migrated_params(wf_module), {"foo": "bar"}) load_module.assert_not_called()
def test_execute_tempdir_not_in_tmpfs(self, fake_load_module): # /tmp is RAM; /var/tmp is disk. Assert big files go on disk. workflow = Workflow.create_and_init() tab = workflow.tabs.first() delta1 = workflow.last_delta ModuleVersion.create_or_replace_from_spec({ "id_name": "mod", "name": "Mod", "category": "Clean", "parameters": [] }) tab.wf_modules.create( order=0, slug="step-1", last_relevant_delta_id=delta1.id - 1, module_id_name="mod", ) result2 = RenderResult(arrow_table({"B": [2]})) fake_load_module.return_value.migrate_params.return_value = {} fake_load_module.return_value.render.return_value = result2 self._execute(workflow) self.assertRegex( str(fake_load_module.return_value.render.call_args[1]["basedir"]), r"/var/tmp/", )
def test_change_parameters_on_hard_deleted_wf_module(self): workflow = Workflow.create_and_init() ModuleVersion.create_or_replace_from_spec({ "id_name": "loadurl", "name": "loadurl", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }) wf_module = workflow.tabs.first().wf_modules.create( order=0, slug="step-1", module_id_name="loadurl", last_relevant_delta_id=workflow.last_delta_id, params={"url": ""}, ) wf_module.delete() cmd = self.run_with_async_db( commands.do( ChangeParametersCommand, workflow_id=workflow.id, wf_module=wf_module, new_values={"url": "https://example.com"}, )) self.assertIsNone(cmd)
def test_execute_race_delete_workflow(self, fake_load_module): workflow = Workflow.create_and_init() tab = workflow.tabs.first() ModuleVersion.create_or_replace_from_spec({ "id_name": "mod", "name": "Mod", "category": "Clean", "parameters": [] }) tab.wf_modules.create( order=0, slug="step-1", last_relevant_delta_id=workflow.last_delta_id, module_id_name="mod", ) tab.wf_modules.create( order=1, slug="step-2", last_relevant_delta_id=workflow.last_delta_id, module_id_name="mod", ) def load_module_and_delete(module_version): Workflow.objects.filter(id=workflow.id).delete() fake_module = Mock(LoadedModule) fake_module.migrate_params.return_value = {} fake_module.render.return_value = RenderResult( arrow_table({"A": [1]})) return fake_module fake_load_module.side_effect = load_module_and_delete with self.assertRaises(UnneededExecution): self._execute(workflow)
def test_execute_migrate_params_module_error_gives_default_params( self, fake_load_module): workflow = Workflow.create_and_init() tab = workflow.tabs.first() delta1 = workflow.last_delta ModuleVersion.create_or_replace_from_spec({ "id_name": "mod", "name": "Mod", "category": "Clean", "parameters": [{ "type": "string", "id_name": "x", "default": "def" }], }) tab.wf_modules.create( order=0, slug="step-1", last_relevant_delta_id=delta1.id, module_id_name="mod", ) def render(*args, params, **kwargs): self.assertEqual(params, Params({"x": "def"})) # default params return RenderResult(arrow_table({"A": [1]})) # make migrate_params() raise an error. fake_load_module.return_value.migrate_params.side_effect = ModuleExitedError( -9, "") fake_load_module.return_value.render.side_effect = render self._execute(workflow) fake_load_module.return_value.render.assert_called()
def test_set_secret_error_not_a_secret(self): user = User.objects.create() workflow = Workflow.create_and_init(owner=user) ModuleVersion.create_or_replace_from_spec( { "id_name": "g", "name": "g", "category": "Clean", "parameters": [{"id_name": "string_secret", "type": "string"}], } ) wf_module = workflow.tabs.first().wf_modules.create( module_id_name="g", order=0, slug="step-1", params={"string_secret": "bar"}, secrets={}, ) response = self.run_handler( set_secret, user=user, workflow=workflow, wfModuleId=wf_module.id, param="string_secret", secret="foo", ) self.assertResponse( response, error="BadRequest: param is not a secret string parameter" ) wf_module.refresh_from_db() self.assertEqual(wf_module.params, {"string_secret": "bar"}) self.assertEqual(wf_module.secrets, {})
def test_change_dependent_wf_modules(self): # tab slug: tab-1 workflow = Workflow.create_and_init(selected_tab_position=2) workflow.tabs.create(position=1, slug="tab-2") workflow.tabs.create(position=2, slug="tab-3") # Create `wf_module` depending on tabs 2+3 (and their order) ModuleVersion.create_or_replace_from_spec({ "id_name": "x", "name": "X", "category": "Clean", "parameters": [{ "id_name": "tabs", "type": "multitab" }], }) wf_module = workflow.tabs.first().wf_modules.create( order=0, slug="step-1", module_id_name="x", params={"tabs": ["tab-2", "tab-3"]}, last_relevant_delta_id=workflow.last_delta_id, ) cmd = self.run_with_async_db( commands.do( ReorderTabsCommand, workflow_id=workflow.id, new_order=["tab-3", "tab-1", "tab-2"], )) wf_module.refresh_from_db() self.assertEqual(wf_module.last_relevant_delta_id, cmd.id)
def test_delete_deltas_without_init_delta(self): workflow = Workflow.objects.create(name="A") tab = workflow.tabs.create(position=0) self.run_with_async_db( commands.do( ChangeWorkflowTitleCommand, workflow_id=workflow.id, new_value="B" ) ) ModuleVersion.create_or_replace_from_spec( {"id_name": "x", "name": "x", "category": "Clean", "parameters": []} ) self.run_with_async_db( commands.do( AddModuleCommand, workflow_id=workflow.id, tab=tab, slug="step-1", module_id_name="x", position=0, param_values={}, ) ) self.run_with_async_db( commands.do( ChangeWorkflowTitleCommand, workflow_id=workflow.id, new_value="C" ) ) workflow.delete() self.assertTrue(True) # no crash
def test_change_last_relevant_delta_ids_of_self_wf_modules(self): """ Module render() accepts a `tab_name` argument: test it sees a new one. """ workflow = Workflow.create_and_init() delta_id = workflow.last_delta_id tab = workflow.tabs.first() # Add a WfModule that relies on `tab.name` through its 'render' method. ModuleVersion.create_or_replace_from_spec({ "id_name": "x", "name": "x", "category": "Clean", "parameters": [] }) wf_module = tab.wf_modules.create(order=0, slug="step-1", module_id_name="x", last_relevant_delta_id=delta_id) cmd = self.run_with_async_db( commands.do( SetTabNameCommand, workflow_id=workflow.id, tab=tab, new_name=tab.name + "X", )) wf_module.refresh_from_db() self.assertEqual(wf_module.last_relevant_delta_id, cmd.id)
def test_add_module_missing_tab(self): user = User.objects.create(username="******", email="*****@*****.**") workflow = Workflow.create_and_init(owner=user) other_workflow = Workflow.create_and_init(owner=user) # Create a "honeypot" tab -- make sure the module doesn't get inserted # in the other workflow's 'tab-2'! other_workflow.tabs.create(position=1, slug="tab-2") ModuleVersion.create_or_replace_from_spec({ "id_name": "amodule", "name": "A Module", "category": "Clean", "parameters": [{ "id_name": "foo", "type": "string" }], }) response = self.run_handler( add_module, user=user, workflow=workflow, tabSlug="tab-2", slug="step-1", position=3, moduleIdName="amodule", paramValues={"foo": "bar"}, ) self.assertResponse(response, error="DoesNotExist: Tab not found")
def test_add_module_viewer_access_denied(self): workflow = Workflow.create_and_init(public=True) # tab-1 ModuleVersion.create_or_replace_from_spec({ "id_name": "amodule", "name": "A Module", "category": "Clean", "parameters": [{ "id_name": "foo", "type": "string" }], }) response = self.run_handler( add_module, workflow=workflow, tabSlug="tab-1", slug="step-1", position=3, moduleIdName="amodule", paramValues={"foo": "bar"}, ) self.assertResponse(response, error="AuthError: no write access to workflow")
def test_add_module_invalid_position(self): user = User.objects.create(username="******", email="*****@*****.**") workflow = Workflow.create_and_init(owner=user) # tab-1 ModuleVersion.create_or_replace_from_spec({ "id_name": "amodule", "name": "A Module", "category": "Clean", "parameters": [{ "id_name": "foo", "type": "string" }], }) response = self.run_handler( add_module, user=user, workflow=workflow, tabSlug="tab-1", slug="step-1", position="foo", moduleIdName="amodule", paramValues={"foo": "bar"}, ) self.assertResponse(response, error="BadRequest: position must be a Number")
def test_fetch_integration(self, send_update, queue_render): queue_render.side_effect = async_value(None) send_update.side_effect = async_value(None) workflow = Workflow.create_and_init() ModuleVersion.create_or_replace_from_spec( {"id_name": "mod", "name": "Mod", "category": "Clean", "parameters": []}, source_version_hash="abc123", ) wf_module = workflow.tabs.first().wf_modules.create( order=0, slug="step-1", module_id_name="mod" ) minio.put_bytes( minio.ExternalModulesBucket, "mod/abc123/code.py", b"import pandas as pd\ndef fetch(params): return pd.DataFrame({'A': [1]})\ndef render(table, params): return table", ) cjwstate.modules.init_module_system() now = timezone.now() with self.assertLogs(level=logging.INFO): self.run_with_async_db( fetch.fetch(workflow_id=workflow.id, wf_module_id=wf_module.id, now=now) ) wf_module.refresh_from_db() so = wf_module.stored_objects.get(stored_at=wf_module.stored_data_version) with minio.temporarily_download(so.bucket, so.key) as parquet_path: table = pyarrow.parquet.read_table(str(parquet_path), use_threads=False) assert_arrow_table_equals(table, {"A": [1]}) workflow.refresh_from_db() queue_render.assert_called_with(workflow.id, workflow.last_delta_id) send_update.assert_called()
def test_internal_wrong_cache_version_calls_migrate_params( self, load_module): workflow = Workflow.create_and_init() wf_module = workflow.tabs.first().wf_modules.create( order=0, module_id_name="yay", params={"foo": "bar"}, cached_migrated_params={"foo": "bar"}, cached_migrated_params_module_version="v2", ) ModuleVersion.create_or_replace_from_spec( { "id_name": "yay", "name": "Yay", "category": "Clean", "parameters_version": 3, "parameters": [{ "id_name": "foo", "type": "string" }], }, source_version_hash="internal", # magic string ) load_module.return_value.migrate_params.return_value = {"foo": "baz"} self.assertEqual(get_migrated_params(wf_module), {"foo": "baz"}) load_module.assert_called() # and assert we've cached things self.assertEqual(wf_module.cached_migrated_params, {"foo": "baz"}) self.assertEqual(wf_module.cached_migrated_params_module_version, "v3") wf_module.refresh_from_db() self.assertEqual(wf_module.cached_migrated_params, {"foo": "baz"}) self.assertEqual(wf_module.cached_migrated_params_module_version, "v3")
def test_delete_secret_ignore_non_secret(self): user = User.objects.create() workflow = Workflow.create_and_init(owner=user) ModuleVersion.create_or_replace_from_spec( { "id_name": "googlesheets", "name": "g", "category": "Clean", "parameters": [TestGoogleSecret, {"id_name": "foo", "type": "string"}], } ) wf_module = workflow.tabs.first().wf_modules.create( module_id_name="googlesheets", order=0, slug="step-1", params={"foo": "bar"}, secrets={"google_credentials": {"name": "a", "secret": "hello"}}, ) response = self.run_handler( delete_secret, user=user, workflow=workflow, wfModuleId=wf_module.id, param="foo", ) self.assertResponse(response, data=None) wf_module.refresh_from_db() self.assertEqual(wf_module.params, {"foo": "bar"}) self.assertEqual( wf_module.secrets, {"google_credentials": {"name": "a", "secret": "hello"}} )
def test_workflow_anonymous_user(self): # Looking at example workflow as anonymous should create a new workflow num_workflows = Workflow.objects.count() self.other_workflow_public.example = True self.other_workflow_public.save() # Also ensure the anonymous users can't access the Python module; first we need to load it ModuleVersion.create_or_replace_from_spec( { "id_name": "pythoncode", "name": "Python", "category": "Clean", "parameters": [], } ) request = self._build_get( "/workflows/%d/" % self.other_workflow_public.id, user=AnonymousUser() ) response = render_workflow(request, workflow_id=self.other_workflow_public.id) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual( Workflow.objects.count(), num_workflows + 1 ) # should have duplicated the wf with this API call # Ensure the anonymous users can't access the Python module self.assertNotContains(response, '"pythoncode"')
def test_delete_secret_happy_path(self, send_update): send_update.return_value = async_noop() user = User.objects.create() workflow = Workflow.create_and_init(owner=user) ModuleVersion.create_or_replace_from_spec( { "id_name": "googlesheets", "name": "g", "category": "Clean", "parameters": [TestGoogleSecret], } ) wf_module = workflow.tabs.first().wf_modules.create( module_id_name="googlesheets", order=0, slug="step-1", secrets={"google_credentials": {"name": "a", "secret": "hello"}}, ) response = self.run_handler( delete_secret, user=user, workflow=workflow, wfModuleId=wf_module.id, param="google_credentials", ) self.assertResponse(response, data=None) wf_module.refresh_from_db() self.assertEqual(wf_module.secrets, {}) send_update.assert_called() delta = send_update.call_args[0][1] self.assertEqual(delta.steps[wf_module.id].secrets, {})
def test_set_params_invalid_params(self): user = User.objects.create(username="******", email="*****@*****.**") workflow = Workflow.create_and_init(owner=user) wf_module = workflow.tabs.first().wf_modules.create( order=0, slug="step-1", module_id_name="x" ) ModuleVersion.create_or_replace_from_spec( { "id_name": "x", "name": "x", "category": "Clean", "parameters": [{"id_name": "foo", "type": "string"}], } ) response = self.run_handler( set_params, user=user, workflow=workflow, wfModuleId=wf_module.id, values={"foo1": "bar"}, ) self.assertResponse( response, error=( "ValueError: Value {'foo1': 'bar'} has wrong names: " "expected names {'foo'}" ), )
def test_workflow_anonymous_user(self): # Looking at example workflow as anonymous should create a new workflow public_workflow = Workflow.create_and_init( name="Other workflow public", owner=self.otheruser, public=True, example=True, ) # Also ensure the anonymous users can't access the Python module; first we need to load it ModuleVersion.create_or_replace_from_spec({ "id_name": "pythoncode", "name": "Python", "category": "Clean", "parameters": [], }) # don't log in response = self.client.get("/workflows/%d/" % public_workflow.id) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual( Workflow.objects.filter(owner=None).count(), 1) # should have duplicated the wf with this API call # Ensure the anonymous users can't access the Python module self.assertNotContains(response, '"pythoncode"')
def test_generate_secret_access_token_wrong_param_type_gives_null(self): user = User.objects.create() workflow = Workflow.create_and_init(owner=user) ModuleVersion.create_or_replace_from_spec( { "id_name": "googlesheets", "name": "g", "category": "Clean", "parameters": [TestGoogleSecret], } ) wf_module = workflow.tabs.first().wf_modules.create( module_id_name="googlesheets", order=0, slug="step-1", params={"s": '{"name":"a","secret":"hello"}'}, ) response = self.run_handler( generate_secret_access_token, user=user, workflow=workflow, wfModuleId=wf_module.id, param="a", ) self.assertResponse(response, data={"token": None})
def test_oauth1a_token_request_denied(self, lookup): lookup.return_value = Mock(oauth.OAuthService) lookup.return_value.generate_redirect_url_and_state.side_effect = oauth.TokenRequestDenied( "no!", {} ) ModuleVersion.create_or_replace_from_spec( { "id_name": "twitter", "name": "", "category": "Clean", "parameters": [ { "id_name": "twitter_credentials", "type": "secret", "secret_logic": {"provider": "oauth1a", "service": "twitter"}, } ], } ) user = User.objects.create(username="******", email="*****@*****.**") self.client.force_login(user) workflow = Workflow.create_and_init(owner=user) wf_module = workflow.tabs.first().wf_modules.create( module_id_name="twitter", order=0, slug="step-1" ) response = self.client.get( f"/oauth/create-secret/{workflow.id}/{wf_module.id}/twitter_credentials/" ) self.assertEqual(response.status_code, 403) self.assertRegex( response.content, b"The authorization server refused to let you log in" )
def test_set_params(self): user = User.objects.create(username="******", email="*****@*****.**") workflow = Workflow.create_and_init(owner=user) wf_module = workflow.tabs.first().wf_modules.create( order=0, slug="step-1", module_id_name="x" ) ModuleVersion.create_or_replace_from_spec( { "id_name": "x", "name": "x", "category": "Clean", "parameters": [{"id_name": "foo", "type": "string"}], } ) response = self.run_handler( set_params, user=user, workflow=workflow, wfModuleId=wf_module.id, values={"foo": "bar"}, ) self.assertResponse(response, data=None) command = ChangeParametersCommand.objects.first() self.assertEquals(command.new_values, {"foo": "bar"}) self.assertEquals(command.old_values, {}) self.assertEquals(command.wf_module_id, wf_module.id) self.assertEquals(command.workflow_id, workflow.id)
def test_execute_migrate_params_invalid_params_are_coerced( self, fake_load_module): workflow = Workflow.create_and_init() tab = workflow.tabs.first() delta1 = workflow.last_delta ModuleVersion.create_or_replace_from_spec({ "id_name": "mod", "name": "Mod", "category": "Clean", "parameters": [{ "type": "string", "id_name": "x" }], }) tab.wf_modules.create( order=0, slug="step-1", last_relevant_delta_id=delta1.id, module_id_name="mod", ) def render(*args, params, **kwargs): self.assertEqual(params, Params({"x": "2"})) return RenderResult(arrow_table({"A": [1]})) # make migrate_params() return an int instead of a str. Assume # ParamDType.coerce() will cast it to str. fake_load_module.return_value.migrate_params.return_value = {"x": 2} fake_load_module.return_value.render.side_effect = render self._execute(workflow) fake_load_module.return_value.render.assert_called()
def test_generate_secret_access_token_no_service_gives_error(self): user = User.objects.create() workflow = Workflow.create_and_init(owner=user) ModuleVersion.create_or_replace_from_spec( { "id_name": "googlesheets", "name": "g", "category": "Clean", "parameters": [TestGoogleSecret], } ) wf_module = workflow.tabs.first().wf_modules.create( module_id_name="googlesheets", order=0, slug="step-1", secrets={"google_credentials": {"name": "a", "secret": "hello"}}, ) response = self.run_handler( generate_secret_access_token, user=user, workflow=workflow, wfModuleId=wf_module.id, param="google_credentials", ) self.assertResponse(response, error=("AuthError: we only support twitter"))
def test_email_no_delta_when_not_changed(self, email, fake_load_module): workflow = Workflow.objects.create() tab = workflow.tabs.create(position=0) delta1 = InitWorkflowCommand.create(workflow) ModuleVersion.create_or_replace_from_spec({ "id_name": "mod", "name": "Mod", "category": "Clean", "parameters": [] }) wf_module = tab.wf_modules.create( order=0, slug="step-1", last_relevant_delta_id=delta1.id, module_id_name="mod", notifications=True, ) cache_render_result(workflow, wf_module, delta1.id, RenderResult(arrow_table({"A": [1]}))) # Make a new delta, so we need to re-render. Give it the same output. delta2 = InitWorkflowCommand.create(workflow) wf_module.last_relevant_delta_id = delta2.id wf_module.save(update_fields=["last_relevant_delta_id"]) fake_loaded_module = Mock(LoadedModule) fake_load_module.return_value = fake_loaded_module fake_loaded_module.migrate_params.return_value = {} fake_loaded_module.render.return_value = RenderResult( arrow_table({"A": [1]})) self._execute(workflow) email.assert_not_called()
def test_generate_secret_access_token_happy_path(self, factory): service = Mock(oauth.OAuth2) service.generate_access_token_or_str_error.return_value = { "access_token": "a-token", "refresh_token": "something we must never share", } factory.return_value = service user = User.objects.create() workflow = Workflow.create_and_init(owner=user) ModuleVersion.create_or_replace_from_spec( { "id_name": "googlesheets", "name": "g", "category": "Clean", "parameters": [TestGoogleSecret], } ) wf_module = workflow.tabs.first().wf_modules.create( module_id_name="googlesheets", order=0, slug="step-1", secrets={"google_credentials": {"name": "a", "secret": "hello"}}, ) response = self.run_handler( generate_secret_access_token, user=user, workflow=workflow, wfModuleId=wf_module.id, param="google_credentials", ) self.assertResponse(response, data={"token": "a-token"})
def test_change_parameters_on_soft_deleted_tab(self): workflow = Workflow.objects.create() delta = InitWorkflowCommand.create(workflow) tab = workflow.tabs.create(position=0, is_deleted=True) ModuleVersion.create_or_replace_from_spec({ "id_name": "loadurl", "name": "loadurl", "category": "Clean", "parameters": [{ "id_name": "url", "type": "string" }], }) wf_module = tab.wf_modules.create( order=0, slug="step-1", module_id_name="loadurl", last_relevant_delta_id=delta.id, params={"url": ""}, ) cmd = self.run_with_async_db( commands.do( ChangeParametersCommand, workflow_id=workflow.id, wf_module=wf_module, new_values={"url": "https://example.com"}, )) self.assertIsNone(cmd)
def test_delete_secret_writer_access_denied(self): user = User.objects.create(email="*****@*****.**") workflow = Workflow.create_and_init(public=True) workflow.acl.create(email=user.email, can_edit=True) ModuleVersion.create_or_replace_from_spec( { "id_name": "googlesheets", "name": "g", "category": "Clean", "parameters": [TestGoogleSecret], } ) wf_module = workflow.tabs.first().wf_modules.create( module_id_name="googlesheets", order=0, slug="step-1", secrets={"google_credentials": {"name": "a", "secret": "hello"}}, ) response = self.run_handler( delete_secret, user=user, workflow=workflow, wfModuleId=wf_module.id, param="google_credentials", ) self.assertResponse(response, error="AuthError: no owner access to workflow")
def test_change_parameters_deny_invalid_params(self, load_module): workflow = Workflow.create_and_init() wf_module = workflow.tabs.first().wf_modules.create( order=0, slug="step-1", module_id_name="x", last_relevant_delta_id=workflow.last_delta_id, params={"x": 1}, ) ModuleVersion.create_or_replace_from_spec({ "id_name": "x", "name": "x", "category": "Clean", "parameters": [{ "id_name": "x", "type": "integer" }], }) load_module.return_value.param_schema = ParamDType.Dict( {"x": ParamDType.Integer()}) load_module.return_value.migrate_params = lambda x: x with self.assertRaises(ValueError): # Now the user requests to change params, giving an invalid param. self.run_with_async_db( commands.do( ChangeParametersCommand, workflow_id=workflow.id, wf_module=wf_module, new_values={"x": "Threeve"}, ))
def test_module_error_raises(self, load_module): workflow = Workflow.create_and_init() wf_module = workflow.tabs.first().wf_modules.create( order=0, module_id_name="yay", params={}) ModuleVersion.create_or_replace_from_spec( { "id_name": "yay", "name": "Yay", "category": "Clean", "parameters": [{ "id_name": "foo", "type": "string" }], }, source_version_hash="abc123", ) load_module.return_value.migrate_params.side_effect = ModuleError with self.assertRaises(ModuleError): get_migrated_params(wf_module) # Assert we wrote nothing to cache self.assertIsNone(wf_module.cached_migrated_params) self.assertIsNone(wf_module.cached_migrated_params_module_version) wf_module.refresh_from_db() self.assertIsNone(wf_module.cached_migrated_params) self.assertIsNone(wf_module.cached_migrated_params_module_version)