def test_change_dependent_wf_modules(self): # tab slug: tab-1 workflow = Workflow.create_and_init(selected_tab_position=2) workflow.tabs.create(position=1, slug='tab-2') workflow.tabs.create(position=2, slug='tab-3') # Create `wf_module` depending on tabs 2+3 (and their order) ModuleVersion.create_or_replace_from_spec({ 'id_name': 'x', 'name': 'X', 'category': 'Clean', 'parameters': [ { 'id_name': 'tabs', 'type': 'multitab' }, ] }) wf_module = workflow.tabs.first().wf_modules.create( order=0, module_id_name='x', params={'tabs': ['tab-2', 'tab-3']}, last_relevant_delta_id=workflow.last_delta_id) cmd = self.run_with_async_db( ReorderTabsCommand.create(workflow=workflow, new_order=['tab-3', 'tab-1', 'tab-2'])) wf_module.refresh_from_db() self.assertEqual(wf_module.last_relevant_delta_id, cmd.id)
def test_fetch(self, queue_fetch, send_delta): future_none = asyncio.Future() future_none.set_result(None) queue_fetch.return_value = future_none send_delta.return_value = future_none user = User.objects.create(username="******", email="*****@*****.**") workflow = Workflow.create_and_init(owner=user) wf_module = workflow.tabs.first().wf_modules.create(order=0, slug="step-1") response = self.run_handler(fetch, user=user, workflow=workflow, wfModuleId=wf_module.id) self.assertResponse(response, data=None) wf_module.refresh_from_db() self.assertEqual(wf_module.is_busy, True) queue_fetch.assert_called_with(wf_module) send_delta.assert_called_with( workflow.id, { "updateWfModules": { str(wf_module.id): { "is_busy": True, "fetch_error": "" } } }, )
def test_set_params(self): user = User.objects.create(username="******", email="*****@*****.**") workflow = Workflow.create_and_init(owner=user) wf_module = workflow.tabs.first().wf_modules.create(order=0, slug="step-1", module_id_name="x") ModuleVersion.create_or_replace_from_spec({ "id_name": "x", "name": "x", "category": "Clean", "parameters": [{ "id_name": "foo", "type": "string" }], }) response = self.run_handler( set_params, user=user, workflow=workflow, wfModuleId=wf_module.id, values={"foo": "bar"}, ) self.assertResponse(response, data=None) command = ChangeParametersCommand.objects.first() self.assertEquals(command.new_values, {"foo": "bar"}) self.assertEquals(command.old_values, {}) self.assertEquals(command.wf_module_id, wf_module.id) self.assertEquals(command.workflow_id, workflow.id)
def test_try_set_autofetch_happy_path(self): user = User.objects.create(username="******", email="*****@*****.**") workflow = Workflow.create_and_init(owner=user) wf_module = workflow.tabs.first().wf_modules.create(order=0, slug="step-1") response = self.run_handler( try_set_autofetch, user=user, workflow=workflow, wfModuleId=wf_module.id, isAutofetch=True, fetchInterval=1200, ) self.assertResponse(response, data={ "isAutofetch": True, "fetchInterval": 1200 }) wf_module.refresh_from_db() self.assertEqual(wf_module.auto_update_data, True) self.assertEqual(wf_module.update_interval, 1200) self.assertLess(wf_module.next_update, timezone.now() + datetime.timedelta(seconds=1202)) self.assertGreater(wf_module.next_update, timezone.now() + datetime.timedelta(seconds=1198))
def test_try_set_autofetch_exceed_quota(self): user = User.objects.create(username="******", email="*****@*****.**") user.user_profile.max_fetches_per_day = 10 user.user_profile.save() workflow = Workflow.create_and_init(owner=user) wf_module = workflow.tabs.first().wf_modules.create(order=0, slug="step-1") response = self.run_handler( try_set_autofetch, user=user, workflow=workflow, wfModuleId=wf_module.id, isAutofetch=True, fetchInterval=300, ) self.assertEqual(response.error, "") self.assertEqual(response.data["quotaExceeded"]["maxFetchesPerDay"], 10) self.assertEqual(response.data["quotaExceeded"]["nFetchesPerDay"], 288) self.assertEqual( response.data["quotaExceeded"]["autofetches"][0]["workflow"]["id"], workflow.id, ) wf_module.refresh_from_db() self.assertEqual(wf_module.auto_update_data, False)
def test_add_module(self): user = User.objects.create(username='******', email='*****@*****.**') workflow = Workflow.create_and_init(owner=user) # with tab-1 module_version = ModuleVersion.create_or_replace_from_spec({ 'id_name': 'amodule', 'name': 'A Module', 'category': 'Clean', 'parameters': [ { 'id_name': 'foo', 'type': 'string' }, ], }) response = self.run_handler(add_module, user=user, workflow=workflow, tabSlug='tab-1', position=3, moduleIdName='amodule', paramValues={'foo': 'bar'}) self.assertResponse(response, data=None) command = AddModuleCommand.objects.first() self.assertEquals(command.wf_module.order, 3) self.assertEquals(command.wf_module.module_version, module_version) self.assertEquals(command.wf_module.get_params()['foo'], 'bar') self.assertEquals(command.wf_module.tab.slug, 'tab-1') self.assertEquals(command.workflow_id, workflow.id)
def test_add_module_param_values_not_object(self): user = User.objects.create(username='******', email='*****@*****.**') workflow = Workflow.create_and_init(owner=user) # tab-1 ModuleVersion.create_or_replace_from_spec({ 'id_name': 'amodule', 'name': 'A Module', 'category': 'Clean', 'parameters': [ { 'id_name': 'foo', 'type': 'string' }, ], }) response = self.run_handler(add_module, user=user, workflow=workflow, tabSlug='tab-1', position=3, moduleIdName='amodule', paramValues='foobar') self.assertResponse(response, error='BadRequest: paramValues must be an Object')
def _get_anonymous_workflow_for(workflow: Workflow, request: HttpRequest) -> Workflow: """If not owner, return a cached duplicate of `workflow`. The duplicate will be married to `request.session.session_key`, and its `.is_anonymous` will return `True`. """ if not request.session.session_key: request.session.create() session_key = request.session.session_key try: return Workflow.objects.get(original_workflow_id=workflow.id, anonymous_owner_session_key=session_key) except Workflow.DoesNotExist: if workflow.example: server.utils.log_user_event_from_request(request, 'Opened Demo Workflow', {'name': workflow.name}) new_workflow = workflow.duplicate_anonymous(session_key) async_to_sync(rabbitmq.queue_render)(new_workflow.id, new_workflow.last_delta_id) return new_workflow
def workflow_detail(request, workflow_id, format=None): if request.method == 'POST': workflow = lookup_workflow_for_write(workflow_id, request) try: valid_fields = {'public'} if not set(request.data.keys()).intersection(valid_fields): raise ValueError('Unknown fields: {}'.format(request.data)) if 'public' in request.data: # TODO this should be a command, so it's undoable workflow.public = request.data['public'] workflow.save(update_fields=['public']) except Exception as e: return JsonResponse({'message': str(e), 'status_code': 400}, status=status.HTTP_400_BAD_REQUEST) return Response(status=status.HTTP_204_NO_CONTENT) elif request.method == 'DELETE': with Workflow.authorized_lookup_and_cooperative_lock( 'owner', request.user, request.session, pk=workflow_id ) as workflow: workflow.delete() return Response(status=status.HTTP_204_NO_CONTENT)
def test_add_module_default_params(self): workflow = Workflow.create_and_init() module_version = ModuleVersion.create_or_replace_from_spec( { 'id_name': 'blah', 'name': 'Blah', 'category': 'Clean', 'parameters': [ { 'id_name': 'a', 'type': 'string', 'default': 'x' }, { 'id_name': 'c', 'type': 'checkbox', 'name': 'C', 'default': True }, ], }, source_version_hash='1.0') cmd = self.run_with_async_db( AddModuleCommand.create(workflow=workflow, tab=workflow.tabs.first(), module_id_name=module_version.id_name, position=0, param_values={})) self.assertEqual(cmd.wf_module.params, {'a': 'x', 'c': True})
def setUp(self): super().setUp() self.workflow = Workflow.create_and_init() self.wf_module = self.workflow.tabs.first().wf_modules.create( order=0, slug="step-1", last_relevant_delta_id=self.workflow.last_delta_id )
def _wf_module_delete_secret_and_build_delta( workflow: Workflow, wf_module: WfModule, param: str) -> Optional[Dict[str, Any]]: """ Write a new secret (or `None`) to `wf_module`, or raise. Return a "delta" for websockets.ws_client_send_delta_async(), or `None` if the database has not been modified. Raise Workflow.DoesNotExist if the Workflow was deleted. """ with workflow.cooperative_lock(): # raises Workflow.DoesNotExist try: wf_module.refresh_from_db() except WfModule.DoesNotExist: return None # no-op if wf_module.secrets.get(param) is None: return None # no-op wf_module.secrets = dict(wf_module.secrets) # shallow copy del wf_module.secrets[param] wf_module.save(update_fields=['secrets']) return { 'updateWfModules': { str(wf_module.id): { 'secrets': wf_module.secret_metadata, } } }
def test_convert_to_uploaded_file_happy_path(self): workflow = Workflow.create_and_init() wf_module = workflow.tabs.first().wf_modules.create(order=0, slug="step-1", module_id_name="x") ipu = wf_module.in_progress_uploads.create() minio.put_bytes(ipu.Bucket, ipu.get_upload_key(), b"1234567") uploaded_file = ipu.convert_to_uploaded_file("test sheet.xlsx") self.assertEqual(uploaded_file.uuid, str(ipu.id)) final_key = wf_module.uploaded_file_prefix + str(ipu.id) + ".xlsx" # New file on S3 has the right bytes and metadata self.assertEqual( minio.get_object_with_data(minio.UserFilesBucket, final_key)["Body"], b"1234567", ) self.assertEqual( minio.client.head_object(Bucket=minio.UserFilesBucket, Key=final_key)["ContentDisposition"], "attachment; filename*=UTF-8''test%20sheet.xlsx", ) # InProgressUpload is completed self.assertEqual(ipu.is_completed, True) ipu.refresh_from_db() self.assertEqual(ipu.is_completed, True) # also on DB # Uploaded file is deleted self.assertFalse( minio.exists(minio.UserFilesBucket, ipu.get_upload_key()))
def test_resume_without_rerunning_unneeded_renders(self, fake_load_module): workflow = Workflow.create_and_init() tab = workflow.tabs.first() delta_id = workflow.last_delta_id # wf_module1: has a valid, cached result wf_module1 = tab.wf_modules.create(order=0, last_relevant_delta_id=delta_id) result1 = ProcessResult(pd.DataFrame({'A': [1]})) wf_module1.cache_render_result(delta_id, result1) # wf_module2: has no cached result (must be rendered) wf_module2 = tab.wf_modules.create(order=1, last_relevant_delta_id=delta_id) fake_loaded_module = Mock(LoadedModule) fake_load_module.return_value = fake_loaded_module result2 = ProcessResult(pd.DataFrame({'A': [2]})) fake_loaded_module.render.return_value = result2 self._execute(workflow) wf_module2.refresh_from_db() actual = wf_module2.cached_render_result.result self.assertEqual(actual, result2) fake_loaded_module.render.assert_called_once() # only with module2
def test_delete_scopes_tab_delete_by_workflow(self): workflow = Workflow.create_and_init() workflow2 = Workflow.create_and_init() # Here's a soft-deleted module on workflow2. Nothing references it. It # "shouldn't" exist. tab = workflow2.tabs.create(position=1) # now delete a delta on workflow1 delta = self.run_with_async_db( ChangeWorkflowTitleCommand.create(workflow=workflow, new_value="1")) self.run_with_async_db(delta.backward()) # fix workflow.last_delta_id delta.delete_with_successors() workflow.delete_orphan_soft_deleted_models() tab.refresh_from_db() # no DoesNotExist: leave workflow2 alone
def test_fetch_wf_module(self, save_result, load_module): result = ProcessResult(pd.DataFrame({'A': [1]}), error='hi') async def fake_fetch(*args, **kwargs): return result fake_module = Mock(LoadedModule) load_module.return_value = fake_module fake_module.fetch.side_effect = fake_fetch workflow = Workflow.create_and_init() wf_module = workflow.tabs.first().wf_modules.create( order=0, next_update=parser.parse('Aug 28 1999 2:24PM UTC'), update_interval=600) now = parser.parse('Aug 28 1999 2:24:02PM UTC') due_for_update = parser.parse('Aug 28 1999 2:34PM UTC') with self.assertLogs(fetch.__name__, logging.DEBUG): self.run_with_async_db( fetch.fetch_wf_module(workflow.id, wf_module, now)) save_result.assert_called_with(workflow.id, wf_module, result) wf_module.refresh_from_db() self.assertEqual(wf_module.last_update_check, now) self.assertEqual(wf_module.next_update, due_for_update)
def test_delete_orphans_does_not_delete_new_tab(self): """ Don't delete a new AddTabCommand's new orphan Tab during creation. We delete orphans Deltas during creation, and we should delete their Tabs/WfModules. But we shouldn't delete _new_ Tabs/WfModules. (We need to order creation and deletion carefully to avoid doing so.) """ workflow = Workflow.create_and_init() # Create a soft-deleted Tab in an orphan Delta (via AddTabCommand) delta1 = self.run_with_async_db( AddTabCommand.create(workflow=workflow, slug="tab-2", name="name-2")) self.run_with_async_db(delta1.backward()) # Now create a new Tab in a new Delta. This will delete delta1, and it # _should_ delete `tab-2`. self.run_with_async_db( AddTabCommand.create(workflow=workflow, slug="tab-3", name="name-3")) with self.assertRaises(Tab.DoesNotExist): delta1.tab.refresh_from_db() # orphan tab was deleted with self.assertRaises(Delta.DoesNotExist): delta1.refresh_from_db()
def _wf_module_delete_secret_and_build_delta( workflow: Workflow, wf_module: WfModule, param: str ) -> Optional[Dict[str, Any]]: """ Write a new secret (or `None`) to `wf_module`, or raise. Return a "delta" for websockets.ws_client_send_delta_async(), or `None` if the database has not been modified. Raise Workflow.DoesNotExist if the Workflow was deleted. """ with workflow.cooperative_lock(): # raises Workflow.DoesNotExist wf_module.refresh_from_db() # may return None if ( wf_module is None or wf_module.secrets.get(param) is None ): return None wf_module.secrets = dict(wf_module.secrets) del wf_module.secrets[param] wf_module.save(update_fields=['secrets']) return { 'updateWfModules': { str(wf_module.id): { 'params': wf_module.get_params().as_dict() } } }
def test_add_module_viewer_access_denied(self): workflow = Workflow.create_and_init(public=True) # tab-1 ModuleVersion.create_or_replace_from_spec({ 'id_name': 'amodule', 'name': 'A Module', 'category': 'Clean', 'parameters': [ { 'id_name': 'foo', 'type': 'string' }, ], }) response = self.run_handler(add_module, workflow=workflow, tabSlug='tab-1', position=3, moduleIdName='amodule', paramValues={'foo': 'bar'}) self.assertResponse(response, error='AuthError: no write access to workflow')
def get_workflow_as_delta_and_needs_render(self): """ Return (apply-delta dict, needs_render), or raise Workflow.DoesNotExist needs_render is a (workflow_id, delta_id) pair. """ with Workflow.authorized_lookup_and_cooperative_lock( 'read', self.scope['user'], self.scope['session'], pk=self.workflow_id) as workflow: request = RequestWrapper(self.scope['user'], self.scope['session']) ret = { 'updateWorkflow': (WorkflowSerializer(workflow, context={ 'request': request }).data), } tabs = list(workflow.live_tabs) ret['updateTabs'] = dict( (tab.slug, TabSerializer(tab).data) for tab in tabs) wf_modules = list(WfModule.live_in_workflow(workflow.id)) ret['updateWfModules'] = dict( (str(wfm.id), WfModuleSerializer(wfm).data) for wfm in wf_modules) if workflow.are_all_render_results_fresh(): needs_render = None else: needs_render = (workflow.id, workflow.last_delta_id) return (ret, needs_render)
def test_set_params_invalid_params(self): user = User.objects.create(username="******", email="*****@*****.**") workflow = Workflow.create_and_init(owner=user) wf_module = workflow.tabs.first().wf_modules.create(order=0, slug="step-1", module_id_name="x") ModuleVersion.create_or_replace_from_spec({ "id_name": "x", "name": "x", "category": "Clean", "parameters": [{ "id_name": "foo", "type": "string" }], }) response = self.run_handler( set_params, user=user, workflow=workflow, wfModuleId=wf_module.id, values={"foo1": "bar"}, ) self.assertResponse( response, error=("ValueError: Value {'foo1': 'bar'} has wrong names: " "expected names {'foo'}"), )
def render_workflow(request: HttpRequest, workflow: Workflow): if (workflow.lesson_slug and _lesson_exists(workflow.lesson_slug) and workflow.owner == request.user): if "/" in workflow.lesson_slug: # /courses/a-course/a-lesson -- no trailing '/' because courses use # relative URLs return redirect("/courses/" + workflow.lesson_slug) else: # /lessons/a-lesson/ return redirect("/lessons/" + workflow.lesson_slug) else: if workflow.example and workflow.owner != request.user: workflow = _get_anonymous_workflow_for(workflow, request) modules = visible_modules(request) init_state = make_init_state(request, workflow=workflow, modules=modules) if not workflow.are_all_render_results_fresh(): # We're returning a Workflow that may have stale WfModules. That's # fine, but are we _sure_ the renderer is about to render them? # Let's double-check. This will handle edge cases such as "we wiped # our caches" or maybe some bugs we haven't thought of. # # This isn't just for bug recovery. ChangeDataVersionCommand won't # queue_render until a client requests it. async_to_sync(rabbitmq.queue_render)(workflow.id, workflow.last_delta_id) return TemplateResponse(request, "workflow.html", {"initState": init_state})
def test_try_set_autofetch_disable_autofetch(self): user = User.objects.create(username="******", email="*****@*****.**") workflow = Workflow.create_and_init(owner=user) wf_module = workflow.tabs.first().wf_modules.create( order=0, slug="step-1", auto_update_data=True, update_interval=1200, next_update=timezone.now(), ) response = self.run_handler( try_set_autofetch, user=user, workflow=workflow, wfModuleId=wf_module.id, isAutofetch=False, fetchInterval=300, ) self.assertResponse(response, data={ "isAutofetch": False, "fetchInterval": 300 }) wf_module.refresh_from_db() self.assertEqual(wf_module.auto_update_data, False) self.assertEqual(wf_module.update_interval, 300) self.assertIsNone(wf_module.next_update)
def test_prepare_upload_happy_path(self): user = User.objects.create(username='******', email='*****@*****.**') workflow = Workflow.create_and_init(owner=user) wf_module = workflow.tabs.first().wf_modules.create( order=0, module_id_name='x', ) data = b'1234567' md5sum = _base64_md5sum(data) response = self.run_handler(prepare_upload, user=user, workflow=workflow, wfModuleId=wf_module.id, filename='abc.csv', nBytes=len(data), base64Md5sum=md5sum) self.assertEqual(response.error, '') wf_module.refresh_from_db() self.assertIsNone(wf_module.inprogress_file_upload_id) self.assertTrue( response.data['key'].startswith(wf_module.uploaded_file_prefix) ) self.assertTrue(response.data['key'] in response.data['url']) http = urllib3.PoolManager() response = http.request('PUT', response.data['url'], body=data, headers=response.data['headers']) self.assertEqual(response.status, 200) # the URL+headers work
def test_try_set_autofetch_allow_exceed_quota_when_reducing(self): user = User.objects.create(username="******", email="*****@*****.**") user.user_profile.max_fetches_per_day = 10 user.user_profile.save() workflow = Workflow.create_and_init(owner=user) wf_module = workflow.tabs.first().wf_modules.create( order=0, slug="step-1", auto_update_data=True, update_interval=300, next_update=timezone.now(), ) response = self.run_handler( try_set_autofetch, user=user, workflow=workflow, wfModuleId=wf_module.id, isAutofetch=True, fetchInterval=600, ) self.assertResponse(response, data={ "isAutofetch": True, "fetchInterval": 600 }) wf_module.refresh_from_db() self.assertEqual(wf_module.update_interval, 600)
def test_complete_upload_happy_path(self, send_delta): user = User.objects.create(username='******', email='*****@*****.**') workflow = Workflow.create_and_init(owner=user) uuid = str(uuidgen.uuid4()) key = f'wf-123/wfm-234/{uuid}.csv' wf_module = workflow.tabs.first().wf_modules.create( order=0, module_id_name='x', inprogress_file_upload_id=None, inprogress_file_upload_key=key, inprogress_file_upload_last_accessed_at=timezone.now(), ) # The user needs to write the file to S3 before calling complete_upload minio.put_bytes( minio.UserFilesBucket, key, b'1234567', ContentDisposition="attachment; filename*=UTF-8''file.csv", ) send_delta.side_effect = async_noop response = self.run_handler(complete_upload, user=user, workflow=workflow, wfModuleId=wf_module.id, key=key) self.assertEqual(response.error, '') self.assertEqual(response.data, {'uuid': uuid}) wf_module.refresh_from_db() self.assertIsNone(wf_module.inprogress_file_upload_id) self.assertIsNone(wf_module.inprogress_file_upload_key) self.assertIsNone(wf_module.inprogress_file_upload_last_accessed_at) uploaded_file: UploadedFile = wf_module.uploaded_files.first() self.assertEqual(uploaded_file.name, 'file.csv') self.assertEqual(uploaded_file.uuid, uuid) self.assertEqual(uploaded_file.size, 7) self.assertEqual(uploaded_file.bucket, minio.UserFilesBucket) self.assertEqual(uploaded_file.key, key)
def test_generate_secret_access_token_writer_access_denied(self): user = User.objects.create(email="*****@*****.**") workflow = Workflow.create_and_init(public=True) workflow.acl.create(email=user.email, can_edit=True) ModuleVersion.create_or_replace_from_spec({ "id_name": "g", "name": "g", "category": "Clean", "parameters": [TestGoogleSecret], }) wf_module = workflow.tabs.first().wf_modules.create(module_id_name="g", order=0, slug="step-1") response = self.run_handler( generate_secret_access_token, user=user, workflow=workflow, wfModuleId=wf_module.id, param="google_credentials", ) self.assertResponse(response, error="AuthError: no owner access to workflow")
def test_abort_upload(self): user = User.objects.create(username='******', email='*****@*****.**') workflow = Workflow.create_and_init(owner=user) uuid = str(uuidgen.uuid4()) key = f'wf-123/wfm-234/{uuid}.csv' wf_module = workflow.tabs.first().wf_modules.create( order=0, module_id_name='x', inprogress_file_upload_id=None, inprogress_file_upload_key=key, inprogress_file_upload_last_accessed_at=timezone.now(), ) # let's pretend the user has uploaded at least partial data. minio.put_bytes( minio.UserFilesBucket, key, b'1234567', ContentDisposition="attachment; filename*=UTF-8''file.csv", ) response = self.run_handler(abort_upload, user=user, workflow=workflow, wfModuleId=wf_module.id, key=key) self.assertResponse(response, data=None) wf_module.refresh_from_db() self.assertIsNone(wf_module.inprogress_file_upload_id) self.assertIsNone(wf_module.inprogress_file_upload_key) self.assertIsNone(wf_module.inprogress_file_upload_last_accessed_at) # Ensure the file is deleted from S3 self.assertFalse(minio.exists(minio.UserFilesBucket, key))
def test_generate_secret_access_token_wrong_param_type_gives_null(self): user = User.objects.create() workflow = Workflow.create_and_init(owner=user) ModuleVersion.create_or_replace_from_spec({ "id_name": "g", "name": "g", "category": "Clean", "parameters": [TestGoogleSecret], }) wf_module = workflow.tabs.first().wf_modules.create( module_id_name="g", order=0, slug="step-1", params={"s": '{"name":"a","secret":"hello"}'}, ) response = self.run_handler( generate_secret_access_token, user=user, workflow=workflow, wfModuleId=wf_module.id, param="a", ) self.assertResponse(response, data={"token": None})
def test_clean_file_no_uploaded_file(self): workflow = Workflow.create_and_init() tab = workflow.tabs.first() wfm = tab.wf_modules.create(module_id_name='uploadfile', order=0) context = RenderContext(workflow.id, wfm.id, None, None, None) result = clean_value(ParamDType.File(), str(uuid.uuid4()), context) self.assertIsNone(result)