def setUp(self): super(WorkflowViewTests, self).setUp() # log in self.log_patcher = patch('server.utils.log_user_event') self.log_patch = self.log_patcher.start() self.factory = APIRequestFactory() self.workflow1 = add_new_workflow('Workflow 1') self.workflow2 = add_new_workflow('Workflow 2') self.module_version1 = add_new_module_version('Module 1') add_new_module_version('Module 2') add_new_module_version('Module 3') # Add another user, with one public and one private workflow self.otheruser = User.objects.create(username='******', email='*****@*****.**', password='******') self.other_workflow_private = Workflow.objects.create( name="Other workflow private", owner=self.otheruser ) self.other_workflow_public = Workflow.objects.create( name="Other workflow public", owner=self.otheruser, public=True )
def test_wf_module_duplicate(self): wfm1 = self.wfmodule1 # store data to test that it is duplicated s1 = wfm1.store_fetched_table(mock_csv_table) s2 = wfm1.store_fetched_table(mock_csv_table2) wfm1.set_fetched_data_version(s2) self.assertEqual(len(wfm1.list_fetched_data_versions()), 2) # duplicate into another workflow, as we would do when duplicating a workflow workflow2 = add_new_workflow("Test Workflow 2") wfm1d = wfm1.duplicate(workflow2) wfm1d.refresh_from_db() # test what we actually have in the db self.assertEqual(wfm1d.workflow, workflow2) self.assertEqual(wfm1d.module_version, wfm1.module_version) self.assertEqual(wfm1d.order, wfm1.order) self.assertEqual(wfm1d.notes, wfm1.notes) self.assertEqual(wfm1d.auto_update_data, wfm1.auto_update_data) self.assertEqual(wfm1d.last_update_check, wfm1.last_update_check) self.assertEqual(wfm1d.update_interval, wfm1.update_interval) self.assertEqual(wfm1d.is_collapsed, wfm1.is_collapsed) self.assertEqual(wfm1d.stored_data_version, wfm1.stored_data_version) # parameters should be duplicated self.assertEqual( ParameterVal.objects.filter(wf_module=wfm1d).count(), ParameterVal.objects.filter(wf_module=wfm1).count()) # Stored data should contain a clone of content only, not complete version history self.assertIsNotNone(wfm1d.stored_data_version) self.assertEqual(wfm1d.stored_data_version, wfm1.stored_data_version) self.assertTrue(wfm1d.retrieve_fetched_table().equals( wfm1.retrieve_fetched_table())) self.assertEqual(len(wfm1d.list_fetched_data_versions()), 1)
async def test_deny_other_users_workflow(self): other_workflow = add_new_workflow( 'Workflow 2', owner=create_test_user('other', '*****@*****.**') ) comm = WebsocketCommunicator(self.application, f'/workflows/{other_workflow.id}/') connected, _ = await comm.connect() self.assertFalse(connected)
def test_load_and_dispatch(self): test_dir = self.fake_github_clone() import_module_from_directory('https://github.com/account/reponame', 'reponame', '123456', test_dir) # Module and ModuleVersion should have loaded -- these will raise exception if they don't exist module = Module.objects.get(id_name=self.importable_id_name) module_version = ModuleVersion.objects.get(module=module) # Create a test workflow that uses this imported module workflow = add_new_workflow('Dynamic Dispatch Test Workflow') wfm = add_new_wf_module(workflow, module_version, order=1) # These will fail if we haven't correctly loaded the json describing # the parameters stringparam = get_param_by_id_name('test', wf_module=wfm) colparam = get_param_by_id_name('test_column', wf_module=wfm) multicolparam = get_param_by_id_name('test_multicolumn', wf_module=wfm) # Does it render right? test_csv = 'Class,M,F,Other\n' \ 'math,10,12,100\n' \ 'english,,7\,200\n' \ 'history,11,13,\n' \ 'economics,20,20,20' test_table = pd.read_csv(io.StringIO(test_csv), header=0, skipinitialspace=True) test_table_out = test_table.copy() test_table_out['M'] *= 2 test_table_out[['F', 'Other']] *= 3 colparam.set_value('M') # double this multicolparam.set_value('F,Other') # triple these with self.assertLogs(dynamicdispatch.__name__): result = module_dispatch_render(module_version, wfm.get_params(), test_table, None) self.assertEqual(result, ProcessResult(test_table_out)) # Test that bad column parameter values are removed colparam.set_value('missing_column_name') multicolparam.set_value('Other,junk_column_name') test_table_out = test_table.copy() # multicolumn parameter has only one valid col test_table_out[['Other']] *= 3 result = module_dispatch_render(module_version, wfm.get_params(), test_table, None) self.assertEqual(result, ProcessResult(test_table_out)) # if the module crashes, we should get an error with a line number stringparam.set_value('crashme') result = module_dispatch_render(module_version, wfm.get_params(), test_table, None) self.assertEqual( result, ProcessResult( error='ValueError: we crashed! at line 7 of importable.py'))
def setUp(self): super(ChannelTests, self).setUp() self.user = create_test_user(username='******', email='*****@*****.**') self.workflow = add_new_workflow('Workflow 1') self.wf_id = self.workflow.id self.module = add_new_module_version('Module') self.wf_module = add_new_wf_module(self.workflow, self.module) self.application = self.mock_auth_middleware(create_url_router())
def setUp(self): super(UpdatesTests, self).setUp() # log in self.workflow = add_new_workflow('Update scan') loadurl = load_module_version('loadurl') self.wfm1 = add_new_wf_module(self.workflow, loadurl, order=0) self.wfm2 = add_new_wf_module(self.workflow, loadurl, order=1) self.wfm3 = add_new_wf_module(self.workflow, loadurl, order=2) # fake out the current time so we can run the test just-so self.nowtime = parser.parse('Aug 28 1999 2:35PM UTC')
def test_secret_not_duplicated(self): val1 = self.secret_val() val1.set_value({'name': 'foo', 'secret': 'foo'}) workflow2 = add_new_workflow("Test Workflow 2") wfmodule2 = WfModule.objects.create(module_version=self.module_version, workflow=workflow2, order=0) val2 = val1.duplicate(wfmodule2) self.assertEqual(val2.get_value(), None) self.assertEqual(val2.get_secret(), None)
def test_missing_module(self): # If the WfModule references a Module that does not exist, we should # get a placeholder workflow = add_new_workflow('Missing module') wfm = add_new_wf_module(workflow, None, 0) response = self.client.get('/api/wfmodules/%d/' % wfm.id) self.assertIs(response.status_code, status.HTTP_200_OK) parsed = json.loads(response.content) self.assertEqual(parsed['module_version']['module'], None) response = self.client.get('/api/wfmodules/%d/render' % wfm.id) self.assertIs(response.status_code, status.HTTP_200_OK) self.assertEqual(json.loads(response.content), empty_data_json)
def test_auth(self): # Create otheruser and try to access workflow owned by default user other_user = create_test_user(username='******', email='*****@*****.**') wf = add_new_workflow('New Workflow', owner=other_user) wfm = load_and_add_module('concaturl', workflow=wf) result = store_external_workflow( wfm, f'https://app.workbenchdata.com/workflows/{self.ext_wfm.workflow_id}/' ) self.assertEqual( result, ProcessResult(error='Access denied to the target workflow'))
def setUp(self): super(DispatchTests, self).setUp() # log in self.test_csv = 'Class,M,F\n' \ 'math,10,12\n' \ 'english,,7\n' \ 'history,11,13\n' \ 'economics,20,20' self.test_table = pd.read_csv(io.StringIO(self.test_csv)) self.test_table_MF = self.test_table[['M', 'F']] self.workflow = add_new_workflow('dispatch tests wf') self.wfm = load_and_add_module('selectcolumns', workflow=self.workflow) get_param_by_id_name('colnames').set_value('M,F')
def test_duplicate_module(self): version = self.wfm.store_fetched_table(self.csv_table) self.wfm.set_fetched_data_version(version) self.assertEqual(len(self.wfm.list_fetched_data_versions()), 1) wfm2 = self.wfm.duplicate(add_new_workflow('workflow 2')) wfm2.refresh_from_db() self.assertEqual(len(self.wfm.list_fetched_data_versions()), 1) self.assertEqual(len(wfm2.list_fetched_data_versions()), 1) self.assertEqual(wfm2.status, WfModule.READY) result = UploadFile.render(wfm2, None) self.assertEqual(result, ProcessResult(self.csv_table))
def test_duplicate(self): # Create a new WfModule of the same type, that has no parametervals yet # This is the context where duplicate is normally called, when # duplicating a WfModule workflow2 = add_new_workflow("Test Workflow 2") wfmodule2 = WfModule.objects.create(module_version=self.module_version, workflow=workflow2, order=0) sp = ParameterVal.objects.get(pk=self.stringID) spd = sp.duplicate(wfmodule2) self.assertEqual(spd.wf_module, wfmodule2) self.assertEqual(sp.parameter_spec, spd.parameter_spec) self.assertEqual(sp.value, spd.value) self.assertEqual(sp.items, spd.items) self.assertEqual(sp.visible, spd.visible) self.assertEqual(sp.order, spd.order)
def test_updates_module_version(self): test_dir = self.fake_github_clone() import_module_from_directory("https://test_url_of_test_module", "importable", "111111", test_dir) module_version_q = ModuleVersion.objects.filter(module__id_name=self.importable_id_name) self.assertEqual(module_version_q.count(), 1) # Create a test workflow that uses this imported module module_version = module_version_q.first() workflow = add_new_workflow('updates_module_version workflow') wfm = add_new_wf_module(workflow, module_version, order=1) # import "new" version (different version hash) test_dir = self.fake_github_clone() import_module_from_directory("https://test_url_of_test_module", "importable", "222222", test_dir) self.assertEqual(module_version_q.count(), 2) # should have updated the wfm to newly imported version wfm.refresh_from_db() self.assertEqual(wfm.module_version.source_version_hash, "222222")
def test_load_and_dispatch(self): try: test_dir = self.fake_github_clone() import_module_from_directory('https://github.com/account/reponame', 'reponame', '123456', test_dir) # Module and ModuleVersion should have loaded -- these will raise # exception if they don't exist module = Module.objects.get(id_name=self.importable_id_name) module_version = ModuleVersion.objects.get(module=module) # Create a test workflow that uses this imported module workflow = add_new_workflow('Dynamic Dispatch Test Workflow') wfm = add_new_wf_module(workflow, module_version, order=1) # These will fail if we haven't correctly loaded the json # describing the parameters colparam = get_param_by_id_name('test_column', wf_module=wfm) multicolparam = get_param_by_id_name('test_multicolumn', wf_module=wfm) # Does it render right? test_csv = 'Class,M,F,Other\n' \ 'math,10,12,100\n' \ 'english,,7\,200\n' \ 'history,11,13,\n' \ 'economics,20,20,20' test_table = pd.read_csv(io.StringIO(test_csv), header=0, skipinitialspace=True) test_table_out = test_table.copy() test_table_out['M'] *= 2 test_table_out[['F', 'Other']] *= 3 colparam.set_value('M') # double this multicolparam.set_value('F,Other') # triple these with self.assertLogs(): lm = LoadedModule.for_module_version_sync(module_version) result = lm.render(wfm.get_params(), test_table, None) self.assertEqual(result, ProcessResult(test_table_out)) finally: server.models.loaded_module.load_external_module.cache_clear()
def test_missing_module(self): workflow = add_new_workflow('Missing module') wfm = add_new_wf_module(workflow, None, 0) result = module_dispatch_render(wfm, mock_csv_table) self.assertEqual(result, ProcessResult())
def createTestWorkflow(self): # Create a WfModule with one parameter of each type self.module_version = add_new_module_version("TestModule") self.moduleID = self.module_version.module.id stringSpec = ParameterSpec.objects.create( name='StringParam', id_name='stringparam', module_version=self.module_version, type=ParameterSpec.STRING, def_value='foo', placeholder='placeholder') stringSpecEmpty = ParameterSpec.objects.create( name='StringParamEmpty', id_name='stringparamempty', module_version=self.module_version, type=ParameterSpec.STRING) integerSpec = ParameterSpec.objects.create( name='IntegerParam', id_name='integerparam', module_version=self.module_version, type=ParameterSpec.INTEGER, def_value='42') floatSpec = ParameterSpec.objects.create( name='FloatParam', id_name='floatparam', module_version=self.module_version, type=ParameterSpec.FLOAT, def_value='10.11') checkboxSpec = ParameterSpec.objects.create( name='CheckboxParam', id_name='checkboxparam', module_version=self.module_version, type=ParameterSpec.CHECKBOX, def_value='1') menuSpec = ParameterSpec.objects.create( name='MenuParam', id_name='menuparam', module_version=self.module_version, type=ParameterSpec.MENU, def_items='Item A|Item B|Item C', def_value='1') # should refer to Item B radioSpec = ParameterSpec.objects.create( name='RadioParam', id_name='radioparam', module_version=self.module_version, type=ParameterSpec.RADIO, def_items='Item A|Item B|Item C', def_value='0') # should refer to Item A self.workflow = add_new_workflow(name="Test Workflow") self.workflowID = self.workflow.id self.wfmodule = WfModule.objects.create( module_version=self.module_version, workflow=self.workflow, order=0) self.wfmoduleID = self.wfmodule.id # set non-default values for vals in order to reveal certain types of bugs stringVal = ParameterVal.objects.create(parameter_spec=stringSpec, wf_module=self.wfmodule, value='fooval') self.stringID = stringVal.id emptyStringVal = ParameterVal.objects.create( parameter_spec=stringSpecEmpty, wf_module=self.wfmodule) self.stringemptyID = emptyStringVal.id integerVal = ParameterVal.objects.create(parameter_spec=integerSpec, wf_module=self.wfmodule, value='10') self.integerID = integerVal.id floatVal = ParameterVal.objects.create(parameter_spec=floatSpec, wf_module=self.wfmodule, value='3.14159') self.floatID = floatVal.id checkboxVal = ParameterVal.objects.create(parameter_spec=checkboxSpec, wf_module=self.wfmodule, value='True') self.checkboxID = checkboxVal.id menuVal = ParameterVal.objects.create(parameter_spec=menuSpec, wf_module=self.wfmodule, value='2', items=menuSpec.def_items) self.menuID = menuVal.id radioVal = ParameterVal.objects.create(parameter_spec=radioSpec, wf_module=self.wfmodule, value='0', items=radioSpec.def_items) self.radioID = radioVal.id
def setUp(self): # Define two types of modules we are going to use self.csv = load_module_version('pastecsv') self.workflow = add_new_workflow('My Undoable Workflow')