def test_workflow_reorder_modules(self): wfm1 = add_new_wf_module(self.workflow1, self.module_version1, 0) wfm2 = add_new_wf_module(self.workflow1, self.module_version1, 1) wfm3 = add_new_wf_module(self.workflow1, self.module_version1, 2) # your basic reordering request = self._build_patch('/api/workflows/%d/' % self.workflow1.id, data=[{'id': wfm1.id, 'order': 2}, {'id': wfm2.id, 'order': 0}, {'id': wfm3.id, 'order': 1}], format='json', user=self.user) response = workflow_detail(request, pk = self.workflow1.id) self.assertIs(response.status_code, status.HTTP_204_NO_CONTENT) self.assertEqual(list(WfModule.objects.order_by('order').values_list('id', flat=True)), [wfm2.id, wfm3.id, wfm1.id]) # bad data should generate a 400 error # (we don't test every possible failure case, ReorderModulesCommand tests does that) request = self._build_patch('/api/workflows/%d/' % self.workflow1.id, data=[{'problem':'bad data'}], format='json', user=self.user) response = workflow_detail(request, pk = self.workflow1.id) self.assertIs(response.status_code, status.HTTP_400_BAD_REQUEST)
def setUp(self): super(UpdatesTests, self).setUp() # log in self.workflow = add_new_workflow('Update scan') loadurl = load_module_version('loadurl') self.wfm1 = add_new_wf_module(self.workflow, loadurl, order=0) self.wfm2 = add_new_wf_module(self.workflow, loadurl, order=1) self.wfm3 = add_new_wf_module(self.workflow, loadurl, order=2) # fake out the current time so we can run the test just-so self.nowtime = parser.parse('Aug 28 1999 2:35PM UTC')
def test_workflow_duplicate(self): # Create workflow with two WfModules wf1 = create_testdata_workflow() self.assertNotEqual(wf1.owner, self.otheruser) # should owned by user created by LoggedInTestCase module_version1 = add_new_module_version('Module 1') add_new_wf_module(wf1, module_version1, 1) # order=1 self.assertEqual(wf1.wf_modules.count(), 2) wf2 = wf1.duplicate(self.otheruser) self.assertNotEqual(wf1.id, wf2.id) self.assertEqual(wf2.owner, self.otheruser) self.assertEqual(wf2.name, "Copy of " + wf1.name) self.assertIsNone(wf2.last_delta) # no undo history self.assertFalse(wf2.public) self.assertEqual(wf1.wf_modules.count(), wf2.wf_modules.count())
def createTestWorkflow(self): # Create a standard test workflow, but it has only one module so add two more to test render pipeline # Throw a missing value in there to test handling of NA values test_csv = 'Class,M,F\n' \ 'math,10,12\n' \ 'english,,7\n' \ 'history,11,13\n' \ 'economics,20,20' self.test_table = pd.read_csv(io.StringIO(test_csv), header=0, skipinitialspace=True) self.workflow1 = create_testdata_workflow(csv_text=test_csv) self.pspec11 = ParameterSpec.objects.get(id_name='csv') self.module2_version = add_new_module_version('Module 2', dispatch='NOP') self.pspec21 = add_new_parameter_spec(self.module2_version, ParameterSpec.STRING, def_value='foo') self.pspec22 = add_new_parameter_spec(self.module2_version, ParameterSpec.FLOAT, def_value=3.14) self.pspec23 = add_new_parameter_spec(self.module2_version, ParameterSpec.INTEGER, def_value=42) self.pspec24 = add_new_parameter_spec(self.module2_version, ParameterSpec.CHECKBOX, def_value=True) self.pspec25 = ParameterSpec.objects.create( module_version=self.module2_version, type=ParameterSpec.MENU, def_menu_items='Apple|Banana|Kittens', def_value='1') self.module3_version = add_new_module_version('Module 3', dispatch='double_M_col') self.pspec31 = add_new_parameter_spec(self.module3_version, ParameterSpec.BUTTON) self.wfmodule1 = WfModule.objects.get(order=0) self.wfmodule2 = add_new_wf_module(self.workflow1, self.module2_version, 1) self.wfmodule3 = add_new_wf_module(self.workflow1, self.module3_version, 2)
def setUp(self): # Create a workflow with two modules self.workflow = create_testdata_workflow() # defined by create_testdata_workflow self.module_version = ModuleVersion.objects.first() self.module1 = WfModule.objects.first() self.module2 = add_new_wf_module(self.workflow, self.module_version, order=1)
def test_load_and_dispatch(self): test_dir = self.fake_github_clone() import_module_from_directory('https://github.com/account/reponame', 'reponame', '123456', test_dir) # Module and ModuleVersion should have loaded -- these will raise exception if they don't exist module = Module.objects.get(id_name=self.importable_id_name) module_version = ModuleVersion.objects.get(module=module) # Create a test workflow that uses this imported module workflow = add_new_workflow('Dynamic Dispatch Test Workflow') wfm = add_new_wf_module(workflow, module_version, order=1) # These will fail if we haven't correctly loaded the json describing # the parameters stringparam = get_param_by_id_name('test', wf_module=wfm) colparam = get_param_by_id_name('test_column', wf_module=wfm) multicolparam = get_param_by_id_name('test_multicolumn', wf_module=wfm) # Does it render right? test_csv = 'Class,M,F,Other\n' \ 'math,10,12,100\n' \ 'english,,7\,200\n' \ 'history,11,13,\n' \ 'economics,20,20,20' test_table = pd.read_csv(io.StringIO(test_csv), header=0, skipinitialspace=True) test_table_out = test_table.copy() test_table_out['M'] *= 2 test_table_out[['F', 'Other']] *= 3 colparam.set_value('M') # double this multicolparam.set_value('F,Other') # triple these with self.assertLogs(dynamicdispatch.__name__): result = module_dispatch_render(module_version, wfm.get_params(), test_table, None) self.assertEqual(result, ProcessResult(test_table_out)) # Test that bad column parameter values are removed colparam.set_value('missing_column_name') multicolparam.set_value('Other,junk_column_name') test_table_out = test_table.copy() # multicolumn parameter has only one valid col test_table_out[['Other']] *= 3 result = module_dispatch_render(module_version, wfm.get_params(), test_table, None) self.assertEqual(result, ProcessResult(test_table_out)) # if the module crashes, we should get an error with a line number stringparam.set_value('crashme') result = module_dispatch_render(module_version, wfm.get_params(), test_table, None) self.assertEqual( result, ProcessResult( error='ValueError: we crashed! at line 7 of importable.py'))
def setUp(self): super(ChannelTests, self).setUp() self.user = create_test_user(username='******', email='*****@*****.**') self.workflow = add_new_workflow('Workflow 1') self.wf_id = self.workflow.id self.module = add_new_module_version('Module') self.wf_module = add_new_wf_module(self.workflow, self.module) self.application = self.mock_auth_middleware(create_url_router())
def test_wf_module_delete(self): # add a new one to delete; don't mess with other tests wfmodule4 = add_new_wf_module(self.workflow1, self.module2_version, 3) self.workflow1.selected_wf_module = 3 self.workflow1.save() response = self.client.delete('/api/wfmodules/%d' % wfmodule4.id) self.assertIs(response.status_code, status.HTTP_204_NO_CONTENT) self.assertFalse(WfModule.objects.filter(workflow=self.workflow1, pk=wfmodule4.id).exists()) # must really be gone # also check that deleting the selected module nullifies workflow.selected_wf_module self.workflow1.refresh_from_db() self.assertEqual(self.workflow1.selected_wf_module, 2)
def setUp(self): super().setUp() clear_db() self.user = create_test_user(username='******', email='*****@*****.**') self.workflow = Workflow.objects.create(name='Workflow 1', owner=self.user) self.wf_id = self.workflow.id self.module = add_new_module_version('Module') self.wf_module = add_new_wf_module(self.workflow, self.module) self.application = self.mock_auth_middleware(create_url_router()) self.communicators = []
def test_missing_module(self): # If the WfModule references a Module that does not exist, we should # get a placeholder workflow = add_new_workflow('Missing module') wfm = add_new_wf_module(workflow, None, 0) response = self.client.get('/api/wfmodules/%d/' % wfm.id) self.assertIs(response.status_code, status.HTTP_200_OK) parsed = json.loads(response.content) self.assertEqual(parsed['module_version']['module'], None) response = self.client.get('/api/wfmodules/%d/render' % wfm.id) self.assertIs(response.status_code, status.HTTP_200_OK) self.assertEqual(json.loads(response.content), empty_data_json)
def setUp(self): super().setUp() self.workflow = create_testdata_workflow() self.wfm1 = WfModule.objects.first() self.wfm2 = add_new_wf_module(self.workflow, ModuleVersion.objects.first(), 1) # order = 1 self.test_data = 'stored data'.encode() self.metadata = 'metadataish' # Use a more realistic test table with lots of data of different types # mock data wasn't finding bugs related to dict-type columns fname = os.path.join(settings.BASE_DIR, 'server/tests/test_data/sfpd.json') sfpd = json.load(open(fname)) self.test_table = pd.DataFrame(sfpd) sanitize_dataframe(self.test_table)
def test_updates_module_version(self): test_dir = self.fake_github_clone() import_module_from_directory("https://test_url_of_test_module", "importable", "111111", test_dir) module_version_q = ModuleVersion.objects.filter(module__id_name=self.importable_id_name) self.assertEqual(module_version_q.count(), 1) # Create a test workflow that uses this imported module module_version = module_version_q.first() workflow = add_new_workflow('updates_module_version workflow') wfm = add_new_wf_module(workflow, module_version, order=1) # import "new" version (different version hash) test_dir = self.fake_github_clone() import_module_from_directory("https://test_url_of_test_module", "importable", "222222", test_dir) self.assertEqual(module_version_q.count(), 2) # should have updated the wfm to newly imported version wfm.refresh_from_db() self.assertEqual(wfm.module_version.source_version_hash, "222222")
def test_load_and_dispatch(self): try: test_dir = self.fake_github_clone() import_module_from_directory('https://github.com/account/reponame', 'reponame', '123456', test_dir) # Module and ModuleVersion should have loaded -- these will raise # exception if they don't exist module = Module.objects.get(id_name=self.importable_id_name) module_version = ModuleVersion.objects.get(module=module) # Create a test workflow that uses this imported module workflow = add_new_workflow('Dynamic Dispatch Test Workflow') wfm = add_new_wf_module(workflow, module_version, order=1) # These will fail if we haven't correctly loaded the json # describing the parameters colparam = get_param_by_id_name('test_column', wf_module=wfm) multicolparam = get_param_by_id_name('test_multicolumn', wf_module=wfm) # Does it render right? test_csv = 'Class,M,F,Other\n' \ 'math,10,12,100\n' \ 'english,,7\,200\n' \ 'history,11,13,\n' \ 'economics,20,20,20' test_table = pd.read_csv(io.StringIO(test_csv), header=0, skipinitialspace=True) test_table_out = test_table.copy() test_table_out['M'] *= 2 test_table_out[['F', 'Other']] *= 3 colparam.set_value('M') # double this multicolparam.set_value('F,Other') # triple these with self.assertLogs(): lm = LoadedModule.for_module_version_sync(module_version) result = lm.render(wfm.get_params(), test_table, None) self.assertEqual(result, ProcessResult(test_table_out)) finally: server.models.loaded_module.load_external_module.cache_clear()
def test_missing_module(self): workflow = add_new_workflow('Missing module') wfm = add_new_wf_module(workflow, None, 0) result = module_dispatch_render(wfm, mock_csv_table) self.assertEqual(result, ProcessResult())