def test_execute_revision_0(self): # Don't crash on a new workflow (rev=0, no caches) workflow = create_testdata_workflow(table_csv) wf_module2 = load_and_add_module('selectcolumns', workflow=workflow) result = execute_wfmodule(wf_module2) self.assertEqual(result, ProcessResult(table_dataframe)) self.assertEqual(cached_render_result_revision_list(workflow), [0, 0])
def setUp(self): # Create a workflow with two modules self.workflow = create_testdata_workflow() # defined by create_testdata_workflow self.module_version = ModuleVersion.objects.first() self.module1 = WfModule.objects.first() self.module2 = add_new_wf_module(self.workflow, self.module_version, order=1)
def test_email_no_delta_when_not_changed(self, email): workflow = create_testdata_workflow(table_csv) wf_module1 = workflow.wf_modules.first() wf_module1.notifications = True wf_module1.save() async_to_sync(execute_workflow)(workflow) # sends one email async_to_sync(execute_workflow)(workflow) # should not email email.assert_called_once()
def test_execute_cache_hit(self): workflow = create_testdata_workflow(table_csv) wf_module2 = load_and_add_module('selectcolumns', workflow=workflow) # Execute -- which should cache the result expected = execute_wfmodule(wf_module2) with mock.patch('server.dispatch.module_dispatch_render') as mdr: result = execute_wfmodule(wf_module2) self.assertFalse(mdr.called) self.assertEqual(result, expected)
def setUp(self): super(DuplicateColumnFromTableTests, self).setUp() # A reference table for correctness checking # Performing date conversion here does not help tests as default test # WF does not parse dates self.table = reference_table.copy() self.workflow = create_testdata_workflow(csv_text=test_csv) self.wf_module = load_and_add_module('duplicatecolumn', workflow=self.workflow) self.colnames_pval = get_param_by_id_name('colnames')
def test_execute_revision_0(self): # Don't crash on a new workflow (rev=0, no caches) workflow = create_testdata_workflow(table_csv) wf_module2 = load_and_add_module('selectcolumns', workflow=workflow) async_to_sync(execute_workflow)(workflow) wf_module2.refresh_from_db() result = wf_module2.get_cached_render_result().result self.assertEqual(result, ProcessResult(table_dataframe)) self.assertEqual(cached_render_result_revision_list(workflow), [0, 0])
def test_pandas_13258(self): # simple test case where Pandas produces int64 column type, and json # conversion throws ValueError # https://github.com/pandas-dev/pandas/issues/13258#issuecomment-326671257 workflow = create_testdata_workflow(csv_text='A,B,C,D\n1,2,3,4') response = self.client.get('/api/wfmodules/%d/render' % workflow.wf_modules.first().id) self.assertIs(response.status_code, 200) self.assertEqual(json.loads(response.content)['column_types'], ['number', 'number', 'number', 'number'])
def test_pandas_no_header(self): # When no header row, Pandas uses int64s as column names, and # json.dumps(list(table)) throws ValueError workflow = create_testdata_workflow(csv_text='1,2,3,4\n1,2,3,4') response = self.client.get('/api/wfmodules/%d/render' % workflow.wf_modules.first().id) self.assertIs(response.status_code, 200) self.assertEqual(json.loads(response.content)['columns'], ['1', '2', '3', '4']) self.assertEqual(json.loads(response.content)['column_types'], ['number', 'number', 'number', 'number'])
def test_email_delta(self, email): workflow = create_testdata_workflow(table_csv) wf_module1 = workflow.wf_modules.first() wf_module1.notifications = True wf_module1.save() async_to_sync(execute_workflow)(workflow) email.assert_called() wf_module1.refresh_from_db() self.assertTrue(wf_module1.has_unseen_notification)
def setUp(self): super().setUp() # log in self.scraped_table = simple_result_table self.urls = list(self.scraped_table['url']) # create a workflow that feeds our urls via PasteCSV into a URLScraper self.url_table = pd.DataFrame(self.urls, columns=['url']) self.expected_url_table_result = ProcessResult(self.url_table) self.expected_url_table_result.sanitize_in_place() url_csv = 'url\n' + '\n'.join(self.urls) workflow = create_testdata_workflow(url_csv) self.wfmodule = load_and_add_module('urlscraper', workflow=workflow)
def test_execute_cache_hit(self): workflow = create_testdata_workflow(table_csv) wf_module2 = load_and_add_module('selectcolumns', workflow=workflow) async_to_sync(execute_workflow)(workflow) wf_module2.refresh_from_db() result1 = wf_module2.get_cached_render_result().result with patch('server.dispatch.module_dispatch_render') as mdr: async_to_sync(execute_workflow)(workflow) wf_module2.refresh_from_db() result2 = wf_module2.get_cached_render_result().result self.assertFalse(mdr.called) self.assertEqual(result2, result1)
def test_workflow_duplicate(self): # Create workflow with two WfModules wf1 = create_testdata_workflow() self.assertNotEqual(wf1.owner, self.otheruser) # should owned by user created by LoggedInTestCase module_version1 = add_new_module_version('Module 1') add_new_wf_module(wf1, module_version1, 1) # order=1 self.assertEqual(wf1.wf_modules.count(), 2) wf2 = wf1.duplicate(self.otheruser) self.assertNotEqual(wf1.id, wf2.id) self.assertEqual(wf2.owner, self.otheruser) self.assertEqual(wf2.name, "Copy of " + wf1.name) self.assertIsNone(wf2.last_delta) # no undo history self.assertFalse(wf2.public) self.assertEqual(wf1.wf_modules.count(), wf2.wf_modules.count())
def createTestWorkflow(self): # Create a standard test workflow, but it has only one module so add two more to test render pipeline # Throw a missing value in there to test handling of NA values test_csv = 'Class,M,F\n' \ 'math,10,12\n' \ 'english,,7\n' \ 'history,11,13\n' \ 'economics,20,20' self.test_table = pd.read_csv(io.StringIO(test_csv), header=0, skipinitialspace=True) self.workflow1 = create_testdata_workflow(csv_text=test_csv) self.pspec11 = ParameterSpec.objects.get(id_name='csv') self.module2_version = add_new_module_version('Module 2', dispatch='NOP') self.pspec21 = add_new_parameter_spec(self.module2_version, ParameterSpec.STRING, def_value='foo') self.pspec22 = add_new_parameter_spec(self.module2_version, ParameterSpec.FLOAT, def_value=3.14) self.pspec23 = add_new_parameter_spec(self.module2_version, ParameterSpec.INTEGER, def_value=42) self.pspec24 = add_new_parameter_spec(self.module2_version, ParameterSpec.CHECKBOX, def_value=True) self.pspec25 = ParameterSpec.objects.create( module_version=self.module2_version, type=ParameterSpec.MENU, def_menu_items='Apple|Banana|Kittens', def_value='1') self.module3_version = add_new_module_version('Module 3', dispatch='double_M_col') self.pspec31 = add_new_parameter_spec(self.module3_version, ParameterSpec.BUTTON) self.wfmodule1 = WfModule.objects.get(order=0) self.wfmodule2 = add_new_wf_module(self.workflow1, self.module2_version, 1) self.wfmodule3 = add_new_wf_module(self.workflow1, self.module3_version, 2)
def setUp(self): super().setUp() self.workflow = create_testdata_workflow() self.wfm1 = WfModule.objects.first() self.wfm2 = add_new_wf_module(self.workflow, ModuleVersion.objects.first(), 1) # order = 1 self.test_data = 'stored data'.encode() self.metadata = 'metadataish' # Use a more realistic test table with lots of data of different types # mock data wasn't finding bugs related to dict-type columns fname = os.path.join(settings.BASE_DIR, 'server/tests/test_data/sfpd.json') sfpd = json.load(open(fname)) self.test_table = pd.DataFrame(sfpd) sanitize_dataframe(self.test_table)
def test_execute_new_revision(self): workflow = create_testdata_workflow(table_csv) wf_module2 = load_and_add_module('selectcolumns', workflow=workflow) async_to_sync(execute_workflow)(workflow) pval = get_param_by_id_name('colnames', wf_module=wf_module2) pval.set_value('A') wf_module2.last_relevant_delta_id = 2 wf_module2.save(update_fields=['last_relevant_delta_id']) async_to_sync(execute_workflow)(workflow) wf_module2.refresh_from_db() result = wf_module2.get_cached_render_result().result self.assertEqual(result, ProcessResult(table_dataframe[['A']])) self.assertEqual(cached_render_result_revision_list(workflow), [0, 2])
def test_execute_new_revision(self): workflow = create_testdata_workflow(table_csv) wf_module2 = load_and_add_module('selectcolumns', workflow=workflow) # Add command, modifying revision pval = get_param_by_id_name('colnames', wf_module=wf_module2) ChangeParameterCommand.create(pval, 'A') self.assertEqual(cached_render_result_revision_list(workflow), [None, None]) wf_module1 = workflow.wf_modules.first() wf_module1.last_relevant_delta_id = 1 wf_module1.save() wf_module2.last_relevant_delta_id = 2 wf_module2.save() result = execute_wfmodule(wf_module2) self.assertEqual(result, ProcessResult(table_dataframe[['A']])) self.assertEqual(cached_render_result_revision_list(workflow), [1, 2])
def test_resume_without_rerunning_unneeded_renders(self): workflow = create_testdata_workflow(table_csv) wf_module1 = workflow.wf_modules.first() wf_module2 = load_and_add_module('selectcolumns', workflow=workflow, last_relevant_delta_id=1) wf_module1.last_relevant_delta_id = 1 wf_module1.save() expected = execute_wfmodule(wf_module2) wf_module2.refresh_from_db() wf_module2.last_relevant_delta_id = 2 wf_module2.save() with mock.patch('server.dispatch.module_dispatch_render') as mdr: mdr.return_value = expected result = execute_wfmodule(wf_module2) mdr.assert_called_once() self.assertEqual(result, expected)
def setUp(self): super().setUp() self.workflow = create_testdata_workflow()
def setUp(self): self.workflow = create_testdata_workflow()
def test_execute_mark_unreachable(self, send_delta_async): send_delta_async.return_value = fake_future workflow = create_testdata_workflow(table_csv) # Default pythoncode value is passthru wf_module2 = load_and_add_module('pythoncode', workflow=workflow) wf_module3 = load_and_add_module('selectcolumns', workflow=workflow, param_values={'drop_or_keep': 1, 'colnames': 'A,B'}) async_to_sync(execute_workflow)(workflow) # Should set status of all modules to 'ok' wf_module3.refresh_from_db() self.assertEqual(wf_module3.status, 'ok') # Update parameter. Now module 2 will return an error. wf_module2.parameter_vals.get(parameter_spec__id_name='code') \ .set_value('=ERROR') wf_module2.last_relevant_delta_id = 2 wf_module3.last_relevant_delta_id = 2 wf_module2.save(update_fields=['last_relevant_delta_id']) wf_module3.save(update_fields=['last_relevant_delta_id']) # (more integration-test-y) now their statuses are 'busy' because they # await render (and not because they're fetching) wf_module2.refresh_from_db() self.assertEqual(wf_module2.status, 'busy') self.assertEqual(wf_module2.is_busy, False) # is_busy is for fetch wf_module3.refresh_from_db() self.assertEqual(wf_module3.status, 'busy') self.assertEqual(wf_module2.is_busy, False) # is_busy is for fetch async_to_sync(execute_workflow)(workflow) # Now we expect module 2 to have 'error', 3 to have 'unreachable' wf_module2.refresh_from_db() self.assertEqual(wf_module2.status, 'error') wf_module3.refresh_from_db() self.assertEqual(wf_module3.status, 'unreachable') # send_delta_async.assert_called_with(workflow.id, { # 'updateWfModules': { # str(wf_module2.id): { # 'error_msg': 'ERROR', # 'status': 'error', # 'quick_fixes': [], # 'output_columns': [], # 'last_relevant_delta_id': # wf_module2.last_relevant_delta_id # } # } # }) send_delta_async.assert_called_with(workflow.id, { 'updateWfModules': { str(wf_module3.id): { 'error_msg': '', 'status': 'unreachable', 'quick_fixes': [], 'output_columns': [], 'output_n_rows': 0, 'last_relevant_delta_id': wf_module3.last_relevant_delta_id, 'cached_render_result_delta_id': wf_module2.last_relevant_delta_id, } } })
def setUp(self): super(ReorderFromTableTests, self).setUp() self.workflow = create_testdata_workflow(csv_text=test_csv) self.wf_module = load_and_add_module('reorder', workflow=self.workflow) self.history_pval = get_param_by_id_name('reorder-history')
def setUp(self): super(RenameFromTableTests, self).setUp() self.workflow = create_testdata_workflow(csv_text=test_csv) self.wf_module = load_and_add_module('rename', workflow=self.workflow) self.entries_pval = get_param_by_id_name('rename-entries')