def test_execute_revision_0(self):
# Don't crash on a new workflow (rev=0, no caches)
workflow = create_testdata_workflow(table_csv)
wf_module2 = load_and_add_module('selectcolumns', workflow=workflow)
result = execute_wfmodule(wf_module2)
self.assertEqual(result, ProcessResult(table_dataframe))
self.assertEqual(cached_render_result_revision_list(workflow), [0, 0])
def setUp(self):
# Create a workflow with two modules
self.workflow = create_testdata_workflow()
# defined by create_testdata_workflow
self.module_version = ModuleVersion.objects.first()
self.module1 = WfModule.objects.first()
self.module2 = add_new_wf_module(self.workflow, self.module_version,
order=1)
def test_email_no_delta_when_not_changed(self, email):
workflow = create_testdata_workflow(table_csv)
wf_module1 = workflow.wf_modules.first()
wf_module1.notifications = True
wf_module1.save()
async_to_sync(execute_workflow)(workflow) # sends one email
async_to_sync(execute_workflow)(workflow) # should not email
email.assert_called_once()
def test_execute_cache_hit(self):
workflow = create_testdata_workflow(table_csv)
wf_module2 = load_and_add_module('selectcolumns', workflow=workflow)
# Execute -- which should cache the result
expected = execute_wfmodule(wf_module2)
with mock.patch('server.dispatch.module_dispatch_render') as mdr:
result = execute_wfmodule(wf_module2)
self.assertFalse(mdr.called)
self.assertEqual(result, expected)
def setUp(self):
super(DuplicateColumnFromTableTests, self).setUp()
# A reference table for correctness checking
# Performing date conversion here does not help tests as default test
# WF does not parse dates
self.table = reference_table.copy()
self.workflow = create_testdata_workflow(csv_text=test_csv)
self.wf_module = load_and_add_module('duplicatecolumn',
workflow=self.workflow)
self.colnames_pval = get_param_by_id_name('colnames')
def test_execute_revision_0(self):
# Don't crash on a new workflow (rev=0, no caches)
workflow = create_testdata_workflow(table_csv)
wf_module2 = load_and_add_module('selectcolumns', workflow=workflow)
async_to_sync(execute_workflow)(workflow)
wf_module2.refresh_from_db()
result = wf_module2.get_cached_render_result().result
self.assertEqual(result, ProcessResult(table_dataframe))
self.assertEqual(cached_render_result_revision_list(workflow), [0, 0])
def test_pandas_13258(self):
# simple test case where Pandas produces int64 column type, and json
# conversion throws ValueError
# https://github.com/pandas-dev/pandas/issues/13258#issuecomment-326671257
workflow = create_testdata_workflow(csv_text='A,B,C,D\n1,2,3,4')
response = self.client.get('/api/wfmodules/%d/render' %
workflow.wf_modules.first().id)
self.assertIs(response.status_code, 200)
self.assertEqual(json.loads(response.content)['column_types'],
['number', 'number', 'number', 'number'])
def test_pandas_no_header(self):
# When no header row, Pandas uses int64s as column names, and
# json.dumps(list(table)) throws ValueError
workflow = create_testdata_workflow(csv_text='1,2,3,4\n1,2,3,4')
response = self.client.get('/api/wfmodules/%d/render' %
workflow.wf_modules.first().id)
self.assertIs(response.status_code, 200)
self.assertEqual(json.loads(response.content)['columns'],
['1', '2', '3', '4'])
self.assertEqual(json.loads(response.content)['column_types'],
['number', 'number', 'number', 'number'])
def test_email_delta(self, email):
workflow = create_testdata_workflow(table_csv)
wf_module1 = workflow.wf_modules.first()
wf_module1.notifications = True
wf_module1.save()
async_to_sync(execute_workflow)(workflow)
email.assert_called()
wf_module1.refresh_from_db()
self.assertTrue(wf_module1.has_unseen_notification)
def setUp(self):
super().setUp() # log in
self.scraped_table = simple_result_table
self.urls = list(self.scraped_table['url'])
# create a workflow that feeds our urls via PasteCSV into a URLScraper
self.url_table = pd.DataFrame(self.urls, columns=['url'])
self.expected_url_table_result = ProcessResult(self.url_table)
self.expected_url_table_result.sanitize_in_place()
url_csv = 'url\n' + '\n'.join(self.urls)
workflow = create_testdata_workflow(url_csv)
self.wfmodule = load_and_add_module('urlscraper', workflow=workflow)
def test_execute_cache_hit(self):
workflow = create_testdata_workflow(table_csv)
wf_module2 = load_and_add_module('selectcolumns', workflow=workflow)
async_to_sync(execute_workflow)(workflow)
wf_module2.refresh_from_db()
result1 = wf_module2.get_cached_render_result().result
with patch('server.dispatch.module_dispatch_render') as mdr:
async_to_sync(execute_workflow)(workflow)
wf_module2.refresh_from_db()
result2 = wf_module2.get_cached_render_result().result
self.assertFalse(mdr.called)
self.assertEqual(result2, result1)
def test_workflow_duplicate(self):
# Create workflow with two WfModules
wf1 = create_testdata_workflow()
self.assertNotEqual(wf1.owner, self.otheruser) # should owned by user created by LoggedInTestCase
module_version1 = add_new_module_version('Module 1')
add_new_wf_module(wf1, module_version1, 1) # order=1
self.assertEqual(wf1.wf_modules.count(), 2)
wf2 = wf1.duplicate(self.otheruser)
self.assertNotEqual(wf1.id, wf2.id)
self.assertEqual(wf2.owner, self.otheruser)
self.assertEqual(wf2.name, "Copy of " + wf1.name)
self.assertIsNone(wf2.last_delta) # no undo history
self.assertFalse(wf2.public)
self.assertEqual(wf1.wf_modules.count(), wf2.wf_modules.count())
def createTestWorkflow(self):
# Create a standard test workflow, but it has only one module so add two more to test render pipeline
# Throw a missing value in there to test handling of NA values
test_csv = 'Class,M,F\n' \
'math,10,12\n' \
'english,,7\n' \
'history,11,13\n' \
'economics,20,20'
self.test_table = pd.read_csv(io.StringIO(test_csv),
header=0,
skipinitialspace=True)
self.workflow1 = create_testdata_workflow(csv_text=test_csv)
self.pspec11 = ParameterSpec.objects.get(id_name='csv')
self.module2_version = add_new_module_version('Module 2',
dispatch='NOP')
self.pspec21 = add_new_parameter_spec(self.module2_version,
ParameterSpec.STRING,
def_value='foo')
self.pspec22 = add_new_parameter_spec(self.module2_version,
ParameterSpec.FLOAT,
def_value=3.14)
self.pspec23 = add_new_parameter_spec(self.module2_version,
ParameterSpec.INTEGER,
def_value=42)
self.pspec24 = add_new_parameter_spec(self.module2_version,
ParameterSpec.CHECKBOX,
def_value=True)
self.pspec25 = ParameterSpec.objects.create(
module_version=self.module2_version,
type=ParameterSpec.MENU,
def_menu_items='Apple|Banana|Kittens',
def_value='1')
self.module3_version = add_new_module_version('Module 3',
dispatch='double_M_col')
self.pspec31 = add_new_parameter_spec(self.module3_version,
ParameterSpec.BUTTON)
self.wfmodule1 = WfModule.objects.get(order=0)
self.wfmodule2 = add_new_wf_module(self.workflow1,
self.module2_version, 1)
self.wfmodule3 = add_new_wf_module(self.workflow1,
self.module3_version, 2)
def setUp(self):
super().setUp()
self.workflow = create_testdata_workflow()
self.wfm1 = WfModule.objects.first()
self.wfm2 = add_new_wf_module(self.workflow,
ModuleVersion.objects.first(),
1) # order = 1
self.test_data = 'stored data'.encode()
self.metadata = 'metadataish'
# Use a more realistic test table with lots of data of different types
# mock data wasn't finding bugs related to dict-type columns
fname = os.path.join(settings.BASE_DIR,
'server/tests/test_data/sfpd.json')
sfpd = json.load(open(fname))
self.test_table = pd.DataFrame(sfpd)
sanitize_dataframe(self.test_table)
def test_execute_new_revision(self):
workflow = create_testdata_workflow(table_csv)
wf_module2 = load_and_add_module('selectcolumns', workflow=workflow)
async_to_sync(execute_workflow)(workflow)
pval = get_param_by_id_name('colnames', wf_module=wf_module2)
pval.set_value('A')
wf_module2.last_relevant_delta_id = 2
wf_module2.save(update_fields=['last_relevant_delta_id'])
async_to_sync(execute_workflow)(workflow)
wf_module2.refresh_from_db()
result = wf_module2.get_cached_render_result().result
self.assertEqual(result, ProcessResult(table_dataframe[['A']]))
self.assertEqual(cached_render_result_revision_list(workflow), [0, 2])
def test_execute_new_revision(self):
workflow = create_testdata_workflow(table_csv)
wf_module2 = load_and_add_module('selectcolumns', workflow=workflow)
# Add command, modifying revision
pval = get_param_by_id_name('colnames', wf_module=wf_module2)
ChangeParameterCommand.create(pval, 'A')
self.assertEqual(cached_render_result_revision_list(workflow),
[None, None])
wf_module1 = workflow.wf_modules.first()
wf_module1.last_relevant_delta_id = 1
wf_module1.save()
wf_module2.last_relevant_delta_id = 2
wf_module2.save()
result = execute_wfmodule(wf_module2)
self.assertEqual(result, ProcessResult(table_dataframe[['A']]))
self.assertEqual(cached_render_result_revision_list(workflow), [1, 2])
def test_resume_without_rerunning_unneeded_renders(self):
workflow = create_testdata_workflow(table_csv)
wf_module1 = workflow.wf_modules.first()
wf_module2 = load_and_add_module('selectcolumns',
workflow=workflow,
last_relevant_delta_id=1)
wf_module1.last_relevant_delta_id = 1
wf_module1.save()
expected = execute_wfmodule(wf_module2)
wf_module2.refresh_from_db()
wf_module2.last_relevant_delta_id = 2
wf_module2.save()
with mock.patch('server.dispatch.module_dispatch_render') as mdr:
mdr.return_value = expected
result = execute_wfmodule(wf_module2)
mdr.assert_called_once()
self.assertEqual(result, expected)
def setUp(self):
super().setUp()
self.workflow = create_testdata_workflow()
def setUp(self):
self.workflow = create_testdata_workflow()
def test_execute_mark_unreachable(self, send_delta_async):
send_delta_async.return_value = fake_future
workflow = create_testdata_workflow(table_csv)
# Default pythoncode value is passthru
wf_module2 = load_and_add_module('pythoncode', workflow=workflow)
wf_module3 = load_and_add_module('selectcolumns', workflow=workflow,
param_values={'drop_or_keep': 1,
'colnames': 'A,B'})
async_to_sync(execute_workflow)(workflow)
# Should set status of all modules to 'ok'
wf_module3.refresh_from_db()
self.assertEqual(wf_module3.status, 'ok')
# Update parameter. Now module 2 will return an error.
wf_module2.parameter_vals.get(parameter_spec__id_name='code') \
.set_value('=ERROR')
wf_module2.last_relevant_delta_id = 2
wf_module3.last_relevant_delta_id = 2
wf_module2.save(update_fields=['last_relevant_delta_id'])
wf_module3.save(update_fields=['last_relevant_delta_id'])
# (more integration-test-y) now their statuses are 'busy' because they
# await render (and not because they're fetching)
wf_module2.refresh_from_db()
self.assertEqual(wf_module2.status, 'busy')
self.assertEqual(wf_module2.is_busy, False) # is_busy is for fetch
wf_module3.refresh_from_db()
self.assertEqual(wf_module3.status, 'busy')
self.assertEqual(wf_module2.is_busy, False) # is_busy is for fetch
async_to_sync(execute_workflow)(workflow)
# Now we expect module 2 to have 'error', 3 to have 'unreachable'
wf_module2.refresh_from_db()
self.assertEqual(wf_module2.status, 'error')
wf_module3.refresh_from_db()
self.assertEqual(wf_module3.status, 'unreachable')
# send_delta_async.assert_called_with(workflow.id, {
# 'updateWfModules': {
# str(wf_module2.id): {
# 'error_msg': 'ERROR',
# 'status': 'error',
# 'quick_fixes': [],
# 'output_columns': [],
# 'last_relevant_delta_id':
# wf_module2.last_relevant_delta_id
# }
# }
# })
send_delta_async.assert_called_with(workflow.id, {
'updateWfModules': {
str(wf_module3.id): {
'error_msg': '',
'status': 'unreachable',
'quick_fixes': [],
'output_columns': [],
'output_n_rows': 0,
'last_relevant_delta_id':
wf_module3.last_relevant_delta_id,
'cached_render_result_delta_id':
wf_module2.last_relevant_delta_id,
}
}
})
def setUp(self):
super(ReorderFromTableTests, self).setUp()
self.workflow = create_testdata_workflow(csv_text=test_csv)
self.wf_module = load_and_add_module('reorder', workflow=self.workflow)
self.history_pval = get_param_by_id_name('reorder-history')
def setUp(self):
super(RenameFromTableTests, self).setUp()
self.workflow = create_testdata_workflow(csv_text=test_csv)
self.wf_module = load_and_add_module('rename', workflow=self.workflow)
self.entries_pval = get_param_by_id_name('rename-entries')
