def test_workflow_reorder_modules(self):
wfm1 = add_new_wf_module(self.workflow1, self.module_version1, 0)
wfm2 = add_new_wf_module(self.workflow1, self.module_version1, 1)
wfm3 = add_new_wf_module(self.workflow1, self.module_version1, 2)
# your basic reordering
request = self._build_patch('/api/workflows/%d/' % self.workflow1.id,
data=[{'id': wfm1.id, 'order': 2},
{'id': wfm2.id, 'order': 0},
{'id': wfm3.id, 'order': 1}],
format='json',
user=self.user)
response = workflow_detail(request, pk = self.workflow1.id)
self.assertIs(response.status_code, status.HTTP_204_NO_CONTENT)
self.assertEqual(list(WfModule.objects.order_by('order').values_list('id', flat=True)),
[wfm2.id, wfm3.id, wfm1.id])
# bad data should generate a 400 error
# (we don't test every possible failure case, ReorderModulesCommand tests does that)
request = self._build_patch('/api/workflows/%d/' % self.workflow1.id,
data=[{'problem':'bad data'}],
format='json',
user=self.user)
response = workflow_detail(request, pk = self.workflow1.id)
self.assertIs(response.status_code, status.HTTP_400_BAD_REQUEST)
def setUp(self):
super(UpdatesTests, self).setUp() # log in
self.workflow = add_new_workflow('Update scan')
loadurl = load_module_version('loadurl')
self.wfm1 = add_new_wf_module(self.workflow, loadurl, order=0)
self.wfm2 = add_new_wf_module(self.workflow, loadurl, order=1)
self.wfm3 = add_new_wf_module(self.workflow, loadurl, order=2)
# fake out the current time so we can run the test just-so
self.nowtime = parser.parse('Aug 28 1999 2:35PM UTC')
def test_workflow_duplicate(self):
# Create workflow with two WfModules
wf1 = create_testdata_workflow()
self.assertNotEqual(wf1.owner, self.otheruser) # should owned by user created by LoggedInTestCase
module_version1 = add_new_module_version('Module 1')
add_new_wf_module(wf1, module_version1, 1) # order=1
self.assertEqual(wf1.wf_modules.count(), 2)
wf2 = wf1.duplicate(self.otheruser)
self.assertNotEqual(wf1.id, wf2.id)
self.assertEqual(wf2.owner, self.otheruser)
self.assertEqual(wf2.name, "Copy of " + wf1.name)
self.assertIsNone(wf2.last_delta) # no undo history
self.assertFalse(wf2.public)
self.assertEqual(wf1.wf_modules.count(), wf2.wf_modules.count())
def createTestWorkflow(self):
# Create a standard test workflow, but it has only one module so add two more to test render pipeline
# Throw a missing value in there to test handling of NA values
test_csv = 'Class,M,F\n' \
'math,10,12\n' \
'english,,7\n' \
'history,11,13\n' \
'economics,20,20'
self.test_table = pd.read_csv(io.StringIO(test_csv),
header=0,
skipinitialspace=True)
self.workflow1 = create_testdata_workflow(csv_text=test_csv)
self.pspec11 = ParameterSpec.objects.get(id_name='csv')
self.module2_version = add_new_module_version('Module 2',
dispatch='NOP')
self.pspec21 = add_new_parameter_spec(self.module2_version,
ParameterSpec.STRING,
def_value='foo')
self.pspec22 = add_new_parameter_spec(self.module2_version,
ParameterSpec.FLOAT,
def_value=3.14)
self.pspec23 = add_new_parameter_spec(self.module2_version,
ParameterSpec.INTEGER,
def_value=42)
self.pspec24 = add_new_parameter_spec(self.module2_version,
ParameterSpec.CHECKBOX,
def_value=True)
self.pspec25 = ParameterSpec.objects.create(
module_version=self.module2_version,
type=ParameterSpec.MENU,
def_menu_items='Apple|Banana|Kittens',
def_value='1')
self.module3_version = add_new_module_version('Module 3',
dispatch='double_M_col')
self.pspec31 = add_new_parameter_spec(self.module3_version,
ParameterSpec.BUTTON)
self.wfmodule1 = WfModule.objects.get(order=0)
self.wfmodule2 = add_new_wf_module(self.workflow1,
self.module2_version, 1)
self.wfmodule3 = add_new_wf_module(self.workflow1,
self.module3_version, 2)
def setUp(self):
# Create a workflow with two modules
self.workflow = create_testdata_workflow()
# defined by create_testdata_workflow
self.module_version = ModuleVersion.objects.first()
self.module1 = WfModule.objects.first()
self.module2 = add_new_wf_module(self.workflow, self.module_version,
order=1)
def test_load_and_dispatch(self):
test_dir = self.fake_github_clone()
import_module_from_directory('https://github.com/account/reponame',
'reponame', '123456', test_dir)
# Module and ModuleVersion should have loaded -- these will raise exception if they don't exist
module = Module.objects.get(id_name=self.importable_id_name)
module_version = ModuleVersion.objects.get(module=module)
# Create a test workflow that uses this imported module
workflow = add_new_workflow('Dynamic Dispatch Test Workflow')
wfm = add_new_wf_module(workflow, module_version, order=1)
# These will fail if we haven't correctly loaded the json describing
# the parameters
stringparam = get_param_by_id_name('test', wf_module=wfm)
colparam = get_param_by_id_name('test_column', wf_module=wfm)
multicolparam = get_param_by_id_name('test_multicolumn', wf_module=wfm)
# Does it render right?
test_csv = 'Class,M,F,Other\n' \
'math,10,12,100\n' \
'english,,7\,200\n' \
'history,11,13,\n' \
'economics,20,20,20'
test_table = pd.read_csv(io.StringIO(test_csv),
header=0,
skipinitialspace=True)
test_table_out = test_table.copy()
test_table_out['M'] *= 2
test_table_out[['F', 'Other']] *= 3
colparam.set_value('M') # double this
multicolparam.set_value('F,Other') # triple these
with self.assertLogs(dynamicdispatch.__name__):
result = module_dispatch_render(module_version, wfm.get_params(),
test_table, None)
self.assertEqual(result, ProcessResult(test_table_out))
# Test that bad column parameter values are removed
colparam.set_value('missing_column_name')
multicolparam.set_value('Other,junk_column_name')
test_table_out = test_table.copy()
# multicolumn parameter has only one valid col
test_table_out[['Other']] *= 3
result = module_dispatch_render(module_version, wfm.get_params(),
test_table, None)
self.assertEqual(result, ProcessResult(test_table_out))
# if the module crashes, we should get an error with a line number
stringparam.set_value('crashme')
result = module_dispatch_render(module_version, wfm.get_params(),
test_table, None)
self.assertEqual(
result,
ProcessResult(
error='ValueError: we crashed! at line 7 of importable.py'))
def setUp(self):
super(ChannelTests, self).setUp()
self.user = create_test_user(username='******',
email='*****@*****.**')
self.workflow = add_new_workflow('Workflow 1')
self.wf_id = self.workflow.id
self.module = add_new_module_version('Module')
self.wf_module = add_new_wf_module(self.workflow, self.module)
self.application = self.mock_auth_middleware(create_url_router())
def test_wf_module_delete(self):
# add a new one to delete; don't mess with other tests
wfmodule4 = add_new_wf_module(self.workflow1, self.module2_version, 3)
self.workflow1.selected_wf_module = 3
self.workflow1.save()
response = self.client.delete('/api/wfmodules/%d' % wfmodule4.id)
self.assertIs(response.status_code, status.HTTP_204_NO_CONTENT)
self.assertFalse(WfModule.objects.filter(workflow=self.workflow1, pk=wfmodule4.id).exists()) # must really be gone
# also check that deleting the selected module nullifies workflow.selected_wf_module
self.workflow1.refresh_from_db()
self.assertEqual(self.workflow1.selected_wf_module, 2)
def setUp(self):
super().setUp()
clear_db()
self.user = create_test_user(username='******',
email='*****@*****.**')
self.workflow = Workflow.objects.create(name='Workflow 1',
owner=self.user)
self.wf_id = self.workflow.id
self.module = add_new_module_version('Module')
self.wf_module = add_new_wf_module(self.workflow, self.module)
self.application = self.mock_auth_middleware(create_url_router())
self.communicators = []
def test_missing_module(self):
# If the WfModule references a Module that does not exist, we should
# get a placeholder
workflow = add_new_workflow('Missing module')
wfm = add_new_wf_module(workflow, None, 0)
response = self.client.get('/api/wfmodules/%d/' % wfm.id)
self.assertIs(response.status_code, status.HTTP_200_OK)
parsed = json.loads(response.content)
self.assertEqual(parsed['module_version']['module'], None)
response = self.client.get('/api/wfmodules/%d/render' % wfm.id)
self.assertIs(response.status_code, status.HTTP_200_OK)
self.assertEqual(json.loads(response.content), empty_data_json)
def setUp(self):
super().setUp()
self.workflow = create_testdata_workflow()
self.wfm1 = WfModule.objects.first()
self.wfm2 = add_new_wf_module(self.workflow,
ModuleVersion.objects.first(),
1) # order = 1
self.test_data = 'stored data'.encode()
self.metadata = 'metadataish'
# Use a more realistic test table with lots of data of different types
# mock data wasn't finding bugs related to dict-type columns
fname = os.path.join(settings.BASE_DIR,
'server/tests/test_data/sfpd.json')
sfpd = json.load(open(fname))
self.test_table = pd.DataFrame(sfpd)
sanitize_dataframe(self.test_table)
def test_updates_module_version(self):
test_dir = self.fake_github_clone()
import_module_from_directory("https://test_url_of_test_module", "importable", "111111", test_dir)
module_version_q = ModuleVersion.objects.filter(module__id_name=self.importable_id_name)
self.assertEqual(module_version_q.count(), 1)
# Create a test workflow that uses this imported module
module_version = module_version_q.first()
workflow = add_new_workflow('updates_module_version workflow')
wfm = add_new_wf_module(workflow, module_version, order=1)
# import "new" version (different version hash)
test_dir = self.fake_github_clone()
import_module_from_directory("https://test_url_of_test_module", "importable", "222222", test_dir)
self.assertEqual(module_version_q.count(), 2)
# should have updated the wfm to newly imported version
wfm.refresh_from_db()
self.assertEqual(wfm.module_version.source_version_hash, "222222")
def test_load_and_dispatch(self):
try:
test_dir = self.fake_github_clone()
import_module_from_directory('https://github.com/account/reponame',
'reponame', '123456', test_dir)
# Module and ModuleVersion should have loaded -- these will raise
# exception if they don't exist
module = Module.objects.get(id_name=self.importable_id_name)
module_version = ModuleVersion.objects.get(module=module)
# Create a test workflow that uses this imported module
workflow = add_new_workflow('Dynamic Dispatch Test Workflow')
wfm = add_new_wf_module(workflow, module_version, order=1)
# These will fail if we haven't correctly loaded the json
# describing the parameters
colparam = get_param_by_id_name('test_column', wf_module=wfm)
multicolparam = get_param_by_id_name('test_multicolumn',
wf_module=wfm)
# Does it render right?
test_csv = 'Class,M,F,Other\n' \
'math,10,12,100\n' \
'english,,7\,200\n' \
'history,11,13,\n' \
'economics,20,20,20'
test_table = pd.read_csv(io.StringIO(test_csv), header=0,
skipinitialspace=True)
test_table_out = test_table.copy()
test_table_out['M'] *= 2
test_table_out[['F', 'Other']] *= 3
colparam.set_value('M') # double this
multicolparam.set_value('F,Other') # triple these
with self.assertLogs():
lm = LoadedModule.for_module_version_sync(module_version)
result = lm.render(wfm.get_params(), test_table, None)
self.assertEqual(result, ProcessResult(test_table_out))
finally:
server.models.loaded_module.load_external_module.cache_clear()
def test_missing_module(self):
workflow = add_new_workflow('Missing module')
wfm = add_new_wf_module(workflow, None, 0)
result = module_dispatch_render(wfm, mock_csv_table)
self.assertEqual(result, ProcessResult())
