def setUp(self):
super(WorkflowViewTests, self).setUp() # log in
self.log_patcher = patch('server.utils.log_user_event')
self.log_patch = self.log_patcher.start()
self.factory = APIRequestFactory()
self.workflow1 = add_new_workflow('Workflow 1')
self.workflow2 = add_new_workflow('Workflow 2')
self.module_version1 = add_new_module_version('Module 1')
add_new_module_version('Module 2')
add_new_module_version('Module 3')
# Add another user, with one public and one private workflow
self.otheruser = User.objects.create(username='******',
email='*****@*****.**',
password='******')
self.other_workflow_private = Workflow.objects.create(
name="Other workflow private",
owner=self.otheruser
)
self.other_workflow_public = Workflow.objects.create(
name="Other workflow public",
owner=self.otheruser,
public=True
)
def test_wf_module_duplicate(self):
wfm1 = self.wfmodule1
# store data to test that it is duplicated
s1 = wfm1.store_fetched_table(mock_csv_table)
s2 = wfm1.store_fetched_table(mock_csv_table2)
wfm1.set_fetched_data_version(s2)
self.assertEqual(len(wfm1.list_fetched_data_versions()), 2)
# duplicate into another workflow, as we would do when duplicating a workflow
workflow2 = add_new_workflow("Test Workflow 2")
wfm1d = wfm1.duplicate(workflow2)
wfm1d.refresh_from_db() # test what we actually have in the db
self.assertEqual(wfm1d.workflow, workflow2)
self.assertEqual(wfm1d.module_version, wfm1.module_version)
self.assertEqual(wfm1d.order, wfm1.order)
self.assertEqual(wfm1d.notes, wfm1.notes)
self.assertEqual(wfm1d.auto_update_data, wfm1.auto_update_data)
self.assertEqual(wfm1d.last_update_check, wfm1.last_update_check)
self.assertEqual(wfm1d.update_interval, wfm1.update_interval)
self.assertEqual(wfm1d.is_collapsed, wfm1.is_collapsed)
self.assertEqual(wfm1d.stored_data_version, wfm1.stored_data_version)
# parameters should be duplicated
self.assertEqual(
ParameterVal.objects.filter(wf_module=wfm1d).count(),
ParameterVal.objects.filter(wf_module=wfm1).count())
# Stored data should contain a clone of content only, not complete version history
self.assertIsNotNone(wfm1d.stored_data_version)
self.assertEqual(wfm1d.stored_data_version, wfm1.stored_data_version)
self.assertTrue(wfm1d.retrieve_fetched_table().equals(
wfm1.retrieve_fetched_table()))
self.assertEqual(len(wfm1d.list_fetched_data_versions()), 1)
async def test_deny_other_users_workflow(self):
other_workflow = add_new_workflow(
'Workflow 2',
owner=create_test_user('other', '*****@*****.**')
)
comm = WebsocketCommunicator(self.application, f'/workflows/{other_workflow.id}/')
connected, _ = await comm.connect()
self.assertFalse(connected)
def test_load_and_dispatch(self):
test_dir = self.fake_github_clone()
import_module_from_directory('https://github.com/account/reponame',
'reponame', '123456', test_dir)
# Module and ModuleVersion should have loaded -- these will raise exception if they don't exist
module = Module.objects.get(id_name=self.importable_id_name)
module_version = ModuleVersion.objects.get(module=module)
# Create a test workflow that uses this imported module
workflow = add_new_workflow('Dynamic Dispatch Test Workflow')
wfm = add_new_wf_module(workflow, module_version, order=1)
# These will fail if we haven't correctly loaded the json describing
# the parameters
stringparam = get_param_by_id_name('test', wf_module=wfm)
colparam = get_param_by_id_name('test_column', wf_module=wfm)
multicolparam = get_param_by_id_name('test_multicolumn', wf_module=wfm)
# Does it render right?
test_csv = 'Class,M,F,Other\n' \
'math,10,12,100\n' \
'english,,7\,200\n' \
'history,11,13,\n' \
'economics,20,20,20'
test_table = pd.read_csv(io.StringIO(test_csv),
header=0,
skipinitialspace=True)
test_table_out = test_table.copy()
test_table_out['M'] *= 2
test_table_out[['F', 'Other']] *= 3
colparam.set_value('M') # double this
multicolparam.set_value('F,Other') # triple these
with self.assertLogs(dynamicdispatch.__name__):
result = module_dispatch_render(module_version, wfm.get_params(),
test_table, None)
self.assertEqual(result, ProcessResult(test_table_out))
# Test that bad column parameter values are removed
colparam.set_value('missing_column_name')
multicolparam.set_value('Other,junk_column_name')
test_table_out = test_table.copy()
# multicolumn parameter has only one valid col
test_table_out[['Other']] *= 3
result = module_dispatch_render(module_version, wfm.get_params(),
test_table, None)
self.assertEqual(result, ProcessResult(test_table_out))
# if the module crashes, we should get an error with a line number
stringparam.set_value('crashme')
result = module_dispatch_render(module_version, wfm.get_params(),
test_table, None)
self.assertEqual(
result,
ProcessResult(
error='ValueError: we crashed! at line 7 of importable.py'))
def setUp(self):
super(ChannelTests, self).setUp()
self.user = create_test_user(username='******',
email='*****@*****.**')
self.workflow = add_new_workflow('Workflow 1')
self.wf_id = self.workflow.id
self.module = add_new_module_version('Module')
self.wf_module = add_new_wf_module(self.workflow, self.module)
self.application = self.mock_auth_middleware(create_url_router())
def setUp(self):
super(UpdatesTests, self).setUp() # log in
self.workflow = add_new_workflow('Update scan')
loadurl = load_module_version('loadurl')
self.wfm1 = add_new_wf_module(self.workflow, loadurl, order=0)
self.wfm2 = add_new_wf_module(self.workflow, loadurl, order=1)
self.wfm3 = add_new_wf_module(self.workflow, loadurl, order=2)
# fake out the current time so we can run the test just-so
self.nowtime = parser.parse('Aug 28 1999 2:35PM UTC')
def test_secret_not_duplicated(self):
val1 = self.secret_val()
val1.set_value({'name': 'foo', 'secret': 'foo'})
workflow2 = add_new_workflow("Test Workflow 2")
wfmodule2 = WfModule.objects.create(module_version=self.module_version,
workflow=workflow2,
order=0)
val2 = val1.duplicate(wfmodule2)
self.assertEqual(val2.get_value(), None)
self.assertEqual(val2.get_secret(), None)
def test_missing_module(self):
# If the WfModule references a Module that does not exist, we should
# get a placeholder
workflow = add_new_workflow('Missing module')
wfm = add_new_wf_module(workflow, None, 0)
response = self.client.get('/api/wfmodules/%d/' % wfm.id)
self.assertIs(response.status_code, status.HTTP_200_OK)
parsed = json.loads(response.content)
self.assertEqual(parsed['module_version']['module'], None)
response = self.client.get('/api/wfmodules/%d/render' % wfm.id)
self.assertIs(response.status_code, status.HTTP_200_OK)
self.assertEqual(json.loads(response.content), empty_data_json)
def test_auth(self):
# Create otheruser and try to access workflow owned by default user
other_user = create_test_user(username='******',
email='*****@*****.**')
wf = add_new_workflow('New Workflow', owner=other_user)
wfm = load_and_add_module('concaturl', workflow=wf)
result = store_external_workflow(
wfm,
f'https://app.workbenchdata.com/workflows/{self.ext_wfm.workflow_id}/'
)
self.assertEqual(
result,
ProcessResult(error='Access denied to the target workflow'))
def setUp(self):
super(DispatchTests, self).setUp() # log in
self.test_csv = 'Class,M,F\n' \
'math,10,12\n' \
'english,,7\n' \
'history,11,13\n' \
'economics,20,20'
self.test_table = pd.read_csv(io.StringIO(self.test_csv))
self.test_table_MF = self.test_table[['M', 'F']]
self.workflow = add_new_workflow('dispatch tests wf')
self.wfm = load_and_add_module('selectcolumns', workflow=self.workflow)
get_param_by_id_name('colnames').set_value('M,F')
def test_duplicate_module(self):
version = self.wfm.store_fetched_table(self.csv_table)
self.wfm.set_fetched_data_version(version)
self.assertEqual(len(self.wfm.list_fetched_data_versions()), 1)
wfm2 = self.wfm.duplicate(add_new_workflow('workflow 2'))
wfm2.refresh_from_db()
self.assertEqual(len(self.wfm.list_fetched_data_versions()), 1)
self.assertEqual(len(wfm2.list_fetched_data_versions()), 1)
self.assertEqual(wfm2.status, WfModule.READY)
result = UploadFile.render(wfm2, None)
self.assertEqual(result, ProcessResult(self.csv_table))
def test_duplicate(self):
# Create a new WfModule of the same type, that has no parametervals yet
# This is the context where duplicate is normally called, when
# duplicating a WfModule
workflow2 = add_new_workflow("Test Workflow 2")
wfmodule2 = WfModule.objects.create(module_version=self.module_version,
workflow=workflow2,
order=0)
sp = ParameterVal.objects.get(pk=self.stringID)
spd = sp.duplicate(wfmodule2)
self.assertEqual(spd.wf_module, wfmodule2)
self.assertEqual(sp.parameter_spec, spd.parameter_spec)
self.assertEqual(sp.value, spd.value)
self.assertEqual(sp.items, spd.items)
self.assertEqual(sp.visible, spd.visible)
self.assertEqual(sp.order, spd.order)
def test_updates_module_version(self):
test_dir = self.fake_github_clone()
import_module_from_directory("https://test_url_of_test_module", "importable", "111111", test_dir)
module_version_q = ModuleVersion.objects.filter(module__id_name=self.importable_id_name)
self.assertEqual(module_version_q.count(), 1)
# Create a test workflow that uses this imported module
module_version = module_version_q.first()
workflow = add_new_workflow('updates_module_version workflow')
wfm = add_new_wf_module(workflow, module_version, order=1)
# import "new" version (different version hash)
test_dir = self.fake_github_clone()
import_module_from_directory("https://test_url_of_test_module", "importable", "222222", test_dir)
self.assertEqual(module_version_q.count(), 2)
# should have updated the wfm to newly imported version
wfm.refresh_from_db()
self.assertEqual(wfm.module_version.source_version_hash, "222222")
def test_load_and_dispatch(self):
try:
test_dir = self.fake_github_clone()
import_module_from_directory('https://github.com/account/reponame',
'reponame', '123456', test_dir)
# Module and ModuleVersion should have loaded -- these will raise
# exception if they don't exist
module = Module.objects.get(id_name=self.importable_id_name)
module_version = ModuleVersion.objects.get(module=module)
# Create a test workflow that uses this imported module
workflow = add_new_workflow('Dynamic Dispatch Test Workflow')
wfm = add_new_wf_module(workflow, module_version, order=1)
# These will fail if we haven't correctly loaded the json
# describing the parameters
colparam = get_param_by_id_name('test_column', wf_module=wfm)
multicolparam = get_param_by_id_name('test_multicolumn',
wf_module=wfm)
# Does it render right?
test_csv = 'Class,M,F,Other\n' \
'math,10,12,100\n' \
'english,,7\,200\n' \
'history,11,13,\n' \
'economics,20,20,20'
test_table = pd.read_csv(io.StringIO(test_csv), header=0,
skipinitialspace=True)
test_table_out = test_table.copy()
test_table_out['M'] *= 2
test_table_out[['F', 'Other']] *= 3
colparam.set_value('M') # double this
multicolparam.set_value('F,Other') # triple these
with self.assertLogs():
lm = LoadedModule.for_module_version_sync(module_version)
result = lm.render(wfm.get_params(), test_table, None)
self.assertEqual(result, ProcessResult(test_table_out))
finally:
server.models.loaded_module.load_external_module.cache_clear()
def test_missing_module(self):
workflow = add_new_workflow('Missing module')
wfm = add_new_wf_module(workflow, None, 0)
result = module_dispatch_render(wfm, mock_csv_table)
self.assertEqual(result, ProcessResult())
def createTestWorkflow(self):
# Create a WfModule with one parameter of each type
self.module_version = add_new_module_version("TestModule")
self.moduleID = self.module_version.module.id
stringSpec = ParameterSpec.objects.create(
name='StringParam',
id_name='stringparam',
module_version=self.module_version,
type=ParameterSpec.STRING,
def_value='foo',
placeholder='placeholder')
stringSpecEmpty = ParameterSpec.objects.create(
name='StringParamEmpty',
id_name='stringparamempty',
module_version=self.module_version,
type=ParameterSpec.STRING)
integerSpec = ParameterSpec.objects.create(
name='IntegerParam',
id_name='integerparam',
module_version=self.module_version,
type=ParameterSpec.INTEGER,
def_value='42')
floatSpec = ParameterSpec.objects.create(
name='FloatParam',
id_name='floatparam',
module_version=self.module_version,
type=ParameterSpec.FLOAT,
def_value='10.11')
checkboxSpec = ParameterSpec.objects.create(
name='CheckboxParam',
id_name='checkboxparam',
module_version=self.module_version,
type=ParameterSpec.CHECKBOX,
def_value='1')
menuSpec = ParameterSpec.objects.create(
name='MenuParam',
id_name='menuparam',
module_version=self.module_version,
type=ParameterSpec.MENU,
def_items='Item A|Item B|Item C',
def_value='1') # should refer to Item B
radioSpec = ParameterSpec.objects.create(
name='RadioParam',
id_name='radioparam',
module_version=self.module_version,
type=ParameterSpec.RADIO,
def_items='Item A|Item B|Item C',
def_value='0') # should refer to Item A
self.workflow = add_new_workflow(name="Test Workflow")
self.workflowID = self.workflow.id
self.wfmodule = WfModule.objects.create(
module_version=self.module_version,
workflow=self.workflow,
order=0)
self.wfmoduleID = self.wfmodule.id
# set non-default values for vals in order to reveal certain types of bugs
stringVal = ParameterVal.objects.create(parameter_spec=stringSpec,
wf_module=self.wfmodule,
value='fooval')
self.stringID = stringVal.id
emptyStringVal = ParameterVal.objects.create(
parameter_spec=stringSpecEmpty, wf_module=self.wfmodule)
self.stringemptyID = emptyStringVal.id
integerVal = ParameterVal.objects.create(parameter_spec=integerSpec,
wf_module=self.wfmodule,
value='10')
self.integerID = integerVal.id
floatVal = ParameterVal.objects.create(parameter_spec=floatSpec,
wf_module=self.wfmodule,
value='3.14159')
self.floatID = floatVal.id
checkboxVal = ParameterVal.objects.create(parameter_spec=checkboxSpec,
wf_module=self.wfmodule,
value='True')
self.checkboxID = checkboxVal.id
menuVal = ParameterVal.objects.create(parameter_spec=menuSpec,
wf_module=self.wfmodule,
value='2',
items=menuSpec.def_items)
self.menuID = menuVal.id
radioVal = ParameterVal.objects.create(parameter_spec=radioSpec,
wf_module=self.wfmodule,
value='0',
items=radioSpec.def_items)
self.radioID = radioVal.id
def setUp(self):
# Define two types of modules we are going to use
self.csv = load_module_version('pastecsv')
self.workflow = add_new_workflow('My Undoable Workflow')
