def test_reload_external_module(self):
# source_version => external
mini_module = {
'name': 'Test',
'id_name': 'test',
'category': 'Cats',
'source_version': 'f00dbeef',
'parameters': [
{
'name': 'URL',
'id_name': 'url',
'type': 'string',
},
]
}
m1 = load_module_from_dict(mini_module)
self.assertEqual(m1.source_version_hash, mini_module['source_version'])
pspec1 = ParameterSpec.objects.get(id_name='url', module_version=m1)
# Create a wf_module that references this module. Should create a new ParameterVal
wf = add_new_workflow(name='Worky')
wfm = add_new_wf_module(workflow=wf, module_version=m1, order=1)
ParameterVal.objects.get(parameter_spec=pspec1)
mini_module2 = {
'name':
'Test',
'id_name':
'test',
'category':
'Cats',
'source_version':
'deadbeef',
'parameters': [{
'name': 'URL',
'id_name': 'url',
'type': 'string',
}, {
'name': 'New Integer',
'id_name': 'newint',
'type': 'integer'
}]
}
# loading new definition should create new module_version, should not add/alter old parameter values
m2 = load_module_from_dict(mini_module2)
self.assertNotEqual(m1.id, m2.id)
with self.assertRaises(ParameterVal.DoesNotExist):
ParameterVal.objects.get(parameter_spec__id_name='newint')
# load same version again, should re-use module_version
m3 = load_module_from_dict(mini_module2)
self.assertEqual(m2.id, m3.id)
# load a different version
mini_module4 = copy.deepcopy(mini_module2)
mini_module4['source_version'] = 'f0f0beef'
m4 = load_module_from_dict(mini_module4)
self.assertNotEqual(m3.id, m4.id)
def test_reload_internal_module(self):
self.assertEqual(len(Module.objects.all()), 0) # we should be starting with no modules
m1 = load_module_from_dict(self.loadcsv)
url_spec1 = ParameterSpec.objects.get(id_name='url')
ParameterSpec.objects.get(id_name='fetch')
self.assertEqual(m1.source_version_hash, '1.0') # internal modules get this version
# create wf_modules in two different workflows that reference this module
wf1 = add_new_workflow(name='Worky')
wfm1 = add_new_wf_module(workflow=wf1, module_version=m1, order=1)
wf2 = add_new_workflow(name='Worky 2')
wfm2 = add_new_wf_module(workflow=wf2, module_version=m1, order=1)
# precondition: corresponding parameter val exists for each wfm with correct default
# also tested in test_wfmodule.py but whatevs, won't hurt
url_pval1 = ParameterVal.objects.get(parameter_spec=url_spec1, wf_module=wfm1)
self.assertEqual(url_pval1.value, 'http://foo.com')
url_pval2 = ParameterVal.objects.get(parameter_spec=url_spec1, wf_module=wfm2)
self.assertEqual(url_pval2.value, 'http://foo.com')
# load the revised module, check that it ends up with the same primary key
m2 = load_module_from_dict(self.loadcsv2)
self.assertEqual(m1.id, m2.id)
self.assertEqual(m2.module.help_url, '')
# button pspec should be gone
with self.assertRaises(ParameterSpec.DoesNotExist):
ParameterSpec.objects.get(id_name='fetch')
# parametervals should now point to spec with new type, order, default value
# and have value=default value because type changed
self.assertEqual(ParameterSpec.objects.filter(id_name='url').count(), 1)
url_spec2 = ParameterSpec.objects.get(id_name='url')
self.assertEqual(url_spec2.type, ParameterSpec.INTEGER)
self.assertEqual(url_spec2.order, 1)
self.assertEqual(url_spec2.def_value, '42')
url_pval1.refresh_from_db()
self.assertEqual(url_pval1.value, '42')
self.assertEqual(url_pval1.parameter_spec.id, url_spec2.id)
url_pval2.refresh_from_db()
self.assertEqual(url_pval2.value, '42')
self.assertEqual(url_pval2.parameter_spec.id, url_spec2.id)
# new Menu parameter should exist, with corresponding new values in WfModules
menu_spec = ParameterSpec.objects.get(id_name='caketype')
self.assertEqual(menu_spec.type, ParameterSpec.MENU)
self.assertEqual(menu_spec.def_value, '1')
self.assertEqual(menu_spec.def_menu_items, 'Cheese|Chocolate')
self.assertEqual(menu_spec.order, 0)
menu_pval1 = ParameterVal.objects.get(parameter_spec=menu_spec, wf_module=wfm1)
self.assertEqual(menu_pval1.value, '1')
self.assertEqual(menu_pval1.order, 0)
menu_pval2 = ParameterVal.objects.get(parameter_spec=menu_spec, wf_module=wfm2)
self.assertEqual(menu_pval2.value, '1')
self.assertEqual(menu_pval1.order, 0)
# load the old one again, just for kicks (and to test updating a previously updated module_version)
m2 = load_module_from_dict(self.loadcsv)
def test_module_reload(self):
self.assertEqual(len(Module.objects.all()),
0) # we should be starting with no modules
m1 = load_module_from_dict(self.loadcsv)
url_spec1 = ParameterSpec.objects.get(id_name='url')
button_spec1 = ParameterSpec.objects.get(id_name='fetch')
# create wf_modules in two different workflows that reference this module
wf1 = add_new_workflow(name='Worky')
wfm1 = add_new_wf_module(workflow=wf1, module_version=m1, order=1)
wfm1.create_default_parameters()
wf2 = add_new_workflow(name='Worky 2')
wfm2 = add_new_wf_module(workflow=wf2, module_version=m1, order=1)
wfm2.create_default_parameters()
# precondition: corresponding parameter val exists for each wfm with correct default
# also tested in test_wfmodule.py but whatevs, won't hurt
url_pval1 = ParameterVal.objects.get(parameter_spec=url_spec1,
wf_module=wfm1)
self.assertEqual(url_pval1.value, 'http://foo.com')
url_pval2 = ParameterVal.objects.get(parameter_spec=url_spec1,
wf_module=wfm2)
self.assertEqual(url_pval2.value, 'http://foo.com')
# load the revised module, check that it ends up with the same primary key
m2 = load_module_from_dict(self.loadcsv2)
self.assertEqual(m1.id, m2.id)
# button pspec should be gone
with self.assertRaises(ParameterSpec.DoesNotExist):
ParameterSpec.objects.get(id_name='fetch')
# url spec should still exist with same id, new type, new order
# existing parameterval should have new default values
url_spec2 = ParameterSpec.objects.get(id_name='url')
self.assertEqual(url_spec1.id, url_spec2.id)
self.assertEqual(url_spec2.type, ParameterSpec.INTEGER)
self.assertEqual(url_spec2.order, 1)
url_pval1.refresh_from_db()
self.assertEqual(url_pval1.value, '42')
url_pval2.refresh_from_db()
self.assertEqual(url_pval2.value, '42')
# new Menu parameter should exist, with corresponding new values in WfModules
menu_spec = ParameterSpec.objects.get(id_name='caketype')
self.assertEqual(menu_spec.type, ParameterSpec.MENU)
self.assertEqual(menu_spec.def_value, '1')
self.assertEqual(menu_spec.def_menu_items, 'Cheese|Chocolate')
self.assertEqual(menu_spec.order, 0)
menu_pval1 = ParameterVal.objects.get(parameter_spec=menu_spec,
wf_module=wfm1)
self.assertEqual(menu_pval1.value, '1')
self.assertEqual(menu_pval1.order, 0)
menu_pval2 = ParameterVal.objects.get(parameter_spec=menu_spec,
wf_module=wfm2)
self.assertEqual(menu_pval2.value, '1')
self.assertEqual(menu_pval1.order, 0)
def test_load_valid(self):
self.assertEqual(len(Module.objects.all()),
0) # we should be starting with no modules
load_module_from_dict(self.loadcsv)
# basic properties
self.assertEqual(len(Module.objects.all()), 1)
m = Module.objects.all()[0]
self.assertEqual(m.name, 'Load CSV')
self.assertEqual(m.id_name, 'loadcsv')
self.assertEqual(m.help_url, 'http://help.com/help')
# parameters
pspecs = ParameterSpec.objects.all()
self.assertEqual(len(pspecs), 5)
url_spec = ParameterSpec.objects.get(id_name='url')
self.assertEqual(url_spec.name, 'URL')
self.assertEqual(url_spec.id_name, 'url')
self.assertEqual(url_spec.type, ParameterSpec.STRING)
self.assertEqual(url_spec.def_value, 'http://foo.com')
self.assertEqual(url_spec.def_visible, True)
self.assertEqual(url_spec.ui_only, False)
self.assertEqual(url_spec.multiline, False)
self.assertEqual(url_spec.derived_data, False)
self.assertEqual(url_spec.order, 0)
name = ParameterSpec.objects.get(id_name='name')
self.assertEqual(name.name, 'Name')
self.assertEqual(name.id_name, 'name')
self.assertEqual(name.type, ParameterSpec.STRING)
self.assertEqual(name.placeholder, 'Type in a name and hit enter')
button_spec = ParameterSpec.objects.get(id_name='fetch')
self.assertEqual(button_spec.name, 'Fetch')
self.assertEqual(button_spec.id_name, 'fetch')
self.assertEqual(button_spec.type, ParameterSpec.BUTTON)
self.assertEqual(button_spec.def_visible, False)
self.assertEqual(button_spec.ui_only, True)
self.assertEqual(button_spec.order, 2)
# check missing default has a default, and that multiline works
nodef_spec = ParameterSpec.objects.get(id_name='nodefault')
self.assertEqual(nodef_spec.type, ParameterSpec.STRING)
self.assertEqual(nodef_spec.def_value, '')
self.assertEqual(nodef_spec.multiline, True)
self.assertEqual(nodef_spec.derived_data, True)
# Make sure checkbox loads with correct default value
# This tests boolean -> string conversion (JSON is boolean, def_value is string)
cb_spec = ParameterSpec.objects.get(id_name='doitcheckbox')
self.assertEqual(cb_spec.type, ParameterSpec.CHECKBOX)
self.assertEqual(cb_spec.def_value, 'True')
def test_missing_keys(self):
module_keys = ['name','id_name','category']
for k in module_keys:
missing = copy.deepcopy(self.loadcsv)
del missing[k]
with self.assertRaises(ValueError):
load_module_from_dict(missing)
param_keys = ['name','id_name','type']
for k in param_keys:
missing = copy.deepcopy(self.loadcsv)
del missing['parameters'][0][k]
with self.assertRaises(ValueError):
load_module_from_dict(missing)
def load_and_add_module_from_dict(module_dict, workflow=None):
if not workflow:
workflow = add_new_workflow('Workflow 1')
module_version = load_module_from_dict(module_dict)
num_modules = WfModule.objects.filter(workflow=workflow).count()
wf_module = add_new_wf_module(workflow, module_version, order=num_modules)
return wf_module
def load_and_add_module(workflow, module_spec):
if not workflow:
workflow = add_new_workflow('Workflow 1')
module_version = load_module_from_dict(module_spec)
wf_module = add_new_wf_module(workflow, module_version, 1) # 1 = order after PasteCSV from create_mock_workflow
wf_module.create_default_parameters()
return wf_module
def test_load_valid(self):
self.assertEqual(len(Module.objects.all()),
0) # we should be starting with no modules
load_module_from_dict(self.loadcsv)
# basic properties
self.assertEqual(len(Module.objects.all()), 1)
m = Module.objects.all()[0]
self.assertEqual(m.name, 'Load CSV')
self.assertEqual(m.id_name, 'loadcsv')
# parameters
pspecs = ParameterSpec.objects.all()
self.assertEqual(len(pspecs), 3)
url_spec = ParameterSpec.objects.get(id_name='url')
self.assertEqual(url_spec.name, 'URL')
self.assertEqual(url_spec.id_name, 'url')
self.assertEqual(url_spec.type, ParameterSpec.STRING)
self.assertEqual(url_spec.def_string, 'http://foo.com')
self.assertEqual(url_spec.def_visible, True)
self.assertEqual(url_spec.ui_only, False)
self.assertEqual(url_spec.multiline, False)
self.assertEqual(url_spec.derived_data, False)
self.assertEqual(url_spec.order, 0)
button_spec = ParameterSpec.objects.get(id_name='fetch')
self.assertEqual(button_spec.name, 'Fetch')
self.assertEqual(button_spec.id_name, 'fetch')
self.assertEqual(button_spec.type, ParameterSpec.BUTTON)
self.assertEqual(button_spec.def_visible, False)
self.assertEqual(button_spec.ui_only, True)
self.assertEqual(button_spec.order, 1)
# check missing default has a default, and that multiline works
nodef_spec = ParameterSpec.objects.get(id_name='nodefault')
self.assertEqual(nodef_spec.type, ParameterSpec.STRING)
self.assertEqual(nodef_spec.def_string, '')
self.assertEqual(nodef_spec.multiline, True)
self.assertEqual(nodef_spec.derived_data, True)
def load_and_add_module_from_dict(module_dict,
workflow=None,
param_values={},
last_relevant_delta_id=0):
if not workflow:
workflow = add_new_workflow('Workflow 1')
module_version = load_module_from_dict(module_dict)
num_modules = WfModule.objects.filter(workflow=workflow).count()
wf_module = add_new_wf_module(
workflow,
module_version,
param_values=param_values,
last_relevant_delta_id=last_relevant_delta_id,
order=num_modules)
return wf_module
def test_condui(self):
# A very barebones module to test conditional UI loading
cond_ui_valid = {
'name':
'CondUI1',
'id_name':
'condui1',
'category':
'Analyze',
'parameters': [{
'name': 'cond_menu',
'id_name': 'cond_menu',
'type': 'menu',
'menu_items': 'cond1|cond2|cond3'
}, {
'name': 'cond_test',
'id_name': 'cond_test',
'type': 'checkbox',
'visible_if': {
'id_name': 'cond_menu',
'value': 'cond1|cond3'
}
}]
}
self.assertEqual(len(Module.objects.all()), 0)
load_module_from_dict(cond_ui_valid)
cond_spec = ParameterSpec.objects.get(id_name='cond_test')
cond_spec_visibility = json.loads(cond_spec.visible_if)
self.assertEqual(cond_spec_visibility,
cond_ui_valid['parameters'][1]['visible_if'])
new_cond_ui = copy.copy(cond_ui_valid)
del new_cond_ui['parameters'][1]['visible_if']['value']
with self.assertRaises(ValueError):
load_module_from_dict(
new_cond_ui
) # this also tests that db is still valid if load fails
new_cond_ui['parameters'][1]['visible_if']['value'] = 'cond1|cond2'
load_module_from_dict(new_cond_ui)
cond_spec_new = ParameterSpec.objects.get(id_name='cond_test')
cond_spec_visibility_new = json.loads(cond_spec_new.visible_if)
self.assertEqual(cond_spec_visibility_new,
new_cond_ui['parameters'][1]['visible_if'])
def import_module_from_directory(url,
reponame,
version,
importdir,
force_reload=False):
destination_directory = None
ui_info = {}
try:
# check that right files exist
extension_file_mapping = validate_module_structure(importdir)
python_file = extension_file_mapping['py']
json_file = extension_file_mapping['json']
html_file = extension_file_mapping.get('html', None)
# load json file
module_config = get_module_config_from_json(url,
extension_file_mapping,
importdir)
id_name = module_config['id_name']
# Don't allow importing the same version twice, unless forced
if not force_reload:
if ModuleVersion.objects.filter(module__id_name=id_name,
source_version_hash=version):
raise ValidationError(f'Version {version} of module {url}'
' has already been imported')
# Don't allow loading a module with the same id_name from a different
# repo. Prevents replacement attacks.
module_urls = get_already_imported_module_urls()
if module_config["id_name"] in module_urls \
and url != module_urls[module_config["id_name"]]:
source = module_urls[module_config["id_name"]]
if source == '':
source = "Internal"
raise ValidationError(
f"Module {module_config['id_name']} has already been loaded"
f' from a different repo: {source}')
module_config['source_version'] = version
module_config['link'] = url
if 'author' not in module_config:
module_config['author'] = retrieve_author(url)
# Ensure that modules are categorised properly – if a module category
# isn't one of our pre-defined categories, then we just set it to
# other.
if module_config["category"] not in get_categories():
module_config["category"] = "Other"
# The core work of creating a module
destination_directory = create_destination_directory(id_name, version)
move_files_to_final_location(destination_directory, importdir,
[json_file, python_file, html_file])
add_boilerplate_and_check_syntax(destination_directory, python_file)
validate_python_functions(destination_directory, python_file)
# If that succeeds, initialise module in our database
module_version = load_module_from_dict(module_config)
# clean-up
shutil.rmtree(importdir)
if force_reload:
dynamicdispatch.load_module.cache_clear()
# For now, our policy is to update all wfmodules to this just-imported
# version
module = module_version.module
for wfm in WfModule.objects.filter(module_version__module=module):
update_wfm_parameters_to_new_version(wfm, module_version)
except Exception as e:
log_message('Error importing module %s: %s' % (url, str(e)))
if destination_directory is not None:
try:
shutil.rmtree(destination_directory)
except:
pass
try:
shutil.rmtree(destination_directory + '-original')
except:
pass
raise
# return data that we probably want displayed in the UI.
ui_info["category"] = module_config["category"]
ui_info["project"] = reponame
ui_info["author"] = module_config["author"]
ui_info["name"] = module_config["name"]
return ui_info
def test_missing_keys(self):
with self.assertRaises(ValueError):
load_module_from_dict(self.missing_name)
with self.assertRaises(ValueError):
load_module_from_dict(self.missing_id_name)
with self.assertRaises(ValueError):
load_module_from_dict(self.missing_category)
with self.assertRaises(ValueError):
load_module_from_dict(self.missing_param_name)
with self.assertRaises(ValueError):
load_module_from_dict(self.missing_param_id_name)
with self.assertRaises(ValueError):
load_module_from_dict(self.missing_param_type)
def load_module_version(filename):
return load_module_from_dict(load_module_dict(filename))
