def test_reload_external_module(self): # source_version => external mini_module = { 'name': 'Test', 'id_name': 'test', 'category': 'Cats', 'source_version': 'f00dbeef', 'parameters': [ { 'name': 'URL', 'id_name': 'url', 'type': 'string', }, ] } m1 = load_module_from_dict(mini_module) self.assertEqual(m1.source_version_hash, mini_module['source_version']) pspec1 = ParameterSpec.objects.get(id_name='url', module_version=m1) # Create a wf_module that references this module. Should create a new ParameterVal wf = add_new_workflow(name='Worky') wfm = add_new_wf_module(workflow=wf, module_version=m1, order=1) ParameterVal.objects.get(parameter_spec=pspec1) mini_module2 = { 'name': 'Test', 'id_name': 'test', 'category': 'Cats', 'source_version': 'deadbeef', 'parameters': [{ 'name': 'URL', 'id_name': 'url', 'type': 'string', }, { 'name': 'New Integer', 'id_name': 'newint', 'type': 'integer' }] } # loading new definition should create new module_version, should not add/alter old parameter values m2 = load_module_from_dict(mini_module2) self.assertNotEqual(m1.id, m2.id) with self.assertRaises(ParameterVal.DoesNotExist): ParameterVal.objects.get(parameter_spec__id_name='newint') # load same version again, should re-use module_version m3 = load_module_from_dict(mini_module2) self.assertEqual(m2.id, m3.id) # load a different version mini_module4 = copy.deepcopy(mini_module2) mini_module4['source_version'] = 'f0f0beef' m4 = load_module_from_dict(mini_module4) self.assertNotEqual(m3.id, m4.id)
def test_reload_internal_module(self): self.assertEqual(len(Module.objects.all()), 0) # we should be starting with no modules m1 = load_module_from_dict(self.loadcsv) url_spec1 = ParameterSpec.objects.get(id_name='url') ParameterSpec.objects.get(id_name='fetch') self.assertEqual(m1.source_version_hash, '1.0') # internal modules get this version # create wf_modules in two different workflows that reference this module wf1 = add_new_workflow(name='Worky') wfm1 = add_new_wf_module(workflow=wf1, module_version=m1, order=1) wf2 = add_new_workflow(name='Worky 2') wfm2 = add_new_wf_module(workflow=wf2, module_version=m1, order=1) # precondition: corresponding parameter val exists for each wfm with correct default # also tested in test_wfmodule.py but whatevs, won't hurt url_pval1 = ParameterVal.objects.get(parameter_spec=url_spec1, wf_module=wfm1) self.assertEqual(url_pval1.value, 'http://foo.com') url_pval2 = ParameterVal.objects.get(parameter_spec=url_spec1, wf_module=wfm2) self.assertEqual(url_pval2.value, 'http://foo.com') # load the revised module, check that it ends up with the same primary key m2 = load_module_from_dict(self.loadcsv2) self.assertEqual(m1.id, m2.id) self.assertEqual(m2.module.help_url, '') # button pspec should be gone with self.assertRaises(ParameterSpec.DoesNotExist): ParameterSpec.objects.get(id_name='fetch') # parametervals should now point to spec with new type, order, default value # and have value=default value because type changed self.assertEqual(ParameterSpec.objects.filter(id_name='url').count(), 1) url_spec2 = ParameterSpec.objects.get(id_name='url') self.assertEqual(url_spec2.type, ParameterSpec.INTEGER) self.assertEqual(url_spec2.order, 1) self.assertEqual(url_spec2.def_value, '42') url_pval1.refresh_from_db() self.assertEqual(url_pval1.value, '42') self.assertEqual(url_pval1.parameter_spec.id, url_spec2.id) url_pval2.refresh_from_db() self.assertEqual(url_pval2.value, '42') self.assertEqual(url_pval2.parameter_spec.id, url_spec2.id) # new Menu parameter should exist, with corresponding new values in WfModules menu_spec = ParameterSpec.objects.get(id_name='caketype') self.assertEqual(menu_spec.type, ParameterSpec.MENU) self.assertEqual(menu_spec.def_value, '1') self.assertEqual(menu_spec.def_menu_items, 'Cheese|Chocolate') self.assertEqual(menu_spec.order, 0) menu_pval1 = ParameterVal.objects.get(parameter_spec=menu_spec, wf_module=wfm1) self.assertEqual(menu_pval1.value, '1') self.assertEqual(menu_pval1.order, 0) menu_pval2 = ParameterVal.objects.get(parameter_spec=menu_spec, wf_module=wfm2) self.assertEqual(menu_pval2.value, '1') self.assertEqual(menu_pval1.order, 0) # load the old one again, just for kicks (and to test updating a previously updated module_version) m2 = load_module_from_dict(self.loadcsv)
def test_module_reload(self): self.assertEqual(len(Module.objects.all()), 0) # we should be starting with no modules m1 = load_module_from_dict(self.loadcsv) url_spec1 = ParameterSpec.objects.get(id_name='url') button_spec1 = ParameterSpec.objects.get(id_name='fetch') # create wf_modules in two different workflows that reference this module wf1 = add_new_workflow(name='Worky') wfm1 = add_new_wf_module(workflow=wf1, module_version=m1, order=1) wfm1.create_default_parameters() wf2 = add_new_workflow(name='Worky 2') wfm2 = add_new_wf_module(workflow=wf2, module_version=m1, order=1) wfm2.create_default_parameters() # precondition: corresponding parameter val exists for each wfm with correct default # also tested in test_wfmodule.py but whatevs, won't hurt url_pval1 = ParameterVal.objects.get(parameter_spec=url_spec1, wf_module=wfm1) self.assertEqual(url_pval1.value, 'http://foo.com') url_pval2 = ParameterVal.objects.get(parameter_spec=url_spec1, wf_module=wfm2) self.assertEqual(url_pval2.value, 'http://foo.com') # load the revised module, check that it ends up with the same primary key m2 = load_module_from_dict(self.loadcsv2) self.assertEqual(m1.id, m2.id) # button pspec should be gone with self.assertRaises(ParameterSpec.DoesNotExist): ParameterSpec.objects.get(id_name='fetch') # url spec should still exist with same id, new type, new order # existing parameterval should have new default values url_spec2 = ParameterSpec.objects.get(id_name='url') self.assertEqual(url_spec1.id, url_spec2.id) self.assertEqual(url_spec2.type, ParameterSpec.INTEGER) self.assertEqual(url_spec2.order, 1) url_pval1.refresh_from_db() self.assertEqual(url_pval1.value, '42') url_pval2.refresh_from_db() self.assertEqual(url_pval2.value, '42') # new Menu parameter should exist, with corresponding new values in WfModules menu_spec = ParameterSpec.objects.get(id_name='caketype') self.assertEqual(menu_spec.type, ParameterSpec.MENU) self.assertEqual(menu_spec.def_value, '1') self.assertEqual(menu_spec.def_menu_items, 'Cheese|Chocolate') self.assertEqual(menu_spec.order, 0) menu_pval1 = ParameterVal.objects.get(parameter_spec=menu_spec, wf_module=wfm1) self.assertEqual(menu_pval1.value, '1') self.assertEqual(menu_pval1.order, 0) menu_pval2 = ParameterVal.objects.get(parameter_spec=menu_spec, wf_module=wfm2) self.assertEqual(menu_pval2.value, '1') self.assertEqual(menu_pval1.order, 0)
def test_load_valid(self): self.assertEqual(len(Module.objects.all()), 0) # we should be starting with no modules load_module_from_dict(self.loadcsv) # basic properties self.assertEqual(len(Module.objects.all()), 1) m = Module.objects.all()[0] self.assertEqual(m.name, 'Load CSV') self.assertEqual(m.id_name, 'loadcsv') self.assertEqual(m.help_url, 'http://help.com/help') # parameters pspecs = ParameterSpec.objects.all() self.assertEqual(len(pspecs), 5) url_spec = ParameterSpec.objects.get(id_name='url') self.assertEqual(url_spec.name, 'URL') self.assertEqual(url_spec.id_name, 'url') self.assertEqual(url_spec.type, ParameterSpec.STRING) self.assertEqual(url_spec.def_value, 'http://foo.com') self.assertEqual(url_spec.def_visible, True) self.assertEqual(url_spec.ui_only, False) self.assertEqual(url_spec.multiline, False) self.assertEqual(url_spec.derived_data, False) self.assertEqual(url_spec.order, 0) name = ParameterSpec.objects.get(id_name='name') self.assertEqual(name.name, 'Name') self.assertEqual(name.id_name, 'name') self.assertEqual(name.type, ParameterSpec.STRING) self.assertEqual(name.placeholder, 'Type in a name and hit enter') button_spec = ParameterSpec.objects.get(id_name='fetch') self.assertEqual(button_spec.name, 'Fetch') self.assertEqual(button_spec.id_name, 'fetch') self.assertEqual(button_spec.type, ParameterSpec.BUTTON) self.assertEqual(button_spec.def_visible, False) self.assertEqual(button_spec.ui_only, True) self.assertEqual(button_spec.order, 2) # check missing default has a default, and that multiline works nodef_spec = ParameterSpec.objects.get(id_name='nodefault') self.assertEqual(nodef_spec.type, ParameterSpec.STRING) self.assertEqual(nodef_spec.def_value, '') self.assertEqual(nodef_spec.multiline, True) self.assertEqual(nodef_spec.derived_data, True) # Make sure checkbox loads with correct default value # This tests boolean -> string conversion (JSON is boolean, def_value is string) cb_spec = ParameterSpec.objects.get(id_name='doitcheckbox') self.assertEqual(cb_spec.type, ParameterSpec.CHECKBOX) self.assertEqual(cb_spec.def_value, 'True')
def test_missing_keys(self): module_keys = ['name','id_name','category'] for k in module_keys: missing = copy.deepcopy(self.loadcsv) del missing[k] with self.assertRaises(ValueError): load_module_from_dict(missing) param_keys = ['name','id_name','type'] for k in param_keys: missing = copy.deepcopy(self.loadcsv) del missing['parameters'][0][k] with self.assertRaises(ValueError): load_module_from_dict(missing)
def load_and_add_module_from_dict(module_dict, workflow=None): if not workflow: workflow = add_new_workflow('Workflow 1') module_version = load_module_from_dict(module_dict) num_modules = WfModule.objects.filter(workflow=workflow).count() wf_module = add_new_wf_module(workflow, module_version, order=num_modules) return wf_module
def load_and_add_module(workflow, module_spec): if not workflow: workflow = add_new_workflow('Workflow 1') module_version = load_module_from_dict(module_spec) wf_module = add_new_wf_module(workflow, module_version, 1) # 1 = order after PasteCSV from create_mock_workflow wf_module.create_default_parameters() return wf_module
def test_load_valid(self): self.assertEqual(len(Module.objects.all()), 0) # we should be starting with no modules load_module_from_dict(self.loadcsv) # basic properties self.assertEqual(len(Module.objects.all()), 1) m = Module.objects.all()[0] self.assertEqual(m.name, 'Load CSV') self.assertEqual(m.id_name, 'loadcsv') # parameters pspecs = ParameterSpec.objects.all() self.assertEqual(len(pspecs), 3) url_spec = ParameterSpec.objects.get(id_name='url') self.assertEqual(url_spec.name, 'URL') self.assertEqual(url_spec.id_name, 'url') self.assertEqual(url_spec.type, ParameterSpec.STRING) self.assertEqual(url_spec.def_string, 'http://foo.com') self.assertEqual(url_spec.def_visible, True) self.assertEqual(url_spec.ui_only, False) self.assertEqual(url_spec.multiline, False) self.assertEqual(url_spec.derived_data, False) self.assertEqual(url_spec.order, 0) button_spec = ParameterSpec.objects.get(id_name='fetch') self.assertEqual(button_spec.name, 'Fetch') self.assertEqual(button_spec.id_name, 'fetch') self.assertEqual(button_spec.type, ParameterSpec.BUTTON) self.assertEqual(button_spec.def_visible, False) self.assertEqual(button_spec.ui_only, True) self.assertEqual(button_spec.order, 1) # check missing default has a default, and that multiline works nodef_spec = ParameterSpec.objects.get(id_name='nodefault') self.assertEqual(nodef_spec.type, ParameterSpec.STRING) self.assertEqual(nodef_spec.def_string, '') self.assertEqual(nodef_spec.multiline, True) self.assertEqual(nodef_spec.derived_data, True)
def load_and_add_module_from_dict(module_dict, workflow=None, param_values={}, last_relevant_delta_id=0): if not workflow: workflow = add_new_workflow('Workflow 1') module_version = load_module_from_dict(module_dict) num_modules = WfModule.objects.filter(workflow=workflow).count() wf_module = add_new_wf_module( workflow, module_version, param_values=param_values, last_relevant_delta_id=last_relevant_delta_id, order=num_modules) return wf_module
def test_condui(self): # A very barebones module to test conditional UI loading cond_ui_valid = { 'name': 'CondUI1', 'id_name': 'condui1', 'category': 'Analyze', 'parameters': [{ 'name': 'cond_menu', 'id_name': 'cond_menu', 'type': 'menu', 'menu_items': 'cond1|cond2|cond3' }, { 'name': 'cond_test', 'id_name': 'cond_test', 'type': 'checkbox', 'visible_if': { 'id_name': 'cond_menu', 'value': 'cond1|cond3' } }] } self.assertEqual(len(Module.objects.all()), 0) load_module_from_dict(cond_ui_valid) cond_spec = ParameterSpec.objects.get(id_name='cond_test') cond_spec_visibility = json.loads(cond_spec.visible_if) self.assertEqual(cond_spec_visibility, cond_ui_valid['parameters'][1]['visible_if']) new_cond_ui = copy.copy(cond_ui_valid) del new_cond_ui['parameters'][1]['visible_if']['value'] with self.assertRaises(ValueError): load_module_from_dict( new_cond_ui ) # this also tests that db is still valid if load fails new_cond_ui['parameters'][1]['visible_if']['value'] = 'cond1|cond2' load_module_from_dict(new_cond_ui) cond_spec_new = ParameterSpec.objects.get(id_name='cond_test') cond_spec_visibility_new = json.loads(cond_spec_new.visible_if) self.assertEqual(cond_spec_visibility_new, new_cond_ui['parameters'][1]['visible_if'])
def import_module_from_directory(url, reponame, version, importdir, force_reload=False): destination_directory = None ui_info = {} try: # check that right files exist extension_file_mapping = validate_module_structure(importdir) python_file = extension_file_mapping['py'] json_file = extension_file_mapping['json'] html_file = extension_file_mapping.get('html', None) # load json file module_config = get_module_config_from_json(url, extension_file_mapping, importdir) id_name = module_config['id_name'] # Don't allow importing the same version twice, unless forced if not force_reload: if ModuleVersion.objects.filter(module__id_name=id_name, source_version_hash=version): raise ValidationError(f'Version {version} of module {url}' ' has already been imported') # Don't allow loading a module with the same id_name from a different # repo. Prevents replacement attacks. module_urls = get_already_imported_module_urls() if module_config["id_name"] in module_urls \ and url != module_urls[module_config["id_name"]]: source = module_urls[module_config["id_name"]] if source == '': source = "Internal" raise ValidationError( f"Module {module_config['id_name']} has already been loaded" f' from a different repo: {source}') module_config['source_version'] = version module_config['link'] = url if 'author' not in module_config: module_config['author'] = retrieve_author(url) # Ensure that modules are categorised properly – if a module category # isn't one of our pre-defined categories, then we just set it to # other. if module_config["category"] not in get_categories(): module_config["category"] = "Other" # The core work of creating a module destination_directory = create_destination_directory(id_name, version) move_files_to_final_location(destination_directory, importdir, [json_file, python_file, html_file]) add_boilerplate_and_check_syntax(destination_directory, python_file) validate_python_functions(destination_directory, python_file) # If that succeeds, initialise module in our database module_version = load_module_from_dict(module_config) # clean-up shutil.rmtree(importdir) if force_reload: dynamicdispatch.load_module.cache_clear() # For now, our policy is to update all wfmodules to this just-imported # version module = module_version.module for wfm in WfModule.objects.filter(module_version__module=module): update_wfm_parameters_to_new_version(wfm, module_version) except Exception as e: log_message('Error importing module %s: %s' % (url, str(e))) if destination_directory is not None: try: shutil.rmtree(destination_directory) except: pass try: shutil.rmtree(destination_directory + '-original') except: pass raise # return data that we probably want displayed in the UI. ui_info["category"] = module_config["category"] ui_info["project"] = reponame ui_info["author"] = module_config["author"] ui_info["name"] = module_config["name"] return ui_info
def test_missing_keys(self): with self.assertRaises(ValueError): load_module_from_dict(self.missing_name) with self.assertRaises(ValueError): load_module_from_dict(self.missing_id_name) with self.assertRaises(ValueError): load_module_from_dict(self.missing_category) with self.assertRaises(ValueError): load_module_from_dict(self.missing_param_name) with self.assertRaises(ValueError): load_module_from_dict(self.missing_param_id_name) with self.assertRaises(ValueError): load_module_from_dict(self.missing_param_type)
def load_module_version(filename): return load_module_from_dict(load_module_dict(filename))