def test_load_twice(self): test_dir = self.fake_github_clone() import_module_from_directory("https://test_url_of_test_module", "importable", "123456", test_dir) test_dir = self.fake_github_clone() # import moves files, so get same files again with self.assertRaises(ValidationError): import_module_from_directory("https://test_url_of_test_module", "importable", "123456", test_dir)
def test_load_twice_fails(self): """loading the same version of the same module twice should fail""" test_dir = self._test_module_path("importable") with self.assertLogs(): import_module_from_directory("123456", Path(test_dir)) with self.assertRaises(ValueError): import_module_from_directory("123456", Path(test_dir))
def test_load_invalid_code(self): test_dir = self.fake_github_clone('test_data/bad_json_module') with self.assertRaises(ValidationError): import_module_from_directory("https://test_url_of_test_module", "bad_json_module", "123456", test_dir) test_dir = self.fake_github_clone('test_data/bad_py_module') with self.assertRaises(ValidationError): import_module_from_directory("https://test_url_of_test_module", "bad_py_module", "123456", test_dir)
def test_already_imported(self): test_dir = self.fake_github_clone() import_module_from_directory("https://github.com/account/importable1", "importable1", "123456", test_dir) test_dir = self.fake_github_clone() # import moves files, so get same files again with self.assertRaises(ValidationError): with self.assertLogs(): import_module_from_directory("https://github.com/account/importable2", "importable2", "123456", test_dir)
def test_load_and_dispatch(self): test_dir = self.fake_github_clone() import_module_from_directory('https://github.com/account/reponame', 'reponame', '123456', test_dir) # Module and ModuleVersion should have loaded -- these will raise exception if they don't exist module = Module.objects.get(id_name=self.importable_id_name) module_version = ModuleVersion.objects.get(module=module) # Create a test workflow that uses this imported module workflow = add_new_workflow('Dynamic Dispatch Test Workflow') wfm = add_new_wf_module(workflow, module_version, order=1) # These will fail if we haven't correctly loaded the json describing # the parameters stringparam = get_param_by_id_name('test', wf_module=wfm) colparam = get_param_by_id_name('test_column', wf_module=wfm) multicolparam = get_param_by_id_name('test_multicolumn', wf_module=wfm) # Does it render right? test_csv = 'Class,M,F,Other\n' \ 'math,10,12,100\n' \ 'english,,7\,200\n' \ 'history,11,13,\n' \ 'economics,20,20,20' test_table = pd.read_csv(io.StringIO(test_csv), header=0, skipinitialspace=True) test_table_out = test_table.copy() test_table_out['M'] *= 2 test_table_out[['F', 'Other']] *= 3 colparam.set_value('M') # double this multicolparam.set_value('F,Other') # triple these with self.assertLogs(dynamicdispatch.__name__): result = module_dispatch_render(module_version, wfm.get_params(), test_table, None) self.assertEqual(result, ProcessResult(test_table_out)) # Test that bad column parameter values are removed colparam.set_value('missing_column_name') multicolparam.set_value('Other,junk_column_name') test_table_out = test_table.copy() # multicolumn parameter has only one valid col test_table_out[['Other']] *= 3 result = module_dispatch_render(module_version, wfm.get_params(), test_table, None) self.assertEqual(result, ProcessResult(test_table_out)) # if the module crashes, we should get an error with a line number stringparam.set_value('crashme') result = module_dispatch_render(module_version, wfm.get_params(), test_table, None) self.assertEqual( result, ProcessResult( error='ValueError: we crashed! at line 7 of importable.py'))
def test_load_twice_force_relaod(self): test_dir = self.fake_github_clone() import_module_from_directory("https://test_url_of_test_module", "importable", "123456", test_dir) self.assertEqual(ModuleVersion.objects.filter(module__id_name=self.importable_id_name).count(), 1) test_dir = self.fake_github_clone() # import moves files, so get same files again import_module_from_directory("https://test_url_of_test_module", "importable", "123456", test_dir, force_reload=True) # should replace existing module_version, not add a new one self.assertEqual(ModuleVersion.objects.filter(module__id_name=self.importable_id_name).count(), 1)
def test_load_twice_force_relaod(self): """We will do a reload of same version if force_reload==True""" test_dir = self._test_module_path('importable') with self.assertLogs(): import_module_from_directory('develop', Path(test_dir)) with self.assertLogs(): import_module_from_directory('develop', Path(test_dir), force_reload=True) # should replace existing module_version, not add a new one self.assertEqual(ModuleVersion.objects.count(), 1)
def reload(): print(f'Reloading {basename}') # import_module_from_directory is unintuitive: it _destroys_ the input # directory. tmpdir = tempfile.mkdtemp() shutil.rmtree(tmpdir) shutil.copytree(directory, tmpdir) import_module_from_directory(pretend_git_url, basename, 'develop', tmpdir, force_reload=True) print('Reloaded')
def reload(): logger.info(f'Reloading...') with tempfile.TemporaryDirectory() as tmpdir: importdir = os.path.join(tmpdir, 'importme') shutil.copytree(directory, importdir) shutil.rmtree(os.path.join(importdir, '.git'), ignore_errors=True) shutil.rmtree(os.path.join(importdir, '.eggs'), ignore_errors=True) try: import_module_from_directory('develop', pathlib.Path(importdir), force_reload=True) except Exception: logger.exception('Error loading module')
def test_validate_detect_python_syntax_errors(self): test_dir = MockDir({ "badpy.json": json.dumps( dict( name="Syntax-error Python", id_name="badpy", category="Clean", parameters=[], )).encode("utf-8"), "badpy.py": b'def render(table, params):\n cols = split(","', }) with self.assertRaises(ModuleCompileError): import_module_from_directory("123456", test_dir)
def reload(): logger.info(f"Reloading...") with tempfile.TemporaryDirectory() as tmpdir: importdir = os.path.join(tmpdir, "importme") shutil.copytree(directory, importdir) shutil.rmtree(os.path.join(importdir, ".git"), ignore_errors=True) shutil.rmtree(os.path.join(importdir, ".eggs"), ignore_errors=True) shutil.rmtree(os.path.join(importdir, "node_modules"), ignore_errors=True) try: import_module_from_directory( "develop", pathlib.Path(importdir), force_reload=True ) except Exception: logger.exception("Error loading module")
def test_updates_module_version(self): test_dir = self.fake_github_clone() import_module_from_directory("https://test_url_of_test_module", "importable", "111111", test_dir) module_version_q = ModuleVersion.objects.filter(module__id_name=self.importable_id_name) self.assertEqual(module_version_q.count(), 1) # Create a test workflow that uses this imported module module_version = module_version_q.first() workflow = add_new_workflow('updates_module_version workflow') wfm = add_new_wf_module(workflow, module_version, order=1) # import "new" version (different version hash) test_dir = self.fake_github_clone() import_module_from_directory("https://test_url_of_test_module", "importable", "222222", test_dir) self.assertEqual(module_version_q.count(), 2) # should have updated the wfm to newly imported version wfm.refresh_from_db() self.assertEqual(wfm.module_version.source_version_hash, "222222")
def test_load_and_dispatch(self): try: test_dir = self.fake_github_clone() import_module_from_directory('https://github.com/account/reponame', 'reponame', '123456', test_dir) # Module and ModuleVersion should have loaded -- these will raise # exception if they don't exist module = Module.objects.get(id_name=self.importable_id_name) module_version = ModuleVersion.objects.get(module=module) # Create a test workflow that uses this imported module workflow = add_new_workflow('Dynamic Dispatch Test Workflow') wfm = add_new_wf_module(workflow, module_version, order=1) # These will fail if we haven't correctly loaded the json # describing the parameters colparam = get_param_by_id_name('test_column', wf_module=wfm) multicolparam = get_param_by_id_name('test_multicolumn', wf_module=wfm) # Does it render right? test_csv = 'Class,M,F,Other\n' \ 'math,10,12,100\n' \ 'english,,7\,200\n' \ 'history,11,13,\n' \ 'economics,20,20,20' test_table = pd.read_csv(io.StringIO(test_csv), header=0, skipinitialspace=True) test_table_out = test_table.copy() test_table_out['M'] *= 2 test_table_out[['F', 'Other']] *= 3 colparam.set_value('M') # double this multicolparam.set_value('F,Other') # triple these with self.assertLogs(): lm = LoadedModule.for_module_version_sync(module_version) result = lm.render(wfm.get_params(), test_table, None) self.assertEqual(result, ProcessResult(test_table_out)) finally: server.models.loaded_module.load_external_module.cache_clear()
def test_load_and_dispatch(self): try: test_dir = self._test_module_path("importable") with self.assertLogs(): module_version = import_module_from_directory( "123456", Path(test_dir)) # Create a test workflow that uses this imported module workflow = Workflow.objects.create() tab = workflow.tabs.create(position=0) wfm = tab.wf_modules.create( order=0, slug="step-1", module_id_name=module_version.id_name, params={ **module_version.default_params, "test_column": "M", # double this "test_multicolumn": ["F", "Other"], # triple these }, ) # Does it render right? test_table = pd.DataFrame({ "Class": ["math", "english", "history", "economics"], "M": [10, np.nan, 11, 20], "F": [12, 7, 13, 20], "Other": [100, 100, 13, 20], }) expected = pd.DataFrame({ "Class": ["math", "english", "history", "economics"], "M": [20, np.nan, 22, 40], "F": [36, 21, 39, 60], "Other": [300, 300, 39, 60], }) with self.assertLogs(): lm = LoadedModule.for_module_version_sync(module_version) result = lm.render(ProcessResult(test_table), wfm.get_params(), "x", None) self.assertEqual(result.error, "") assert_frame_equal(result.dataframe, expected) finally: server.models.loaded_module.load_external_module.cache_clear()
def test_load_and_dispatch(self): try: test_dir = self._test_module_path('importable') with self.assertLogs(): module_version = import_module_from_directory('123456', Path(test_dir)) # Create a test workflow that uses this imported module workflow = Workflow.objects.create() tab = workflow.tabs.create(position=0) wfm = tab.wf_modules.create( order=0, module_id_name=module_version.id_name, params={ **module_version.default_params, 'test_column': 'M', # double this 'test_multicolumn': ['F', 'Other'] # triple these } ) # Does it render right? test_table = pd.DataFrame({ 'Class': ['math', 'english', 'history', 'economics'], 'M': [10, np.nan, 11, 20], 'F': [12, 7, 13, 20], 'Other': [100, 100, 13, 20], }) expected = pd.DataFrame({ 'Class': ['math', 'english', 'history', 'economics'], 'M': [20, np.nan, 22, 40], 'F': [36, 21, 39, 60], 'Other': [300, 300, 39, 60], }) with self.assertLogs(): lm = LoadedModule.for_module_version_sync(module_version) result = lm.render(ProcessResult(test_table), wfm.get_params(), 'x', None) self.assertEqual(result.error, '') assert_frame_equal(result.dataframe, expected) finally: server.models.loaded_module.load_external_module.cache_clear()
def test_load_and_dispatch(self): try: test_dir = self._test_module_path('importable') with self.assertLogs(): module_version = import_module_from_directory( '123456', Path(test_dir)) # Create a test workflow that uses this imported module workflow = Workflow.objects.create() tab = workflow.tabs.create(position=0) wfm = tab.wf_modules.create(order=0, module_id_name=module_version.id_name, params=module_version.default_params) # Does it render right? test_csv = 'Class,M,F,Other\n' \ 'math,10,12,100\n' \ 'english,,7,200\n' \ 'history,11,13,\n' \ 'economics,20,20,20' test_table = pd.read_csv(io.StringIO(test_csv), header=0, skipinitialspace=True) test_table_out = test_table.copy() test_table_out['M'] *= 2 test_table_out[['F', 'Other']] *= 3 wfm.params = { **wfm.params, 'test_column': 'M', # double this 'test_multicolumn': 'F,Other' # triple these } wfm.save(update_fields=['params']) with self.assertLogs(): lm = LoadedModule.for_module_version_sync(module_version) result = lm.render(ProcessResult(test_table), wfm.get_params(), 'x', None) self.assertEqual(result, ProcessResult(test_table_out)) finally: server.models.loaded_module.load_external_module.cache_clear()
def test_validate_invalid_spec(self): test_dir = self._test_module_path("bad_json_module") with self.assertRaises(ValueError): import_module_from_directory("123456", Path(test_dir))
def test_validate_detect_python_syntax_errors(self): test_dir = self._test_module_path('bad_py_module') with self.assertRaises(ValueError): import_module_from_directory('123456', Path(test_dir))