def test_load_twice(self):
test_dir = self.fake_github_clone()
import_module_from_directory("https://test_url_of_test_module", "importable", "123456", test_dir)
test_dir = self.fake_github_clone() # import moves files, so get same files again
with self.assertRaises(ValidationError):
import_module_from_directory("https://test_url_of_test_module", "importable", "123456", test_dir)
def test_load_twice_fails(self):
"""loading the same version of the same module twice should fail"""
test_dir = self._test_module_path("importable")
with self.assertLogs():
import_module_from_directory("123456", Path(test_dir))
with self.assertRaises(ValueError):
import_module_from_directory("123456", Path(test_dir))
def test_load_invalid_code(self):
test_dir = self.fake_github_clone('test_data/bad_json_module')
with self.assertRaises(ValidationError):
import_module_from_directory("https://test_url_of_test_module", "bad_json_module", "123456", test_dir)
test_dir = self.fake_github_clone('test_data/bad_py_module')
with self.assertRaises(ValidationError):
import_module_from_directory("https://test_url_of_test_module", "bad_py_module", "123456", test_dir)
def test_already_imported(self):
test_dir = self.fake_github_clone()
import_module_from_directory("https://github.com/account/importable1", "importable1", "123456", test_dir)
test_dir = self.fake_github_clone() # import moves files, so get same files again
with self.assertRaises(ValidationError):
with self.assertLogs():
import_module_from_directory("https://github.com/account/importable2", "importable2", "123456", test_dir)
def test_load_and_dispatch(self):
test_dir = self.fake_github_clone()
import_module_from_directory('https://github.com/account/reponame',
'reponame', '123456', test_dir)
# Module and ModuleVersion should have loaded -- these will raise exception if they don't exist
module = Module.objects.get(id_name=self.importable_id_name)
module_version = ModuleVersion.objects.get(module=module)
# Create a test workflow that uses this imported module
workflow = add_new_workflow('Dynamic Dispatch Test Workflow')
wfm = add_new_wf_module(workflow, module_version, order=1)
# These will fail if we haven't correctly loaded the json describing
# the parameters
stringparam = get_param_by_id_name('test', wf_module=wfm)
colparam = get_param_by_id_name('test_column', wf_module=wfm)
multicolparam = get_param_by_id_name('test_multicolumn', wf_module=wfm)
# Does it render right?
test_csv = 'Class,M,F,Other\n' \
'math,10,12,100\n' \
'english,,7\,200\n' \
'history,11,13,\n' \
'economics,20,20,20'
test_table = pd.read_csv(io.StringIO(test_csv),
header=0,
skipinitialspace=True)
test_table_out = test_table.copy()
test_table_out['M'] *= 2
test_table_out[['F', 'Other']] *= 3
colparam.set_value('M') # double this
multicolparam.set_value('F,Other') # triple these
with self.assertLogs(dynamicdispatch.__name__):
result = module_dispatch_render(module_version, wfm.get_params(),
test_table, None)
self.assertEqual(result, ProcessResult(test_table_out))
# Test that bad column parameter values are removed
colparam.set_value('missing_column_name')
multicolparam.set_value('Other,junk_column_name')
test_table_out = test_table.copy()
# multicolumn parameter has only one valid col
test_table_out[['Other']] *= 3
result = module_dispatch_render(module_version, wfm.get_params(),
test_table, None)
self.assertEqual(result, ProcessResult(test_table_out))
# if the module crashes, we should get an error with a line number
stringparam.set_value('crashme')
result = module_dispatch_render(module_version, wfm.get_params(),
test_table, None)
self.assertEqual(
result,
ProcessResult(
error='ValueError: we crashed! at line 7 of importable.py'))
def test_load_twice_force_relaod(self):
test_dir = self.fake_github_clone()
import_module_from_directory("https://test_url_of_test_module", "importable", "123456", test_dir)
self.assertEqual(ModuleVersion.objects.filter(module__id_name=self.importable_id_name).count(), 1)
test_dir = self.fake_github_clone() # import moves files, so get same files again
import_module_from_directory("https://test_url_of_test_module", "importable", "123456", test_dir, force_reload=True)
# should replace existing module_version, not add a new one
self.assertEqual(ModuleVersion.objects.filter(module__id_name=self.importable_id_name).count(), 1)
def test_load_twice_force_relaod(self):
"""We will do a reload of same version if force_reload==True"""
test_dir = self._test_module_path('importable')
with self.assertLogs():
import_module_from_directory('develop', Path(test_dir))
with self.assertLogs():
import_module_from_directory('develop', Path(test_dir),
force_reload=True)
# should replace existing module_version, not add a new one
self.assertEqual(ModuleVersion.objects.count(), 1)
def reload():
print(f'Reloading {basename}')
# import_module_from_directory is unintuitive: it _destroys_ the input
# directory.
tmpdir = tempfile.mkdtemp()
shutil.rmtree(tmpdir)
shutil.copytree(directory, tmpdir)
import_module_from_directory(pretend_git_url,
basename,
'develop',
tmpdir,
force_reload=True)
print('Reloaded')
def reload():
logger.info(f'Reloading...')
with tempfile.TemporaryDirectory() as tmpdir:
importdir = os.path.join(tmpdir, 'importme')
shutil.copytree(directory, importdir)
shutil.rmtree(os.path.join(importdir, '.git'), ignore_errors=True)
shutil.rmtree(os.path.join(importdir, '.eggs'), ignore_errors=True)
try:
import_module_from_directory('develop',
pathlib.Path(importdir),
force_reload=True)
except Exception:
logger.exception('Error loading module')
def test_validate_detect_python_syntax_errors(self):
test_dir = MockDir({
"badpy.json":
json.dumps(
dict(
name="Syntax-error Python",
id_name="badpy",
category="Clean",
parameters=[],
)).encode("utf-8"),
"badpy.py":
b'def render(table, params):\n cols = split(","',
})
with self.assertRaises(ModuleCompileError):
import_module_from_directory("123456", test_dir)
def reload():
logger.info(f"Reloading...")
with tempfile.TemporaryDirectory() as tmpdir:
importdir = os.path.join(tmpdir, "importme")
shutil.copytree(directory, importdir)
shutil.rmtree(os.path.join(importdir, ".git"), ignore_errors=True)
shutil.rmtree(os.path.join(importdir, ".eggs"), ignore_errors=True)
shutil.rmtree(os.path.join(importdir, "node_modules"), ignore_errors=True)
try:
import_module_from_directory(
"develop", pathlib.Path(importdir), force_reload=True
)
except Exception:
logger.exception("Error loading module")
def test_updates_module_version(self):
test_dir = self.fake_github_clone()
import_module_from_directory("https://test_url_of_test_module", "importable", "111111", test_dir)
module_version_q = ModuleVersion.objects.filter(module__id_name=self.importable_id_name)
self.assertEqual(module_version_q.count(), 1)
# Create a test workflow that uses this imported module
module_version = module_version_q.first()
workflow = add_new_workflow('updates_module_version workflow')
wfm = add_new_wf_module(workflow, module_version, order=1)
# import "new" version (different version hash)
test_dir = self.fake_github_clone()
import_module_from_directory("https://test_url_of_test_module", "importable", "222222", test_dir)
self.assertEqual(module_version_q.count(), 2)
# should have updated the wfm to newly imported version
wfm.refresh_from_db()
self.assertEqual(wfm.module_version.source_version_hash, "222222")
def test_load_and_dispatch(self):
try:
test_dir = self.fake_github_clone()
import_module_from_directory('https://github.com/account/reponame',
'reponame', '123456', test_dir)
# Module and ModuleVersion should have loaded -- these will raise
# exception if they don't exist
module = Module.objects.get(id_name=self.importable_id_name)
module_version = ModuleVersion.objects.get(module=module)
# Create a test workflow that uses this imported module
workflow = add_new_workflow('Dynamic Dispatch Test Workflow')
wfm = add_new_wf_module(workflow, module_version, order=1)
# These will fail if we haven't correctly loaded the json
# describing the parameters
colparam = get_param_by_id_name('test_column', wf_module=wfm)
multicolparam = get_param_by_id_name('test_multicolumn',
wf_module=wfm)
# Does it render right?
test_csv = 'Class,M,F,Other\n' \
'math,10,12,100\n' \
'english,,7\,200\n' \
'history,11,13,\n' \
'economics,20,20,20'
test_table = pd.read_csv(io.StringIO(test_csv), header=0,
skipinitialspace=True)
test_table_out = test_table.copy()
test_table_out['M'] *= 2
test_table_out[['F', 'Other']] *= 3
colparam.set_value('M') # double this
multicolparam.set_value('F,Other') # triple these
with self.assertLogs():
lm = LoadedModule.for_module_version_sync(module_version)
result = lm.render(wfm.get_params(), test_table, None)
self.assertEqual(result, ProcessResult(test_table_out))
finally:
server.models.loaded_module.load_external_module.cache_clear()
def test_load_and_dispatch(self):
try:
test_dir = self._test_module_path("importable")
with self.assertLogs():
module_version = import_module_from_directory(
"123456", Path(test_dir))
# Create a test workflow that uses this imported module
workflow = Workflow.objects.create()
tab = workflow.tabs.create(position=0)
wfm = tab.wf_modules.create(
order=0,
slug="step-1",
module_id_name=module_version.id_name,
params={
**module_version.default_params,
"test_column": "M", # double this
"test_multicolumn": ["F", "Other"], # triple these
},
)
# Does it render right?
test_table = pd.DataFrame({
"Class": ["math", "english", "history", "economics"],
"M": [10, np.nan, 11, 20],
"F": [12, 7, 13, 20],
"Other": [100, 100, 13, 20],
})
expected = pd.DataFrame({
"Class": ["math", "english", "history", "economics"],
"M": [20, np.nan, 22, 40],
"F": [36, 21, 39, 60],
"Other": [300, 300, 39, 60],
})
with self.assertLogs():
lm = LoadedModule.for_module_version_sync(module_version)
result = lm.render(ProcessResult(test_table), wfm.get_params(),
"x", None)
self.assertEqual(result.error, "")
assert_frame_equal(result.dataframe, expected)
finally:
server.models.loaded_module.load_external_module.cache_clear()
def test_load_and_dispatch(self):
try:
test_dir = self._test_module_path('importable')
with self.assertLogs():
module_version = import_module_from_directory('123456',
Path(test_dir))
# Create a test workflow that uses this imported module
workflow = Workflow.objects.create()
tab = workflow.tabs.create(position=0)
wfm = tab.wf_modules.create(
order=0,
module_id_name=module_version.id_name,
params={
**module_version.default_params,
'test_column': 'M', # double this
'test_multicolumn': ['F', 'Other'] # triple these
}
)
# Does it render right?
test_table = pd.DataFrame({
'Class': ['math', 'english', 'history', 'economics'],
'M': [10, np.nan, 11, 20],
'F': [12, 7, 13, 20],
'Other': [100, 100, 13, 20],
})
expected = pd.DataFrame({
'Class': ['math', 'english', 'history', 'economics'],
'M': [20, np.nan, 22, 40],
'F': [36, 21, 39, 60],
'Other': [300, 300, 39, 60],
})
with self.assertLogs():
lm = LoadedModule.for_module_version_sync(module_version)
result = lm.render(ProcessResult(test_table), wfm.get_params(),
'x', None)
self.assertEqual(result.error, '')
assert_frame_equal(result.dataframe, expected)
finally:
server.models.loaded_module.load_external_module.cache_clear()
def test_load_and_dispatch(self):
try:
test_dir = self._test_module_path('importable')
with self.assertLogs():
module_version = import_module_from_directory(
'123456', Path(test_dir))
# Create a test workflow that uses this imported module
workflow = Workflow.objects.create()
tab = workflow.tabs.create(position=0)
wfm = tab.wf_modules.create(order=0,
module_id_name=module_version.id_name,
params=module_version.default_params)
# Does it render right?
test_csv = 'Class,M,F,Other\n' \
'math,10,12,100\n' \
'english,,7,200\n' \
'history,11,13,\n' \
'economics,20,20,20'
test_table = pd.read_csv(io.StringIO(test_csv),
header=0,
skipinitialspace=True)
test_table_out = test_table.copy()
test_table_out['M'] *= 2
test_table_out[['F', 'Other']] *= 3
wfm.params = {
**wfm.params,
'test_column': 'M', # double this
'test_multicolumn': 'F,Other' # triple these
}
wfm.save(update_fields=['params'])
with self.assertLogs():
lm = LoadedModule.for_module_version_sync(module_version)
result = lm.render(ProcessResult(test_table), wfm.get_params(),
'x', None)
self.assertEqual(result, ProcessResult(test_table_out))
finally:
server.models.loaded_module.load_external_module.cache_clear()
def test_validate_invalid_spec(self):
test_dir = self._test_module_path("bad_json_module")
with self.assertRaises(ValueError):
import_module_from_directory("123456", Path(test_dir))
def test_validate_detect_python_syntax_errors(self):
test_dir = self._test_module_path('bad_py_module')
with self.assertRaises(ValueError):
import_module_from_directory('123456', Path(test_dir))
