def test_validate_missing_py(self):
dirpath = MockDir({
'module.json': b'',
})
with self.assertRaisesRegex(ValueError,
'Missing ".py" module-code file'):
ModuleFiles.load_from_dirpath(dirpath)
def test_validate_missing_json(self):
dirpath = MockDir({
'module.py': b'',
})
with self.assertRaisesRegex(
ValueError, 'Missing ".json" or ".yaml" module-spec file'):
ModuleFiles.load_from_dirpath(dirpath)
def test_validate_extra_py(self):
dirpath = MockDir({
'module.json': b'{}',
'module.py': b'',
'module2.py': b'',
})
with self.assertRaisesRegex(ValueError, 'Multiple.*py.*files'):
ModuleFiles.load_from_dirpath(dirpath)
def test_validate_max_1_js(self):
dirpath = MockDir({
'module.json': b'',
'module.py': b'',
'module.js': b'',
'extra.js': b'',
})
with self.assertRaisesRegex(ValueError, 'Multiple.*js.*files'):
ModuleFiles.load_from_dirpath(dirpath)
def test_validate_max_1_html(self):
dirpath = MockDir(
{
"module.json": b"",
"module.py": b"",
"module.html": b"",
"extra.html": b"",
}
)
with self.assertRaisesRegex(ValueError, "Multiple.*html.*files"):
ModuleFiles.load_from_dirpath(dirpath)
def test_ignore_test_py(self):
dirpath = MockDir({
'module.json': b'',
'module.py': b'',
'test_module.py': b'',
})
module_files = ModuleFiles.load_from_dirpath(dirpath)
self.assertEqual(module_files.code.name, 'module.py')
def test_happy_path(self):
dirpath = MockDir(
{"module.json": b"", "module.py": b"", "module.html": b"", "module.js": b""}
)
module_files = ModuleFiles.load_from_dirpath(dirpath)
self.assertEqual(module_files.spec.name, "module.json")
self.assertEqual(module_files.code.name, "module.py")
self.assertEqual(module_files.html.name, "module.html")
self.assertEqual(module_files.javascript.name, "module.js")
def test_ignore_package_json(self):
dirpath = MockDir({
'module.json': b'',
'module.py': b'',
'package.json': b'',
'package-lock.json': b'',
})
module_files = ModuleFiles.load_from_dirpath(dirpath)
self.assertEqual(module_files.spec.name, 'module.json')
def test_ignore_package_json(self):
dirpath = MockDir(
{
"module.json": b"",
"module.py": b"",
"package.json": b"",
"package-lock.json": b"",
}
)
module_files = ModuleFiles.load_from_dirpath(dirpath)
self.assertEqual(module_files.spec.name, "module.json")
def test_happy_path(self):
dirpath = MockDir({
'module.json': b'',
'module.py': b'',
'module.html': b'',
'module.js': b'',
})
module_files = ModuleFiles.load_from_dirpath(dirpath)
self.assertEqual(module_files.spec.name, 'module.json')
self.assertEqual(module_files.code.name, 'module.py')
self.assertEqual(module_files.html.name, 'module.html')
self.assertEqual(module_files.javascript.name, 'module.js')
def import_module_from_directory(version: str,
importdir: Path,
force_reload=False):
module_files = ModuleFiles.load_from_dirpath(importdir) # raise ValueError
spec = ModuleSpec.load_from_path(module_files.spec) # raise ValueError
validate_python_functions(module_files.code) # raise ValueError
if not force_reload:
# Don't allow importing the same version twice
try:
ModuleVersion.objects.get(id_name=spec.id_name,
source_version_hash=version)
raise ValueError(f'Version {version} of module {spec.id_name}'
' has already been imported')
except ModuleVersion.DoesNotExist:
# this is what we want
pass
if module_files.javascript:
js_module = module_files.javascript.read_text(encoding='utf-8')
else:
js_module = ''
# Copy whole directory to S3
prefix = '%s/%s/' % (spec.id_name, version)
try:
# If files already exist, delete them so we can overwrite them.
#
# This can race: a worker may be loading the code to execute it. But
# races are unlikely to affect anybody because:
#
# * If force_reload=True we're in dev or test, where we control
# everything.
# * Otherwise, we've already checked there's no ModuleVersion, so
# probably nothing is trying and load what we're deleting here.
minio.remove_recursive(minio.ExternalModulesBucket, prefix)
except FileNotFoundError:
pass # common case: we aren't overwriting code
minio.fput_directory_contents(minio.ExternalModulesBucket, prefix,
Path(importdir))
# If that succeeds, initialise module in our database
module_version = ModuleVersion.create_or_replace_from_spec(
spec, source_version_hash=version, js_module=js_module)
logger.info('Imported module %s' % spec.id_name)
return module_version
def test_ignore_test_py(self):
dirpath = MockDir({"module.json": b"", "module.py": b"", "test_module.py": b""})
module_files = ModuleFiles.load_from_dirpath(dirpath)
self.assertEqual(module_files.code.name, "module.py")
def test_validate_extra_py(self):
dirpath = MockDir({"module.json": b"{}", "module.py": b"", "module2.py": b""})
with self.assertRaisesRegex(ValueError, "Multiple.*py.*files"):
ModuleFiles.load_from_dirpath(dirpath)