def run_test(self, source):
with util.create_modules(self.module_name) as mapping:
with open(mapping[self.module_name], 'wb') as file:
file.write(source)
loader = self.machinery.SourceFileLoader(self.module_name,
mapping[self.module_name])
return self.load(loader)
def test_checked_hash_based_pyc(self):
with util.create_modules('_temp') as mapping:
source = mapping['_temp']
pyc = self.util.cache_from_source(source)
with open(source, 'wb') as fp:
fp.write(b'state = "old"')
os.utime(source, (50, 50))
py_compile.compile(
source,
invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
)
loader = self.machinery.SourceFileLoader('_temp', source)
mod = types.ModuleType('_temp')
mod.__spec__ = self.util.spec_from_loader('_temp', loader)
loader.exec_module(mod)
self.assertEqual(mod.state, 'old')
# Write a new source with the same mtime and size as before.
with open(source, 'wb') as fp:
fp.write(b'state = "new"')
os.utime(source, (50, 50))
loader.exec_module(mod)
self.assertEqual(mod.state, 'new')
with open(pyc, 'rb') as fp:
data = fp.read()
self.assertEqual(int.from_bytes(data[4:8], 'little'), 0b11)
self.assertEqual(
self.util.source_hash(b'state = "new"'),
data[8:16],
)
def test_timestamp_overflow(self):
# When a modification timestamp is larger than 2**32, it should be
# truncated rather than raise an OverflowError.
with util.create_modules('_temp') as mapping:
source = mapping['_temp']
compiled = self.util.cache_from_source(source)
with open(source, 'w', encoding='utf-8') as f:
f.write("x = 5")
try:
os.utime(source, (2**33 - 5, 2**33 - 5))
except OverflowError:
self.skipTest("cannot set modification time to large integer")
except OSError as e:
if e.errno != getattr(errno, 'EOVERFLOW', None):
raise
self.skipTest(
"cannot set modification time to large integer ({})".
format(e))
loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
# PEP 451
module = types.ModuleType('_temp')
module.__spec__ = self.util.spec_from_loader('_temp', loader)
loader.exec_module(module)
self.assertEqual(module.x, 5)
self.assertTrue(os.path.exists(compiled))
os.unlink(compiled)
# PEP 302
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
mod = loader.load_module('_temp')
# Sanity checks.
self.assertEqual(mod.__cached__, compiled)
self.assertEqual(mod.x, 5)
# The pyc file was created.
self.assertTrue(os.path.exists(compiled))
def _test_partial_size(self, test, *, del_source=False):
with util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode('_temp',
mapping,
lambda bc: bc[:15],
del_source=del_source)
test('_temp', mapping, bc_path)
def test_overridden_unchecked_hash_based_pyc(self):
with util.create_modules('_temp') as mapping, \
unittest.mock.patch('_imp.check_hash_based_pycs', 'always'):
source = mapping['_temp']
pyc = self.util.cache_from_source(source)
with open(source, 'wb') as fp:
fp.write(b'state = "old"')
os.utime(source, (50, 50))
py_compile.compile(
source,
invalidation_mode=py_compile.PycInvalidationMode.
UNCHECKED_HASH,
)
loader = self.machinery.SourceFileLoader('_temp', source)
mod = types.ModuleType('_temp')
mod.__spec__ = self.util.spec_from_loader('_temp', loader)
loader.exec_module(mod)
self.assertEqual(mod.state, 'old')
# Update the source file, which should be ignored.
with open(source, 'wb') as fp:
fp.write(b'state = "new"')
loader.exec_module(mod)
self.assertEqual(mod.state, 'new')
with open(pyc, 'rb') as fp:
data = fp.read()
self.assertEqual(int.from_bytes(data[4:8], 'little'), 0b1)
self.assertEqual(
self.util.source_hash(b'state = "new"'),
data[8:16],
)
def run_test(self, test, create=None, *, compile_=None, unlink=None):
"""Test the finding of 'test' with the creation of modules listed in
'create'.
Any names listed in 'compile_' are byte-compiled. Modules
listed in 'unlink' have their source files deleted.
"""
if create is None:
create = {test}
with util.create_modules(*create) as mapping:
if compile_:
for name in compile_:
py_compile.compile(mapping[name])
if unlink:
for name in unlink:
os.unlink(mapping[name])
try:
make_legacy_pyc(mapping[name])
except OSError as error:
# Some tests do not set compile_=True so the source
# module will not get compiled and there will be no
# PEP 3147 pyc file to rename.
if error.errno != errno.ENOENT:
raise
loader = self.import_(mapping['.root'], test)
self.assertTrue(hasattr(loader, 'load_module'))
return loader
def test_dir_removal_handling(self):
mod = 'mod'
with util.create_modules(mod) as mapping:
finder = self.get_finder(mapping['.root'])
found = self._find(finder, 'mod', loader_only=True)
self.assertIsNotNone(found)
found = self._find(finder, 'mod', loader_only=True)
self.assertIsNone(found)
def _test_partial_magic(self, test, *, del_source=False):
# When their are less than 4 bytes to a .pyc, regenerate it if
# possible, else raise ImportError.
with util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode('_temp',
mapping,
lambda bc: bc[:3],
del_source=del_source)
test('_temp', mapping, bc_path)
def _test_no_marshal(self, *, del_source=False):
with util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode('_temp',
mapping,
lambda bc: bc[:16],
del_source=del_source)
file_path = mapping['_temp'] if not del_source else bc_path
with self.assertRaises(EOFError):
self.import_(file_path, '_temp')
def test_bad_syntax(self):
with util.create_modules('_temp') as mapping:
with open(mapping['_temp'], 'w', encoding='utf-8') as file:
file.write('=')
loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
with self.assertRaises(SyntaxError):
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
loader.load_module('_temp')
self.assertNotIn('_temp', sys.modules)
def run_test(self, line_ending):
module_name = '_temp'
source_lines = [b"a = 42", b"b = -13", b'']
source = line_ending.join(source_lines)
with util.create_modules(module_name) as mapping:
with open(mapping[module_name], 'wb') as file:
file.write(source)
loader = self.machinery.SourceFileLoader(module_name,
mapping[module_name])
return self.load(loader, module_name)
def _test_partial_hash(self, test, *, del_source=False):
with util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode(
'_temp',
mapping,
lambda bc: bc[:13],
del_source=del_source,
invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
)
test('_temp', mapping, bc_path)
with util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode(
'_temp',
mapping,
lambda bc: bc[:13],
del_source=del_source,
invalidation_mode=py_compile.PycInvalidationMode.
UNCHECKED_HASH,
)
test('_temp', mapping, bc_path)
def sensitivity_test(self):
"""Look for a module with matching and non-matching sensitivity."""
sensitive_pkg = 'sensitive.{0}'.format(self.name)
insensitive_pkg = 'insensitive.{0}'.format(self.name.lower())
context = util.create_modules(insensitive_pkg, sensitive_pkg)
with context as mapping:
sensitive_path = os.path.join(mapping['.root'], 'sensitive')
insensitive_path = os.path.join(mapping['.root'], 'insensitive')
sensitive_finder = self.finder(sensitive_path)
insensitive_finder = self.finder(insensitive_path)
return self.find(sensitive_finder), self.find(insensitive_finder)
def source_using_bytecode(seconds, repeat):
"""Source w/ bytecode: small"""
name = '__importlib_test_benchmark__'
with util.create_modules(name) as mapping:
sys.meta_path.append(importlib.machinery.PathFinder)
loader = (importlib.machinery.SourceFileLoader,
importlib.machinery.SOURCE_SUFFIXES)
sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader))
py_compile.compile(mapping[name])
assert os.path.exists(imp.cache_from_source(mapping[name]))
yield from bench(name, lambda: sys.modules.pop(name), repeat=repeat,
seconds=seconds)
def _test_non_code_marshal(self, *, del_source=False):
with util.create_modules('_temp') as mapping:
bytecode_path = self.manipulate_bytecode(
'_temp',
mapping,
lambda bc: bc[:16] + marshal.dumps(b'abcd'),
del_source=del_source)
file_path = mapping['_temp'] if not del_source else bytecode_path
with self.assertRaises(ImportError) as cm:
self.import_(file_path, '_temp')
self.assertEqual(cm.exception.name, '_temp')
self.assertEqual(cm.exception.path, bytecode_path)
def source_using_bytecode(seconds, repeat):
"""Source w/ bytecode: small"""
name = '__importlib_test_benchmark__'
with util.create_modules(name) as mapping:
sys.meta_path.append(importlib.machinery.PathFinder)
loader = (importlib.machinery.SourceFileLoader,
importlib.machinery.SOURCE_SUFFIXES)
sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader))
py_compile.compile(mapping[name])
assert os.path.exists(imp.cache_from_source(mapping[name]))
yield from bench(name,
lambda: sys.modules.pop(name),
repeat=repeat,
seconds=seconds)
def test_module(self):
with util.create_modules('_temp') as mapping:
loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
module = loader.load_module('_temp')
self.assertIn('_temp', sys.modules)
check = {
'__name__': '_temp',
'__file__': mapping['_temp'],
'__package__': ''
}
for attr, value in check.items():
self.assertEqual(getattr(module, attr), value)
def source_writing_bytecode(seconds, repeat):
"""Source writing bytecode: small"""
assert not sys.dont_write_bytecode
name = '__importlib_test_benchmark__'
with util.create_modules(name) as mapping:
sys.meta_path.append(importlib.machinery.PathFinder)
loader = (importlib.machinery.SourceFileLoader,
importlib.machinery.SOURCE_SUFFIXES)
sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader))
def cleanup():
sys.modules.pop(name)
os.unlink(imp.cache_from_source(mapping[name]))
for result in bench(name, cleanup, repeat=repeat, seconds=seconds):
assert not os.path.exists(imp.cache_from_source(mapping[name]))
yield result
def test_old_timestamp(self):
# When the timestamp is older than the source, bytecode should be
# regenerated.
zeros = b'\x00\x00\x00\x00'
with util.create_modules('_temp') as mapping:
py_compile.compile(mapping['_temp'])
bytecode_path = self.util.cache_from_source(mapping['_temp'])
with open(bytecode_path, 'r+b') as bytecode_file:
bytecode_file.seek(8)
bytecode_file.write(zeros)
self.import_(mapping['_temp'], '_temp')
source_mtime = os.path.getmtime(mapping['_temp'])
source_timestamp = self.importlib._pack_uint32(source_mtime)
with open(bytecode_path, 'rb') as bytecode_file:
bytecode_file.seek(8)
self.assertEqual(bytecode_file.read(4), source_timestamp)
def source_wo_bytecode(seconds, repeat):
"""Source w/o bytecode: small"""
sys.dont_write_bytecode = True
try:
name = '__importlib_test_benchmark__'
# Clears out sys.modules and puts an entry at the front of sys.path.
with util.create_modules(name) as mapping:
assert not os.path.exists(imp.cache_from_source(mapping[name]))
sys.meta_path.append(importlib.machinery.PathFinder)
loader = (importlib.machinery.SourceFileLoader,
importlib.machinery.SOURCE_SUFFIXES)
sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader))
yield from bench(name, lambda: sys.modules.pop(name), repeat=repeat,
seconds=seconds)
finally:
sys.dont_write_bytecode = False
def test_read_only_bytecode(self):
# When bytecode is read-only but should be rewritten, fail silently.
with util.create_modules('_temp') as mapping:
# Create bytecode that will need to be re-created.
py_compile.compile(mapping['_temp'])
bytecode_path = self.util.cache_from_source(mapping['_temp'])
with open(bytecode_path, 'r+b') as bytecode_file:
bytecode_file.seek(0)
bytecode_file.write(b'\x00\x00\x00\x00')
# Make the bytecode read-only.
os.chmod(bytecode_path, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
try:
# Should not raise OSError!
self.import_(mapping['_temp'], '_temp')
finally:
# Make writable for eventual clean-up.
os.chmod(bytecode_path, stat.S_IWUSR)
def source_writing_bytecode(seconds, repeat):
"""Source writing bytecode: small"""
assert not sys.dont_write_bytecode
name = '__importlib_test_benchmark__'
with util.create_modules(name) as mapping:
sys.meta_path.append(importlib.machinery.PathFinder)
loader = (importlib.machinery.SourceFileLoader,
importlib.machinery.SOURCE_SUFFIXES)
sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader))
def cleanup():
sys.modules.pop(name)
os.unlink(imp.cache_from_source(mapping[name]))
for result in bench(name, cleanup, repeat=repeat, seconds=seconds):
assert not os.path.exists(imp.cache_from_source(mapping[name]))
yield result
def source_wo_bytecode(seconds, repeat):
"""Source w/o bytecode: small"""
sys.dont_write_bytecode = True
try:
name = '__importlib_test_benchmark__'
# Clears out sys.modules and puts an entry at the front of sys.path.
with util.create_modules(name) as mapping:
assert not os.path.exists(imp.cache_from_source(mapping[name]))
sys.meta_path.append(importlib.machinery.PathFinder)
loader = (importlib.machinery.SourceFileLoader,
importlib.machinery.SOURCE_SUFFIXES)
sys.path_hooks.append(
importlib.machinery.FileFinder.path_hook(loader))
yield from bench(name,
lambda: sys.modules.pop(name),
repeat=repeat,
seconds=seconds)
finally:
sys.dont_write_bytecode = False
def test_module_reuse(self):
with util.create_modules('_temp') as mapping:
loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
module = loader.load_module('_temp')
module_id = id(module)
module_dict_id = id(module.__dict__)
with open(mapping['_temp'], 'w', encoding='utf-8') as file:
file.write("testing_var = 42\n")
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
module = loader.load_module('_temp')
self.assertIn(
'testing_var', module.__dict__, "'testing_var' not in "
"{0}".format(list(module.__dict__.keys())))
self.assertEqual(module, sys.modules['_temp'])
self.assertEqual(id(module), module_id)
self.assertEqual(id(module.__dict__), module_dict_id)
def test_state_after_failure(self):
# A failed reload should leave the original module intact.
attributes = ('__file__', '__path__', '__package__')
value = '<test>'
name = '_temp'
with util.create_modules(name) as mapping:
orig_module = types.ModuleType(name)
for attr in attributes:
setattr(orig_module, attr, value)
with open(mapping[name], 'w', encoding='utf-8') as file:
file.write('+++ bad syntax +++')
loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
with self.assertRaises(SyntaxError):
loader.exec_module(orig_module)
for attr in attributes:
self.assertEqual(getattr(orig_module, attr), value)
with self.assertRaises(SyntaxError):
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
loader.load_module(name)
for attr in attributes:
self.assertEqual(getattr(orig_module, attr), value)
def test_overridden_checked_hash_based_pyc(self):
with util.create_modules('_temp') as mapping, \
unittest.mock.patch('_imp.check_hash_based_pycs', 'never'):
source = mapping['_temp']
pyc = self.util.cache_from_source(source)
with open(source, 'wb') as fp:
fp.write(b'state = "old"')
os.utime(source, (50, 50))
py_compile.compile(
source,
invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
)
loader = self.machinery.SourceFileLoader('_temp', source)
mod = types.ModuleType('_temp')
mod.__spec__ = self.util.spec_from_loader('_temp', loader)
loader.exec_module(mod)
self.assertEqual(mod.state, 'old')
# Write a new source with the same mtime and size as before.
with open(source, 'wb') as fp:
fp.write(b'state = "new"')
os.utime(source, (50, 50))
loader.exec_module(mod)
self.assertEqual(mod.state, 'old')
def _test_bad_magic(self, test, *, del_source=False):
with util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode(
'_temp', mapping, lambda bc: b'\x00\x00\x00\x00' + bc[4:])
test('_temp', mapping, bc_path)
def test_success_legacy(self):
with util.create_modules('dummy') as mapping:
self.assertTrue(
hasattr(self.path_hook()(mapping['.root']), 'find_module'))
def test_failure(self):
with util.create_modules('blah') as mapping:
nothing = self.import_(mapping['.root'], 'sdfsadsadf')
self.assertIsNone(nothing)
def test_module_in_package(self):
with util.create_modules('pkg.__init__', 'pkg.sub') as mapping:
pkg_dir = os.path.dirname(mapping['pkg.__init__'])
loader = self.import_(pkg_dir, 'pkg.sub')
self.assertTrue(hasattr(loader, 'load_module'))