def run_test(self, source): with util.create_modules(self.module_name) as mapping: with open(mapping[self.module_name], 'wb') as file: file.write(source) loader = self.machinery.SourceFileLoader(self.module_name, mapping[self.module_name]) return self.load(loader)
def test_checked_hash_based_pyc(self): with util.create_modules('_temp') as mapping: source = mapping['_temp'] pyc = self.util.cache_from_source(source) with open(source, 'wb') as fp: fp.write(b'state = "old"') os.utime(source, (50, 50)) py_compile.compile( source, invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH, ) loader = self.machinery.SourceFileLoader('_temp', source) mod = types.ModuleType('_temp') mod.__spec__ = self.util.spec_from_loader('_temp', loader) loader.exec_module(mod) self.assertEqual(mod.state, 'old') # Write a new source with the same mtime and size as before. with open(source, 'wb') as fp: fp.write(b'state = "new"') os.utime(source, (50, 50)) loader.exec_module(mod) self.assertEqual(mod.state, 'new') with open(pyc, 'rb') as fp: data = fp.read() self.assertEqual(int.from_bytes(data[4:8], 'little'), 0b11) self.assertEqual( self.util.source_hash(b'state = "new"'), data[8:16], )
def test_timestamp_overflow(self): # When a modification timestamp is larger than 2**32, it should be # truncated rather than raise an OverflowError. with util.create_modules('_temp') as mapping: source = mapping['_temp'] compiled = self.util.cache_from_source(source) with open(source, 'w', encoding='utf-8') as f: f.write("x = 5") try: os.utime(source, (2**33 - 5, 2**33 - 5)) except OverflowError: self.skipTest("cannot set modification time to large integer") except OSError as e: if e.errno != getattr(errno, 'EOVERFLOW', None): raise self.skipTest( "cannot set modification time to large integer ({})". format(e)) loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) # PEP 451 module = types.ModuleType('_temp') module.__spec__ = self.util.spec_from_loader('_temp', loader) loader.exec_module(module) self.assertEqual(module.x, 5) self.assertTrue(os.path.exists(compiled)) os.unlink(compiled) # PEP 302 with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) mod = loader.load_module('_temp') # Sanity checks. self.assertEqual(mod.__cached__, compiled) self.assertEqual(mod.x, 5) # The pyc file was created. self.assertTrue(os.path.exists(compiled))
def _test_partial_size(self, test, *, del_source=False): with util.create_modules('_temp') as mapping: bc_path = self.manipulate_bytecode('_temp', mapping, lambda bc: bc[:15], del_source=del_source) test('_temp', mapping, bc_path)
def test_overridden_unchecked_hash_based_pyc(self): with util.create_modules('_temp') as mapping, \ unittest.mock.patch('_imp.check_hash_based_pycs', 'always'): source = mapping['_temp'] pyc = self.util.cache_from_source(source) with open(source, 'wb') as fp: fp.write(b'state = "old"') os.utime(source, (50, 50)) py_compile.compile( source, invalidation_mode=py_compile.PycInvalidationMode. UNCHECKED_HASH, ) loader = self.machinery.SourceFileLoader('_temp', source) mod = types.ModuleType('_temp') mod.__spec__ = self.util.spec_from_loader('_temp', loader) loader.exec_module(mod) self.assertEqual(mod.state, 'old') # Update the source file, which should be ignored. with open(source, 'wb') as fp: fp.write(b'state = "new"') loader.exec_module(mod) self.assertEqual(mod.state, 'new') with open(pyc, 'rb') as fp: data = fp.read() self.assertEqual(int.from_bytes(data[4:8], 'little'), 0b1) self.assertEqual( self.util.source_hash(b'state = "new"'), data[8:16], )
def run_test(self, test, create=None, *, compile_=None, unlink=None): """Test the finding of 'test' with the creation of modules listed in 'create'. Any names listed in 'compile_' are byte-compiled. Modules listed in 'unlink' have their source files deleted. """ if create is None: create = {test} with util.create_modules(*create) as mapping: if compile_: for name in compile_: py_compile.compile(mapping[name]) if unlink: for name in unlink: os.unlink(mapping[name]) try: make_legacy_pyc(mapping[name]) except OSError as error: # Some tests do not set compile_=True so the source # module will not get compiled and there will be no # PEP 3147 pyc file to rename. if error.errno != errno.ENOENT: raise loader = self.import_(mapping['.root'], test) self.assertTrue(hasattr(loader, 'load_module')) return loader
def test_dir_removal_handling(self): mod = 'mod' with util.create_modules(mod) as mapping: finder = self.get_finder(mapping['.root']) found = self._find(finder, 'mod', loader_only=True) self.assertIsNotNone(found) found = self._find(finder, 'mod', loader_only=True) self.assertIsNone(found)
def _test_partial_magic(self, test, *, del_source=False): # When their are less than 4 bytes to a .pyc, regenerate it if # possible, else raise ImportError. with util.create_modules('_temp') as mapping: bc_path = self.manipulate_bytecode('_temp', mapping, lambda bc: bc[:3], del_source=del_source) test('_temp', mapping, bc_path)
def _test_no_marshal(self, *, del_source=False): with util.create_modules('_temp') as mapping: bc_path = self.manipulate_bytecode('_temp', mapping, lambda bc: bc[:16], del_source=del_source) file_path = mapping['_temp'] if not del_source else bc_path with self.assertRaises(EOFError): self.import_(file_path, '_temp')
def test_bad_syntax(self): with util.create_modules('_temp') as mapping: with open(mapping['_temp'], 'w', encoding='utf-8') as file: file.write('=') loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) with self.assertRaises(SyntaxError): with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) loader.load_module('_temp') self.assertNotIn('_temp', sys.modules)
def run_test(self, line_ending): module_name = '_temp' source_lines = [b"a = 42", b"b = -13", b''] source = line_ending.join(source_lines) with util.create_modules(module_name) as mapping: with open(mapping[module_name], 'wb') as file: file.write(source) loader = self.machinery.SourceFileLoader(module_name, mapping[module_name]) return self.load(loader, module_name)
def _test_partial_hash(self, test, *, del_source=False): with util.create_modules('_temp') as mapping: bc_path = self.manipulate_bytecode( '_temp', mapping, lambda bc: bc[:13], del_source=del_source, invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH, ) test('_temp', mapping, bc_path) with util.create_modules('_temp') as mapping: bc_path = self.manipulate_bytecode( '_temp', mapping, lambda bc: bc[:13], del_source=del_source, invalidation_mode=py_compile.PycInvalidationMode. UNCHECKED_HASH, ) test('_temp', mapping, bc_path)
def sensitivity_test(self): """Look for a module with matching and non-matching sensitivity.""" sensitive_pkg = 'sensitive.{0}'.format(self.name) insensitive_pkg = 'insensitive.{0}'.format(self.name.lower()) context = util.create_modules(insensitive_pkg, sensitive_pkg) with context as mapping: sensitive_path = os.path.join(mapping['.root'], 'sensitive') insensitive_path = os.path.join(mapping['.root'], 'insensitive') sensitive_finder = self.finder(sensitive_path) insensitive_finder = self.finder(insensitive_path) return self.find(sensitive_finder), self.find(insensitive_finder)
def source_using_bytecode(seconds, repeat): """Source w/ bytecode: small""" name = '__importlib_test_benchmark__' with util.create_modules(name) as mapping: sys.meta_path.append(importlib.machinery.PathFinder) loader = (importlib.machinery.SourceFileLoader, importlib.machinery.SOURCE_SUFFIXES) sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader)) py_compile.compile(mapping[name]) assert os.path.exists(imp.cache_from_source(mapping[name])) yield from bench(name, lambda: sys.modules.pop(name), repeat=repeat, seconds=seconds)
def _test_non_code_marshal(self, *, del_source=False): with util.create_modules('_temp') as mapping: bytecode_path = self.manipulate_bytecode( '_temp', mapping, lambda bc: bc[:16] + marshal.dumps(b'abcd'), del_source=del_source) file_path = mapping['_temp'] if not del_source else bytecode_path with self.assertRaises(ImportError) as cm: self.import_(file_path, '_temp') self.assertEqual(cm.exception.name, '_temp') self.assertEqual(cm.exception.path, bytecode_path)
def test_module(self): with util.create_modules('_temp') as mapping: loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) module = loader.load_module('_temp') self.assertIn('_temp', sys.modules) check = { '__name__': '_temp', '__file__': mapping['_temp'], '__package__': '' } for attr, value in check.items(): self.assertEqual(getattr(module, attr), value)
def source_writing_bytecode(seconds, repeat): """Source writing bytecode: small""" assert not sys.dont_write_bytecode name = '__importlib_test_benchmark__' with util.create_modules(name) as mapping: sys.meta_path.append(importlib.machinery.PathFinder) loader = (importlib.machinery.SourceFileLoader, importlib.machinery.SOURCE_SUFFIXES) sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader)) def cleanup(): sys.modules.pop(name) os.unlink(imp.cache_from_source(mapping[name])) for result in bench(name, cleanup, repeat=repeat, seconds=seconds): assert not os.path.exists(imp.cache_from_source(mapping[name])) yield result
def test_old_timestamp(self): # When the timestamp is older than the source, bytecode should be # regenerated. zeros = b'\x00\x00\x00\x00' with util.create_modules('_temp') as mapping: py_compile.compile(mapping['_temp']) bytecode_path = self.util.cache_from_source(mapping['_temp']) with open(bytecode_path, 'r+b') as bytecode_file: bytecode_file.seek(8) bytecode_file.write(zeros) self.import_(mapping['_temp'], '_temp') source_mtime = os.path.getmtime(mapping['_temp']) source_timestamp = self.importlib._pack_uint32(source_mtime) with open(bytecode_path, 'rb') as bytecode_file: bytecode_file.seek(8) self.assertEqual(bytecode_file.read(4), source_timestamp)
def source_wo_bytecode(seconds, repeat): """Source w/o bytecode: small""" sys.dont_write_bytecode = True try: name = '__importlib_test_benchmark__' # Clears out sys.modules and puts an entry at the front of sys.path. with util.create_modules(name) as mapping: assert not os.path.exists(imp.cache_from_source(mapping[name])) sys.meta_path.append(importlib.machinery.PathFinder) loader = (importlib.machinery.SourceFileLoader, importlib.machinery.SOURCE_SUFFIXES) sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader)) yield from bench(name, lambda: sys.modules.pop(name), repeat=repeat, seconds=seconds) finally: sys.dont_write_bytecode = False
def test_read_only_bytecode(self): # When bytecode is read-only but should be rewritten, fail silently. with util.create_modules('_temp') as mapping: # Create bytecode that will need to be re-created. py_compile.compile(mapping['_temp']) bytecode_path = self.util.cache_from_source(mapping['_temp']) with open(bytecode_path, 'r+b') as bytecode_file: bytecode_file.seek(0) bytecode_file.write(b'\x00\x00\x00\x00') # Make the bytecode read-only. os.chmod(bytecode_path, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) try: # Should not raise OSError! self.import_(mapping['_temp'], '_temp') finally: # Make writable for eventual clean-up. os.chmod(bytecode_path, stat.S_IWUSR)
def source_wo_bytecode(seconds, repeat): """Source w/o bytecode: small""" sys.dont_write_bytecode = True try: name = '__importlib_test_benchmark__' # Clears out sys.modules and puts an entry at the front of sys.path. with util.create_modules(name) as mapping: assert not os.path.exists(imp.cache_from_source(mapping[name])) sys.meta_path.append(importlib.machinery.PathFinder) loader = (importlib.machinery.SourceFileLoader, importlib.machinery.SOURCE_SUFFIXES) sys.path_hooks.append( importlib.machinery.FileFinder.path_hook(loader)) yield from bench(name, lambda: sys.modules.pop(name), repeat=repeat, seconds=seconds) finally: sys.dont_write_bytecode = False
def test_module_reuse(self): with util.create_modules('_temp') as mapping: loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) module = loader.load_module('_temp') module_id = id(module) module_dict_id = id(module.__dict__) with open(mapping['_temp'], 'w', encoding='utf-8') as file: file.write("testing_var = 42\n") with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) module = loader.load_module('_temp') self.assertIn( 'testing_var', module.__dict__, "'testing_var' not in " "{0}".format(list(module.__dict__.keys()))) self.assertEqual(module, sys.modules['_temp']) self.assertEqual(id(module), module_id) self.assertEqual(id(module.__dict__), module_dict_id)
def test_state_after_failure(self): # A failed reload should leave the original module intact. attributes = ('__file__', '__path__', '__package__') value = '<test>' name = '_temp' with util.create_modules(name) as mapping: orig_module = types.ModuleType(name) for attr in attributes: setattr(orig_module, attr, value) with open(mapping[name], 'w', encoding='utf-8') as file: file.write('+++ bad syntax +++') loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) with self.assertRaises(SyntaxError): loader.exec_module(orig_module) for attr in attributes: self.assertEqual(getattr(orig_module, attr), value) with self.assertRaises(SyntaxError): with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) loader.load_module(name) for attr in attributes: self.assertEqual(getattr(orig_module, attr), value)
def test_overridden_checked_hash_based_pyc(self): with util.create_modules('_temp') as mapping, \ unittest.mock.patch('_imp.check_hash_based_pycs', 'never'): source = mapping['_temp'] pyc = self.util.cache_from_source(source) with open(source, 'wb') as fp: fp.write(b'state = "old"') os.utime(source, (50, 50)) py_compile.compile( source, invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH, ) loader = self.machinery.SourceFileLoader('_temp', source) mod = types.ModuleType('_temp') mod.__spec__ = self.util.spec_from_loader('_temp', loader) loader.exec_module(mod) self.assertEqual(mod.state, 'old') # Write a new source with the same mtime and size as before. with open(source, 'wb') as fp: fp.write(b'state = "new"') os.utime(source, (50, 50)) loader.exec_module(mod) self.assertEqual(mod.state, 'old')
def _test_bad_magic(self, test, *, del_source=False): with util.create_modules('_temp') as mapping: bc_path = self.manipulate_bytecode( '_temp', mapping, lambda bc: b'\x00\x00\x00\x00' + bc[4:]) test('_temp', mapping, bc_path)
def test_success_legacy(self): with util.create_modules('dummy') as mapping: self.assertTrue( hasattr(self.path_hook()(mapping['.root']), 'find_module'))
def test_failure(self): with util.create_modules('blah') as mapping: nothing = self.import_(mapping['.root'], 'sdfsadsadf') self.assertIsNone(nothing)
def test_module_in_package(self): with util.create_modules('pkg.__init__', 'pkg.sub') as mapping: pkg_dir = os.path.dirname(mapping['pkg.__init__']) loader = self.import_(pkg_dir, 'pkg.sub') self.assertTrue(hasattr(loader, 'load_module'))