def check_object_path(path):
if not is_bytecode_extension(path) and is_python_source(path):
try:
import importlib
return importlib.util.cache_from_source(path, optimization='')
bytecode_path = importlib.util.cache_from_source(path,
optimization="")
if osp.exists(bytecode_path):
return bytecode_path
except:
try:
import imp
imp.cache_from_source(path, debug_override=False)
except:
pass
pass
basename = osp.basename(path)[0:-3]
if PYTHON3:
spath = path
else:
spath = path.decode("utf-8")
path = tempfile.mkstemp(prefix=basename + "-",
suffix=".pyc",
text=False)[1]
py_compile.compile(spath, cfile=path, doraise=True)
if not is_bytecode_extension(path):
raise ValueError(
"path %s must point to a Python source that can be compiled, or Python bytecode (.pyc, .pyo)\n"
% path)
return path
def check_object_path(path):
if path.endswith(".py"):
try:
import importlib
return importlib.util.cache_from_source(path,
optimization='')
except:
try:
import imp
imp.cache_from_source(path, debug_override=False)
except:
pass
pass
basename = os.path.basename(path)[0:-3]
if PYTHON3:
spath = path
else:
spath = path.decode('utf-8')
path = tempfile.mkstemp(prefix=basename + '-',
suffix='.pyc', text=False)[1]
py_compile.compile(spath, cfile=path, doraise=True)
if not path.endswith(".pyc") and not path.endswith(".pyo"):
raise ValueError("path %s must point to a .py or .pyc file\n" %
path)
return path
def test_add_patch_to_sitecustomize():
"""
Test adding monkey patch snippet to sitecustomize.py (using TestPipEnvironment)
"""
env = reset_env(sitecustomize=patch_urlopen)
if uses_pycache:
# caught py32 with an outdated __pycache__ file after a sitecustomize update (after python should have updated it)
# https://github.com/pypa/pip/pull/893#issuecomment-16426701
# now we delete the cache file to be sure in TestPipEnvironment._add_to_sitecustomize
# it should not exist after creating the env
cache_path = imp.cache_from_source(env.lib_path / 'sitecustomize.py')
assert not os.path.isfile(cache_path)
debug_content = open(env.lib_path / 'sitecustomize.py').read()
result = env.run('python', '-c',
"import os; print(os.path.isdir.__module__)")
if uses_pycache:
# if this next assert fails, let's have the modified time to look at
cache_path = imp.cache_from_source(env.lib_path / 'sitecustomize.py')
src_mtime = os.stat(env.lib_path / 'sitecustomize.py').st_mtime
cache_mtime = os.stat(cache_path).st_mtime
debug_content += "src mtime: %s, cache mtime: %s" % (src_mtime,
cache_mtime)
assert "sitecustomize" == result.stdout.strip(), result.stdout
def test_package___cached___from_pyc(self):
# Like test___cached__ but ensuring __cached__ when imported from a
# PEP 3147 pyc file.
def cleanup():
rmtree('pep3147')
os.mkdir('pep3147')
self.addCleanup(cleanup)
unload('pep3147.foo')
unload('pep3147')
# Touch the __init__.py
with open(os.path.join('pep3147', '__init__.py'), 'w'):
pass
with open(os.path.join('pep3147', 'foo.py'), 'w'):
pass
importlib.invalidate_caches()
m = __import__('pep3147.foo')
unload('pep3147.foo')
unload('pep3147')
importlib.invalidate_caches()
m = __import__('pep3147.foo')
init_pyc = imp.cache_from_source(
os.path.join('pep3147', '__init__.py'))
self.assertEqual(m.__cached__, os.path.join(os.curdir, init_pyc))
foo_pyc = imp.cache_from_source(os.path.join('pep3147', 'foo.py'))
self.assertEqual(sys.modules['pep3147.foo'].__cached__,
os.path.join(os.curdir, foo_pyc))
def test_package___cached___from_pyc(self):
# Like test___cached__ but ensuring __cached__ when imported from a
# PEP 3147 pyc file.
def cleanup():
rmtree("pep3147")
unload("pep3147.foo")
unload("pep3147")
os.mkdir("pep3147")
self.addCleanup(cleanup)
# Touch the __init__.py
with open(os.path.join("pep3147", "__init__.py"), "w"):
pass
with open(os.path.join("pep3147", "foo.py"), "w"):
pass
importlib.invalidate_caches()
m = __import__("pep3147.foo")
unload("pep3147.foo")
unload("pep3147")
importlib.invalidate_caches()
m = __import__("pep3147.foo")
init_pyc = imp.cache_from_source(os.path.join("pep3147", "__init__.py"))
self.assertEqual(m.__cached__, os.path.join(os.curdir, init_pyc))
foo_pyc = imp.cache_from_source(os.path.join("pep3147", "foo.py"))
self.assertEqual(sys.modules["pep3147.foo"].__cached__, os.path.join(os.curdir, foo_pyc))
def test_package___cached___from_pyc(self):
# Like test___cached__ but ensuring __cached__ when imported from a
# PEP 3147 pyc file.
def cleanup():
rmtree('pep3147')
unload('pep3147.foo')
unload('pep3147')
os.mkdir('pep3147')
self.addCleanup(cleanup)
# Touch the __init__.py
with open(os.path.join('pep3147', '__init__.py'), 'w'):
pass
with open(os.path.join('pep3147', 'foo.py'), 'w'):
pass
importlib.invalidate_caches()
m = __import__('pep3147.foo')
unload('pep3147.foo')
unload('pep3147')
importlib.invalidate_caches()
m = __import__('pep3147.foo')
init_pyc = imp.cache_from_source(os.path.join('pep3147',
'__init__.py'))
self.assertEqual(m.__cached__, os.path.join(os.curdir, init_pyc))
foo_pyc = imp.cache_from_source(os.path.join('pep3147', 'foo.py'))
self.assertEqual(sys.modules['pep3147.foo'].__cached__,
os.path.join(os.curdir, foo_pyc))
def check_object_path(path):
if path.endswith(".py"):
try:
import importlib
bytecode_path = importlib.util.cache_from_source(path,
optimization="")
if osp.exists(bytecode_path):
return bytecode_path
except:
try:
import imp
imp.cache_from_source(path, debug_override=False)
except:
pass
pass
basename = osp.basename(path)[0:-3]
if PYTHON3:
spath = path
else:
spath = path.decode("utf-8")
path = tempfile.mkstemp(prefix=basename + "-",
suffix=".pyc",
text=False)[1]
py_compile.compile(spath, cfile=path, doraise=True)
if not path.endswith(".pyc") and not path.endswith(".pyo"):
raise ValueError("path %s must point to a .py or .pyc file\n" % path)
return path
def find_bytecode_files(python_file):
"""
Find the byte code file(s) generated from a Python file.
:param python_file: The pathname of a ``*.py`` file (a string).
:returns: A generator of pathnames (strings).
Starting from Python 3.2 byte code files are written according to `PEP
3147`_ which also defines :py:func:`imp.cache_from_source()` to locate
(optimized) byte code files. When this function is available it is used,
when it's not available the corresponding ``*.pyc`` and/or ``*.pyo`` files
are located manually by :py:func:`find_bytecode_files()`.
.. _PEP 3147: https://www.python.org/dev/peps/pep-3147/
"""
if HAS_PEP_3147:
bytecode_file = imp.cache_from_source(python_file, True)
if os.path.isfile(bytecode_file):
yield bytecode_file
optimized_bytecode_file = imp.cache_from_source(python_file, False)
if os.path.isfile(optimized_bytecode_file):
yield optimized_bytecode_file
else:
for suffix in ("c", "o"):
bytecode_file = python_file + suffix
if os.path.isfile(bytecode_file):
yield bytecode_file
def find_bytecode_files(python_file):
"""
Find the byte code file(s) generated from a Python file.
:param python_file: The pathname of a ``*.py`` file (a string).
:returns: A generator of pathnames (strings).
Starting from Python 3.2 byte code files are written according to `PEP
3147`_ which also defines :func:`imp.cache_from_source()` to locate
(optimized) byte code files. When this function is available it is used,
when it's not available the corresponding ``*.pyc`` and/or ``*.pyo`` files
are located manually by :func:`find_bytecode_files()`.
.. _PEP 3147: https://www.python.org/dev/peps/pep-3147/
"""
if HAS_PEP_3147:
bytecode_file = imp.cache_from_source(python_file, True)
if os.path.isfile(bytecode_file):
yield bytecode_file
optimized_bytecode_file = imp.cache_from_source(python_file, False)
if os.path.isfile(optimized_bytecode_file):
yield optimized_bytecode_file
else:
for suffix in ('c', 'o'):
bytecode_file = python_file + suffix
if os.path.isfile(bytecode_file):
yield bytecode_file
def setUp(self):
self.directory = tempfile.mkdtemp()
self.source_path = os.path.join(self.directory, '_test.py')
self.bc_path = imp.cache_from_source(self.source_path)
with open(self.source_path, 'w') as file:
file.write('x = 123\n')
self.source_path2 = os.path.join(self.directory, '_test2.py')
self.bc_path2 = imp.cache_from_source(self.source_path2)
shutil.copyfile(self.source_path, self.source_path2)
def test_cache_from_source(self):
# Given the path to a .py file, return the path to its PEP 3147
# defined .pyc file (i.e. under __pycache__).
self.assertEqual(
imp.cache_from_source('/foo/bar/baz/qux.py', True),
'/foo/bar/baz/__pycache__/qux.{}.pyc'.format(self.tag))
# Directory with a dot, filename without dot
self.assertEqual(
imp.cache_from_source('/foo.bar/file', True),
'/foo.bar/__pycache__/file{}.pyc'.format(self.tag))
def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1):
success = 1
name = os.path.basename(fullname)
if ddir is not None:
dfile = os.path.join(ddir, name)
else:
dfile = None
if rx is not None:
mo = rx.search(fullname)
if mo:
return success
if legacy:
cfile = fullname + ('c' if __debug__ else 'o')
else:
if optimize >= 0:
cfile = imp.cache_from_source(fullname, debug_override=not optimize)
else:
cfile = imp.cache_from_source(fullname)
cache_dir = os.path.dirname(cfile)
(head, tail) = (name[:-3], name[-3:])
if os.path.isfile(fullname) and tail == '.py':
if not force:
try:
mtime = int(os.stat(fullname).st_mtime)
expect = struct.pack('<4sl', imp.get_magic(), mtime)
with open(cfile, 'rb') as chandle:
actual = chandle.read(8)
if expect == actual:
return success
except IOError:
pass
if not quiet:
print('Compiling {!r}...'.format(fullname))
try:
ok = py_compile.compile(fullname, cfile, dfile, True, optimize=optimize)
except py_compile.PyCompileError as err:
if quiet:
print('*** Error compiling {!r}...'.format(fullname))
else:
print('*** ', end='')
msg = err.msg.encode(sys.stdout.encoding, errors='backslashreplace')
msg = msg.decode(sys.stdout.encoding)
print(msg)
success = 0
except (SyntaxError, UnicodeError, IOError) as e:
if quiet:
print('*** Error compiling {!r}...'.format(fullname))
else:
print('*** ', end='')
print(e.__class__.__name__ + ':', e)
success = 0
if ok == 0:
success = 0
return success
def test_byte_compile(self):
project_dir, dist = self.create_dist()
os.chdir(project_dir)
cmd = install_lib(dist)
cmd.compile = cmd.optimize = 1
f = os.path.join(project_dir, 'foo.py')
self.write_file(f, '# python file')
cmd.byte_compile([f])
pyc_file = imp.cache_from_source('foo.py', debug_override=True)
pyo_file = imp.cache_from_source('foo.py', debug_override=False)
self.assertTrue(os.path.exists(pyc_file))
self.assertTrue(os.path.exists(pyo_file))
def setUp(self):
self.directory = tempfile.mkdtemp()
self.source_path = os.path.join(self.directory, '_test.py')
self.bc_path = imp.cache_from_source(self.source_path)
with open(self.source_path, 'w') as file:
file.write('x = 123\n')
self.source_path2 = os.path.join(self.directory, '_test2.py')
self.bc_path2 = imp.cache_from_source(self.source_path2)
shutil.copyfile(self.source_path, self.source_path2)
self.subdirectory = os.path.join(self.directory, '_subdir')
os.mkdir(self.subdirectory)
self.source_path3 = os.path.join(self.subdirectory, '_test3.py')
shutil.copyfile(self.source_path, self.source_path3)
def test_byte_compile(self):
project_dir, dist = self.create_dist()
os.chdir(project_dir)
cmd = install_lib(dist)
cmd.compile = cmd.optimize = 1
f = os.path.join(project_dir, 'foo.py')
self.write_file(f, '# python file')
cmd.byte_compile([f])
pyc_file = imp.cache_from_source('foo.py', debug_override=True)
pyo_file = imp.cache_from_source('foo.py', debug_override=False)
self.assertTrue(os.path.exists(pyc_file))
self.assertTrue(os.path.exists(pyo_file))
def test_optimize(self):
# make sure compiling with different optimization settings than the
# interpreter's creates the correct file names
optimize = 1 if __debug__ else 0
compileall.compile_dir(self.directory, quiet=True, optimize=optimize)
cached = imp.cache_from_source(self.source_path,
debug_override=not optimize)
self.assertTrue(os.path.isfile(cached))
cached2 = imp.cache_from_source(self.source_path2,
debug_override=not optimize)
self.assertTrue(os.path.isfile(cached2))
cached3 = imp.cache_from_source(self.source_path3,
debug_override=not optimize)
self.assertTrue(os.path.isfile(cached3))
def test_optimize(self):
# make sure compiling with different optimization settings than the
# interpreter's creates the correct file names
optimize = 1 if __debug__ else 0
compileall.compile_dir(self.directory, quiet=True, optimize=optimize)
cached = imp.cache_from_source(self.source_path,
debug_override=not optimize)
self.assertTrue(os.path.isfile(cached))
cached2 = imp.cache_from_source(self.source_path2,
debug_override=not optimize)
self.assertTrue(os.path.isfile(cached2))
cached3 = imp.cache_from_source(self.source_path3,
debug_override=not optimize)
self.assertTrue(os.path.isfile(cached3))
def forget(modname):
"""'Forget' a module was ever imported.
This removes the module from sys.modules and deletes any PEP 3147 or
legacy .pyc and .pyo files.
"""
unload(modname)
for dirname in sys.path:
source = os.path.join(dirname, modname + '.py')
# It doesn't matter if they exist or not, unlink all possible
# combinations of PEP 3147 and legacy pyc and pyo files.
unlink(source + 'c')
unlink(source + 'o')
unlink(imp.cache_from_source(source, debug_override=True))
unlink(imp.cache_from_source(source, debug_override=False))
def _bytecode_filenames(self, py_filenames):
bytecode_files = []
for py_file in py_filenames:
# Since build_py handles package data installation, the
# list of outputs can contain more than just .py files.
# Make sure we only report bytecode for the .py files.
ext = os.path.splitext(os.path.normcase(py_file))[1]
if ext != PYTHON_SOURCE_EXTENSION:
continue
if self.compile:
bytecode_files.append(imp.cache_from_source(py_file, True))
if self.optimize:
bytecode_files.append(imp.cache_from_source(py_file, False))
return bytecode_files
def test_cache_from_source_override(self):
# When debug_override is not None, it can be any true-ish or false-ish
# value.
path = os.path.join('foo', 'bar', 'baz.py')
partial_expect = os.path.join('foo', 'bar', '__pycache__',
'baz.{}.py'.format(self.tag))
self.assertEqual(imp.cache_from_source(path, []), partial_expect + 'o')
self.assertEqual(imp.cache_from_source(path, [17]),
partial_expect + 'c')
# However if the bool-ishness can't be determined, the exception
# propagates.
class Bearish:
def __bool__(self): raise RuntimeError
with self.assertRaises(RuntimeError):
imp.cache_from_source('/foo/bar/baz.py', Bearish())
def forget(modname):
"""'Forget' a module was ever imported.
This removes the module from sys.modules and deletes any PEP 3147 or
legacy .pyc and .pyo files.
"""
unload(modname)
for dirname in sys.path:
source = os.path.join(dirname, modname + os.extsep + 'py')
# It doesn't matter if they exist or not, unlink all possible
# combinations of PEP 3147 and legacy pyc and pyo files.
unlink(source + 'c')
unlink(source + 'o')
unlink(imp.cache_from_source(source, debug_override=True))
unlink(imp.cache_from_source(source, debug_override=False))
def test_cache_from_source_override(self):
# When debug_override is not None, it can be any true-ish or false-ish
# value.
path = os.path.join('foo', 'bar', 'baz.py')
partial_expect = os.path.join('foo', 'bar', '__pycache__',
'baz.{}.py'.format(self.tag))
self.assertEqual(imp.cache_from_source(path, []), partial_expect + 'o')
self.assertEqual(imp.cache_from_source(path, [17]),
partial_expect + 'c')
# However if the bool-ishness can't be determined, the exception
# propagates.
class Bearish:
def __bool__(self): raise RuntimeError
with self.assertRaises(RuntimeError):
imp.cache_from_source('/foo/bar/baz.py', Bearish())
def test_run(self):
script_path = self.tmp_py()
pyscript = PythonScript.create(script_path)
pyscript.run(self.engine, 1)
pyscript.run(self.engine, -1)
self.assertRaises(exceptions.ScriptError, pyscript.run, self.engine, 0)
self.assertRaises(exceptions.ScriptError, pyscript._func, 'foobar')
# clean pyc file
if six.PY3:
os.remove(imp.cache_from_source(script_path))
else:
os.remove(script_path + 'c')
# test deprecated upgrade/downgrade with no arguments
contents = open(script_path, 'r').read()
f = open(script_path, 'w')
f.write(contents.replace("upgrade(migrate_engine)", "upgrade()"))
f.close()
pyscript = PythonScript(script_path)
pyscript._module = None
try:
pyscript.run(self.engine, 1)
pyscript.run(self.engine, -1)
except exceptions.ScriptError:
pass
else:
self.fail()
def test_timestamp_overflow(self):
# A modification timestamp larger than 2**32 should not be a problem
# when importing a module (issue #11235).
sys.path.insert(0, os.curdir)
try:
source = TESTFN + ".py"
compiled = imp.cache_from_source(source)
with open(source, 'w') as f:
pass
try:
os.utime(source, (2**33 - 5, 2**33 - 5))
except OverflowError:
self.skipTest("cannot set modification time to large integer")
except OSError as e:
if e.errno != getattr(errno, 'EOVERFLOW', None):
raise
self.skipTest(
"cannot set modification time to large integer ({})".
format(e))
__import__(TESTFN)
# The pyc file was created.
os.stat(compiled)
finally:
del sys.path[0]
remove_files(TESTFN)
def test_file_from_empty_string_dir(self):
# Loading a module found from an empty string entry on sys.path should
# not only work, but keep all attributes relative.
file_path = "_temp.py"
with open(file_path, "w") as file:
file.write("# test file for importlib")
try:
with util.uncache("_temp"):
loader = machinery.SourceFileLoader("_temp", file_path)
mod = loader.load_module("_temp")
self.assertEqual(file_path, mod.__file__)
self.assertEqual(imp.cache_from_source(file_path), mod.__cached__)
finally:
os.unlink(file_path)
pycache = os.path.dirname(imp.cache_from_source(file_path))
shutil.rmtree(pycache)
def test_cache_from_source(self):
# Given the path to a .py file, return the path to its PEP 3147
# defined .pyc file (i.e. under __pycache__).
path = os.path.join('foo', 'bar', 'baz', 'qux.py')
expect = os.path.join('foo', 'bar', 'baz', '__pycache__',
'qux.{}.pyc'.format(self.tag))
self.assertEqual(imp.cache_from_source(path, True), expect)
def test_cache_from_source_optimized(self):
# Given the path to a .py file, return the path to its PEP 3147
# defined .pyo file (i.e. under __pycache__).
path = os.path.join('foo', 'bar', 'baz', 'qux.py')
expect = os.path.join('foo', 'bar', 'baz', '__pycache__',
'qux.{}.pyo'.format(self.tag))
self.assertEqual(imp.cache_from_source(path, False), expect)
def test_run(self):
script_path = self.tmp_py()
pyscript = PythonScript.create(script_path)
pyscript.run(self.engine, 1)
pyscript.run(self.engine, -1)
self.assertRaises(exceptions.ScriptError, pyscript.run, self.engine, 0)
self.assertRaises(exceptions.ScriptError, pyscript._func, 'foobar')
# clean pyc file
if six.PY3:
os.remove(imp.cache_from_source(script_path))
else:
os.remove(script_path + 'c')
# test deprecated upgrade/downgrade with no arguments
contents = open(script_path, 'r').read()
f = open(script_path, 'w')
f.write(contents.replace("upgrade(migrate_engine)", "upgrade()"))
f.close()
pyscript = PythonScript(script_path)
pyscript._module = None
try:
pyscript.run(self.engine, 1)
pyscript.run(self.engine, -1)
except exceptions.ScriptError:
pass
else:
self.fail()
def pyc_file_from_path(path):
"""Given a python source path, locate the .pyc.
"""
if has_pep3147():
if py35:
import importlib
candidate = importlib.util.cache_from_source(path)
else:
import imp
candidate = imp.cache_from_source(path)
if os.path.exists(candidate):
return candidate
# even for pep3147, fall back to the old way of finding .pyc files,
# to support sourceless operation
filepath, ext = os.path.splitext(path)
for ext in get_current_bytecode_suffixes():
if os.path.exists(filepath + ext):
return filepath + ext
else:
return None
def test_include_bad_file(self):
rc, out, err = self.assertRunNotOK(
'-i', os.path.join(self.directory, 'nosuchfile'), self.pkgdir)
self.assertRegex(out, b'rror.*nosuchfile')
self.assertNotRegex(err, b'Traceback')
self.assertFalse(
os.path.exists(imp.cache_from_source(self.pkgdir_cachedir)))
def test_pyc_always_writable(self):
# Initially read-only .pyc files on Windows used to cause problems
# with later updates, see issue #6074 for details
with _ready_to_import() as (name, path):
# Write a Python file, make it read-only and import it
with open(path, 'w') as f:
f.write("x = 'original'\n")
# Tweak the mtime of the source to ensure pyc gets updated later
s = os.stat(path)
os.utime(path, (s.st_atime, s.st_mtime-100000000))
os.chmod(path, 0o400)
m = __import__(name)
self.assertEqual(m.x, 'original')
# Change the file and then reimport it
os.chmod(path, 0o600)
with open(path, 'w') as f:
f.write("x = 'rewritten'\n")
unload(name)
importlib.invalidate_caches()
m = __import__(name)
self.assertEqual(m.x, 'rewritten')
# Now delete the source file and check the pyc was rewritten
unlink(path)
unload(name)
importlib.invalidate_caches()
if __debug__:
bytecode_only = path + "c"
else:
bytecode_only = path + "o"
os.rename(imp.cache_from_source(path), bytecode_only)
m = __import__(name)
self.assertEqual(m.x, 'rewritten')
def manipulate_bytecode(self,
name,
mapping,
manipulator,
*,
del_source=False):
"""Manipulate the bytecode of a module by passing it into a callable
that returns what to use as the new bytecode."""
try:
del sys.modules['_temp']
except KeyError:
pass
py_compile.compile(mapping[name])
if not del_source:
bytecode_path = imp.cache_from_source(mapping[name])
else:
os.unlink(mapping[name])
bytecode_path = make_legacy_pyc(mapping[name])
if manipulator:
with open(bytecode_path, 'rb') as file:
bc = file.read()
new_bc = manipulator(bc)
with open(bytecode_path, 'wb') as file:
if new_bc is not None:
file.write(new_bc)
return bytecode_path
def test_timestamp_overflow(self):
# When a modification timestamp is larger than 2**32, it should be
# truncated rather than raise an OverflowError.
with source_util.create_modules('_temp') as mapping:
source = mapping['_temp']
compiled = imp.cache_from_source(source)
with open(source, 'w') as f:
f.write("x = 5")
try:
os.utime(source, (2**33 - 5, 2**33 - 5))
except OverflowError:
self.skipTest("cannot set modification time to large integer")
except OSError as e:
if e.errno != getattr(errno, 'EOVERFLOW', None):
raise
self.skipTest(
"cannot set modification time to large integer ({})".
format(e))
loader = machinery.SourceFileLoader('_temp', mapping['_temp'])
mod = loader.load_module('_temp')
# Sanity checks.
self.assertEqual(mod.__cached__, compiled)
self.assertEqual(mod.x, 5)
# The pyc file was created.
os.stat(compiled)
def get_compiled_binary(fp):
if sys.version_info >= (3, 4):
return importlib.util.cache_from_source(fp)
else:
import imp
return imp.cache_from_source(fp)
def test_pyc_always_writable(self):
# Initially read-only .pyc files on Windows used to cause problems
# with later updates, see issue #6074 for details
with _ready_to_import() as (name, path):
# Write a Python file, make it read-only and import it
with open(path, 'w') as f:
f.write("x = 'original'\n")
# Tweak the mtime of the source to ensure pyc gets updated later
s = os.stat(path)
os.utime(path, (s.st_atime, s.st_mtime - 100000000))
os.chmod(path, 0o400)
m = __import__(name)
self.assertEqual(m.x, 'original')
# Change the file and then reimport it
os.chmod(path, 0o600)
with open(path, 'w') as f:
f.write("x = 'rewritten'\n")
unload(name)
importlib.invalidate_caches()
m = __import__(name)
self.assertEqual(m.x, 'rewritten')
# Now delete the source file and check the pyc was rewritten
unlink(path)
unload(name)
importlib.invalidate_caches()
if __debug__:
bytecode_only = path + "c"
else:
bytecode_only = path + "o"
os.rename(imp.cache_from_source(path), bytecode_only)
m = __import__(name)
self.assertEqual(m.x, 'rewritten')
def test_cache_from_source_override(self):
# When debug_override is not None, it can be any true-ish or false-ish
# value.
self.assertEqual(
imp.cache_from_source('/foo/bar/baz.py', []),
'/foo/bar/__pycache__/baz.{}.pyo'.format(self.tag))
self.assertEqual(
imp.cache_from_source('/foo/bar/baz.py', [17]),
'/foo/bar/__pycache__/baz.{}.pyc'.format(self.tag))
# However if the bool-ishness can't be determined, the exception
# propagates.
class Bearish:
def __bool__(self): raise RuntimeError
self.assertRaises(
RuntimeError,
imp.cache_from_source, '/foo/bar/baz.py', Bearish())
def test_cache_from_source_override(self):
# When debug_override is not None, it can be any true-ish or false-ish
# value.
self.assertEqual(imp.cache_from_source('/foo/bar/baz.py', []),
'/foo/bar/__pycache__/baz.{}.pyo'.format(self.tag))
self.assertEqual(imp.cache_from_source('/foo/bar/baz.py', [17]),
'/foo/bar/__pycache__/baz.{}.pyc'.format(self.tag))
# However if the bool-ishness can't be determined, the exception
# propagates.
class Bearish:
def __bool__(self):
raise RuntimeError
self.assertRaises(RuntimeError, imp.cache_from_source,
'/foo/bar/baz.py', Bearish())
def test_include_bad_file(self):
rc, out, err = self.assertRunNotOK(
'-i', os.path.join(self.directory, 'nosuchfile'), self.pkgdir)
self.assertRegex(out, b'rror.*nosuchfile')
self.assertNotRegex(err, b'Traceback')
self.assertFalse(os.path.exists(imp.cache_from_source(
self.pkgdir_cachedir)))
def _get_codename(self, pathname, basename):
def _compile(file, optimize=-1):
import py_compile
if self.debug:
print('Compiling', file)
try:
py_compile.compile(file, doraise=True, optimize=optimize)
except py_compile.PyCompileError as err:
print(err.msg)
return False
return True
file_py = pathname + '.py'
file_pyc = pathname + '.pyc'
file_pyo = pathname + '.pyo'
pycache_pyc = imp.cache_from_source(file_py, True)
pycache_pyo = imp.cache_from_source(file_py, False)
if self._optimize == -1:
if os.path.isfile(file_pyo) and os.stat(file_pyo).st_mtime >= os.stat(file_py).st_mtime:
arcname = fname = file_pyo
elif os.path.isfile(file_pyc) and os.stat(file_pyc).st_mtime >= os.stat(file_py).st_mtime:
arcname = fname = file_pyc
elif os.path.isfile(pycache_pyc) and os.stat(pycache_pyc).st_mtime >= os.stat(file_py).st_mtime:
fname = pycache_pyc
arcname = file_pyc
elif os.path.isfile(pycache_pyo) and os.stat(pycache_pyo).st_mtime >= os.stat(file_py).st_mtime:
fname = pycache_pyo
arcname = file_pyo
elif _compile(file_py):
fname = pycache_pyc if __debug__ else pycache_pyo
arcname = file_pyc if __debug__ else file_pyo
else:
fname = arcname = file_py
else:
if self._optimize == 0:
fname = pycache_pyc
arcname = file_pyc
else:
fname = pycache_pyo
arcname = file_pyo
if not (os.path.isfile(fname) and os.stat(fname).st_mtime >= os.stat(file_py).st_mtime or _compile(file_py, optimize=self._optimize)):
fname = arcname = file_py
archivename = os.path.split(arcname)[1]
if basename:
archivename = '%s/%s' % (basename, archivename)
return (fname, archivename)
def _bytecode_filenames(self, py_filenames):
bytecode_files = []
for py_file in py_filenames:
# Since build_py handles package data installation, the
# list of outputs can contain more than just .py files.
# Make sure we only report bytecode for the .py files.
ext = os.path.splitext(os.path.normcase(py_file))[1]
if ext != PYTHON_SOURCE_EXTENSION:
continue
if self.compile:
bytecode_files.append(
imp.cache_from_source(py_file, debug_override=True))
if self.optimize > 0:
bytecode_files.append(
imp.cache_from_source(py_file, debug_override=False))
return bytecode_files
def test_file_from_empty_string_dir(self):
# Loading a module found from an empty string entry on sys.path should
# not only work, but keep all attributes relative.
file_path = '_temp.py'
with open(file_path, 'w') as file:
file.write("# test file for importlib")
try:
with util.uncache('_temp'):
loader = machinery.SourceFileLoader('_temp', file_path)
mod = loader.load_module('_temp')
self.assertEqual(file_path, mod.__file__)
self.assertEqual(imp.cache_from_source(file_path),
mod.__cached__)
finally:
os.unlink(file_path)
pycache = os.path.dirname(imp.cache_from_source(file_path))
shutil.rmtree(pycache)
def _cache_path(filename):
import imp
if hasattr(imp, "cache_from_source"):
return imp.cache_from_source(filename) + "kle.cache"
else:
return fullname + ".cache"
def test_cache_from_source(self):
import imp, sys
tag = sys.implementation.cache_tag
pycfile = imp.cache_from_source('a/b/c.py')
assert pycfile == 'a/b/__pycache__/c.%s.pyc' % tag
assert imp.source_from_cache('a/b/__pycache__/c.%s.pyc' %
tag) == 'a/b/c.py'
raises(ValueError, imp.source_from_cache, 'a/b/c.py')
def test_file_from_empty_string_dir(self):
# Loading a module found from an empty string entry on sys.path should
# not only work, but keep all attributes relative.
file_path = u'_temp.py'
with open(file_path, u'w') as file:
file.write(u"# test file for importlib_full")
try:
with util.uncache(u'_temp'):
loader = _bootstrap._SourceFileLoader(u'_temp', file_path)
mod = loader.load_module(u'_temp')
self.assertEqual(file_path, mod.__file__)
self.assertEqual(imp.cache_from_source(file_path),
mod.__cached__)
finally:
os.unlink(file_path)
pycache = os.path.dirname(imp.cache_from_source(file_path))
shutil.rmtree(pycache)
def source_using_bytecode(seconds, repeat):
u"""Bytecode w/ source: simple"""
name = u"__importlib_full_test_benchmark__"
with source_util.create_modules(name) as mapping:
py_compile.compile(mapping[name])
assert os.path.exists(imp.cache_from_source(mapping[name]))
for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat, seconds=seconds):
yield result
def compile_file(fullname, ddir=None, force=0, rx=None, quiet=False,
legacy=False):
"""Byte-compile one file.
Arguments (only fullname is required):
fullname: the file to byte-compile
ddir: if given, the directory name compiled in to the
byte-code file.
force: if True, force compilation, even if timestamps are up-to-date
quiet: if True, be quiet during compilation
legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
"""
success = 1
name = os.path.basename(fullname)
if ddir is not None:
dfile = os.path.join(ddir, name)
else:
dfile = None
if rx is not None:
mo = rx.search(fullname)
if mo:
return success
if os.path.isfile(fullname):
if legacy:
cfile = fullname + ('c' if __debug__ else 'o')
else:
cfile = imp.cache_from_source(fullname)
cache_dir = os.path.dirname(cfile)
try:
os.mkdir(cache_dir)
except OSError, error:
if error.errno != errno.EEXIST:
raise
head, tail = name[:-3], name[-3:]
if tail == '.py':
if not force:
try:
mtime = int(os.stat(fullname).st_mtime)
expect = struct.pack('<4sl', imp.get_magic(), mtime)
with open(cfile, 'rb') as chandle:
actual = chandle.read(8)
if expect == actual:
return success
except IOError:
pass
if not quiet:
print 'Compiling', fullname, '...'
try:
ok = py_compile.compile(fullname, cfile, dfile, True)
except py_compile.PyCompileError,err:
if quiet:
print 'Compiling', fullname, '...'
print err.msg
success = 0
except IOError, e:
print "Sorry", e
success = 0
def doneIter(self):
self.__modDeserial.doneIter()
# remove the temporary module and its compiled version (*.pyc)
os.remove(self.__filename)
try: # py3.2+ uses special folder/filename for .pyc files
from imp import cache_from_source
os.remove(cache_from_source(self.__filename))
except ImportError:
os.remove(self.__filename + 'c')
def test_no_bytecode(self):
# If no bytecode exists then move on to the source.
self.loader.bytecode_path = "<does not exist>"
# Sanity check
with self.assertRaises(IOError):
bytecode_path = imp.cache_from_source(self.path)
self.loader.get_data(bytecode_path)
code_object = self.loader.get_code(self.name)
self.verify_code(code_object, bytecode_written=True)
def test_d_runtime_error(self):
bazfn = script_helper.make_script(self.pkgdir, 'baz', 'raise Exception')
self.assertRunOK('-q', '-d', 'dinsdale', self.pkgdir)
fn = script_helper.make_script(self.pkgdir, 'bing', 'import baz')
pyc = imp.cache_from_source(bazfn)
os.rename(pyc, os.path.join(self.pkgdir, 'baz.pyc'))
os.remove(bazfn)
rc, out, err = script_helper.assert_python_failure(fn)
self.assertRegex(err, b'File "dinsdale')
def doneIter(self):
self.__modDeserial.doneIter()
# remove the temporary module and its compiled version (*.pyc)
os.remove(self.__filename)
try: # py3.2+ uses special folder/filename for .pyc files
from imp import cache_from_source
os.remove(cache_from_source(self.__filename))
except ImportError:
os.remove(self.__filename + 'c')
def test_missing_source(self):
# With PEP 3147 cache layout, removing the source but leaving the pyc
# file does not satisfy the import.
__import__(TESTFN)
pyc_file = imp.cache_from_source(self.source)
self.assertTrue(os.path.exists(pyc_file))
os.remove(self.source)
forget(TESTFN)
self.assertRaises(ImportError, __import__, TESTFN)
def test_missing_source(self):
# With PEP 3147 cache layout, removing the source but leaving the pyc
# file does not satisfy the import.
__import__(TESTFN)
pyc_file = imp.cache_from_source(self.source)
self.assertTrue(os.path.exists(pyc_file))
os.remove(self.source)
forget(TESTFN)
self.assertRaises(ImportError, __import__, TESTFN)
def test_byte_compile(self):
project_dir, dist = self.create_dist()
os.chdir(project_dir)
cmd = install_lib(dist)
cmd.compile = True
cmd.optimize = 1
f = os.path.join(project_dir, 'foo.py')
self.write_file(f, '# python file')
cmd.byte_compile([f])
if sys.version_info[:2] == (3, 1):
pyc_file = 'foo.pyc'
pyo_file = 'foo.pyo'
else:
pyc_file = imp.cache_from_source('foo.py', True)
pyo_file = imp.cache_from_source('foo.py', False)
self.assertTrue(os.path.exists(pyc_file))
self.assertTrue(os.path.exists(pyo_file))
def __init__(self, path, magic=imp.get_magic()):
super().__init__(path)
self.bytecode_path = imp.cache_from_source(self.path)
data = bytearray(magic)
data.extend(marshal._w_long(self.source_mtime))
code_object = compile(self.source, self.path, 'exec',
dont_inherit=True)
data.extend(marshal.dumps(code_object))
self.bytecode = bytes(data)
self.written = {}
def get_outputs(self, include_bytecode=True):
modules = self.find_all_modules()
outputs = []
for package, module, module_file in modules:
package = package.split('.')
filename = self.get_module_outfile(self.build_lib, package, module)
outputs.append(filename)
if include_bytecode:
if self.compile:
outputs.append(imp.cache_from_source(filename, True))
if self.optimize:
outputs.append(imp.cache_from_source(filename, False))
outputs += [
os.path.join(build_dir, filename)
for package, src_dir, build_dir, filenames in self.data_files
for filename in filenames]
return outputs
def _bcompile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
with tokenize.open(file) as f:
try:
timestamp = int(os.fstat(f.fileno()).st_mtime)
except AttributeError:
timestamp = int(os.stat(file).st_mtime)
codestring = f.read()
try:
codeobject = builtins.compile(codestring,
dfile or file,
'exec',
optimize=optimize)
except Exception as err:
py_exc = py_compile.PyCompileError(err.__class__, err, dfile
or file)
if doraise:
raise py_exc
else:
sys.stderr.write(py_exc.msg + '\n')
return
if cfile is None:
if optimize >= 0:
cfile = imp.cache_from_source(file,
debug_override=not optimize)
else:
cfile = imp.cache_from_source(file)
try:
os.makedirs(os.path.dirname(cfile))
except OSError as error:
if error.errno != errno.EEXIST:
raise
fc = io.BytesIO()
try:
fc.write(b'\0\0\0\0')
py_compile.wr_long(fc, timestamp)
marshal.dump(codeobject, fc)
fc.flush()
fc.seek(0, 0)
fc.write(py_compile.MAGIC)
return fc.getvalue()
finally:
fc.close()
def setUp(self, *, is_package=True, **kwargs):
self.package = 'pkg'
if is_package:
self.path = os.path.join(self.package, '__init__.py')
self.name = self.package
else:
module_name = 'mod'
self.path = os.path.join(self.package, '.'.join(['mod', 'py']))
self.name = '.'.join([self.package, module_name])
self.cached = imp.cache_from_source(self.path)
self.loader = self.loader_mock(self.path, **kwargs)