def test_cache_from_source_override(self):
# When debug_override is not None, it can be any true-ish or false-ish
# value.
path = os.path.join('foo', 'bar', 'baz.py')
partial_expect = os.path.join('foo', 'bar', '__pycache__',
'baz.{}.py'.format(self.tag))
self.assertEqual(util.cache_from_source(path, []), partial_expect + 'o')
self.assertEqual(util.cache_from_source(path, [17]),
partial_expect + 'c')
# However if the bool-ishness can't be determined, the exception
# propagates.
class Bearish:
def __bool__(self): raise RuntimeError
with self.assertRaises(RuntimeError):
util.cache_from_source('/foo/bar/baz.py', Bearish())
def test_bin_hyc():
_, err = run_cmd("hyc", expect=0)
assert err == ''
_, err = run_cmd("hyc -", expect=0)
assert err == ''
output, _ = run_cmd("hyc -h")
assert "usage" in output
path = "tests/resources/argparse_ex.hy"
output, _ = run_cmd("hyc " + path)
assert "Compiling" in output
assert os.path.exists(cache_from_source(path))
rm(cache_from_source(path))
def test_bin_hy_circular_macro_require():
"""Confirm that macros can require themselves during expansion and when
run from the command line."""
# First, with no bytecode
test_file = "tests/resources/bin/circular_macro_require.hy"
rm(cache_from_source(test_file))
assert not os.path.exists(cache_from_source(test_file))
output, _ = run_cmd("hy {}".format(test_file))
assert "42" == output.strip()
# Now, with bytecode
assert os.path.exists(cache_from_source(test_file))
output, _ = run_cmd("hy {}".format(test_file))
assert "42" == output.strip()
def test_bin_hyc_file_sys_path():
# similar to test_bin_hy_file_sys_path, test hyc and hy2py to make sure
# they can find the relative import at compile time
# https://github.com/hylang/hy/issues/2021
test_file = 'tests/resources/relative_import_compile_time.hy'
file_relative_path = os.path.realpath(os.path.dirname(test_file))
for binary in ("hy", "hyc", "hy2py"):
# Ensure we hit the compiler
rm(cache_from_source(test_file))
assert not os.path.exists(cache_from_source(file_relative_path))
output, _ = run_cmd(f"{binary} {test_file}")
assert file_relative_path in output
def test_cache_from_source_optimized(self):
# Given the path to a .py file, return the path to its PEP 3147
# defined .pyo file (i.e. under __pycache__).
path = os.path.join('foo', 'bar', 'baz', 'qux.py')
expect = os.path.join('foo', 'bar', 'baz', '__pycache__',
'qux.{}.pyo'.format(self.tag))
self.assertEqual(util.cache_from_source(path, False), expect)
def make_pymodule_path(filename, optimization=None):
path = os.path.dirname(filename)
file = os.path.basename(filename)
mod, ext = os.path.splitext(file)
if sys.hexversion >= 0x3050000:
fullpath = cache_from_source(filename, optimization=optimization)
elif sys.hexversion >= 0x3040000:
fullpath = cache_from_source(filename, ext=='.pyc')
elif sys.hexversion >= 0x3020000:
import imp
modname = mod+"."+imp.get_tag()+ext
fullpath = os.path.join(path,'__pycache__',modname)
else:
fullpath = filename
return fullpath
def make_pymodule_path(filename, optimization=None):
path = os.path.dirname(filename)
file = os.path.basename(filename)
mod, ext = os.path.splitext(file)
if sys.hexversion >= 0x3050000:
fullpath = cache_from_source(filename, optimization=optimization)
elif sys.hexversion >= 0x3040000:
fullpath = cache_from_source(filename, ext == '.pyc')
elif sys.hexversion >= 0x3020000:
import imp
modname = mod + "." + imp.get_tag() + ext
fullpath = os.path.join(path, '__pycache__', modname)
else:
fullpath = filename
return fullpath
def test_cache_from_source_optimized(self):
# Given the path to a .py file, return the path to its PEP 3147
# defined .pyo file (i.e. under __pycache__).
path = os.path.join('foo', 'bar', 'baz', 'qux.py')
expect = os.path.join('foo', 'bar', 'baz', '__pycache__',
'qux.{}.pyo'.format(self.tag))
self.assertEqual(util.cache_from_source(path, False), expect)
def check_py(self, mod_name, zip_name, zip_path, existing_attr, **kwargs):
filename = asset_path(zip_name, zip_path)
# In build.gradle, .pyc pre-compilation is disabled for app.zip, so it will generate
# __pycache__ directories.
if zip_name == APP_ZIP:
cache_filename = cache_from_source(filename)
origin = filename
else:
cache_filename = None
origin = filename + "c"
mod = self.check_module(mod_name, filename, cache_filename, origin, **kwargs)
self.assertNotPredicate(exists, filename)
if cache_filename is None:
self.assertNotPredicate(exists, cache_from_source(filename))
new_attr = "check_py_attr"
self.assertFalse(hasattr(mod, new_attr))
setattr(mod, new_attr, 1)
delattr(mod, existing_attr)
reload(mod) # Should reuse existing module object.
self.assertEqual(1, getattr(mod, new_attr))
self.assertTrue(hasattr(mod, existing_attr))
if cache_filename:
# A valid .pyc should not be written again. (We can't use the set_mode technique
# here because failure to write a .pyc is silently ignored.)
with self.assertNotModifies(cache_filename):
mod = self.clean_reload(mod)
self.assertFalse(hasattr(mod, new_attr))
# And if the header matches, the code in the .pyc should be used, whatever it is.
header = self.read_pyc_header(cache_filename)
with open(cache_filename, "wb") as pyc_file:
pyc_file.write(header)
code = compile(f"{new_attr} = 2", "<test>", "exec")
marshal.dump(code, pyc_file)
mod = self.clean_reload(mod)
self.assertEqual(2, getattr(mod, new_attr))
self.assertFalse(hasattr(mod, existing_attr))
# A .pyc with mismatching header timestamp should be written again.
new_header = header[0:8] + b"\x00\x01\x02\x03" + header[12:]
self.assertNotEqual(new_header, header)
self.write_pyc_header(cache_filename, new_header)
with self.assertModifies(cache_filename):
self.clean_reload(mod)
self.assertEqual(header, self.read_pyc_header(cache_filename))
def test_bin_hy_macro_require():
"""Confirm that a `require` will load macros into the non-module namespace
(i.e. `exec(code, locals)`) used by `runpy.run_path`.
In other words, this confirms that the AST generated for a `require` will
load macros into the unnamed namespace its run in."""
# First, with no bytecode
test_file = "tests/resources/bin/require_and_eval.hy"
rm(cache_from_source(test_file))
assert not os.path.exists(cache_from_source(test_file))
output, _ = run_cmd("hy {}".format(test_file))
assert "abc" == output.strip()
# Now, with bytecode
assert os.path.exists(cache_from_source(test_file))
output, _ = run_cmd("hy {}".format(test_file))
assert "abc" == output.strip()
def test_import_file_creates_no_bytecode(example_pikefile_path):
bytecode_path = cache_from_source(str(example_pikefile_path))
assert not os.path.exists(bytecode_path)
import_file(example_pikefile_path)
assert not os.path.exists(bytecode_path)
bytecode_dir = os.path.dirname(bytecode_path)
assert bytecode_dir.endswith(_PYCACHE)
assert not os.path.exists(bytecode_dir)
def test_cache_from_source_override(self):
# When debug_override is not None, it can be any true-ish or false-ish
# value.
path = os.path.join('foo', 'bar', 'baz.py')
partial_expect = os.path.join('foo', 'bar', '__pycache__',
'baz.{}.py'.format(self.tag))
self.assertEqual(util.cache_from_source(path, []),
partial_expect + 'o')
self.assertEqual(util.cache_from_source(path, [17]),
partial_expect + 'c')
# However if the bool-ishness can't be determined, the exception
# propagates.
class Bearish:
def __bool__(self):
raise RuntimeError
with self.assertRaises(RuntimeError):
util.cache_from_source('/foo/bar/baz.py', Bearish())
def _rename_python3_bytecode(self, pure_python):
# "If the py source file is missing, the pyc file inside
# __pycache__ will be ignored. This eliminates the problem of
# accidental stale pyc file imports."
# https://www.python.org/dev/peps/pep-3147/
from importlib.util import cache_from_source
bytecode = py.path.local(cache_from_source(str(pure_python)))
target_dir, target_fn = pure_python.dirname, pure_python.basename
target_fn = target_fn.replace('.py', '.pyc')
target = py.path.local(target_dir).join(target_fn)
bytecode.copy(target)
def check_py(self, mod_name, zip_name, zip_path, existing_attr, **kwargs):
filename = asset_path(zip_name, zip_path)
# build.gradle has pyc { src false }, so APP_ZIP will generate __pycache__ directories.
cache_filename = cache_from_source(
filename) if zip_name == APP_ZIP else None
mod = self.check_module(mod_name, filename, cache_filename, **kwargs)
self.assertNotPredicate(exists, filename)
if cache_filename is None:
self.assertNotPredicate(exists, cache_from_source(filename))
new_attr = "check_py_attr"
self.assertFalse(hasattr(mod, new_attr))
setattr(mod, new_attr, 1)
delattr(mod, existing_attr)
reload(mod) # Should reuse existing module object.
self.assertEqual(1, getattr(mod, new_attr))
self.assertTrue(hasattr(mod, existing_attr))
if cache_filename:
# A valid .pyc should not be written again.
with self.assertNotModifies(cache_filename):
mod = self.clean_reload(mod)
self.assertFalse(hasattr(mod, new_attr))
# And if the header matches, the code in the .pyc should be used, whatever it is.
header = self.read_pyc_header(cache_filename)
with open(cache_filename, "wb") as pyc_file:
pyc_file.write(header)
code = compile(f"{new_attr} = 2", "<test>", "exec")
marshal.dump(code, pyc_file)
mod = self.clean_reload(mod)
self.assertEqual(2, getattr(mod, new_attr))
self.assertFalse(hasattr(mod, existing_attr))
# A .pyc with mismatching header timestamp should be written again.
new_header = header[0:8] + b"\x00\x01\x02\x03" + header[12:]
self.assertNotEqual(new_header, header)
self.write_pyc_header(cache_filename, new_header)
with self.assertModifies(cache_filename):
self.clean_reload(mod)
self.assertEqual(header, self.read_pyc_header(cache_filename))
def cleanup_py_files(path):
"""Cleanup python file and associated byte-compiled file"""
path = Path(path)
if path.exists():
path.unlink()
assert not path.exists()
cached_file = Path(cache_from_source(str(path)))
if cached_file.exists():
cached_file.unlink()
assert not cached_file.exists()
def load_from_apyc(filename):
apyc_filename = cache_from_source(filename)
with open(apyc_filename, "rb") as pyc:
pycmagic = pyc.read(4)
pycmodtime = unpack('=L', pyc.read(4))[0]
pycsrcsize = unpack('=L', pyc.read(4))[0]
codeobj = marshal.load(pyc)
return codeobj
def cleanup_py_files(path):
"""Cleanup python file and associated byte-compiled file"""
path = Path(path)
if path.exists():
path.unlink()
assert not path.exists()
cached_file = Path(cache_from_source(str(path)))
if cached_file.exists():
cached_file.unlink()
assert not cached_file.exists()
def byte_compile(self, arcname, content):
py_path = os.path.join(*arcname.split('/'))
pyc_path = cache_from_source(py_path)
arcname_pyc = '/'.join(pyc_path.split(os.path.sep))
diagnostic_name = posixpath.join(self.egg_name, arcname)
with tempfile.NamedTemporaryFile() as dst:
with tempfile.NamedTemporaryFile() as src:
src.write(content)
src.flush()
py_compile.compile(src.name, dst.name, diagnostic_name, True)
return arcname_pyc, dst.read()
def da_cache_from_source(source_path, optimization=None):
"""Given the path to a .da file, return the path to its .pyc file.
This function modifies the filename generated by importlib's
`cache_from_source` to include the DistAlgo version number.
"""
bytecode_path = util.cache_from_source(source_path, optimization=optimization)
bytecode_dir, bytecode_file = os.path.split(bytecode_path)
components = bytecode_file.split('.')
components.insert(-1, 'da-{}'.format(common.__version__))
bytecode_file = '.'.join(components)
return os.path.join(bytecode_dir, bytecode_file)
def da_cache_from_source(source_path, optimization=None):
"""Given the path to a .da file, return the path to its .pyc file.
This function modifies the filename generated by importlib's
`cache_from_source` to include the DistAlgo version number.
"""
bytecode_path = util.cache_from_source(source_path,
optimization=optimization)
bytecode_dir, bytecode_file = os.path.split(bytecode_path)
components = bytecode_file.split('.')
components.insert(-1, 'da-{}'.format(common.__version__))
bytecode_file = '.'.join(components)
return os.path.join(bytecode_dir, bytecode_file)
def cache_from_source(path, debug_override=None):
"""Given the path to a .py file, return the path to its .pyc file.
The .py file does not need to exist; this simply returns the path to the
.pyc file calculated as if the .py file were imported.
If debug_override is not None, then it must be a boolean and is used in
place of sys.flags.optimize.
If sys.implementation.cache_tag is None then NotImplementedError is raised.
"""
with warnings.catch_warnings():
warnings.simplefilter('ignore')
return util.cache_from_source(path, debug_override)
def test_bin_hy_byte_compile(scenario, cmd_fmt):
modname = "tests.resources.bin.bytecompile"
fpath = modname.replace(".", "/") + ".hy"
if scenario == 'prevent_by_option':
cmd_fmt.insert(1, '-B')
cmd = ' '.join(cmd_fmt).format(**locals())
rm(cache_from_source(fpath))
if scenario == "prevent_by_force":
# Keep Hy from being able to byte-compile the module by
# creating a directory at the target location.
os.mkdir(cache_from_source(fpath))
# Whether or not we can byte-compile the module, we should be able
# to run it.
output, _ = run_cmd(cmd, dontwritebytecode=(scenario == "prevent_by_env"))
assert "Hello from macro" in output
assert "The macro returned: boink" in output
if scenario == "normal":
# That should've byte-compiled the module.
assert os.path.exists(cache_from_source(fpath))
elif scenario == "prevent_by_env" or scenario == "prevent_by_option":
# No byte-compiled version should've been created.
assert not os.path.exists(cache_from_source(fpath))
# When we run the same command again, and we've byte-compiled the
# module, the byte-compiled version should be run instead of the
# source, in which case the macro shouldn't be run.
output, _ = run_cmd(cmd)
assert ("Hello from macro" in output) ^ (scenario == "normal")
assert "The macro returned: boink" in output
def cache_from_source(path, debug_override=None):
"""**DEPRECATED**
Given the path to a .py file, return the path to its .pyc/.pyo file.
The .py file does not need to exist; this simply returns the path to the
.pyc/.pyo file calculated as if the .py file were imported. The extension
will be .pyc unless sys.flags.optimize is non-zero, then it will be .pyo.
If debug_override is not None, then it must be a boolean and is used in
place of sys.flags.optimize.
If sys.implementation.cache_tag is None then NotImplementedError is raised.
"""
return util.cache_from_source(path, debug_override)
def cache_from_source(path, debug_override=None):
"""**DEPRECATED**
Given the path to a .py file, return the path to its .pyc/.pyo file.
The .py file does not need to exist; this simply returns the path to the
.pyc/.pyo file calculated as if the .py file were imported. The extension
will be .pyc unless sys.flags.optimize is non-zero, then it will be .pyo.
If debug_override is not None, then it must be a boolean and is used in
place of sys.flags.optimize.
If sys.implementation.cache_tag is None then NotImplementedError is raised.
"""
return util.cache_from_source(path, debug_override)
def load_header_and_prog(self) -> Tuple[Header, T]:
suffix = self.suffix()
searched = []
suffixes = suffix if isinstance(suffix, tuple) else (suffix, )
for suffix in suffixes:
src_file = self.file.with_suffix(suffix)
cache_file_path = cache_from_source(str(src_file.absolute()))
cache_file = Path(cache_file_path).with_suffix('.pie.pyc')
cache_file.parent.mkdir(exist_ok=True, parents=True)
cache = None
if cache_file.exists():
with cache_file.open('rb') as f:
cache_bytes = f.read()
header, prog_bytes = self.header_cls(
).get_header_and_prog_bytes(cache_bytes)
program = self.load_program(prog_bytes)
cache = header, program
if src_file.exists():
with src_file.open('rb') as f:
src = f.read()
need_recompile = True
if cache:
# verify cache
header, program = cache
if not header.is_out_of_date(src):
need_recompile = False
if need_recompile:
header = self.header_cls().from_source(src)
program = self.source_to_prog(src, src_file)
with cache_file.open('wb') as f:
contents = self.dump_program(program)
contents = header.add_header(contents)
f.write(contents)
else:
searched.append(str(src_file.absolute()))
continue
# noinspection PyUnboundLocalVariable
return header, program
raise FileNotFoundError('any of\n- {}'.format('\n- '.join(searched)))
def cache_from_source(path, debug_override=None):
"""**DEPRECATED**
Given the path to a .py file, return the path to its .pyc file.
The .py file does not need to exist; this simply returns the path to the
.pyc file calculated as if the .py file were imported.
If debug_override is not None, then it must be a boolean and is used in
place of sys.flags.optimize.
If sys.implementation.cache_tag is None then NotImplementedError is raised.
"""
with warnings.catch_warnings():
warnings.simplefilter('ignore')
return util.cache_from_source(path, debug_override)
def check_for_apyc(filename):
apyc_filename = cache_from_source(filename)
if isfile(apyc_filename):
with open(apyc_filename, "rb") as pyc:
pycmagic = pyc.read(4)
pycmodtime = unpack('=L', pyc.read(4))[0]
pycsrcsize = unpack('=L', pyc.read(4))[0]
srcstats = stat(filename)
# check magic number?
#print(int(srcstats.st_mtime), pycmodtime, srcstats.st_size, pycsrcsize, sep="\n")
return int(srcstats.st_mtime) == pycmodtime \
and srcstats.st_size == pycsrcsize
return False
def add(self, path: str) -> None:
head, tail = os.path.split(path)
# we normalize the head to resolve parent directory symlinks, but not
# the tail, since we only want to uninstall symlinks, not their targets
path = os.path.join(normalize_path(head), os.path.normcase(tail))
if not os.path.exists(path):
return
if self._permitted(path):
self.paths.add(path)
else:
self._refuse.add(path)
# __pycache__ files can show up after 'installed-files.txt' is created,
# due to imports
if os.path.splitext(path)[1] == '.py':
self.add(cache_from_source(path))
def test_local(self):
'''Load a local template *.py file'''
with tempfile.NamedTemporaryFile(prefix='answer_',
suffix='.py') as my_template:
my_template.write(b'''import dbus
BUS_NAME = 'universe.Ultimate'
MAIN_OBJ = '/'
MAIN_IFACE = 'universe.Ultimate'
SYSTEM_BUS = False
def load(mock, parameters):
mock.AddMethods(MAIN_IFACE, [('Answer', '', 'i', 'ret = 42')])
''')
my_template.flush()
(p_mock, dbus_ultimate) = self.spawn_server_template(
my_template.name, stdout=subprocess.PIPE)
self.addCleanup(p_mock.wait)
self.addCleanup(p_mock.terminate)
self.addCleanup(p_mock.stdout.close)
# ensure that we don't use/write any .pyc files, they are dangerous
# in a world-writable directory like /tmp
self.assertFalse(os.path.exists(my_template.name + 'c'))
try:
from importlib.util import cache_from_source
self.assertFalse(
os.path.exists(cache_from_source(my_template.name)))
except ImportError:
# python < 3.4
pass
self.assertEqual(dbus_ultimate.Answer(), 42)
# should appear in introspection
xml = dbus_ultimate.Introspect()
self.assertIn('<interface name="universe.Ultimate">', xml)
self.assertIn('<method name="Answer">', xml)
# should not have ObjectManager API by default
self.assertRaises(dbus.exceptions.DBusException,
dbus_ultimate.GetManagedObjects)
def write_apyc(filename, codeobj):
apyc_filename = cache_from_source(filename)
srcstats = stat(filename)
with open(apyc_filename, "wb") as pyc:
pyc.write(MAGIC_NUMBER)
# write last mod time
srcmtime = int(srcstats.st_mtime)
bsrcmtime = pack('=L', srcmtime)
pyc.write(bsrcmtime)
# write size
srcsize = srcstats.st_size
bsrcsize = pack('=L', srcsize)
pyc.write(bsrcsize)
# write bytecode
marshal.dump(codeobj, pyc)
def test_local(self):
'''Load a local template *.py file'''
with tempfile.NamedTemporaryFile(prefix='answer_', suffix='.py') as my_template:
my_template.write(b'''import dbus
BUS_NAME = 'universe.Ultimate'
MAIN_OBJ = '/'
MAIN_IFACE = 'universe.Ultimate'
SYSTEM_BUS = False
def load(mock, parameters):
mock.AddMethods(MAIN_IFACE, [('Answer', '', 'i', 'ret = 42')])
''')
my_template.flush()
(p_mock, dbus_ultimate) = self.spawn_server_template(
my_template.name, stdout=subprocess.PIPE)
self.addCleanup(p_mock.wait)
self.addCleanup(p_mock.terminate)
self.addCleanup(p_mock.stdout.close)
# ensure that we don't use/write any .pyc files, they are dangerous
# in a world-writable directory like /tmp
self.assertFalse(os.path.exists(my_template.name + 'c'))
try:
from importlib.util import cache_from_source
self.assertFalse(os.path.exists(cache_from_source(my_template.name)))
except ImportError:
# python < 3.4
pass
self.assertEqual(dbus_ultimate.Answer(), 42)
# should appear in introspection
xml = dbus_ultimate.Introspect()
self.assertIn('<interface name="universe.Ultimate">', xml)
self.assertIn('<method name="Answer">', xml)
# should not have ObjectManager API by default
self.assertRaises(dbus.exceptions.DBusException,
dbus_ultimate.GetManagedObjects)
def import_file(path):
""" Loads modules for a file (.py, .zip, .egg) """
directory, filename = os.path.split(path)
name, ext = os.path.splitext(filename)
names_to_import = []
tmp_python_path = None
if ext in (".py", ): # , '.pyc'):
if directory not in sys.path:
tmp_python_path = directory
names_to_import.append(name)
if ext == ".py": # Ensure that no pyc file will be reused
cache_file = cache_from_source(path)
with ignoring(OSError):
os.remove(cache_file)
if ext in (".egg", ".zip", ".pyz"):
if path not in sys.path:
sys.path.insert(0, path)
if sys.version_info >= (3, 6):
names = (mod_info.name
for mod_info in pkgutil.iter_modules([path]))
else:
names = (mod_info[1] for mod_info in pkgutil.iter_modules([path]))
names_to_import.extend(names)
loaded = []
if not names_to_import:
logger.warning("Found nothing to import from %s", filename)
else:
importlib.invalidate_caches()
if tmp_python_path is not None:
sys.path.insert(0, tmp_python_path)
try:
for name in names_to_import:
logger.info("Reload module %s from %s file", name, ext)
loaded.append(importlib.reload(importlib.import_module(name)))
finally:
if tmp_python_path is not None:
sys.path.remove(tmp_python_path)
return loaded
def reload_all(locals_=None, max_depth=5):
for k, v in sys.modules.items():
if not hasattr(v, '__file__'):
continue
elif v.__file__.startswith(SYS_PREFIX):
continue
elif k == '__main__':
continue
elif v.__file__ == __file__:
continue
if sys.version_info[0] == 2:
if re.match(".+\.pyc$", v.__file__):
py_file_path = v.__file__[:-1]
if os.path.exists(py_file_path):
if os.path.getmtime(py_file_path) > os.path.getmtime(
v.__file__):
print("reload %s" % (v.__file__))
reload(v)
reload_class_if_exist(v, locals_, max_depth)
elif re.match(".+\.py$", v.__file__):
pyc_file_path = v.__file__ + "c"
if os.path.exists(pyc_file_path):
if os.path.getmtime(
v.__file__) > os.path.getmtime(pyc_file_path):
print("reload %s" % (v.__file__))
reload(v)
reload_class_if_exist(v, locals_, max_depth)
elif sys.version_info[0] == 3:
if re.match(".+\.py$", v.__file__):
pyc_file_path = cache_from_source(v.__file__)
if os.path.exists(pyc_file_path):
if os.path.getmtime(
v.__file__) > os.path.getmtime(pyc_file_path):
print("reload %s" % (v.__file__))
reload(v)
reload_class_if_exist(v, locals_, max_depth)
def add(pth):
paths.add(normalize_path(pth))
if os.path.splitext(pth)[1] == '.py' and uses_pycache:
add(cache_from_source(pth))
def rewrite_file(self, contents):
compiled_path = cache_from_source(self.module_path)
if os.path.exists(compiled_path):
os.remove(compiled_path)
with open(self.module_path, 'w') as f:
f.write(contents)
from importlib.util import cache_from_source
import os.path
import py_compile
import sys
if len(sys.argv) != 3:
print('Usage: compile.py <module.py> <outdir>')
sys.exit(1)
src = sys.argv[1]
outdir = sys.argv[2]
# Compile src file, but store result in current directory.
cfile = cache_from_source(os.path.basename(src))
cfile = os.path.join(outdir, cfile)
py_compile.compile(src, cfile=cfile, doraise=True)
def test_cache_from_source_no_dot(self):
# Directory with a dot, filename without dot.
path = os.path.join('foo.bar', 'file')
expect = os.path.join('foo.bar', '__pycache__',
'file{}.pyc'.format(self.tag))
self.assertEqual(util.cache_from_source(path, True), expect)
def test_sep_altsep_and_sep_cache_from_source(self):
# Windows path and PEP 3147 where sep is right of altsep.
self.assertEqual(
util.cache_from_source('\\foo\\bar\\baz/qux.py', True),
'\\foo\\bar\\baz\\__pycache__\\qux.{}.pyc'.format(self.tag))
import sys
import warnings
from importlib import import_module
from importlib.util import cache_from_source
from pathlib import Path
import pytest
from typeguard.importhook import install_import_hook
this_dir = Path(__file__).parent
dummy_module_path = this_dir / 'dummymodule.py'
cached_module_path = Path(
cache_from_source(str(dummy_module_path), optimization='typeguard'))
@pytest.fixture(scope='module')
def dummymodule():
if cached_module_path.exists():
cached_module_path.unlink()
sys.path.insert(0, str(this_dir))
try:
with install_import_hook('dummymodule'):
with warnings.catch_warnings():
warnings.filterwarnings('error', module='typeguard')
module = import_module('dummymodule')
return module
finally:
sys.path.remove(str(this_dir))
def optimized_cache_from_source(path, debug_override=None):
return cache_from_source(path, debug_override, optimization='typeguard')
def test_cache_from_source_cwd(self):
path = 'foo.py'
expect = os.path.join('__pycache__', 'foo.{}.pyc'.format(self.tag))
self.assertEqual(util.cache_from_source(path, True), expect)
def test_cache_from_source_no_cache_tag(self):
# No cache tag means NotImplementedError.
with support.swap_attr(sys.implementation, 'cache_tag', None):
with self.assertRaises(NotImplementedError):
util.cache_from_source('whatever.py')
def recompile(filename):
"""Create a .pyc by disassembling the file and assembling it again, printing
a message that the reassembled file was loaded."""
# Most of the code here based on the compile.py module.
import os
import imp
import marshal
import struct
try:
from importlib.util import cache_from_source
except ImportError:
if python_version < '3':
def cache_from_source(filename):
return filename + 'c'
else:
def cache_from_source(filename):
dir, base = os.path.split(filename)
base, ext = os.path.splitext(base)
x, y = sys.version_info[:2]
p = os.path.join(dir, '__pycache__',
base + '.cpython-{0}{1}.pyc'.format(x, y))
return p
f = open(filename, 'U')
try:
st = os.fstat(f.fileno())
except AttributeError:
timestamp = os.stat(filename)
timestamp = int(st.st_mtime)
sourcesize = st.st_size
codestring = f.read()
f.close()
if codestring and codestring[-1] != '\n':
codestring = codestring + '\n'
try:
codeobject = compile(codestring, filename, 'exec', 0, 1)
except SyntaxError as e:
print("Skipping %s - syntax error: %s." % (filename, e),
file=sys.stderr)
return
cod = Code.from_code(codeobject)
message = "reassembled %r imported.\n" % filename
cod.code[:0] = [ # __import__('sys').stderr.write(message)
(LOAD_GLOBAL, '__import__'),
(LOAD_CONST, 'sys'),
(CALL_FUNCTION, 1),
(LOAD_ATTR, 'stderr'),
(LOAD_ATTR, 'write'),
(LOAD_CONST, message),
(CALL_FUNCTION, 1),
(POP_TOP, None),
]
try:
codeobject2 = cod.to_code()
except:
print(cod.code)
print('Failed to compile')
raise
fc = open(cache_from_source(filename), 'wb')
fc.write(b'\0\0\0\0')
fc.write(struct.pack('<l', timestamp))
if python_version >= '3':
fc.write(struct.pack('<l', sourcesize))
try:
marshal.dump(codeobject2, fc)
except:
print(cod.code)
print('Failed on {0}'.format(type(codeobject2)))
raise
fc.flush()
fc.seek(0, 0)
fc.write(imp.get_magic())
fc.close()