def compile_src_string_to_pyc_string(src, filename, python_exe, mode="exec"):
"""Compile Python source code to pyc data.
This may use py_compile if the src is for the same version as we're running,
or else it spawns an external process to produce a .pyc file. The generated
bytecode (.pyc file) is read and both it and any temporary files are deleted.
Args:
src: Python sourcecode
filename: Name of the source file. For error messages.
python_exe: Tuple of a path to a Python interpreter and command-line flags.
mode: Same as __builtin__.compile: "exec" if source consists of a
sequence of statements, "eval" if it consists of a single expression,
or "single" if it consists of a single interactive statement.
Returns:
The compiled pyc file as a binary string.
Raises:
CompileError: If we find a syntax error in the file.
IOError: If our compile script failed.
"""
tempfile_options = {"mode": "w", "suffix": ".py", "delete": False}
if six.PY3:
tempfile_options.update({"encoding": "utf-8"})
else:
tempfile_options.update({"mode": "wb"})
fi = tempfile.NamedTemporaryFile(**tempfile_options)
try:
if six.PY3:
fi.write(src)
else:
fi.write(src.encode("utf-8"))
fi.close()
# In order to be able to compile pyc files for both Python 2 and Python 3,
# we spawn an external process.
# We pass -E to ignore the environment so that PYTHONPATH and sitecustomize
# on some people's systems don't mess with the interpreter.
exe, flags = python_exe
cmd = [exe] + flags + ["-E", "-", fi.name, filename or fi.name, mode]
compile_script_src = pytype_source_utils.load_pytype_file(
COMPILE_SCRIPT)
p = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
bytecode, _ = p.communicate(compile_script_src)
assert p.poll() == 0, "Child process failed"
finally:
os.unlink(fi.name)
first_byte = six.indexbytes(bytecode, 0)
if first_byte == 0: # compile OK
return bytecode[1:]
elif first_byte == 1: # compile error
code = bytecode[1:] # type: bytes
raise CompileError(compat.native_str(code))
else:
raise IOError("_compile.py produced invalid result")
def _load_file(self, path):
if self._env_home:
filename = os.path.join(self._env_home, path)
with open(filename, "rb") as f:
return filename, f.read()
else:
filepath = os.path.join(self._root, path)
data = pytype_source_utils.load_pytype_file(filepath)
return filepath, data
def read_blacklist(self):
"""Read the typeshed blacklist."""
if self._env_home:
raise NotImplementedError(
"Can't read blacklist outside ./typeshed")
data = pytype_source_utils.load_pytype_file(
"typeshed/tests/pytype_blacklist.txt")
# |data| is raw byte data.
for line in data.splitlines():
line = line.decode("utf-8")
line = line[:line.find("#")].strip()
if line:
yield line
def GetPredefinedFile(pytd_subdir, module, extension=".pytd",
as_package=False):
"""Get the contents of a predefined PyTD, typically with a file name *.pytd.
Arguments:
pytd_subdir: the directory, typically "builtins" or "stdlib"
module: module name (e.g., "sys" or "__builtins__")
extension: either ".pytd" or ".py"
as_package: try the module as a directory with an __init__ file
Returns:
The contents of the file
Raises:
IOError: if file not found
"""
parts = module.split(".")
if as_package:
parts.append("__init__")
mod_path = os.path.join(*parts) + extension
path = os.path.join("pytd", pytd_subdir, mod_path)
return path, pytype_source_utils.load_pytype_file(path)
def compile_src_string_to_pyc_string(src,
filename,
python_version,
python_exe,
mode="exec"):
"""Compile Python source code to pyc data.
This may use py_compile if the src is for the same version as we're running,
or else it spawns an external process to produce a .pyc file. The generated
bytecode (.pyc file) is read and both it and any temporary files are deleted.
Args:
src: Python sourcecode
filename: Name of the source file. For error messages.
python_version: Python version, (major, minor).
python_exe: Tuple of a path to a Python interpreter and command-line flags.
mode: Same as __builtin__.compile: "exec" if source consists of a
sequence of statements, "eval" if it consists of a single expression,
or "single" if it consists of a single interactive statement.
Returns:
The compiled pyc file as a binary string.
Raises:
CompileError: If we find a syntax error in the file.
IOError: If our compile script failed.
"""
if python_version == sys.version_info[:2] and (
sys.version_info.major != 2 or not utils.USE_ANNOTATIONS_BACKPORT):
# Optimization: calling compile_bytecode directly is faster than spawning a
# subprocess. We can do this only when the host and target versions match
# and we don't need the patched 2.7 interpreter.
output = six.BytesIO()
compile_bytecode.compile_src_to_pyc(src, filename or "<>", output,
mode)
bytecode = output.getvalue()
else:
tempfile_options = {"mode": "w", "suffix": ".py", "delete": False}
if six.PY3:
tempfile_options.update({"encoding": "utf-8"})
else:
tempfile_options.update({"mode": "wb"})
fi = tempfile.NamedTemporaryFile(**tempfile_options)
try:
if six.PY3:
fi.write(src)
else:
fi.write(src.encode("utf-8"))
fi.close()
# In order to be able to compile pyc files for a different Python version
# from the one we're running under, we spawn an external process.
# We pass -E to ignore the environment so that PYTHONPATH and
# sitecustomize on some people's systems don't mess with the interpreter.
exe, flags = python_exe
cmd = [exe
] + flags + ["-E", "-", fi.name, filename or fi.name, mode]
compile_script_src = pytype_source_utils.load_pytype_file(
COMPILE_SCRIPT)
p = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
bytecode, _ = p.communicate(compile_script_src)
assert p.poll() == 0, "Child process failed"
finally:
os.unlink(fi.name)
first_byte = six.indexbytes(bytecode, 0)
if first_byte == 0: # compile OK
return bytecode[1:]
elif first_byte == 1: # compile error
code = bytecode[1:] # type: bytes
raise CompileError(compat.native_str(code))
else:
raise IOError("_compile.py produced invalid result")
def compile_src_string_to_pyc_string(src,
filename,
python_version,
python_exe,
mode="exec"):
"""Compile Python source code to pyc data.
This may use py_compile if the src is for the same version as we're running,
or else it spawns an external process to produce a .pyc file. The generated
bytecode (.pyc file) is read and both it and any temporary files are deleted.
Args:
src: Python sourcecode
filename: Name of the source file. For error messages.
python_version: Python version, (major, minor). E.g. (2, 7). Will be used
to determine the Python executable to call.
python_exe: Path to a Python interpreter, or None. If this is
None, the system "pythonX.X" interpreter will be used.
mode: Same as __builtin__.compile: "exec" if source consists of a
sequence of statements, "eval" if it consists of a single expression,
or "single" if it consists of a single interactive statement.
Returns:
The compiled pyc file as a binary string.
Raises:
CompileError: If we find a syntax error in the file.
IOError: If our compile script failed.
"""
tempfile_options = {"mode": "w", "suffix": ".py", "delete": False}
if six.PY3:
tempfile_options.update({"encoding": "utf-8"})
fi = tempfile.NamedTemporaryFile(**tempfile_options)
try:
fi.write(src)
fi.close()
# In order to be able to compile pyc files for both Python 2 and Python 3,
# we spawn an external process.
if python_exe:
# Allow python_exe to contain parameters (E.g. "-T")
exe = python_exe.split()
else:
exe = ["python" + ".".join(map(str, python_version))]
# We pass -E to ignore the environment so that PYTHONPATH and sitecustomize
# on some people's systems don't mess with the interpreter.
cmd = exe + ["-E", "-", fi.name, filename or fi.name, mode]
compile_script_src = pytype_source_utils.load_pytype_file(
COMPILE_SCRIPT)
p = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
bytecode, _ = p.communicate(compile_script_src)
assert p.poll() == 0, "Child process failed"
finally:
os.unlink(fi.name)
first_byte = six.indexbytes(bytecode, 0)
if first_byte == 0: # compile OK
return bytecode[1:]
elif first_byte == 1: # compile error
raise CompileError(bytecode[1:].decode("utf-8"))
else:
raise IOError("_compile.py produced invalid result")
