def iter_pth_paths(filename):
"""Given a .pth file, extract and yield all inner paths without honoring imports.
This shadows Python's site.py behavior, which is invoked at interpreter startup.
"""
try:
f = open(filename, "rU" if PY2 else "r") # noqa
except IOError:
return
dirname = os.path.dirname(filename)
known_paths = set()
with f:
for i, line in enumerate(f, start=1):
line = line.rstrip()
if not line or line.startswith("#"):
continue
elif line.startswith(("import ", "import\t")):
# One important side effect of executing import lines can be alteration of the
# sys.path directly or indirectly as a programmatic way to add sys.path entries
# in contrast to the standard .pth mechanism of including fixed paths as
# individual lines in the file. Here we capture all such programmatic attempts
# to expand the sys.path and report the additions.
original_sys_path = sys.path[:]
try:
# N.B.: Setting sys.path to empty is ok since all the .pth files we find and
# execute have already been found and executed by our ambient sys.executable
# when it started up before running this PEX file. As such, all symbols imported
# by the .pth files then will still be available now as cached in sys.modules.
sys.path = []
exec_function(line, globals_map={})
for path in sys.path:
yield path
except Exception as e:
# NB: import lines are routinely abused with extra code appended using `;` so
# the class of exceptions that might be raised in broader than ImportError. As
# such we catch broadly here.
TRACER.log(
"Error executing line {linenumber} of {pth_file} with content:\n"
"{content}\n"
"Error was:\n"
"{error}".format(linenumber=i,
pth_file=filename,
content=line,
error=e),
V=9,
)
# Defer error handling to the higher level site.py logic invoked at startup.
return
finally:
sys.path = original_sys_path
else:
extras_dir, extras_dir_case_insensitive = makepath(
dirname, line)
if extras_dir_case_insensitive not in known_paths and os.path.exists(
extras_dir):
yield extras_dir
known_paths.add(extras_dir_case_insensitive)
def readdpackage(sitedir, name):
"""Process a .pth file within the site-packages directory:
For each line in the file, if it doesnt start with 'import ', combine it with sitedir to a path
and remove that from sys.path.
"""
# All this code is inspired from site.addpackage
fullname = os.path.join(sitedir, name)
try:
f = open(fullname, "rU")
except IOError:
return
with f:
for n, line in enumerate(f):
if line.startswith("#") or line.startswith(
("import ", "import\t")):
continue
try:
line = line.rstrip()
dir, dircase = site.makepath(sitedir, line)
# If the path is already there, we remove it (to reinsert it in the proper place)
print("Re-adding {dir} in front of sys.path".format(
**locals()))
if dir in sys.path:
sys.path.remove(dir)
sys.path.insert(1, dir)
except Exception as err:
print("Error processing line {:d} of {}:\n".format(
n + 1, fullname),
file=sys.stderr)
for record in traceback.format_exception(*sys.exc_info()):
for line in record.splitlines():
print(' ' + line, file=sys.stderr)
print("\nRemainder of file ignored", file=sys.stderr)
break
def _main():
import os
import site
import sys
from distutils.sysconfig import get_python_lib
def K(n, *args):
return args[n]
paths = os.environ.get('PYTHONHOMESITE')
if paths is None:
paths = []
else:
paths = paths.split(os.pathsep)
known_paths = set()
for path in sys.path:
try:
if os.path.exists(path):
known_paths.add(K(1, *site.makepath(path)))
except TypeError:
continue
for p in paths:
for plat_specific in (False, True):
np = get_python_lib(plat_specific=plat_specific, prefix=p)
if os.path.isdir(np):
site.addsitedir(np, known_paths)
def iter_pth_paths(filename):
"""Given a .pth file, extract and yield all inner paths without honoring imports. This shadows
python's site.py behavior, which is invoked at interpreter startup."""
try:
f = open(filename, 'rU') # noqa
except IOError:
return
dirname = os.path.dirname(filename)
known_paths = set()
with f:
for line in f:
line = line.rstrip()
if not line or line.startswith('#'):
continue
elif line.startswith(('import ', 'import\t')):
try:
exec_function(line)
continue
except Exception:
# Defer error handling to the higher level site.py logic invoked at startup.
return
else:
extras_dir, extras_dir_case_insensitive = makepath(
dirname, line)
if extras_dir_case_insensitive not in known_paths and os.path.exists(
extras_dir):
yield extras_dir
known_paths.add(extras_dir_case_insensitive)
def test_init_pathinfo(self):
dir_set = site._init_pathinfo()
for entry in [site.makepath(path)[1] for path in sys.path
if path and os.path.isdir(path)]:
self.assertIn(entry, dir_set,
"%s from sys.path not found in set returned "
"by _init_pathinfo(): %s" % (entry, dir_set))
def iter_pth_paths(filename):
"""Given a .pth file, extract and yield all inner paths without honoring imports. This shadows
python's site.py behavior, which is invoked at interpreter startup."""
try:
f = open(filename, 'rU') # noqa
except IOError:
return
dirname = os.path.dirname(filename)
known_paths = set()
with f:
for line in f:
line = line.rstrip()
if not line or line.startswith('#'):
continue
elif line.startswith(('import ', 'import\t')):
try:
exec_function(line, globals_map={})
continue
except Exception:
# NB: import lines are routinely abused with extra code appended using `;` so the class of
# exceptions that might be raised in broader than ImportError. As such we cacth broadly
# here.
# Defer error handling to the higher level site.py logic invoked at startup.
return
else:
extras_dir, extras_dir_case_insensitive = makepath(dirname, line)
if extras_dir_case_insensitive not in known_paths and os.path.exists(extras_dir):
yield extras_dir
known_paths.add(extras_dir_case_insensitive)
def iter_pth_paths(filename):
"""Given a .pth file, extract and yield all inner paths without honoring imports. This shadows
python's site.py behavior, which is invoked at interpreter startup."""
try:
f = open(filename, 'rU') # noqa
except IOError:
return
dirname = os.path.dirname(filename)
known_paths = set()
with f:
for line in f:
line = line.rstrip()
if not line or line.startswith('#'):
continue
elif line.startswith(('import ', 'import\t')):
try:
exec_function(line)
continue
except Exception:
# Defer error handling to the higher level site.py logic invoked at startup.
return
else:
extras_dir, extras_dir_case_insensitive = makepath(dirname, line)
if extras_dir_case_insensitive not in known_paths and os.path.exists(extras_dir):
yield extras_dir
known_paths.add(extras_dir_case_insensitive)
def iter_pth_paths(filename):
"""Given a .pth file, extract and yield all inner paths without honoring imports. This shadows
python's site.py behavior, which is invoked at interpreter startup."""
try:
f = open(filename, 'rU') # noqa
except IOError:
return
dirname = os.path.dirname(filename)
known_paths = set()
with f:
for line in f:
line = line.rstrip()
if not line or line.startswith('#'):
continue
elif line.startswith(('import ', 'import\t')):
try:
exec_function(line, globals_map={})
continue
except Exception:
# NB: import lines are routinely abused with extra code appended using `;` so the class of
# exceptions that might be raised in broader than ImportError. As such we cacth broadly
# here.
# Defer error handling to the higher level site.py logic invoked at startup.
return
else:
extras_dir, extras_dir_case_insensitive = makepath(
dirname, line)
if extras_dir_case_insensitive not in known_paths and os.path.exists(
extras_dir):
yield extras_dir
known_paths.add(extras_dir_case_insensitive)
def test_init_pathinfo(self):
dir_set = site._init_pathinfo()
for entry in [site.makepath(path)[1] for path in sys.path
if path and os.path.isdir(path)]:
self.assertIn(entry, dir_set,
"%s from sys.path not found in set returned "
"by _init_pathinfo(): %s" % (entry, dir_set))
def test_makepath(self):
path_parts = 'Beginning', 'End'
original_dir = os.path.join(*path_parts)
abs_dir, norm_dir = site.makepath(*path_parts)
self.assertEqual(os.path.abspath(original_dir), abs_dir)
if original_dir == os.path.normcase(original_dir):
self.assertEqual(abs_dir, norm_dir)
else:
self.assertEqual(os.path.normcase(abs_dir), norm_dir)
def test_makepath(self):
# Test makepath() have an absolute path for its first return value
# and a case-normalized version of the absolute path for its
# second value.
path_parts = ("Beginning", "End")
original_dir = os.path.join(*path_parts)
abs_dir, norm_dir = site.makepath(*path_parts)
self.assertEqual(os.path.abspath(original_dir), abs_dir)
if original_dir == os.path.normcase(original_dir):
self.assertEqual(abs_dir, norm_dir)
else:
self.assertEqual(os.path.normcase(abs_dir), norm_dir)
def test_makepath(self):
# Test makepath() have an absolute path for its first return value
# and a case-normalized version of the absolute path for its
# second value.
path_parts = ("Beginning", "End")
original_dir = os.path.join(*path_parts)
abs_dir, norm_dir = site.makepath(*path_parts)
self.assertEqual(os.path.abspath(original_dir), abs_dir)
if original_dir == os.path.normcase(original_dir):
self.assertEqual(abs_dir, norm_dir)
else:
self.assertEqual(os.path.normcase(abs_dir), norm_dir)
def _init_pathinfo(syspath):
"""
Return a set containing all existing directory entries from C{syspath}.
@param syspath: A list of filesystem path strings to directories containing
Python packages and modules.
"""
d = set()
for dir in syspath:
try:
if os.path.isdir(dir):
dir, dircase = makepath(dir)
d.add(dircase)
except TypeError:
continue
return d
def readdsitedir(sitedir):
"""Add 'sitedir' argument to sys.path if missing and handle .pth files in
'sitedir'"""
sitedir, sitedircase = site.makepath(sitedir)
print("Re-adding {sitedir} in front of sys.path".format(**locals()))
if sitedir in sys.path:
sys.path.remove(sitedir) # Remove path component
sys.path.insert(1, sitedir) # Insert path component in front
try:
names = os.listdir(sitedir)
except os.error:
return
dotpth = os.extsep + "pth"
names = [name for name in names if name.endswith(dotpth)]
for name in sorted(names):
readdpackage(sitedir, name)
return
def addsitedir(sitedir, syspath):
"""
Add C{sitedir} argument to C{syspath} argument if missing and handle .pth
files in C{sitedir}.
@return: A list of filesystem path strings.
"""
known_paths = _init_pathinfo(syspath)
reset = 1
sitedir, sitedircase = makepath(sitedir)
if not sitedircase in known_paths:
syspath.append(sitedir) # Add path component
names = os.listdir(sitedir)
names.sort()
for name in names:
if name.endswith(os.extsep + "pth"):
addpackage(sitedir, name, known_paths, syspath)
if reset:
known_paths = None
return known_paths
def addsitedir(sitedir, known_paths=None, prepend=False):
if known_paths is None:
known_paths = site._init_pathinfo()
reset = True
else:
reset = False
sitedir, sitedircase = site.makepath(sitedir)
if sitedircase not in known_paths:
_add_to_syspath(sitedir, prepend)
known_paths.add(sitedircase)
try:
names = os.listdir(sitedir)
except OSError:
return
names = [name for name in names if name.endswith(".pth")]
for name in sorted(names):
_addpackage(sitedir, name, known_paths, prepend)
if reset:
known_paths = None
return known_paths
def _addpackage(sitedir, name, known_paths, prepend):
if known_paths is None:
known_paths = site._init_pathinfo()
reset = True
else:
reset = False
fullname = os.path.join(sitedir, name)
try:
f = open(fullname, "rb")
except OSError:
return
with f:
for n, line in enumerate(f):
line = line.decode()
if line.startswith("#"):
continue
try:
if line.startswith(("import ", "import\t")):
exec(line)
continue
line = line.rstrip()
directory, dircase = site.makepath(sitedir, line)
if dircase not in known_paths and os.path.exists(directory):
_add_to_syspath(directory, prepend)
known_paths.add(dircase)
except Exception:
sys.stderr.write("Error processing line {:d} of {}:\n".format(
n + 1, fullname))
import traceback
for record in traceback.format_exception(*sys.exc_info()):
for line2 in record.splitlines():
sys.stderr.write(" " + line2 + "\n")
sys.stderr.write("\nRemainder of file ignored\n")
break
if reset:
known_paths = None
return known_paths
def addpackage(sitedir, name, known_paths, syspath):
"""
Add a new path to C{known_paths} by combining C{sitedir} and C{name} or
execute C{sitedir} if it starts with 'import'.
@return: C{known_paths}, or None if the given new path does not exist.
"""
fullname = os.path.join(sitedir, name)
f = open(fullname, "rU")
try:
for line in f:
if line.startswith("#"):
continue
if line.startswith("import"):
exec line
continue
line = line.rstrip()
dir, dircase = makepath(sitedir, line)
if not dircase in known_paths and os.path.exists(dir):
syspath.append(dir)
known_paths.add(dircase)
finally:
f.close()
return known_paths
def pth_file_tests(self, pth_file):
"""Contain common code for testing results of reading a .pth file"""
self.assertIn(pth_file.imported, sys.modules,
"%s not in sys.modules" % pth_file.imported)
self.assertIn(site.makepath(pth_file.good_dir_path)[0], sys.path)
self.assertFalse(os.path.exists(pth_file.bad_dir_path))
def pth_file_tests(self, pth_file):
"""Contain common code for testing results of reading a .pth file"""
self.failUnless(pth_file.imported in sys.modules,
"%s not in sys.path" % pth_file.imported)
self.failUnless(site.makepath(pth_file.good_dir_path)[0] in sys.path)
self.failUnless(not os.path.exists(pth_file.bad_dir_path))
def update_event(self, inp=-1):
self.set_output_val(0, site.makepath())
if not sitedir.endswith('site-packages'):
continue
# find pth files
try:
names = os.listdir(sitedir)
except os.error:
continue
dotpth = os.extsep + "pth"
pths = [name for name in names if name.endswith(dotpth)]
for pth in pths:
fullname = os.path.join(sitedir, pth)
try:
f = open(fullname, "rU")
except IOError:
continue
with f:
for n, line in enumerate(f):
if line.startswith("#"):
continue
if line.startswith(("import ", "import\t")):
continue
line = line.rstrip()
dir, dircase = site.makepath(sitedir, line)
if not dircase in sys.path:
sys.path.insert(path_idx + 1, dir)
