def run_in_syspy(f):
"""
Decorator to run a function in the system python
:param f:
:return:
"""
fname = f.__name__
code_lines = inspect.getsource(f).splitlines()
code = dedent("\n".join(code_lines[1:])) # strip this decorator
# add call to the function and print it's result
code += dedent("""\n
import sys
args = sys.argv[1:]
result = {fname}(*args)
print("%r" % result)
""").format(fname=fname)
env = os.environ
python = findsyspy()
logger.debug("Create function for system python\n%s" % code)
def call_f(*args):
cmd = [python, '-c', code] + list(args)
output = subprocess.check_output(cmd, env=env).decode('utf-8')
result = ast.literal_eval(output)
return result
return call_f
def load_module(self, name):
"""
Only lets modules in allowed_modules be loaded, others
will get an ImportError
"""
# Get the name relative to SITEDIR ..
filepath = self.module_info[1]
fullname = splitext( \
relpath(filepath, self.sitedir) \
)[0].replace(os.sep, '.')
modulename = filename_to_module(fullname)
if modulename not in allowed_modules:
if remember_blocks:
blocked_imports.add(fullname)
if log_blocks:
raise ImportError("Vext blocked import of '%s'" % modulename)
else:
# Standard error message
raise ImportError("No module named %s" % modulename)
if name not in sys.modules:
try:
logger.debug("load_module %s %s", name, self.module_info)
module = imp.load_module(name, *self.module_info)
except Exception as e:
logger.debug(e)
raise
sys.modules[fullname] = module
return sys.modules[fullname]
def check_sysdeps(vext_files):
"""
Check that imports in 'test_imports' succeed
otherwise display message in 'install_hints'
"""
@run_in_syspy
def run(*modules):
result = {}
for m in modules:
if m:
try:
__import__(m)
result[m] = True
except ImportError:
result[m] = False
return result
success = True
for vext_file in vext_files:
with open(vext_file) as f:
vext = open_spec(f)
install_hint = " ".join(vext.get('install_hints', ['System dependencies not found']))
modules = vext.get('test_import', '')
logger.debug("%s test imports of: %s", vext_file, modules)
result = run(*modules)
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
for k, v in result.items():
logger.debug("%s: %s", k, v)
if not all(result.values()):
success = False
print(install_hint)
return success
def load_module(self, name):
"""
Only lets modules in allowed_modules be loaded, others
will get an ImportError
"""
# Get the name relative to SITEDIR ..
filepath = self.module_info[1]
fullname = splitext( \
relpath(filepath, self.sitedir) \
)[0].replace(os.sep, '.')
modulename = filename_to_module(fullname)
if modulename not in allowed_modules:
if remember_blocks:
blocked_imports.add(fullname)
if log_blocks:
raise ImportError("Vext blocked import of '%s'" % modulename)
else:
# Standard error message
raise ImportError("No module named %s" % modulename)
if name not in sys.modules:
try:
logger.debug("load_module %s %s", name, self.module_info)
module = imp.load_module(name, *self.module_info)
except Exception as e:
logger.debug(e)
raise
sys.modules[fullname] = module
return sys.modules[fullname]
def check_sysdeps(vext_files):
"""
Check that imports in 'test_imports' succeed
otherwise display message in 'install_hints'
"""
@run_in_syspy
def run(*modules):
result = {}
for m in modules:
if m:
try:
__import__(m)
result[m] = True
except ImportError:
result[m] = False
return result
success = True
for vext_file in vext_files:
with open(vext_file) as f:
vext = open_spec(f)
install_hint = " ".join(vext.get('install_hints', ['System dependencies not found']))
modules = vext.get('test_import', '')
logger.debug("%s test imports of: %s", vext_file, modules)
result = run(*modules)
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
for k, v in result.items():
logger.debug("%s: %s", k, v)
if not all(result.values()):
success = False
print(install_hint)
return success
def __init__(self, path_entry):
self.path_entry = path_entry
sitedir = getsyssitepackages()
if path_entry in (sitedir, GatekeeperFinder.PATH_TRIGGER):
logger.debug("handle entry %s", path_entry)
return
else:
logger.debug("not handling entry %s", path_entry)
raise ImportError()
def __init__(self, path_entry):
self.path_entry = path_entry
sitedir = getsyssitepackages()
if path_entry in (sitedir, GatekeeperFinder.PATH_TRIGGER):
logger.debug("handle entry %s", path_entry)
return
else:
logger.debug("not handling entry %s", path_entry)
raise ImportError()
def in_venv():
"""
:return: True if in running from a virtualenv
Has to detect the case where the python binary is run
directly, so VIRTUAL_ENV may not be set
"""
global _in_venv
if _in_venv is not None:
return _in_venv
if not (os.path.isfile(ORIG_PREFIX_TXT) or os.path.isfile(PY_VENV_CFG)):
logger.debug("in_venv no orig_prefix_txt [%s]", ORIG_PREFIX_TXT)
logger.debug("in_venv no py_venv_cfg [%s]", PY_VENV_CFG)
# TODO - check this is actually valid !
_in_venv = False
return _in_venv
if 'VIRTUAL_ENV' in os.environ:
logger.debug("in_venv VIRTUAL_ENV set.")
_in_venv = True
else:
# Find first python in path ... if its not this one,
# ...we are in a different python
python = basename(sys.executable)
for p in os.environ['PATH'].split(os.pathsep):
py_path = join(p, python)
if isfile(py_path):
logger.debug("in_venv py_at [%s] return: %s", (py_path, sys.executable != py_path))
_in_venv = sys.executable != py_path
break
return _in_venv
def __init__(self, path_entry):
self.path_entry = path_entry
try:
# Wrap in exception handler in case something in a .pth file causes an exception.
sitedir = getsyssitepackages()
except Exception as e:
sys.stderr.write(
"Vext disabled: There was an issue getting the system site packages.\n"
)
raise ImportError()
if path_entry in (sitedir, GatekeeperFinder.PATH_TRIGGER):
logger.debug("handle entry %s", path_entry)
return
else:
logger.debug("not handling entry %s", path_entry)
raise ImportError()
def getsyssitepackages():
"""
:return: list of site-packages from system python
"""
global _syssitepackages
if not _syssitepackages:
if not in_venv():
_syssitepackages = get_python_lib()
return _syssitepackages
@run_in_syspy
def run(*args):
import site
return site.getsitepackages()
output = run()
_syssitepackages = output
logger.debug("system site packages: %s", _syssitepackages)
return _syssitepackages
def getsyssitepackages():
"""
:return: list of site-packages from system python
"""
global _syssitepackages
if not _syssitepackages:
if not in_venv():
_syssitepackages = get_python_lib()
return _syssitepackages
@run_in_syspy
def run(*args):
import site
return site.getsitepackages()
output = run()
_syssitepackages = output
logger.debug("system site packages: %s", _syssitepackages)
return _syssitepackages
def in_venv():
"""
:return: True if in running from a virtualenv
Has to detect the case where the python binary is run
directly, so VIRTUAL_ENV may not be set
"""
global _in_venv
if _in_venv is not None:
return _in_venv
if not os.path.isfile(orig_prefix):
logger.debug("in_venv no orig_prefix [%s]", orig_prefix)
# TODO - check this is actually valid !
_in_venv = False
return _in_venv
if 'VIRTUAL_ENV' in os.environ:
logger.debug("in_venv VIRTUAL_ENV set.")
_in_venv = True
else:
# Find first python in path ... if its not this one,
# ...we are in a different python
python = basename(sys.executable)
for p in os.environ['PATH'].split(os.pathsep):
py_path = join(p, python)
if isfile(py_path):
logger.debug("in_venv py_at [%s] return: %s", (py_path, sys.executable != py_path))
_in_venv = sys.executable != py_path
break
return _in_venv
def install_vexts(vext_files, verify=True):
"""
copy vext_file to sys.prefix + '/share/vext/specs'
(PIP7 seems to remove data_files so we recreate something similar here)
"""
if verify and not check_sysdeps(vext_files):
return
spec_dir = join(prefix, 'share/vext/specs')
try:
makedirs(spec_dir)
except OSError:
pass
for vext_file in vext_files:
dest = normpath(join(spec_dir, basename(vext_file)))
try:
logger.debug("%s > %s" % (vext_file, dest))
copyfile(vext_file, dest)
yield vext_file, dest
except IOError as e:
logger.error("Could not copy %s %r" % (vext_file, e))
def install_vexts(vext_files, verify=True):
"""
copy vext_file to sys.prefix + '/share/vext/specs'
(PIP7 seems to remove data_files so we recreate something similar here)
"""
if verify and not check_sysdeps(vext_files):
return
spec_dir = join(prefix, 'share/vext/specs')
try:
makedirs(spec_dir)
except OSError as e:
if not isdir(spec_dir):
logger.error("Error making spec directory [%s]: %r" % (spec_dir, e))
for vext_file in vext_files:
dest = normpath(join(spec_dir, basename(vext_file)))
try:
logger.debug("%s > %s" % (vext_file, dest))
copyfile(vext_file, dest)
yield vext_file, dest
except IOError as e:
logger.error("Could not copy %s %r" % (vext_file, e))
def load_specs():
bad_specs = set()
last_error = None
for fn in spec_files():
logger.debug("load spec: %s", fn)
if fn in bad_specs:
# Don't try and load the same bad spec twice
continue
try:
spec = open_spec(open(fn))
for module in spec['modules']:
logger.debug("allow module: %s", module)
allowed_modules.add(module)
for path_name in spec.get('extra_paths', []):
extra_path = get_extra_path(path_name)
if isdir(extra_path):
os.environ['PATH'] += env_t(os.pathsep + extra_path)
sys.path.append(extra_path)
added_dirs.add(extra_path)
else:
logger.warn("Skipped adding nonexistant path: {0}".format(
extra_path))
sys_sitedirs = getsyssitepackages()
for sys_sitedir in sys_sitedirs:
with fixup_paths():
for pth in [pth for pth in spec['pths'] or [] if pth]:
try:
logger.debug("open pth: %s", pth)
pth_file = join(sys_sitedir, pth)
addpackage(sys_sitedir, pth_file, added_dirs)
init_path() # TODO
except IOError as e:
# Path files are optional..
logging.debug('No pth found at %s', pth_file)
pass
except Exception as e:
bad_specs.add(fn)
err_msg = 'error loading spec %s: %s' % (fn, e)
if last_error != err_msg:
logging.error(err_msg)
last_error = err_msg
if bad_specs:
raise VextError('Error loading spec files: %s' % ', '.join(bad_specs))
def load_specs():
bad_specs = set()
last_error = None
for fn in spec_files():
logger.debug("load spec: %s", fn)
if fn in bad_specs:
# Don't try and load the same bad spec twice
continue
try:
spec = open_spec(open(fn))
for module in spec['modules']:
logger.debug("allow module: %s", module)
allowed_modules.add(module)
for path_name in spec.get('extra_paths', []):
extra_path = get_extra_path(path_name)
if isdir(extra_path):
os.environ['PATH'] += env_t(os.pathsep + extra_path)
sys.path.append(extra_path)
added_dirs.add(extra_path)
else:
logger.warn("Could not add extra path: {0}".format(extra_path))
sys_sitedirs = getsyssitepackages()
for sys_sitedir in sys_sitedirs:
for pth in [pth for pth in spec['pths'] or [] if pth]:
try:
logger.debug("open pth: %s", pth)
pth_file = join(sys_sitedir, pth)
addpackage(sys_sitedir, pth_file, added_dirs)
init_path() # TODO
except IOError as e:
# Path files are optional..
logging.debug('No pth found at %s', pth_file)
pass
except Exception as e:
bad_specs.add(fn)
err_msg = 'error loading spec %s: %s' % (fn, e)
if last_error != err_msg:
logging.error(err_msg)
last_error = err_msg
if bad_specs:
raise VextError('Error loading spec files: %s' % ', '.join(bad_specs))
def find_module(self, fullname, path=None):
# TODO Need lots of unit tests around this
module = sys.modules.get(fullname)
if module and hasattr(module, "load_module"):
# After reload module can be missing load_module
return module
sitedirs = getsyssitepackages()
# Check paths other than system sitepackages first
other_paths = [
p for p in sys.path
if p not in sitedirs + [GatekeeperFinder.PATH_TRIGGER, '.']
]
try:
for other_path in other_paths:
try:
module_info = imp.find_module(fullname, [other_path])
if module_info:
logger.debug("found module %s in other path [%s]",
fullname, other_path)
return
except ImportError:
continue
else:
raise ImportError()
except ImportError:
try:
# Now check if in site packages and needs gatekeeping
for sitedir in sitedirs:
try:
module_info = imp.find_module(
fullname, [sitedir, self.path_entry])
if module_info:
logger.debug(
"found module %s in sitedir or subdirectory [%s]",
fullname, sitedir)
return GateKeeperLoader(module_info, sitedir)
except ImportError:
logger.debug("%s not found in: %s", fullname,
os.pathsep.join(other_paths))
continue
except ImportError:
### TODO
# Need to debug why this catch is needed, removing it stops things working...
# something is subtly weird here.
return
def installed_packages(self):
""" :return: list of installed packages """
packages = []
CMDLINE = [sys.executable, "-mpip", "freeze"]
try:
for package in subprocess.check_output(CMDLINE) \
.decode('utf-8'). \
splitlines():
for comparator in ["==", ">=", "<=", "<", ">"]:
if comparator in package:
# installed package names usually look like Pillow==2.8.1
# ignore others, like external packages that pip show
# won't understand
name = package.partition(comparator)[0]
packages.append(name)
except RuntimeError as e:
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Exception checking existing packages.")
logger.debug("cmdline: %s", CMDLINE)
ex_type, ex, tb = sys.exc_info()
traceback.print_tb(tb)
logger.debug()
return packages
def find_module(self, fullname, path=None):
# TODO Need lots of unit tests around this
if fullname in sys.modules:
return sys.modules[fullname]
sitedirs = getsyssitepackages()
# Check paths other than system sitepackages first
other_paths = [p for p in sys.path if p not in sitedirs + [GatekeeperFinder.PATH_TRIGGER, '.']]
try:
for other_path in other_paths:
try:
module_info = imp.find_module(fullname, [other_path])
if module_info:
logger.debug("found module %s in other path [%s]", fullname, other_path)
return
except ImportError:
continue
else:
raise ImportError()
except ImportError:
try:
# Now check if in site packages and needs gatekeeping
for sitedir in sitedirs:
try:
module_info = imp.find_module(fullname, [sitedir, self.path_entry])
if module_info:
logger.debug("found module %s in sitedir or subdirectory [%s]", fullname, sitedir)
return GateKeeperLoader(module_info, sitedir)
except ImportError:
logger.debug("%s not found in: %s", fullname, os.pathsep.join(other_paths))
continue
except ImportError:
### TODO
# Need to debug why this catch is needed, removing it stops things working...
# something is subtly weird here.
return
def run(self):
"""
Need to find any pre-existing vext contained in dependent packages
and install them
example:
you create a setup.py with install_requires["vext.gi"]:
- vext.gi gets installed using bdist_egg
- vext itself is now called with bdist_egg and we end up here
Vext now needs to find and install .vext files in vext.gi
[or any other files that depend on vext]
:return:
"""
logger.debug("vext InstallLib [started]")
# Find packages that depend on vext and check for .vext files...
logger.debug("find_vext_files")
vext_files = self.find_vext_files()
logger.debug("manually_install_vext: ", vext_files)
self.manually_install_vext(vext_files)
logger.debug("enable vext")
self.enable_vext()
logger.debug("install_lib.run")
install_lib.run(self)
logger.debug("vext InstallLib [finished]")