def __init__(
self, index_url="https://pypi.python.org/simple", hosts=('*',),
ca_bundle=None, verify_ssl=True, *args, **kw
):
Environment.__init__(self,*args,**kw)
self.index_url = index_url + "/"[:not index_url.endswith('/')]
self.scanned_urls = {}
self.fetched_urls = {}
self.package_pages = {}
self.allows = re.compile('|'.join(map(translate,hosts))).match
self.to_scan = []
if verify_ssl and ssl_support.is_available and (ca_bundle or ssl_support.find_ca_bundle()):
self.opener = ssl_support.opener_for(ca_bundle)
def _init_environment(self): dist_folders = map(lambda d: os.path.join(os.getcwd(), self.mod_folder, d), os.listdir(self.mod_folder)) dist_folders = filter(lambda f: os.path.exists(os.path.join(f, "EGG-INFO")), dist_folders) dists = map(lambda f: Distribution.from_filename(f) , dist_folders) # self.pkg_env = Environment() for dist in dists: self.pkg_env.add(dist)
def __init__(
self, index_url="https://pypi.org/simple/", hosts=('*',),
ca_bundle=None, verify_ssl=True, *args, **kw
):
Environment.__init__(self, *args, **kw)
self.index_url = index_url + "/" [:not index_url.endswith('/')]
self.scanned_urls = {}
self.fetched_urls = {}
self.package_pages = {}
self.allows = re.compile('|'.join(map(translate, hosts))).match
self.to_scan = []
use_ssl = (
verify_ssl
and ssl_support.is_available
and (ca_bundle or ssl_support.find_ca_bundle())
)
if use_ssl:
self.opener = ssl_support.opener_for(ca_bundle)
else:
self.opener = urllib.request.urlopen
def checker_pex(self, interpreter):
# TODO(John Sirois): Formalize in pants.base?
pants_dev_mode = os.environ.get('PANTS_DEV')
if pants_dev_mode:
checker_id = self.checker_target.transitive_invalidation_hash()
else:
checker_id = hash_all([self._CHECKER_REQ])
pex_path = os.path.join(self.workdir, 'checker', checker_id, str(interpreter.identity))
if not os.path.exists(pex_path):
with self.context.new_workunit(name='build-checker'):
with safe_concurrent_creation(pex_path) as chroot:
pex_builder = PexBuilderWrapper.Factory.create(
builder=PEXBuilder(path=chroot, interpreter=interpreter),
log=self.context.log)
# Constraining is required to guard against the case where the user
# has a pexrc file set.
pex_builder.add_interpreter_constraint(str(interpreter.identity.requirement))
if pants_dev_mode:
pex_builder.add_sources_from(self.checker_target)
req_libs = [tgt for tgt in self.checker_target.closure()
if isinstance(tgt, PythonRequirementLibrary)]
pex_builder.add_requirement_libs_from(req_libs=req_libs)
else:
try:
# The checker is already on sys.path, eg: embedded in pants.pex.
platform = Platform.current()
platform_name = platform.platform
env = Environment(search_path=sys.path,
platform=platform_name,
python=interpreter.version_string)
working_set = WorkingSet(entries=sys.path)
for dist in working_set.resolve([Requirement.parse(self._CHECKER_REQ)], env=env):
pex_builder.add_direct_requirements(dist.requires())
# NB: We add the dist location instead of the dist itself to make sure its a
# distribution style pex knows how to package.
pex_builder.add_dist_location(dist.location)
pex_builder.add_direct_requirements([self._CHECKER_REQ])
except (DistributionNotFound, PEXBuilder.InvalidDistribution):
# We need to resolve the checker from a local or remote distribution repo.
pex_builder.add_resolved_requirements(
[PythonRequirement(self._CHECKER_REQ)])
pex_builder.set_entry_point(self._CHECKER_ENTRYPOINT)
pex_builder.freeze()
return PEX(pex_path, interpreter=interpreter)
def main():
failed = []
print('Getting environment')
env = Environment()
for pkgname in env:
print('Checking %s' % pkgname)
if pkgname in PROJECTS['bad']:
print('\tMarked as BAD. FAIL')
failed.append(pkgname)
continue
elif pkgname in PROJECTS['good']:
print('\tMarked as GOOD. PASS')
continue
for pkg in env[pkgname]:
had_conclusive_result = False
print('\tChecking version %s' % pkg.version)
for check in CHECKS:
print('\t\tRunning check: %s...' % check.__name__)
res = check(pkg)
if res is True:
print('\t\tPASS')
had_conclusive_result = True
break
elif res is False:
print('\t\tFAIL')
had_conclusive_result = True
failed.append(pkg)
break
elif res is None:
print('\t\t\tINCONCLUSIVE')
continue
else:
print('\t\t\tINVALID RESULT: %s' % res)
failed.append(pkg)
break
if not had_conclusive_result:
failed.append(pkg)
print('\t\tNO CONCLUSIVE RESULTS. FAIL')
print()
if failed:
print('At least one package failed.')
for fail in failed:
print('\tFailure: %s' % fail)
sys.exit(1)
else:
print('All packages passed license check')
def clean_version():
""" Keep only most recent version of each package """
env = Environment()
for project_name in env._distmap.keys():
installed_version = [d.version for d in env[project_name]]
if (installed_version):
max_version = max(installed_version, key=parse_version)
for dist in env[project_name]:
if parse_version(dist.version) < parse_version(max_version):
remove_egg(project_name, dist)
def test_list_one_installed(environment: Environment):
env = Environment(search_path=[])
environment.return_value = env
metadata = """Metadata-Version: 2.1
Name: chaostoolkit-some-stuff
Version: 0.1.0
Summary: Chaos Toolkit some package
Home-page: http://chaostoolkit.org
Author: chaostoolkit Team
Author-email: [email protected]
License: Apache License 2.0
"""
env.add(
Distribution(project_name="chaostoolkit-some-stuff",
version="0.1.0",
metadata=InMemoryMetadata({"PKG-INFO": metadata})))
extensions = list_extensions()
assert len(extensions) == 1
ext = extensions[0]
assert ext.name == "chaostoolkit-some-stuff"
assert ext.version == "0.1.0"
def run(self):
self.run_command('build_tests')
this_dir = os.path.normpath(
os.path.abspath(os.path.dirname(__file__)))
lib_dirs = glob.glob(os.path.join(this_dir, 'build', 'lib*'))
test_dir = os.path.join(this_dir, 'build', 'tests')
env = Environment(search_path=lib_dirs)
distributions = env["nose"]
assert len(distributions) == 1, (
"Incorrect number of distributions found")
dist = distributions[0]
dist.activate()
sys.path.insert(0, test_dir)
setuptools.command.test.test.run(self)
def main():
""" Run the application. """
# Find all additional eggs.
environment = Environment(EGG_PATH)
distributions, errors = working_set.find_plugins(environment)
if len(errors) > 0:
raise SystemError('cannot add eggs %s' % errors)
logger.debug('added eggs %s' % distributions)
# Add them to the working set.
map(working_set.add, distributions)
# Create and run the application.
return run()
def finalize_options(self):
if self.version:
print 'distribute %s' % get_distribution('distribute').version
sys.exit()
py_version = sys.version.split()[0]
self.config_vars = {
'dist_name': self.distribution.get_name(),
'dist_version': self.distribution.get_version(),
'dist_fullname': self.distribution.get_fullname(),
'py_version': py_version,
'py_version_short': py_version[0:3],
'py_version_nodot': py_version[0] + py_version[2],
}
self._expand('install_dir')
normpath = map(normalize_path, sys.path)
self.index_url = self.index_url or "http://pypi.python.org/simple"
hosts = ['*']
if self.package_index is None:
self.package_index = self.create_index(
self.index_url,
hosts=hosts,
)
self.local_index = Environment(sys.path)
if self.find_links is not None:
if isinstance(self.find_links, basestring):
self.find_links = self.find_links.split()
else:
self.find_links = []
self.package_index.add_find_links(self.find_links)
if not self.args:
raise DistutilsArgError(
"No urls, filenames, or requirements specified (see --help)")
self.outputs = []
def _load_eggs(self, search_path):
# Redefine where to search entry point.
distributions, errors = working_set.find_plugins(
Environment(search_path)
)
if errors:
logger.warn('could not load %s', errors)
map(working_set.add, distributions)
# Load each entry point one by one
for entry in working_set.iter_entry_points('rdiffweb.plugins'):
# Get unicode plugin name
module_name = entry.name
if isinstance(module_name, bytes):
module_name = module_name.decode('ascii')
# Plugin is enabled. Load it.
logger.debug('loading module plugin [%s] from [%r]', module_name, entry.module_name)
try:
yield (module_name, entry.load(), entry.dist)
except:
logger.error('fail to load module plugin [%s] from [%r]', module_name, entry.module_name, exc_info=1)
def list_extensions() -> List[ExtensionInfo]:
"""
List all installed Chaos Toolkit extensions in the current environment.
Notice, for now we can only list extensions that start with `chaostoolkit-`
in their package name.
This is not as powerful and solid as we want it to be. The trick is that we
can't rely on any metadata inside extensions to tell us they exist and
what functionnality they provide either. Python has the concept of trove
classifiers on packages but we can't extend them yet so they are of no use
to us.
In a future version, we will provide a mechanism from packages to support
a better detection.
"""
infos = []
distros = Environment()
seen = []
for key in distros:
for dist in distros[key]:
if dist.has_metadata('PKG-INFO'):
m = dist.get_metadata('PKG-INFO')
info = message_from_string(m)
name = info["Name"]
if name == "chaostoolkit-lib":
continue
if name in seen:
continue
seen.append(name)
if name.startswith("chaostoolkit-"):
ext = ExtensionInfo(name=name,
version=info["Version"],
summary=info["Summary"],
license=info["License"],
author=info["Author"],
url=info["Home-page"])
infos.append(ext)
return infos
def initPluginEnv(options, path):
from pkg_resources import working_set, Environment
# if options is passed in, use prefs to determine what to bypass
# otherwise all plugins are added to the working_set
if options is not None:
prefs = loadPrefs(options)
pluginPrefs = prefs.get('plugins', None)
else:
prefs = None
pluginPrefs = None
plugin_env = Environment(path)
plugin_eggs = []
# remove uninstalled plugins from prefs
if pluginPrefs is not None:
for project_name in pluginPrefs.keys():
if project_name not in plugin_env:
del prefs['plugins'][project_name]
prefs.write()
# add active plugins and l10n eggs to working set
for project_name in sorted(plugin_env):
for egg in plugin_env[project_name]:
if egg.has_metadata('resources.ini'):
working_set.add(egg) # possible l10n egg
elif (pluginPrefs is None
or pluginPrefs.get(project_name) != 'inactive'):
working_set.add(egg) # possible plugin egg
plugin_eggs.append(egg)
break
return plugin_env, plugin_eggs
def environment(self):
return Environment([str(self.sp)])
def already_satisfied(req: str) -> bool:
requirement = Requirement(req)
environment = Environment()
return any(dist in requirement for dist in environment[requirement.name])
def run(plugins=[], use_eggs=True, egg_path=[], image_path=[], template_path=[], startup_task="", application_name="Omnivore", debug_log=False, document_class=None):
"""Start the application
:param plugins: list of user plugins
:param use_eggs Boolean: search for setuptools plugins and plugins in local eggs?
:param egg_path: list of user-specified paths to search for more plugins
:param startup_task string: task factory identifier for task shown in initial window
:param application_name string: change application name instead of default Omnivore
"""
EnthoughtWxApp.mac_menubar_app_name = application_name
_app = EnthoughtWxApp(redirect=False)
if False: # enable this to use FilterEvent
_app.FilterEvent = _app.FilterEventMouseWheel
# Enthought library imports.
from envisage.api import PluginManager
from envisage.core_plugin import CorePlugin
# Local imports.
from omnivore.framework.application import FrameworkApplication
from omnivore.framework.plugin import OmnivoreTasksPlugin, OmnivoreMainPlugin
from omnivore.file_type.plugin import FileTypePlugin
from omnivore import get_image_path
from omnivore.utils.jobs import get_global_job_manager
# Include standard plugins
core_plugins = [ CorePlugin(), OmnivoreTasksPlugin(), OmnivoreMainPlugin(), FileTypePlugin() ]
if sys.platform == "darwin":
from omnivore.framework.osx_plugin import OSXMenuBarPlugin
core_plugins.append(OSXMenuBarPlugin())
import omnivore.file_type.recognizers
core_plugins.extend(omnivore.file_type.recognizers.plugins)
import omnivore.plugins
core_plugins.extend(omnivore.plugins.plugins)
# Add the user's plugins
core_plugins.extend(plugins)
# Check basic command line args
default_parser = argparse.ArgumentParser(description="Default Parser")
default_parser.add_argument("--no-eggs", dest="use_eggs", action="store_false", default=True, help="Do not load plugins from python eggs")
options, extra_args = default_parser.parse_known_args()
# The default is to use the specified plugins as well as any found
# through setuptools and any local eggs (if an egg_path is specified).
# Egg/setuptool plugin searching is turned off by the use_eggs parameter.
default = PluginManager(
plugins = core_plugins,
)
if use_eggs and options.use_eggs:
from pkg_resources import Environment, working_set
from envisage.api import EggPluginManager
from envisage.composite_plugin_manager import CompositePluginManager
# Find all additional eggs and add them to the working set
environment = Environment(egg_path)
distributions, errors = working_set.find_plugins(environment)
if len(errors) > 0:
raise SystemError('cannot add eggs %s' % errors)
logger = logging.getLogger()
logger.debug('added eggs %s' % distributions)
map(working_set.add, distributions)
# The plugin manager specifies which eggs to include and ignores all others
egg = EggPluginManager(
include = [
'omnivore.tasks',
]
)
plugin_manager = CompositePluginManager(
plugin_managers=[default, egg]
)
else:
plugin_manager = default
# Add omnivore icons after all image paths to allow user icon themes to take
# precidence
from pyface.resource_manager import resource_manager
import os
image_paths = image_path[:]
image_paths.append(get_image_path("icons"))
image_paths.append(get_image_path("../omnivore8bit/icons"))
resource_manager.extra_paths.extend(image_paths)
from omnivore.templates import template_subdirs
template_subdirs.extend(template_path)
kwargs = {}
if startup_task:
kwargs['startup_task'] = startup_task
if application_name:
kwargs['name'] = application_name
if document_class:
kwargs['document_class'] = document_class
# Create a debugging log
if debug_log:
filename = app.get_log_file_name("debug")
handler = logging.FileHandler(filename)
logger = logging.getLogger('')
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
# Turn off omnivore log debug messages by default
log = logging.getLogger("omnivore")
log.setLevel(logging.INFO)
# check for logging stuff again to pick up any new loggers loaded since
# startup
import omnivore.utils.wx.error_logger as error_logger
if "-d" in extra_args:
i = extra_args.index("-d")
error_logger.enable_loggers(extra_args[i+1])
app = FrameworkApplication(plugin_manager=plugin_manager, command_line_args=extra_args, **kwargs)
app.run()
job_manager = get_global_job_manager()
if job_manager is not None:
job_manager.shutdown()
def configure(self, settings):
"""
Finds all discoverable plugins and configures them. Plugins are
discoverable if they are in the normal python module path, or in
the path specified by 'plugin directory'.
"""
# load plugins
section = settings.section('server')
self.pluginsdir = os.path.join(section.getPath("plugin directory"))
if self.pluginsdir:
logger.debug("loading plugins from %s" % self.pluginsdir)
working_set.add_entry(self.pluginsdir)
env = Environment([
self.pluginsdir,
])
else:
env = Environment([])
self._eggs, errors = working_set.find_plugins(env)
# load plugin eggs
for p in self._eggs:
working_set.add(p)
logger.info("loaded plugin egg '%s'" % p)
for e in errors:
logger.info("failed to load plugin egg '%s'" % e)
# load all discovered plugins for each type
for ep in working_set.iter_entry_points("terane.plugin"):
# if no config section exists, then don't load the plugin
if not settings.hasSection("plugin:%s" % ep.name):
continue
try:
# load and configure the plugin
_Plugin = ep.load()
if not IPlugin.implementedBy(_Plugin):
raise Exception("plugin '%s' doesn't implement IPlugin" %
ep.name)
plugin = _Plugin()
plugin.setName(ep.name)
plugin.setServiceParent(self)
section = settings.section("plugin:%s" % ep.name)
plugin.configure(section)
logger.info("loaded plugin '%s'" % ep.name)
# find all plugin components
for impl, spec, name in plugin.listComponents():
if not ILoadable.implementedBy(impl):
raise Exception(
"component %s:%s in plugin %s doesn't implement ILoadable"
% (spec.__name__, name, ep.name))
if not isinstance(spec, InterfaceClass):
raise TypeError("spec must be an Interface")
if (spec, name) in self._components:
raise KeyError("component %s:%s already exists" %
(spec.__name__, name))
# a little extra syntax here to make sure the lambda expression
# passed as the factory function has the appropriate variables bound
# in its scope
def _makeTrampoline(impl=impl, plugin=plugin):
def _trampoline(*args, **kwds):
logger.trace("allocating new %s from plugin %s" %
(impl.__name__, plugin.name))
return impl(plugin, *args, **kwds)
return _trampoline
self._components[(spec,
name)] = _makeTrampoline(impl, plugin)
logger.trace("added component %s:%s" %
(spec.__name__, name))
except ConfigureError:
raise
except Exception, e:
logger.exception(e)
logger.warning("failed to load plugin '%s'" % ep.name)
class DistributionsManager(object):
def __init__(self):
self.mod_folder = os.path.join(os.getcwd(), "raduga_modules")
if not os.path.exists(self.mod_folder):
os.mkdir(self.mod_folder)
self._init_environment()
def _init_environment(self):
dist_folders = map(lambda d: os.path.join(os.getcwd(), self.mod_folder, d), os.listdir(self.mod_folder))
dist_folders = filter(lambda f: os.path.exists(os.path.join(f, "EGG-INFO")), dist_folders)
dists = map(lambda f: Distribution.from_filename(f) , dist_folders)
#
self.pkg_env = Environment()
for dist in dists: self.pkg_env.add(dist)
def _add_to_environment(self, egg_folder):
dist = Distribution.from_filename(egg_folder)
self.pkg_env.add(dist)
def _match_req(self, req):
return self.pkg_env.best_match(req, working_set)
def _flatten_reqs(self, *req_sets):
# req_sets further in the list take precedence
reqs = {}
for rset in req_sets:
for sreq in rset:
req = Requirement.parse(sreq)
reqs[req.key] = req
return reqs.values()
@contextmanager
def requirement_loader(self, *req_sets):
# Save sys.path and sys.modules, to be restored later
import sys, copy
old_path = copy.copy(sys.path)
old_sys_modules = sys.modules.keys()
# Find distributions for all the requirements
req_dists = []
reqs = self._flatten_reqs(req_sets)
for req in reqs:
match = self._match_req(req)
if match is None:
raise RuntimeError("Unable to find distribution matching %s" % str(req))
req_dists.append(match)
# Activate the distributions, return control
for req in req_dists: req.activate()
yield
# Restore sys path and modules
sys.path = old_path
for modname in sys.modules.keys():
if not modname in old_sys_modules:
del sys.modules[modname]
def install_dist(self, path):
setup_py = os.path.join(os.getcwd(), path, "setup.py")
if not os.path.isfile(setup_py):
raise RuntimeError("Folder %s doesn't have a setup file" % path)
with self._build_egg_env(path) as tempdir:
import subprocess, zipfile
subprocess.check_call(["python", setup_py, "bdist_egg", "--dist-dir=%s" % tempdir])
egg = os.listdir(tempdir)[0] # egg will be the single entry in the temp folder
# TODO: check if exactly that same egg is installed
eggf = os.path.join(self.mod_folder, egg) # target egg folder
os.mkdir(eggf)
eggz = zipfile.ZipFile(os.path.join(tempdir, egg))
eggz.extractall(eggf)
@contextmanager
def _build_egg_env(self, path):
import tempfile, shutil
old_cwd = os.getcwd()
os.chdir(path)
tempdir = tempfile.mkdtemp()
yield tempdir
shutil.rmtree(tempdir)
os.chdir(old_cwd)
"overlay": "p2ner.%s.overlay",
"serveroverlay": "p2ner.%s.serveroverlay",
"pipeelement": "p2ner.%s.pipeelement",
"ui": "p2ner.%s.ui",
'interface': 'p2ner.%s.interface',
"input": "p2ner.%s.input",
"output": "p2ner.%s.output",
"plugin": "p2ner.%s.plugin",
"stats": "p2ner.%s.stats",
"flowcontrol": "p2ner.%s.flowcontrol"
}
COMPONENTS_DIR = os.path.dirname(os.path.abspath(sys.argv[0]))
USER_COMPONENTS_DIR = ""
working_set.add_entry(COMPONENTS_DIR)
pkg_env = Environment([COMPONENTS_DIR])
for name in pkg_env:
working_set.require(pkg_env[name][0].key)
sys.path.append(COMPONENTS_DIR)
def getComponents(ctype):
"""Get all the components of a given type
:param ctype: the component's type
:type ctype: string
:returns: {"ComponentName":ComponentClass,...}
:rtype: dict
"""
def __init__(self, context, request):
super().__init__(context, request)
environment = self.environment = Environment()
environment.scan()
FileMetadata,
PathMetadata,
ResourceManager,
Requirement,
AvailableDistributions,
DistInfoDistribution,
Distribution,
)
from xmlrpc.client import ServerProxy
import requests
import tablib.core
from Other import OrderedSet
from operator import itemgetter
from tablib import Dataset
Distributions = Environment(search_path=sys.path)
Distributions.scan(search_path=sys.path)
# print(len(Distributions._distmap))
import pkg_resources
Cache = OrderedSet()
AD = AvailableDistributions()
print(len(AD._distmap))
refiners = "AND,OR,ANDNOT,ANDMAYBE,NOT".split(",")
def boost(term, value):
return "{}^{}".format(term, value)
class Bootstrapper(object):
def __init__(self,
buildout_version,
eggs_dir='bootstrap-eggs',
bootstrap_config=None,
output_bootstrap_config=None,
offline=False,
python=sys.executable,
buildout_dir=None,
buildout_config='buildout.cfg',
force_setuptools_path=None,
force_distribute=None,
force_setuptools=None,
error=None):
"""Initializations.
Right after instantiation, the requirements for ``setuptools`` and
``zc.buildout`` are fully known.
:param error: callable to issue end-user error messages and quit.
"""
self.init_directories(buildout_dir, eggs_dir)
self.error = error
self.init_python_info(python)
self.bootstrap_config = bootstrap_config
self.output_bootstrap_config = output_bootstrap_config
if buildout_version is None:
buildout_version = self.read_bootstrap_config()
self.buildout_version = buildout_version
self.init_reqs(buildout_version,
force_setuptools_path=force_setuptools_path,
force_distribute=force_distribute,
force_setuptools=force_setuptools)
self.init_internal_attrs()
self.buildout_config = buildout_config
self.offline = offline
def init_directories(self, buildout_dir, eggs_dir):
if buildout_dir is None:
buildout_dir = os.getcwd()
self.buildout_dir = buildout_dir
self.eggs_dir = os.path.abspath(os.path.join(buildout_dir, eggs_dir))
if not os.path.exists(self.eggs_dir):
os.makedirs(self.eggs_dir)
def bootstrap(self):
# actually calling the property right now
logger.info(
"Starting bootstrap stage 1 for %s (%s) "
"and " + str(self.buildout_req), self.python, self.python_version)
if self.setuptools_path is None:
self.setuptools_path = self.ensure_req(self.setuptools_req)
paths = dict(setuptools_path=self.setuptools_path,
buildout_path=self.ensure_req(self.buildout_req))
oldpwd = os.getcwd()
os.chdir(self.buildout_dir)
try:
boot_fname = 'bootstrap_stage2.py'
with open(boot_fname, 'w') as bootf:
bootf.write(bootstrap_script_tmpl % paths)
logger.info("Starting bootstrap stage 2 (running %r)", boot_fname)
cmd = [self.python, boot_fname, '-c', self.buildout_config]
cmd.append('buildout:eggs-directory=' + self.eggs_dir)
if self.offline:
cmd.append("-o")
logger.debug("Exact stage2 command is %r", cmd)
subprocess.check_call(cmd)
self.clean()
self.dump_bootstrap_config()
self.remove_develop_eggs()
finally:
os.chdir(oldpwd) # crucial for tests
def read_bootstrap_config(self):
"""Read buildout version from a bootstrap INI file."""
config = self.bootstrap_config
if config is None:
return
ini_path = os.path.join(self.buildout_dir, config)
if not os.path.exists(ini_path):
logger.info(
"No bootstrap configuration file found at %r "
"proceeding without any prescribed buildout version.",
ini_path)
return
logger.info("Reading buildout version from %r", ini_path)
parser = ConfigParser()
try:
parser.read(ini_path)
return parser.get('bootstrap', 'buildout-version').strip()
except Exception as exc:
logger.warn(
"Exception while reading bootstrap configuration "
"file %r, %s. Ignoring the file", ini_path, exc)
def dump_bootstrap_config(self):
"""Write the used buildout version in the bootstrap config file.
If a precise version has been required, at this stage we know
it works, just dump it.
Otherwise, take the version after stage2, there's no reason it should
not work.
:param out_path: path to the produced file, relative to the buildout
directory
"""
out_path = self.output_bootstrap_config
if out_path is None:
return
b_version = self.buildout_version
# we have already changed directory to buildout directory
# TODO make it independent of cwd
try:
if b_version is None:
# not passed on command line, not read from config file
# TODO don't hardcode exe path (not so easy)
cmd = ["bin/buildout", "--version"]
cmd_str = ' '.join(cmd)
logger.info(
"No buildout version has been specified on "
"command line nor in bootstrap config file. "
"Using the one stage 2 produced (%s)", cmd_str)
buildout = subprocess.Popen(cmd, stdout=subprocess.PIPE)
out = buildout.communicate()[0]
if buildout.returncode == 0:
b_version = out.decode().split()[-1]
else:
raise RuntimeError(cmd_str + " has errors")
with open(out_path, 'w') as ini:
ini.write('\n'.join((
"# Produced by unibootstrap",
"# at time (UTC): " + datetime.utcnow().isoformat(),
"# with Python " + sys.version.splitlines()[0],
"",
"[bootstrap]",
"buildout-version = " + b_version,
)))
except Exception as exc:
logger.error("Could not write used "
"buildout version in %r (%s)", out_path, exc)
logger.info("Wrote buildout version %s to %s", b_version, out_path)
def init_env(self):
self.ws = WorkingSet(entries=())
self.env = Environment(search_path=[self.eggs_dir])
def init_reqs(self,
buildout_version,
force_setuptools_path=None,
force_setuptools=None,
force_distribute=None):
"""Sets wished requirement attributes.
These attributes are instances of ``Requirement`` (all needed
workarounds for that have already been done), plus this method
also creates the attributes that will be used in
:meth:`grab_req`
:param buildout_version: same as in :func:`guess_versions`
:param force_setuptools_path: if set, this setuptools distribution
will be used both to grab zc.buildout
and in stage2 of bootstrap. This has
precedence over the other 'force'
arguments below.
:param force_setuptools: force a setuptools (not distribute) req, and
make the necessary preparations for it.
:param force_distribute: force a distribute (not setuptools) req, and
make the necessary preparations for it.
"""
self.init_env()
buildout_rhs, setuptools, setuptools_rhs = guess_versions(
buildout_version)
self.buildout_req = Requirement.parse('zc.buildout' + buildout_rhs)
# this is almost the CLI, but that lets us control which one is
# executed from PYTHONPATH (finding the equivalent executable would be
# more hazardeous)
self._ez_install = (
sys.executable,
'-c',
"from setuptools.command.easy_install import main; main()",
)
if force_setuptools_path is not None:
logger.info(
"Using the forced setuptools distribution at %r "
"both for preparations and in stage2 bootstrap",
force_setuptools_path)
self.setuptools_path = force_setuptools_path
self._ez_install_pypath = force_setuptools_path
return
self.setuptools_path = None
if force_setuptools is not None and force_distribute is not None:
# should be excluded upstream, last-time check
logger.critical("Got force_setuptools=%r AND force_distribute=%r",
force_setuptools, force_distribute)
raise ValueError("Can't force both on setuptools and distribute !")
if force_setuptools is not None:
setuptools, setuptools_rhs = 'setuptools', force_setuptools
if force_distribute is not None:
setuptools, setuptools_rhs = 'distribute', force_distribute
# actually, installing distribute with any version of setuptools or
# itself happens to work... provided that the magic buildout marker
# that tells is setup no to touch the global site-packages is there
# otherwise, it'd try the rename an .egg_info, and would fail for non-
# privileged users, even for a local egg production.
# Usually, the shell would set it, thankfully it's not libc's concern
if setuptools == 'distribute':
os.environ['_'] = 'buildout_unibootstrap'
if DISTRIBUTE and setuptools == 'setuptools':
self.setuptools_req = self._setuptools_req(setuptools_rhs)
self.init_ez_install_distribute_to_setuptools()
return
self.setuptools_req = Requirement.parse(setuptools + setuptools_rhs)
self._ez_install_pypath = None
def ensure_req(self, req):
"""Make sure that requirement is satisfied and return location.
Either we have it already, or we install it.
"""
# we don't use obtain() because it's not clear how it would
# not use the working set with the full sys.path instead of our empty
# one (fearing also slight behaviour changes across versions)
dist = self.env.best_match(req, self.ws)
if dist is None:
dist = self.grab_req(req)
self.init_env()
dist = self.env.best_match(req, self.ws)
if dist is None:
raise LookupError(req)
logger.info("Requirement %s fulfilled at %s", req, dist.location)
return dist.location
def grab_req(self, req):
"""Install a requirement to self.eggs_dir.
We don't use the internal API of setuptools and spawn of subprocess
because:
- we might use a different version of setuptools than the one we import
from here
- the command-line (or surface) API is obviously very stable
"""
if self.offline:
# actually, we would have a small ability to find setuptools
# eggs that are present locally, but that's too complicated for
# now and has few chances of success
self.error("%s is not available in specified dists "
"directory %s, and can't be downloaded "
"(offline mode is requested)" % (req, self.eggs_dir))
logger.info("%s not available locally, attempting to download", req)
os_env = dict(os.environ)
pypath = self._ez_install_pypath
if pypath is not None:
os_env['PYTHONPATH'] = pypath
subprocess.check_call(self._ez_install +
('-qamxd', self.eggs_dir, str(req)),
env=os_env)
def _setuptools_req(self, req_rhs):
"""Counter distribute's hack that replaces setuptools requirements.
if distribute is not around, this is simply a normal requirement
parsing.
"""
if not DISTRIBUTE:
return Requirement.parse('setuptools' + req_rhs)
req = Requirement.parse('willbesetuptools' + req_rhs)
req.key = req.project_name = 'setuptools'
return req
def init_ez_install_distribute_to_setuptools(self):
"""Collection of workarounds to grab setuptools with distribute.
Here it gets dirty.
Worse than just being unable to grab setuptools, distribute
is very confused also for other distributions that require setuptools.
Therefore, we look for any available setuptools package (typically,
an egg), and run easy_install from it.
This sets the 'ez_install_pypath'.
"""
# first, let's see if we have any version of setuptools around
dist = self.env.best_match(self._setuptools_req(''), self.ws)
if dist is not None and dist.project_name == 'setuptools':
self._ez_install_pypath = dist.location
return
setuptools_egg_rx = re.compile( # glob not precise enough
r'setuptools-\d+[.]\d+.*?-py' + self.python_version + '.egg')
# if virtualenv is around, it often comes with a bundled egg
try:
import virtualenv
except ImportError:
pass
else:
for venv_support in (
# Debian 6 & 7 python-virtualenv package:
'/usr/share/python-virtualenv',
# generic install of virtualenv:
os.path.join(os.path.dirname(virtualenv.__file__),
'virtualenv_support')):
# TODO look for wheels, too, more recent virtualenv versions
# have them
if os.path.isdir(venv_support):
for fname in os.listdir(venv_support):
if setuptools_egg_rx.match(fname):
self._ez_install_pypath = os.path.join(
venv_support, fname)
return
# last chance: old-school harcoded downloading.
self._tmpdir = tempfile.mkdtemp()
# Up to now in my tests, this minimial version of setuptools is
# able to download all known setuptools versions and is the one
# used in many versions of virtualenv (prior to wheels)
# kept for reference for now, must retest with distribute
# min_pkg = 'setuptools-0.6c11-py%s.egg' % self.python_version
min_pkg = 'setuptools-5.5.1-py2.py3-none-any.whl'
dl = urlopen('https://pypi.python.org/packages/3.4/s/setuptools/' +
min_pkg).read()
pypath = self._ez_install_pypath = os.path.join(self._tmpdir, min_pkg)
with open(pypath, 'wb') as pkg:
pkg.write(dl)
def clean(self):
tmpdir = self._tmpdir
if tmpdir is not None and os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
def remove_develop_eggs(self):
"""This saves a lot of potential trouble."""
# TODO read it from buildout and remove if below buildout_dir
develop_eggs = os.path.join(self.buildout_dir, 'develop_eggs')
if os.path.isdir(develop_eggs):
shutil.rmtree(develop_eggs)
def init_python_info(self, python):
"""Used the passed executable path to get version and absolute path.
The x.y version string of the target Python."""
cmd = [
python, '-c',
'import sys; print(sys.version); print(sys.executable)'
]
try: # no check_output in Python 2.6
p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
except OSError:
self.error("No wished Python %r executable on PATH" % python)
out = p.communicate()[0]
if p.returncode != 0:
self.error("The target Python executable has errors, "
"command was " + ' '.join(cmd))
lines = out.decode().splitlines()
self.python_version = '.'.join(lines[0].split()[0].split('.', 2)[:2])
self.python = lines[-1]
def init_internal_attrs(self):
self._tmpdir = None # for cleanups
self._pyversion = None # for property
def __init__(self, groups=_plugin_groups, search_path=None):
super(PkgResourcesFactory, self).__init__()
self._have_new_types = True
self._groups = copy.copy(groups)
self._search_path = search_path
self.env = Environment(search_path)
class FigmentatorFactory:
"""
This class acts as a factory and a registry for Figmentators. You simply register a
Figmentator to the factory, then ask for the Figmentator by SuggestionType. It only
allows a single unique Figmentator per SuggestionType. It is also responsible for
calling the startup/shutdown of each Figmentator.
"""
def __init__(self):
""" Create the factory """
self.lock = Lock()
self.env = Environment()
self.loop = get_event_loop()
self.figmentators_by_type = {}
self.settings = FigmentatorFactorySettings()
@property
def figmentators(self):
""" Return a list of all the figmentators """
return list(self.figmentators_by_type.values())
async def get(self, suggestion_type: SuggestionType) -> Figmentator:
"""
Get a Figmentator by it's type. If there are valid settings for a
Figmentator it will instatiate it (and install any needed requirements.
"""
async with self.lock:
if suggestion_type not in self.figmentators_by_type:
if (suggestion_type not in self.settings.figmentators # pylint:disable=unsupported-membership-test
):
raise ValueError(
f"Cannot create Figmentator of type {suggestion_type}")
settings = self.settings.figmentators[ # pylint:disable=unsubscriptable-object
suggestion_type]
requirements = await self.loop.run_in_executor(
None, # use default executor
working_set.resolve, # resolve model requirements
settings.requires, # list of requirements
self.env, # environment
self.installer, # installer for missing requirements
True, # whether to replace conflicting requirements
)
for requirement in requirements:
working_set.add(requirement, replace=True)
try:
model_cls = settings.cls.resolve()
except ImportError as e:
raise e
if not issubclass(model_cls, Figmentator):
raise ValueError(
"model_cls must be a subclass of Figmentator")
figmentator = model_cls(suggestion_type)
await self.loop.run_in_executor(None, figmentator.startup,
settings.properties)
self.figmentators_by_type[suggestion_type] = figmentator
return self.figmentators_by_type[suggestion_type]
async def remove(self, figmentator: Figmentator):
"""
Get a Figmentator by it's type. If there are valid settings for a
Figmentator it will instatiate it (and install any needed requirements.
"""
async with self.lock:
suggestion_type = figmentator.suggestion_type
assert suggestion_type in self.figmentators_by_type
figmentator.shutdown()
del self.figmentators_by_type[suggestion_type]
def get_distribution(self,
requirement: Requirement) -> Optional[Distribution]:
"""
Get a distribution that statisfies the requirement. Use
find_distributions so it looks on disk, thus can find newly installed
pacakges. """
for distribution in find_distributions(self.settings.install_dir):
if distribution in requirement:
return distribution
return None
def installer(self, requirement: Requirement) -> Optional[Distribution]:
""" A method for using easy_install to install a requirement """
for dist in self.env[requirement.key]:
if dist not in requirement:
self.env.remove(dist)
# Use easy_install despite being deprecated as it is the only way to
# have multi-version package support. See:
# https://packaging.python.org/guides/multi-version-installs/
# https://packaging.python.org/discussions/pip-vs-easy-install/
with shadow_argv([
"",
"easy_install",
"--install-dir",
self.settings.install_dir,
str(requirement),
]):
setup()
distribution = self.get_distribution(requirement)
if distribution:
working_set.add(distribution, replace=True)
return distribution
def pkg_environment(path):
return Environment(path)
def init_env(self):
self.ws = WorkingSet(entries=())
self.env = Environment(search_path=[self.eggs_dir])
class Bootstrapper(object):
def __init__(self, buildout_version, eggs_dir='bootstrap-eggs',
bootstrap_config=None,
output_bootstrap_config=None,
offline=False,
python=sys.executable,
buildout_dir=None,
buildout_config='buildout.cfg',
force_setuptools_path=None,
force_distribute=None,
force_setuptools=None,
error=None):
"""Initializations.
Right after instantiation, the requirements for ``setuptools`` and
``zc.buildout`` are fully known.
:param error: callable to issue end-user error messages and quit.
"""
self.init_directories(buildout_dir, eggs_dir)
self.error = error
self.init_python_info(python)
self.bootstrap_config = bootstrap_config
self.output_bootstrap_config = output_bootstrap_config
if buildout_version is None:
buildout_version = self.read_bootstrap_config()
self.buildout_version = buildout_version
self.init_reqs(buildout_version,
force_setuptools_path=force_setuptools_path,
force_distribute=force_distribute,
force_setuptools=force_setuptools)
self.init_internal_attrs()
self.buildout_config = buildout_config
self.offline = offline
def init_directories(self, buildout_dir, eggs_dir):
if buildout_dir is None:
buildout_dir = os.getcwd()
self.buildout_dir = buildout_dir
self.eggs_dir = os.path.abspath(os.path.join(buildout_dir, eggs_dir))
if not os.path.exists(self.eggs_dir):
os.makedirs(self.eggs_dir)
def bootstrap(self):
# actually calling the property right now
logger.info("Starting bootstrap stage 1 for %s (%s) "
"and " + str(self.buildout_req),
self.python, self.python_version)
if self.setuptools_path is None:
self.setuptools_path = self.ensure_req(self.setuptools_req)
paths = dict(setuptools_path=self.setuptools_path,
buildout_path=self.ensure_req(self.buildout_req))
oldpwd = os.getcwd()
os.chdir(self.buildout_dir)
try:
boot_fname = 'bootstrap_stage2.py'
with open(boot_fname, 'w') as bootf:
bootf.write(bootstrap_script_tmpl % paths)
logger.info("Starting bootstrap stage 2 (running %r)",
boot_fname)
cmd = [self.python, boot_fname, '-c', self.buildout_config]
cmd.append('buildout:eggs-directory=' + self.eggs_dir)
if self.offline:
cmd.append("-o")
logger.debug("Exact stage2 command is %r", cmd)
subprocess.check_call(cmd)
self.clean()
self.dump_bootstrap_config()
self.remove_develop_eggs()
finally:
os.chdir(oldpwd) # crucial for tests
def read_bootstrap_config(self):
"""Read buildout version from a bootstrap INI file."""
config = self.bootstrap_config
if config is None:
return
ini_path = os.path.join(self.buildout_dir, config)
if not os.path.exists(ini_path):
logger.info("No bootstrap configuration file found at %r "
"proceeding without any prescribed buildout version.",
ini_path)
return
logger.info("Reading buildout version from %r", ini_path)
parser = ConfigParser()
try:
parser.read(ini_path)
return parser.get('bootstrap', 'buildout-version').strip()
except Exception as exc:
logger.warn("Exception while reading bootstrap configuration "
"file %r, %s. Ignoring the file",
ini_path, exc)
def dump_bootstrap_config(self):
"""Write the used buildout version in the bootstrap config file.
If a precise version has been required, at this stage we know
it works, just dump it.
Otherwise, take the version after stage2, there's no reason it should
not work.
:param out_path: path to the produced file, relative to the buildout
directory
"""
out_path = self.output_bootstrap_config
if out_path is None:
return
b_version = self.buildout_version
# we have already changed directory to buildout directory
# TODO make it independent of cwd
try:
if b_version is None:
# not passed on command line, not read from config file
# TODO don't hardcode exe path (not so easy)
cmd = ["bin/buildout", "--version"]
cmd_str = ' '.join(cmd)
logger.info("No buildout version has been specified on "
"command line nor in bootstrap config file. "
"Using the one stage 2 produced (%s)",
cmd_str)
buildout = subprocess.Popen(cmd, stdout=subprocess.PIPE)
out = buildout.communicate()[0]
if buildout.returncode == 0:
b_version = out.decode().split()[-1]
else:
raise RuntimeError(cmd_str + " has errors")
with open(out_path, 'w') as ini:
ini.write('\n'.join((
"# Produced by unibootstrap",
"# at time (UTC): " + datetime.utcnow().isoformat(),
"# with Python " + sys.version.splitlines()[0],
"",
"[bootstrap]",
"buildout-version = " + b_version,
)))
except Exception as exc:
logger.error("Could not write used "
"buildout version in %r (%s)", out_path, exc)
logger.info("Wrote buildout version %s to %s", b_version, out_path)
def init_env(self):
self.ws = WorkingSet(entries=())
self.env = Environment(search_path=[self.eggs_dir])
def init_reqs(self, buildout_version,
force_setuptools_path=None,
force_setuptools=None, force_distribute=None):
"""Sets wished requirement attributes.
These attributes are instances of ``Requirement`` (all needed
workarounds for that have already been done), plus this method
also creates the attributes that will be used in
:meth:`grab_req`
:param buildout_version: same as in :func:`guess_versions`
:param force_setuptools_path: if set, this setuptools distribution
will be used both to grab zc.buildout
and in stage2 of bootstrap. This has
precedence over the other 'force'
arguments below.
:param force_setuptools: force a setuptools (not distribute) req, and
make the necessary preparations for it.
:param force_distribute: force a distribute (not setuptools) req, and
make the necessary preparations for it.
"""
self.init_env()
buildout_rhs, setuptools, setuptools_rhs = guess_versions(
buildout_version)
self.buildout_req = Requirement.parse('zc.buildout' + buildout_rhs)
# this is almost the CLI, but that lets us control which one is
# executed from PYTHONPATH (finding the equivalent executable would be
# more hazardeous)
self._ez_install = (
sys.executable, '-c',
"from setuptools.command.easy_install import main; main()",
)
if force_setuptools_path is not None:
logger.info("Using the forced setuptools distribution at %r "
"both for preparations and in stage2 bootstrap",
force_setuptools_path)
self.setuptools_path = force_setuptools_path
self._ez_install_pypath = force_setuptools_path
return
self.setuptools_path = None
if force_setuptools is not None and force_distribute is not None:
# should be excluded upstream, last-time check
logger.critical("Got force_setuptools=%r AND force_distribute=%r",
force_setuptools, force_distribute)
raise ValueError("Can't force both on setuptools and distribute !")
if force_setuptools is not None:
setuptools, setuptools_rhs = 'setuptools', force_setuptools
if force_distribute is not None:
setuptools, setuptools_rhs = 'distribute', force_distribute
# actually, installing distribute with any version of setuptools or
# itself happens to work... provided that the magic buildout marker
# that tells is setup no to touch the global site-packages is there
# otherwise, it'd try the rename an .egg_info, and would fail for non-
# privileged users, even for a local egg production.
# Usually, the shell would set it, thankfully it's not libc's concern
if setuptools == 'distribute':
os.environ['_'] = 'buildout_unibootstrap'
if DISTRIBUTE and setuptools == 'setuptools':
self.setuptools_req = self._setuptools_req(setuptools_rhs)
self.init_ez_install_distribute_to_setuptools()
return
self.setuptools_req = Requirement.parse(setuptools + setuptools_rhs)
self._ez_install_pypath = None
def ensure_req(self, req):
"""Make sure that requirement is satisfied and return location.
Either we have it already, or we install it.
"""
# we don't use obtain() because it's not clear how it would
# not use the working set with the full sys.path instead of our empty
# one (fearing also slight behaviour changes across versions)
dist = self.env.best_match(req, self.ws)
if dist is None:
dist = self.grab_req(req)
self.init_env()
dist = self.env.best_match(req, self.ws)
if dist is None:
raise LookupError(req)
logger.info("Requirement %s fulfilled at %s", req, dist.location)
return dist.location
def grab_req(self, req):
"""Install a requirement to self.eggs_dir.
We don't use the internal API of setuptools and spawn of subprocess
because:
- we might use a different version of setuptools than the one we import
from here
- the command-line (or surface) API is obviously very stable
"""
if self.offline:
# actually, we would have a small ability to find setuptools
# eggs that are present locally, but that's too complicated for
# now and has few chances of success
self.error("%s is not available in specified dists "
"directory %s, and can't be downloaded "
"(offline mode is requested)" % (req, self.eggs_dir))
logger.info("%s not available locally, attempting to download", req)
os_env = dict(os.environ)
pypath = self._ez_install_pypath
if pypath is not None:
os_env['PYTHONPATH'] = pypath
subprocess.check_call(self._ez_install +
('-qamxd', self.eggs_dir, str(req)),
env=os_env)
def _setuptools_req(self, req_rhs):
"""Counter distribute's hack that replaces setuptools requirements.
if distribute is not around, this is simply a normal requirement
parsing.
"""
if not DISTRIBUTE:
return Requirement.parse('setuptools' + req_rhs)
req = Requirement.parse('willbesetuptools' + req_rhs)
req.key = req.project_name = 'setuptools'
return req
def init_ez_install_distribute_to_setuptools(self):
"""Collection of workarounds to grab setuptools with distribute.
Here it gets dirty.
Worse than just being unable to grab setuptools, distribute
is very confused also for other distributions that require setuptools.
Therefore, we look for any available setuptools package (typically,
an egg), and run easy_install from it.
This sets the 'ez_install_pypath'.
"""
# first, let's see if we have any version of setuptools around
dist = self.env.best_match(self._setuptools_req(''), self.ws)
if dist is not None and dist.project_name == 'setuptools':
self._ez_install_pypath = dist.location
return
setuptools_egg_rx = re.compile( # glob not precise enough
r'setuptools-\d+[.]\d+.*?-py' + self.python_version + '.egg')
# if virtualenv is around, it often comes with a bundled egg
try:
import virtualenv
except ImportError:
pass
else:
for venv_support in (
# Debian 6 & 7 python-virtualenv package:
'/usr/share/python-virtualenv',
# generic install of virtualenv:
os.path.join(os.path.dirname(virtualenv.__file__),
'virtualenv_support')):
# TODO look for wheels, too, more recent virtualenv versions
# have them
if os.path.isdir(venv_support):
for fname in os.listdir(venv_support):
if setuptools_egg_rx.match(fname):
self._ez_install_pypath = os.path.join(
venv_support, fname)
return
# last chance: old-school harcoded downloading.
self._tmpdir = tempfile.mkdtemp()
# Up to now in my tests, this minimial version of setuptools is
# able to download all known setuptools versions and is the one
# used in many versions of virtualenv (prior to wheels)
# kept for reference for now, must retest with distribute
# min_pkg = 'setuptools-0.6c11-py%s.egg' % self.python_version
min_pkg = 'setuptools-5.5.1-py2.py3-none-any.whl'
dl = urlopen('https://pypi.python.org/packages/3.4/s/setuptools/' +
min_pkg).read()
pypath = self._ez_install_pypath = os.path.join(self._tmpdir, min_pkg)
with open(pypath, 'wb') as pkg:
pkg.write(dl)
def clean(self):
tmpdir = self._tmpdir
if tmpdir is not None and os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
def remove_develop_eggs(self):
"""This saves a lot of potential trouble."""
# TODO read it from buildout and remove if below buildout_dir
develop_eggs = os.path.join(self.buildout_dir, 'develop_eggs')
if os.path.isdir(develop_eggs):
shutil.rmtree(develop_eggs)
def init_python_info(self, python):
"""Used the passed executable path to get version and absolute path.
The x.y version string of the target Python."""
cmd = [python, '-c',
'import sys; print(sys.version); print(sys.executable)']
try: # no check_output in Python 2.6
p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
except OSError:
self.error("No wished Python %r executable on PATH" % python)
out = p.communicate()[0]
if p.returncode != 0:
self.error("The target Python executable has errors, "
"command was " + ' '.join(cmd))
lines = out.decode().splitlines()
self.python_version = '.'.join(lines[0].split()[0].split('.', 2)[:2])
self.python = lines[-1]
def init_internal_attrs(self):
self._tmpdir = None # for cleanups
self._pyversion = None # for property
def init_env(self):
self.ws = WorkingSet(entries=())
self.env = Environment(search_path=[self.eggs_dir])
# -*- coding: utf-8 -*-
from base64 import b64encode
from couchdb.http import ResourceConflict
from json import dumps
from libnacl.secret import SecretBox
from logging import getLogger
from openprocurement.api.utils import context_unpack, json_view, APIResource
from pyramid.security import Allow
from pkg_resources import Environment
from itertools import chain
PKG_ENV = Environment()
PKG_VERSIONS = dict(
chain.from_iterable([[(x.project_name, x.version) for x in PKG_ENV[i]]
for i in PKG_ENV]))
LOGGER = getLogger(__package__)
class Root(object):
__name__ = None
__parent__ = None
__acl__ = [
(Allow, 'g:archivarius', 'dump_resource'),
(Allow, 'g:archivarius', 'delete_resource'),
]
def __init__(self, request):
self.request = request
self.db = request.registry.db
def fetch_dist(self, requirement):
"""Fetch an egg needed for building.
Use pip/wheel to fetch/build a wheel."""
get_distribution("pip")
get_distribution("wheel")
# Ignore environment markers; if supplied, it is required.
requirement = Requirement.parse(str(requirement))
requirement.marker = None
# Take easy_install options into account, but do not override relevant
# pip environment variables (like PIP_INDEX_URL or PIP_QUIET); they'll
# take precedence.
if "PIP_QUIET" in environ or "PIP_VERBOSE" in environ:
quiet = False
else:
quiet = True
index_url = None
find_links = []
environment = Environment()
for dist in find_distributions(self.dist_dir):
if dist in requirement and environment.can_add(dist):
return dist
with TemporaryDirectory() as tmpdir:
cmd = [
executable,
"-m",
"pip",
"--disable-pip-version-check",
"wheel",
"--no-deps",
"-w",
tmpdir,
]
if quiet:
cmd.append("--quiet")
if "PIP_NO_CACHE_DIR" not in environ and self.no_cache_dir:
cmd.append("--no-cache-dir")
if index_url is not None:
cmd.extend(("--index-url", index_url))
if find_links is not None:
for link in find_links:
cmd.extend(("--find-links", link))
# If requirement is a PEP 508 direct URL, directly pass
# the URL to pip, as `req @ url` does not work on the
# command line.
if requirement.url:
cmd.append(requirement.url)
else:
cmd.append(str(requirement))
check_call(cmd)
with ZipFile(glob(path.join(tmpdir, "*.whl"))[0], "r") as zf:
zf.extractall(self.dist_dir)
pattern = re_compile(
translate(
f'{requirement.project_name.replace("-","_")}-*.dist-info'
),
IGNORECASE,
)
dist_path = [
path.join(self.dist_dir, x) for x in listdir(self.dist_dir) if
path.isdir(path.join(self.dist_dir, x)) and match(pattern, x)
][0]
root = path.dirname(dist_path)
return Distribution.from_location(
root,
path.basename(dist_path),
PathMetadata(root, dist_path),
precedence=BINARY_DIST,
)
def resolve(
self,
requirements: Sequence[Requirement],
env: Optional[Environment] = None,
installer: Optional[Callable[[str], Distribution]] = None,
replace_conflicting: Optional[bool] = False,
extras: List[str] = None,
) -> List[Distribution]:
"""List all distributions needed to (recursively) meet `requirements`
`requirements` must be a sequence of ``Requirement`` objects. `env`,
if supplied, should be an ``Environment`` instance. If
not supplied, it defaults to all distributions available within any
entry or distribution in the working set. `installer`, if supplied,
will be invoked with each requirement that cannot be met by an
already-installed distribution; it should return a ``Distribution`` or
``None``.
Unless `replace_conflicting=True`, raises a VersionConflict exception
if
any requirements are found on the path that have the correct name but
the wrong version. Otherwise, if an `installer` is supplied it will be
invoked to obtain the correct version of the requirement and activate
it.
`extras` is a list of the extras to be used with these requirements.
This is important because extra requirements may look like `my_req;
extra = "my_extra"`, which would otherwise be interpreted as a purely
optional requirement. Instead, we want to be able to assert that these
requirements are truly required.
"""
# set up the stack
requirements = list(requirements)[::-1]
# set of processed requirements
processed = {}
# key -> dist
best = {}
resolved = []
requirement_extras = _ReqExtras()
# Mapping of requirement to set of distributions that required it;
# useful for reporting info about conflicts.
required_by = defaultdict(set)
while requirements:
# process dependencies breadth-first
requirement = requirements.pop(0)
if requirement in processed:
# Ignore cyclic or redundant dependencies
continue
if not requirement_extras.markers_pass(requirement, extras):
continue
dist = best.get(requirement.key)
if dist is None:
# Find the best distribution and add it to the map
dist = self.by_key.get(requirement.key)
if dist is None or (dist not in requirement and replace_conflicting):
ws = self
if env is None:
if dist is None:
env = Environment(self.entries)
else:
# Use an empty environment and workingset to avoid
# any further conflicts with the conflicting
# distribution
env = Environment([])
ws = WorkingSet([])
dist = best[requirement.key] = env.best_match(
requirement,
ws,
installer,
replace_conflicting=replace_conflicting,
)
if dist is None:
requirers = required_by.get(requirement, None)
raise DistributionNotFound(requirement, requirers)
resolved.append(dist)
if dist not in requirement:
# Oops, the "best" so far conflicts with a dependency
dependent_requirement = required_by[requirement]
raise VersionConflict(dist, requirement).with_context(
dependent_requirement
)
# push the new requirements onto the stack
new_requirements = [
requirement
for requirement in dist.requires(requirement.extras)[::-1]
if requirement.key not in self.excludes
]
requirements.extend(new_requirements)
# Register the new requirements needed by requirement
for new_requirement in new_requirements:
required_by[new_requirement].add(requirement.project_name)
requirement_extras[new_requirement] = requirement.extras
processed[requirement] = True
# return list of distros to activate
return resolved
