def inspect(self):
pkg = self.package
if self.setup_path and not self.egg_path:
log.info(style_note('Building Python egg-info'))
res = call_setup_py(self.setup_path, ['egg_info'],
env=pkg.fresh_environ(),
indent=True,
verbosity=1)
if res:
raise RuntimeError('Could not build Python package')
self.egg_path = find_in_tree(pkg.build_path, '*.egg-info', 'dir')
if not self.egg_path:
log.warning('Could not find newly created *.egg-info')
if self.egg_path:
requires_path = os.path.join(self.egg_path, 'requires.txt')
if os.path.exists(requires_path):
for line in open(requires_path, 'rb'):
line = line.strip()
if not line:
continue
if line.startswith('['):
break
name = re.split('\W', line)[0].lower()
log.debug('%s depends on %s' % (pkg.name, name))
pkg.dependencies.append(
Package(name=name, url='pypi:%s' % name))
def create_if_not_exists(self):
python = os.path.join(self.path, 'bin', 'python')
if not os.path.exists(python):
makedirs(self.path)
log.info(style_note('Creating Python virtualenv', self.path))
if hasattr(virtualenv, 'cli_run'):
# New API (in which there isn't really any API)
virtualenv.cli_run(
['--no-pip', '--no-wheel', '--no-setuptools', self.path])
else:
# Old API
virtualenv.create_environment(self.path,
no_setuptools=True,
no_pip=True)
if not os.path.exists(python + '-config'):
version = get_default_python().version
names = (
'python{}.{}-config'.format(*version),
'python{}-config'.format(*version),
'python-config',
)
prefix = getattr(sys, 'real_prefix', sys.prefix)
for name in names:
old_path = os.path.join(prefix, 'bin', name)
if os.path.exists(old_path):
for name in names:
new_path = os.path.join(self.path, 'bin', name)
self.rewrite_shebang_or_link(old_path, new_path)
break
else:
log.warning('Could not find python-config')
def check_ff_safety(self, rev='HEAD', ignore_permissions=True):
# Check the status of the work tree and index.
status_ok = True
for idx, tree, name in self.status(
ignore_permissions=ignore_permissions):
if idx or tree:
log.error('uncomitted changes:')
self.git('status')
status_ok = False
break
# Make sure we haven't forked.
ahead, behind = self.distance(self.head, rev)
if ahead and behind:
log.error('you and the repo have forked')
status_ok = False
elif ahead:
log.warning(
'you are %s commits ahead of the remote repo; please `vee push`'
% ahead)
status_ok = False
elif behind:
log.info('You are %d commits behind.' % behind)
return status_ok
def relocate_pkgconfig(root):
# Do trivial rewrites of pkgconfig files.
pkg_config = os.path.join(root, 'lib', 'pkgconfig')
if os.path.exists(pkg_config):
for name in os.listdir(pkg_config):
if not name.endswith('.pc'):
continue
path = os.path.join(pkg_config, name)
log.info(path)
lines = list(open(path))
for i, line in enumerate(lines):
if re.match(r'^prefix=([^\$]+)\s*$', line):
lines[i] = 'prefix=%s\n' % root
break
else:
with log.indent():
log.warning('No obvious prefix to replace')
continue
# As silly as this seems, *.pc files we have seen have their
# write flag removed, but we still own them (since we just installed
# them). Quickest way to fix: delete them.
if not os.access(path, os.W_OK):
os.unlink(path)
with open(path, 'w') as fh:
fh.writelines(lines)
def factory(cls, step, pkg):
if step not in ('build', 'install'):
return
makefile = find_in_tree(pkg.build_path, 'Makefile')
configure = find_in_tree(pkg.build_path, 'configure') if step == 'build' else None
configure_ac = find_in_tree(pkg.build_path, 'configure.ac') if step == 'build' else None
# We generally provide 'install' via get_next, but if you specify
# --build-sh it will come looking here, and so we must redo that
# logic.
if step == 'install':
if makefile:
# HACK: I was too lazy too refactor the logic for --make-install,
# so I just use this API.
return cls(pkg, (None, None, makefile)).get_next(step)
else:
return
# Warn about both autoconf conflict states.
if configure_ac and not pkg.autoconf:
log.warning('autoconf detected, but --autoconf flag is not set on package')
if configure and pkg.autoconf:
log.warning('--autoconf flag is set on package but ./configure was found')
# Only return with configure.ac iff the user set --autoconf
if (configure_ac and pkg.autoconf) or configure or makefile:
return cls(pkg, (configure_ac, configure, makefile))
def create_if_not_exists(self):
python = os.path.join(self.path, 'bin', 'python')
if not os.path.exists(python):
makedirs(self.path)
print style('Creating Python virtualenv', 'blue',
bold=True), style(self.path, bold=True)
virtualenv.create_environment(self.path,
no_setuptools=True,
no_pip=True)
if not os.path.exists(python + '-config'):
names = (
'python%d.%d-config' % sys.version_info[:2],
'python%d-config' % sys.version_info[0],
'python-config',
)
prefix = getattr(sys, 'real_prefix', sys.prefix)
for name in names:
old_path = os.path.join(prefix, 'bin', name)
if os.path.exists(old_path):
for name in names:
new_path = os.path.join(self.path, 'bin', name)
self.rewrite_shebang_or_link(old_path, new_path)
break
else:
log.warning('Could not find python-config')
def install(self):
pkg = self.package
if pkg.pseudo_homebrew:
homebrew = Homebrew(home=pkg.home)
version = pkg.revision.split('+')[0]
pkg.install_path = os.path.join(homebrew.cellar, pkg.name, version)
log.info(style_note('Re-installing into Homebrew', 'as %s/%s' % (pkg.name, version)))
pkg._assert_paths(install=True)
if pkg.make_install:
log.warning('--make-install specified, but no Makefile found.')
if os.path.exists(pkg.install_path):
log.warning('Removing existing install', pkg.install_path)
shutil.rmtree(pkg.install_path)
if pkg.hard_link:
log.info(style_note('Installing via hard-link', 'to ' + pkg.install_path))
linktree(pkg.build_path_to_install, pkg.install_path_from_build, symlinks=True)
else:
log.info(style_note('Installing via copy', 'to ' + pkg.install_path))
shutil.copytree(pkg.build_path_to_install, pkg.install_path_from_build, symlinks=True)
def rescan(args):
home = args.assert_home()
con = home.db.connect()
args.force = True
pairs = []
if args.names:
for name in args.names:
row = con.execute(
'SELECT path FROM development_packages WHERE name = ? OR name = ?',
[name, os.path.basename(os.path.abspath(name))]).fetchone()
if not row:
log.warning('No dev package named %s' % name)
continue
path = row[0]
if not os.path.exists(path):
log.warning('Dev package %s not longer exists at %s' %
(name, path))
continue
pairs.append((name, path))
else:
pairs = list(
con.execute('SELECT name, path FROM development_packages'))
for name, path in pairs:
args.name = name
args.path = path
init(args, do_add=True)
def callback_target():
try:
fh = os.fdopen(prfd)
except:
os.close(prfd)
raise
while True:
arg_count = fh.readline().strip()
if not arg_count:
break
arg_count = int(arg_count)
args = []
for arg_i in range(arg_count):
arg_len = int(fh.readline())
args.append(fh.read(arg_len))
name = args[0]
if name in callbacks:
try:
res = callbacks[name](*args[1:])
except Exception as e:
log.exception('exception in callback %s: %s' % (name, e))
res = None
else:
log.warning('no callback %s' % name)
res = None
if res is None:
os.write(pwfd, '0\n')
else:
res = str(res)
os.write(pwfd, '%s\n' % len(res))
os.write(pwfd, res)
def parse_file(self, source):
if source == '-':
source = sys.stdin
elif isinstance(source, basestring):
source = open(source, 'r')
line_iter = iter(source)
for line in line_iter:
line = line.rstrip()
while line.endswith('\\'):
line = line[:-1] + next(line_iter).rstrip()
m = re.match(r'^(\s*)([^#]*?)(\s*#.*)?$', line)
before, spec, after = m.groups()
before = before or ''
after = after or ''
if not spec:
self.append((before, '', after))
continue
# Note: This will freak out with colons in comments.
# TODO: Pull parsing from PyHAML.
m = re.match(r'^%\s*(if|elif|else|endif)\s*(.*?):?\s*$', spec)
if m:
type_, expr = m.groups()
self.append((before, Control(type_, expr), after))
continue
m = re.match(r'^(\w+)=(\S.*)$', spec)
if m:
name, value = m.groups()
self._cumulative_environ[name] = value
self.append((before, Envvar(name, value), after))
continue
m = re.match(r'^([\w-]+): (\S.*)$', spec)
if m:
header = Header(*m.groups())
self.headers[header.name] = header
self.append((before, header, after))
continue
try:
pkg = Package(spec, home=self.home)
except RequirementParseError as e:
log.warning('parse error: %s' % e)
self.append(('', '', '# RequirementParseError: %s' % e.args))
self.append(('', '', '# ' + line.strip()))
continue
for k, v in self._cumulative_environ.iteritems():
pkg.base_environ.setdefault(k, v)
self.append((before, pkg, after))
self._guess_names()
def shared_libraries(self, rescan=False):
self._assert_paths(install=True)
if not self.installed:
raise RuntimeError('cannot find libraries if not installed')
if not self.id:
# I'm not sure if this is a big deal, but I want to see when
# it is happening.
log.warning('Finding shared libraries before package is in database.')
return libs.get_installed_shared_libraries(self.home.db.connect(), self.id_or_persist(), self.install_path, rescan)
def get_next(self, step):
if step != 'install':
return
if self.makefile_path:
if self.package.make_install:
return self
else:
log.warning('Skipping `make install` and installing full package.\n'
'Usually you will want to specify one of:\n'
' --make-install\n'
' --build-subdir PATH\n'
' --install-subdir PATH'
)
def _install_wheel(self, pkg):
pkg._assert_paths(install=True)
if pkg.package_path.endswith('.whl'):
log.info(
style_note("Found Python Wheel",
os.path.basename(self.dist_info_dir)))
else:
log.info(
style_note("Found dist-info",
os.path.basename(self.dist_info_dir)))
log.warning("Bare dist-info does not appear to be a wheel.")
wheel_dir, dist_info_name = os.path.split(self.dist_info_dir)
wheel_name = os.path.splitext(dist_info_name)[0]
# Lets just take advantage of pip!
# The only reason we're reading into pip like this is because we
# would rather just do this part, rather than have it go through
# the full process with the *.whl file. If this breaks, feel
# free to do something like:
# pip install --force-reinstall --prefix {pkg.install_path} --no-deps {pkg.package_path}
# along with:
# --no-warn-script-location
# --disable-pip-version-check
# We delay the import just in case the bootstrap is borked.
from pip._internal.operations.install.wheel import install_wheel
from pip._internal.locations import get_scheme
# We may to trick pip into installing into another version's directories.
scheme = get_scheme(self.name, prefix=pkg.install_path)
version = get_default_python().version
src_python = '{}python{}.{}{}'.format(os.path.sep, sys.version_info[0],
sys.version_info[1], os.path.sep)
dst_python = '{}python{}.{}{}'.format(os.path.sep, version[0],
version[1], os.path.sep)
if src_python != dst_python:
for k in 'platlib', 'purelib', 'headers', 'scripts', 'data':
setattr(scheme, k,
getattr(scheme, k).replace(src_python, dst_python))
req = DummyPipRequirement()
req.name = wheel_name
install_wheel(pkg.name, pkg.package_path, scheme,
'<VEE dummy request>')
def _install_setup(self, pkg):
pkg._assert_paths(install=True)
site_packages = get_default_python().rel_site_packages
install_site_packages = os.path.join(pkg.install_path, site_packages)
# Setup the PYTHONPATH to point to the "install" directory.
env = pkg.fresh_environ()
env['PYTHONPATH'] = join_env_path(install_site_packages,
env.get('PYTHONPATH'))
if os.path.exists(pkg.install_path):
log.warning('Removing existing install: ' + pkg.install_path)
shutil.rmtree(pkg.install_path)
os.makedirs(install_site_packages)
log.info(
style_note('Installing Python package',
'to ' + install_site_packages))
cmd = [
'install',
'--root',
pkg.install_path, # Better than prefix
'--prefix',
'.',
# At one point we forced everything into `lib`, so we don't get a
# `lib64`. Virtualenv symlinked them together anyways. But then we
# switched to using pip's internals to unpack wheels, and it would
# place stuff into both `lib` and `lib64`. So we don't really
# know where we stand on this anymore.
'--install-lib',
site_packages,
'--single-version-externally-managed',
]
if not pkg.defer_setup_build:
cmd.append('--skip-build')
res = call_setup_py(self.setup_path,
cmd,
env=env,
indent=True,
verbosity=1)
if res:
raise RuntimeError('Could not install Python package')
def list_(args):
home = args.assert_home()
rows = list(home.db.execute('SELECT * FROM repositories'))
if not rows:
log.warning('No repositories.')
return
max_len = max(len(row['name']) for row in rows)
for row in rows:
repo = EnvironmentRepo(row, home=home)
if repo.exists:
log.info(
style_note(
repo.name,
'%s/%s' % (repo.remote_name, repo.branch_name),
repo.remotes().get(repo.remote_name, '') +
' --default' if row['is_default'] else '',
))
def update(self, force=False):
log.info(style_note('Updating repo', self.name))
self.clone_if_not_exists()
if self.remote_name not in self.remotes():
log.warning('"%s" does not have remote "%s"' % (self.name, self.remote_name))
return True
rev = self.fetch()
if not force and not self.check_ff_safety(rev):
log.error('Cannot fast-forward; skipping.')
return False
self.checkout(force=force)
return True
def install(self):
if not self.setup_path:
return super(PythonBuilder, self).install()
pkg = self.package
pkg._assert_paths(install=True)
install_site_packages = os.path.join(pkg.install_path, site_packages)
# Setup the PYTHONPATH to point to the "install" directory.
env = pkg.fresh_environ()
env['PYTHONPATH'] = '%s:%s' % (install_site_packages,
env.get('PYTHONPATH', ''))
if os.path.exists(pkg.install_path):
log.warning('Removing existing install', pkg.install_path)
shutil.rmtree(pkg.install_path)
os.makedirs(install_site_packages)
log.info(
style_note('Installing Python package',
'to ' + install_site_packages))
cmd = [
'install',
'--root',
pkg.install_path, # Better than prefix
'--prefix',
'.',
'--install-lib',
site_packages, # So that we don't get lib64; virtualenv symlinks them together anyways.
'--single-version-externally-managed',
]
if not pkg.defer_setup_build:
cmd.append('--skip-build')
res = call_setup_py(self.setup_path,
cmd,
env=env,
indent=True,
verbosity=1)
if res:
raise RuntimeError('Could not install Python package')
def persist_in_db(self, con=None):
self._set_names(package=True, build=True, install=True)
if not self.installed:
log.warning('%s does not appear to be installed to %s' % (self.name, self.install_path))
raise ValueError('cannot record requirement that is not installed')
con = con or self.home.db.connect()
with con:
exists = self.id is not None
res = super(Package, self).persist_in_db(con=con)
if exists:
con.execute('DELETE FROM package_dependencies WHERE depender_id = ?', [self.id])
for dep in self.dependencies:
dep_id = dep.id_or_persist(con=con)
log.debug('Recorded %s -> %s dependency as %d' % (
self.name, dep.name, dep_id
))
con.execute('INSERT INTO package_dependencies (depender_id, dependee_id) VALUES (?, ?)', [
self.id, dep_id
])
return res
def rescan(args):
home = args.assert_home()
con = home.db.connect()
args.force = True
repos = []
if args.names:
for name in args.names:
dev_repo = home.find_development_package(name)
if not dev_repo:
log.warning("No dev package: {}".format(name))
continue
repos.append(dev_repo)
else:
repos = list(home.iter_development_packages())
for repo in repos:
args.name = os.path.basename(repo.work_tree)
args.path = repo.work_tree
init(args, do_add=True)
def restore_from_row(self, row, ignore=None):
try:
if self.id and self.id != row['id']:
log.warning('Restoring from a mismatched ID; %s %d != %d' %
(self.__tablename__, self.id, row['id']))
self.id = row['id']
except KeyError:
pass
for col in self.__columns__:
try:
val = row[col.name]
except KeyError:
continue
if ignore and col.name in ignore:
continue
if col._restore:
col._restore(self, val)
else:
self.__dict__[col.name] = val
def parse_file(self, source, filename=None, alt_open=None, _depth=0):
open_ = alt_open or open
if source == '-':
filename = filename or '<stdin>'
source = sys.stdin
elif isinstance(source, str):
filename = filename or source
source = open_(source)
self.filename = self.filename or filename
def append(x):
self._append(ManifestItem(x, prefix, suffix, filename, line_i + 1))
line_iter = iter(source)
for line_i, line in enumerate(line_iter):
line = line.rstrip()
while line.endswith('\\'):
line = line[:-1] + next(line_iter).rstrip()
m = re.match(r'^(\s*)([^#]*?)(\s*#.*)?$', line)
prefix, spec, suffix = m.groups()
if not spec:
append(None)
continue
# Note: This will freak out with colons in comments.
# TODO: Pull parsing from PyHAML.
m = re.match(r'^%\s*(if|elif|else|endif)\s*(.*?):?\s*$', spec)
if m:
type_, expr = m.groups()
append(Control(type_, expr))
continue
m = re.match(r'^%\s*(set|eval|expr)\s+(.+?)\s*$', spec)
if m:
type_, source = m.groups()
append(Expression(source, type_))
continue
m = re.match(r'^%\s*include\s+(.+?)\s*$', spec)
if m:
raw_path = m.group(1)
path = os.path.normpath(raw_path).strip('/')
if raw_path != path:
raise ValueError("Malformed include path.", raw_path)
if self.filename:
path = os.path.join(os.path.dirname(self.filename), path)
other = Manifest(repo=self.repo, home=self.home)
other.parse_file(path, alt_open=alt_open, _depth=_depth + 1)
append(Include(raw_path, other))
continue
m = re.match(r'^(\w+)=(\S.*)$', spec)
if m:
name, value = m.groups()
self._cumulative_environ[name] = value
append(Envvar(name, value))
continue
m = re.match(r'^([\w-]+): (\S.*)$', spec)
if m:
header = Header(*m.groups())
self.headers[header.name] = header
append(header)
continue
try:
pkg = Package(spec, context=self, home=self.home)
except RequirementParseError as e:
log.warning('parse error: %s' % e)
self._append('', '', '# RequirementParseError: %s' % e.args)
self._append('', '', '# ' + line.strip())
continue
for k, v in self._cumulative_environ.items():
pkg.base_environ.setdefault(k, v)
append(pkg)
def __init__(self,
args=None,
*,
home=None,
set=None,
dev=False,
context=None,
parent=None,
source=None,
**kwargs):
super(Package, self).__init__()
source = source or parent
# Must be early due to some properties using this.
self.home = home = home or (source.home if source else None)
if not home:
raise ValueError("Package requires home")
self.context = context = context or (source.context
if source else None)
if not context and not dev:
raise ValueError(
"Package requires context (Manifest) when not dev")
if args and kwargs:
raise ValueError('specify either args OR kwargs')
if isinstance(args, self.__class__):
kwargs = args.to_kwargs()
args = None
elif isinstance(args, dict):
kwargs = args
args = None
if args:
if isinstance(args, six.string_types):
args = shlex.split(args)
if isinstance(args, (list, tuple)):
try:
requirement_parser.parse_args(args, namespace=self)
except RequirementParseError as e:
raise RequirementParseError("%s in %s" % (e.args[0], args))
elif isinstance(args, argparse.Namespace):
for action in requirement_parser._actions:
name = action.dest
setattr(self, name, getattr(args, name))
else:
raise TypeError(
"args must be one of (str, list, tuple, dict); got {}".
format(args.__class__))
else:
for action in requirement_parser._actions:
name = action.dest
# Version is a bit special, and should not have a default applied
# here, otherwise to_kwargs will clear it out.
if name in ('version', ):
try:
value = kwargs.pop(name)
except KeyError:
continue
else:
value = kwargs.pop(name, action.default)
setattr(self, name, value)
if kwargs:
raise ValueError("too many kwargs: {}".format(list(kwargs)))
assert self.url
# Assert we have a name.
if self.name:
if self.name.lower() != self.name:
log.warning("package name {!r} was not lowercase".format(
self.name))
self.name = self.name.lower()
else:
self.name = guess_name(self.url)
# TODO: Deprecate these.
self.dependencies = []
self.set = set
# Variant relationships.
self.parent = parent
self._children = None
self._child_is_self = None
# Make sure to make copies of anything that is mutable.
self.base_environ = self.base_environ.copy(
) if self.base_environ else {}
self.environ = self.environ.copy() if self.environ else {}
self.config = self.config[:] if self.config else []
# Initialize other state not covered by the argument parser.
# TODO: Should this come from the parent?
self.link_id = None
self.package_name = self.build_name = None
self.package_path = self.build_path = self.install_path = None
# Share some state with the parent.
if parent:
self.meta = parent.meta # Directly shared.
self.pipeline = parent.pipeline.copy(self)
else:
self.meta = context.load_meta(self.name) if context else None
self.url = self.get_meta('url') or self.url
self.version = self.get_meta('version') or self.version
self._init_pipeline(dev=dev)
def resolve_existing(self, env=None, weak=False):
"""Check against the database to see if this was already installed."""
if self.id is not None:
raise ValueError('requirement already in database')
cur = self.home.db.cursor()
# Dependencies are deferred.
deferred = self.url.startswith('deferred:')
if deferred:
deferred_id = int(self.url.split(':')[1])
cur.execute('SELECT * from packages WHERE id = ?', [deferred_id])
else:
clauses = ['install_path IS NOT NULL']
values = []
if not weak and self.url:
clauses.append('url = ?')
values.append(self.url)
for name in ('name', 'etag', 'install_name'):
if getattr(self, name):
clauses.append('%s = ?' % name)
values.append(getattr(self, name))
clause = ' AND '.join(clauses)
# log.debug('SELECT FROM packages WHERE %s' % ' AND '.join('%s = %r' % (c.replace(' = ?', ''), v) for c, v in zip(clauses[1:], values)), verbosity=2)
if env:
values.append(env.id_or_persist())
cur.execute(
'''
SELECT packages.*, links.id as link_id FROM packages
LEFT OUTER JOIN links ON packages.id = links.package_id
WHERE %s AND links.environment_id = ?
ORDER BY links.created_at DESC, packages.created_at DESC
''' % clause, values)
else:
cur.execute(
'''
SELECT * FROM packages
WHERE %s
ORDER BY packages.created_at DESC
''' % clause, values)
for row in cur:
# Make sure it has enough provisions.
provides = Provision(row['provides'])
if any(
provides.get(key, None) != value
for key, value in self.provides.items()):
log.debug(
'Found %s (%d) whose provisions %s do not satisfy %s' % (
self.name or row['name'],
row['id'],
row['provides'],
self.provides,
),
verbosity=2)
continue
# Make sure it has enough requirements.
requires = RequirementSet(row['requires'])
reqs_ok = True
for name in self.requires:
try:
reqs = requires[name]
except KeyError:
reqs_ok = False
continue
if any(
reqs.get(key, None) != value
for key, value in self.requires.items()):
reqs_ok = False
continue
if not reqs_ok:
log.debug(
'Found %s (%d) whose requirements %s do not satisfy %s' % (
self.name or row['name'],
row['id'],
row['requires'],
self.requires,
),
verbosity=2)
if not os.path.exists(row['install_path']):
log.warning(
'Found %s (%d) does not exist at %s' %
(self.name or row['name'], row['id'], row['install_path']))
continue
break
else:
if deferred:
raise ValueError(
'deferred package %d no longer exists; consider `vee gc`' %
deferred_id)
return
log.debug('Found %s (%d%s%s) at %s' % (
self.name or row['name'],
row['id'],
' weakly' if weak else '',
' in env %d' % env.id if env else '',
row['install_path'],
))
self.restore_from_row(row)
self.link_id = row.get('link_id')
if deferred:
self._init_pipeline()
self._load_dependencies(cur)
return True
def _install_one(self, names, name, link_env, reinstall, relink, no_deps):
try:
pkg = self[name]
except KeyError:
# print(', '.join(sorted(self.keys())))
raise
reinstall_this = name in reinstall
relink_this = name in relink
if name not in self._extracted:
try:
# Between every step, take a look to see if we now have
# enough information to tell that it is already installed.
pkg.assert_uninstalled(uninstall=reinstall_this)
pkg.pipeline.run_to('fetch')
pkg.assert_uninstalled(uninstall=reinstall_this)
pkg.pipeline.run_to('extract')
pkg.assert_uninstalled(uninstall=reinstall_this)
pkg.pipeline.run_to('inspect')
pkg.assert_uninstalled(uninstall=reinstall_this)
except AlreadyInstalled:
self._installed.add(name)
finally:
self._extracted.add(name)
# Loop around for dependencies. We insert dependencies, and the
# package itself, back into the names to check. If we get back to
# a name that we have already deferred in this manner, we continue
# anyways, since that means there is a dependency cycle. We assume
# that dependency order is resolved in the requirements file.
deferred = False
deps_installed = True
insert_i = 0
for i, dep in ([] if no_deps else enumerate(pkg.dependencies)):
# Since resolution is rather loose in here (only by name, not URL)
# we want to replace dependencies with their concrete variant to
# ease recording that into the database.
dep = self.resolve(dep, weak=True)
pkg.dependencies[i] = dep
self._parent_names.setdefault(dep.name, pkg.name)
# We must check if the dependency errored, otherwise we will still
# end up in an infinite loop.
if dep.name in self._errored:
log.warning('Skipping due to error in %s' % dep.name)
self._errored.add(pkg.name)
return
if dep.name not in self._installed:
key = (name, dep.name)
if key not in self._deferred:
log.debug('%s needs %s; deferring install' %
(name, dep.name))
self._deferred.add(key)
deferred = True
else:
log.debug('%s needs %s, but install was already deferred' %
(name, dep.name))
deps_installed = False
names.insert(insert_i, dep.name)
insert_i += 1
if deferred:
names.insert(insert_i, name)
return
pre_build_deps = pkg.dependencies[:]
if name not in self._installed:
try:
pkg.pipeline.run_to('build')
pkg.pipeline.run_to('install')
pkg.pipeline.run_to('relocate')
except AlreadyInstalled:
pass
pkg.pipeline.run_to('optlink')
self._installed.add(name)
# We need to build/install Homebrew packages before we can decide
# which of their optional dependencies will be used. The relocation
# process can also determine other dependencies. We need to run
# these new ones through the pipe too.
if pkg.dependencies != pre_build_deps:
log.debug('%s has changed dependencies after build/install')
names.insert(insert_i, name)
return
if pkg.virtual:
return
if name not in self._persisted:
# We need to wait to persist until all dependencies are
# installed.
if not deps_installed:
log.debug('%s cannot persist without all dependencies' %
(name, ))
names.insert(insert_i, name)
return
pkg.persist_in_db()
pkg.shared_libraries() # TODO: Move this earlier?
self._persisted.add(name)
if link_env and name not in self._linked:
try:
pkg.link(link_env, force=relink_this)
except AlreadyLinked:
pass
self._linked.add(name)
def install(self,
names=None,
link_env=None,
reinstall=False,
relink=False,
no_deps=False):
# I'd love to split this method into an "install" and "link" step, but
# then we'd need to reimplement the dependency resolution. That would
# be a good idea to do anyways, but... meh.
if isinstance(names, str):
names = [names]
names = list(names if names else self.keys())
for name in names:
if name not in self:
raise KeyError(name)
if not isinstance(reinstall, set):
reinstall = set(
names if no_deps else self.keys()) if reinstall else set()
if not isinstance(relink, set):
relink = set(
names if no_deps else self.keys()) if relink else set()
while names:
name = names.pop(0)
self._parent_names.setdefault(name, None)
parent_chain = []
tip = name
while tip and tip not in parent_chain:
parent_chain.append(tip)
tip = self._parent_names.get(tip)
parent_chain = parent_chain[1:]
print(
'==>', style(name, 'blue'),
style('(%s)' % ' < '.join(parent_chain), faint=True)
if parent_chain else '')
with log.indent():
# Avoid infinite error loops.
if name in self._errored:
log.warning('Skipping due to previous error.')
continue
try:
self._install_one(names, name, link_env, reinstall, relink,
no_deps)
except PipelineError as e:
self._errored.add(name)
log.error(str(e))
continue
except Exception as e:
self._errored.add(name)
print_cli_exc(e, verbose=True)
log.exception('Exception while processing %s' % name)
continue
if self._errored:
log.warning('There were errors in: %s' %
', '.join(sorted(self._errored)))
def inspect(self, pkg):
if self.setup_path:
stdout = call_setup_py(self.setup_path, ['egg_info'],
env=pkg.fresh_environ(),
stdout=True).decode()
m = re.search(r'writing requirements to (.+?)\n', stdout)
if not m:
log.debug("No requirements")
return
requirements_path = os.path.join(os.path.dirname(self.setup_path),
m.group(1))
for line in open(requirements_path):
# Stop once we get to the "extras".
if line.startswith('['):
break
m = re.match(r'^([\w\.-]+)', line)
if m:
name = m.group(1).lower()
log.debug('%s depends on %s' % (pkg.name, name))
pkg.add_dependency(name=name, url='pypi:%s' % name.lower())
if self.dist_info_dir:
for line in open(os.path.join(self.dist_info_dir, 'METADATA')):
line = line.strip()
if not line:
break # We're at the end of the headers.
key, value = line.split(': ', 1)
key = key.lower()
if key == 'requires-dist':
# Environmental markers look like `FOO; extra == 'BAR'`.
if ';' in value:
value, raw_marker = value.split(';')
value = value.strip()
# We delay the import just in case the bootstrap is borked.
from packaging.markers import Marker
marker = Marker(raw_marker)
if not marker.evaluate({'extra': None}):
continue
m = re.match(r'([\w-]+)(?:\s+\(([^)]+)\))?', value)
if not m:
log.warning(
'Could not parse requires-dist {!r}'.format(value))
continue
dep_name, version_expr = m.groups()
pkg.add_dependency(
name=dep_name,
url='pypi:{}'.format(dep_name),
version=version_expr,
)
def fetch(self, pkg):
meta = self._get_meta(pkg)
all_releases = [(Version(v), rs) for v, rs in meta['releases'].items()]
all_releases.sort(reverse=True)
if not all_releases:
raise ValueError('no releases of {} (any version) on the PyPI'.format(self.name))
if pkg.version:
version_expr = VersionExpr(pkg.version)
matching_releases = [(v, rs) for v, rs in all_releases if version_expr.eval(v)]
if not matching_releases:
for v, rs in all_releases:
print(v)
raise ValueError('no releases of {} {} on the PyPI'.format(self.name, pkg.version))
else:
matching_releases = all_releases
supported_tags = get_supported_tags()
usable_releases = []
for version, releases in matching_releases:
for release in releases:
if release['packagetype'] == 'sdist':
usable_releases.append((version, 0, release))
continue
elif release['packagetype'] == 'bdist_wheel':
m = re.match(r'^(.+)-([^-]+)-([^-]+)-([^-]+)-([^-]+)\.whl$', release['filename'])
if not m:
log.warning("Could not parse wheel filename: {}".format(release['filename']))
name, version_tag, python_tag, abi_tag, platform_tags = m.groups()
# Platform tags can have multiple seperated by dots.
for platform_tag in platform_tags.split('.'):
tags = (python_tag, abi_tag, platform_tag)
if tags in supported_tags:
break
else:
continue
usable_releases.append((version, 1, release))
if not usable_releases:
raise ValueError('no usable release of %s %s on the PyPI;' % (self.name, expr if pkg.version else '(any version)'))
usable_releases.sort(key=lambda x: x[:2])
version, _, release = usable_releases[-1]
pkg.version = str(version)
if release.get('md5_digest'):
pkg.checksum = 'md5:%s' % release['md5_digest']
pkg.package_name = os.path.join(self.name, os.path.basename(release['url']))
pkg._assert_paths(package=True)
if os.path.exists(pkg.package_path):
log.info(style_note('Already downloaded', release['url']))
return
log.info(style_note('Downloading', release['url']))
download(release['url'], pkg.package_path)
def _relocate_darwin_library(lib_path, con, flags, include, exclude, dry_run,
target_cache):
auto = 'auto' in flags
lib_id, lib_deps = get_dependencies(lib_path)
id_versions = set(
name_variants(os.path.basename(lib_id),
version_only=True)) if lib_id else set()
lib_versions = set(
name_variants(os.path.basename(lib_path), version_only=True))
cmd = ['install_name_tool']
if lib_id != lib_path:
log.info('id %s' % (lib_path), verbosity=1)
cmd.extend(('-id', lib_path))
lib_def, lib_undef = get_symbols(lib_path)
for dep_i, dep_path in enumerate(lib_deps):
if dep_path == lib_id:
log.warning('The ID is included?! %s' % lib_path)
cmd.extend(('-change', dep_path, lib_path))
continue
# If the dependency is similarly named to the library itself, then we
# assume it is its own dependency. Which I don't understand...
dep_versions = set(
name_variants(os.path.basename(dep_path), version_only=True))
if dep_versions.intersection(id_versions) or dep_versions.intersection(
lib_versions):
log.warning('Library depends on itself?! %s' % dep_path)
cmd.extend(('-change', dep_path, lib_path))
continue
do_exclude = any(dep_path.startswith(x) for x in exclude)
if not do_exclude and os.path.exists(dep_path):
log.debug('skipping %s' % dep_path)
continue
dep_name = os.path.basename(dep_path)
targets = []
for variant in name_variants(dep_name):
if variant in target_cache:
targets.extend(target_cache[variant])
if auto:
cur = con.execute(
'SELECT path FROM shared_libraries WHERE name = ? ORDER BY created_at DESC',
[variant])
new_targets = target_cache.setdefault(variant, [])
new_targets.extend([row[0] for row in cur])
targets.extend(new_targets)
# Go searching for the "best" relocation target.
# The one with the most defined symbols missing from the lib wins
# (essentially; it is more complex then that below). We also, dubiously,
# accept libraries which provide no matching symbols as long as they
# don't introduct any conflicts. There are a TON of these in FFmpeg.
best_score = -1
best_target = None
seen_targets = set()
for target in targets:
if target in seen_targets:
continue
seen_targets.add(target)
if not os.path.exists(target):
continue
tar_def, tar_undef = get_symbols(target)
pros = len(tar_def.intersection(lib_undef))
shared = len(tar_def.intersection(lib_def))
cons = len(lib_undef.intersection(lib_def))
log.debug('+%d ~%d -%d %s' % (pros, shared, cons, target),
verbosity=2)
if pros - shared - cons > best_score:
best_score = pros - shared - cons
best_target = (pros, shared, cons, target)
if best_target is None:
log.warning('No relocation targets for %s' % dep_path)
continue
if best_score < 0:
log.warning('No positive relocation targets for %s' % dep_path)
continue
if best_target[1] or best_target[2]:
log.warning('Best target has %s collisions for %s' %
(best_target[1] + best_target[2], dep_path))
target = best_target[3]
log.info('change %s -> %s' % (dep_name, target), verbosity=1)
cmd.extend(('-change', dep_path, target))
if len(cmd) > 1 and not dry_run:
cmd.append(lib_path)
s = os.stat(lib_path)
if not s.st_mode & stat.S_IWUSR:
os.chmod(lib_path, s.st_mode | stat.S_IWUSR)
call(cmd)
os.chmod(lib_path, s.st_mode)
else:
call(cmd)
def gc(args):
home = args.assert_home()
con = home.db.connect()
with con:
repo_ids = {}
for row in con.execute('SELECT id, name from repositories'):
repo_ids[row['name']] = row['id']
envs_by_id = {}
log.info(style_note('Cleaning environments'))
for row in con.execute(
'SELECT id, name, path, repository_id from environments ORDER BY created_at ASC'
):
id_, name, path, repo_id = row
if not os.path.exists(path):
log.info('environment does not exist at %s; deleting' % (path))
if not args.dry_run:
delete_environment(con, id_)
continue
# Track for later.
envs_by_id.setdefault(repo_id, []).append((id_, name, path))
# The rest is making sure the repo_id and commit are correct.
if repo_id:
continue
m = re.match(r'(\w+)/commits/([0-9a-f]{7,8}(?:-dirty)?)$', name)
if not m:
log.warning(
'%s (%d) does not appear to be managed by git; skipping' %
(name, id_))
continue
repo_name, commit_name = m.groups()
repo_id = repo_ids.get(repo_name)
if not repo_id:
log.warning('repo %s does not exist for %s (%d); skipping' %
(repo_name, name, id_))
continue
log.info('Fixing repo relationship for %s (%d)' % (name, id_))
if not args.dry_run:
con.execute(
'UPDATE environments SET repository_id = ?, repository_commit = ? WHERE id = ?',
[repo_id, commit_name, id_])
if args.prune_environments:
log.info(style_note('Pruning old environments'))
for repo_id, envs in sorted(envs_by_id.iteritems()):
for id_, name, path in envs[:-args.keep_latest]:
log.info('Deleting %s (%d)' % (name, id_))
if not args.dry_run:
shutil.rmtree(path)
delete_environment(con, id_)
log.info(style_note('Cleaning installed packages'))
package_ids = []
install_paths_to_id = {}
for row in con.execute(
'SELECT id, name, install_path, build_path from packages ORDER by created_at DESC'
):
id_, name, install_path, build_path = row
log.debug('%s %s %s' % (id_, name, install_path))
if not os.path.exists(install_path):
log.info('%s no longer exists at %s; deleting' %
(name, install_path))
if not args.dry_run:
delete_package(con, id_)
continue
real_id = install_paths_to_id.get(install_path)
if real_id:
log.info('%s %d is a duplicate of %s; deleting' %
(name, id_, real_id))
# TODO: update any links or package_dependencies which to point to this.
if not args.dry_run:
delete_package(con, id_)
continue
install_paths_to_id[install_path] = id_
if args.prune_orphaned_packages:
row = con.execute(
'SELECT count(1) FROM links WHERE package_id = ?',
[id_]).fetchone()
if not row[0]:
log.info('%s (%d) is not linked; deleting' % (name, id_))
if not args.dry_run:
if build_path and os.path.exists(build_path):
shutil.rmtree(build_path)
shutil.rmtree(install_path)
delete_package(con, id_)
def upgrade(self,
dirty=False,
subset=None,
reinstall=False,
relink=False,
no_deps=False,
force_branch_link=True):
self.clone_if_not_exists()
try:
head = self.head
except CalledProcessError:
log.warning(style_warning('no commits in repository'))
head = None
try:
remote_head = self.rev_parse('%s/%s' %
(self.remote_name, self.branch_name))
except ValueError:
log.warning(
style_warning('tracked %s/%s does not exist in self' %
(self.remote_name, self.branch_name)))
remote_head = None
if remote_head and head != remote_head:
log.warning(
style_warning('%s repo not checked out to %s/%s' %
(self.name, self.remote_name, self.branch_name)))
dirty = bool(list(self.status()))
if not dirty and self.is_dirty():
log.error('%s repo is dirty; force with --dirty' % self.name)
return False
env = self.get_environment()
req_set = self.load_requirements()
pkg_set = PackageSet(env=env, home=self.home)
# Register the whole set, so that dependencies are pulled from here instead
# of weakly resolved from installed packages.
# TODO: This blanket reinstalls things, even if no_deps is set.
pkg_set.resolve_set(req_set, check_existing=not reinstall)
# Install and/or link.
pkg_set.install(subset or None,
link_env=env,
reinstall=reinstall,
relink=relink,
no_deps=no_deps)
if pkg_set._errored and not force_branch_link:
log.warning(
style_warning(
"Not creating branch or version links; force with --force-branch-link"
))
return False
# Create a symlink by branch.
path_by_branch = self.home._abs_path('environments', self.name,
self.branch_name)
if os.path.lexists(path_by_branch):
os.unlink(path_by_branch)
makedirs(os.path.dirname(path_by_branch))
os.symlink(env.path, path_by_branch)
# Create a symlink by version.
version = req_set.headers.get('Version')
if version:
path_by_version = self.home._abs_path(
'environments', self.name, 'versions',
version.value + ('-dirty' if dirty else ''))
if os.path.lexists(path_by_version):
os.unlink(path_by_version)
makedirs(os.path.dirname(path_by_version))
os.symlink(env.path, path_by_version)
return True
