def build(self):
log.info(style_note('source %s' % os.path.basename(self.build_sh)))
pkg = self.package
pkg._assert_paths(build=True, install=True)
env = pkg.fresh_environ()
env.update(
VEE=pkg.home.root,
VEE_BUILD_PATH=pkg.build_path,
VEE_INSTALL_NAME=pkg.install_name,
VEE_INSTALL_PATH=pkg.install_path,
)
# TODO: somehow derive this from --build-sh provided script.
cwd = os.path.dirname(self.build_sh)
envfile = os.path.join(cwd, 'vee-env-' + os.urandom(8).encode('hex'))
call([
'bash', '-c',
'. %s; env | grep VEE > %s' %
(os.path.basename(self.build_sh), envfile)
],
env=env,
cwd=cwd)
env = list(open(envfile))
env = dict(line.strip().split('=', 1) for line in env)
os.unlink(envfile)
pkg.build_subdir = env.get('VEE_BUILD_SUBDIR') or ''
pkg.install_prefix = env.get('VEE_INSTALL_PREFIX') or ''
def git(args, *command):
"""Run a ``git`` command on a environment repository's git repository.
(Sorry for the name collision.)
e.g.::
$ vee git -r primary status
On branch master
Your branch is behind 'origin/master' by 1 commit, and can be fast-forwarded.
(use "git pull" to update your local branch)
nothing to commit, working directory clean
"""
home = args.assert_home()
repo = home.get_env_repo(args.repo)
if args.stree:
call(['stree', repo.work_tree])
return
if not command:
print style_error('please provide a git command')
return 1
makedirs(repo.work_tree)
repo.git(*command, verbosity=0, indent=False)
def _relocate_linux_library(lib_path, include, dry_run):
rpath = ':'.join(include)
log.info('set rpath to %s' % rpath)
if dry_run:
return
call(['patchelf', '--set-rpath', rpath, lib_path])
def install(self, pkg):
pkg._assert_paths(install=True)
# TODO: Find the Ruby version.
root = os.path.join(pkg.install_path, 'lib/ruby/2.0.0')
makedirs(root)
cmd = ['gem', 'install', pkg.name]
if pkg.config:
cmd.append('--')
cmd.extend(pkg.render_template(x) for x in pkg.config)
call(cmd, env={'GEM_HOME': root})
def get_dependencies(path):
ids = _parse_otool(call(['otool', '-D', path], stdout=True))
id_ = ids[0] if ids else None
deps = _parse_otool(call(['otool', '-L', path], stdout=True))
# Trim off the ID.
if deps and id_ and deps[0] == id_:
deps = deps[1:]
return id_, deps
def get_symbols(path):
if path not in _symbol_cache:
undefined = set()
defined = set()
raw = call(['nm', '-gP', path], stdout=True)
for line in raw.splitlines():
line = line.strip()
if not line:
continue
name, type_, _ = line.split(None, 2)
# Some symbols can safely be ignored. E.g. "_main", which we
# sometimes see left over in some shared libs.
if name in IGNORE_SYMBOLS:
continue
if type_ in 'TDBC':
defined.add(name)
elif type_ == 'U':
undefined.add(name)
_symbol_cache[path] = frozenset(defined), frozenset(undefined)
return _symbol_cache[path]
def system_gems(self):
gems = {}
out = call(['gem', 'list', '--no-details'], stdout=True)
for line in out.splitlines():
m = re.match(r'^(\w+) \((.+?)\)', line.strip())
if m:
gems[m.group(1)] = m.group(2)
return gems
def call_setup_py(setup_py, args, **kwargs):
kwargs['cwd'] = os.path.dirname(setup_py)
cmd = [
'python', '-c',
'import sys, setuptools; sys.argv[0]=__file__=%r; execfile(__file__)' %
os.path.basename(setup_py)
]
cmd.extend(args)
return call(cmd, **kwargs)
def __call__(self, cmd, *args, **kwargs):
brew_head = os.environ.get('VEE_HOMEBREW_COMMIT')
core_head = os.environ.get('VEE_HOMEBREW_CORE_COMMIT')
bin_ = os.path.join(self.repo.work_tree, 'bin', 'brew')
if self.repo.clone_if_not_exists(shallow=not brew_head):
# Homebrew should be updated the first time, since it has gotten
# a little more complicated. We call it directly so that
# it can update itself here, and then our pin will drag it back.
call((bin_, 'update'))
if brew_head:
if self.repo.is_shallow:
self.repo.fetch(shallow=False)
self.repo.checkout(brew_head)
if core_head:
repo = GitRepo(
os.path.join(self.repo.work_tree, 'Library', 'Taps',
'homebrew', 'homebrew-core'),
'https://github.com/homebrew/homebrew-core.git',
)
repo.clone_if_not_exists(shallow=False)
if repo.is_shallow:
repo.fetch(shallow=False)
repo.checkout(core_head)
# We need to own the homebrew cache so that we can control permissions.
kwargs['env'] = env = kwargs.get('env', os.environ).copy()
env.setdefault('HOMEBREW_CACHE',
os.path.join(self.repo.work_tree, 'Cache'))
env.setdefault('HOMEBREW_LOGS',
os.path.join(self.repo.work_tree, 'Logs'))
# Keep our pin.
if brew_head or core_head:
env['HOMEBREW_NO_AUTO_UPDATE'] = '1'
res = call((bin_, cmd) + args, _frame=1, **kwargs)
if cmd in ('install', 'uninstall'):
self._info.pop(args[0], None)
return res
def install(self):
pkg = self.package
pkg._assert_paths(install=True)
log.info(style_note('make install'))
if call(
['make', 'install', '-j4'],
cwd=os.path.dirname(self.makefile_path),
env=pkg.fresh_environ(),
):
raise RuntimeError('Could not `make install` package')
def git(self, *cmd, **kw):
stderr = []
kw['stderr'] = [kw.get('stderr'), stderr.append]
kw['check'] = False # We will do it ourselves.
kw.setdefault('decode', True)
e = None
try:
# If we can, run as if we are within the work tree.
if self.work_tree and self.git_dir == os.path.join(
self.work_tree, '.git'):
kw['cwd'] = self.work_tree
res = call(('git', ) + cmd, **kw)
else:
res = call(('git', '--git-dir', self.git_dir, '--work-tree',
self.work_tree) + cmd, **kw)
except CalledProcessError as e:
res = e.returncode
stderr = b''.join(stderr).rstrip().decode()
fatal = []
nonfatal = False
for line in stderr.splitlines():
if line.startswith('fatal:'):
fatal.append(line[6:].strip())
elif line.strip():
nonfatal = True
if fatal:
raise GitError(*fatal,
errno=res,
detail=stderr if nonfatal else None)
if isinstance(res, int) and res:
raise GitError('%s returned %d; %s' %
(' '.join(cmd), res, stderr.strip()),
errno=res,
detail=stderr if nonfatal else None)
if e: # This should never happen.
raise
return res
def extract(self):
"""Extract the package into the (cleaned) build directory."""
pkg = self.package
pkg._assert_paths(build=True)
if pkg.checksum:
log.info(style_note('Verifying checksum',
'of ' + pkg.package_path),
verbosity=1)
assert_file_checksum(pkg.package_path, pkg.checksum)
log.info(
style_note('Expanding %s to' % self.archive_type, pkg.build_path))
pkg._clean_build_path()
# gzip-ed Tarballs.
if self.archive_type == 'tar+gzip':
call(['tar', 'xzf', pkg.package_path], cwd=pkg.build_path)
# bzip-ed Tarballs.
elif self.archive_type == 'tar+bzip':
call(['tar', 'xjf', pkg.package_path], cwd=pkg.build_path)
# Zip files (and Python wheels).
elif self.archive_type == 'zip':
call(['unzip', pkg.package_path], cwd=pkg.build_path)
def clone_if_not_exists(self, remote_url=None, shallow=True):
"""Assert that the repo has been cloned. Return True if it did not exist."""
self.remote_url = remote_url or self.remote_url
if self.exists:
return False
if not self.remote_url:
raise ValueError('git repo %r does not exist; need remote url' %
self.git_dir)
if os.path.exists(self.work_tree):
call(['git', 'init', '--bare', self.git_dir])
self.git('remote', 'add', 'origin', self.remote_url)
self.git('config', '--unset', 'core.bare')
if shallow:
# TODO: non-master
self.git('pull', '--ff-only', '--depth=1', 'origin',
get_default_branch())
else:
# TODO: non-master
self.git('pull', '--ff-only', 'origin', get_default_branch())
elif shallow:
log.info(style_note('Cloning shallow', self.remote_url))
call(
['git', 'clone', '--depth=1', self.remote_url, self.work_tree])
else:
log.info(style_note('Cloning', self.remote_url))
call(['git', 'clone', self.remote_url, self.work_tree])
return True
def __call__(self, cmd, *args, **kwargs):
self.repo.clone_if_not_exists()
bin = os.path.join(self.repo.work_tree, 'bin', 'brew')
# We need to own the homebrew cache so that we can control permissions.
kwargs['env'] = env = kwargs.get('env', os.environ).copy()
env.setdefault('HOMEBREW_CACHE',
os.path.join(self.repo.work_tree, 'Cache'))
res = call((bin, cmd) + args, _frame=1, **kwargs)
if cmd in ('install', 'uninstall'):
self._info.pop(args[0], None)
return res
def install(self):
log.info(style_note('source %s' % os.path.basename(self.install_sh)))
pkg = self.package
pkg._assert_paths(build=True, install=True)
env = pkg.fresh_environ()
env.update(
VEE=pkg.home.root,
VEE_BUILD_PATH=pkg.build_path,
VEE_INSTALL_NAME=pkg.install_name,
VEE_INSTALL_PATH=pkg.install_path,
)
cwd = os.path.dirname(self.install_sh)
with log.indent():
call([
'bash', '-c',
'source "%s" "%s"' % (self.install_sh, pkg.install_path)
],
env=env,
cwd=cwd)
def call_setup_py(setup_py, args, **kwargs):
kwargs['cwd'] = os.path.dirname(setup_py)
kwargs.setdefault('vee_in_env', True)
executable = get_default_python().executable
cmd = [
executable, '-sc', '''import sys
sys.path.append({!r})
import setuptools
sys.argv[0] = __file__ = {!r}
exec(compile(open(__file__).read(), __file__, 'exec'))
'''.format(get_base_site_packages(), os.path.basename(setup_py))
]
cmd.extend(('--command-packages', 'vee.distutils'))
cmd.extend(args)
return call(cmd, **kwargs)
def installed_packages(self):
if _installed_packages:
return _installed_packages
packages = _installed_packages
out = call(['rpm', '-qa'], stdout=True)
for line in out.splitlines():
line = line.strip().lower()
if not line:
continue
packages.add(line)
chunks = line.split('-')
for i in range(1, len(chunks)):
packages.add('-'.join(chunks[:i]))
chunks = line.split('.')
for i in range(1, len(chunks)):
packages.add('.'.join(chunks[:i]))
return packages
def build(self):
pkg = self.package
env = None
if self.configure_ac_path and not self.configure_path:
bootstrap = os.path.join(os.path.dirname(self.configure_ac_path), 'bootstrap')
if os.path.exists(bootstrap):
log.info(style_note('./bootstrap', '(autoreconf)'))
cmd = ['./bootstrap']
else:
log.info(style_note('autoreconf'))
cmd = ['autoreconf', '--install', '--force']
env = env or pkg.fresh_environ()
call(cmd, cwd=os.path.dirname(self.configure_ac_path), env=env)
pkg.build_subdir = os.path.dirname(self.configure_ac_path)
# Need to look for it again.
self.configure_path = self.configure_path or find_in_tree(pkg.build_path, 'configure')
if self.configure_path:
log.info(style_note('./configure'))
pkg._assert_paths(install=True)
cmd = ['./configure', '--prefix', pkg.install_path]
cmd.extend(pkg.config)
env = env or pkg.fresh_environ()
call(cmd, cwd=os.path.dirname(self.configure_path), env=env)
pkg.build_subdir = os.path.dirname(self.configure_path)
# Need to look for it again.
self.makefile_path = self.makefile_path or find_in_tree(pkg.build_path, 'Makefile')
if self.makefile_path:
log.info(style_note('make'))
env = env or pkg.fresh_environ()
call(['make', '-j4'], cwd=os.path.dirname(self.makefile_path), env=env)
pkg.build_subdir = os.path.dirname(self.makefile_path)
def _relocate_darwin_library(lib_path, con, flags, include, exclude, dry_run,
target_cache):
auto = 'auto' in flags
lib_id, lib_deps = get_dependencies(lib_path)
id_versions = set(
name_variants(os.path.basename(lib_id),
version_only=True)) if lib_id else set()
lib_versions = set(
name_variants(os.path.basename(lib_path), version_only=True))
cmd = ['install_name_tool']
if lib_id != lib_path:
log.info('id %s' % (lib_path), verbosity=1)
cmd.extend(('-id', lib_path))
lib_def, lib_undef = get_symbols(lib_path)
for dep_i, dep_path in enumerate(lib_deps):
if dep_path == lib_id:
log.warning('The ID is included?! %s' % lib_path)
cmd.extend(('-change', dep_path, lib_path))
continue
# If the dependency is similarly named to the library itself, then we
# assume it is its own dependency. Which I don't understand...
dep_versions = set(
name_variants(os.path.basename(dep_path), version_only=True))
if dep_versions.intersection(id_versions) or dep_versions.intersection(
lib_versions):
log.warning('Library depends on itself?! %s' % dep_path)
cmd.extend(('-change', dep_path, lib_path))
continue
do_exclude = any(dep_path.startswith(x) for x in exclude)
if not do_exclude and os.path.exists(dep_path):
log.debug('skipping %s' % dep_path)
continue
dep_name = os.path.basename(dep_path)
targets = []
for variant in name_variants(dep_name):
if variant in target_cache:
targets.extend(target_cache[variant])
if auto:
cur = con.execute(
'SELECT path FROM shared_libraries WHERE name = ? ORDER BY created_at DESC',
[variant])
new_targets = target_cache.setdefault(variant, [])
new_targets.extend([row[0] for row in cur])
targets.extend(new_targets)
# Go searching for the "best" relocation target.
# The one with the most defined symbols missing from the lib wins
# (essentially; it is more complex then that below). We also, dubiously,
# accept libraries which provide no matching symbols as long as they
# don't introduct any conflicts. There are a TON of these in FFmpeg.
best_score = -1
best_target = None
seen_targets = set()
for target in targets:
if target in seen_targets:
continue
seen_targets.add(target)
if not os.path.exists(target):
continue
tar_def, tar_undef = get_symbols(target)
pros = len(tar_def.intersection(lib_undef))
shared = len(tar_def.intersection(lib_def))
cons = len(lib_undef.intersection(lib_def))
log.debug('+%d ~%d -%d %s' % (pros, shared, cons, target),
verbosity=2)
if pros - shared - cons > best_score:
best_score = pros - shared - cons
best_target = (pros, shared, cons, target)
if best_target is None:
log.warning('No relocation targets for %s' % dep_path)
continue
if best_score < 0:
log.warning('No positive relocation targets for %s' % dep_path)
continue
if best_target[1] or best_target[2]:
log.warning('Best target has %s collisions for %s' %
(best_target[1] + best_target[2], dep_path))
target = best_target[3]
log.info('change %s -> %s' % (dep_name, target), verbosity=1)
cmd.extend(('-change', dep_path, target))
if len(cmd) > 1 and not dry_run:
cmd.append(lib_path)
s = os.stat(lib_path)
if not s.st_mode & stat.S_IWUSR:
os.chmod(lib_path, s.st_mode | stat.S_IWUSR)
call(cmd)
os.chmod(lib_path, s.st_mode)
else:
call(cmd)
