def build(self): log.info(style_note('source %s' % os.path.basename(self.build_sh))) pkg = self.package pkg._assert_paths(build=True, install=True) env = pkg.fresh_environ() env.update( VEE=pkg.home.root, VEE_BUILD_PATH=pkg.build_path, VEE_INSTALL_NAME=pkg.install_name, VEE_INSTALL_PATH=pkg.install_path, ) # TODO: somehow derive this from --build-sh provided script. cwd = os.path.dirname(self.build_sh) envfile = os.path.join(cwd, 'vee-env-' + os.urandom(8).encode('hex')) call([ 'bash', '-c', '. %s; env | grep VEE > %s' % (os.path.basename(self.build_sh), envfile) ], env=env, cwd=cwd) env = list(open(envfile)) env = dict(line.strip().split('=', 1) for line in env) os.unlink(envfile) pkg.build_subdir = env.get('VEE_BUILD_SUBDIR') or '' pkg.install_prefix = env.get('VEE_INSTALL_PREFIX') or ''
def git(args, *command): """Run a ``git`` command on a environment repository's git repository. (Sorry for the name collision.) e.g.:: $ vee git -r primary status On branch master Your branch is behind 'origin/master' by 1 commit, and can be fast-forwarded. (use "git pull" to update your local branch) nothing to commit, working directory clean """ home = args.assert_home() repo = home.get_env_repo(args.repo) if args.stree: call(['stree', repo.work_tree]) return if not command: print style_error('please provide a git command') return 1 makedirs(repo.work_tree) repo.git(*command, verbosity=0, indent=False)
def _relocate_linux_library(lib_path, include, dry_run): rpath = ':'.join(include) log.info('set rpath to %s' % rpath) if dry_run: return call(['patchelf', '--set-rpath', rpath, lib_path])
def install(self, pkg): pkg._assert_paths(install=True) # TODO: Find the Ruby version. root = os.path.join(pkg.install_path, 'lib/ruby/2.0.0') makedirs(root) cmd = ['gem', 'install', pkg.name] if pkg.config: cmd.append('--') cmd.extend(pkg.render_template(x) for x in pkg.config) call(cmd, env={'GEM_HOME': root})
def get_dependencies(path): ids = _parse_otool(call(['otool', '-D', path], stdout=True)) id_ = ids[0] if ids else None deps = _parse_otool(call(['otool', '-L', path], stdout=True)) # Trim off the ID. if deps and id_ and deps[0] == id_: deps = deps[1:] return id_, deps
def get_symbols(path): if path not in _symbol_cache: undefined = set() defined = set() raw = call(['nm', '-gP', path], stdout=True) for line in raw.splitlines(): line = line.strip() if not line: continue name, type_, _ = line.split(None, 2) # Some symbols can safely be ignored. E.g. "_main", which we # sometimes see left over in some shared libs. if name in IGNORE_SYMBOLS: continue if type_ in 'TDBC': defined.add(name) elif type_ == 'U': undefined.add(name) _symbol_cache[path] = frozenset(defined), frozenset(undefined) return _symbol_cache[path]
def system_gems(self): gems = {} out = call(['gem', 'list', '--no-details'], stdout=True) for line in out.splitlines(): m = re.match(r'^(\w+) \((.+?)\)', line.strip()) if m: gems[m.group(1)] = m.group(2) return gems
def call_setup_py(setup_py, args, **kwargs): kwargs['cwd'] = os.path.dirname(setup_py) cmd = [ 'python', '-c', 'import sys, setuptools; sys.argv[0]=__file__=%r; execfile(__file__)' % os.path.basename(setup_py) ] cmd.extend(args) return call(cmd, **kwargs)
def __call__(self, cmd, *args, **kwargs): brew_head = os.environ.get('VEE_HOMEBREW_COMMIT') core_head = os.environ.get('VEE_HOMEBREW_CORE_COMMIT') bin_ = os.path.join(self.repo.work_tree, 'bin', 'brew') if self.repo.clone_if_not_exists(shallow=not brew_head): # Homebrew should be updated the first time, since it has gotten # a little more complicated. We call it directly so that # it can update itself here, and then our pin will drag it back. call((bin_, 'update')) if brew_head: if self.repo.is_shallow: self.repo.fetch(shallow=False) self.repo.checkout(brew_head) if core_head: repo = GitRepo( os.path.join(self.repo.work_tree, 'Library', 'Taps', 'homebrew', 'homebrew-core'), 'https://github.com/homebrew/homebrew-core.git', ) repo.clone_if_not_exists(shallow=False) if repo.is_shallow: repo.fetch(shallow=False) repo.checkout(core_head) # We need to own the homebrew cache so that we can control permissions. kwargs['env'] = env = kwargs.get('env', os.environ).copy() env.setdefault('HOMEBREW_CACHE', os.path.join(self.repo.work_tree, 'Cache')) env.setdefault('HOMEBREW_LOGS', os.path.join(self.repo.work_tree, 'Logs')) # Keep our pin. if brew_head or core_head: env['HOMEBREW_NO_AUTO_UPDATE'] = '1' res = call((bin_, cmd) + args, _frame=1, **kwargs) if cmd in ('install', 'uninstall'): self._info.pop(args[0], None) return res
def install(self): pkg = self.package pkg._assert_paths(install=True) log.info(style_note('make install')) if call( ['make', 'install', '-j4'], cwd=os.path.dirname(self.makefile_path), env=pkg.fresh_environ(), ): raise RuntimeError('Could not `make install` package')
def git(self, *cmd, **kw): stderr = [] kw['stderr'] = [kw.get('stderr'), stderr.append] kw['check'] = False # We will do it ourselves. kw.setdefault('decode', True) e = None try: # If we can, run as if we are within the work tree. if self.work_tree and self.git_dir == os.path.join( self.work_tree, '.git'): kw['cwd'] = self.work_tree res = call(('git', ) + cmd, **kw) else: res = call(('git', '--git-dir', self.git_dir, '--work-tree', self.work_tree) + cmd, **kw) except CalledProcessError as e: res = e.returncode stderr = b''.join(stderr).rstrip().decode() fatal = [] nonfatal = False for line in stderr.splitlines(): if line.startswith('fatal:'): fatal.append(line[6:].strip()) elif line.strip(): nonfatal = True if fatal: raise GitError(*fatal, errno=res, detail=stderr if nonfatal else None) if isinstance(res, int) and res: raise GitError('%s returned %d; %s' % (' '.join(cmd), res, stderr.strip()), errno=res, detail=stderr if nonfatal else None) if e: # This should never happen. raise return res
def extract(self): """Extract the package into the (cleaned) build directory.""" pkg = self.package pkg._assert_paths(build=True) if pkg.checksum: log.info(style_note('Verifying checksum', 'of ' + pkg.package_path), verbosity=1) assert_file_checksum(pkg.package_path, pkg.checksum) log.info( style_note('Expanding %s to' % self.archive_type, pkg.build_path)) pkg._clean_build_path() # gzip-ed Tarballs. if self.archive_type == 'tar+gzip': call(['tar', 'xzf', pkg.package_path], cwd=pkg.build_path) # bzip-ed Tarballs. elif self.archive_type == 'tar+bzip': call(['tar', 'xjf', pkg.package_path], cwd=pkg.build_path) # Zip files (and Python wheels). elif self.archive_type == 'zip': call(['unzip', pkg.package_path], cwd=pkg.build_path)
def clone_if_not_exists(self, remote_url=None, shallow=True): """Assert that the repo has been cloned. Return True if it did not exist.""" self.remote_url = remote_url or self.remote_url if self.exists: return False if not self.remote_url: raise ValueError('git repo %r does not exist; need remote url' % self.git_dir) if os.path.exists(self.work_tree): call(['git', 'init', '--bare', self.git_dir]) self.git('remote', 'add', 'origin', self.remote_url) self.git('config', '--unset', 'core.bare') if shallow: # TODO: non-master self.git('pull', '--ff-only', '--depth=1', 'origin', get_default_branch()) else: # TODO: non-master self.git('pull', '--ff-only', 'origin', get_default_branch()) elif shallow: log.info(style_note('Cloning shallow', self.remote_url)) call( ['git', 'clone', '--depth=1', self.remote_url, self.work_tree]) else: log.info(style_note('Cloning', self.remote_url)) call(['git', 'clone', self.remote_url, self.work_tree]) return True
def __call__(self, cmd, *args, **kwargs): self.repo.clone_if_not_exists() bin = os.path.join(self.repo.work_tree, 'bin', 'brew') # We need to own the homebrew cache so that we can control permissions. kwargs['env'] = env = kwargs.get('env', os.environ).copy() env.setdefault('HOMEBREW_CACHE', os.path.join(self.repo.work_tree, 'Cache')) res = call((bin, cmd) + args, _frame=1, **kwargs) if cmd in ('install', 'uninstall'): self._info.pop(args[0], None) return res
def install(self): log.info(style_note('source %s' % os.path.basename(self.install_sh))) pkg = self.package pkg._assert_paths(build=True, install=True) env = pkg.fresh_environ() env.update( VEE=pkg.home.root, VEE_BUILD_PATH=pkg.build_path, VEE_INSTALL_NAME=pkg.install_name, VEE_INSTALL_PATH=pkg.install_path, ) cwd = os.path.dirname(self.install_sh) with log.indent(): call([ 'bash', '-c', 'source "%s" "%s"' % (self.install_sh, pkg.install_path) ], env=env, cwd=cwd)
def call_setup_py(setup_py, args, **kwargs): kwargs['cwd'] = os.path.dirname(setup_py) kwargs.setdefault('vee_in_env', True) executable = get_default_python().executable cmd = [ executable, '-sc', '''import sys sys.path.append({!r}) import setuptools sys.argv[0] = __file__ = {!r} exec(compile(open(__file__).read(), __file__, 'exec')) '''.format(get_base_site_packages(), os.path.basename(setup_py)) ] cmd.extend(('--command-packages', 'vee.distutils')) cmd.extend(args) return call(cmd, **kwargs)
def installed_packages(self): if _installed_packages: return _installed_packages packages = _installed_packages out = call(['rpm', '-qa'], stdout=True) for line in out.splitlines(): line = line.strip().lower() if not line: continue packages.add(line) chunks = line.split('-') for i in range(1, len(chunks)): packages.add('-'.join(chunks[:i])) chunks = line.split('.') for i in range(1, len(chunks)): packages.add('.'.join(chunks[:i])) return packages
def build(self): pkg = self.package env = None if self.configure_ac_path and not self.configure_path: bootstrap = os.path.join(os.path.dirname(self.configure_ac_path), 'bootstrap') if os.path.exists(bootstrap): log.info(style_note('./bootstrap', '(autoreconf)')) cmd = ['./bootstrap'] else: log.info(style_note('autoreconf')) cmd = ['autoreconf', '--install', '--force'] env = env or pkg.fresh_environ() call(cmd, cwd=os.path.dirname(self.configure_ac_path), env=env) pkg.build_subdir = os.path.dirname(self.configure_ac_path) # Need to look for it again. self.configure_path = self.configure_path or find_in_tree(pkg.build_path, 'configure') if self.configure_path: log.info(style_note('./configure')) pkg._assert_paths(install=True) cmd = ['./configure', '--prefix', pkg.install_path] cmd.extend(pkg.config) env = env or pkg.fresh_environ() call(cmd, cwd=os.path.dirname(self.configure_path), env=env) pkg.build_subdir = os.path.dirname(self.configure_path) # Need to look for it again. self.makefile_path = self.makefile_path or find_in_tree(pkg.build_path, 'Makefile') if self.makefile_path: log.info(style_note('make')) env = env or pkg.fresh_environ() call(['make', '-j4'], cwd=os.path.dirname(self.makefile_path), env=env) pkg.build_subdir = os.path.dirname(self.makefile_path)
def _relocate_darwin_library(lib_path, con, flags, include, exclude, dry_run, target_cache): auto = 'auto' in flags lib_id, lib_deps = get_dependencies(lib_path) id_versions = set( name_variants(os.path.basename(lib_id), version_only=True)) if lib_id else set() lib_versions = set( name_variants(os.path.basename(lib_path), version_only=True)) cmd = ['install_name_tool'] if lib_id != lib_path: log.info('id %s' % (lib_path), verbosity=1) cmd.extend(('-id', lib_path)) lib_def, lib_undef = get_symbols(lib_path) for dep_i, dep_path in enumerate(lib_deps): if dep_path == lib_id: log.warning('The ID is included?! %s' % lib_path) cmd.extend(('-change', dep_path, lib_path)) continue # If the dependency is similarly named to the library itself, then we # assume it is its own dependency. Which I don't understand... dep_versions = set( name_variants(os.path.basename(dep_path), version_only=True)) if dep_versions.intersection(id_versions) or dep_versions.intersection( lib_versions): log.warning('Library depends on itself?! %s' % dep_path) cmd.extend(('-change', dep_path, lib_path)) continue do_exclude = any(dep_path.startswith(x) for x in exclude) if not do_exclude and os.path.exists(dep_path): log.debug('skipping %s' % dep_path) continue dep_name = os.path.basename(dep_path) targets = [] for variant in name_variants(dep_name): if variant in target_cache: targets.extend(target_cache[variant]) if auto: cur = con.execute( 'SELECT path FROM shared_libraries WHERE name = ? ORDER BY created_at DESC', [variant]) new_targets = target_cache.setdefault(variant, []) new_targets.extend([row[0] for row in cur]) targets.extend(new_targets) # Go searching for the "best" relocation target. # The one with the most defined symbols missing from the lib wins # (essentially; it is more complex then that below). We also, dubiously, # accept libraries which provide no matching symbols as long as they # don't introduct any conflicts. There are a TON of these in FFmpeg. best_score = -1 best_target = None seen_targets = set() for target in targets: if target in seen_targets: continue seen_targets.add(target) if not os.path.exists(target): continue tar_def, tar_undef = get_symbols(target) pros = len(tar_def.intersection(lib_undef)) shared = len(tar_def.intersection(lib_def)) cons = len(lib_undef.intersection(lib_def)) log.debug('+%d ~%d -%d %s' % (pros, shared, cons, target), verbosity=2) if pros - shared - cons > best_score: best_score = pros - shared - cons best_target = (pros, shared, cons, target) if best_target is None: log.warning('No relocation targets for %s' % dep_path) continue if best_score < 0: log.warning('No positive relocation targets for %s' % dep_path) continue if best_target[1] or best_target[2]: log.warning('Best target has %s collisions for %s' % (best_target[1] + best_target[2], dep_path)) target = best_target[3] log.info('change %s -> %s' % (dep_name, target), verbosity=1) cmd.extend(('-change', dep_path, target)) if len(cmd) > 1 and not dry_run: cmd.append(lib_path) s = os.stat(lib_path) if not s.st_mode & stat.S_IWUSR: os.chmod(lib_path, s.st_mode | stat.S_IWUSR) call(cmd) os.chmod(lib_path, s.st_mode) else: call(cmd)