def develop(self, pkg):
log.info(style_note('Building scripts'))
cmd = ['vee_develop']
if call_setup_py(self.setup_path, cmd):
raise RuntimeError('Could not build scripts')
egg_info = find_in_tree(os.path.dirname(self.setup_path), '*.egg-info',
'dir')
if not egg_info:
raise RuntimeError('Could not find built egg-info')
dirs_to_link = set()
for line in open(os.path.join(egg_info, 'top_level.txt')):
dirs_to_link.add(os.path.dirname(line.strip()))
for name in sorted(dirs_to_link):
log.info(style_note("Adding ./%s to $PYTHONPATH" % name))
pkg.environ['PYTHONPATH'] = join_env_path(
'./' + name, pkg.environ.get('PYTHONPATH', '@'))
scripts = os.path.join(os.path.dirname(self.setup_path), 'build',
'scripts')
if os.path.exists(scripts):
log.info(style_note("Adding ./build/scripts to $PATH"))
pkg.environ['PATH'] = join_env_path('./build/scripts',
pkg.environ.get('PATH', '@'))
def install(self):
pkg = self.package
if pkg.pseudo_homebrew:
homebrew = Homebrew(home=pkg.home)
version = pkg.revision.split('+')[0]
pkg.install_path = os.path.join(homebrew.cellar, pkg.name, version)
log.info(style_note('Re-installing into Homebrew', 'as %s/%s' % (pkg.name, version)))
pkg._assert_paths(install=True)
if pkg.make_install:
log.warning('--make-install specified, but no Makefile found.')
if os.path.exists(pkg.install_path):
log.warning('Removing existing install', pkg.install_path)
shutil.rmtree(pkg.install_path)
if pkg.hard_link:
log.info(style_note('Installing via hard-link', 'to ' + pkg.install_path))
linktree(pkg.build_path_to_install, pkg.install_path_from_build, symlinks=True)
else:
log.info(style_note('Installing via copy', 'to ' + pkg.install_path))
shutil.copytree(pkg.build_path_to_install, pkg.install_path_from_build, symlinks=True)
def develop(self):
pkg = self.package
log.info(style_note('Building scripts'))
cmd = [
'build_scripts',
'-e',
'/usr/bin/env VEE=%s VEE_PYTHON=%s dev python' %
(os.environ.get("VEE", ''), os.environ.get('VEE_PYTHON', '')),
'install_scripts',
'-d',
'build/scripts',
]
if call_setup_py(self.setup_path, cmd):
raise RuntimeError('Could not build scripts')
egg_info = find_in_tree(os.path.dirname(self.setup_path), '*.egg-info',
'dir')
if not egg_info:
raise RuntimeError('Could not find built egg-info')
dirs_to_link = set()
for line in open(os.path.join(egg_info, 'top_level.txt')):
dirs_to_link.add(os.path.dirname(line.strip()))
for name in sorted(dirs_to_link):
log.info(style_note("Adding ./%s to $PYTHONPATH" % name))
pkg.environ['PYTHONPATH'] = join_env_path(
'./' + name, pkg.environ.get('PYTHONPATH', '@'))
scripts = os.path.join(os.path.dirname(self.setup_path), 'build',
'scripts')
if os.path.exists(scripts):
log.info(style_note("Adding ./build/scripts to $PATH"))
pkg.environ['PATH'] = join_env_path('./build/scripts',
pkg.environ.get('PATH', '@'))
def extract(self):
"""Extract the package into the (cleaned) build directory."""
pkg = self.package
pkg._assert_paths(build=True)
if pkg.checksum:
log.info(style_note('Verifying checksum',
'of ' + pkg.package_path),
verbosity=1)
assert_file_checksum(pkg.package_path, pkg.checksum)
log.info(
style_note('Expanding %s to' % self.archive_type, pkg.build_path))
pkg._clean_build_path()
# gzip-ed Tarballs.
if self.archive_type == 'tar+gzip':
call(['tar', 'xzf', pkg.package_path], cwd=pkg.build_path)
# bzip-ed Tarballs.
elif self.archive_type == 'tar+bzip':
call(['tar', 'xjf', pkg.package_path], cwd=pkg.build_path)
# Zip files (and Python wheels).
elif self.archive_type == 'zip':
call(['unzip', pkg.package_path], cwd=pkg.build_path)
def list_(args):
home = args.assert_home()
if args.availible:
for req, url in iter_availible_requirements(home):
log.info(style_note(req.name, str(req)))
return
for dev_pkg in sorted(home.iter_development_packages(),
key=lambda p: p.name.lower()):
if args.glob and not fnmatch.fnmatch(dev_pkg.name, args.glob):
continue
path = dev_pkg.work_tree.replace(home.dev_root, '$VEE_DEV').replace(
home.root, '$VEE')
log.info(style_note(dev_pkg.name, path))
if args.show_environ:
for k, v in sorted(
render_envvars(dev_pkg.environ,
dev_pkg.work_tree).items()):
if os.environ.get(k):
v = v.replace(os.environ[k], '$' + k)
v = v.replace(home.dev_root, '$VEE_DEV')
v = v.replace(home.root, '$VEE')
log.info(style(' %s=' % k) + v)
def list_(args):
home = args.assert_home()
if args.availible:
for req, url in iter_availible_requirements(home):
print style_note(req.name, str(req))
return
if args.glob:
cur = home.db.execute(
'SELECT * FROM development_packages WHERE name GLOB ? ORDER BY lower(name)',
[args.glob])
else:
cur = home.db.execute(
'SELECT * FROM development_packages ORDER BY lower(name)')
for dev_pkg in sorted(home.iter_development_packages(),
key=lambda p: p.name.lower()):
if args.glob and not fnmatch.fnmatch(dev_pkg.name, args.glob):
continue
path = dev_pkg.work_tree.replace(home.dev_root, '$VEE_DEV').replace(
home.root, '$VEE')
print style_note(dev_pkg.name, path)
if args.show_environ:
for k, v in sorted(
render_envvars(dev_pkg.environ,
dev_pkg.work_tree).iteritems()):
if os.environ.get(k):
v = v.replace(os.environ[k], '$' + k)
v = v.replace(home.dev_root, '$VEE_DEV')
v = v.replace(home.root, '$VEE')
print style(' %s=' % k) + v
def clone_if_not_exists(self, remote_url=None, shallow=True):
"""Assert that the repo has been cloned. Return True if it did not exist."""
self.remote_url = remote_url or self.remote_url
if self.exists:
return False
if not self.remote_url:
raise ValueError('git repo %r does not exist; need remote url' %
self.git_dir)
if os.path.exists(self.work_tree):
call(['git', 'init', '--bare', self.git_dir])
self.git('remote', 'add', 'origin', self.remote_url)
self.git('config', '--unset', 'core.bare')
if shallow:
# TODO: non-master
self.git('pull', '--ff-only', '--depth=1', 'origin',
get_default_branch())
else:
# TODO: non-master
self.git('pull', '--ff-only', 'origin', get_default_branch())
elif shallow:
log.info(style_note('Cloning shallow', self.remote_url))
call(
['git', 'clone', '--depth=1', self.remote_url, self.work_tree])
else:
log.info(style_note('Cloning', self.remote_url))
call(['git', 'clone', self.remote_url, self.work_tree])
return True
def fetch(self, pkg):
pkg._assert_paths(package=True)
if os.path.exists(pkg.package_path):
log.info(style_note('Already downloaded', pkg.url))
return
log.info(style_note('Downloading', pkg.url))
download(pkg.url, pkg.package_path)
def find_command(name, warn=False):
path = which(name)
if path:
print style_note('%s:' % name, path)
else:
if warn:
print style_warning('cannot find %s' % name)
else:
print style_error('cannot find %s' % name)
return 1
def develop(self):
pkg = self.package
for name in ('bin', 'scripts'):
path = os.path.join(pkg.build_path, name)
if os.path.exists(path):
log.info(style_note("Adding ./%s to $PATH" % name))
pkg.environ['PATH'] = join_env_path('./' + name, pkg.environ.get('PATH', '@'))
def inspect(self):
pkg = self.package
if self.setup_path and not self.egg_path:
log.info(style_note('Building Python egg-info'))
res = call_setup_py(self.setup_path, ['egg_info'],
env=pkg.fresh_environ(),
indent=True,
verbosity=1)
if res:
raise RuntimeError('Could not build Python package')
self.egg_path = find_in_tree(pkg.build_path, '*.egg-info', 'dir')
if not self.egg_path:
log.warning('Could not find newly created *.egg-info')
if self.egg_path:
requires_path = os.path.join(self.egg_path, 'requires.txt')
if os.path.exists(requires_path):
for line in open(requires_path, 'rb'):
line = line.strip()
if not line:
continue
if line.startswith('['):
break
name = re.split('\W', line)[0].lower()
log.debug('%s depends on %s' % (pkg.name, name))
pkg.dependencies.append(
Package(name=name, url='pypi:%s' % name))
def build(self):
log.info(style_note('source %s' % os.path.basename(self.build_sh)))
pkg = self.package
pkg._assert_paths(build=True, install=True)
env = pkg.fresh_environ()
env.update(
VEE=pkg.home.root,
VEE_BUILD_PATH=pkg.build_path,
VEE_INSTALL_NAME=pkg.install_name,
VEE_INSTALL_PATH=pkg.install_path,
)
# TODO: somehow derive this from --build-sh provided script.
cwd = os.path.dirname(self.build_sh)
envfile = os.path.join(cwd, 'vee-env-' + os.urandom(8).encode('hex'))
call([
'bash', '-c',
'. %s; env | grep VEE > %s' %
(os.path.basename(self.build_sh), envfile)
],
env=env,
cwd=cwd)
env = list(open(envfile))
env = dict(line.strip().split('=', 1) for line in env)
os.unlink(envfile)
pkg.build_subdir = env.get('VEE_BUILD_SUBDIR') or ''
pkg.install_prefix = env.get('VEE_INSTALL_PREFIX') or ''
def link(self, env, force=False):
self._assert_paths(install=True)
if not force:
self._assert_unlinked(env)
log.info(style_note('Linking into %s' % env.name))
env.link_directory(self.install_path)
self._record_link(env)
def checkout(self,
revision,
branch=None,
force=False,
fetch=False,
ignore_permissions=True):
commit = self.rev_parse(revision, fetch=fetch)
if self.head == commit:
return
log.info(style_note('Checking out', '%s [%s]' % (revision, commit)))
cmd = []
if ignore_permissions:
cmd.extend(('-c', 'core.fileMode=false'))
cmd.append('checkout')
if force:
cmd.append('--force')
if branch: # Make this branch if it doesn't exist.
cmd.extend(('-B', branch))
cmd.append(revision)
self.git(*cmd)
self.git('submodule',
'update',
'--init',
'--checkout',
'--recursive',
silent=False)
self._head = commit
def git(args, *command):
if not (args.all or args.name):
log.error("Please provide -n NAME or --all.")
return 1
if not command:
log.error('Please provide a git command.')
return 1
home = args.assert_home()
retcode = 0
if args.all:
dev_pkgs = home.iter_development_packages()
else:
dev_pkgs = []
for name in args.names:
dev_pkg = home.find_development_package(name)
if not dev_pkg:
log.error("Could not find dev package: {!r}.".format(name))
return 2
dev_pkgs.append(dev_pkg)
for dev_pkg in dev_pkgs:
log.info(style_note(dev_pkg.name, ' '.join(command)))
try:
dev_pkg.git(*command, verbosity=0, indent=False)
except Exception as e:
print_cli_exc(e)
retcode = 1
return retcode
def create_if_not_exists(self):
python = os.path.join(self.path, 'bin', 'python')
if not os.path.exists(python):
makedirs(self.path)
log.info(style_note('Creating Python virtualenv', self.path))
if hasattr(virtualenv, 'cli_run'):
# New API (in which there isn't really any API)
virtualenv.cli_run(
['--no-pip', '--no-wheel', '--no-setuptools', self.path])
else:
# Old API
virtualenv.create_environment(self.path,
no_setuptools=True,
no_pip=True)
if not os.path.exists(python + '-config'):
version = get_default_python().version
names = (
'python{}.{}-config'.format(*version),
'python{}-config'.format(*version),
'python-config',
)
prefix = getattr(sys, 'real_prefix', sys.prefix)
for name in names:
old_path = os.path.join(prefix, 'bin', name)
if os.path.exists(old_path):
for name in names:
new_path = os.path.join(self.path, 'bin', name)
self.rewrite_shebang_or_link(old_path, new_path)
break
else:
log.warning('Could not find python-config')
def list_(args):
home = args.assert_home()
rows = list(home.db.execute('SELECT * FROM repositories'))
if not rows:
print style_warning('No repositories.')
return
max_len = max(len(row['name']) for row in rows)
for row in rows:
env_repo = EnvironmentRepo(row, home=home)
if env_repo.exists:
print style_note(
env_repo.name,
'%s/%s' % (env_repo.remote_name, env_repo.branch_name),
env_repo.remotes().get(env_repo.remote_name, '') +
' --default' if row['is_default'] else '',
)
def _install_wheel(self, pkg):
pkg._assert_paths(install=True)
if pkg.package_path.endswith('.whl'):
log.info(
style_note("Found Python Wheel",
os.path.basename(self.dist_info_dir)))
else:
log.info(
style_note("Found dist-info",
os.path.basename(self.dist_info_dir)))
log.warning("Bare dist-info does not appear to be a wheel.")
wheel_dir, dist_info_name = os.path.split(self.dist_info_dir)
wheel_name = os.path.splitext(dist_info_name)[0]
# Lets just take advantage of pip!
# The only reason we're reading into pip like this is because we
# would rather just do this part, rather than have it go through
# the full process with the *.whl file. If this breaks, feel
# free to do something like:
# pip install --force-reinstall --prefix {pkg.install_path} --no-deps {pkg.package_path}
# along with:
# --no-warn-script-location
# --disable-pip-version-check
# We delay the import just in case the bootstrap is borked.
from pip._internal.operations.install.wheel import install_wheel
from pip._internal.locations import get_scheme
# We may to trick pip into installing into another version's directories.
scheme = get_scheme(self.name, prefix=pkg.install_path)
version = get_default_python().version
src_python = '{}python{}.{}{}'.format(os.path.sep, sys.version_info[0],
sys.version_info[1], os.path.sep)
dst_python = '{}python{}.{}{}'.format(os.path.sep, version[0],
version[1], os.path.sep)
if src_python != dst_python:
for k in 'platlib', 'purelib', 'headers', 'scripts', 'data':
setattr(scheme, k,
getattr(scheme, k).replace(src_python, dst_python))
req = DummyPipRequirement()
req.name = wheel_name
install_wheel(pkg.name, pkg.package_path, scheme,
'<VEE dummy request>')
def find_command(name):
path = which(name)
if path:
print(style_note(repr(path) + ":"))
return 0
else:
print(style(name + ":", 'yellow'), None)
return 1
def optlink(self, pkg):
if pkg.name:
opt_link = pkg.home._abs_path('opt', pkg.name)
log.info(style_note('Linking to opt/%s' % pkg.name))
if os.path.lexists(opt_link):
os.unlink(opt_link)
makedirs(os.path.dirname(opt_link))
os.symlink(pkg.install_path, opt_link)
def build(self):
pkg = self.package
env = None
if self.configure_ac_path and not self.configure_path:
bootstrap = os.path.join(os.path.dirname(self.configure_ac_path), 'bootstrap')
if os.path.exists(bootstrap):
log.info(style_note('./bootstrap', '(autoreconf)'))
cmd = ['./bootstrap']
else:
log.info(style_note('autoreconf'))
cmd = ['autoreconf', '--install', '--force']
env = env or pkg.fresh_environ()
call(cmd, cwd=os.path.dirname(self.configure_ac_path), env=env)
pkg.build_subdir = os.path.dirname(self.configure_ac_path)
# Need to look for it again.
self.configure_path = self.configure_path or find_in_tree(pkg.build_path, 'configure')
if self.configure_path:
log.info(style_note('./configure'))
pkg._assert_paths(install=True)
cmd = ['./configure', '--prefix', pkg.install_path]
cmd.extend(pkg.config)
env = env or pkg.fresh_environ()
call(cmd, cwd=os.path.dirname(self.configure_path), env=env)
pkg.build_subdir = os.path.dirname(self.configure_path)
# Need to look for it again.
self.makefile_path = self.makefile_path or find_in_tree(pkg.build_path, 'Makefile')
if self.makefile_path:
log.info(style_note('make'))
env = env or pkg.fresh_environ()
call(['make', '-j4'], cwd=os.path.dirname(self.makefile_path), env=env)
pkg.build_subdir = os.path.dirname(self.makefile_path)
def relocate_package(pkg):
if pkg.relocate:
log.info(style_note('Relocating'))
with log.indent():
libs.relocate(pkg.install_path,
con=pkg.home.db.connect(),
spec=pkg.render_template(pkg.relocate),
)
if pkg.set_rpath and sys.platform.startswith('linux'):
rpath = pkg.render_template(pkg.set_rpath)
log.info(style_note('Setting RPATH to', rpath))
with log.indent():
libs.relocate(pkg.install_path,
con=pkg.home.db.connect(),
spec=rpath,
)
def install(self):
pkg = self.package
pkg._assert_paths(install=True)
log.info(style_note('make install'))
if call(
['make', 'install', '-j4'],
cwd=os.path.dirname(self.makefile_path),
env=pkg.fresh_environ(),
):
raise RuntimeError('Could not `make install` package')
def fetch(self):
pkg = self.package
meta = self._meta()
all_releases = [(Version(v), rs)
for v, rs in meta['releases'].iteritems()]
all_releases.sort(reverse=True)
if pkg.revision:
expr = VersionExpr(pkg.revision)
matching_releases = [(v, rs) for v, rs in all_releases
if expr.eval(v)]
log.debug(
'%s matched %s' %
(expr, ','.join(str(v)
for v, _ in matching_releases) or 'none'))
else:
matching_releases = all_releases
for version, releases in matching_releases:
release = next(
(r for r in releases if r['packagetype'] == 'sdist'), None)
if release:
break
else:
raise ValueError('no sdist %s %s on the PyPI;' %
(self.name, expr if pkg.revision else '(any)'))
pkg.revision = str(version)
if release.get('md5_digest'):
pkg.checksum = 'md5:%s' % release['md5_digest']
pkg.package_name = os.path.join(self.name,
os.path.basename(release['url']))
pkg._assert_paths(package=True)
if os.path.exists(pkg.package_path):
log.info(style_note('Already downloaded', release['url']))
return
log.info(style_note('Downloading', release['url']))
download(release['url'], pkg.package_path)
def inspect(self):
log.info(
style_note('Inspecting %s' %
os.path.basename(self.requirements_txt)))
pkg = self.package
for line in open(self.requirements_txt):
line = line.strip()
if not line or line[0] == '#':
continue
pkg.dependencies.append(Package(line, home=pkg.home))
def _meta(self):
pkg = self.package
path = pkg.home._abs_path('packages', 'pypi', self.name, 'meta.json')
if not os.path.exists(path):
log.info(style_note('Looking up %s on PyPI' % self.name))
url = PYPI_URL_PATTERN % self.name
res = urllib2.urlopen(url)
makedirs(os.path.dirname(path))
with open(path, 'wb') as fh:
fh.write(res.read())
return json.load(open(path, 'rb'))
def _get_meta(self, pkg):
path = pkg.home._abs_path('packages', 'pypi', self.name, 'meta.json')
log.info(style_note('Looking up %s on PyPI' % self.name))
url = PYPI_URL_PATTERN % self.name
res = http_request('GET', url)
body = res.data
meta = json.loads(body)
return meta
def relocate(self):
pkg = self.package
if not pkg.pseudo_homebrew:
return
# --pseudo-homebrew is first handled by the generic.install, which
# sets the install_path to be in the Homebrew cellar. We finish the
# job by switching to that version.
log.info(style_note('Switching Homebrew to %s %s' % (pkg.name, self.version)))
self.brew('switch', pkg.name, self.version)
def build(self, pkg):
if self.setup_path:
# Some packages need to be built at the same time as installing.
# Anything which uses the distutils install_clib command, for instance...
if pkg.defer_setup_build:
log.info(style_note('Deferring build to install stage'))
return
log.info(style_note('Building Python package'))
cmd = ['build']
cmd.extend(pkg.config)
res = call_setup_py(self.setup_path,
cmd,
env=pkg.fresh_environ(),
indent=True,
verbosity=1)
if res:
raise RuntimeError('Could not build Python package')
def develop(self):
log.info(style_note('source %s' % os.path.basename(self.develop_sh)))
pkg = self.package
def setenv(name, value):
log.info('vee develop setenv %s "%s"' % (name, value))
pkg.environ[name] = value
with log.indent():
bash_source(os.path.basename(self.develop_sh),
callbacks=dict(vee_develop_setenv=setenv),
cwd=os.path.dirname(self.develop_sh))