def install(args):
"""Install the given requirements without linking them into an environment.
This is a low-level command, and is generally unused.
Examples:
# Install a single package.
vee install [email protected]:westernx/sgmock
# Install multiple packages.
vee install [email protected]:westernx/sgmock [email protected]:westernx/sgsession \\
http:/example.org/path/to/tarball.tgz --make-install
# Install from a requirement set.
vee install path/to/requirements.txt
"""
home = args.assert_home()
if not args.requirements:
raise ValueError('please provide requirements to install')
reqs = Requirements(args.requirements, home=home)
pkgs = PackageSet(home=home)
# TODO: Resolve everything at once like upgrade does.
for req in reqs.iter_packages():
pkg = pkgs.resolve(req, check_existing=not args.force)
try:
pkgs.install(pkg.name, reinstall=args.force)
except AlreadyInstalled:
print style('Already installed', 'blue', bold=True), style(str(pkg.freeze()), bold=True)
def test_local_envvars(self):
reqs = Requirements()
reqs.parse_file('''
url -e KEY=VALUE
'''.strip().splitlines())
flat = ''.join(reqs.iter_dump()).strip()
self.assertEqual(flat, 'file:url --environ=KEY=VALUE')
def test_global_envvars(self):
req_set = Requirements()
req_set.parse_file('''
first
KEY=VALUE1
second
KEY=VALUE2
third
'''.strip().splitlines())
reqs = list(req_set.iter_packages())
self.assertEqual(reqs[0].base_environ, {})
self.assertEqual(reqs[1].base_environ, {'KEY': 'VALUE1'})
self.assertEqual(reqs[2].base_environ, {'KEY': 'VALUE2'})
def test_else(self):
reqs = Requirements()
reqs.parse_file('''
% if 0:
zero
% elif 0:
one
% else:
two
% endif
'''.strip().splitlines())
pkgs = list(reqs.iter_packages())
self.assertEqual(len(pkgs), 1)
self.assertEqual(pkgs[0].name, 'two')
def test_prefix_function(self):
home = self.home()
reqs = Requirements()
reqs.parse_file('''
first --revision 1.0
second -e "FIRST=$(prefix first)"
'''.strip().splitlines())
pkgs = PackageSet(home=home)
pkgs.resolve_set(reqs)
first = pkgs['first']
first.install_path = '/path/to/first'
second = pkgs['second']
env = second.fresh_environ()
self.assertEqual(env['FIRST'], '/path/to/first')
def test_platforms(self):
req_set = Requirements()
req_set.parse_file('''
before
% if MACOS:
macos
% elif LINUX:
linux
% endif
after
'''.strip().splitlines())
print req_set
reqs = list(req_set.iter_packages())
self.assertEqual(reqs[0].name, 'before')
if sys.platform == 'darwin':
self.assertEqual(reqs[1].name, 'macos')
elif sys.platform == 'linux2':
self.assertEqual(reqs[1].name, 'linux')
self.assertEqual(reqs[2].name, 'after')
def load_requirements(self, revision=None):
reqs = Requirements(env_repo=self, home=self.home)
if revision is not None:
contents = self.show(revision, 'requirements.txt')
if contents:
reqs.parse_file(contents.splitlines())
else:
if os.path.exists(self._req_path):
reqs.parse_file(self._req_path)
return reqs
def exec_(args):
"""Construct an environment, and either export it or run a command in it.
e.g.::
# Run in the default repository.
$ vee exec $command
# Run within a given repository.
$ vee exec --repo named_repo $command
# Run within a named environment.
$ vee exec -e named_environ $command
# Run within a constructed runtime for a set of requirements.
$ vee exec -r requirements.txt $command
# Export the default environment.
$ vee exec --export
export LD_LIBRARY_PATH="/usr/local/vee/lib:$LD_LIBRARY_PATH"
export PATH="/usr/local/vee/bin:$PATH"
export PYTHONPATH="/usr/local/vee/lib/python2.7/site-packages"
"""
home = args.assert_home()
# TODO: seed these with the current values.
repo_names = []
env_names = []
environ_diff = {}
if args.bootstrap:
bootstrap = os.environ['VEE_EXEC_ARGS'].split() if os.environ.get(
'VEE_EXEC_ARGS') else []
# This is gross, but easier than building another parser. It does mean
# that we expect this variable must be set by ourselves.
while bootstrap:
arg = bootstrap.pop(0)
if arg in ('--dev', ):
setattr(args, arg[2:], True)
elif arg in ('--requirements', '--repo', '--environment'):
v = getattr(args, arg[2:], None) or []
v.append(bootstrap.pop(0))
setattr(args, arg[2:], v)
else:
print >> sys.stderr, 'cannot bootstrap', arg
if args.dev:
# Store the original flags as provided so that --bootstrap can pick it back up.
bootstrap = os.environ['VEE_EXEC_ARGS'].split() if os.environ.get(
'VEE_EXEC_ARGS') else []
if '--dev' not in bootstrap:
bootstrap.append('--dev')
for attr in 'requirements', 'repo', 'environment':
for value in getattr(args, attr, None) or ():
bootstrap.append('--' + attr)
bootstrap.append(value)
environ_diff['VEE_EXEC_ARGS'] = ' '.join(bootstrap)
if not (args.export or args.command or args.prefix):
raise ValueError(
'Must either --prefix, --export, or provide a command')
# Default to the default repo.
if not (args.requirements or args.environment or args.repos):
args.repos = [None]
paths = []
# Named (or default) repos.
for name in args.repos or ():
repo = home.get_env_repo(name or None) # Allow '' to be the default.
args.environment = args.environment or []
args.environment.append('%s/%s' % (repo.name, repo.branch_name))
repo_names.append(repo.name)
# Requirements and requirement sets.
req_args = []
for arg in args.requirements or ():
req_args.extend(arg.split(','))
req_set = Requirements(home=home)
req_set.parse_args(req_args)
pkg_set = PackageSet(home=home)
for req in req_set.iter_packages():
pkg = pkg_set.resolve(req)
pkg._assert_paths(install=True)
if not pkg.installed:
raise NotInstalled(pkg.install_path)
paths.append(pkg.install_path)
# Named environments.
for name in args.environment or ():
env = Environment(name, home=home)
paths.append(env.path)
env_names.append(name)
if args.prefix:
for path in paths:
print path
return
environ_diff.update(guess_envvars(paths))
if args.dev:
for pkg in home.iter_development_packages(exists=True, search=True):
if pkg.environ:
environ_diff.update(
render_envvars(pkg.environ, pkg.work_tree, environ_diff))
# Add the current virtualenv.
venv = os.environ.get('VIRTUAL_ENV')
if venv:
environ_diff['PYTHONPATH'] = '%s:%s' % (
os.path.join(venv, 'lib', 'python%d.%d' % sys.version_info[:2],
'site-packages'),
environ_diff.get('PYTHONPATH', ''), # This is sloppy.
)
# More environment variables.
command = args.command or []
while command and re.match(r'^\w+=', command[0]):
k, v = command.pop(0).split('=', 1)
environ_diff[k] = v
# Make sure setuptools is bootstrapped.
bootstrap_environ(environ_diff)
environ_diff['VEE_EXEC_PATH'] = ':'.join(paths)
environ_diff['VEE_EXEC_REPO'] = ','.join(repo_names)
environ_diff['VEE_EXEC_ENV'] = ','.join(env_names)
environ_diff['VEE_EXEC_PREFIX'] = paths[0]
# Print it out instead of running it.
if args.export:
for k, v in sorted(environ_diff.iteritems()):
existing = os.environ.get(k)
# Since we modify os.environ in __init__ to bootstrap the vendored
# packages, swaping out the original values will not include the
# bootstrap. So we are tricking the code so that it still includes it.
if k == 'PYTHONPATH' and existing.endswith(vendor_path):
existing += (':' if existing else '') + vendor_path
if existing is not None and not k.startswith('VEE_EXEC'):
v = v.replace(existing, '$' + k)
print 'export %s="%s"' % (k, v)
return
environ = os.environ.copy()
environ.update(environ_diff)
os.execvpe(args.command[0], args.command, environ)
def add_requirements(self, raw, insert=False, commit=True):
old = Requirements(home=self.home)
path = os.path.join(self.path, 'requirements.txt')
if os.path.exists(path):
old.parse_file(path)
new = Requirements(home=self.home, file=StringIO(raw))
new_urls = set()
new_names = set()
for req in new.iter_packages():
new_names.add(req.name or guess_name(req.url))
new_urls.add(req.url)
for prefix, element, postfix in old:
if (not isinstance(element, Package) or
(element.name or guess_name(element.url)) not in new_names
or element.url not in new_urls):
if insert:
new.append((prefix, element, postfix))
else:
new.insert(0, (prefix, element, postfix))
with open(path, 'wb') as fh:
for line in new.iter_dump():
fh.write(line)
if commit:
self.commit('add requirements')
def init(args, do_clone=False, do_install=False, do_add=False, is_find=False):
do_init = not (do_clone or do_install or do_add)
name = args.name
home = args.assert_home()
con = home.db.connect()
# Make sure there are no other packages already, and clear out old ones
# which no longer exist.
for row in con.execute('SELECT * FROM development_packages WHERE name = ?',
[name]):
if not args.force and os.path.exists(os.path.join(row['path'],
'.git')):
if is_find:
print style_note('"%s" already exists:' % name, row['path'])
return
else:
print style_error('"%s" already exists:' % name, row['path'])
return 1
else:
con.execute('DELETE FROM development_packages WHERE id = ?',
[row['id']])
path = os.path.abspath(args.path or os.path.join(home.dev_root, name))
dev_repo = GitRepo(path)
if do_init:
print style_note('Initing %s' % dev_repo.work_tree)
makedirs(dev_repo.work_tree)
dev_repo.git('init')
elif do_clone:
print style_note('Cloning %s' % args.url)
makedirs(dev_repo.work_tree)
dev_repo.clone_if_not_exists(args.url)
elif do_install:
# Find an existing tool.
# TODO: put more of this into EnvironmentRepo or Requirements
env_repo = home.get_env_repo(args.repo)
req_path = os.path.join(env_repo.work_tree, 'requirements.txt')
reqs = Requirements(req_path, home=home)
for req in reqs.iter_packages():
if req.name.lower() == name.lower():
# Make sure it is a Git package.
url = normalize_git_url(req.url, prefix=False)
if url:
break
else:
print style_error('Could not find git-based "%s" in "%s" repo.' %
(name, env_repo.name))
return 2
print style_note('Found %s in %s' % (name, env_repo.name), str(req))
makedirs(dev_repo.work_tree)
dev_repo.clone_if_not_exists(url, shallow=False)
elif do_add:
print style_note('Adding %s from %s' % (name, path))
if not os.path.exists(path):
log.error('%s does not exist' % path)
return 1
package = Package([path], home=home, dev=True)
try:
package.pipeline.run_to('develop')
except Exception as e:
print_cli_exc(e)
return 1
print style_note('Linking dev package', name, path)
con.execute(
'INSERT INTO development_packages (name, path, environ) VALUES (?, ?, ?)',
[name, path, json.dumps(package.environ)])
dev_pkg = DevPackage(
{
'name': name,
'path': path,
'environ': package.environ
}, home=home)
dev_pkg.save_tag()
def link(args):
"""Link the given requirement or requirements into the given environment,
e.g.::
# Install and link a single package.
$ vee link [email protected]:westernx/sgmock
# Install and link multiple packages.
$ vee link [email protected]:westernx/sgmock [email protected]:westernx/sgsession \\
http:/example.org/path/to/tarball.tgz --make-install
# Install and link from a requirement set.
$ vee link path/to/requirements.txt
"""
if args.no_install and args.re_install:
raise ValueError(
'please use only one of --no-install and --re-install')
home = args.assert_home()
if sum(
int(bool(x))
for x in (args.repo, args.environment, args.directory)) > 1:
raise ValueError(
'use only one of --repo, --environment, or --directory')
if args.environment:
env = Environment(args.environment, home=home)
elif args.directory:
env = Environment(os.path.abspath(args.directory), home=home)
else:
env_repo = home.get_env_repo(args.repo)
env = Environment('%s/%s' % (env_repo.name, env_repo.branch_name),
home=home)
if args.raw:
for dir_ in args.requirements:
print style('Linking', 'blue', bold=True), style(dir_, bold=True)
env.link_directory(dir_)
return
req_set = Requirements(args.requirements, home=home)
pkg_set = PackageSet(env=env, home=home)
# Register the whole set, so that dependencies are pulled from here instead
# of weakly resolved from installed packages.
pkg_set.resolve_set(req_set)
for req in req_set.iter_packages():
# Skip if it wasn't requested.
if args.subset and req.name not in args.subset:
continue
log.info(style('==> %s' % req.name, 'blue'))
pkg = pkg_set.resolve(req, check_existing=not args.reinstall)
if args.no_install and not pkg.installed:
raise CliError('not installed: %s' % req)
try:
with log.indent():
pkg_set.install(pkg.name,
link_env=env,
reinstall=args.reinstall,
relink=args.force)
except AlreadyInstalled:
pass
except AlreadyLinked as e:
log.info(style('Already linked ', 'blue') + str(req), verbosity=1)
except Exception as e:
print_cli_exc(e, verbose=True)
log.exception('Failed to link %s' % req)
continue