def install(args):
"""Install the given requirements without linking them into an environment.
This is a low-level command, and is generally unused.
Examples:
# Install a single package.
vee install [email protected]:westernx/sgmock
# Install multiple packages.
vee install [email protected]:westernx/sgmock [email protected]:westernx/sgsession \\
http:/example.org/path/to/tarball.tgz --make-install
# Install from a requirement set.
vee install path/to/requirements.txt
"""
home = args.assert_home()
if not args.requirements:
raise ValueError('please provide requirements to install')
reqs = Requirements(args.requirements, home=home)
pkgs = PackageSet(home=home)
# TODO: Resolve everything at once like upgrade does.
for req in reqs.iter_packages():
pkg = pkgs.resolve(req, check_existing=not args.force)
try:
pkgs.install(pkg.name, reinstall=args.force)
except AlreadyInstalled:
print style('Already installed', 'blue', bold=True), style(str(pkg.freeze()), bold=True)
def iter_availible_requirements(home):
repo = home.get_repo()
req_set = repo.requirement_set()
pkg_set = PackageSet(home=home)
for req in req_set.iter_packages():
pkg = pkg_set.resolve(req, check_existing=False)
if pkg.type != 'git':
return
yield req, normalize_git_url(req.url, prefix=False)
def test_prefix_function(self):
home = self.home()
reqs = Manifest(home=home)
reqs.parse_file('''
first --version 1.0
second -e "FIRST=$(prefix first)"
'''.strip().splitlines())
pkgs = PackageSet(home=home)
pkgs.resolve_set(reqs)
first = pkgs['first']
first.install_path = '/path/to/first'
second = pkgs['second']
env = second.fresh_environ()
self.assertEqual(env['FIRST'], '/path/to/first')
def exec_(args):
"""Construct an environment, and either export it or run a command in it.
e.g.::
# Run in the default repository.
$ vee exec $command
# Run within a given repository.
$ vee exec --repo named_repo $command
# Run within a named environment.
$ vee exec -e named_environ $command
# Run within a constructed runtime for a set of requirements.
$ vee exec -r requirements.txt $command
# Export the default environment.
$ vee exec --export
export LD_LIBRARY_PATH="/usr/local/vee/lib:$LD_LIBRARY_PATH"
export PATH="/usr/local/vee/bin:$PATH"
export PYTHONPATH="/usr/local/vee/lib/python2.7/site-packages"
"""
home = args.assert_home()
# TODO: seed these with the current values.
repo_names = []
env_names = []
environ_diff = {}
if args.bootstrap:
bootstrap = os.environ['VEE_EXEC_ARGS'].split() if os.environ.get(
'VEE_EXEC_ARGS') else []
# This is gross, but easier than building another parser. It does mean
# that we expect this variable must be set by ourselves.
while bootstrap:
arg = bootstrap.pop(0)
if arg in ('--dev', ):
setattr(args, arg[2:], True)
elif arg in ('--requirements', '--repo', '--environment'):
v = getattr(args, arg[2:], None) or []
v.append(bootstrap.pop(0))
setattr(args, arg[2:], v)
else:
print >> sys.stderr, 'cannot bootstrap', arg
if args.dev:
# Store the original flags as provided so that --bootstrap can pick it back up.
bootstrap = os.environ['VEE_EXEC_ARGS'].split() if os.environ.get(
'VEE_EXEC_ARGS') else []
if '--dev' not in bootstrap:
bootstrap.append('--dev')
for attr in 'requirements', 'repo', 'environment':
for value in getattr(args, attr, None) or ():
bootstrap.append('--' + attr)
bootstrap.append(value)
environ_diff['VEE_EXEC_ARGS'] = ' '.join(bootstrap)
if not (args.export or args.command or args.prefix):
raise ValueError(
'Must either --prefix, --export, or provide a command')
# Default to the default repo.
if not (args.requirements or args.environment or args.repos):
args.repos = [None]
paths = []
# Named (or default) repos.
for name in args.repos or ():
repo = home.get_env_repo(name or None) # Allow '' to be the default.
args.environment = args.environment or []
args.environment.append('%s/%s' % (repo.name, repo.branch_name))
repo_names.append(repo.name)
# Requirements and requirement sets.
req_args = []
for arg in args.requirements or ():
req_args.extend(arg.split(','))
req_set = Requirements(home=home)
req_set.parse_args(req_args)
pkg_set = PackageSet(home=home)
for req in req_set.iter_packages():
pkg = pkg_set.resolve(req)
pkg._assert_paths(install=True)
if not pkg.installed:
raise NotInstalled(pkg.install_path)
paths.append(pkg.install_path)
# Named environments.
for name in args.environment or ():
env = Environment(name, home=home)
paths.append(env.path)
env_names.append(name)
if args.prefix:
for path in paths:
print path
return
environ_diff.update(guess_envvars(paths))
if args.dev:
for pkg in home.iter_development_packages(exists=True, search=True):
if pkg.environ:
environ_diff.update(
render_envvars(pkg.environ, pkg.work_tree, environ_diff))
# Add the current virtualenv.
venv = os.environ.get('VIRTUAL_ENV')
if venv:
environ_diff['PYTHONPATH'] = '%s:%s' % (
os.path.join(venv, 'lib', 'python%d.%d' % sys.version_info[:2],
'site-packages'),
environ_diff.get('PYTHONPATH', ''), # This is sloppy.
)
# More environment variables.
command = args.command or []
while command and re.match(r'^\w+=', command[0]):
k, v = command.pop(0).split('=', 1)
environ_diff[k] = v
# Make sure setuptools is bootstrapped.
bootstrap_environ(environ_diff)
environ_diff['VEE_EXEC_PATH'] = ':'.join(paths)
environ_diff['VEE_EXEC_REPO'] = ','.join(repo_names)
environ_diff['VEE_EXEC_ENV'] = ','.join(env_names)
environ_diff['VEE_EXEC_PREFIX'] = paths[0]
# Print it out instead of running it.
if args.export:
for k, v in sorted(environ_diff.iteritems()):
existing = os.environ.get(k)
# Since we modify os.environ in __init__ to bootstrap the vendored
# packages, swaping out the original values will not include the
# bootstrap. So we are tricking the code so that it still includes it.
if k == 'PYTHONPATH' and existing.endswith(vendor_path):
existing += (':' if existing else '') + vendor_path
if existing is not None and not k.startswith('VEE_EXEC'):
v = v.replace(existing, '$' + k)
print 'export %s="%s"' % (k, v)
return
environ = os.environ.copy()
environ.update(environ_diff)
os.execvpe(args.command[0], args.command, environ)
def upgrade(self,
dirty=False,
subset=None,
reinstall=False,
relink=False,
no_deps=False,
force_branch_link=True):
self.clone_if_not_exists()
try:
head = self.head
except CalledProcessError:
log.warning(style_warning('no commits in repository'))
head = None
try:
remote_head = self.rev_parse('%s/%s' %
(self.remote_name, self.branch_name))
except ValueError:
log.warning(
style_warning('tracked %s/%s does not exist in self' %
(self.remote_name, self.branch_name)))
remote_head = None
if remote_head and head != remote_head:
log.warning(
style_warning('%s repo not checked out to %s/%s' %
(self.name, self.remote_name, self.branch_name)))
dirty = bool(list(self.status()))
if not dirty and self.is_dirty():
log.error('%s repo is dirty; force with --dirty' % self.name)
return False
env = self.get_environment()
req_set = self.load_requirements()
pkg_set = PackageSet(env=env, home=self.home)
# Register the whole set, so that dependencies are pulled from here instead
# of weakly resolved from installed packages.
# TODO: This blanket reinstalls things, even if no_deps is set.
pkg_set.resolve_set(req_set, check_existing=not reinstall)
# Install and/or link.
pkg_set.install(subset or None,
link_env=env,
reinstall=reinstall,
relink=relink,
no_deps=no_deps)
if pkg_set._errored and not force_branch_link:
log.warning(
style_warning(
"Not creating branch or version links; force with --force-branch-link"
))
return False
# Create a symlink by branch.
path_by_branch = self.home._abs_path('environments', self.name,
self.branch_name)
if os.path.lexists(path_by_branch):
os.unlink(path_by_branch)
makedirs(os.path.dirname(path_by_branch))
os.symlink(env.path, path_by_branch)
# Create a symlink by version.
version = req_set.headers.get('Version')
if version:
path_by_version = self.home._abs_path(
'environments', self.name, 'versions',
version.value + ('-dirty' if dirty else ''))
if os.path.lexists(path_by_version):
os.unlink(path_by_version)
makedirs(os.path.dirname(path_by_version))
os.symlink(env.path, path_by_version)
return True
def commit(args):
home = args.assert_home()
repo = home.get_repo(args.repo)
if not repo.status():
log.error('Nothing to commit.')
return 1
if args.semver_level is None:
args.semver_level = 0 if args.micro else 2
if args.message is None:
dev_pkgs = {pkg.name: pkg for pkg in home.iter_development_packages()}
pkg_set = PackageSet(home=home)
by_name = {}
for revision, name in [
(None, 'work'),
('HEAD', 'head'),
]:
for req in repo.load_manifest(revision=revision).iter_packages():
pkg = pkg_set.resolve(req, check_existing=False)
if pkg.fetch_type != 'git':
continue
by_name.setdefault(pkg.name, {})[name] = req
commits = []
for pkg_name, reqs in sorted(by_name.items()):
new = reqs['work']
old = reqs['head']
if new.version == old.version:
continue
dev = dev_pkgs.get(pkg_name)
if not dev:
continue
for line in dev.git('log',
'--pretty=%cI %h %s',
'{}...{}'.format(old.version, new.version),
stdout=True).splitlines():
line = line.strip()
if not line:
continue
time, hash_, subject = line.split(' ', 2)
commits.append((time, pkg_name,
'[{} {}] {}'.format(pkg_name, hash_, subject)))
if commits:
if len(commits) == 1:
default_message = [commits[0][2]]
else:
pkg_names = set(c[1] for c in commits)
default_message = [
'{} commit{} in {} package{}: {}.'.format(
len(commits),
's' if len(commits) != 1 else '',
len(pkg_names),
's' if len(pkg_names) != 1 else '',
', '.join(sorted(pkg_names)),
), ''
]
for c in commits:
default_message.append(c[2])
else:
default_message = [default_messages[args.semver_level]]
fd, path = tempfile.mkstemp('.txt', 'vee-commit-msg.')
with open(path, 'w') as fh:
fh.write('\n'.join(default_message))
fh.write('''
# Please enter the commit message for your changes. Lines starting
# with '#' will be ignored, and an empty message aborts the commit.
''')
editor = os.environ.get('EDITOR', 'vim')
editor_args = [editor, path]
if editor == 'vim':
editor_args.insert(1, r'+syntax match Comment "^\s*#.*$"')
code = subprocess.call(editor_args)
message = open(path).readlines()
os.close(fd)
os.unlink(path)
if code:
log.error("Editor ({}) failed".format(editor),
"and returned code {}".format(code))
return
message = [
line.rstrip() for line in message
if not line.lstrip().startswith('#')
]
message = '\n'.join(message).strip()
if not message:
return
args.message = message
repo.commit(args.message, args.semver_level)
def add(args):
home = args.assert_home()
env_repo = home.get_env_repo(args.repo)
req_set = env_repo.load_requirements()
pkg_set = PackageSet(home=home)
baked_any = None
if args.update:
baked_any = False
for req in req_set.iter_packages():
pkg = pkg_set.resolve(req, check_existing=False)
if pkg.fetch_type != 'git':
continue
print style_note('Fetching', str(req))
pkg.repo.fetch('origin',
'master') # TODO: track these another way?
if pkg.repo.check_ff_safety('origin/master'):
pkg.repo.checkout('origin/master')
head = pkg.repo.head[:8]
if head != req.revision:
req.revision = pkg.repo.head[:8]
print style_note('Updated', str(req))
baked_any = True
if args.bake_installed:
baked_any = False
for req in req_set.iter_packages():
pkg = pkg_set.resolve(req)
if pkg.fetch_type != 'git':
continue
repo = pkg.pipeline.steps['fetch'].repo
if req.name and req.name == guess_name(req.url):
req.name = None
baked_any = True
print style_note('Unset redundant name', req.name)
if pkg.installed and req.revision != repo.head[:8]:
req.revision = repo.head[:8]
baked_any = True
print style_note('Pinned', req.name, req.revision)
if args.checksum:
baked_any = False
for req in req_set.iter_packages():
pkg = pkg_set.resolve(req)
if pkg.checksum:
continue
if not pkg.package_path or not os.path.isfile(pkg.package_path):
continue
req.checksum = checksum_file(pkg.package_path)
print style_note('Checksummed', pkg.name, req.checksum)
baked_any = True
if baked_any is not None:
if baked_any:
env_repo.dump_requirements(req_set)
else:
print style_note('No changes.')
return
row = home.get_development_record(os.path.abspath(args.package))
if not row:
raise ValueError('No development package %r' % args.package)
dev_repo = GitRepo(row['path'])
# Get the normalized origin.
dev_remote_urls = set()
for url in dev_repo.remotes().itervalues():
url = normalize_git_url(url, prefer='scp') or url
log.debug('adding dev remote url: %s' % url)
dev_remote_urls.add(url)
if not dev_remote_urls:
print style_error('No git remotes for %s' % row['path'])
return 1
for req in req_set.iter_packages(eval_control=False):
# We only deal with git packages.
pkg = pkg_set.resolve(req, check_existing=False)
if pkg.fetch_type != 'git':
continue
req_url = normalize_git_url(req.url, prefer='scp')
log.debug('does match package url?: %s' % req_url)
if req_url in dev_remote_urls:
if req.revision == dev_repo.head[:8]:
print style_note('No change to', str(req))
else:
req.revision = dev_repo.head[:8]
print style_note('Updated', str(req))
break
else:
if not args.init:
print '{error}: No required package {name}; would match one of:'.format(
error=style('Error', 'red'),
name=style(args.package, bold=True))
for url in sorted(dev_remote_urls):
print ' {}'.format(url)
print 'Use {} to setup: git+{} --revision {}'.format(
style('vee add --init %s' % args.package, 'green'),
dev_repo.remotes()['origin'], dev_repo.head[:8])
return 1
req = Package(
url=normalize_git_url(dev_repo.remotes()['origin'], prefix=True),
revision=dev_repo.head[:8],
home=home,
)
req_set.append(('', req, ''))
env_repo.dump_requirements(req_set)
def link(args):
"""Link the given requirement or requirements into the given environment,
e.g.::
# Install and link a single package.
$ vee link [email protected]:vfxetc/sgmock
# Install and link multiple packages.
$ vee link [email protected]:vfxetc/sgmock [email protected]:vfxetc/sgsession \\
http:/example.org/path/to/tarball.tgz --make-install
# Install and link from a requirement set.
$ vee link path/to/manifest.txt
"""
if args.no_install and args.re_install:
raise ValueError(
'please use only one of --no-install and --re-install')
home = args.assert_home()
if sum(
int(bool(x))
for x in (args.repo, args.environment, args.directory)) > 1:
raise ValueError(
'use only one of --repo, --environment, or --directory')
if args.environment:
env = Environment(args.environment, home=home)
elif args.directory:
env = Environment(os.path.abspath(args.directory), home=home)
else:
repo = home.get_repo(args.repo)
env = Environment('%s/%s' % (repo.name, repo.branch_name), home=home)
if args.raw:
for dir_ in args.requirements:
log.info(style_note('Linking', dir_))
env.link_directory(dir_)
return
manifest = Manifest(args.requirements, home=home)
pkg_set = PackageSet(env=env, home=home)
# Register the whole set, so that dependencies are pulled from here instead
# of weakly resolved from installed packages.
pkg_set.resolve_set(manifest)
for req in manifest.iter_packages():
# Skip if it wasn't requested.
if args.subset and req.name not in args.subset:
continue
log.info(style('==> %s' % req.name, 'blue'))
pkg = pkg_set.resolve(req, check_existing=not args.reinstall)
if args.no_install and not pkg.installed:
raise CliError('not installed: %s' % req)
try:
with log.indent():
pkg_set.install(pkg.name,
link_env=env,
reinstall=args.reinstall,
relink=args.force)
except AlreadyInstalled:
pass
except AlreadyLinked as e:
log.info(style('Already linked ', 'blue') + str(req), verbosity=1)
except Exception as e:
print_cli_exc(e, verbose=True)
log.exception('Failed to link %s' % req)
continue
def status(args):
home = args.assert_home()
env_repo = home.get_env_repo(args.repo)
pkg_set = PackageSet(home=home)
by_name = {}
# Dev packages.
for row in home.db.execute('SELECT * FROM development_packages'):
row = dict(row)
if not os.path.exists(row['path']):
continue
dev_repo = GitRepo(row['path'])
row['remotes'] = dev_repo.remotes()
by_name.setdefault(row['name'], {})['dev'] = row
# Current requirements.
for revision, name in [
(None, 'work'),
('HEAD', 'head'),
]:
for req in env_repo.load_requirements(
revision=revision).iter_packages():
pkg = pkg_set.resolve(req, check_existing=False)
if pkg.fetch_type != 'git':
continue
by_name.setdefault(pkg.name, {})[name] = req
by_name = by_name.items()
by_name.sort(key=lambda x: x[0].lower())
if args.names:
by_name = [x for x in by_name if x[0] in args.names]
for name, everything in by_name:
dev_row = everything.get('dev')
work_req = everything.get('work')
head_req = everything.get('head')
has_dev = dev_row is not None
only_has_dev = has_dev and not (work_req or head_req)
# Skip dev-only stuff most of the time.
if only_has_dev and not args.all_dev:
continue
# Title.
print '%s %s' % (style(
'%s %s' % ('==>' if has_dev else '-->', name),
fg='blue'), '(dev only)' if only_has_dev else '')
# Status of requirements.
if work_req and head_req and str(work_req) == str(head_req):
if args.verbose:
print '=== %s' % work_req
else:
# Print a lovely coloured diff of the specific arguments that
# are changing.
# TODO: make this environment relative to the context.
head_args = head_req.to_args(
exclude=('base_environ', )) if head_req else []
work_args = work_req.to_args(
exclude=('base_environ', )) if work_req else []
differ = difflib.SequenceMatcher(None, head_args, work_args)
opcodes = differ.get_opcodes()
if head_req is not None:
print style('---', fg='red', bold=True),
for tag, i1, i2, j1, j2 in opcodes:
if tag in ('replace', 'delete'):
print style(' '.join(head_args[i1:i2]),
fg='red',
bold=True)
elif tag in ('equal', ):
print ' '.join(head_args[i1:i2]),
if work_req is not None:
print style('+++', fg='green', bold=True),
for tag, i1, i2, j1, j2 in opcodes:
if tag in ('replace', 'insert'):
print style(' '.join(work_args[j1:j2]),
fg='green',
bold=True)
elif tag in ('equal', ):
print ' '.join(work_args[j1:j2]),
if dev_row:
if 'warning' in dev_row:
print dev_row['warning']
if 'origin' in dev_row['remotes']:
dev_row['remote_name'] = 'origin'
else:
remote_names = sorted(dev_row['remotes'])
dev_row['remote_name'] = remote_names[0]
if len(remote_names) != 1:
print ' ' + style_warning(
'More that one non-origin remote; picking %s' %
dev_row['remote_name'])
dev_repo = dev_row and GitRepo(dev_row['path'])
if dev_repo and not dev_repo.exists:
print style_warning('Git repo does not exist.')
dev_row = dev_repo = None
if dev_repo:
if dev_repo.status():
print ' ' + style_warning('Work tree is dirty.')
if args.fetch:
dev_remote_head = dev_repo.fetch(dev_row['remote_name'],
'master')
else:
dev_remote_head = dev_repo.rev_parse(dev_row['remote_name'] +
'/master')
# Check your local dev vs. its remote.
dev_local, dev_remote = dev_repo.distance(dev_repo.head,
dev_remote_head)
summarize_rev_distance(
dev_local,
dev_remote,
local_name=name,
local_verb='is',
remote_name='%s/master' % dev_row['remote_name'],
behind_action='please pull or `vee dev ff %s`' % name,
)
if dev_repo and work_req and work_req.revision:
# Check your local dev vs the required revision
try:
pkg_revision = dev_repo.rev_parse(work_req.revision)
pkg_local, pkg_remote = dev_repo.distance(
dev_repo.head, pkg_revision)
summarize_rev_distance(
pkg_local,
pkg_remote,
local_name=name,
local_verb='is',
remote_name='%s repo' % env_repo.name,
ahead_action='you may `vee add %s`' % name,
behind_action='please `vee dev checkout --repo %s %s`' %
(env_repo.name, name),
)
except Exception as e:
print ' ' + format_cli_exc(e)