def install(args): """Install the given requirements without linking them into an environment. This is a low-level command, and is generally unused. Examples: # Install a single package. vee install [email protected]:westernx/sgmock # Install multiple packages. vee install [email protected]:westernx/sgmock [email protected]:westernx/sgsession \\ http:/example.org/path/to/tarball.tgz --make-install # Install from a requirement set. vee install path/to/requirements.txt """ home = args.assert_home() if not args.requirements: raise ValueError('please provide requirements to install') reqs = Requirements(args.requirements, home=home) pkgs = PackageSet(home=home) # TODO: Resolve everything at once like upgrade does. for req in reqs.iter_packages(): pkg = pkgs.resolve(req, check_existing=not args.force) try: pkgs.install(pkg.name, reinstall=args.force) except AlreadyInstalled: print style('Already installed', 'blue', bold=True), style(str(pkg.freeze()), bold=True)
def iter_availible_requirements(home): repo = home.get_repo() req_set = repo.requirement_set() pkg_set = PackageSet(home=home) for req in req_set.iter_packages(): pkg = pkg_set.resolve(req, check_existing=False) if pkg.type != 'git': return yield req, normalize_git_url(req.url, prefix=False)
def exec_(args): """Construct an environment, and either export it or run a command in it. e.g.:: # Run in the default repository. $ vee exec $command # Run within a given repository. $ vee exec --repo named_repo $command # Run within a named environment. $ vee exec -e named_environ $command # Run within a constructed runtime for a set of requirements. $ vee exec -r requirements.txt $command # Export the default environment. $ vee exec --export export LD_LIBRARY_PATH="/usr/local/vee/lib:$LD_LIBRARY_PATH" export PATH="/usr/local/vee/bin:$PATH" export PYTHONPATH="/usr/local/vee/lib/python2.7/site-packages" """ home = args.assert_home() # TODO: seed these with the current values. repo_names = [] env_names = [] environ_diff = {} if args.bootstrap: bootstrap = os.environ['VEE_EXEC_ARGS'].split() if os.environ.get( 'VEE_EXEC_ARGS') else [] # This is gross, but easier than building another parser. It does mean # that we expect this variable must be set by ourselves. while bootstrap: arg = bootstrap.pop(0) if arg in ('--dev', ): setattr(args, arg[2:], True) elif arg in ('--requirements', '--repo', '--environment'): v = getattr(args, arg[2:], None) or [] v.append(bootstrap.pop(0)) setattr(args, arg[2:], v) else: print >> sys.stderr, 'cannot bootstrap', arg if args.dev: # Store the original flags as provided so that --bootstrap can pick it back up. bootstrap = os.environ['VEE_EXEC_ARGS'].split() if os.environ.get( 'VEE_EXEC_ARGS') else [] if '--dev' not in bootstrap: bootstrap.append('--dev') for attr in 'requirements', 'repo', 'environment': for value in getattr(args, attr, None) or (): bootstrap.append('--' + attr) bootstrap.append(value) environ_diff['VEE_EXEC_ARGS'] = ' '.join(bootstrap) if not (args.export or args.command or args.prefix): raise ValueError( 'Must either --prefix, --export, or provide a command') # Default to the default repo. if not (args.requirements or args.environment or args.repos): args.repos = [None] paths = [] # Named (or default) repos. for name in args.repos or (): repo = home.get_env_repo(name or None) # Allow '' to be the default. args.environment = args.environment or [] args.environment.append('%s/%s' % (repo.name, repo.branch_name)) repo_names.append(repo.name) # Requirements and requirement sets. req_args = [] for arg in args.requirements or (): req_args.extend(arg.split(',')) req_set = Requirements(home=home) req_set.parse_args(req_args) pkg_set = PackageSet(home=home) for req in req_set.iter_packages(): pkg = pkg_set.resolve(req) pkg._assert_paths(install=True) if not pkg.installed: raise NotInstalled(pkg.install_path) paths.append(pkg.install_path) # Named environments. for name in args.environment or (): env = Environment(name, home=home) paths.append(env.path) env_names.append(name) if args.prefix: for path in paths: print path return environ_diff.update(guess_envvars(paths)) if args.dev: for pkg in home.iter_development_packages(exists=True, search=True): if pkg.environ: environ_diff.update( render_envvars(pkg.environ, pkg.work_tree, environ_diff)) # Add the current virtualenv. venv = os.environ.get('VIRTUAL_ENV') if venv: environ_diff['PYTHONPATH'] = '%s:%s' % ( os.path.join(venv, 'lib', 'python%d.%d' % sys.version_info[:2], 'site-packages'), environ_diff.get('PYTHONPATH', ''), # This is sloppy. ) # More environment variables. command = args.command or [] while command and re.match(r'^\w+=', command[0]): k, v = command.pop(0).split('=', 1) environ_diff[k] = v # Make sure setuptools is bootstrapped. bootstrap_environ(environ_diff) environ_diff['VEE_EXEC_PATH'] = ':'.join(paths) environ_diff['VEE_EXEC_REPO'] = ','.join(repo_names) environ_diff['VEE_EXEC_ENV'] = ','.join(env_names) environ_diff['VEE_EXEC_PREFIX'] = paths[0] # Print it out instead of running it. if args.export: for k, v in sorted(environ_diff.iteritems()): existing = os.environ.get(k) # Since we modify os.environ in __init__ to bootstrap the vendored # packages, swaping out the original values will not include the # bootstrap. So we are tricking the code so that it still includes it. if k == 'PYTHONPATH' and existing.endswith(vendor_path): existing += (':' if existing else '') + vendor_path if existing is not None and not k.startswith('VEE_EXEC'): v = v.replace(existing, '$' + k) print 'export %s="%s"' % (k, v) return environ = os.environ.copy() environ.update(environ_diff) os.execvpe(args.command[0], args.command, environ)
def commit(args): home = args.assert_home() repo = home.get_repo(args.repo) if not repo.status(): log.error('Nothing to commit.') return 1 if args.semver_level is None: args.semver_level = 0 if args.micro else 2 if args.message is None: dev_pkgs = {pkg.name: pkg for pkg in home.iter_development_packages()} pkg_set = PackageSet(home=home) by_name = {} for revision, name in [ (None, 'work'), ('HEAD', 'head'), ]: for req in repo.load_manifest(revision=revision).iter_packages(): pkg = pkg_set.resolve(req, check_existing=False) if pkg.fetch_type != 'git': continue by_name.setdefault(pkg.name, {})[name] = req commits = [] for pkg_name, reqs in sorted(by_name.items()): new = reqs['work'] old = reqs['head'] if new.version == old.version: continue dev = dev_pkgs.get(pkg_name) if not dev: continue for line in dev.git('log', '--pretty=%cI %h %s', '{}...{}'.format(old.version, new.version), stdout=True).splitlines(): line = line.strip() if not line: continue time, hash_, subject = line.split(' ', 2) commits.append((time, pkg_name, '[{} {}] {}'.format(pkg_name, hash_, subject))) if commits: if len(commits) == 1: default_message = [commits[0][2]] else: pkg_names = set(c[1] for c in commits) default_message = [ '{} commit{} in {} package{}: {}.'.format( len(commits), 's' if len(commits) != 1 else '', len(pkg_names), 's' if len(pkg_names) != 1 else '', ', '.join(sorted(pkg_names)), ), '' ] for c in commits: default_message.append(c[2]) else: default_message = [default_messages[args.semver_level]] fd, path = tempfile.mkstemp('.txt', 'vee-commit-msg.') with open(path, 'w') as fh: fh.write('\n'.join(default_message)) fh.write(''' # Please enter the commit message for your changes. Lines starting # with '#' will be ignored, and an empty message aborts the commit. ''') editor = os.environ.get('EDITOR', 'vim') editor_args = [editor, path] if editor == 'vim': editor_args.insert(1, r'+syntax match Comment "^\s*#.*$"') code = subprocess.call(editor_args) message = open(path).readlines() os.close(fd) os.unlink(path) if code: log.error("Editor ({}) failed".format(editor), "and returned code {}".format(code)) return message = [ line.rstrip() for line in message if not line.lstrip().startswith('#') ] message = '\n'.join(message).strip() if not message: return args.message = message repo.commit(args.message, args.semver_level)
def add(args): home = args.assert_home() env_repo = home.get_env_repo(args.repo) req_set = env_repo.load_requirements() pkg_set = PackageSet(home=home) baked_any = None if args.update: baked_any = False for req in req_set.iter_packages(): pkg = pkg_set.resolve(req, check_existing=False) if pkg.fetch_type != 'git': continue print style_note('Fetching', str(req)) pkg.repo.fetch('origin', 'master') # TODO: track these another way? if pkg.repo.check_ff_safety('origin/master'): pkg.repo.checkout('origin/master') head = pkg.repo.head[:8] if head != req.revision: req.revision = pkg.repo.head[:8] print style_note('Updated', str(req)) baked_any = True if args.bake_installed: baked_any = False for req in req_set.iter_packages(): pkg = pkg_set.resolve(req) if pkg.fetch_type != 'git': continue repo = pkg.pipeline.steps['fetch'].repo if req.name and req.name == guess_name(req.url): req.name = None baked_any = True print style_note('Unset redundant name', req.name) if pkg.installed and req.revision != repo.head[:8]: req.revision = repo.head[:8] baked_any = True print style_note('Pinned', req.name, req.revision) if args.checksum: baked_any = False for req in req_set.iter_packages(): pkg = pkg_set.resolve(req) if pkg.checksum: continue if not pkg.package_path or not os.path.isfile(pkg.package_path): continue req.checksum = checksum_file(pkg.package_path) print style_note('Checksummed', pkg.name, req.checksum) baked_any = True if baked_any is not None: if baked_any: env_repo.dump_requirements(req_set) else: print style_note('No changes.') return row = home.get_development_record(os.path.abspath(args.package)) if not row: raise ValueError('No development package %r' % args.package) dev_repo = GitRepo(row['path']) # Get the normalized origin. dev_remote_urls = set() for url in dev_repo.remotes().itervalues(): url = normalize_git_url(url, prefer='scp') or url log.debug('adding dev remote url: %s' % url) dev_remote_urls.add(url) if not dev_remote_urls: print style_error('No git remotes for %s' % row['path']) return 1 for req in req_set.iter_packages(eval_control=False): # We only deal with git packages. pkg = pkg_set.resolve(req, check_existing=False) if pkg.fetch_type != 'git': continue req_url = normalize_git_url(req.url, prefer='scp') log.debug('does match package url?: %s' % req_url) if req_url in dev_remote_urls: if req.revision == dev_repo.head[:8]: print style_note('No change to', str(req)) else: req.revision = dev_repo.head[:8] print style_note('Updated', str(req)) break else: if not args.init: print '{error}: No required package {name}; would match one of:'.format( error=style('Error', 'red'), name=style(args.package, bold=True)) for url in sorted(dev_remote_urls): print ' {}'.format(url) print 'Use {} to setup: git+{} --revision {}'.format( style('vee add --init %s' % args.package, 'green'), dev_repo.remotes()['origin'], dev_repo.head[:8]) return 1 req = Package( url=normalize_git_url(dev_repo.remotes()['origin'], prefix=True), revision=dev_repo.head[:8], home=home, ) req_set.append(('', req, '')) env_repo.dump_requirements(req_set)
def link(args): """Link the given requirement or requirements into the given environment, e.g.:: # Install and link a single package. $ vee link [email protected]:vfxetc/sgmock # Install and link multiple packages. $ vee link [email protected]:vfxetc/sgmock [email protected]:vfxetc/sgsession \\ http:/example.org/path/to/tarball.tgz --make-install # Install and link from a requirement set. $ vee link path/to/manifest.txt """ if args.no_install and args.re_install: raise ValueError( 'please use only one of --no-install and --re-install') home = args.assert_home() if sum( int(bool(x)) for x in (args.repo, args.environment, args.directory)) > 1: raise ValueError( 'use only one of --repo, --environment, or --directory') if args.environment: env = Environment(args.environment, home=home) elif args.directory: env = Environment(os.path.abspath(args.directory), home=home) else: repo = home.get_repo(args.repo) env = Environment('%s/%s' % (repo.name, repo.branch_name), home=home) if args.raw: for dir_ in args.requirements: log.info(style_note('Linking', dir_)) env.link_directory(dir_) return manifest = Manifest(args.requirements, home=home) pkg_set = PackageSet(env=env, home=home) # Register the whole set, so that dependencies are pulled from here instead # of weakly resolved from installed packages. pkg_set.resolve_set(manifest) for req in manifest.iter_packages(): # Skip if it wasn't requested. if args.subset and req.name not in args.subset: continue log.info(style('==> %s' % req.name, 'blue')) pkg = pkg_set.resolve(req, check_existing=not args.reinstall) if args.no_install and not pkg.installed: raise CliError('not installed: %s' % req) try: with log.indent(): pkg_set.install(pkg.name, link_env=env, reinstall=args.reinstall, relink=args.force) except AlreadyInstalled: pass except AlreadyLinked as e: log.info(style('Already linked ', 'blue') + str(req), verbosity=1) except Exception as e: print_cli_exc(e, verbose=True) log.exception('Failed to link %s' % req) continue
def status(args): home = args.assert_home() env_repo = home.get_env_repo(args.repo) pkg_set = PackageSet(home=home) by_name = {} # Dev packages. for row in home.db.execute('SELECT * FROM development_packages'): row = dict(row) if not os.path.exists(row['path']): continue dev_repo = GitRepo(row['path']) row['remotes'] = dev_repo.remotes() by_name.setdefault(row['name'], {})['dev'] = row # Current requirements. for revision, name in [ (None, 'work'), ('HEAD', 'head'), ]: for req in env_repo.load_requirements( revision=revision).iter_packages(): pkg = pkg_set.resolve(req, check_existing=False) if pkg.fetch_type != 'git': continue by_name.setdefault(pkg.name, {})[name] = req by_name = by_name.items() by_name.sort(key=lambda x: x[0].lower()) if args.names: by_name = [x for x in by_name if x[0] in args.names] for name, everything in by_name: dev_row = everything.get('dev') work_req = everything.get('work') head_req = everything.get('head') has_dev = dev_row is not None only_has_dev = has_dev and not (work_req or head_req) # Skip dev-only stuff most of the time. if only_has_dev and not args.all_dev: continue # Title. print '%s %s' % (style( '%s %s' % ('==>' if has_dev else '-->', name), fg='blue'), '(dev only)' if only_has_dev else '') # Status of requirements. if work_req and head_req and str(work_req) == str(head_req): if args.verbose: print '=== %s' % work_req else: # Print a lovely coloured diff of the specific arguments that # are changing. # TODO: make this environment relative to the context. head_args = head_req.to_args( exclude=('base_environ', )) if head_req else [] work_args = work_req.to_args( exclude=('base_environ', )) if work_req else [] differ = difflib.SequenceMatcher(None, head_args, work_args) opcodes = differ.get_opcodes() if head_req is not None: print style('---', fg='red', bold=True), for tag, i1, i2, j1, j2 in opcodes: if tag in ('replace', 'delete'): print style(' '.join(head_args[i1:i2]), fg='red', bold=True) elif tag in ('equal', ): print ' '.join(head_args[i1:i2]), if work_req is not None: print style('+++', fg='green', bold=True), for tag, i1, i2, j1, j2 in opcodes: if tag in ('replace', 'insert'): print style(' '.join(work_args[j1:j2]), fg='green', bold=True) elif tag in ('equal', ): print ' '.join(work_args[j1:j2]), if dev_row: if 'warning' in dev_row: print dev_row['warning'] if 'origin' in dev_row['remotes']: dev_row['remote_name'] = 'origin' else: remote_names = sorted(dev_row['remotes']) dev_row['remote_name'] = remote_names[0] if len(remote_names) != 1: print ' ' + style_warning( 'More that one non-origin remote; picking %s' % dev_row['remote_name']) dev_repo = dev_row and GitRepo(dev_row['path']) if dev_repo and not dev_repo.exists: print style_warning('Git repo does not exist.') dev_row = dev_repo = None if dev_repo: if dev_repo.status(): print ' ' + style_warning('Work tree is dirty.') if args.fetch: dev_remote_head = dev_repo.fetch(dev_row['remote_name'], 'master') else: dev_remote_head = dev_repo.rev_parse(dev_row['remote_name'] + '/master') # Check your local dev vs. its remote. dev_local, dev_remote = dev_repo.distance(dev_repo.head, dev_remote_head) summarize_rev_distance( dev_local, dev_remote, local_name=name, local_verb='is', remote_name='%s/master' % dev_row['remote_name'], behind_action='please pull or `vee dev ff %s`' % name, ) if dev_repo and work_req and work_req.revision: # Check your local dev vs the required revision try: pkg_revision = dev_repo.rev_parse(work_req.revision) pkg_local, pkg_remote = dev_repo.distance( dev_repo.head, pkg_revision) summarize_rev_distance( pkg_local, pkg_remote, local_name=name, local_verb='is', remote_name='%s repo' % env_repo.name, ahead_action='you may `vee add %s`' % name, behind_action='please `vee dev checkout --repo %s %s`' % (env_repo.name, name), ) except Exception as e: print ' ' + format_cli_exc(e)