class GitTransport(PipelineStep): factory_priority = 1000 @classmethod def factory(cls, step, pkg): if step == 'init' and re.match(r'^git[:+]', pkg.url): return cls(pkg) def get_next(self, step): if step == 'fetch': return self def init(self): pkg = self.package pkg.url = normalize_git_url(pkg.url, prefix=True) or pkg.url pkg._assert_paths(package=True) self.repo = GitRepo(work_tree=pkg.package_path, remote_url=re.sub(r'^git[:\+]', '', pkg.url)) def fetch(self): pkg = self.package self.repo.clone_if_not_exists() self.repo.checkout(pkg.revision or 'HEAD', fetch=True) pkg.revision = self.repo.head[:8]
class GitTransport(PipelineStep): factory_priority = 1000 @classmethod def factory(cls, step, pkg): if step == 'init' and re.match(r'^git[:+]', pkg.url): return cls() def get_next(self, step, pkg): if step == 'fetch': return self def init(self, pkg): pkg.url = normalize_git_url(pkg.url, prefix=True) or pkg.url pkg._assert_paths(package=True) self.repo = GitRepo(work_tree=pkg.package_path, remote_url=re.sub(r'^git[:\+]', '', pkg.url)) # Resolve branches by fetching. if pkg.version and not re.match(r'^[0-9a-f]{8,}$', pkg.version): self.repo.clone_if_not_exists() rev = self.repo.fetch(ref=pkg.version) pkg.version = rev[:8] def fetch(self, pkg): self.repo.clone_if_not_exists() self.repo.checkout(pkg.version or 'HEAD', fetch=True) self.repo.git('submodule', 'update', '--init') pkg.version = self.repo.head[:8]
def __init__(self, path=None, repo=None, home=None): self.name = 'linuxbrew' if sys.platform.startswith( 'linux') else 'homebrew' if repo: self.repo = repo else: work_tree = (path or os.environ.get('VEE_HOMEBREW') or home._abs_path('packages', self.name)) remote_url = 'https://github.com/{}/brew.git'.format(self.name) self.repo = GitRepo(work_tree=work_tree, remote_url=remote_url) self._info = {}
def init(self, pkg): pkg.url = normalize_git_url(pkg.url, prefix=True) or pkg.url pkg._assert_paths(package=True) self.repo = GitRepo(work_tree=pkg.package_path, remote_url=re.sub(r'^git[:\+]', '', pkg.url)) # Resolve branches by fetching. if pkg.version and not re.match(r'^[0-9a-f]{8,}$', pkg.version): self.repo.clone_if_not_exists() rev = self.repo.fetch(ref=pkg.version) pkg.version = rev[:8]
def __init__(self, name, home=None): self.name = name self.path = os.path.abspath( os.path.join(__file__, '..', '..', '..', 'sandbox', 'repos', name)) makedirs(self.path) self.repo = GitRepo(self.path) self.repo.git('init', silent=True, stdout=True) if home is None: from tests import home self.home = home self._rev_count = None
def __init__(self, name, template, defaults=None, path=None): self.name = name self.template = os.path.abspath( os.path.join(__file__, '..', '..', 'package-templates', template)) self.path = os.path.abspath( os.path.join(__file__, '..', '..', '..', 'sandbox', 'packages', path or name)) makedirs(self.path) self.repo = GitRepo(self.path) self.repo.git('init', silent=True, stdout=True) self.defaults = defaults or {} self.defaults.setdefault('NAME', self.name) self.defaults.setdefault('VERSION', '1.0.0') self.rev_count = 0
class Homebrew(object): def __init__(self, path=None, repo=None, home=None): self.name = 'linuxbrew' if sys.platform.startswith( 'linux') else 'homebrew' if repo: self.repo = repo else: work_tree = (path or os.environ.get('VEE_HOMEBREW') or home._abs_path('packages', self.name)) remote_url = 'https://github.com/Homebrew/%s.git' % self.name self.repo = GitRepo(work_tree=work_tree, remote_url=remote_url) self._info = {} @cached_property def cellar(self): return os.path.join(self.repo.work_tree, 'Cellar') def __call__(self, cmd, *args, **kwargs): self.repo.clone_if_not_exists() bin = os.path.join(self.repo.work_tree, 'bin', 'brew') # We need to own the homebrew cache so that we can control permissions. kwargs['env'] = env = kwargs.get('env', os.environ).copy() env.setdefault('HOMEBREW_CACHE', os.path.join(self.repo.work_tree, 'Cache')) res = call((bin, cmd) + args, _frame=1, **kwargs) if cmd in ('install', 'uninstall'): self._info.pop(args[0], None) return res def info(self, name, force=False): if self._info is None: self._info = {} if force or name not in self._info: self._info[name] = json.loads( self('info', '--json=v1', name, stdout=True))[0] return self._info[name]
def __call__(self, cmd, *args, **kwargs): brew_head = os.environ.get('VEE_HOMEBREW_COMMIT') core_head = os.environ.get('VEE_HOMEBREW_CORE_COMMIT') bin_ = os.path.join(self.repo.work_tree, 'bin', 'brew') if self.repo.clone_if_not_exists(shallow=not brew_head): # Homebrew should be updated the first time, since it has gotten # a little more complicated. We call it directly so that # it can update itself here, and then our pin will drag it back. call((bin_, 'update')) if brew_head: if self.repo.is_shallow: self.repo.fetch(shallow=False) self.repo.checkout(brew_head) if core_head: repo = GitRepo( os.path.join(self.repo.work_tree, 'Library', 'Taps', 'homebrew', 'homebrew-core'), 'https://github.com/homebrew/homebrew-core.git', ) repo.clone_if_not_exists(shallow=False) if repo.is_shallow: repo.fetch(shallow=False) repo.checkout(core_head) # We need to own the homebrew cache so that we can control permissions. kwargs['env'] = env = kwargs.get('env', os.environ).copy() env.setdefault('HOMEBREW_CACHE', os.path.join(self.repo.work_tree, 'Cache')) env.setdefault('HOMEBREW_LOGS', os.path.join(self.repo.work_tree, 'Logs')) # Keep our pin. if brew_head or core_head: env['HOMEBREW_NO_AUTO_UPDATE'] = '1' res = call((bin_, cmd) + args, _frame=1, **kwargs) if cmd in ('install', 'uninstall'): self._info.pop(args[0], None) return res
def init(args, do_clone=False, do_install=False, do_add=False, is_find=False): do_init = not (do_clone or do_install or do_add) name = args.name home = args.assert_home() con = home.db.connect() path = os.path.abspath(args.path or os.path.join(home.dev_root, name)) dev_repo = GitRepo(path) if do_init: log.info(style_note('Initing %s' % dev_repo.work_tree)) makedirs(dev_repo.work_tree) dev_repo.git('init') elif do_clone: log.info(style_note('Cloning %s' % args.url)) makedirs(dev_repo.work_tree) dev_repo.clone_if_not_exists(args.url) elif do_install: # Find an existing tool. # TODO: put more of this into EnvironmentRepo or Manifest repo = home.get_repo(args.repo) manifest_path = os.path.join(repo.work_tree, 'manifest.txt') manifest = Manifest(manifest_path, home=home) for req in manifest.iter_packages(): if req.name.lower() == name.lower(): # Make sure it is a Git package. url = normalize_git_url(req.url, prefix=False) if url: break else: log.error('Could not find git-based "%s" in "%s" repo.' % (name, repo.name)) return 2 log.info(style_note('Found %s in %s' % (name, repo.name), str(req))) makedirs(dev_repo.work_tree) dev_repo.clone_if_not_exists(url, shallow=False) elif do_add: log.info(style_note('Adding %s from %s' % (name, path))) if not os.path.exists(path): log.error('%s does not exist' % path) return 1 package = Package([path], home=home, dev=True) try: package.pipeline.run_to('develop') except Exception as e: print_cli_exc(e) return 1 log.info(style_note('Linking dev package', name, path)) dev_pkg = DevPackage( { 'name': name, 'path': path, 'environ': package.environ }, home=home) dev_pkg.save_tag()
class MockPackage(object): def __init__(self, name, template, defaults=None, path=None): self.name = name self.template = os.path.abspath( os.path.join(__file__, '..', '..', 'package-templates', template)) self.path = os.path.abspath( os.path.join(__file__, '..', '..', '..', 'sandbox', 'packages', path or name)) makedirs(self.path) self.repo = GitRepo(self.path) self.repo.git('init', silent=True, stdout=True) self.defaults = defaults or {} self.defaults.setdefault('NAME', self.name) self.defaults.setdefault('VERSION', '1.0.0') self.defaults.setdefault('PYTHON', '{}.{}'.format(*get_default_python().version)) self.rev_count = 0 def clone(self, path): return MockPackage(self.name, os.path.basename(self.template), self.defaults.copy(), path) @property def git_url(self): return 'git+' + self.path @property def url(self): return mock_url(self.path + '.tgz') def rev_list(self): try: return self.repo.git('rev-list', '--all', silent=True, stdout=True, stderr=True)[0].strip().split() except GitError: return [] def render(self, **kwargs): self.rev_count += 1 params = self.defaults.copy() params.update(kwargs) params.update(REVNO=self.rev_count, ) def render_contents(contents): return re.sub(r'MOCK([A-Z0-9]+)', lambda m: str(params.get(m.group(1)) or ''), contents) ignore_path = os.path.join(self.template, 'mockignore') if os.path.exists(ignore_path): patterns = [x.strip() for x in open(ignore_path)] + ['mockignore'] pattern = re.compile('|'.join( fnmatch.translate(x) for x in patterns if x)) else: pattern = None for dir_path, dir_names, file_names in os.walk(self.template): for file_name in file_names: if pattern and pattern.match(file_name): continue src_path = os.path.join(dir_path, file_name) rel_path = os.path.relpath(src_path, self.template) dst_path = os.path.join(self.path, render_contents(rel_path)) makedirs(os.path.dirname(dst_path)) contents = render_contents(open(src_path, 'r').read()) with open(dst_path, 'w') as fh: fh.write(contents) shutil.copystat(src_path, dst_path) def commit(self, message=None): self.repo.git('add', '--', self.path, silent=True, stdout=True) self.repo.git('commit', '-m', message or 'Rendered from template', silent=True, stdout=True) self.rev_count = (self.rev_count or 0) + 1 def render_commit(self, message=None, **kwargs): self.render(**kwargs) self.commit(message)
def add(args): home = args.assert_home() env_repo = home.get_env_repo(args.repo) req_set = env_repo.load_requirements() pkg_set = PackageSet(home=home) baked_any = None if args.update: baked_any = False for req in req_set.iter_packages(): pkg = pkg_set.resolve(req, check_existing=False) if pkg.fetch_type != 'git': continue print style_note('Fetching', str(req)) pkg.repo.fetch('origin', 'master') # TODO: track these another way? if pkg.repo.check_ff_safety('origin/master'): pkg.repo.checkout('origin/master') head = pkg.repo.head[:8] if head != req.revision: req.revision = pkg.repo.head[:8] print style_note('Updated', str(req)) baked_any = True if args.bake_installed: baked_any = False for req in req_set.iter_packages(): pkg = pkg_set.resolve(req) if pkg.fetch_type != 'git': continue repo = pkg.pipeline.steps['fetch'].repo if req.name and req.name == guess_name(req.url): req.name = None baked_any = True print style_note('Unset redundant name', req.name) if pkg.installed and req.revision != repo.head[:8]: req.revision = repo.head[:8] baked_any = True print style_note('Pinned', req.name, req.revision) if args.checksum: baked_any = False for req in req_set.iter_packages(): pkg = pkg_set.resolve(req) if pkg.checksum: continue if not pkg.package_path or not os.path.isfile(pkg.package_path): continue req.checksum = checksum_file(pkg.package_path) print style_note('Checksummed', pkg.name, req.checksum) baked_any = True if baked_any is not None: if baked_any: env_repo.dump_requirements(req_set) else: print style_note('No changes.') return row = home.get_development_record(os.path.abspath(args.package)) if not row: raise ValueError('No development package %r' % args.package) dev_repo = GitRepo(row['path']) # Get the normalized origin. dev_remote_urls = set() for url in dev_repo.remotes().itervalues(): url = normalize_git_url(url, prefer='scp') or url log.debug('adding dev remote url: %s' % url) dev_remote_urls.add(url) if not dev_remote_urls: print style_error('No git remotes for %s' % row['path']) return 1 for req in req_set.iter_packages(eval_control=False): # We only deal with git packages. pkg = pkg_set.resolve(req, check_existing=False) if pkg.fetch_type != 'git': continue req_url = normalize_git_url(req.url, prefer='scp') log.debug('does match package url?: %s' % req_url) if req_url in dev_remote_urls: if req.revision == dev_repo.head[:8]: print style_note('No change to', str(req)) else: req.revision = dev_repo.head[:8] print style_note('Updated', str(req)) break else: if not args.init: print '{error}: No required package {name}; would match one of:'.format( error=style('Error', 'red'), name=style(args.package, bold=True)) for url in sorted(dev_remote_urls): print ' {}'.format(url) print 'Use {} to setup: git+{} --revision {}'.format( style('vee add --init %s' % args.package, 'green'), dev_repo.remotes()['origin'], dev_repo.head[:8]) return 1 req = Package( url=normalize_git_url(dev_repo.remotes()['origin'], prefix=True), revision=dev_repo.head[:8], home=home, ) req_set.append(('', req, '')) env_repo.dump_requirements(req_set)
class Homebrew(object): def __init__(self, path=None, repo=None, home=None): self.name = 'linuxbrew' if sys.platform.startswith( 'linux') else 'homebrew' if repo: self.repo = repo else: work_tree = (path or os.environ.get('VEE_HOMEBREW') or home._abs_path('packages', self.name)) remote_url = 'https://github.com/{}/brew.git'.format(self.name) self.repo = GitRepo(work_tree=work_tree, remote_url=remote_url) self._info = {} @cached_property def cellar(self): return os.path.join(self.repo.work_tree, 'Cellar') def assert_tapped(self, name): path = os.path.join(self.repo.work_tree, 'Library', 'Taps', name) if not os.path.exists(path): self('tap', name) def __call__(self, cmd, *args, **kwargs): brew_head = os.environ.get('VEE_HOMEBREW_COMMIT') core_head = os.environ.get('VEE_HOMEBREW_CORE_COMMIT') bin_ = os.path.join(self.repo.work_tree, 'bin', 'brew') if self.repo.clone_if_not_exists(shallow=not brew_head): # Homebrew should be updated the first time, since it has gotten # a little more complicated. We call it directly so that # it can update itself here, and then our pin will drag it back. call((bin_, 'update')) if brew_head: if self.repo.is_shallow: self.repo.fetch(shallow=False) self.repo.checkout(brew_head) if core_head: repo = GitRepo( os.path.join(self.repo.work_tree, 'Library', 'Taps', 'homebrew', 'homebrew-core'), 'https://github.com/homebrew/homebrew-core.git', ) repo.clone_if_not_exists(shallow=False) if repo.is_shallow: repo.fetch(shallow=False) repo.checkout(core_head) # We need to own the homebrew cache so that we can control permissions. kwargs['env'] = env = kwargs.get('env', os.environ).copy() env.setdefault('HOMEBREW_CACHE', os.path.join(self.repo.work_tree, 'Cache')) env.setdefault('HOMEBREW_LOGS', os.path.join(self.repo.work_tree, 'Logs')) # Keep our pin. if brew_head or core_head: env['HOMEBREW_NO_AUTO_UPDATE'] = '1' res = call((bin_, cmd) + args, _frame=1, **kwargs) if cmd in ('install', 'uninstall'): self._info.pop(args[0], None) return res def info(self, name, force=False): if self._info is None: self._info = {} if force or name not in self._info: self._info[name] = json.loads( self('info', '--json=v1', name, stdout=True))[0] return self._info[name]
class MockRepo(object): def __init__(self, name, home=None): self.name = name self.path = os.path.abspath( os.path.join(__file__, '..', '..', '..', 'sandbox', 'repos', name)) makedirs(self.path) self.repo = GitRepo(self.path) self.repo.git('init', silent=True, stdout=True) if home is None: from tests import home self.home = home self._rev_count = None def rev_list(self): try: res = self.repo.git('rev-list', '--all', silent=True, stdout=True, stderr=True) return res[0].strip().split() except CalledProcessError: return [] @property def rev_count(self): if self._rev_count is None: self._rev_count = len(self.rev_list()) return self._rev_count def add_requirements(self, raw, insert=False, commit=True): old = Requirements(home=self.home) path = os.path.join(self.path, 'requirements.txt') if os.path.exists(path): old.parse_file(path) new = Requirements(home=self.home, file=StringIO(raw)) new_urls = set() new_names = set() for req in new.iter_packages(): new_names.add(req.name or guess_name(req.url)) new_urls.add(req.url) for prefix, element, postfix in old: if (not isinstance(element, Package) or (element.name or guess_name(element.url)) not in new_names or element.url not in new_urls): if insert: new.append((prefix, element, postfix)) else: new.insert(0, (prefix, element, postfix)) with open(path, 'wb') as fh: for line in new.iter_dump(): fh.write(line) if commit: self.commit('add requirements') def commit(self, message): self.repo.git('add', 'requirements.txt', silent=True, stdout=True) self.repo.git('commit', '-m', message or 'do something', silent=True, stdout=True) self._rev_count = (self._rev_count or 0) + 1
def init(args, do_clone=False, do_install=False, do_add=False, is_find=False): do_init = not (do_clone or do_install or do_add) name = args.name home = args.assert_home() con = home.db.connect() # Make sure there are no other packages already, and clear out old ones # which no longer exist. for row in con.execute('SELECT * FROM development_packages WHERE name = ?', [name]): if not args.force and os.path.exists(os.path.join(row['path'], '.git')): if is_find: print style_note('"%s" already exists:' % name, row['path']) return else: print style_error('"%s" already exists:' % name, row['path']) return 1 else: con.execute('DELETE FROM development_packages WHERE id = ?', [row['id']]) path = os.path.abspath(args.path or os.path.join(home.dev_root, name)) dev_repo = GitRepo(path) if do_init: print style_note('Initing %s' % dev_repo.work_tree) makedirs(dev_repo.work_tree) dev_repo.git('init') elif do_clone: print style_note('Cloning %s' % args.url) makedirs(dev_repo.work_tree) dev_repo.clone_if_not_exists(args.url) elif do_install: # Find an existing tool. # TODO: put more of this into EnvironmentRepo or Requirements env_repo = home.get_env_repo(args.repo) req_path = os.path.join(env_repo.work_tree, 'requirements.txt') reqs = Requirements(req_path, home=home) for req in reqs.iter_packages(): if req.name.lower() == name.lower(): # Make sure it is a Git package. url = normalize_git_url(req.url, prefix=False) if url: break else: print style_error('Could not find git-based "%s" in "%s" repo.' % (name, env_repo.name)) return 2 print style_note('Found %s in %s' % (name, env_repo.name), str(req)) makedirs(dev_repo.work_tree) dev_repo.clone_if_not_exists(url, shallow=False) elif do_add: print style_note('Adding %s from %s' % (name, path)) if not os.path.exists(path): log.error('%s does not exist' % path) return 1 package = Package([path], home=home, dev=True) try: package.pipeline.run_to('develop') except Exception as e: print_cli_exc(e) return 1 print style_note('Linking dev package', name, path) con.execute( 'INSERT INTO development_packages (name, path, environ) VALUES (?, ?, ?)', [name, path, json.dumps(package.environ)]) dev_pkg = DevPackage( { 'name': name, 'path': path, 'environ': package.environ }, home=home) dev_pkg.save_tag()
def init(self): pkg = self.package pkg.url = normalize_git_url(pkg.url, prefix=True) or pkg.url pkg._assert_paths(package=True) self.repo = GitRepo(work_tree=pkg.package_path, remote_url=re.sub(r'^git[:\+]', '', pkg.url))
def status(args): home = args.assert_home() env_repo = home.get_env_repo(args.repo) pkg_set = PackageSet(home=home) by_name = {} # Dev packages. for row in home.db.execute('SELECT * FROM development_packages'): row = dict(row) if not os.path.exists(row['path']): continue dev_repo = GitRepo(row['path']) row['remotes'] = dev_repo.remotes() by_name.setdefault(row['name'], {})['dev'] = row # Current requirements. for revision, name in [ (None, 'work'), ('HEAD', 'head'), ]: for req in env_repo.load_requirements( revision=revision).iter_packages(): pkg = pkg_set.resolve(req, check_existing=False) if pkg.fetch_type != 'git': continue by_name.setdefault(pkg.name, {})[name] = req by_name = by_name.items() by_name.sort(key=lambda x: x[0].lower()) if args.names: by_name = [x for x in by_name if x[0] in args.names] for name, everything in by_name: dev_row = everything.get('dev') work_req = everything.get('work') head_req = everything.get('head') has_dev = dev_row is not None only_has_dev = has_dev and not (work_req or head_req) # Skip dev-only stuff most of the time. if only_has_dev and not args.all_dev: continue # Title. print '%s %s' % (style( '%s %s' % ('==>' if has_dev else '-->', name), fg='blue'), '(dev only)' if only_has_dev else '') # Status of requirements. if work_req and head_req and str(work_req) == str(head_req): if args.verbose: print '=== %s' % work_req else: # Print a lovely coloured diff of the specific arguments that # are changing. # TODO: make this environment relative to the context. head_args = head_req.to_args( exclude=('base_environ', )) if head_req else [] work_args = work_req.to_args( exclude=('base_environ', )) if work_req else [] differ = difflib.SequenceMatcher(None, head_args, work_args) opcodes = differ.get_opcodes() if head_req is not None: print style('---', fg='red', bold=True), for tag, i1, i2, j1, j2 in opcodes: if tag in ('replace', 'delete'): print style(' '.join(head_args[i1:i2]), fg='red', bold=True) elif tag in ('equal', ): print ' '.join(head_args[i1:i2]), if work_req is not None: print style('+++', fg='green', bold=True), for tag, i1, i2, j1, j2 in opcodes: if tag in ('replace', 'insert'): print style(' '.join(work_args[j1:j2]), fg='green', bold=True) elif tag in ('equal', ): print ' '.join(work_args[j1:j2]), if dev_row: if 'warning' in dev_row: print dev_row['warning'] if 'origin' in dev_row['remotes']: dev_row['remote_name'] = 'origin' else: remote_names = sorted(dev_row['remotes']) dev_row['remote_name'] = remote_names[0] if len(remote_names) != 1: print ' ' + style_warning( 'More that one non-origin remote; picking %s' % dev_row['remote_name']) dev_repo = dev_row and GitRepo(dev_row['path']) if dev_repo and not dev_repo.exists: print style_warning('Git repo does not exist.') dev_row = dev_repo = None if dev_repo: if dev_repo.status(): print ' ' + style_warning('Work tree is dirty.') if args.fetch: dev_remote_head = dev_repo.fetch(dev_row['remote_name'], 'master') else: dev_remote_head = dev_repo.rev_parse(dev_row['remote_name'] + '/master') # Check your local dev vs. its remote. dev_local, dev_remote = dev_repo.distance(dev_repo.head, dev_remote_head) summarize_rev_distance( dev_local, dev_remote, local_name=name, local_verb='is', remote_name='%s/master' % dev_row['remote_name'], behind_action='please pull or `vee dev ff %s`' % name, ) if dev_repo and work_req and work_req.revision: # Check your local dev vs the required revision try: pkg_revision = dev_repo.rev_parse(work_req.revision) pkg_local, pkg_remote = dev_repo.distance( dev_repo.head, pkg_revision) summarize_rev_distance( pkg_local, pkg_remote, local_name=name, local_verb='is', remote_name='%s repo' % env_repo.name, ahead_action='you may `vee add %s`' % name, behind_action='please `vee dev checkout --repo %s %s`' % (env_repo.name, name), ) except Exception as e: print ' ' + format_cli_exc(e)