def __init__(self, store, remote, stdin=sys.stdin, stdout=sys.stdout): self._store = store self._repo = get_repo(remote) if isinstance(self._repo, bundlerepo): self._repo.init(self._store) self._remote = remote self._helper = IOLogger(logging.getLogger('remote-helper'), stdin, stdout) self._dry_run = False self._branchmap = None self._bookmarks = {} self._HEAD = 'branches/default/tip' self._has_unknown_heads = False GRAFT = { None: False, 'false': False, 'true': True, } try: self._graft = Git.config('cinnabar.graft', remote=remote.name, values=GRAFT) except InvalidConfig as e: logging.error(e.message) return 1 if Git.config('cinnabar.graft-refs') is not None: logging.warn( 'The cinnabar.graft-refs configuration is deprecated.\n' 'Please unset it.' )
def reclone(args): '''reclone all mercurial remotes''' from cinnabar.cmd.rollback import do_rollback git_config = {} metadata_commit = Git.resolve_ref('refs/cinnabar/metadata') if metadata_commit: git_config['cinnabar.previous-metadata'] = \ metadata_commit.decode('ascii') # TODO: Avoid resetting at all, possibly leaving the repo with no metadata # if this is interrupted somehow. do_rollback(NULL_NODE_ID.decode('ascii')) for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'): config, url = line.split() name = config[len('remote.'):-len('.url')] skip_pref = 'remote.%s.skipDefaultUpdate' % name.decode('ascii') if (url.startswith((b'hg::', b'hg://')) and Git.config(skip_pref) != 'true'): Git.run('remote', 'update', '--prune', fsdecode(name), config=git_config) git_config = {} print('Please note that reclone left your local branches untouched.') print('They may be based on entirely different commits.')
def __init__(self, store, remote, stdin=sys.stdin, stdout=sys.stdout): self._store = store self._repo = get_repo(remote) if isinstance(self._repo, bundlerepo): self._repo.init(self._store) self._remote = remote self._helper = IOLogger(logging.getLogger('remote-helper'), stdin, stdout) self._branchmap = None self._bookmarks = {} self._HEAD = 'branches/default/tip' self._has_unknown_heads = False GRAFT = { None: False, 'false': False, 'true': True, } try: self._graft = Git.config('cinnabar.graft', remote=remote.name, values=GRAFT) except InvalidConfig as e: logging.error(e.message) return 1 if Git.config('cinnabar.graft-refs') is not None: logging.warn( 'The cinnabar.graft-refs configuration is deprecated.\n' 'Please unset it.' )
def __init__(self, store, remote, stdin=bytes_stdin, stdout=bytes_stdout): super(GitRemoteHelper, self).__init__(stdin, stdout) self._store = store self._repo = get_repo(remote) if isinstance(self._repo, bundlerepo): self._repo.init(self._store) self._remote = remote self._head_template = None self._tip_template = None self._bookmark_template = None self._branchmap = None self._bookmarks = {} self._has_unknown_heads = False GRAFT = { None: False, b'false': False, b'true': True, } try: self._graft = Git.config('cinnabar.graft', remote=remote.name, values=GRAFT) except InvalidConfig as e: logging.error(str(e)) return 1 if Git.config('cinnabar.graft-refs') is not None: logging.warn( 'The cinnabar.graft-refs configuration is deprecated.\n' 'Please unset it.' )
def setUp(self): self.git_dir = os.environ.get('GIT_DIR') tmpdir = tempfile.mkdtemp() Git.run('init', '--bare', tmpdir, stdout=open(os.devnull, 'w')) os.environ['GIT_DIR'] = tmpdir os.environ['GIT_CINNABAR_EXPERIMENTS'] = \ 'store' if self.NEW_STORE else '' self.assertEquals( GitHgHelper.supports((b'store', b'new')), self.NEW_STORE)
def do_rollback(ref): checked = Git.resolve_ref('refs/cinnabar/checked') if ref: sha1 = Git.resolve_ref(ref) if not sha1: logging.error('Invalid ref: %s', ref) return 1 if sha1 != NULL_NODE_ID: # Validate that the sha1 is in the history of the current metadata metadata = Git.resolve_ref('refs/cinnabar/metadata') while metadata and metadata != sha1: previous_metadata = get_previous_metadata(metadata) if checked == metadata: checked = previous_metadata metadata = previous_metadata if not metadata: logging.error('Cannot rollback to %s, it is not in the ' 'history of the current metadata.', ref) return 1 else: metadata = Git.resolve_ref('refs/cinnabar/metadata') if metadata: sha1 = get_previous_metadata(metadata) or NULL_NODE_ID else: sha1 = NULL_NODE_ID if checked and checked == metadata: checked = sha1 refs = VersionedDict( (ref, commit) for commit, ref in Git.for_each_ref('refs/cinnabar', 'refs/notes/cinnabar') ) for ref in refs: if sha1 == NULL_NODE_ID or ref not in (b'refs/cinnabar/checked', b'refs/cinnabar/broken'): del refs[ref] if sha1 != NULL_NODE_ID: refs[b'refs/cinnabar/metadata'] = sha1 if checked: refs[b'refs/cinnabar/checked'] = checked for line in Git.ls_tree(sha1): mode, typ, commit, path = line refs[b'refs/cinnabar/replace/%s' % path] = commit for status, ref, commit in refs.iterchanges(): if status == VersionedDict.REMOVED: Git.delete_ref(ref) else: Git.update_ref(ref, commit) GitHgHelper.close(rollback=False) return 0
def run(self): from cinnabar import VERSION from cinnabar.git import Git, GitProcess from distutils.version import StrictVersion parent_dir = os.path.dirname(os.path.dirname(__file__)) if not os.path.exists(os.path.join(parent_dir, '.git')) or \ check_enabled('no-version-check') or \ not interval_expired('version-check', 86400, globl=True): return REPO = 'https://github.com/glandium/git-cinnabar' devnull = open(os.devnull, 'wb') if VERSION.endswith('a'): _, _, extra = StrictVersion(VERSION[:-1]).version ref = 'refs/heads/next' if extra == 0 else 'refs/heads/master' for line in Git.iter('ls-remote', REPO, ref, stderr=devnull): sha1, head = line.split() if head != ref: continue proc = GitProcess('-C', parent_dir, 'merge-base', '--is-ancestor', sha1, 'HEAD', stdout=devnull, stderr=devnull) if proc.wait() != 0: self.message = ( 'The `{}` branch of git-cinnabar was updated. ' 'Please update your copy.\n' 'You can switch to the `release` branch if you want ' 'to reduce these update notifications.'.format( ref.partition('refs/heads/')[-1])) break else: version = StrictVersion(VERSION) newer_version = version for line in Git.iter('ls-remote', REPO, 'refs/tags/*', stderr=devnull): sha1, tag = line.split() tag = tag.partition('refs/tags/')[-1] try: v = StrictVersion(tag) except ValueError: continue if v > newer_version: newer_version = v if newer_version != version: self.message = ('New git-cinnabar version available: {} ' '(current version: {})'.format( newer_version, version))
def do_rollback(ref): checked = Git.resolve_ref('refs/cinnabar/checked') if ref: sha1 = Git.resolve_ref(ref) if not sha1: logging.error('Invalid ref: %s', ref) return 1 if sha1 != NULL_NODE_ID: # Validate that the sha1 is in the history of the current metadata metadata = Git.resolve_ref('refs/cinnabar/metadata') while metadata and metadata != sha1: previous_metadata = get_previous_metadata(metadata) if checked == metadata: checked = previous_metadata metadata = previous_metadata if not metadata: logging.error('Cannot rollback to %s, it is not in the ' 'history of the current metadata.', ref) return 1 else: metadata = Git.resolve_ref('refs/cinnabar/metadata') if metadata: sha1 = get_previous_metadata(metadata) or NULL_NODE_ID else: sha1 = NULL_NODE_ID if checked and checked == metadata: checked = sha1 refs = VersionedDict( (ref, commit) for commit, ref in Git.for_each_ref('refs/cinnabar', 'refs/notes/cinnabar') ) for ref in refs: if ref not in ('refs/cinnabar/checked', 'refs/cinnabar/broken'): del refs[ref] if sha1 != NULL_NODE_ID: refs['refs/cinnabar/metadata'] = sha1 if checked: refs['refs/cinnabar/checked'] = checked for line in Git.ls_tree(sha1): mode, typ, commit, path = line refs['refs/cinnabar/replace/%s' % path] = commit for status, ref, commit in refs.iterchanges(): if status == VersionedDict.REMOVED: Git.delete_ref(ref) else: Git.update_ref(ref, commit) GitHgHelper.close(rollback=False) return 0
def reclone(args): '''reclone all mercurial remotes''' from cinnabar.cmd.rollback import do_rollback do_rollback(NULL_NODE_ID) for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'): config, url = line.split() name = config[len('remote.'):-len('.url')] skip_pref = 'remote.%s.skipDefaultUpdate' % name if (url.startswith(('hg::', 'hg://')) and Git.config(skip_pref) != 'true'): Git.run('remote', 'update', '--prune', name) print 'Please note that reclone left your local branches untouched.' print 'They may be based on entirely different commits.'
def manifest_diff(a, b, base_path=''): base_path = base_path.rstrip('/') start = len(base_path) + bool(base_path) for line in Git.diff_tree(a, b, base_path, recursive=True): mode_before, mode_after, sha1_before, sha1_after, status, path = line if sha1_before != sha1_after: yield path[start:], sha1_after, sha1_before
def get_checked_metadata(num): if not checked_metadata: return None commit = Git.resolve_ref('{}^{}'.format( checked_metadata.decode('ascii'), num)) if commit: return GitCommit(commit)
def getbundle(repo, store, heads, branch_names): if isinstance(repo, bundlerepo): bundle = repo._unbundler else: common = findcommon(repo, store, store.heads(branch_names)) logging.info('common: %s', common) bundle = None got_partial = False if not common: if not store._has_metadata and not store._graft: manifest = Git.config('cinnabar.clone') if manifest is None and repo.capable('cinnabarclone'): manifest = repo._call('cinnabarclone') if manifest: got_partial = do_cinnabarclone(repo, manifest, store) if not got_partial: if check_enabled('cinnabarclone'): raise Exception('cinnabarclone failed.') logging.warn('Falling back to normal clone.') if not got_partial and repo.capable('clonebundles'): bundle = get_clonebundle(repo) got_partial = bool(bundle) if not got_partial and check_enabled('clonebundles'): raise Exception('clonebundles failed.') if bundle: bundle = unbundler(bundle) # Manual move semantics apply_bundle = BundleApplier(bundle) del bundle apply_bundle(store) if not changegroup: BundleHelper.close() if got_partial: # Eliminate the heads that we got from the clonebundle or # cinnabarclone. heads = [h for h in heads if not store.changeset_ref(h)] if not heads: return common = findcommon(repo, store, store.heads(branch_names)) logging.info('common: %s', common) kwargs = {} if unbundle20 and repo.capable('bundle2'): bundle2caps = { 'HG20': (), 'changegroup': ('01', '02'), } kwargs['bundlecaps'] = set(( 'HG20', 'bundle2=%s' % urllib.quote(encodecaps(bundle2caps)))) bundle = repo.getbundle('bundle', heads=[unhexlify(h) for h in heads], common=[unhexlify(h) for h in common], **kwargs) bundle = unbundler(bundle) # Manual move semantics apply_bundle = BundleApplier(bundle) del bundle apply_bundle(store)
def get_repo(remote): if remote.parsed_url.scheme == 'file': path = remote.parsed_url.path if sys.platform == 'win32': # TODO: This probably needs more thought. path = path.lstrip('/') if not os.path.isdir(path): return bundlerepo(path) if not changegroup or Git.config('cinnabar.experiments') == 'true': if not changegroup: logging.warning('Mercurial libraries not found. Falling back to ' 'native access.') logging.warning( 'Native access to mercurial repositories is experimental!') try: return HelperRepo(remote.url) except NoHelperException: raise Exception('Native access to mercurial repositories requires ' 'the helper.') if changegroup and remote.parsed_url.scheme == 'file': repo = localpeer(get_ui(), path) else: repo = hg.peer(get_ui(), {}, remote.url) assert repo.capable('getbundle') return repo
def manifest_diff(a, b, base_path=''): base_path = base_path.rstrip('/') start = len(base_path) + bool(base_path) for line in Git.diff_tree(a, b, base_path): mode_before, mode_after, sha1_before, sha1_after, status, path = line if sha1_before != sha1_after: yield path[start:], sha1_after, sha1_before
def bundle(args): '''create a mercurial bundle''' bundle_commits = list((c, p) for c, t, p in GitHgHelper.rev_list( '--topo-order', '--full-history', '--parents', '--reverse', *args.rev)) if bundle_commits: # TODO: better UX. For instance, this will fail with an exception when # the parent commit doesn't have mercurial metadata. GRAFT = { None: False, 'false': False, 'true': True, } try: graft = Git.config('cinnabar.graft', values=GRAFT) except InvalidConfig as e: logging.error(e.message) return 1 store = PushStore(graft=graft) if args.version == 1: b2caps = {} elif args.version == 2: b2caps = { 'HG20': (), 'changegroup': ('01', '02'), } with open(args.path, 'wb') as fh: if not b2caps: fh.write('HG10UN') for data in create_bundle(store, bundle_commits, b2caps): fh.write(data) store.close(rollback=True)
def fetch(args): '''fetch a changeset from a mercurial remote''' remote = args.remote revs = args.revs full_revs = [] for rev in revs: if not re.match('[0-9a-f]{40}$', rev.lower()): if remote.startswith('hg:'): url = remote else: url = Git.config('remote.%s.url' % remote) if not url: print >>sys.stderr, "Unknown remote:", remote return 1 if url.startswith('hg::'): url = url[4:] repo = get_repo(Remote(remote, url)) if repo.capable('lookup'): rev = hexlify(repo.lookup(rev)) else: print >>sys.stderr, ( 'Remote repository does not support the "lookup" command. ' 'Please use a non-abbreviated mercurial revision.') return 1 full_revs.append(rev) refs = ['hg/revs/%s' % r for r in full_revs] proc = GitProcess('fetch', remote, *refs, stdout=sys.stdout, config={'cinnabar.fetch': ' '.join(full_revs)}) return proc.wait()
def fetch(args): '''fetch a changeset from a mercurial remote''' remote = args.remote revs = args.revs full_revs = [] for rev in revs: if not re.match('[0-9a-f]{40}$', rev.lower()): if remote.startswith('hg:'): url = remote else: url = Git.config('remote.%s.url' % remote) if not url: print >> sys.stderr, "Unknown remote:", remote return 1 if url.startswith('hg::'): url = url[4:] repo = get_repo(Remote(remote, url)) if repo.capable('lookup'): rev = hexlify(repo.lookup(rev)) else: print >> sys.stderr, ( 'Remote repository does not support the "lookup" command. ' 'Please use a non-abbreviated mercurial revision.') return 1 full_revs.append(rev) refs = ['hg/revs/%s' % r for r in full_revs] proc = GitProcess('fetch', remote, *refs, stdout=sys.stdout, config={'cinnabar.fetch': ' '.join(full_revs)}) return proc.wait()
def test_module_version(self): module = one( Git.iter('ls-tree', 'HEAD', 'cinnabar', cwd=os.path.join(os.path.dirname(__file__), '..'))) self.assertEqual(CmdVersion.module_version(), split_ls_tree(module)[2])
def test_helper_version(self): helper = one( Git.iter('ls-tree', 'HEAD', 'helper', cwd=os.path.join(os.path.dirname(__file__), '..'))) self.assertEqual(CmdVersion.helper_version()[1], split_ls_tree(helper)[2])
def test_cinnabar_version(self): desc = one(Git.iter('describe', '--tags', 'HEAD')) version = Version(CmdVersion.cinnabar_version()) if '-' in desc: last_tag, n, sha1 = desc.rsplit('-', 2) self.assertGreater(version, Version(last_tag)) else: self.assertEqual(version, Version(desc))
def run(self): from cinnabar import VERSION from cinnabar.git import Git, GitProcess from distutils.version import StrictVersion parent_dir = os.path.dirname(os.path.dirname(__file__)) if not os.path.exists(os.path.join(parent_dir, '.git')) or \ check_enabled('no-version-check') or \ not interval_expired('version-check', 86400): return REPO = 'https://github.com/glandium/git-cinnabar' devnull = open(os.devnull, 'wb') if VERSION.endswith('a'): _, _, extra = StrictVersion(VERSION[:-1]).version ref = 'refs/heads/next' if extra == 0 else 'refs/heads/master' for line in Git.iter('ls-remote', REPO, ref, stderr=devnull): sha1, head = line.split() if head != ref: continue proc = GitProcess( '-C', parent_dir, 'merge-base', '--is-ancestor', sha1, 'HEAD', stdout=devnull, stderr=devnull) if proc.wait() != 0: self.message = ( 'The `{}` branch of git-cinnabar was updated. ' 'Please update your copy.\n' 'You can switch to the `release` branch if you want ' 'to reduce these update notifications.' .format(ref.partition('refs/heads/')[-1])) break else: version = StrictVersion(VERSION) newer_version = version for line in Git.iter('ls-remote', REPO, 'refs/tags/*', stderr=devnull): sha1, tag = line.split() tag = tag.partition('refs/tags/')[-1] try: v = StrictVersion(tag) except ValueError: continue if v > newer_version: newer_version = v if newer_version != version: self.message = ( 'New version available: {} (current version: {})' .format(newer_version, version))
def do_cinnabarclone(repo, manifest, store): url = None for line in manifest.splitlines(): line = line.strip() spec, _, params = line.partition(' ') params = { k: v for k, _, v in (p.partition('=') for p in params.split()) } graft = params.pop('graft', None) if params: # Future proofing: ignore lines with unknown params, even if we # support some that are present. continue if store._graft: # When grafting, ignore lines without a graft revision. if not graft: continue graft = graft.split(',') revs = list(Git.iter('rev-parse', '--revs-only', *graft)) if len(revs) != len(graft): continue # We apparently have all the grafted revisions locally, ensure # they're actually reachable. if not any(Git.iter( 'rev-list', '--branches', '--tags', '--remotes', '--max-count=1', '--ancestry-path', '--stdin', stdin=('^{}^@'.format(c) for c in graft))): continue url, _, branch = spec.partition('#') url, branch = (url.split('#', 1) + [None])[:2] if url: break if not url: logging.warn('Server advertizes cinnabarclone but didn\'t provide ' 'a git repository url to fetch from.') return False parsed_url = urlparse(url) if parsed_url.scheme not in ('http', 'https', 'git'): logging.warn('Server advertizes cinnabarclone but provided a non ' 'http/https git repository. Skipping.') return False sys.stderr.write('Fetching cinnabar metadata from %s\n' % url) return store.merge(url, repo.url(), branch)
def old_helper_head(): from cinnabar.git import Git from cinnabar.helper import GitHgHelper version = GitHgHelper.VERSION return list(Git.iter( 'log', 'HEAD', '--format=%H', '--pickaxe-regex', '-S', '#define CMD_VERSION {}'.format(version), cwd=os.path.join(os.path.dirname(__file__), '..')))[-1]
def old_helper_hash(head): from cinnabar.git import Git, split_ls_tree from cinnabar.util import one return split_ls_tree( one( Git.iter('ls-tree', head, 'helper', cwd=os.path.join(os.path.dirname(__file__), '..'))))[2]
def run(func, args): reexec = None assert not experiment('python3') or sys.version_info[0] != 2 if os.environ.pop('GIT_CINNABAR_COVERAGE', None): if not reexec: reexec = [sys.executable] reexec.extend(['-m', 'coverage', 'run', '--append']) init_logging() if reexec: reexec.append(os.path.abspath(sys.argv[0])) reexec.extend(sys.argv[1:]) os.execlp(reexec[0], *reexec) assert False if check_enabled('memory') or check_enabled('cpu'): reporter = MemoryCPUReporter(memory=check_enabled('memory'), cpu=check_enabled('cpu')) version_check = VersionCheck() try: from cinnabar.git import Git objectformat = Git.config('extensions.objectformat') or 'sha1' if objectformat != 'sha1': sys.stderr.write( 'Git repository uses unsupported %s object format\n' % objectformat) retcode = 65 # Data format error else: retcode = func(args) except Abort as e: # These exceptions are normal abort and require no traceback retcode = 1 logging.error(str(e)) except Exception as e: # Catch all exceptions and provide a nice message retcode = 70 # Internal software error message = getattr(e, 'message', None) or getattr(e, 'reason', None) if check_enabled('traceback') or not message: traceback.print_exc() else: logging.error(message) sys.stderr.write( 'Run the command again with ' '`git -c cinnabar.check=traceback <command>` to see the ' 'full traceback.\n') finally: if check_enabled('memory') or check_enabled('cpu'): reporter.shutdown() version_check.join() if check_enabled('no-mercurial'): if any( k.startswith('mercurial.') or k == 'mercurial' for k in sys.modules): sys.stderr.write('Mercurial libraries were loaded!') retcode = 70 sys.exit(retcode)
def list(self, arg=None): tags = sorted(self._store.tags()) # git fetch does a check-connection that calls # `git rev-list --objects --stdin --not --all` with the list of # sha1s from the list we're about to give it. With no refs on these # exact sha1s, the rev-list can take a long time on large repos. # So we temporarily create refs to make that rev-list faster. for tag, ref in tags: Git.update_ref(b'refs/cinnabar/refs/tags/' + tag, ref) GitHgHelper.reload() for tag, ref in tags: self._helper.write(b'%s refs/tags/%s\n' % (ref, tag)) self._helper.write(b'\n') self._helper.flush() # Now remove the refs. The deletion will only actually be committed # on the store close in main(), after git is done doing # check-connection. for tag, _ in tags: Git.delete_ref(b'refs/cinnabar/refs/tags/' + tag)
def do_rollback(ref): sha1 = Git.resolve_ref(ref) if not sha1: logging.error('Invalid ref: %s', ref) return 1 if sha1 != NULL_NODE_ID: # Validate that the sha1 is in the history of the current metadata metadata = Git.resolve_ref('refs/cinnabar/metadata') while metadata: if sha1 == metadata: break commit = GitCommit(metadata) flags = commit.body.split(' ') if len(commit.parents) == 5 + ('files-meta' in flags): metadata = commit.parents[-1] else: metadata = None if not metadata: logging.error( 'Cannot rollback to %s, it is not in the history of ' 'the current metadata.', ref) return 1 refs = VersionedDict((ref, commit) for commit, ref in Git.for_each_ref( 'refs/cinnabar', 'refs/notes/cinnabar')) for ref in refs: del refs[ref] if sha1 != NULL_NODE_ID: refs['refs/cinnabar/metadata'] = sha1 for line in Git.ls_tree(sha1): mode, typ, commit, path = line refs['refs/cinnabar/replace/%s' % path] = commit for status, ref, commit in refs.iterchanges(): if status == VersionedDict.REMOVED: Git.delete_ref(ref) else: Git.update_ref(ref, commit) Git._close_update_ref() return 0
def local_bases(): for c in Git.iter('rev-list', '--stdin', '--topo-order', '--full-history', '--boundary', *(w for w in what if w), stdin=heads()): if c[0] != '-': continue yield store.hg_changeset(c[1:]) for w in what: rev = store.hg_changeset(w) if rev: yield rev
def get_changes(tree, parents, all=False): if not parents: for line in Git.ls_tree(tree, recursive=True): mode, typ, sha1, path = line yield path, sha1, () elif len(parents) == 1: for path, node, parent in manifest_diff(parents[0], tree): yield path, node, (parent,) else: diff = manifest_diff2_all if all else manifest_diff2 for path, node, parents in diff(parents[0], parents[1], tree): yield path, node, parents
def get_changes(tree, parents, base_path=''): if not parents: for line in Git.ls_tree(tree, base_path, recursive=True): mode, typ, sha1, path = line yield path[len(base_path) + 1:] if base_path else path, sha1, () elif len(parents) == 1: for path, node, parent in manifest_diff(parents[0], tree, base_path): yield path, node, (parent,) else: for path, node, parents in manifest_diff2(parents[0], parents[1], tree, base_path): yield path, node, parents
def get_changes(tree, parents, all=False): if not parents: for line in Git.ls_tree(tree, recursive=True): mode, typ, sha1, path = line yield path, sha1, () elif len(parents) == 1: for path, node, parent in manifest_diff(parents[0], tree): yield path, node, (parent, ) else: diff = manifest_diff2_all if all else manifest_diff2 for path, node, parents in diff(parents[0], parents[1], tree): yield path, node, parents
def git2hg(args): '''convert git sha1 to corresponding mercurial sha1''' for sha1, ref in Git.for_each_ref('refs/cinnabar/replace'): Git._replace[ref[22:]] = sha1 for arg in args.sha1: data = GitHgHelper.git2hg(arg) if data: assert data.startswith('changeset ') print data[10:10 + args.abbrev] else: print NULL_NODE_ID[:args.abbrev]
def git2hg(args): '''convert git sha1 to corresponding mercurial sha1''' for sha1, ref in Git.for_each_ref('refs/cinnabar/replace'): Git._replace[ref[22:]] = sha1 for arg in args.sha1: data = GitHgHelper.git2hg(arg) if data: assert data.startswith('changeset ') print data[10:50] else: print NULL_NODE_ID
def get_changes(tree, parents, base_path=''): if not parents: for line in Git.ls_tree(tree, base_path, recursive=True): mode, typ, sha1, path = line yield path[3:], sha1, () elif len(parents) == 1: for path, node, parent in manifest_diff(parents[0], tree, base_path): yield path, node, (parent,) else: for path, node, parents in manifest_diff2(parents[0], parents[1], tree, base_path): yield path, node, parents
def rollback(args): '''rollback cinnabar metadata state''' if args.fsck and args.committish: logging.error('Cannot use --fsck along a commit.') return 1 if args.fsck: committish = Git.resolve_ref('refs/cinnabar/checked') if not committish: logging.error('No successful fsck has been recorded. ' 'Cannot rollback.') else: committish = args.committish return do_rollback(committish)
def main(args): cmd = args.pop(0) if cmd == 'data': store = GitHgStore() if args[0] == '-c': sys.stdout.write(store.changeset(args[1]).data) elif args[0] == '-m': sys.stdout.write(store.manifest(args[1]).data) store.close() elif cmd == 'fsck': return fsck(args) elif cmd == 'reclone': for ref in Git.for_each_ref('refs/cinnabar', 'refs/remote-hg', 'refs/notes/cinnabar', 'refs/notes/remote-hg/git2hg', format='%(refname)'): Git.delete_ref(ref) Git.close() for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'): config, url = line.split() name = config[len('remote.'):-len('.url')] skip_pref = 'remote.%s.skipDefaultUpdate' % name if (url.startswith('hg::') and Git.config(skip_pref, 'bool') != 'true'): Git.run('remote', 'update', '--prune', name) print 'Please note that reclone left your local branches untouched.' print 'They may be based on entirely different commits.' elif cmd == 'hg2git': for arg in args: print GitHgHelper.hg2git(arg) elif cmd == 'git2hg': for arg in args: data = GitHgHelper.git2hg(arg) if data: data = ChangesetData.parse(data) print data.get('changeset', NULL_NODE_ID) else: print NULL_NODE_ID else: print >> sys.stderr, 'Unknown command:', cmd return 1
def reclone(args): '''reclone all mercurial remotes''' from cinnabar.cmd.rollback import do_rollback git_config = {} metadata_commit = Git.resolve_ref('refs/cinnabar/metadata') if metadata_commit: git_config['cinnabar.previous-metadata'] = metadata_commit # TODO: Avoid resetting at all, possibly leaving the repo with no metadata # if this is interrupted somehow. do_rollback(NULL_NODE_ID) for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'): config, url = line.split() name = config[len('remote.'):-len('.url')] skip_pref = 'remote.%s.skipDefaultUpdate' % name if (url.startswith(('hg::', 'hg://')) and Git.config(skip_pref) != 'true'): Git.run('remote', 'update', '--prune', name, config=git_config) git_config = {} print 'Please note that reclone left your local branches untouched.' print 'They may be based on entirely different commits.'
def main(args): cmd = args.pop(0) if cmd == 'data': store = GitHgStore() if args[0] == '-c': sys.stdout.write(store.changeset(args[1]).data) elif args[0] == '-m': sys.stdout.write(store.manifest(args[1]).data) store.close() elif cmd == 'fsck': return fsck(args) elif cmd == 'reclone': for ref in Git.for_each_ref('refs/cinnabar', 'refs/remote-hg', 'refs/notes/cinnabar', 'refs/notes/remote-hg/git2hg', format='%(refname)'): Git.delete_ref(ref) Git.close() for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'): config, url = line.split() name = config[len('remote.'):-len('.url')] skip_pref = 'remote.%s.skipDefaultUpdate' % name if (url.startswith('hg::') and Git.config(skip_pref, 'bool') != 'true'): Git.run('remote', 'update', '--prune', name) print 'Please note that reclone left your local branches untouched.' print 'They may be based on entirely different commits.' elif cmd == 'hg2git': for arg in args: print GitHgHelper.hg2git(arg) elif cmd == 'git2hg': for arg in args: data = GitHgHelper.git2hg(arg) if data: data = ChangesetData.parse(data) print data.get('changeset', NULL_NODE_ID) else: print NULL_NODE_ID else: print >>sys.stderr, 'Unknown command:', cmd return 1
def find_user_password(self, realm, authuri): try: return url_passwordmgr.find_user_password(self, realm, authuri) except error.Abort: # Assume error.Abort is only thrown from the base class's # find_user_password itself, which reflects that authentication # information is missing and mercurial would want to get it # from user input, but can't because the ui isn't interactive. credentials = dict( line.split(b'=', 1) for line in Git.iter( 'credential', 'fill', stdin=b'url=%s' % authuri)) username = credentials.get(b'username') password = credentials.get(b'password') if not username or not password: raise return username, password
def get_clonebundle(repo): url = Git.config('cinnabar.clonebundle', remote=repo.remote) if not url: url = get_clonebundle_url(repo) if not url: return None parsed_url = urlparse(url) if parsed_url.scheme not in (b'http', b'https'): logging.warn('Server advertizes clone bundle but provided a non ' 'http/https url. Skipping.') return None sys.stderr.write('Getting clone bundle from %s\n' % fsdecode(url)) return get_bundle(url)
def old_helper_head(): from cinnabar import VERSION version = VERSION if version.endswith('a'): from distutils.version import StrictVersion v = StrictVersion(VERSION[:-1]).version + (0, 0, 0) if v[2] == 0: from cinnabar.git import Git from cinnabar.helper import GitHgHelper version = GitHgHelper.VERSION return list(Git.iter( 'log', 'HEAD', '--format=%H', '-S', '#define CMD_VERSION {}'.format(version), cwd=os.path.join(os.path.dirname(__file__), '..')))[-1] version = '{}.{}.{}'.format(v[0], v[1], v[2] - 1) return version
def old_compatible_python(): '''Find the oldest version of the python code that is compatible with the current helper''' from cinnabar.git import Git with open(os.path.join(os.path.dirname(__file__), '..', 'helper', 'cinnabar-helper.c')) as fh: min_version = None for l in fh: if l.startswith('#define MIN_CMD_VERSION'): min_version = l.rstrip().split()[-1][:2] break if not min_version: raise Exception('Cannot find MIN_CMD_VERSION') return list(Git.iter( 'log', 'HEAD', '--format=%H', '-S', 'class GitHgHelper(BaseHelper):\n VERSION = {}'.format(min_version), cwd=os.path.join(os.path.dirname(__file__), '..')))[-1]
def old_compatible_python(): '''Find the oldest version of the python code that is compatible with the current helper''' from cinnabar.git import Git with open(os.path.join(os.path.dirname(__file__), '..', 'helper', 'cinnabar-helper.c')) as fh: min_version = None for l in fh: if l.startswith('#define MIN_CMD_VERSION'): min_version = l.rstrip().split()[-1][:2] break if not min_version: raise Exception('Cannot find MIN_CMD_VERSION') return list(Git.iter( 'log', 'HEAD', '--format=%H', '-S', 'class GitHgHelper(BaseHelper):\n VERSION = {}'.format(min_version), cwd=os.path.join(os.path.dirname(__file__), '..')))[-1].decode()
def old_helper_head(): from cinnabar import VERSION from distutils.version import StrictVersion version = VERSION if version.endswith('a'): v = StrictVersion(VERSION[:-1]).version if v[2] == 0: from cinnabar.git import Git from cinnabar.helper import GitHgHelper version = GitHgHelper.VERSION return list(Git.iter( 'log', 'HEAD', '--format=%H', '-S', '#define CMD_VERSION {}'.format(version), cwd=os.path.join(os.path.dirname(__file__), '..')))[-1].decode() else: v = StrictVersion(VERSION).version return '{}.{}.{}'.format(v[0], v[1], max(v[2] - 1, 0))
def find_user_password(self, realm, authuri): try: return url_passwordmgr.find_user_password(self, realm, authuri) except error.Abort: # Assume error.Abort is only thrown from the base class's # find_user_password itself, which reflects that authentication # information is missing and mercurial would want to get it # from user input, but can't because the ui isn't interactive. credentials = dict( line.split('=', 1) for line in Git.iter('credential', 'fill', stdin='url=%s' % authuri) ) username = credentials.get('username') password = credentials.get('password') if not username or not password: raise return username, password
def close(self, rollback=False): if rollback: self._closed = True if self._closed: return for manifest in self._push_manifests.itervalues(): self.store_manifest(manifest) ls = one(Git.ls_tree(self.manifest_ref(manifest.node), 'git')) if self._manifest_git_tree[manifest.node] == EMPTY_TREE and not ls: pass else: mode, typ, sha1, path = ls assert sha1 == self._manifest_git_tree[manifest.node] for file in self._push_files.itervalues(): if isinstance(self._files[file.node], Mark): mark = self._fast_import.new_mark() self._fast_import.put_blob(data=file.data, mark=mark) self._files[file.node] = Mark(mark) super(PushStore, self).close()
def get_clonebundle(repo): url = Git.config('cinnabar.clonebundle') if not url: url = get_clonebundle_url(repo) if not url: return None parsed_url = urlparse(url) if parsed_url.scheme not in ('http', 'https'): logging.warn('Server advertizes clone bundle but provided a non ' 'http/https url. Skipping.') return None sys.stderr.write('Getting clone bundle from %s\n' % url) reader = None if not changegroup: reader = BundleHelper.connect(url) if not reader: BundleHelper.close() if not reader: reader = HTTPReader(url) return unbundle_fh(reader, url)
def fsck(args): parser = argparse.ArgumentParser() parser.add_argument( '--manifests', action='store_true', help='Validate manifests hashes') parser.add_argument( '--files', action='store_true', help='Validate files hashes') parser.add_argument( 'commit', nargs='*', help='Specific commit or changeset to check') args = parser.parse_args(args) status = { 'broken': False, 'fixed': False, } def info(message): sys.stderr.write('\r') print message def fix(message): status['fixed'] = True info(message) def report(message): status['broken'] = True info(message) store = GitHgStore() store.init_fast_import(lambda: FastImport()) if args.commit: all_hg2git = {} all_notes = set() commits = set() all_git_commits = {} for c in args.commit: data = store.read_changeset_data(c) if data: all_notes.add(c) commits.add(c) c = data['changeset'] commit = GitHgHelper.hg2git(c) if commit == NULL_NODE_ID and not data: info('Unknown commit or changeset: %s' % c) return 1 if commit != NULL_NODE_ID: all_hg2git[c] = commit, 'commit' if not data: data = store.read_changeset_data(commit) commits.add(commit) if data: all_notes.add(commit) all_git_commits = Git.iter( 'log', '--no-walk=unsorted', '--stdin', '--format=%T %H', stdin=commits) else: all_hg2git = { path.replace('/', ''): (filesha1, intern(typ)) for mode, typ, filesha1, path in progress_iter('Reading %d mercurial to git mappings', Git.ls_tree('refs/cinnabar/hg2git', recursive=True)) } all_notes = set(path.replace('/', '') for mode, typ, filesha1, path in progress_iter( 'Reading %d commit to changeset mappings', Git.ls_tree('refs/notes/cinnabar', recursive=True))) manifest_commits = OrderedDict((m, None) for m in progress_iter( 'Reading %d manifest trees', Git.iter('rev-list', '--full-history', '--topo-order', 'refs/cinnabar/manifest')) ) all_git_heads = Git.for_each_ref('refs/cinnabar/branches', format='%(refname)') all_git_commits = Git.iter('log', '--topo-order', '--full-history', '--reverse', '--stdin', '--format=%T %H', stdin=all_git_heads) store._hg2git_cache = {p: s for p, (s, t) in all_hg2git.iteritems()} seen_changesets = set() seen_manifests = set() seen_manifest_refs = {} seen_files = set() seen_notes = set() hg_manifest = None dag = gitdag() for line in progress_iter('Checking %d changesets', all_git_commits): tree, node = line.split(' ') if node not in all_notes: report('Missing note for git commit: ' + node) continue seen_notes.add(node) changeset_data = store.read_changeset_data(node) changeset = changeset_data['changeset'] if 'extra' in changeset_data: extra = changeset_data['extra'] header, message = GitHgHelper.cat_file( 'commit', node).split('\n\n', 1) header = dict(l.split(' ', 1) for l in header.splitlines()) if 'committer' in extra: committer_info = store.hg_author_info(header['committer']) committer = '%s %d %d' % committer_info if (committer != extra['committer'] and header['committer'] != extra['committer'] and committer_info[0] != extra['committer']): report('Committer mismatch between commit and metadata for' ' changeset %s' % changeset) if committer == extra['committer']: fix('Fixing useless committer metadata for changeset %s' % changeset) del changeset_data['extra']['committer'] store._changesets[changeset] = LazyString(node) if header['committer'] != header['author'] and not extra: fix('Fixing useless empty extra metadata for changeset %s' % changeset) del changeset_data['extra'] store._changesets[changeset] = LazyString(node) seen_changesets.add(changeset) changeset_ref = store.changeset_ref(changeset) if not changeset_ref: report('Missing changeset in hg2git branch: %s' % changeset) elif str(changeset_ref) != node: report('Commit mismatch for changeset %s\n' ' hg2git: %s\n commit: %s' % (changeset, changeset_ref, node)) hg_changeset = store.changeset(changeset, include_parents=True) sha1 = hg_changeset.sha1 if hg_changeset.node != sha1: try_fixup = False if (changeset, sha1) in ( ('8c557b7c03a4a753e5c163038f04862e9f65fce1', '249b59139de8e08abeb6c4e261a137c756e7af0e'), ('ffdee4a4eb7fc7cae80dfc4cb2fe0c3178773dcf', '415e9d2eac83d508bf58a4df585c5f6b2b0f44ed'), ): header = hg_changeset.data.split('\n', 4) start = sum(len(h) for h in header[:3]) + 1 changeset_data['patch'] = ((start, start + 1, '1'),) try_fixup = True # Some know cases of corruptions involve a whitespace after the # timezone. Adding an empty extra metadata works around those. elif 'extra' not in changeset_data: changeset_data['extra'] = {} try_fixup = True if try_fixup: hg_changeset = store.changeset(changeset, include_parents=True) sha1 = hg_changeset.sha1 if hg_changeset.node == sha1: fix('Fixing known sha1 mismatch for changeset %s' % changeset) store._changesets[changeset] = LazyString(node) if hg_changeset.node != sha1: report('Sha1 mismatch for changeset %s' % changeset) dag.add(hg_changeset.node, (hg_changeset.parent1, hg_changeset.parent2), changeset_data.get('extra', {}).get('branch', 'default')) manifest = changeset_data['manifest'] if manifest in seen_manifests: continue seen_manifests.add(manifest) manifest_ref = store.manifest_ref(manifest) if manifest_ref: seen_manifest_refs[manifest_ref] = manifest if not manifest_ref: report('Missing manifest in hg2git branch: %s' % manifest) elif not args.commit and manifest_ref not in manifest_commits: report('Missing manifest commit in manifest branch: %s' % manifest_ref) if args.manifests or args.files: parents = tuple( store.read_changeset_data(store.changeset_ref(p))['manifest'] for p in (hg_changeset.parent1, hg_changeset.parent2) if p != NULL_NODE_ID ) if args.manifests: try: with GitHgHelper.query('check-manifest', manifest, *parents) as stdout: if stdout.readline().strip() != 'ok': report('Sha1 mismatch for manifest %s' % manifest) except NoHelperException: hg_manifest = store.manifest(manifest) hg_manifest.set_parents(*parents) if hg_manifest.node != hg_manifest.sha1: report('Sha1 mismatch for manifest %s' % manifest) git_ls = one(Git.ls_tree(manifest_ref, 'git')) if git_ls: mode, typ, sha1, path = git_ls else: header, message = GitHgHelper.cat_file( 'commit', manifest_ref).split('\n\n', 1) header = dict(l.split(' ', 1) for l in header.splitlines()) if header['tree'] == EMPTY_TREE: sha1 = EMPTY_TREE else: report('Missing git tree in manifest commit %s' % manifest_ref) sha1 = None if sha1 and sha1 != tree: report('Tree mismatch between manifest commit %s and commit %s' % (manifest_ref, node)) if args.files: changes = get_changes( manifest_ref, tuple(store.manifest_ref(p) for p in parents), 'hg') for path, hg_file, hg_fileparents in changes: if hg_file != NULL_NODE_ID and hg_file not in seen_files: file = store.file(hg_file) file.set_parents(*hg_fileparents) if file.node != file.sha1: report('Sha1 mismatch for file %s in manifest %s' % (hg_file, manifest_ref)) seen_files.add(hg_file) if args.files: all_hg2git = set(all_hg2git.iterkeys()) else: all_hg2git = set(k for k, (s, t) in all_hg2git.iteritems() if t == 'commit') adjusted = {} if not args.commit: dangling = set(manifest_commits) - set(seen_manifest_refs) if dangling: def iter_manifests(): removed_one = False yielded = False previous = None for obj in reversed(manifest_commits): if obj in dangling: fix('Removing metadata commit %s with no hg2git entry' % obj) removed_one = True else: if removed_one: yield obj, previous yielded = True previous = obj if removed_one and not yielded: yield obj, False for obj, parent in progress_iter('Adjusting %d metadata commits', iter_manifests()): mark = store._fast_import.new_mark() if parent is False: Git.update_ref('refs/cinnabar/manifest', obj) continue elif parent: parents = (adjusted.get(parent, parent),) with store._fast_import.commit( ref='refs/cinnabar/manifest', parents=parents, mark=mark) as commit: mode, typ, tree, path = store._fast_import.ls(obj) commit.filemodify('', tree, typ='tree') adjusted[obj] = Mark(mark) dangling = all_hg2git - seen_changesets - seen_manifests - seen_files if dangling or adjusted: with store._fast_import.commit( ref='refs/cinnabar/hg2git', parents=('refs/cinnabar/hg2git^0',)) as commit: for obj in dangling: fix('Removing dangling metadata for ' + obj) commit.filedelete(sha1path(obj)) for obj, mark in progress_iter( 'Updating hg2git for %d metadata commits', adjusted.iteritems()): commit.filemodify(sha1path(seen_manifest_refs[obj]), mark, typ='commit') dangling = all_notes - seen_notes if dangling: with store._fast_import.commit( ref='refs/notes/cinnabar', parents=('refs/notes/cinnabar^0',)) as commit: for c in dangling: fix('Removing dangling note for commit ' + c) # That's brute force, but meh. for l in range(0, 10): commit.filedelete(sha1path(c, l)) if status['broken']: info('Your git-cinnabar repository appears to be corrupted. There\n' 'are known issues in older revisions that have been fixed.\n' 'Please try running the following command to reset:\n' ' git cinnabar reclone\n\n' 'Please note this command may change the commit sha1s. Your\n' 'local branches will however stay untouched.\n' 'Please report any corruption that fsck would detect after a\n' 'reclone.') if not args.commit: info('Checking head references...') computed_heads = defaultdict(set) for branch, head in dag.all_heads(): computed_heads[branch].add(head) for branch in sorted(dag.tags()): stored_heads = store.heads({branch}) for head in computed_heads[branch] - stored_heads: fix('Adding missing head %s in branch %s' % (head, branch)) store.add_head(head) for head in stored_heads - computed_heads[branch]: fix('Removing non-head reference to %s in branch %s' % (head, branch)) store._hgheads.remove((branch, head)) store.close() if status['broken']: return 1 if status['fixed']: return 2 return 0
def fsck_quick(): status = FsckStatus() store = GitHgStore() metadata_commit = Git.resolve_ref('refs/cinnabar/metadata') if not metadata_commit: status.info( 'There does not seem to be any git-cinnabar metadata.\n' 'Is this a git-cinnabar clone?' ) return 1 commit = GitCommit(metadata_commit) if commit.body != 'files-meta unified-manifests-v2': status.info( 'The git-cinnabar metadata is incompatible with this version.\n' 'Please use the git-cinnabar version it was used with last.\n' ) return 1 if len(commit.parents) > 6 or len(commit.parents) < 5: status.report('The git-cinnabar metadata seems to be corrupted in ' 'unexpected ways.\n') return 1 changesets, manifests, hg2git, git2hg, files_meta = commit.parents[:5] commit = GitCommit(changesets) heads = OrderedDict( (node, branch) for node, _, branch in (d.partition(' ') for d in commit.body.splitlines())) if len(heads) != len(commit.parents): status.report('The git-cinnabar metadata seems to be corrupted in ' 'unexpected ways.\n') return 1 manifest_nodes = [] parents = None fix_changeset_heads = False # TODO: Check that the recorded heads are actually dag heads. for c, changeset_node in progress_iter( 'Checking {} changeset heads', izip(commit.parents, heads)): gitsha1 = GitHgHelper.hg2git(changeset_node) if gitsha1 == NULL_NODE_ID: status.report('Missing hg2git metadata for changeset %s' % changeset_node) continue if gitsha1 != c: if parents is None: parents = set(commit.parents) if gitsha1 not in parents: status.report( 'Inconsistent metadata:\n' ' Head metadata says changeset %s maps to %s\n' ' but hg2git metadata says it maps to %s' % (changeset_node, c, gitsha1)) continue fix_changeset_heads = True changeset = store._changeset(c, include_parents=True) if not changeset: status.report('Missing git2hg metadata for git commit %s' % c) continue if changeset.node != changeset_node: if changeset.node not in heads: status.report( 'Inconsistent metadata:\n' ' Head metadata says %s maps to changeset %s\n' ' but git2hg metadata says it maps to changeset %s' % (c, changeset_node, changeset.node)) continue fix_changeset_heads = True if changeset.node != changeset.sha1: status.report('Sha1 mismatch for changeset %s' % changeset.node) continue changeset_branch = changeset.branch or 'default' if heads[changeset.node] != changeset_branch: status.report( 'Inconsistent metadata:\n' ' Head metadata says changeset %s is in branch %s\n' ' but git2hg metadata says it is in branch %s' % (changeset.node, heads[changeset.node], changeset_branch)) continue manifest_nodes.append(changeset.manifest) if status('broken'): return 1 # Rebuilding manifests benefits from limiting the difference with # the last rebuilt manifest. Similarly, building the list of unique # files in all manifests benefits from that too. # Unfortunately, the manifest heads are not ordered in a topological # relevant matter, and the differences between two consecutive manifests # can be much larger than they could be. The consequence is spending a # large amount of time rebuilding the manifests and gathering the files # list. It's actually faster to attempt to reorder them according to # some heuristics first, such that the differences are smaller. # Here, we use the depth from the root node(s) to reorder the manifests. # This doesn't give the most optimal ordering, but it's already much # faster. On a clone of multiple mozilla-* repositories with > 1400 heads, # it's close to an order of magnitude difference on the "Checking # manifests" loop. depths = {} roots = [] manifest_queue = [] for m, _, parents in progress_iter( 'Loading {} manifests', GitHgHelper.rev_list( '--topo-order', '--reverse', '--full-history', '%s^@' % manifests)): manifest_queue.append((m, parents)) if parents: depth = {} for p in parents: for root, num in depths[p].iteritems(): if root in depth: depth[root] = max(depth[root], num + 1) else: depth[root] = num + 1 depths[m] = depth del depth else: depths[m] = {m: 0} roots.append(m) if status('broken'): return 1 # TODO: check that all manifest_nodes gathered above are available in the # manifests dag, and that the dag heads are the recorded heads. manifests_commit = GitCommit(manifests) depths = [ [depths[p].get(r, 0) for r in roots] for p in manifests_commit.parents ] manifests_commit_parents = [ p for _, p in sorted(zip(depths, manifests_commit.parents)) ] previous = None all_interesting = set() for m in progress_iter('Checking {} manifest heads', manifests_commit_parents): c = GitCommit(m) if not SHA1_RE.match(c.body): status.report('Invalid manifest metadata in git commit %s' % m) continue gitsha1 = GitHgHelper.hg2git(c.body) if gitsha1 == NULL_NODE_ID: status.report('Missing hg2git metadata for manifest %s' % c.body) continue if not GitHgHelper.check_manifest(c.body): status.report('Sha1 mismatch for manifest %s' % c.body) files = {} if previous: for _, _, before, after, d, path in GitHgHelper.diff_tree( previous, m): if d in 'AM' and before != after and \ (path, after) not in all_interesting: files[path] = after else: for _, t, sha1, path in GitHgHelper.ls_tree(m, recursive=True): if (path, sha1) not in all_interesting: files[path] = sha1 all_interesting.update(files.iteritems()) previous = m if status('broken'): return 1 progress = Progress('Checking {} files') while all_interesting and manifest_queue: (m, parents) = manifest_queue.pop() changes = get_changes(m, parents, all=True) for path, hg_file, hg_fileparents in changes: if hg_fileparents[1:] == (hg_file,): continue elif hg_fileparents[:1] == (hg_file,): continue # Reaching here means the file received a modification compared # to its parents. If it's a file we're going to check below, # it means we don't need to check its parents if somehow they were # going to be checked. If it's not a file we're going to check # below, it's because it's either a file we weren't interested in # in the first place, or it's the parent of a file we have checked. # Either way, we aren't interested in the parents. for p in hg_fileparents: all_interesting.discard((path, p)) if (path, hg_file) not in all_interesting: continue all_interesting.remove((path, hg_file)) if not GitHgHelper.check_file(hg_file, *hg_fileparents): p = store.manifest_path(path) status.report( 'Sha1 mismatch for file %s\n' ' revision %s' % (p, hg_file)) print_parents = ' '.join(p for p in hg_fileparents if p != NULL_NODE_ID) if print_parents: status.report(' with parent%s %s' % ( 's' if len(print_parents) > 41 else '', print_parents)) progress.progress() progress.finish() if all_interesting: status.info('Could not find the following files:') for path, sha1 in sorted(all_interesting): p = store.manifest_path(path) status.info(' %s %s' % (sha1, path)) status.info( 'This might be a bug in `git cinnabar fsck`. Please open ' 'an issue, with the message above, on\n' 'https://github.com/glandium/git-cinnabar/issues') return 1 if status('broken'): status.info( 'Your git-cinnabar repository appears to be corrupted.\n' 'Please open an issue, with the information above, on\n' 'https://github.com/glandium/git-cinnabar/issues') Git.update_ref('refs/cinnabar/broken', metadata_commit) if Git.resolve_ref('refs/cinnabar/checked'): status.info( '\nThen please try to run `git cinnabar rollback --fsck` to ' 'restore last known state, and to update from the mercurial ' 'repository.') else: status.info('\nThen please try to run `git cinnabar reclone`.') status.info( '\nPlease note this may affect the commit sha1s of mercurial ' 'changesets, and may require to rebase your local branches.') status.info( '\nAlternatively, you may start afresh with a new clone. In any ' 'case, please keep this corrupted repository around for further ' 'debugging.') return 1 refresh = [] if fix_changeset_heads: status.fix('Fixing changeset heads metadata order.') refresh.append('refs/cinnabar/changesets') interval_expired('fsck', 0) store.close(refresh=refresh) GitHgHelper._helper = False metadata_commit = Git.resolve_ref('refs/cinnabar/metadata') Git.update_ref('refs/cinnabar/checked', metadata_commit) return 0
def push(repo, store, what, repo_heads, repo_branches): store.init_fast_import() def heads(): for sha1 in store.heads(repo_branches): yield '^%s' % store.changeset_ref(sha1) def local_bases(): for c in Git.iter('rev-list', '--stdin', '--topo-order', '--full-history', '--boundary', *(w for w in what if w), stdin=heads()): if c[0] != '-': continue yield store.hg_changeset(c[1:]) for w in what: rev = store.hg_changeset(w) if rev: yield rev common = findcommon(repo, store, set(local_bases())) logging.info('common: %s' % common) def revs(): for sha1 in common: yield '^%s' % store.changeset_ref(sha1) push_commits = list(Git.iter('rev-list', '--stdin', '--topo-order', '--full-history', '--parents', '--reverse', *(w for w in what if w), stdin=revs())) pushed = False if push_commits: has_root = any(len(p) == 40 for p in push_commits) force = all(v[1] for v in what.values()) if has_root and repo_heads: if not force: raise Exception('Cannot push a new root') else: logging.warn('Pushing a new root') if force: repo_heads = ['force'] else: if not repo_heads: repo_heads = [NULL_NODE_ID] repo_heads = [unhexlify(h) for h in repo_heads] if repo.local(): repo.local().ui.setconfig('server', 'validate', True) b2caps = bundle2caps(repo) if unbundle20 else {} if b2caps and (repo.url().startswith(('http://', 'https://')) or not isinstance(repo, HelperRepo)): b2caps['replycaps'] = True cg = create_bundle(store, push_commits, b2caps) if not isinstance(repo, HelperRepo): cg = util.chunkbuffer(cg) if not b2caps: cg = cg1unpacker(cg, 'UN') reply = repo.unbundle(cg, repo_heads, '') if unbundle20 and isinstance(reply, unbundle20): parts = iter(reply.iterparts()) for part in parts: if part.type == 'output': sys.stderr.write(part.read()) elif part.type == 'reply:changegroup': # TODO: should check params['in-reply-to'] reply = int(part.params['return']) else: logging.getLogger('bundle2').warning( 'ignoring bundle2 part: %s', part.type) pushed = reply != 0 return gitdag(push_commits) if pushed else ()
def download(args): '''download a prebuilt helper''' helper = 'git-cinnabar-helper' system = args.system machine = args.machine if system.startswith('MSYS_NT'): system = 'Windows' if system == 'Darwin': system = 'macOS' elif system == 'Windows': helper += '.exe' if machine == 'AMD64': machine = 'x86_64' available = ( ('Linux', 'x86_64'), ('macOS', 'x86_64'), ('Windows', 'x86_64'), ('Windows', 'x86'), ) if args.list: for system, machine in available: print "%s/%s" % (system, machine) return 0 if (system, machine) not in available: print >>sys.stderr, 'No download available for %s/%s' % (system, machine) return 1 if args.dev is False: version = VERSION if version.endswith('a'): # For version x.y.za, download a development helper args.dev = '' script_path = os.path.dirname(os.path.abspath(sys.argv[0])) if args.dev is not False: sha1 = helper_hash() if sha1 is None: print >>sys.stderr, ( 'Cannot find the right development helper for this ' 'version of git cinnabar.') return 1 url = 'https://index.taskcluster.net/v1/task/github' url += '.glandium.git-cinnabar.helper.' url += '{}.{}.{}.{}'.format( sha1, system.lower(), machine, args.dev.lower() if args.dev else '').rstrip('.') url += '/artifacts/public/{}'.format(helper) else: if system in ('Windows', 'macOS'): ext = 'zip' else: ext = 'tar.xz' REPO_BASE = 'https://github.com/glandium' url = '%s/git-cinnabar/releases/download/%s/git-cinnabar.%s.%s.%s' % ( REPO_BASE, version, system.lower(), machine.lower(), ext) if args.url: print url return 0 if args.output: d = os.path.dirname(args.output) else: d = script_path if not os.access(d, os.W_OK): d = os.path.join(os.path.expanduser('~'), '.git-cinnabar') try: os.makedirs(d) except Exception: pass if not os.path.isdir(d): print >>sys.stderr, ( 'Cannot write to either %s or %s.' % (d, script_path)) return 1 print 'Downloading from %s...' % url try: reader = HTTPReader(url) except HTTPError: # Try again, just in case try: reader = HTTPReader(url) except HTTPError as e: print >>sys.stderr, ( 'Download failed with status code %d\n' % e.code) print >>sys.stderr, 'Error body was:\n\n%s' % e.read() return 1 class ReaderProgress(object): def __init__(self, reader, length=None): self._reader = reader self._length = length self._read = 0 self._pos = 0 self._buf = '' self._progress = Progress(' {}%' if self._length else ' {} bytes') def read(self, length): # See comment above tell if self._pos < self._read: assert self._read - self._pos <= 8 assert length <= len(self._buf) data = self._buf[:length] self._buf = self._buf[length:] self._pos += length else: assert self._read == self._pos data = self._reader.read(length) self._read += len(data) self._pos = self._read # Keep the last 8 bytes we read for GzipFile self._buf = data[-8:] self.progress() return data def progress(self): if self._length: count = self._read * 100 / self._length else: count = self._read self._progress.progress(count) def finish(self): self._progress.finish() # GzipFile wants to seek to the end of the file and back, so we add # enough tell/seek support to make it happy. It also rewinds 8 bytes # for the CRC, so we also handle that. def tell(self): return self._pos def seek(self, pos, how=os.SEEK_SET): if how == os.SEEK_END: self._pos = self._length + pos elif how == os.SEEK_SET: self._pos = pos elif how == os.SEEK_CUR: self._pos += pos else: raise NotImplementedError() return self._pos encoding = reader.fh.headers.get('Content-Encoding', 'identity') helper_content = ReaderProgress(reader, reader.length) if encoding == 'gzip': class WrapGzipFile(GzipFile): def finish(self): self.fileobj.finish() helper_content = WrapGzipFile(mode='rb', fileobj=helper_content) if args.dev is False: content = StringIO() copyfileobj(helper_content, content) if hasattr(helper_content, 'finish'): helper_content.finish() content.seek(0) print 'Extracting %s...' % helper if ext == 'zip': zip = zipfile.ZipFile(content, 'r') info = zip.getinfo('git-cinnabar/%s' % helper) helper_content = ReaderProgress(zip.open(info), info.file_size) elif ext == 'tar.xz': class UntarProgress(ReaderProgress): def __init__(self, content, helper): self._proc = subprocess.Popen( ['tar', '-JxO', 'git-cinnabar/%s' % helper], stdin=subprocess.PIPE, stdout=subprocess.PIPE) super(UntarProgress, self).__init__(self._proc.stdout) def send(stdin, content): copyfileobj(content, stdin) stdin.close() self._thread = threading.Thread( target=send, args=(self._proc.stdin, content)) self._thread.start() def finish(self): self._proc.wait() self._thread.join() super(UntarProgress, self).finish() helper_content = UntarProgress(content, helper) else: assert False fd, path = tempfile.mkstemp(prefix=helper, dir=d) fh = os.fdopen(fd, 'wb') success = False try: copyfileobj(helper_content, fh) success = True finally: if hasattr(helper_content, 'finish'): helper_content.finish() fh.close() if success: mode = os.stat(path).st_mode if args.output: helper_path = args.output else: helper_path = os.path.join(d, helper) try: # on Windows it's necessary to remove the file first. os.remove(helper_path) except OSError as exc: if exc.errno != errno.ENOENT: raise pass os.rename(path, helper_path) # Add executable bits wherever read bits are set mode = mode | ((mode & 0444) >> 2) os.chmod(helper_path, mode) if not args.no_config: Git.run('config', '--global', 'cinnabar.helper', os.path.abspath(helper_path)) else: os.unlink(path) return 0
def test_module_version(self): module = one(Git.iter( 'ls-tree', 'HEAD', 'cinnabar', cwd=os.path.join(os.path.dirname(__file__), '..'))) self.assertEqual(CmdVersion.module_version(), split_ls_tree(module)[2])
def fsck(args): '''check cinnabar metadata consistency''' if not args.commit and not args.full: return fsck_quick() status = FsckStatus() store = GitHgStore() if args.full and args.commit: logging.error('Cannot pass both --full and a commit') return 1 if args.commit: commits = set() all_git_commits = {} for c in args.commit: cs = store.hg_changeset(c) if cs: commits.add(c) c = cs.node commit = GitHgHelper.hg2git(c) if commit == NULL_NODE_ID and not cs: status.info('Unknown commit or changeset: %s' % c) return 1 if not cs: cs = store.hg_changeset(commit) commits.add(commit) all_git_commits = GitHgHelper.rev_list('--no-walk=unsorted', *commits) else: all_refs = dict((ref, sha1) for sha1, ref in Git.for_each_ref('refs/cinnabar')) if 'refs/cinnabar/metadata' in all_refs: git_heads = '%s^^@' % all_refs['refs/cinnabar/metadata'] else: assert False all_git_commits = GitHgHelper.rev_list( '--topo-order', '--full-history', '--reverse', git_heads) dag = gitdag() GitHgHelper.reset_heads('manifests') full_file_check = FileFindParents.logger.isEnabledFor(logging.DEBUG) for node, tree, parents in progress_iter('Checking {} changesets', all_git_commits): node = store._replace.get(node, node) hg_node = store.hg_changeset(node) if not hg_node: status.report('Missing note for git commit: ' + node) continue GitHgHelper.seen('git2hg', node) changeset_data = store.changeset(hg_node) changeset = changeset_data.node GitHgHelper.seen('hg2git', changeset) changeset_ref = store.changeset_ref(changeset) if not changeset_ref: status.report('Missing changeset in hg2git branch: %s' % changeset) continue elif str(changeset_ref) != node: status.report('Commit mismatch for changeset %s\n' ' hg2git: %s\n commit: %s' % (changeset, changeset_ref, node)) hg_changeset = store.changeset(changeset, include_parents=True) if hg_changeset.node != hg_changeset.sha1: status.report('Sha1 mismatch for changeset %s' % changeset) dag.add(hg_changeset.node, (hg_changeset.parent1, hg_changeset.parent2), changeset_data.branch or 'default') raw_changeset = Changeset.from_git_commit(node) patcher = ChangesetPatcher.from_diff(raw_changeset, changeset_data) if patcher != store.read_changeset_data(node): status.fix('Adjusted changeset metadata for %s' % changeset) GitHgHelper.set('changeset', changeset, NULL_NODE_ID) GitHgHelper.set('changeset', changeset, node) GitHgHelper.put_blob(patcher, want_sha1=False) GitHgHelper.set('changeset-metadata', changeset, NULL_NODE_ID) GitHgHelper.set('changeset-metadata', changeset, ':1') manifest = changeset_data.manifest if GitHgHelper.seen('hg2git', manifest) or manifest == NULL_NODE_ID: continue manifest_ref = store.manifest_ref(manifest) if not manifest_ref: status.report('Missing manifest in hg2git branch: %s' % manifest) parents = tuple( store.changeset(p).manifest for p in hg_changeset.parents ) git_parents = tuple(store.manifest_ref(p) for p in parents if p != NULL_NODE_ID) # This doesn't change the value but makes the helper track the manifest # dag. GitHgHelper.set('manifest', manifest, manifest_ref) if not GitHgHelper.check_manifest(manifest): status.report('Sha1 mismatch for manifest %s' % manifest) manifest_commit_parents = GitCommit(manifest_ref).parents if sorted(manifest_commit_parents) != sorted(git_parents): # TODO: better error status.report('%s(%s) %s != %s' % (manifest, manifest_ref, manifest_commit_parents, git_parents)) # TODO: check that manifest content matches changeset content changes = get_changes(manifest_ref, git_parents) for path, hg_file, hg_fileparents in changes: if hg_file != NULL_NODE_ID and (hg_file == HG_EMPTY_FILE or GitHgHelper.seen('hg2git', hg_file)): if full_file_check: file = store.file(hg_file, hg_fileparents, git_parents, store.manifest_path(path)) valid = file.node == file.sha1 else: valid = GitHgHelper.check_file(hg_file, *hg_fileparents) if not valid: status.report( 'Sha1 mismatch for file %s in manifest %s' % (hg_file, manifest_ref)) if not args.commit and not status('broken'): store_manifest_heads = set(store._manifest_heads_orig) manifest_heads = set(GitHgHelper.heads('manifests')) if store_manifest_heads != manifest_heads: def iter_manifests(a, b): for h in a - b: yield h for h in b: yield '^%s' % h for m, t, p in GitHgHelper.rev_list( '--topo-order', '--full-history', '--reverse', *iter_manifests(manifest_heads, store_manifest_heads)): status.fix('Missing manifest commit in manifest branch: %s' % m) for m, t, p in GitHgHelper.rev_list( '--topo-order', '--full-history', '--reverse', *iter_manifests(store_manifest_heads, manifest_heads)): status.fix('Removing metadata commit %s with no corresponding ' 'changeset' % (m)) for h in store_manifest_heads - manifest_heads: if GitHgHelper.seen('hg2git', store.hg_manifest(h)): status.fix('Removing non-head reference to %s in manifests' ' metadata.' % h) dangling = () if not args.commit and not status('broken'): dangling = GitHgHelper.dangling('hg2git') for obj in dangling: status.fix('Removing dangling metadata for ' + obj) # Theoretically, we should figure out if they are files, manifests # or changesets and set the right variable accordingly, but in # practice, it makes no difference. Reevaluate when GitHgStore.close # is modified, though. GitHgHelper.set('file', obj, NULL_NODE_ID) GitHgHelper.set('file-meta', obj, NULL_NODE_ID) if not args.commit and not status('broken'): dangling = GitHgHelper.dangling('git2hg') for c in dangling: status.fix('Removing dangling note for commit ' + c) GitHgHelper.set('changeset-metadata', c, NULL_NODE_ID) if status('broken'): status.info( 'Your git-cinnabar repository appears to be corrupted. There\n' 'are known issues in older revisions that have been fixed.\n' 'Please try running the following command to reset:\n' ' git cinnabar reclone\n\n' 'Please note this command may change the commit sha1s. Your\n' 'local branches will however stay untouched.\n' 'Please report any corruption that fsck would detect after a\n' 'reclone.') if not args.commit: status.info('Checking head references...') computed_heads = defaultdict(set) for branch, head in dag.all_heads(): computed_heads[branch].add(head) for branch in sorted(dag.tags()): stored_heads = store.heads({branch}) for head in computed_heads[branch] - stored_heads: status.fix('Adding missing head %s in branch %s' % (head, branch)) store.add_head(head) for head in stored_heads - computed_heads[branch]: status.fix('Removing non-head reference to %s in branch %s' % (head, branch)) del store._hgheads[head] metadata_commit = Git.resolve_ref('refs/cinnabar/metadata') if status('broken'): Git.update_ref('refs/cinnabar/broken', metadata_commit) return 1 if args.full: Git.update_ref('refs/cinnabar/checked', metadata_commit) interval_expired('fsck', 0) store.close() if status('fixed'): return 2 return 0
def test_helper_version(self): helper = one(Git.iter( 'ls-tree', 'HEAD', 'helper', cwd=os.path.join(os.path.dirname(__file__), '..'))) self.assertEqual(CmdVersion.helper_version()[1], split_ls_tree(helper)[2])
def create_hg_metadata(self, commit, parents): if len(parents) > 1: raise Exception('Pushing merges is not supported yet') manifest = GeneratedManifestInfo(NULL_NODE_ID) # TODO: share code with GitHgStore.manifest removed = set() modified = {} copies = {} created = OrderedDict() if parents: parent_changeset_data = self.read_changeset_data(parents[0]) parent_manifest = self.manifest(parent_changeset_data['manifest']) parent_node = parent_manifest.node parent_lines = list(parent_manifest._lines) branch = parent_changeset_data.get('extra', {}).get('branch') line = None for line in Git.diff_tree(parents[0], commit, detect_copy=True, recursive=True): mode_before, mode_after, sha1_before, sha1_after, status, \ path = line status = status[0] if status == 'D': removed.add(path) elif status in 'MT': if sha1_before == sha1_after: modified[path] = (None, self.ATTR[mode_after]) else: modified[path] = (sha1_after, self.ATTR[mode_after]) elif status in 'RC': path1, path2 = path.split('\t', 1) if status == 'R': removed.add(path1) if sha1_after != EMPTY_BLOB: copies[path2] = path1 created[path2] = (sha1_after, self.ATTR[mode_after]) else: assert status == 'A' created[path] = (sha1_after, self.ATTR[mode_after]) if line is None: manifest = parent_manifest parent_lines = [] else: parent_node = NULL_NODE_ID parent_lines = [] branch = None for line in Git.ls_tree(commit, recursive=True): mode, typ, sha1, path = line created[path] = (sha1, self.ATTR[mode]) if copies: copied = {k: () for k in copies.values()} for line in parent_lines: name = str(line.name) if name in copied: copied[name] = line.node iter_created = created.iteritems() next_created = next(iter_created) modified_lines = [] for line in parent_lines: if line.name in removed and line.name not in created: continue mod = modified.get(line.name) if mod: node, attr = mod if attr is None: attr = line.attr if node is None: node = PseudoString(line.node) else: node = self.create_file(node, str(line.node)) line = ManifestLine(line.name, node, attr) modified_lines.append(line) while next_created and next_created[0] < line.name: node, attr = next_created[1] if next_created[0] in copies: copied_name = copies[next_created[0]] node = self.create_copy((copied_name, copied[copied_name]), node) else: node = self.create_file(node) created_line = ManifestLine(next_created[0], node, attr) modified_lines.append(created_line) manifest.append_line(created_line) next_created = next(iter_created) manifest.append_line(line) while next_created: node, attr = next_created[1] if next_created[0] in copies: copied_name = copies[next_created[0]] node = self.create_copy((copied_name, copied[copied_name]), node) else: node = self.create_file(node) created_line = ManifestLine(next_created[0], node, attr) modified_lines.append(created_line) manifest.append_line(created_line) next_created = next(iter_created) commit_data = GitCommit(commit) if manifest.node == NULL_NODE_ID: manifest.set_parents(parent_node) manifest.node = manifest.sha1 manifest.removed = removed manifest.modified = {l.name: (l.node, l.attr) for l in modified_lines} manifest.delta_node = parent_node self._push_manifests[manifest.node] = manifest self.manifest_ref(manifest.node, hg2git=False, create=True) self._manifest_git_tree[manifest.node] = commit_data.tree extra = {} if commit_data.author != commit_data.committer: committer = self.hg_author_info(commit_data.committer) extra['committer'] = '%s %d %d' % committer if branch: extra['branch'] = branch changeset_data = self._changeset_data_cache[commit] = { 'files': sorted(chain(removed, modified, created)), 'manifest': manifest.node, } if extra: changeset_data['extra'] = extra changeset = self._changeset(commit, include_parents=True) if self._graft is True and parents and changeset.data[-1] == '\n': parent_cs = self._changeset(parents[0], skip_patch=True) if 'patch' not in self._changeset_data_cache[parents[0]]: self._graft = False else: patch = self._changeset_data_cache[parents[0]]['patch'][-1] self._graft = (patch[1] == len(parent_cs.data) and parent_cs.data[-1] == '\n') if self._graft: self._graft = 'true' if self._graft == 'true' and changeset.data[-1] == '\n': changeset.data = changeset.data[:-1] changeset_data['patch'] = ( (len(changeset.data), len(changeset.data) + 1, ''), ) changeset_data['changeset'] = changeset.changeset = changeset.node = \ changeset.sha1 self._push_changesets[changeset.node] = changeset # This is a horrible way to do this, but this method is not doing much # better overall anyways. if extra: if 'committer' in extra: del extra['committer'] if not extra: del changeset_data['extra'] self._changesets[changeset.node] = PseudoString(commit)
def main(args): logger = logging.getLogger('-') logger.info(args) assert len(args) == 2 remote, url = args git_dir = os.environ.get('GIT_DIR') if Git.config('core.ignorecase', 'bool') == 'true': sys.stderr.write( 'Your git configuration has core.ignorecase set to "true".\n' 'Usually, this means git detected the file system is case ' 'insensitive.\n' 'Git-cinnabar does not support this setup.\n' 'Either use a case sensitive file system or set ' 'core.ignorecase to "false".\n' ) git_work_tree = os.path.dirname(git_dir) if os.path.abspath(os.getcwd() + os.sep).startswith( os.path.abspath(git_work_tree) + os.sep) or \ remote == 'hg::' + url or tuple( Git.for_each_ref('refs/remotes/%s' % remote)): sys.stderr.write( 'Use the following command to reclone:\n' ' git cinnabar reclone\n' ) else: sys.stderr.write( 'Use the following command to clone:\n' ' git -c core.ignorecase=false clone%(args)s hg::%(url)s ' '%(dir)s\n' % { 'dir': git_work_tree, 'url': url, 'args': ' -o ' + remote if remote != 'origin' else '' } ) return 1 repo = get_repo(url) store = GitHgStore() logger.info(LazyString(lambda: '%s' % store.heads())) helper = IOLogger(logging.getLogger('remote-helper'), sys.stdin, sys.stdout) branchmap = None bookmarks = {} HEAD = 'branches/default/tip' while True: cmd, args = read_cmd(helper) if not cmd: break if cmd == 'capabilities': assert not args helper.write( 'option\n' 'import\n' 'bidi-import\n' 'push\n' 'refspec refs/heads/branches/*:' 'refs/cinnabar/refs/heads/branches/*\n' 'refspec refs/heads/bookmarks/*:' 'refs/cinnabar/refs/heads/bookmarks/*\n' 'refspec HEAD:refs/cinnabar/HEAD\n' '\n' ) helper.flush() elif cmd == 'list': assert not args or args == ['for-push'] if repo.capable('batch'): batch = repo.batch() branchmap = batch.branchmap() heads = batch.heads() bookmarks = batch.listkeys('bookmarks') batch.submit() branchmap = branchmap.value heads = heads.value bookmarks = bookmarks.value else: while True: branchmap = repo.branchmap() heads = repo.heads() if heads == ['\0' * 20]: heads = [] # Some branch heads can be non-heads topologically, but if # some heads don't appear in the branchmap, then something # was pushed to the repo between branchmap() and heads() if set(heads).issubset(set(chain(*branchmap.values()))): break bookmarks = repo.listkeys('bookmarks') branchmap = BranchMap(store, branchmap, heads) unknowns = False for branch in sorted(branchmap.names()): branch_tip = branchmap.tip(branch) for head in sorted(branchmap.heads(branch)): sha1 = branchmap.git_sha1(head) if sha1 == '?': unknowns = True if head == branch_tip: continue helper.write('%s refs/heads/branches/%s/%s\n' % ( sha1, branch, head, )) if branch_tip: helper.write('%s refs/heads/branches/%s/tip\n' % ( branchmap.git_sha1(branch_tip), branch, )) for name, sha1 in sorted(bookmarks.iteritems()): ref = store.changeset_ref(sha1) helper.write( '%s refs/heads/bookmarks/%s\n' % (ref if ref else '?', name) ) if not unknowns: for tag, ref in sorted(store.tags(branchmap.heads())): helper.write('%s refs/tags/%s\n' % (ref, tag)) if '@' in bookmarks: HEAD = 'bookmarks/@' helper.write( '@refs/heads/%s HEAD\n' '\n' % HEAD ) helper.flush() elif cmd == 'option': assert len(args) == 2 name, value = args if name == 'progress': if value == 'true': cinnabar.util.progress = True helper.write('ok\n') elif value == 'false': cinnabar.util.progress = False helper.write('ok\n') else: helper.write('unsupported\n') else: helper.write('unsupported\n') helper.flush() elif cmd == 'import': try: reflog = os.path.join(git_dir, 'logs', 'refs', 'cinnabar') mkpath(reflog) open(os.path.join(reflog, 'hg2git'), 'a').close() open(os.path.join(reflog, 'manifest'), 'a').close() assert len(args) == 1 refs = args while cmd: assert cmd == 'import' cmd, args = read_cmd(helper) assert args is None or len(args) == 1 if args: refs.extend(args) except: # If anything wrong happens before we got all the import # commands, we risk git picking the existing refs/cinnabar # refs. Remove them. for line in Git.for_each_ref('refs/cinnabar/refs/heads', 'refs/cinnabar/HEAD', format='%(refname)'): Git.delete_ref(ref) raise try: def resolve_head(head): if head.startswith('refs/heads/branches/'): head = head[20:] if head[-4:] == '/tip': return branchmap.tip(head[:-4]) return head[-40:] if head.startswith('refs/heads/bookmarks/'): head = head[21:] return bookmarks[head] if head == 'HEAD': return bookmarks.get('@') or branchmap.tip('default') return None wanted_refs = {k: v for k, v in ( (h, resolve_head(h)) for h in refs) if v} heads = wanted_refs.values() if not heads: heads = branchmap.heads() # Older versions would create a symbolic ref for # refs/remote-hg/HEAD. Newer versions don't, and # Git.update_ref doesn't remove the symbolic ref, so it needs # to be removed first. # Since git symbolic-ref only throws an error when the ref is # not symbolic, just try to remove the symbolic ref every time # and ignore errors. tuple(Git.iter('symbolic-ref', '-d', 'refs/remote-hg/HEAD', stderr=open(os.devnull, 'wb'))) refs_orig = {} for line in Git.for_each_ref( 'refs/cinnabar/refs/heads', 'refs/cinnabar/HEAD', format='%(objectname) %(refname)'): sha1, ref = line.split(' ', 1) refs_orig[ref] = sha1 except: # If anything wrong happens before we actually pull, we risk # git pucking the existing refs/cinnabar refs. Remove them. # Unlike in the case above, we now have the list of refs git # is expected, so we can just remove those. for ref in refs: Git.delete_ref('refs/cinnabar/' + ref) raise try: store.init_fast_import(FastImport(sys.stdin, sys.stdout)) getbundle(repo, store, heads, branchmap) except: wanted_refs = {} raise finally: for ref, value in wanted_refs.iteritems(): ref = 'refs/cinnabar/' + ref if ref not in refs_orig or refs_orig[ref] != value: Git.update_ref(ref, store.changeset_ref(value)) for ref in refs_orig: if ref[14:] not in wanted_refs: Git.delete_ref(ref) store.close() if not remote.startswith('hg::'): prune = 'remote.%s.prune' % remote if (Git.config(prune) != 'true' and Git.config('fetch.prune') != 'true'): sys.stderr.write( 'It is recommended that you set "%(conf)s" or ' '"fetch.prune" to "true".\n' ' git config %(conf)s true\n' 'or\n' ' git config fetch.prune true\n' % {'conf': prune} ) if store.tag_changes: sys.stderr.write( '\nRun the following command to update remote tags:\n') if not remote.startswith('hg::'): sys.stderr.write(' git remote update %s\n' % remote) else: sys.stderr.write(' git fetch --tags %s\n' % remote) elif cmd == 'push': if not remote.startswith('hg::'): data_pref = 'remote.%s.cinnabar-data' % remote data = Git.config(data_pref) or 'phase' else: data = 'phase' if data not in ('never', 'phase', 'always'): sys.stderr.write('Invalid value for %s: %s\n' % (data_pref, data)) return 1 refspecs = [] refspecs.extend(args) while True: cmd, args = read_cmd(helper) if not cmd: break assert cmd == 'push' refspecs.extend(args) pushes = {s.lstrip('+'): (d, s.startswith('+')) for s, d in (r.split(':', 1) for r in refspecs)} if isinstance(repo, bundlerepo): for source, (dest, force) in pushes.iteritems(): helper.write('error %s Cannot push to a bundle file\n' % dest) helper.write('\n') helper.flush() else: repo_heads = branchmap.heads() PushStore.adopt(store) pushed = push(repo, store, pushes, repo_heads, branchmap.names()) status = {} for source, (dest, _) in pushes.iteritems(): if dest.startswith('refs/tags/'): if source: status[dest] = 'Pushing tags is unsupported' else: status[dest] = \ 'Deleting remote tags is unsupported' continue if not dest.startswith('refs/heads/bookmarks/'): if source: status[dest] = bool(len(pushed)) else: status[dest] = \ 'Deleting remote branches is unsupported' continue name = dest[21:] if source: source = store.hg_changeset(Git.resolve_ref(source)) \ or '' status[dest] = repo.pushkey( 'bookmarks', name, bookmarks.get(name, ''), source) for source, (dest, force) in pushes.iteritems(): if status[dest] is True: helper.write('ok %s\n' % dest) elif status[dest]: helper.write('error %s %s\n' % (dest, status[dest])) else: helper.write('error %s nothing changed on remote\n' % dest) helper.write('\n') helper.flush() if not pushed: data = False elif data == 'always': data = True elif data == 'phase': phases = repo.listkeys('phases') drafts = {} if not phases.get('publishing', False): drafts = set(p for p, is_draft in phases.iteritems() if int(is_draft)) if not drafts: data = True else: def draft_commits(): for d in drafts: c = store.changeset_ref(d) if c: yield '^%s^@' % c for h in pushed.heads(): yield h args = ['rev-list', '--ancestry-path', '--topo-order', '--stdin'] pushed_drafts = tuple( Git.iter(*args, stdin=draft_commits())) # Theoretically, we could have commits with no # metadata that the remote declares are public, while # the rest of our push is in a draft state. That is # however so unlikely that it's not worth the effort # to support partial metadata storage. data = not bool(pushed_drafts) elif data == 'never': data = False store.close(rollback=not data) store.close()
def helper_hash(head='HEAD'): from cinnabar.git import Git, split_ls_tree from cinnabar.util import one return split_ls_tree(one(Git.iter( 'ls-tree', head, 'helper', cwd=os.path.join(os.path.dirname(__file__), '..'))))[2].decode()
def create_hg_manifest(self, commit, parents): manifest = GeneratedManifestInfo(NULL_NODE_ID) changeset_files = [] if parents: parent_changeset = self.changeset(self.hg_changeset(parents[0])) parent_manifest = self.manifest(parent_changeset.manifest) parent_node = parent_manifest.node if len(parents) == 2: parent2_changeset = self.changeset(self.hg_changeset(parents[1])) parent2_manifest = self.manifest(parent2_changeset.manifest) parent2_node = parent2_manifest.node if parent_node == parent2_node: parents = parents[:1] if not parents: for line in Git.ls_tree(commit, recursive=True): mode, typ, sha1, path = line node = self.create_file(sha1, git_manifest_parents=(), path=path) manifest.append_line(ManifestLine(path, node, self.ATTR[mode]), modified=True) changeset_files.append(path) manifest.set_parents(NULL_NODE_ID) manifest.delta_node = NULL_NODE_ID return manifest, changeset_files elif len(parents) == 2: if not experiment('merge'): raise Exception('Pushing merges is not supported yet') if not self._merge_warn: logging.warning('Pushing merges is experimental.') logging.warning('This may irremediably push bad state to the ' 'mercurial server!') self._merge_warn = 1 git_manifests = (self.manifest_ref(parent_node), self.manifest_ref(parent2_node)) # TODO: this would benefit from less git queries changes = list(get_changes(commit, parents)) files = [(path, mode, sha1) for mode, _, sha1, path in Git.ls_tree(commit, recursive=True)] manifests = sorted_merge(parent_manifest._lines, parent2_manifest._lines, key=lambda i: i.name, non_key=lambda i: i) for line in sorted_merge(files, sorted_merge(changes, manifests)): path, f, (change, (manifest_line_p1, manifest_line_p2)) = line if not f: # File was removed if manifest_line_p1: manifest.removed.add(path) changeset_files.append(path) continue mode, sha1 = f attr = self.ATTR[mode] if manifest_line_p1 and not manifest_line_p2: file_parents = (manifest_line_p1.node,) elif manifest_line_p2 and not manifest_line_p1: file_parents = (manifest_line_p2.node,) elif not manifest_line_p1 and not manifest_line_p2: file_parents = () elif manifest_line_p1.node == manifest_line_p2.node: file_parents = (manifest_line_p1.node,) else: if self._merge_warn == 1: logging.warning('This may take a while...') self._merge_warn = 2 file_parents = (manifest_line_p1.node, manifest_line_p2.node) assert file_parents is not None f = self._create_file_internal( sha1, *file_parents, git_manifest_parents=git_manifests, path=path ) file_parents = tuple(p for p in (f.parent1, f.parent2) if p != NULL_NODE_ID) merged = len(file_parents) == 2 if not merged and file_parents: if self.git_file_ref(file_parents[0]) == sha1: node = file_parents[0] else: merged = True if merged: node = self._store_file_internal(f) else: node = file_parents[0] attr_change = (manifest_line_p1 and manifest_line_p1.attr != attr) manifest.append_line(ManifestLine(path, node, attr), modified=merged or attr_change) if merged or attr_change: changeset_files.append(path) if manifest.data == parent_manifest.data: return parent_manifest, [] manifest.set_parents(parent_node, parent2_node) return manifest, changeset_files def process_diff(diff): for (mode_before, mode_after, sha1_before, sha1_after, status, path) in diff: if status[0] == 'R': yield status[1:], ( '000000', sha1_before, NULL_NODE_ID, 'D') yield path, (mode_after, sha1_before, sha1_after, status) git_diff = sorted( l for l in process_diff(GitHgHelper.diff_tree( parents[0], commit, detect_copy=True)) ) if not git_diff: return parent_manifest, [] parent_lines = OrderedDict((l.name, l) for l in parent_manifest._lines) for line in sorted_merge(parent_lines.iteritems(), git_diff, non_key=lambda i: i[1]): path, manifest_line, change = line if not change: manifest.append_line(manifest_line) continue mode_after, sha1_before, sha1_after, status = change path2 = status[1:] status = status[0] attr = self.ATTR.get(mode_after) if status == 'D': manifest.removed.add(path) changeset_files.append(path) continue if status in 'MT': if sha1_before == sha1_after: node = manifest_line.node else: node = self.create_file( sha1_after, str(manifest_line.node), git_manifest_parents=( self.manifest_ref(parent_node),), path=path) elif status in 'RC': if sha1_after != EMPTY_BLOB: node = self.create_copy( (path2, parent_lines[path2].node), sha1_after, git_manifest_parents=( self.manifest_ref(parent_node),), path=path) else: node = self.create_file( sha1_after, git_manifest_parents=( self.manifest_ref(parent_node),), path=path) else: assert status == 'A' node = self.create_file( sha1_after, git_manifest_parents=( self.manifest_ref(parent_node),), path=path) manifest.append_line(ManifestLine(path, node, attr), modified=True) changeset_files.append(path) manifest.set_parents(parent_node) manifest.delta_node = parent_node return manifest, changeset_files