def run(self): from cinnabar import VERSION from cinnabar.git import Git, GitProcess from distutils.version import StrictVersion parent_dir = os.path.dirname(os.path.dirname(__file__)) if not os.path.exists(os.path.join(parent_dir, '.git')) or \ check_enabled('no-version-check') or \ not interval_expired('version-check', 86400, globl=True): return REPO = 'https://github.com/glandium/git-cinnabar' devnull = open(os.devnull, 'wb') if VERSION.endswith('a'): _, _, extra = StrictVersion(VERSION[:-1]).version ref = 'refs/heads/next' if extra == 0 else 'refs/heads/master' for line in Git.iter('ls-remote', REPO, ref, stderr=devnull): sha1, head = line.split() if head != ref: continue proc = GitProcess('-C', parent_dir, 'merge-base', '--is-ancestor', sha1, 'HEAD', stdout=devnull, stderr=devnull) if proc.wait() != 0: self.message = ( 'The `{}` branch of git-cinnabar was updated. ' 'Please update your copy.\n' 'You can switch to the `release` branch if you want ' 'to reduce these update notifications.'.format( ref.partition('refs/heads/')[-1])) break else: version = StrictVersion(VERSION) newer_version = version for line in Git.iter('ls-remote', REPO, 'refs/tags/*', stderr=devnull): sha1, tag = line.split() tag = tag.partition('refs/tags/')[-1] try: v = StrictVersion(tag) except ValueError: continue if v > newer_version: newer_version = v if newer_version != version: self.message = ('New git-cinnabar version available: {} ' '(current version: {})'.format( newer_version, version))
def reclone(args): '''reclone all mercurial remotes''' from cinnabar.cmd.rollback import do_rollback git_config = {} metadata_commit = Git.resolve_ref('refs/cinnabar/metadata') if metadata_commit: git_config['cinnabar.previous-metadata'] = \ metadata_commit.decode('ascii') # TODO: Avoid resetting at all, possibly leaving the repo with no metadata # if this is interrupted somehow. do_rollback(NULL_NODE_ID.decode('ascii')) for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'): config, url = line.split() name = config[len('remote.'):-len('.url')] skip_pref = 'remote.%s.skipDefaultUpdate' % name.decode('ascii') if (url.startswith((b'hg::', b'hg://')) and Git.config(skip_pref) != 'true'): Git.run('remote', 'update', '--prune', fsdecode(name), config=git_config) git_config = {} print('Please note that reclone left your local branches untouched.') print('They may be based on entirely different commits.')
def test_module_version(self): module = one( Git.iter('ls-tree', 'HEAD', 'cinnabar', cwd=os.path.join(os.path.dirname(__file__), '..'))) self.assertEqual(CmdVersion.module_version(), split_ls_tree(module)[2])
def test_cinnabar_version(self): desc = one(Git.iter('describe', '--tags', 'HEAD')) version = Version(CmdVersion.cinnabar_version()) if '-' in desc: last_tag, n, sha1 = desc.rsplit('-', 2) self.assertGreater(version, Version(last_tag)) else: self.assertEqual(version, Version(desc))
def run(self): from cinnabar import VERSION from cinnabar.git import Git, GitProcess from distutils.version import StrictVersion parent_dir = os.path.dirname(os.path.dirname(__file__)) if not os.path.exists(os.path.join(parent_dir, '.git')) or \ check_enabled('no-version-check') or \ not interval_expired('version-check', 86400): return REPO = 'https://github.com/glandium/git-cinnabar' devnull = open(os.devnull, 'wb') if VERSION.endswith('a'): _, _, extra = StrictVersion(VERSION[:-1]).version ref = 'refs/heads/next' if extra == 0 else 'refs/heads/master' for line in Git.iter('ls-remote', REPO, ref, stderr=devnull): sha1, head = line.split() if head != ref: continue proc = GitProcess( '-C', parent_dir, 'merge-base', '--is-ancestor', sha1, 'HEAD', stdout=devnull, stderr=devnull) if proc.wait() != 0: self.message = ( 'The `{}` branch of git-cinnabar was updated. ' 'Please update your copy.\n' 'You can switch to the `release` branch if you want ' 'to reduce these update notifications.' .format(ref.partition('refs/heads/')[-1])) break else: version = StrictVersion(VERSION) newer_version = version for line in Git.iter('ls-remote', REPO, 'refs/tags/*', stderr=devnull): sha1, tag = line.split() tag = tag.partition('refs/tags/')[-1] try: v = StrictVersion(tag) except ValueError: continue if v > newer_version: newer_version = v if newer_version != version: self.message = ( 'New version available: {} (current version: {})' .format(newer_version, version))
def old_helper_head(): from cinnabar.git import Git from cinnabar.helper import GitHgHelper version = GitHgHelper.VERSION return list(Git.iter( 'log', 'HEAD', '--format=%H', '--pickaxe-regex', '-S', '#define CMD_VERSION {}'.format(version), cwd=os.path.join(os.path.dirname(__file__), '..')))[-1]
def test_helper_version(self): helper = one( Git.iter('ls-tree', 'HEAD', 'helper', cwd=os.path.join(os.path.dirname(__file__), '..'))) self.assertEqual(CmdVersion.helper_version()[1], split_ls_tree(helper)[2])
def do_cinnabarclone(repo, manifest, store): url = None for line in manifest.splitlines(): line = line.strip() spec, _, params = line.partition(' ') params = { k: v for k, _, v in (p.partition('=') for p in params.split()) } graft = params.pop('graft', None) if params: # Future proofing: ignore lines with unknown params, even if we # support some that are present. continue if store._graft: # When grafting, ignore lines without a graft revision. if not graft: continue graft = graft.split(',') revs = list(Git.iter('rev-parse', '--revs-only', *graft)) if len(revs) != len(graft): continue # We apparently have all the grafted revisions locally, ensure # they're actually reachable. if not any(Git.iter( 'rev-list', '--branches', '--tags', '--remotes', '--max-count=1', '--ancestry-path', '--stdin', stdin=('^{}^@'.format(c) for c in graft))): continue url, _, branch = spec.partition('#') url, branch = (url.split('#', 1) + [None])[:2] if url: break if not url: logging.warn('Server advertizes cinnabarclone but didn\'t provide ' 'a git repository url to fetch from.') return False parsed_url = urlparse(url) if parsed_url.scheme not in ('http', 'https', 'git'): logging.warn('Server advertizes cinnabarclone but provided a non ' 'http/https git repository. Skipping.') return False sys.stderr.write('Fetching cinnabar metadata from %s\n' % url) return store.merge(url, repo.url(), branch)
def old_helper_hash(head): from cinnabar.git import Git, split_ls_tree from cinnabar.util import one return split_ls_tree( one( Git.iter('ls-tree', head, 'helper', cwd=os.path.join(os.path.dirname(__file__), '..'))))[2]
def local_bases(): for c in Git.iter('rev-list', '--stdin', '--topo-order', '--full-history', '--boundary', *(w for w in what if w), stdin=heads()): if c[0] != '-': continue yield store.hg_changeset(c[1:]) for w in what: rev = store.hg_changeset(w) if rev: yield rev
def reclone(args): '''reclone all mercurial remotes''' from cinnabar.cmd.rollback import do_rollback do_rollback(NULL_NODE_ID) for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'): config, url = line.split() name = config[len('remote.'):-len('.url')] skip_pref = 'remote.%s.skipDefaultUpdate' % name if (url.startswith(('hg::', 'hg://')) and Git.config(skip_pref) != 'true'): Git.run('remote', 'update', '--prune', name) print 'Please note that reclone left your local branches untouched.' print 'They may be based on entirely different commits.'
def old_helper_head(): from cinnabar import VERSION version = VERSION if version.endswith('a'): from distutils.version import StrictVersion v = StrictVersion(VERSION[:-1]).version + (0, 0, 0) if v[2] == 0: from cinnabar.git import Git from cinnabar.helper import GitHgHelper version = GitHgHelper.VERSION return list(Git.iter( 'log', 'HEAD', '--format=%H', '-S', '#define CMD_VERSION {}'.format(version), cwd=os.path.join(os.path.dirname(__file__), '..')))[-1] version = '{}.{}.{}'.format(v[0], v[1], v[2] - 1) return version
def find_user_password(self, realm, authuri): try: return url_passwordmgr.find_user_password(self, realm, authuri) except error.Abort: # Assume error.Abort is only thrown from the base class's # find_user_password itself, which reflects that authentication # information is missing and mercurial would want to get it # from user input, but can't because the ui isn't interactive. credentials = dict( line.split(b'=', 1) for line in Git.iter( 'credential', 'fill', stdin=b'url=%s' % authuri)) username = credentials.get(b'username') password = credentials.get(b'password') if not username or not password: raise return username, password
def old_compatible_python(): '''Find the oldest version of the python code that is compatible with the current helper''' from cinnabar.git import Git with open(os.path.join(os.path.dirname(__file__), '..', 'helper', 'cinnabar-helper.c')) as fh: min_version = None for l in fh: if l.startswith('#define MIN_CMD_VERSION'): min_version = l.rstrip().split()[-1][:2] break if not min_version: raise Exception('Cannot find MIN_CMD_VERSION') return list(Git.iter( 'log', 'HEAD', '--format=%H', '-S', 'class GitHgHelper(BaseHelper):\n VERSION = {}'.format(min_version), cwd=os.path.join(os.path.dirname(__file__), '..')))[-1].decode()
def old_compatible_python(): '''Find the oldest version of the python code that is compatible with the current helper''' from cinnabar.git import Git with open(os.path.join(os.path.dirname(__file__), '..', 'helper', 'cinnabar-helper.c')) as fh: min_version = None for l in fh: if l.startswith('#define MIN_CMD_VERSION'): min_version = l.rstrip().split()[-1][:2] break if not min_version: raise Exception('Cannot find MIN_CMD_VERSION') return list(Git.iter( 'log', 'HEAD', '--format=%H', '-S', 'class GitHgHelper(BaseHelper):\n VERSION = {}'.format(min_version), cwd=os.path.join(os.path.dirname(__file__), '..')))[-1]
def old_helper_head(): from cinnabar import VERSION from distutils.version import StrictVersion version = VERSION if version.endswith('a'): v = StrictVersion(VERSION[:-1]).version if v[2] == 0: from cinnabar.git import Git from cinnabar.helper import GitHgHelper version = GitHgHelper.VERSION return list(Git.iter( 'log', 'HEAD', '--format=%H', '-S', '#define CMD_VERSION {}'.format(version), cwd=os.path.join(os.path.dirname(__file__), '..')))[-1].decode() else: v = StrictVersion(VERSION).version return '{}.{}.{}'.format(v[0], v[1], max(v[2] - 1, 0))
def main(args): cmd = args.pop(0) if cmd == 'data': store = GitHgStore() if args[0] == '-c': sys.stdout.write(store.changeset(args[1]).data) elif args[0] == '-m': sys.stdout.write(store.manifest(args[1]).data) store.close() elif cmd == 'fsck': return fsck(args) elif cmd == 'reclone': for ref in Git.for_each_ref('refs/cinnabar', 'refs/remote-hg', 'refs/notes/cinnabar', 'refs/notes/remote-hg/git2hg', format='%(refname)'): Git.delete_ref(ref) Git.close() for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'): config, url = line.split() name = config[len('remote.'):-len('.url')] skip_pref = 'remote.%s.skipDefaultUpdate' % name if (url.startswith('hg::') and Git.config(skip_pref, 'bool') != 'true'): Git.run('remote', 'update', '--prune', name) print 'Please note that reclone left your local branches untouched.' print 'They may be based on entirely different commits.' elif cmd == 'hg2git': for arg in args: print GitHgHelper.hg2git(arg) elif cmd == 'git2hg': for arg in args: data = GitHgHelper.git2hg(arg) if data: data = ChangesetData.parse(data) print data.get('changeset', NULL_NODE_ID) else: print NULL_NODE_ID else: print >> sys.stderr, 'Unknown command:', cmd return 1
def find_user_password(self, realm, authuri): try: return url_passwordmgr.find_user_password(self, realm, authuri) except error.Abort: # Assume error.Abort is only thrown from the base class's # find_user_password itself, which reflects that authentication # information is missing and mercurial would want to get it # from user input, but can't because the ui isn't interactive. credentials = dict( line.split('=', 1) for line in Git.iter('credential', 'fill', stdin='url=%s' % authuri) ) username = credentials.get('username') password = credentials.get('password') if not username or not password: raise return username, password
def main(args): cmd = args.pop(0) if cmd == 'data': store = GitHgStore() if args[0] == '-c': sys.stdout.write(store.changeset(args[1]).data) elif args[0] == '-m': sys.stdout.write(store.manifest(args[1]).data) store.close() elif cmd == 'fsck': return fsck(args) elif cmd == 'reclone': for ref in Git.for_each_ref('refs/cinnabar', 'refs/remote-hg', 'refs/notes/cinnabar', 'refs/notes/remote-hg/git2hg', format='%(refname)'): Git.delete_ref(ref) Git.close() for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'): config, url = line.split() name = config[len('remote.'):-len('.url')] skip_pref = 'remote.%s.skipDefaultUpdate' % name if (url.startswith('hg::') and Git.config(skip_pref, 'bool') != 'true'): Git.run('remote', 'update', '--prune', name) print 'Please note that reclone left your local branches untouched.' print 'They may be based on entirely different commits.' elif cmd == 'hg2git': for arg in args: print GitHgHelper.hg2git(arg) elif cmd == 'git2hg': for arg in args: data = GitHgHelper.git2hg(arg) if data: data = ChangesetData.parse(data) print data.get('changeset', NULL_NODE_ID) else: print NULL_NODE_ID else: print >>sys.stderr, 'Unknown command:', cmd return 1
def reclone(args): '''reclone all mercurial remotes''' from cinnabar.cmd.rollback import do_rollback git_config = {} metadata_commit = Git.resolve_ref('refs/cinnabar/metadata') if metadata_commit: git_config['cinnabar.previous-metadata'] = metadata_commit # TODO: Avoid resetting at all, possibly leaving the repo with no metadata # if this is interrupted somehow. do_rollback(NULL_NODE_ID) for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'): config, url = line.split() name = config[len('remote.'):-len('.url')] skip_pref = 'remote.%s.skipDefaultUpdate' % name if (url.startswith(('hg::', 'hg://')) and Git.config(skip_pref) != 'true'): Git.run('remote', 'update', '--prune', name, config=git_config) git_config = {} print 'Please note that reclone left your local branches untouched.' print 'They may be based on entirely different commits.'
def push(self, *refspecs): try: default = 'never' if self._graft else 'phase' values = { None: default, '': default, 'never': 'never', 'phase': 'phase', 'always': 'always', } data = Git.config('cinnabar.data', self._remote.name, values=values) except InvalidConfig as e: logging.error(e.message) return 1 pushes = {s.lstrip('+'): (d, s.startswith('+')) for s, d in (r.split(':', 1) for r in refspecs)} if not self._repo.capable('unbundle'): for source, (dest, force) in pushes.iteritems(): self._helper.write( 'error %s Remote does not support the "unbundle" ' 'capability\n' % dest) self._helper.write('\n') self._helper.flush() else: repo_heads = self._branchmap.heads() PushStore.adopt(self._store, self._graft) pushed = push(self._repo, self._store, pushes, repo_heads, self._branchmap.names()) status = {} for source, (dest, _) in pushes.iteritems(): if dest.startswith('refs/tags/'): if source: status[dest] = 'Pushing tags is unsupported' else: status[dest] = \ 'Deleting remote tags is unsupported' continue if not dest.startswith('refs/heads/bookmarks/'): if source: status[dest] = bool(len(pushed)) else: status[dest] = \ 'Deleting remote branches is unsupported' continue name = unquote(dest[21:]) if source: source = self._store.hg_changeset(Git.resolve_ref(source))\ or '' status[dest] = self._repo.pushkey( 'bookmarks', name, self._bookmarks.get(name, ''), source) for source, (dest, force) in pushes.iteritems(): if status[dest] is True: self._helper.write('ok %s\n' % dest) elif status[dest]: self._helper.write('error %s %s\n' % (dest, status[dest])) else: self._helper.write('error %s nothing changed on remote\n' % dest) self._helper.write('\n') self._helper.flush() if not pushed: data = False elif data == 'always': data = True elif data == 'phase': phases = self._repo.listkeys('phases') drafts = {} if not phases.get('publishing', False): drafts = set(p for p, is_draft in phases.iteritems() if int(is_draft)) if not drafts: data = True else: def draft_commits(): for d in drafts: c = self._store.changeset_ref(d) if c: yield '^%s^@' % c for h in pushed.heads(): yield h args = ['rev-list', '--ancestry-path', '--topo-order', '--stdin'] pushed_drafts = tuple( Git.iter(*args, stdin=draft_commits())) # Theoretically, we could have commits with no # metadata that the remote declares are public, while # the rest of our push is in a draft state. That is # however so unlikely that it's not worth the effort # to support partial metadata storage. data = not bool(pushed_drafts) elif data == 'never': data = False self._store.close(rollback=not data)
def fsck(args): parser = argparse.ArgumentParser() parser.add_argument( '--manifests', action='store_true', help='Validate manifests hashes') parser.add_argument( '--files', action='store_true', help='Validate files hashes') parser.add_argument( 'commit', nargs='*', help='Specific commit or changeset to check') args = parser.parse_args(args) status = { 'broken': False, 'fixed': False, } def info(message): sys.stderr.write('\r') print message def fix(message): status['fixed'] = True info(message) def report(message): status['broken'] = True info(message) store = GitHgStore() store.init_fast_import(lambda: FastImport()) if args.commit: all_hg2git = {} all_notes = set() commits = set() all_git_commits = {} for c in args.commit: data = store.read_changeset_data(c) if data: all_notes.add(c) commits.add(c) c = data['changeset'] commit = GitHgHelper.hg2git(c) if commit == NULL_NODE_ID and not data: info('Unknown commit or changeset: %s' % c) return 1 if commit != NULL_NODE_ID: all_hg2git[c] = commit, 'commit' if not data: data = store.read_changeset_data(commit) commits.add(commit) if data: all_notes.add(commit) all_git_commits = Git.iter( 'log', '--no-walk=unsorted', '--stdin', '--format=%T %H', stdin=commits) else: all_hg2git = { path.replace('/', ''): (filesha1, intern(typ)) for mode, typ, filesha1, path in progress_iter('Reading %d mercurial to git mappings', Git.ls_tree('refs/cinnabar/hg2git', recursive=True)) } all_notes = set(path.replace('/', '') for mode, typ, filesha1, path in progress_iter( 'Reading %d commit to changeset mappings', Git.ls_tree('refs/notes/cinnabar', recursive=True))) manifest_commits = OrderedDict((m, None) for m in progress_iter( 'Reading %d manifest trees', Git.iter('rev-list', '--full-history', '--topo-order', 'refs/cinnabar/manifest')) ) all_git_heads = Git.for_each_ref('refs/cinnabar/branches', format='%(refname)') all_git_commits = Git.iter('log', '--topo-order', '--full-history', '--reverse', '--stdin', '--format=%T %H', stdin=all_git_heads) store._hg2git_cache = {p: s for p, (s, t) in all_hg2git.iteritems()} seen_changesets = set() seen_manifests = set() seen_manifest_refs = {} seen_files = set() seen_notes = set() hg_manifest = None dag = gitdag() for line in progress_iter('Checking %d changesets', all_git_commits): tree, node = line.split(' ') if node not in all_notes: report('Missing note for git commit: ' + node) continue seen_notes.add(node) changeset_data = store.read_changeset_data(node) changeset = changeset_data['changeset'] if 'extra' in changeset_data: extra = changeset_data['extra'] header, message = GitHgHelper.cat_file( 'commit', node).split('\n\n', 1) header = dict(l.split(' ', 1) for l in header.splitlines()) if 'committer' in extra: committer_info = store.hg_author_info(header['committer']) committer = '%s %d %d' % committer_info if (committer != extra['committer'] and header['committer'] != extra['committer'] and committer_info[0] != extra['committer']): report('Committer mismatch between commit and metadata for' ' changeset %s' % changeset) if committer == extra['committer']: fix('Fixing useless committer metadata for changeset %s' % changeset) del changeset_data['extra']['committer'] store._changesets[changeset] = LazyString(node) if header['committer'] != header['author'] and not extra: fix('Fixing useless empty extra metadata for changeset %s' % changeset) del changeset_data['extra'] store._changesets[changeset] = LazyString(node) seen_changesets.add(changeset) changeset_ref = store.changeset_ref(changeset) if not changeset_ref: report('Missing changeset in hg2git branch: %s' % changeset) elif str(changeset_ref) != node: report('Commit mismatch for changeset %s\n' ' hg2git: %s\n commit: %s' % (changeset, changeset_ref, node)) hg_changeset = store.changeset(changeset, include_parents=True) sha1 = hg_changeset.sha1 if hg_changeset.node != sha1: try_fixup = False if (changeset, sha1) in ( ('8c557b7c03a4a753e5c163038f04862e9f65fce1', '249b59139de8e08abeb6c4e261a137c756e7af0e'), ('ffdee4a4eb7fc7cae80dfc4cb2fe0c3178773dcf', '415e9d2eac83d508bf58a4df585c5f6b2b0f44ed'), ): header = hg_changeset.data.split('\n', 4) start = sum(len(h) for h in header[:3]) + 1 changeset_data['patch'] = ((start, start + 1, '1'),) try_fixup = True # Some know cases of corruptions involve a whitespace after the # timezone. Adding an empty extra metadata works around those. elif 'extra' not in changeset_data: changeset_data['extra'] = {} try_fixup = True if try_fixup: hg_changeset = store.changeset(changeset, include_parents=True) sha1 = hg_changeset.sha1 if hg_changeset.node == sha1: fix('Fixing known sha1 mismatch for changeset %s' % changeset) store._changesets[changeset] = LazyString(node) if hg_changeset.node != sha1: report('Sha1 mismatch for changeset %s' % changeset) dag.add(hg_changeset.node, (hg_changeset.parent1, hg_changeset.parent2), changeset_data.get('extra', {}).get('branch', 'default')) manifest = changeset_data['manifest'] if manifest in seen_manifests: continue seen_manifests.add(manifest) manifest_ref = store.manifest_ref(manifest) if manifest_ref: seen_manifest_refs[manifest_ref] = manifest if not manifest_ref: report('Missing manifest in hg2git branch: %s' % manifest) elif not args.commit and manifest_ref not in manifest_commits: report('Missing manifest commit in manifest branch: %s' % manifest_ref) if args.manifests or args.files: parents = tuple( store.read_changeset_data(store.changeset_ref(p))['manifest'] for p in (hg_changeset.parent1, hg_changeset.parent2) if p != NULL_NODE_ID ) if args.manifests: try: with GitHgHelper.query('check-manifest', manifest, *parents) as stdout: if stdout.readline().strip() != 'ok': report('Sha1 mismatch for manifest %s' % manifest) except NoHelperException: hg_manifest = store.manifest(manifest) hg_manifest.set_parents(*parents) if hg_manifest.node != hg_manifest.sha1: report('Sha1 mismatch for manifest %s' % manifest) git_ls = one(Git.ls_tree(manifest_ref, 'git')) if git_ls: mode, typ, sha1, path = git_ls else: header, message = GitHgHelper.cat_file( 'commit', manifest_ref).split('\n\n', 1) header = dict(l.split(' ', 1) for l in header.splitlines()) if header['tree'] == EMPTY_TREE: sha1 = EMPTY_TREE else: report('Missing git tree in manifest commit %s' % manifest_ref) sha1 = None if sha1 and sha1 != tree: report('Tree mismatch between manifest commit %s and commit %s' % (manifest_ref, node)) if args.files: changes = get_changes( manifest_ref, tuple(store.manifest_ref(p) for p in parents), 'hg') for path, hg_file, hg_fileparents in changes: if hg_file != NULL_NODE_ID and hg_file not in seen_files: file = store.file(hg_file) file.set_parents(*hg_fileparents) if file.node != file.sha1: report('Sha1 mismatch for file %s in manifest %s' % (hg_file, manifest_ref)) seen_files.add(hg_file) if args.files: all_hg2git = set(all_hg2git.iterkeys()) else: all_hg2git = set(k for k, (s, t) in all_hg2git.iteritems() if t == 'commit') adjusted = {} if not args.commit: dangling = set(manifest_commits) - set(seen_manifest_refs) if dangling: def iter_manifests(): removed_one = False yielded = False previous = None for obj in reversed(manifest_commits): if obj in dangling: fix('Removing metadata commit %s with no hg2git entry' % obj) removed_one = True else: if removed_one: yield obj, previous yielded = True previous = obj if removed_one and not yielded: yield obj, False for obj, parent in progress_iter('Adjusting %d metadata commits', iter_manifests()): mark = store._fast_import.new_mark() if parent is False: Git.update_ref('refs/cinnabar/manifest', obj) continue elif parent: parents = (adjusted.get(parent, parent),) with store._fast_import.commit( ref='refs/cinnabar/manifest', parents=parents, mark=mark) as commit: mode, typ, tree, path = store._fast_import.ls(obj) commit.filemodify('', tree, typ='tree') adjusted[obj] = Mark(mark) dangling = all_hg2git - seen_changesets - seen_manifests - seen_files if dangling or adjusted: with store._fast_import.commit( ref='refs/cinnabar/hg2git', parents=('refs/cinnabar/hg2git^0',)) as commit: for obj in dangling: fix('Removing dangling metadata for ' + obj) commit.filedelete(sha1path(obj)) for obj, mark in progress_iter( 'Updating hg2git for %d metadata commits', adjusted.iteritems()): commit.filemodify(sha1path(seen_manifest_refs[obj]), mark, typ='commit') dangling = all_notes - seen_notes if dangling: with store._fast_import.commit( ref='refs/notes/cinnabar', parents=('refs/notes/cinnabar^0',)) as commit: for c in dangling: fix('Removing dangling note for commit ' + c) # That's brute force, but meh. for l in range(0, 10): commit.filedelete(sha1path(c, l)) if status['broken']: info('Your git-cinnabar repository appears to be corrupted. There\n' 'are known issues in older revisions that have been fixed.\n' 'Please try running the following command to reset:\n' ' git cinnabar reclone\n\n' 'Please note this command may change the commit sha1s. Your\n' 'local branches will however stay untouched.\n' 'Please report any corruption that fsck would detect after a\n' 'reclone.') if not args.commit: info('Checking head references...') computed_heads = defaultdict(set) for branch, head in dag.all_heads(): computed_heads[branch].add(head) for branch in sorted(dag.tags()): stored_heads = store.heads({branch}) for head in computed_heads[branch] - stored_heads: fix('Adding missing head %s in branch %s' % (head, branch)) store.add_head(head) for head in stored_heads - computed_heads[branch]: fix('Removing non-head reference to %s in branch %s' % (head, branch)) store._hgheads.remove((branch, head)) store.close() if status['broken']: return 1 if status['fixed']: return 2 return 0
def fsck(args): parser = argparse.ArgumentParser() parser.add_argument('--manifests', action='store_true', help='Validate manifests hashes') parser.add_argument('--files', action='store_true', help='Validate files hashes') parser.add_argument('commit', nargs='*', help='Specific commit or changeset to check') args = parser.parse_args(args) status = { 'broken': False, 'fixed': False, } def info(message): sys.stderr.write('\r') print message def fix(message): status['fixed'] = True info(message) def report(message): status['broken'] = True info(message) store = GitHgStore() store.init_fast_import(lambda: FastImport()) if args.commit: all_hg2git = {} all_notes = set() commits = set() all_git_commits = {} for c in args.commit: data = store.read_changeset_data(c) if data: all_notes.add(c) commits.add(c) c = data['changeset'] commit = GitHgHelper.hg2git(c) if commit == NULL_NODE_ID and not data: info('Unknown commit or changeset: %s' % c) return 1 if commit != NULL_NODE_ID: all_hg2git[c] = commit, 'commit' if not data: data = store.read_changeset_data(commit) commits.add(commit) if data: all_notes.add(commit) all_git_commits = Git.iter('log', '--no-walk=unsorted', '--stdin', '--format=%T %H', stdin=commits) else: all_hg2git = { path.replace('/', ''): (filesha1, intern(typ)) for mode, typ, filesha1, path in progress_iter( 'Reading %d mercurial to git mappings', Git.ls_tree('refs/cinnabar/hg2git', recursive=True)) } all_notes = set( path.replace('/', '') for mode, typ, filesha1, path in progress_iter( 'Reading %d commit to changeset mappings', Git.ls_tree('refs/notes/cinnabar', recursive=True))) manifest_commits = OrderedDict((m, None) for m in progress_iter( 'Reading %d manifest trees', Git.iter('rev-list', '--full-history', '--topo-order', 'refs/cinnabar/manifest'))) all_git_heads = Git.for_each_ref('refs/cinnabar/branches', format='%(refname)') all_git_commits = Git.iter('log', '--topo-order', '--full-history', '--reverse', '--stdin', '--format=%T %H', stdin=all_git_heads) store._hg2git_cache = {p: s for p, (s, t) in all_hg2git.iteritems()} seen_changesets = set() seen_manifests = set() seen_manifest_refs = {} seen_files = set() seen_notes = set() hg_manifest = None dag = gitdag() for line in progress_iter('Checking %d changesets', all_git_commits): tree, node = line.split(' ') if node not in all_notes: report('Missing note for git commit: ' + node) continue seen_notes.add(node) changeset_data = store.read_changeset_data(node) changeset = changeset_data['changeset'] if 'extra' in changeset_data: extra = changeset_data['extra'] header, message = GitHgHelper.cat_file('commit', node).split('\n\n', 1) header = dict(l.split(' ', 1) for l in header.splitlines()) if 'committer' in extra: committer_info = store.hg_author_info(header['committer']) committer = '%s %d %d' % committer_info if (committer != extra['committer'] and header['committer'] != extra['committer'] and committer_info[0] != extra['committer']): report('Committer mismatch between commit and metadata for' ' changeset %s' % changeset) if committer == extra['committer']: fix('Fixing useless committer metadata for changeset %s' % changeset) del changeset_data['extra']['committer'] store._changesets[changeset] = LazyString(node) if header['committer'] != header['author'] and not extra: fix('Fixing useless empty extra metadata for changeset %s' % changeset) del changeset_data['extra'] store._changesets[changeset] = LazyString(node) seen_changesets.add(changeset) changeset_ref = store.changeset_ref(changeset) if not changeset_ref: report('Missing changeset in hg2git branch: %s' % changeset) elif str(changeset_ref) != node: report('Commit mismatch for changeset %s\n' ' hg2git: %s\n commit: %s' % (changeset, changeset_ref, node)) hg_changeset = store.changeset(changeset, include_parents=True) sha1 = hg_changeset.sha1 if hg_changeset.node != sha1: try_fixup = False if (changeset, sha1) in ( ('8c557b7c03a4a753e5c163038f04862e9f65fce1', '249b59139de8e08abeb6c4e261a137c756e7af0e'), ('ffdee4a4eb7fc7cae80dfc4cb2fe0c3178773dcf', '415e9d2eac83d508bf58a4df585c5f6b2b0f44ed'), ): header = hg_changeset.data.split('\n', 4) start = sum(len(h) for h in header[:3]) + 1 changeset_data['patch'] = ((start, start + 1, '1'), ) try_fixup = True # Some know cases of corruptions involve a whitespace after the # timezone. Adding an empty extra metadata works around those. elif 'extra' not in changeset_data: changeset_data['extra'] = {} try_fixup = True if try_fixup: hg_changeset = store.changeset(changeset, include_parents=True) sha1 = hg_changeset.sha1 if hg_changeset.node == sha1: fix('Fixing known sha1 mismatch for changeset %s' % changeset) store._changesets[changeset] = LazyString(node) if hg_changeset.node != sha1: report('Sha1 mismatch for changeset %s' % changeset) dag.add(hg_changeset.node, (hg_changeset.parent1, hg_changeset.parent2), changeset_data.get('extra', {}).get('branch', 'default')) manifest = changeset_data['manifest'] if manifest in seen_manifests: continue seen_manifests.add(manifest) manifest_ref = store.manifest_ref(manifest) if manifest_ref: seen_manifest_refs[manifest_ref] = manifest if not manifest_ref: report('Missing manifest in hg2git branch: %s' % manifest) elif not args.commit and manifest_ref not in manifest_commits: report('Missing manifest commit in manifest branch: %s' % manifest_ref) if args.manifests or args.files: parents = tuple( store.read_changeset_data(store.changeset_ref(p))['manifest'] for p in (hg_changeset.parent1, hg_changeset.parent2) if p != NULL_NODE_ID) if args.manifests: try: with GitHgHelper.query('check-manifest', manifest, *parents) as stdout: if stdout.readline().strip() != 'ok': report('Sha1 mismatch for manifest %s' % manifest) except NoHelperException: hg_manifest = store.manifest(manifest) hg_manifest.set_parents(*parents) if hg_manifest.node != hg_manifest.sha1: report('Sha1 mismatch for manifest %s' % manifest) git_ls = one(Git.ls_tree(manifest_ref, 'git')) if git_ls: mode, typ, sha1, path = git_ls else: header, message = GitHgHelper.cat_file('commit', manifest_ref).split( '\n\n', 1) header = dict(l.split(' ', 1) for l in header.splitlines()) if header['tree'] == EMPTY_TREE: sha1 = EMPTY_TREE else: report('Missing git tree in manifest commit %s' % manifest_ref) sha1 = None if sha1 and sha1 != tree: report('Tree mismatch between manifest commit %s and commit %s' % (manifest_ref, node)) if args.files: changes = get_changes( manifest_ref, tuple(store.manifest_ref(p) for p in parents), 'hg') for path, hg_file, hg_fileparents in changes: if hg_file != NULL_NODE_ID and hg_file not in seen_files: file = store.file(hg_file) file.set_parents(*hg_fileparents) if file.node != file.sha1: report('Sha1 mismatch for file %s in manifest %s' % (hg_file, manifest_ref)) seen_files.add(hg_file) if args.files: all_hg2git = set(all_hg2git.iterkeys()) else: all_hg2git = set(k for k, (s, t) in all_hg2git.iteritems() if t == 'commit') adjusted = {} if not args.commit: dangling = set(manifest_commits) - set(seen_manifest_refs) if dangling: def iter_manifests(): removed_one = False yielded = False previous = None for obj in reversed(manifest_commits): if obj in dangling: fix('Removing metadata commit %s with no hg2git entry' % obj) removed_one = True else: if removed_one: yield obj, previous yielded = True previous = obj if removed_one and not yielded: yield obj, False for obj, parent in progress_iter('Adjusting %d metadata commits', iter_manifests()): mark = store._fast_import.new_mark() if parent is False: Git.update_ref('refs/cinnabar/manifest', obj) continue elif parent: parents = (adjusted.get(parent, parent), ) with store._fast_import.commit(ref='refs/cinnabar/manifest', parents=parents, mark=mark) as commit: mode, typ, tree, path = store._fast_import.ls(obj) commit.filemodify('', tree, typ='tree') adjusted[obj] = Mark(mark) dangling = all_hg2git - seen_changesets - seen_manifests - seen_files if dangling or adjusted: with store._fast_import.commit( ref='refs/cinnabar/hg2git', parents=('refs/cinnabar/hg2git^0', )) as commit: for obj in dangling: fix('Removing dangling metadata for ' + obj) commit.filedelete(sha1path(obj)) for obj, mark in progress_iter( 'Updating hg2git for %d metadata commits', adjusted.iteritems()): commit.filemodify(sha1path(seen_manifest_refs[obj]), mark, typ='commit') dangling = all_notes - seen_notes if dangling: with store._fast_import.commit( ref='refs/notes/cinnabar', parents=('refs/notes/cinnabar^0', )) as commit: for c in dangling: fix('Removing dangling note for commit ' + c) # That's brute force, but meh. for l in range(0, 10): commit.filedelete(sha1path(c, l)) if status['broken']: info('Your git-cinnabar repository appears to be corrupted. There\n' 'are known issues in older revisions that have been fixed.\n' 'Please try running the following command to reset:\n' ' git cinnabar reclone\n\n' 'Please note this command may change the commit sha1s. Your\n' 'local branches will however stay untouched.\n' 'Please report any corruption that fsck would detect after a\n' 'reclone.') if not args.commit: info('Checking head references...') computed_heads = defaultdict(set) for branch, head in dag.all_heads(): computed_heads[branch].add(head) for branch in sorted(dag.tags()): stored_heads = store.heads({branch}) for head in computed_heads[branch] - stored_heads: fix('Adding missing head %s in branch %s' % (head, branch)) store.add_head(head) for head in stored_heads - computed_heads[branch]: fix('Removing non-head reference to %s in branch %s' % (head, branch)) store._hgheads.remove((branch, head)) store.close() if status['broken']: return 1 if status['fixed']: return 2 return 0
def findcommon(repo, store, hgheads): logger = logging.getLogger('findcommon') logger.debug(hgheads) if not hgheads: return set() sample_size = 100 sample = _sample(hgheads, sample_size) known = repo.known(unhexlify(h) for h in sample) known = set(h for h, k in izip(sample, known) if k) logger.info('initial sample size: %d' % len(sample)) if len(known) == len(hgheads): logger.info('all heads known') return hgheads git_heads = set(store.changeset_ref(h) for h in hgheads) git_known = set(store.changeset_ref(h) for h in known) logger.debug('known (sub)set: (%d) %s', len(known), LazyCall(sorted, git_known)) args = ['rev-list', '--topo-order', '--full-history', '--parents', '--stdin'] def revs(): for h in git_known: yield '^%s' % h for h in git_heads: if h not in git_known: yield h dag = gitdag(chain(Git.iter(*args, stdin=revs()), git_known)) dag.tag_nodes_and_parents(git_known, 'known') def log_dag(tag): if not logger.isEnabledFor(logging.DEBUG): return logger.debug('%s dag size: %d' % ( tag, sum(1 for n in dag.iternodes(tag)))) heads = sorted(dag.heads(tag)) logger.debug('%s dag heads: (%d) %s' % (tag, len(heads), heads)) roots = sorted(dag.roots(tag)) logger.debug('%s dag roots: (%d) %s' % (tag, len(roots), roots)) log_dag('unknown') log_dag('known') while True: unknown = set(chain(dag.heads(), dag.roots())) if not unknown: break sample = set(_sample(unknown, sample_size)) if len(sample) < sample_size: sample |= set(_sample(set(dag.iternodes()), sample_size - len(sample))) sample = list(sample) hg_sample = [store.hg_changeset(h) for h in sample] known = repo.known(unhexlify(h) for h in hg_sample) unknown = set(h for h, k in izip(sample, known) if not k) known = set(h for h, k in izip(sample, known) if k) logger.info('next sample size: %d' % len(sample)) logger.debug('known (sub)set: (%d) %s', len(known), LazyCall(sorted, known)) logger.debug('unknown (sub)set: (%d) %s', len(unknown), LazyCall(sorted, unknown)) dag.tag_nodes_and_parents(known, 'known') dag.tag_nodes_and_children(unknown, 'unknown') log_dag('unknown') log_dag('known') return [store.hg_changeset(h) for h in dag.heads('known')]
def do_cinnabarclone(repo, manifest, store, limit_schemes=True): GRAFT = { None: None, b'false': False, b'true': True, } try: enable_graft = Git.config('cinnabar.graft', remote=repo.remote, values=GRAFT) except InvalidConfig: enable_graft = None url = None candidates = [] for line in manifest.splitlines(): line = line.strip() if not line: continue spec, _, params = line.partition(b' ') params = { k: v for k, _, v in (p.partition(b'=') for p in params.split()) } graft = params.pop(b'graft', None) if params: # Future proofing: ignore lines with unknown params, even if we # support some that are present. continue # When grafting, ignore lines without a graft revision. if store._graft and not graft: continue # When explicitly disabling graft, ignore lines with a graft revision. if enable_graft is False and graft: continue graft = graft.split(b',') if graft else [] graft_u = [] for g in graft: if SHA1_RE.match(g): graft_u.append(g.decode('ascii')) if len(graft) != len(graft_u): continue if graft: revs = list(Git.iter('rev-parse', '--revs-only', *graft_u)) if len(revs) != len(graft): continue # We apparently have all the grafted revisions locally, ensure # they're actually reachable. if not any( Git.iter('rev-list', '--branches', '--tags', '--remotes', '--max-count=1', '--ancestry-path', '--stdin', stdin=(b'^%s^@' % c for c in graft), stderr=open(os.devnull, 'wb'))): continue candidates.append((spec, len(graft) != 0)) if enable_graft is not False: graft_filters = [True, False] else: graft_filters = [False] for graft_filter in graft_filters: for spec, graft in candidates: if graft == graft_filter: url, _, branch = spec.partition(b'#') url, branch = (url.split(b'#', 1) + [None])[:2] if url: break if url: break if not url: logging.warn('Server advertizes cinnabarclone but didn\'t provide ' 'a git repository url to fetch from.') return False parsed_url = urlparse(url) if limit_schemes and parsed_url.scheme not in (b'http', b'https', b'git'): logging.warn('Server advertizes cinnabarclone but provided a non ' 'http/https git repository. Skipping.') return False sys.stderr.write('Fetching cinnabar metadata from %s\n' % fsdecode(url)) sys.stderr.flush() return store.merge(url, repo.url(), branch)
def push(repo, store, what, repo_heads, repo_branches): store.init_fast_import() def heads(): for sha1 in store.heads(repo_branches): yield '^%s' % store.changeset_ref(sha1) def local_bases(): for c in Git.iter('rev-list', '--stdin', '--topo-order', '--full-history', '--boundary', *(w for w in what if w), stdin=heads()): if c[0] != '-': continue yield store.hg_changeset(c[1:]) for w in what: rev = store.hg_changeset(w) if rev: yield rev common = findcommon(repo, store, set(local_bases())) logging.info('common: %s' % common) def revs(): for sha1 in common: yield '^%s' % store.changeset_ref(sha1) push_commits = list(Git.iter('rev-list', '--stdin', '--topo-order', '--full-history', '--parents', '--reverse', *(w for w in what if w), stdin=revs())) pushed = False if push_commits: has_root = any(len(p) == 40 for p in push_commits) force = all(v[1] for v in what.values()) if has_root and repo_heads: if not force: raise Exception('Cannot push a new root') else: logging.warn('Pushing a new root') if force: repo_heads = ['force'] else: if not repo_heads: repo_heads = [NULL_NODE_ID] repo_heads = [unhexlify(h) for h in repo_heads] if repo.local(): repo.local().ui.setconfig('server', 'validate', True) b2caps = bundle2caps(repo) if unbundle20 else {} if b2caps and (repo.url().startswith(('http://', 'https://')) or not isinstance(repo, HelperRepo)): b2caps['replycaps'] = True cg = create_bundle(store, push_commits, b2caps) if not isinstance(repo, HelperRepo): cg = util.chunkbuffer(cg) if not b2caps: cg = cg1unpacker(cg, 'UN') reply = repo.unbundle(cg, repo_heads, '') if unbundle20 and isinstance(reply, unbundle20): parts = iter(reply.iterparts()) for part in parts: if part.type == 'output': sys.stderr.write(part.read()) elif part.type == 'reply:changegroup': # TODO: should check params['in-reply-to'] reply = int(part.params['return']) else: logging.getLogger('bundle2').warning( 'ignoring bundle2 part: %s', part.type) pushed = reply != 0 return gitdag(push_commits) if pushed else ()
def main(args): logger = logging.getLogger('-') logger.info(args) assert len(args) == 2 remote, url = args git_dir = os.environ.get('GIT_DIR') if Git.config('core.ignorecase', 'bool') == 'true': sys.stderr.write( 'Your git configuration has core.ignorecase set to "true".\n' 'Usually, this means git detected the file system is case ' 'insensitive.\n' 'Git-cinnabar does not support this setup.\n' 'Either use a case sensitive file system or set ' 'core.ignorecase to "false".\n' ) git_work_tree = os.path.dirname(git_dir) if os.path.abspath(os.getcwd() + os.sep).startswith( os.path.abspath(git_work_tree) + os.sep) or \ remote == 'hg::' + url or tuple( Git.for_each_ref('refs/remotes/%s' % remote)): sys.stderr.write( 'Use the following command to reclone:\n' ' git cinnabar reclone\n' ) else: sys.stderr.write( 'Use the following command to clone:\n' ' git -c core.ignorecase=false clone%(args)s hg::%(url)s ' '%(dir)s\n' % { 'dir': git_work_tree, 'url': url, 'args': ' -o ' + remote if remote != 'origin' else '' } ) return 1 repo = get_repo(url) store = GitHgStore() logger.info(LazyString(lambda: '%s' % store.heads())) helper = IOLogger(logging.getLogger('remote-helper'), sys.stdin, sys.stdout) branchmap = None bookmarks = {} HEAD = 'branches/default/tip' while True: cmd, args = read_cmd(helper) if not cmd: break if cmd == 'capabilities': assert not args helper.write( 'option\n' 'import\n' 'bidi-import\n' 'push\n' 'refspec refs/heads/branches/*:' 'refs/cinnabar/refs/heads/branches/*\n' 'refspec refs/heads/bookmarks/*:' 'refs/cinnabar/refs/heads/bookmarks/*\n' 'refspec HEAD:refs/cinnabar/HEAD\n' '\n' ) helper.flush() elif cmd == 'list': assert not args or args == ['for-push'] if repo.capable('batch'): batch = repo.batch() branchmap = batch.branchmap() heads = batch.heads() bookmarks = batch.listkeys('bookmarks') batch.submit() branchmap = branchmap.value heads = heads.value bookmarks = bookmarks.value else: while True: branchmap = repo.branchmap() heads = repo.heads() if heads == ['\0' * 20]: heads = [] # Some branch heads can be non-heads topologically, but if # some heads don't appear in the branchmap, then something # was pushed to the repo between branchmap() and heads() if set(heads).issubset(set(chain(*branchmap.values()))): break bookmarks = repo.listkeys('bookmarks') branchmap = BranchMap(store, branchmap, heads) unknowns = False for branch in sorted(branchmap.names()): branch_tip = branchmap.tip(branch) for head in sorted(branchmap.heads(branch)): sha1 = branchmap.git_sha1(head) if sha1 == '?': unknowns = True if head == branch_tip: continue helper.write('%s refs/heads/branches/%s/%s\n' % ( sha1, branch, head, )) if branch_tip: helper.write('%s refs/heads/branches/%s/tip\n' % ( branchmap.git_sha1(branch_tip), branch, )) for name, sha1 in sorted(bookmarks.iteritems()): ref = store.changeset_ref(sha1) helper.write( '%s refs/heads/bookmarks/%s\n' % (ref if ref else '?', name) ) if not unknowns: for tag, ref in sorted(store.tags(branchmap.heads())): helper.write('%s refs/tags/%s\n' % (ref, tag)) if '@' in bookmarks: HEAD = 'bookmarks/@' helper.write( '@refs/heads/%s HEAD\n' '\n' % HEAD ) helper.flush() elif cmd == 'option': assert len(args) == 2 name, value = args if name == 'progress': if value == 'true': cinnabar.util.progress = True helper.write('ok\n') elif value == 'false': cinnabar.util.progress = False helper.write('ok\n') else: helper.write('unsupported\n') else: helper.write('unsupported\n') helper.flush() elif cmd == 'import': try: reflog = os.path.join(git_dir, 'logs', 'refs', 'cinnabar') mkpath(reflog) open(os.path.join(reflog, 'hg2git'), 'a').close() open(os.path.join(reflog, 'manifest'), 'a').close() assert len(args) == 1 refs = args while cmd: assert cmd == 'import' cmd, args = read_cmd(helper) assert args is None or len(args) == 1 if args: refs.extend(args) except: # If anything wrong happens before we got all the import # commands, we risk git picking the existing refs/cinnabar # refs. Remove them. for line in Git.for_each_ref('refs/cinnabar/refs/heads', 'refs/cinnabar/HEAD', format='%(refname)'): Git.delete_ref(ref) raise try: def resolve_head(head): if head.startswith('refs/heads/branches/'): head = head[20:] if head[-4:] == '/tip': return branchmap.tip(head[:-4]) return head[-40:] if head.startswith('refs/heads/bookmarks/'): head = head[21:] return bookmarks[head] if head == 'HEAD': return bookmarks.get('@') or branchmap.tip('default') return None wanted_refs = {k: v for k, v in ( (h, resolve_head(h)) for h in refs) if v} heads = wanted_refs.values() if not heads: heads = branchmap.heads() # Older versions would create a symbolic ref for # refs/remote-hg/HEAD. Newer versions don't, and # Git.update_ref doesn't remove the symbolic ref, so it needs # to be removed first. # Since git symbolic-ref only throws an error when the ref is # not symbolic, just try to remove the symbolic ref every time # and ignore errors. tuple(Git.iter('symbolic-ref', '-d', 'refs/remote-hg/HEAD', stderr=open(os.devnull, 'wb'))) refs_orig = {} for line in Git.for_each_ref( 'refs/cinnabar/refs/heads', 'refs/cinnabar/HEAD', format='%(objectname) %(refname)'): sha1, ref = line.split(' ', 1) refs_orig[ref] = sha1 except: # If anything wrong happens before we actually pull, we risk # git pucking the existing refs/cinnabar refs. Remove them. # Unlike in the case above, we now have the list of refs git # is expected, so we can just remove those. for ref in refs: Git.delete_ref('refs/cinnabar/' + ref) raise try: store.init_fast_import(FastImport(sys.stdin, sys.stdout)) getbundle(repo, store, heads, branchmap) except: wanted_refs = {} raise finally: for ref, value in wanted_refs.iteritems(): ref = 'refs/cinnabar/' + ref if ref not in refs_orig or refs_orig[ref] != value: Git.update_ref(ref, store.changeset_ref(value)) for ref in refs_orig: if ref[14:] not in wanted_refs: Git.delete_ref(ref) store.close() if not remote.startswith('hg::'): prune = 'remote.%s.prune' % remote if (Git.config(prune) != 'true' and Git.config('fetch.prune') != 'true'): sys.stderr.write( 'It is recommended that you set "%(conf)s" or ' '"fetch.prune" to "true".\n' ' git config %(conf)s true\n' 'or\n' ' git config fetch.prune true\n' % {'conf': prune} ) if store.tag_changes: sys.stderr.write( '\nRun the following command to update remote tags:\n') if not remote.startswith('hg::'): sys.stderr.write(' git remote update %s\n' % remote) else: sys.stderr.write(' git fetch --tags %s\n' % remote) elif cmd == 'push': if not remote.startswith('hg::'): data_pref = 'remote.%s.cinnabar-data' % remote data = Git.config(data_pref) or 'phase' else: data = 'phase' if data not in ('never', 'phase', 'always'): sys.stderr.write('Invalid value for %s: %s\n' % (data_pref, data)) return 1 refspecs = [] refspecs.extend(args) while True: cmd, args = read_cmd(helper) if not cmd: break assert cmd == 'push' refspecs.extend(args) pushes = {s.lstrip('+'): (d, s.startswith('+')) for s, d in (r.split(':', 1) for r in refspecs)} if isinstance(repo, bundlerepo): for source, (dest, force) in pushes.iteritems(): helper.write('error %s Cannot push to a bundle file\n' % dest) helper.write('\n') helper.flush() else: repo_heads = branchmap.heads() PushStore.adopt(store) pushed = push(repo, store, pushes, repo_heads, branchmap.names()) status = {} for source, (dest, _) in pushes.iteritems(): if dest.startswith('refs/tags/'): if source: status[dest] = 'Pushing tags is unsupported' else: status[dest] = \ 'Deleting remote tags is unsupported' continue if not dest.startswith('refs/heads/bookmarks/'): if source: status[dest] = bool(len(pushed)) else: status[dest] = \ 'Deleting remote branches is unsupported' continue name = dest[21:] if source: source = store.hg_changeset(Git.resolve_ref(source)) \ or '' status[dest] = repo.pushkey( 'bookmarks', name, bookmarks.get(name, ''), source) for source, (dest, force) in pushes.iteritems(): if status[dest] is True: helper.write('ok %s\n' % dest) elif status[dest]: helper.write('error %s %s\n' % (dest, status[dest])) else: helper.write('error %s nothing changed on remote\n' % dest) helper.write('\n') helper.flush() if not pushed: data = False elif data == 'always': data = True elif data == 'phase': phases = repo.listkeys('phases') drafts = {} if not phases.get('publishing', False): drafts = set(p for p, is_draft in phases.iteritems() if int(is_draft)) if not drafts: data = True else: def draft_commits(): for d in drafts: c = store.changeset_ref(d) if c: yield '^%s^@' % c for h in pushed.heads(): yield h args = ['rev-list', '--ancestry-path', '--topo-order', '--stdin'] pushed_drafts = tuple( Git.iter(*args, stdin=draft_commits())) # Theoretically, we could have commits with no # metadata that the remote declares are public, while # the rest of our push is in a draft state. That is # however so unlikely that it's not worth the effort # to support partial metadata storage. data = not bool(pushed_drafts) elif data == 'never': data = False store.close(rollback=not data) store.close()
def main(args): logger = logging.getLogger('-') logger.info(args) assert len(args) == 2 remote, url = args git_dir = os.environ.get('GIT_DIR') if Git.config('core.ignorecase', 'bool') == 'true': sys.stderr.write( 'Your git configuration has core.ignorecase set to "true".\n' 'Usually, this means git detected the file system is case ' 'insensitive.\n' 'Git-cinnabar does not support this setup.\n' 'Either use a case sensitive file system or set ' 'core.ignorecase to "false".\n') git_work_tree = os.path.dirname(git_dir) if os.path.abspath(os.getcwd() + os.sep).startswith( os.path.abspath(git_work_tree) + os.sep) or \ remote == 'hg::' + url or tuple( Git.for_each_ref('refs/remotes/%s' % remote)): sys.stderr.write('Use the following command to reclone:\n' ' git cinnabar reclone\n') else: sys.stderr.write( 'Use the following command to clone:\n' ' git -c core.ignorecase=false clone%(args)s hg::%(url)s ' '%(dir)s\n' % { 'dir': git_work_tree, 'url': url, 'args': ' -o ' + remote if remote != 'origin' else '' }) return 1 repo = get_repo(url) store = GitHgStore() logger.info(LazyString(lambda: '%s' % store.heads())) helper = IOLogger(logging.getLogger('remote-helper'), sys.stdin, sys.stdout) branchmap = None bookmarks = {} HEAD = 'branches/default/tip' while True: cmd, args = read_cmd(helper) if not cmd: break if cmd == 'capabilities': assert not args helper.write('option\n' 'import\n' 'bidi-import\n' 'push\n' 'refspec refs/heads/branches/*:' 'refs/cinnabar/refs/heads/branches/*\n' 'refspec refs/heads/bookmarks/*:' 'refs/cinnabar/refs/heads/bookmarks/*\n' 'refspec HEAD:refs/cinnabar/HEAD\n' '\n') helper.flush() elif cmd == 'list': assert not args or args == ['for-push'] if repo.capable('batch'): batch = repo.batch() branchmap = batch.branchmap() heads = batch.heads() bookmarks = batch.listkeys('bookmarks') batch.submit() branchmap = branchmap.value heads = heads.value bookmarks = bookmarks.value else: while True: branchmap = repo.branchmap() heads = repo.heads() if heads == ['\0' * 20]: heads = [] # Some branch heads can be non-heads topologically, but if # some heads don't appear in the branchmap, then something # was pushed to the repo between branchmap() and heads() if set(heads).issubset(set(chain(*branchmap.values()))): break bookmarks = repo.listkeys('bookmarks') branchmap = BranchMap(store, branchmap, heads) unknowns = False for branch in sorted(branchmap.names()): branch_tip = branchmap.tip(branch) for head in sorted(branchmap.heads(branch)): sha1 = branchmap.git_sha1(head) if sha1 == '?': unknowns = True if head == branch_tip: continue helper.write('%s refs/heads/branches/%s/%s\n' % ( sha1, branch, head, )) if branch_tip: helper.write('%s refs/heads/branches/%s/tip\n' % ( branchmap.git_sha1(branch_tip), branch, )) for name, sha1 in sorted(bookmarks.iteritems()): ref = store.changeset_ref(sha1) helper.write('%s refs/heads/bookmarks/%s\n' % (ref if ref else '?', name)) if not unknowns: for tag, ref in sorted(store.tags(branchmap.heads())): helper.write('%s refs/tags/%s\n' % (ref, tag)) if '@' in bookmarks: HEAD = 'bookmarks/@' helper.write('@refs/heads/%s HEAD\n' '\n' % HEAD) helper.flush() elif cmd == 'option': assert len(args) == 2 name, value = args if name == 'progress': if value == 'true': cinnabar.util.progress = True helper.write('ok\n') elif value == 'false': cinnabar.util.progress = False helper.write('ok\n') else: helper.write('unsupported\n') else: helper.write('unsupported\n') helper.flush() elif cmd == 'import': try: reflog = os.path.join(git_dir, 'logs', 'refs', 'cinnabar') mkpath(reflog) open(os.path.join(reflog, 'hg2git'), 'a').close() open(os.path.join(reflog, 'manifest'), 'a').close() assert len(args) == 1 refs = args while cmd: assert cmd == 'import' cmd, args = read_cmd(helper) assert args is None or len(args) == 1 if args: refs.extend(args) except: # If anything wrong happens before we got all the import # commands, we risk git picking the existing refs/cinnabar # refs. Remove them. for line in Git.for_each_ref('refs/cinnabar/refs/heads', 'refs/cinnabar/HEAD', format='%(refname)'): Git.delete_ref(ref) raise try: def resolve_head(head): if head.startswith('refs/heads/branches/'): head = head[20:] if head[-4:] == '/tip': return branchmap.tip(head[:-4]) return head[-40:] if head.startswith('refs/heads/bookmarks/'): head = head[21:] return bookmarks[head] if head == 'HEAD': return bookmarks.get('@') or branchmap.tip('default') return None wanted_refs = { k: v for k, v in ((h, resolve_head(h)) for h in refs) if v } heads = wanted_refs.values() if not heads: heads = branchmap.heads() # Older versions would create a symbolic ref for # refs/remote-hg/HEAD. Newer versions don't, and # Git.update_ref doesn't remove the symbolic ref, so it needs # to be removed first. # Since git symbolic-ref only throws an error when the ref is # not symbolic, just try to remove the symbolic ref every time # and ignore errors. tuple( Git.iter('symbolic-ref', '-d', 'refs/remote-hg/HEAD', stderr=open(os.devnull, 'wb'))) refs_orig = {} for line in Git.for_each_ref( 'refs/cinnabar/refs/heads', 'refs/cinnabar/HEAD', format='%(objectname) %(refname)'): sha1, ref = line.split(' ', 1) refs_orig[ref] = sha1 except: # If anything wrong happens before we actually pull, we risk # git pucking the existing refs/cinnabar refs. Remove them. # Unlike in the case above, we now have the list of refs git # is expected, so we can just remove those. for ref in refs: Git.delete_ref('refs/cinnabar/' + ref) raise try: store.init_fast_import(FastImport(sys.stdin, sys.stdout)) getbundle(repo, store, heads, branchmap) except: wanted_refs = {} raise finally: for ref, value in wanted_refs.iteritems(): ref = 'refs/cinnabar/' + ref if ref not in refs_orig or refs_orig[ref] != value: Git.update_ref(ref, store.changeset_ref(value)) for ref in refs_orig: if ref[14:] not in wanted_refs: Git.delete_ref(ref) store.close() if not remote.startswith('hg::'): prune = 'remote.%s.prune' % remote if (Git.config(prune) != 'true' and Git.config('fetch.prune') != 'true'): sys.stderr.write( 'It is recommended that you set "%(conf)s" or ' '"fetch.prune" to "true".\n' ' git config %(conf)s true\n' 'or\n' ' git config fetch.prune true\n' % {'conf': prune}) if store.tag_changes: sys.stderr.write( '\nRun the following command to update remote tags:\n') if not remote.startswith('hg::'): sys.stderr.write(' git remote update %s\n' % remote) else: sys.stderr.write(' git fetch --tags %s\n' % remote) elif cmd == 'push': if not remote.startswith('hg::'): data_pref = 'remote.%s.cinnabar-data' % remote data = Git.config(data_pref) or 'phase' else: data = 'phase' if data not in ('never', 'phase', 'always'): sys.stderr.write('Invalid value for %s: %s\n' % (data_pref, data)) return 1 refspecs = [] refspecs.extend(args) while True: cmd, args = read_cmd(helper) if not cmd: break assert cmd == 'push' refspecs.extend(args) pushes = { s.lstrip('+'): (d, s.startswith('+')) for s, d in (r.split(':', 1) for r in refspecs) } if isinstance(repo, bundlerepo): for source, (dest, force) in pushes.iteritems(): helper.write('error %s Cannot push to a bundle file\n' % dest) helper.write('\n') helper.flush() else: repo_heads = branchmap.heads() PushStore.adopt(store) pushed = push(repo, store, pushes, repo_heads, branchmap.names()) status = {} for source, (dest, _) in pushes.iteritems(): if dest.startswith('refs/tags/'): if source: status[dest] = 'Pushing tags is unsupported' else: status[dest] = \ 'Deleting remote tags is unsupported' continue if not dest.startswith('refs/heads/bookmarks/'): if source: status[dest] = bool(len(pushed)) else: status[dest] = \ 'Deleting remote branches is unsupported' continue name = dest[21:] if source: source = store.hg_changeset(Git.resolve_ref(source)) \ or '' status[dest] = repo.pushkey('bookmarks', name, bookmarks.get(name, ''), source) for source, (dest, force) in pushes.iteritems(): if status[dest] is True: helper.write('ok %s\n' % dest) elif status[dest]: helper.write('error %s %s\n' % (dest, status[dest])) else: helper.write('error %s nothing changed on remote\n' % dest) helper.write('\n') helper.flush() if not pushed: data = False elif data == 'always': data = True elif data == 'phase': phases = repo.listkeys('phases') drafts = {} if not phases.get('publishing', False): drafts = set(p for p, is_draft in phases.iteritems() if int(is_draft)) if not drafts: data = True else: def draft_commits(): for d in drafts: c = store.changeset_ref(d) if c: yield '^%s^@' % c for h in pushed.heads(): yield h args = [ 'rev-list', '--ancestry-path', '--topo-order', '--stdin' ] pushed_drafts = tuple( Git.iter(*args, stdin=draft_commits())) # Theoretically, we could have commits with no # metadata that the remote declares are public, while # the rest of our push is in a draft state. That is # however so unlikely that it's not worth the effort # to support partial metadata storage. data = not bool(pushed_drafts) elif data == 'never': data = False store.close(rollback=not data) store.close()
def test_module_version(self): module = one(Git.iter( 'ls-tree', 'HEAD', 'cinnabar', cwd=os.path.join(os.path.dirname(__file__), '..'))) self.assertEqual(CmdVersion.module_version(), split_ls_tree(module)[2])
def test_helper_version(self): helper = one(Git.iter( 'ls-tree', 'HEAD', 'helper', cwd=os.path.join(os.path.dirname(__file__), '..'))) self.assertEqual(CmdVersion.helper_version()[1], split_ls_tree(helper)[2])
def helper_hash(head='HEAD'): from cinnabar.git import Git, split_ls_tree from cinnabar.util import one return split_ls_tree(one(Git.iter( 'ls-tree', head, 'helper', cwd=os.path.join(os.path.dirname(__file__), '..'))))[2].decode()