Example #1
0
def data(args):
    '''dump the contents of a mercurial revision'''

    store = GitHgStore()
    if args.changeset and args.manifest:
        print >>sys.stderr, 'Cannot use both -c and -m.'
        return 1
    if args.changeset:
        sys.stdout.write(store.changeset(args.rev).raw_data)
    elif args.manifest:
        sys.stdout.write(store.manifest(args.rev).data)
    else:
        sys.stdout.write(store.file(args.rev).raw_data)
    store.close()
Example #2
0
def data(args):
    '''dump the contents of a mercurial revision'''

    store = GitHgStore()
    if args.changeset and args.manifest:
        print('Cannot use both -c and -m.', file=sys.stderr)
        return 1
    if args.changeset:
        sys.stdout.write(store.changeset(args.rev).raw_data)
    elif args.manifest:
        sys.stdout.write(store.manifest(args.rev).data)
    else:
        sys.stdout.write(store.file(args.rev).raw_data)
    store.close()
Example #3
0
def data(args):
    '''dump the contents of a mercurial revision'''

    try:
        store = GitHgStore()
    except UpgradeException as e:
        print >> sys.stderr, e.message
        return 1
    if args.changeset and args.manifest:
        print >> sys.stderr, 'Cannot use both -c and -m.'
        return 1
    if args.changeset:
        sys.stdout.write(store.changeset(args.rev).raw_data)
    elif args.manifest:
        sys.stdout.write(store.manifest(args.rev).data)
    else:
        sys.stdout.write(store.file(args.rev).raw_data)
    store.close()
Example #4
0
def main(args):
    cmd = args.pop(0)
    if cmd == 'data':
        store = GitHgStore()
        if args[0] == '-c':
            sys.stdout.write(store.changeset(args[1]).data)
        elif args[0] == '-m':
            sys.stdout.write(store.manifest(args[1]).data)
        store.close()
    elif cmd == 'fsck':
        return fsck(args)
    elif cmd == 'reclone':
        for ref in Git.for_each_ref('refs/cinnabar',
                                    'refs/remote-hg',
                                    'refs/notes/cinnabar',
                                    'refs/notes/remote-hg/git2hg',
                                    format='%(refname)'):
            Git.delete_ref(ref)
        Git.close()

        for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'):
            config, url = line.split()
            name = config[len('remote.'):-len('.url')]
            skip_pref = 'remote.%s.skipDefaultUpdate' % name
            if (url.startswith('hg::')
                    and Git.config(skip_pref, 'bool') != 'true'):
                Git.run('remote', 'update', '--prune', name)

        print 'Please note that reclone left your local branches untouched.'
        print 'They may be based on entirely different commits.'
    elif cmd == 'hg2git':
        for arg in args:
            print GitHgHelper.hg2git(arg)
    elif cmd == 'git2hg':
        for arg in args:
            data = GitHgHelper.git2hg(arg)
            if data:
                data = ChangesetData.parse(data)
                print data.get('changeset', NULL_NODE_ID)
            else:
                print NULL_NODE_ID
    else:
        print >> sys.stderr, 'Unknown command:', cmd
        return 1
Example #5
0
def main(args):
    cmd = args.pop(0)
    if cmd == 'data':
        store = GitHgStore()
        if args[0] == '-c':
            sys.stdout.write(store.changeset(args[1]).data)
        elif args[0] == '-m':
            sys.stdout.write(store.manifest(args[1]).data)
        store.close()
    elif cmd == 'fsck':
        return fsck(args)
    elif cmd == 'reclone':
        for ref in Git.for_each_ref('refs/cinnabar', 'refs/remote-hg',
                                    'refs/notes/cinnabar',
                                    'refs/notes/remote-hg/git2hg',
                                    format='%(refname)'):
            Git.delete_ref(ref)
        Git.close()

        for line in Git.iter('config', '--get-regexp', 'remote\..*\.url'):
            config, url = line.split()
            name = config[len('remote.'):-len('.url')]
            skip_pref = 'remote.%s.skipDefaultUpdate' % name
            if (url.startswith('hg::') and
                    Git.config(skip_pref, 'bool') != 'true'):
                Git.run('remote', 'update', '--prune', name)

        print 'Please note that reclone left your local branches untouched.'
        print 'They may be based on entirely different commits.'
    elif cmd == 'hg2git':
        for arg in args:
            print GitHgHelper.hg2git(arg)
    elif cmd == 'git2hg':
        for arg in args:
            data = GitHgHelper.git2hg(arg)
            if data:
                data = ChangesetData.parse(data)
                print data.get('changeset', NULL_NODE_ID)
            else:
                print NULL_NODE_ID
    else:
        print >>sys.stderr, 'Unknown command:', cmd
        return 1
Example #6
0
def upgrade(args):
    '''upgrade cinnabar metadata'''

    try:
        store = GitHgStore()
        print('No metadata to upgrade')
        return 2
    except UpgradeAbort:
        store = UpgradeGitHgStore()

    if not GitHgHelper.upgrade():
        print('Cannot finish upgrading... You may need to reclone.')
        return 1

    print('Finalizing upgrade...')
    store.close(refresh=store.METADATA_REFS)
    print('You may want to run `git cinnabar fsck` to ensure the upgrade '
          'went well.\n')
    return 0
Example #7
0
def upgrade(args):
    '''upgrade cinnabar metadata'''

    try:
        store = GitHgStore()
        print 'No metadata to upgrade'
        return 2
    except UpgradeAbort:
        store = UpgradeGitHgStore()

    if not GitHgHelper.upgrade():
        print 'Cannot finish upgrading... You may need to reclone.'
        return 1

    print 'Finalizing upgrade...'
    store.close()
    print(
        'You may want to run `git cinnabar fsck --manifests --files` to '
        'ensure the upgrade went well.\n'
        'Please be aware this might take a while.')
    return 0
Example #8
0
def upgrade(args):
    '''upgrade cinnabar metadata'''

    try:
        store = GitHgStore()
        print 'No metadata to upgrade'
        return 2
    except UpgradeAbort:
        store = UpgradeGitHgStore()

    if not GitHgHelper.upgrade():
        print 'Cannot finish upgrading... You may need to reclone.'
        return 1

    print 'Finalizing upgrade...'
    store.close(refresh=store.METADATA_REFS)
    print (
        'You may want to run `git cinnabar fsck` to ensure the upgrade '
        'went well.\n'
    )
    return 0
Example #9
0
def unbundle(args):
    '''apply a mercurial bundle to the repository'''
    # Make git emit its error when the current directory is not in a git repo.
    proc = GitProcess('rev-parse')
    ret = proc.wait()
    if ret:
        return ret
    remote = Remote(b'', fsencode(args.url))
    if remote.parsed_url.scheme not in (b'file', b'http', b'https'):
        logging.error('%s urls are not supported.' % remote.parsed_url.scheme)
        return 1
    if args.clonebundle:
        repo = get_repo(remote)
        if not repo.capable(b'clonebundles'):
            logging.error('Repository does not support clonebundles')
            return 1
        bundle = get_clonebundle(repo)
    else:
        bundle = get_bundle(remote.url)

    store = GitHgStore()
    GRAFT = {
        None: False,
        b'false': False,
        b'true': True,
    }
    try:
        graft = Git.config('cinnabar.graft', values=GRAFT)
    except InvalidConfig as e:
        logging.error(str(e))
        return 1
    if graft:
        store.prepare_graft()
    bundle = unbundler(bundle)
    apply_bundle = BundleApplier(bundle)
    del bundle
    apply_bundle(store)
    store.close()
Example #10
0
def fsck(args):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--manifests', action='store_true',
        help='Validate manifests hashes')
    parser.add_argument(
        '--files', action='store_true',
        help='Validate files hashes')
    parser.add_argument(
        'commit', nargs='*',
        help='Specific commit or changeset to check')
    args = parser.parse_args(args)

    status = {
        'broken': False,
        'fixed': False,
    }

    def info(message):
        sys.stderr.write('\r')
        print message

    def fix(message):
        status['fixed'] = True
        info(message)

    def report(message):
        status['broken'] = True
        info(message)

    store = GitHgStore()
    store.init_fast_import(lambda: FastImport())

    if args.commit:
        all_hg2git = {}
        all_notes = set()
        commits = set()
        all_git_commits = {}

        for c in args.commit:
            data = store.read_changeset_data(c)
            if data:
                all_notes.add(c)
                commits.add(c)
                c = data['changeset']
            commit = GitHgHelper.hg2git(c)
            if commit == NULL_NODE_ID and not data:
                info('Unknown commit or changeset: %s' % c)
                return 1
            if commit != NULL_NODE_ID:
                all_hg2git[c] = commit, 'commit'
            if not data:
                data = store.read_changeset_data(commit)
                commits.add(commit)
                if data:
                    all_notes.add(commit)

        all_git_commits = Git.iter(
            'log', '--no-walk=unsorted', '--stdin', '--format=%T %H',
            stdin=commits)
    else:
        all_hg2git = {
            path.replace('/', ''): (filesha1, intern(typ))
            for mode, typ, filesha1, path in
            progress_iter('Reading %d mercurial to git mappings',
                          Git.ls_tree('refs/cinnabar/hg2git', recursive=True))
        }

        all_notes = set(path.replace('/', '') for mode, typ, filesha1, path in
                        progress_iter(
                            'Reading %d commit to changeset mappings',
                            Git.ls_tree('refs/notes/cinnabar',
                                        recursive=True)))

        manifest_commits = OrderedDict((m, None) for m in progress_iter(
            'Reading %d manifest trees',
            Git.iter('rev-list', '--full-history',
                     '--topo-order', 'refs/cinnabar/manifest'))
        )

        all_git_heads = Git.for_each_ref('refs/cinnabar/branches',
                                         format='%(refname)')

        all_git_commits = Git.iter('log', '--topo-order', '--full-history',
                                   '--reverse', '--stdin', '--format=%T %H',
                                   stdin=all_git_heads)

    store._hg2git_cache = {p: s for p, (s, t) in all_hg2git.iteritems()}

    seen_changesets = set()
    seen_manifests = set()
    seen_manifest_refs = {}
    seen_files = set()
    seen_notes = set()

    hg_manifest = None

    dag = gitdag()

    for line in progress_iter('Checking %d changesets', all_git_commits):
        tree, node = line.split(' ')
        if node not in all_notes:
            report('Missing note for git commit: ' + node)
            continue
        seen_notes.add(node)

        changeset_data = store.read_changeset_data(node)
        changeset = changeset_data['changeset']
        if 'extra' in changeset_data:
            extra = changeset_data['extra']
            header, message = GitHgHelper.cat_file(
                'commit', node).split('\n\n', 1)
            header = dict(l.split(' ', 1) for l in header.splitlines())
            if 'committer' in extra:
                committer_info = store.hg_author_info(header['committer'])
                committer = '%s %d %d' % committer_info
                if (committer != extra['committer'] and
                        header['committer'] != extra['committer'] and
                        committer_info[0] != extra['committer']):
                    report('Committer mismatch between commit and metadata for'
                           ' changeset %s' % changeset)
                if committer == extra['committer']:
                    fix('Fixing useless committer metadata for changeset %s'
                        % changeset)
                    del changeset_data['extra']['committer']
                    store._changesets[changeset] = LazyString(node)
            if header['committer'] != header['author'] and not extra:
                fix('Fixing useless empty extra metadata for changeset %s'
                    % changeset)
                del changeset_data['extra']
                store._changesets[changeset] = LazyString(node)

        seen_changesets.add(changeset)
        changeset_ref = store.changeset_ref(changeset)
        if not changeset_ref:
            report('Missing changeset in hg2git branch: %s' % changeset)
        elif str(changeset_ref) != node:
            report('Commit mismatch for changeset %s\n'
                   '  hg2git: %s\n  commit: %s'
                   % (changeset, changeset_ref, node))

        hg_changeset = store.changeset(changeset, include_parents=True)
        sha1 = hg_changeset.sha1
        if hg_changeset.node != sha1:
            try_fixup = False
            if (changeset, sha1) in (
                ('8c557b7c03a4a753e5c163038f04862e9f65fce1',
                 '249b59139de8e08abeb6c4e261a137c756e7af0e'),
                ('ffdee4a4eb7fc7cae80dfc4cb2fe0c3178773dcf',
                 '415e9d2eac83d508bf58a4df585c5f6b2b0f44ed'),
            ):
                header = hg_changeset.data.split('\n', 4)
                start = sum(len(h) for h in header[:3]) + 1
                changeset_data['patch'] = ((start, start + 1, '1'),)
                try_fixup = True

            # Some know cases of corruptions involve a whitespace after the
            # timezone. Adding an empty extra metadata works around those.
            elif 'extra' not in changeset_data:
                changeset_data['extra'] = {}
                try_fixup = True

            if try_fixup:
                hg_changeset = store.changeset(changeset, include_parents=True)
                sha1 = hg_changeset.sha1
                if hg_changeset.node == sha1:
                    fix('Fixing known sha1 mismatch for changeset %s' %
                        changeset)
                    store._changesets[changeset] = LazyString(node)

        if hg_changeset.node != sha1:
            report('Sha1 mismatch for changeset %s' % changeset)

        dag.add(hg_changeset.node,
                (hg_changeset.parent1, hg_changeset.parent2),
                changeset_data.get('extra', {}).get('branch', 'default'))

        manifest = changeset_data['manifest']
        if manifest in seen_manifests:
            continue
        seen_manifests.add(manifest)
        manifest_ref = store.manifest_ref(manifest)
        if manifest_ref:
            seen_manifest_refs[manifest_ref] = manifest
        if not manifest_ref:
            report('Missing manifest in hg2git branch: %s' % manifest)
        elif not args.commit and manifest_ref not in manifest_commits:
            report('Missing manifest commit in manifest branch: %s' %
                   manifest_ref)

        if args.manifests or args.files:
            parents = tuple(
                store.read_changeset_data(store.changeset_ref(p))['manifest']
                for p in (hg_changeset.parent1, hg_changeset.parent2)
                if p != NULL_NODE_ID
            )

        if args.manifests:
            try:
                with GitHgHelper.query('check-manifest', manifest,
                                       *parents) as stdout:
                    if stdout.readline().strip() != 'ok':
                        report('Sha1 mismatch for manifest %s' % manifest)
            except NoHelperException:
                hg_manifest = store.manifest(manifest)
                hg_manifest.set_parents(*parents)
                if hg_manifest.node != hg_manifest.sha1:
                    report('Sha1 mismatch for manifest %s' % manifest)

        git_ls = one(Git.ls_tree(manifest_ref, 'git'))
        if git_ls:
            mode, typ, sha1, path = git_ls
        else:
            header, message = GitHgHelper.cat_file(
                'commit', manifest_ref).split('\n\n', 1)
            header = dict(l.split(' ', 1) for l in header.splitlines())
            if header['tree'] == EMPTY_TREE:
                sha1 = EMPTY_TREE
            else:
                report('Missing git tree in manifest commit %s' % manifest_ref)
                sha1 = None
        if sha1 and sha1 != tree:
            report('Tree mismatch between manifest commit %s and commit %s'
                   % (manifest_ref, node))

        if args.files:
            changes = get_changes(
                manifest_ref, tuple(store.manifest_ref(p) for p in parents),
                'hg')
            for path, hg_file, hg_fileparents in changes:
                if hg_file != NULL_NODE_ID and hg_file not in seen_files:
                    file = store.file(hg_file)
                    file.set_parents(*hg_fileparents)
                    if file.node != file.sha1:
                        report('Sha1 mismatch for file %s in manifest %s'
                               % (hg_file, manifest_ref))
                    seen_files.add(hg_file)

    if args.files:
        all_hg2git = set(all_hg2git.iterkeys())
    else:
        all_hg2git = set(k for k, (s, t) in all_hg2git.iteritems()
                         if t == 'commit')

    adjusted = {}
    if not args.commit:
        dangling = set(manifest_commits) - set(seen_manifest_refs)
        if dangling:
            def iter_manifests():
                removed_one = False
                yielded = False
                previous = None
                for obj in reversed(manifest_commits):
                    if obj in dangling:
                        fix('Removing metadata commit %s with no hg2git entry'
                            % obj)
                        removed_one = True
                    else:
                        if removed_one:
                            yield obj, previous
                            yielded = True
                        previous = obj

                if removed_one and not yielded:
                    yield obj, False

            for obj, parent in progress_iter('Adjusting %d metadata commits',
                                             iter_manifests()):
                mark = store._fast_import.new_mark()
                if parent is False:
                    Git.update_ref('refs/cinnabar/manifest', obj)
                    continue
                elif parent:
                    parents = (adjusted.get(parent, parent),)
                with store._fast_import.commit(
                        ref='refs/cinnabar/manifest',
                        parents=parents, mark=mark) as commit:
                    mode, typ, tree, path = store._fast_import.ls(obj)
                    commit.filemodify('', tree, typ='tree')
                adjusted[obj] = Mark(mark)

    dangling = all_hg2git - seen_changesets - seen_manifests - seen_files
    if dangling or adjusted:
        with store._fast_import.commit(
                ref='refs/cinnabar/hg2git',
                parents=('refs/cinnabar/hg2git^0',)) as commit:
            for obj in dangling:
                fix('Removing dangling metadata for ' + obj)
                commit.filedelete(sha1path(obj))
            for obj, mark in progress_iter(
                    'Updating hg2git for %d metadata commits',
                    adjusted.iteritems()):
                commit.filemodify(sha1path(seen_manifest_refs[obj]), mark,
                                  typ='commit')

    dangling = all_notes - seen_notes
    if dangling:
        with store._fast_import.commit(
                ref='refs/notes/cinnabar',
                parents=('refs/notes/cinnabar^0',)) as commit:
            for c in dangling:
                fix('Removing dangling note for commit ' + c)
                # That's brute force, but meh.
                for l in range(0, 10):
                    commit.filedelete(sha1path(c, l))

    if status['broken']:
        info('Your git-cinnabar repository appears to be corrupted. There\n'
             'are known issues in older revisions that have been fixed.\n'
             'Please try running the following command to reset:\n'
             '  git cinnabar reclone\n\n'
             'Please note this command may change the commit sha1s. Your\n'
             'local branches will however stay untouched.\n'
             'Please report any corruption that fsck would detect after a\n'
             'reclone.')

    if not args.commit:
        info('Checking head references...')
        computed_heads = defaultdict(set)
        for branch, head in dag.all_heads():
            computed_heads[branch].add(head)

        for branch in sorted(dag.tags()):
            stored_heads = store.heads({branch})
            for head in computed_heads[branch] - stored_heads:
                fix('Adding missing head %s in branch %s' %
                    (head, branch))
                store.add_head(head)
            for head in stored_heads - computed_heads[branch]:
                fix('Removing non-head reference to %s in branch %s' %
                    (head, branch))
                store._hgheads.remove((branch, head))

    store.close()

    if status['broken']:
        return 1
    if status['fixed']:
        return 2
    return 0
Example #11
0
def fsck(args):
    parser = argparse.ArgumentParser()
    parser.add_argument('--manifests',
                        action='store_true',
                        help='Validate manifests hashes')
    parser.add_argument('--files',
                        action='store_true',
                        help='Validate files hashes')
    parser.add_argument('commit',
                        nargs='*',
                        help='Specific commit or changeset to check')
    args = parser.parse_args(args)

    status = {
        'broken': False,
        'fixed': False,
    }

    def info(message):
        sys.stderr.write('\r')
        print message

    def fix(message):
        status['fixed'] = True
        info(message)

    def report(message):
        status['broken'] = True
        info(message)

    store = GitHgStore()
    store.init_fast_import(lambda: FastImport())

    if args.commit:
        all_hg2git = {}
        all_notes = set()
        commits = set()
        all_git_commits = {}

        for c in args.commit:
            data = store.read_changeset_data(c)
            if data:
                all_notes.add(c)
                commits.add(c)
                c = data['changeset']
            commit = GitHgHelper.hg2git(c)
            if commit == NULL_NODE_ID and not data:
                info('Unknown commit or changeset: %s' % c)
                return 1
            if commit != NULL_NODE_ID:
                all_hg2git[c] = commit, 'commit'
            if not data:
                data = store.read_changeset_data(commit)
                commits.add(commit)
                if data:
                    all_notes.add(commit)

        all_git_commits = Git.iter('log',
                                   '--no-walk=unsorted',
                                   '--stdin',
                                   '--format=%T %H',
                                   stdin=commits)
    else:
        all_hg2git = {
            path.replace('/', ''): (filesha1, intern(typ))
            for mode, typ, filesha1, path in progress_iter(
                'Reading %d mercurial to git mappings',
                Git.ls_tree('refs/cinnabar/hg2git', recursive=True))
        }

        all_notes = set(
            path.replace('/', '')
            for mode, typ, filesha1, path in progress_iter(
                'Reading %d commit to changeset mappings',
                Git.ls_tree('refs/notes/cinnabar', recursive=True)))

        manifest_commits = OrderedDict((m, None) for m in progress_iter(
            'Reading %d manifest trees',
            Git.iter('rev-list', '--full-history', '--topo-order',
                     'refs/cinnabar/manifest')))

        all_git_heads = Git.for_each_ref('refs/cinnabar/branches',
                                         format='%(refname)')

        all_git_commits = Git.iter('log',
                                   '--topo-order',
                                   '--full-history',
                                   '--reverse',
                                   '--stdin',
                                   '--format=%T %H',
                                   stdin=all_git_heads)

    store._hg2git_cache = {p: s for p, (s, t) in all_hg2git.iteritems()}

    seen_changesets = set()
    seen_manifests = set()
    seen_manifest_refs = {}
    seen_files = set()
    seen_notes = set()

    hg_manifest = None

    dag = gitdag()

    for line in progress_iter('Checking %d changesets', all_git_commits):
        tree, node = line.split(' ')
        if node not in all_notes:
            report('Missing note for git commit: ' + node)
            continue
        seen_notes.add(node)

        changeset_data = store.read_changeset_data(node)
        changeset = changeset_data['changeset']
        if 'extra' in changeset_data:
            extra = changeset_data['extra']
            header, message = GitHgHelper.cat_file('commit',
                                                   node).split('\n\n', 1)
            header = dict(l.split(' ', 1) for l in header.splitlines())
            if 'committer' in extra:
                committer_info = store.hg_author_info(header['committer'])
                committer = '%s %d %d' % committer_info
                if (committer != extra['committer']
                        and header['committer'] != extra['committer']
                        and committer_info[0] != extra['committer']):
                    report('Committer mismatch between commit and metadata for'
                           ' changeset %s' % changeset)
                if committer == extra['committer']:
                    fix('Fixing useless committer metadata for changeset %s' %
                        changeset)
                    del changeset_data['extra']['committer']
                    store._changesets[changeset] = LazyString(node)
            if header['committer'] != header['author'] and not extra:
                fix('Fixing useless empty extra metadata for changeset %s' %
                    changeset)
                del changeset_data['extra']
                store._changesets[changeset] = LazyString(node)

        seen_changesets.add(changeset)
        changeset_ref = store.changeset_ref(changeset)
        if not changeset_ref:
            report('Missing changeset in hg2git branch: %s' % changeset)
        elif str(changeset_ref) != node:
            report('Commit mismatch for changeset %s\n'
                   '  hg2git: %s\n  commit: %s' %
                   (changeset, changeset_ref, node))

        hg_changeset = store.changeset(changeset, include_parents=True)
        sha1 = hg_changeset.sha1
        if hg_changeset.node != sha1:
            try_fixup = False
            if (changeset, sha1) in (
                ('8c557b7c03a4a753e5c163038f04862e9f65fce1',
                 '249b59139de8e08abeb6c4e261a137c756e7af0e'),
                ('ffdee4a4eb7fc7cae80dfc4cb2fe0c3178773dcf',
                 '415e9d2eac83d508bf58a4df585c5f6b2b0f44ed'),
            ):
                header = hg_changeset.data.split('\n', 4)
                start = sum(len(h) for h in header[:3]) + 1
                changeset_data['patch'] = ((start, start + 1, '1'), )
                try_fixup = True

            # Some know cases of corruptions involve a whitespace after the
            # timezone. Adding an empty extra metadata works around those.
            elif 'extra' not in changeset_data:
                changeset_data['extra'] = {}
                try_fixup = True

            if try_fixup:
                hg_changeset = store.changeset(changeset, include_parents=True)
                sha1 = hg_changeset.sha1
                if hg_changeset.node == sha1:
                    fix('Fixing known sha1 mismatch for changeset %s' %
                        changeset)
                    store._changesets[changeset] = LazyString(node)

        if hg_changeset.node != sha1:
            report('Sha1 mismatch for changeset %s' % changeset)

        dag.add(hg_changeset.node,
                (hg_changeset.parent1, hg_changeset.parent2),
                changeset_data.get('extra', {}).get('branch', 'default'))

        manifest = changeset_data['manifest']
        if manifest in seen_manifests:
            continue
        seen_manifests.add(manifest)
        manifest_ref = store.manifest_ref(manifest)
        if manifest_ref:
            seen_manifest_refs[manifest_ref] = manifest
        if not manifest_ref:
            report('Missing manifest in hg2git branch: %s' % manifest)
        elif not args.commit and manifest_ref not in manifest_commits:
            report('Missing manifest commit in manifest branch: %s' %
                   manifest_ref)

        if args.manifests or args.files:
            parents = tuple(
                store.read_changeset_data(store.changeset_ref(p))['manifest']
                for p in (hg_changeset.parent1, hg_changeset.parent2)
                if p != NULL_NODE_ID)

        if args.manifests:
            try:
                with GitHgHelper.query('check-manifest', manifest,
                                       *parents) as stdout:
                    if stdout.readline().strip() != 'ok':
                        report('Sha1 mismatch for manifest %s' % manifest)
            except NoHelperException:
                hg_manifest = store.manifest(manifest)
                hg_manifest.set_parents(*parents)
                if hg_manifest.node != hg_manifest.sha1:
                    report('Sha1 mismatch for manifest %s' % manifest)

        git_ls = one(Git.ls_tree(manifest_ref, 'git'))
        if git_ls:
            mode, typ, sha1, path = git_ls
        else:
            header, message = GitHgHelper.cat_file('commit',
                                                   manifest_ref).split(
                                                       '\n\n', 1)
            header = dict(l.split(' ', 1) for l in header.splitlines())
            if header['tree'] == EMPTY_TREE:
                sha1 = EMPTY_TREE
            else:
                report('Missing git tree in manifest commit %s' % manifest_ref)
                sha1 = None
        if sha1 and sha1 != tree:
            report('Tree mismatch between manifest commit %s and commit %s' %
                   (manifest_ref, node))

        if args.files:
            changes = get_changes(
                manifest_ref, tuple(store.manifest_ref(p) for p in parents),
                'hg')
            for path, hg_file, hg_fileparents in changes:
                if hg_file != NULL_NODE_ID and hg_file not in seen_files:
                    file = store.file(hg_file)
                    file.set_parents(*hg_fileparents)
                    if file.node != file.sha1:
                        report('Sha1 mismatch for file %s in manifest %s' %
                               (hg_file, manifest_ref))
                    seen_files.add(hg_file)

    if args.files:
        all_hg2git = set(all_hg2git.iterkeys())
    else:
        all_hg2git = set(k for k, (s, t) in all_hg2git.iteritems()
                         if t == 'commit')

    adjusted = {}
    if not args.commit:
        dangling = set(manifest_commits) - set(seen_manifest_refs)
        if dangling:

            def iter_manifests():
                removed_one = False
                yielded = False
                previous = None
                for obj in reversed(manifest_commits):
                    if obj in dangling:
                        fix('Removing metadata commit %s with no hg2git entry'
                            % obj)
                        removed_one = True
                    else:
                        if removed_one:
                            yield obj, previous
                            yielded = True
                        previous = obj

                if removed_one and not yielded:
                    yield obj, False

            for obj, parent in progress_iter('Adjusting %d metadata commits',
                                             iter_manifests()):
                mark = store._fast_import.new_mark()
                if parent is False:
                    Git.update_ref('refs/cinnabar/manifest', obj)
                    continue
                elif parent:
                    parents = (adjusted.get(parent, parent), )
                with store._fast_import.commit(ref='refs/cinnabar/manifest',
                                               parents=parents,
                                               mark=mark) as commit:
                    mode, typ, tree, path = store._fast_import.ls(obj)
                    commit.filemodify('', tree, typ='tree')
                adjusted[obj] = Mark(mark)

    dangling = all_hg2git - seen_changesets - seen_manifests - seen_files
    if dangling or adjusted:
        with store._fast_import.commit(
                ref='refs/cinnabar/hg2git',
                parents=('refs/cinnabar/hg2git^0', )) as commit:
            for obj in dangling:
                fix('Removing dangling metadata for ' + obj)
                commit.filedelete(sha1path(obj))
            for obj, mark in progress_iter(
                    'Updating hg2git for %d metadata commits',
                    adjusted.iteritems()):
                commit.filemodify(sha1path(seen_manifest_refs[obj]),
                                  mark,
                                  typ='commit')

    dangling = all_notes - seen_notes
    if dangling:
        with store._fast_import.commit(
                ref='refs/notes/cinnabar',
                parents=('refs/notes/cinnabar^0', )) as commit:
            for c in dangling:
                fix('Removing dangling note for commit ' + c)
                # That's brute force, but meh.
                for l in range(0, 10):
                    commit.filedelete(sha1path(c, l))

    if status['broken']:
        info('Your git-cinnabar repository appears to be corrupted. There\n'
             'are known issues in older revisions that have been fixed.\n'
             'Please try running the following command to reset:\n'
             '  git cinnabar reclone\n\n'
             'Please note this command may change the commit sha1s. Your\n'
             'local branches will however stay untouched.\n'
             'Please report any corruption that fsck would detect after a\n'
             'reclone.')

    if not args.commit:
        info('Checking head references...')
        computed_heads = defaultdict(set)
        for branch, head in dag.all_heads():
            computed_heads[branch].add(head)

        for branch in sorted(dag.tags()):
            stored_heads = store.heads({branch})
            for head in computed_heads[branch] - stored_heads:
                fix('Adding missing head %s in branch %s' % (head, branch))
                store.add_head(head)
            for head in stored_heads - computed_heads[branch]:
                fix('Removing non-head reference to %s in branch %s' %
                    (head, branch))
                store._hgheads.remove((branch, head))

    store.close()

    if status['broken']:
        return 1
    if status['fixed']:
        return 2
    return 0
Example #12
0
def main(args):
    logger = logging.getLogger('-')
    logger.info(args)
    assert len(args) == 2
    remote, url = args
    git_dir = os.environ.get('GIT_DIR')
    if Git.config('core.ignorecase', 'bool') == 'true':
        sys.stderr.write(
            'Your git configuration has core.ignorecase set to "true".\n'
            'Usually, this means git detected the file system is case '
            'insensitive.\n'
            'Git-cinnabar does not support this setup.\n'
            'Either use a case sensitive file system or set '
            'core.ignorecase to "false".\n'
        )
        git_work_tree = os.path.dirname(git_dir)
        if os.path.abspath(os.getcwd() + os.sep).startswith(
                os.path.abspath(git_work_tree) + os.sep) or \
                remote == 'hg::' + url or tuple(
                Git.for_each_ref('refs/remotes/%s' % remote)):
            sys.stderr.write(
                'Use the following command to reclone:\n'
                '  git cinnabar reclone\n'
            )
        else:
            sys.stderr.write(
                'Use the following command to clone:\n'
                '  git -c core.ignorecase=false clone%(args)s hg::%(url)s '
                '%(dir)s\n'
                % {
                    'dir': git_work_tree,
                    'url': url,
                    'args': ' -o ' + remote if remote != 'origin' else ''
                }
            )
        return 1
    repo = get_repo(url)
    store = GitHgStore()
    logger.info(LazyString(lambda: '%s' % store.heads()))
    helper = IOLogger(logging.getLogger('remote-helper'),
                      sys.stdin, sys.stdout)
    branchmap = None
    bookmarks = {}
    HEAD = 'branches/default/tip'

    while True:
        cmd, args = read_cmd(helper)
        if not cmd:
            break

        if cmd == 'capabilities':
            assert not args
            helper.write(
                'option\n'
                'import\n'
                'bidi-import\n'
                'push\n'
                'refspec refs/heads/branches/*:'
                'refs/cinnabar/refs/heads/branches/*\n'
                'refspec refs/heads/bookmarks/*:'
                'refs/cinnabar/refs/heads/bookmarks/*\n'
                'refspec HEAD:refs/cinnabar/HEAD\n'
                '\n'
            )
            helper.flush()
        elif cmd == 'list':
            assert not args or args == ['for-push']

            if repo.capable('batch'):
                batch = repo.batch()
                branchmap = batch.branchmap()
                heads = batch.heads()
                bookmarks = batch.listkeys('bookmarks')
                batch.submit()
                branchmap = branchmap.value
                heads = heads.value
                bookmarks = bookmarks.value
            else:
                while True:
                    branchmap = repo.branchmap()
                    heads = repo.heads()
                    if heads == ['\0' * 20]:
                        heads = []
                    # Some branch heads can be non-heads topologically, but if
                    # some heads don't appear in the branchmap, then something
                    # was pushed to the repo between branchmap() and heads()
                    if set(heads).issubset(set(chain(*branchmap.values()))):
                        break
                bookmarks = repo.listkeys('bookmarks')

            branchmap = BranchMap(store, branchmap, heads)
            unknowns = False
            for branch in sorted(branchmap.names()):
                branch_tip = branchmap.tip(branch)
                for head in sorted(branchmap.heads(branch)):
                    sha1 = branchmap.git_sha1(head)
                    if sha1 == '?':
                        unknowns = True
                    if head == branch_tip:
                        continue
                    helper.write('%s refs/heads/branches/%s/%s\n' % (
                        sha1,
                        branch,
                        head,
                    ))
                if branch_tip:
                    helper.write('%s refs/heads/branches/%s/tip\n' % (
                        branchmap.git_sha1(branch_tip),
                        branch,
                    ))
            for name, sha1 in sorted(bookmarks.iteritems()):
                ref = store.changeset_ref(sha1)
                helper.write(
                    '%s refs/heads/bookmarks/%s\n'
                    % (ref if ref else '?', name)
                )
            if not unknowns:
                for tag, ref in sorted(store.tags(branchmap.heads())):
                    helper.write('%s refs/tags/%s\n' % (ref, tag))

            if '@' in bookmarks:
                HEAD = 'bookmarks/@'
            helper.write(
                '@refs/heads/%s HEAD\n'
                '\n'
                % HEAD
            )
            helper.flush()
        elif cmd == 'option':
            assert len(args) == 2
            name, value = args
            if name == 'progress':
                if value == 'true':
                    cinnabar.util.progress = True
                    helper.write('ok\n')
                elif value == 'false':
                    cinnabar.util.progress = False
                    helper.write('ok\n')
                else:
                    helper.write('unsupported\n')
            else:
                helper.write('unsupported\n')
            helper.flush()
        elif cmd == 'import':
            try:
                reflog = os.path.join(git_dir, 'logs', 'refs', 'cinnabar')
                mkpath(reflog)
                open(os.path.join(reflog, 'hg2git'), 'a').close()
                open(os.path.join(reflog, 'manifest'), 'a').close()
                assert len(args) == 1
                refs = args
                while cmd:
                    assert cmd == 'import'
                    cmd, args = read_cmd(helper)
                    assert args is None or len(args) == 1
                    if args:
                        refs.extend(args)
            except:
                # If anything wrong happens before we got all the import
                # commands, we risk git picking the existing refs/cinnabar
                # refs. Remove them.
                for line in Git.for_each_ref('refs/cinnabar/refs/heads',
                                             'refs/cinnabar/HEAD',
                                             format='%(refname)'):
                    Git.delete_ref(ref)
                raise

            try:
                def resolve_head(head):
                    if head.startswith('refs/heads/branches/'):
                        head = head[20:]
                        if head[-4:] == '/tip':
                            return branchmap.tip(head[:-4])
                        return head[-40:]
                    if head.startswith('refs/heads/bookmarks/'):
                        head = head[21:]
                        return bookmarks[head]
                    if head == 'HEAD':
                        return bookmarks.get('@') or branchmap.tip('default')
                    return None

                wanted_refs = {k: v for k, v in (
                               (h, resolve_head(h)) for h in refs) if v}
                heads = wanted_refs.values()
                if not heads:
                    heads = branchmap.heads()

                # Older versions would create a symbolic ref for
                # refs/remote-hg/HEAD. Newer versions don't, and
                # Git.update_ref doesn't remove the symbolic ref, so it needs
                # to be removed first.
                # Since git symbolic-ref only throws an error when the ref is
                # not symbolic, just try to remove the symbolic ref every time
                # and ignore errors.
                tuple(Git.iter('symbolic-ref', '-d', 'refs/remote-hg/HEAD',
                               stderr=open(os.devnull, 'wb')))

                refs_orig = {}
                for line in Git.for_each_ref(
                        'refs/cinnabar/refs/heads', 'refs/cinnabar/HEAD',
                        format='%(objectname) %(refname)'):
                    sha1, ref = line.split(' ', 1)
                    refs_orig[ref] = sha1
            except:
                # If anything wrong happens before we actually pull, we risk
                # git pucking the existing refs/cinnabar refs. Remove them.
                # Unlike in the case above, we now have the list of refs git
                # is expected, so we can just remove those.
                for ref in refs:
                    Git.delete_ref('refs/cinnabar/' + ref)
                raise

            try:
                store.init_fast_import(FastImport(sys.stdin, sys.stdout))
                getbundle(repo, store, heads, branchmap)
            except:
                wanted_refs = {}
                raise
            finally:
                for ref, value in wanted_refs.iteritems():
                    ref = 'refs/cinnabar/' + ref
                    if ref not in refs_orig or refs_orig[ref] != value:
                        Git.update_ref(ref, store.changeset_ref(value))
                for ref in refs_orig:
                    if ref[14:] not in wanted_refs:
                        Git.delete_ref(ref)

            store.close()

            if not remote.startswith('hg::'):
                prune = 'remote.%s.prune' % remote
                if (Git.config(prune) != 'true' and
                        Git.config('fetch.prune') != 'true'):
                    sys.stderr.write(
                        'It is recommended that you set "%(conf)s" or '
                        '"fetch.prune" to "true".\n'
                        '  git config %(conf)s true\n'
                        'or\n'
                        '  git config fetch.prune true\n'
                        % {'conf': prune}
                    )

            if store.tag_changes:
                sys.stderr.write(
                    '\nRun the following command to update remote tags:\n')
                if not remote.startswith('hg::'):
                    sys.stderr.write('  git remote update %s\n' % remote)
                else:
                    sys.stderr.write('  git fetch --tags %s\n' % remote)

        elif cmd == 'push':
            if not remote.startswith('hg::'):
                data_pref = 'remote.%s.cinnabar-data' % remote
                data = Git.config(data_pref) or 'phase'
            else:
                data = 'phase'

            if data not in ('never', 'phase', 'always'):
                sys.stderr.write('Invalid value for %s: %s\n'
                                 % (data_pref, data))
                return 1

            refspecs = []
            refspecs.extend(args)
            while True:
                cmd, args = read_cmd(helper)
                if not cmd:
                    break
                assert cmd == 'push'
                refspecs.extend(args)
            pushes = {s.lstrip('+'): (d, s.startswith('+'))
                      for s, d in (r.split(':', 1) for r in refspecs)}
            if isinstance(repo, bundlerepo):
                for source, (dest, force) in pushes.iteritems():
                    helper.write('error %s Cannot push to a bundle file\n'
                                 % dest)
                helper.write('\n')
                helper.flush()
            else:
                repo_heads = branchmap.heads()
                PushStore.adopt(store)
                pushed = push(repo, store, pushes, repo_heads,
                              branchmap.names())

                status = {}
                for source, (dest, _) in pushes.iteritems():
                    if dest.startswith('refs/tags/'):
                        if source:
                            status[dest] = 'Pushing tags is unsupported'
                        else:
                            status[dest] = \
                                'Deleting remote tags is unsupported'
                        continue
                    if not dest.startswith('refs/heads/bookmarks/'):
                        if source:
                            status[dest] = bool(len(pushed))
                        else:
                            status[dest] = \
                                'Deleting remote branches is unsupported'
                        continue
                    name = dest[21:]
                    if source:
                        source = store.hg_changeset(Git.resolve_ref(source)) \
                            or ''
                    status[dest] = repo.pushkey(
                        'bookmarks', name, bookmarks.get(name, ''), source)

                for source, (dest, force) in pushes.iteritems():
                    if status[dest] is True:
                        helper.write('ok %s\n' % dest)
                    elif status[dest]:
                        helper.write('error %s %s\n' % (dest, status[dest]))
                    else:
                        helper.write('error %s nothing changed on remote\n'
                                     % dest)
                helper.write('\n')
                helper.flush()

                if not pushed:
                    data = False
                elif data == 'always':
                    data = True
                elif data == 'phase':
                    phases = repo.listkeys('phases')
                    drafts = {}
                    if not phases.get('publishing', False):
                        drafts = set(p for p, is_draft in phases.iteritems()
                                     if int(is_draft))
                    if not drafts:
                        data = True
                    else:
                        def draft_commits():
                            for d in drafts:
                                c = store.changeset_ref(d)
                                if c:
                                    yield '^%s^@' % c
                            for h in pushed.heads():
                                yield h

                        args = ['rev-list', '--ancestry-path', '--topo-order',
                                '--stdin']

                        pushed_drafts = tuple(
                            Git.iter(*args, stdin=draft_commits()))

                        # Theoretically, we could have commits with no
                        # metadata that the remote declares are public, while
                        # the rest of our push is in a draft state. That is
                        # however so unlikely that it's not worth the effort
                        # to support partial metadata storage.
                        data = not bool(pushed_drafts)
                elif data == 'never':
                    data = False

                store.close(rollback=not data)

    store.close()
Example #13
0
def fsck_quick():
    status = FsckStatus()
    store = GitHgStore()

    metadata_commit = Git.resolve_ref('refs/cinnabar/metadata')
    if not metadata_commit:
        status.info(
            'There does not seem to be any git-cinnabar metadata.\n'
            'Is this a git-cinnabar clone?'
        )
        return 1
    commit = GitCommit(metadata_commit)
    if commit.body != 'files-meta unified-manifests-v2':
        status.info(
            'The git-cinnabar metadata is incompatible with this version.\n'
            'Please use the git-cinnabar version it was used with last.\n'
        )
        return 1
    if len(commit.parents) > 6 or len(commit.parents) < 5:
        status.report('The git-cinnabar metadata seems to be corrupted in '
                      'unexpected ways.\n')
        return 1
    changesets, manifests, hg2git, git2hg, files_meta = commit.parents[:5]

    commit = GitCommit(changesets)
    heads = OrderedDict(
        (node, branch)
        for node, _, branch in (d.partition(' ')
                                for d in commit.body.splitlines()))
    if len(heads) != len(commit.parents):
        status.report('The git-cinnabar metadata seems to be corrupted in '
                      'unexpected ways.\n')
        return 1

    manifest_nodes = []

    parents = None
    fix_changeset_heads = False
    # TODO: Check that the recorded heads are actually dag heads.
    for c, changeset_node in progress_iter(
            'Checking {} changeset heads', izip(commit.parents, heads)):
        gitsha1 = GitHgHelper.hg2git(changeset_node)
        if gitsha1 == NULL_NODE_ID:
            status.report('Missing hg2git metadata for changeset %s'
                          % changeset_node)
            continue
        if gitsha1 != c:
            if parents is None:
                parents = set(commit.parents)
            if gitsha1 not in parents:
                status.report(
                    'Inconsistent metadata:\n'
                    '  Head metadata says changeset %s maps to %s\n'
                    '  but hg2git metadata says it maps to %s'
                    % (changeset_node, c, gitsha1))
                continue
            fix_changeset_heads = True
        changeset = store._changeset(c, include_parents=True)
        if not changeset:
            status.report('Missing git2hg metadata for git commit %s' % c)
            continue
        if changeset.node != changeset_node:
            if changeset.node not in heads:
                status.report(
                    'Inconsistent metadata:\n'
                    '  Head metadata says %s maps to changeset %s\n'
                    '  but git2hg metadata says it maps to changeset %s'
                    % (c, changeset_node, changeset.node))
                continue
            fix_changeset_heads = True
        if changeset.node != changeset.sha1:
            status.report('Sha1 mismatch for changeset %s' % changeset.node)
            continue
        changeset_branch = changeset.branch or 'default'
        if heads[changeset.node] != changeset_branch:
            status.report(
                'Inconsistent metadata:\n'
                '  Head metadata says changeset %s is in branch %s\n'
                '  but git2hg metadata says it is in branch %s'
                % (changeset.node, heads[changeset.node], changeset_branch))
            continue
        manifest_nodes.append(changeset.manifest)

    if status('broken'):
        return 1

    # Rebuilding manifests benefits from limiting the difference with
    # the last rebuilt manifest. Similarly, building the list of unique
    # files in all manifests benefits from that too.
    # Unfortunately, the manifest heads are not ordered in a topological
    # relevant matter, and the differences between two consecutive manifests
    # can be much larger than they could be. The consequence is spending a
    # large amount of time rebuilding the manifests and gathering the files
    # list. It's actually faster to attempt to reorder them according to
    # some heuristics first, such that the differences are smaller.
    # Here, we use the depth from the root node(s) to reorder the manifests.
    # This doesn't give the most optimal ordering, but it's already much
    # faster. On a clone of multiple mozilla-* repositories with > 1400 heads,
    # it's close to an order of magnitude difference on the "Checking
    # manifests" loop.
    depths = {}
    roots = []
    manifest_queue = []
    for m, _, parents in progress_iter(
            'Loading {} manifests', GitHgHelper.rev_list(
                '--topo-order', '--reverse', '--full-history', '%s^@'
                % manifests)):
        manifest_queue.append((m, parents))
        if parents:
            depth = {}
            for p in parents:
                for root, num in depths[p].iteritems():
                    if root in depth:
                        depth[root] = max(depth[root], num + 1)
                    else:
                        depth[root] = num + 1
            depths[m] = depth
            del depth
        else:
            depths[m] = {m: 0}
            roots.append(m)

    if status('broken'):
        return 1

    # TODO: check that all manifest_nodes gathered above are available in the
    # manifests dag, and that the dag heads are the recorded heads.
    manifests_commit = GitCommit(manifests)
    depths = [
        [depths[p].get(r, 0) for r in roots]
        for p in manifests_commit.parents
    ]
    manifests_commit_parents = [
        p for _, p in sorted(zip(depths, manifests_commit.parents))
    ]
    previous = None
    all_interesting = set()
    for m in progress_iter('Checking {} manifest heads',
                           manifests_commit_parents):
        c = GitCommit(m)
        if not SHA1_RE.match(c.body):
            status.report('Invalid manifest metadata in git commit %s' % m)
            continue
        gitsha1 = GitHgHelper.hg2git(c.body)
        if gitsha1 == NULL_NODE_ID:
            status.report('Missing hg2git metadata for manifest %s' % c.body)
            continue
        if not GitHgHelper.check_manifest(c.body):
            status.report('Sha1 mismatch for manifest %s' % c.body)

        files = {}
        if previous:
            for _, _, before, after, d, path in GitHgHelper.diff_tree(
                    previous, m):
                if d in 'AM' and before != after and \
                        (path, after) not in all_interesting:
                    files[path] = after
        else:
            for _, t, sha1, path in GitHgHelper.ls_tree(m, recursive=True):
                if (path, sha1) not in all_interesting:
                    files[path] = sha1
        all_interesting.update(files.iteritems())
        previous = m

    if status('broken'):
        return 1

    progress = Progress('Checking {} files')
    while all_interesting and manifest_queue:
        (m, parents) = manifest_queue.pop()
        changes = get_changes(m, parents, all=True)
        for path, hg_file, hg_fileparents in changes:
            if hg_fileparents[1:] == (hg_file,):
                continue
            elif hg_fileparents[:1] == (hg_file,):
                continue
            # Reaching here means the file received a modification compared
            # to its parents. If it's a file we're going to check below,
            # it means we don't need to check its parents if somehow they were
            # going to be checked. If it's not a file we're going to check
            # below, it's because it's either a file we weren't interested in
            # in the first place, or it's the parent of a file we have checked.
            # Either way, we aren't interested in the parents.
            for p in hg_fileparents:
                all_interesting.discard((path, p))
            if (path, hg_file) not in all_interesting:
                continue
            all_interesting.remove((path, hg_file))
            if not GitHgHelper.check_file(hg_file, *hg_fileparents):
                p = store.manifest_path(path)
                status.report(
                    'Sha1 mismatch for file %s\n'
                    '  revision %s' % (p, hg_file))

                print_parents = ' '.join(p for p in hg_fileparents
                                         if p != NULL_NODE_ID)
                if print_parents:
                    status.report('  with parent%s %s' % (
                        's' if len(print_parents) > 41 else '',
                        print_parents))
            progress.progress()
    progress.finish()
    if all_interesting:
        status.info('Could not find the following files:')
        for path, sha1 in sorted(all_interesting):
            p = store.manifest_path(path)
            status.info('  %s %s' % (sha1, path))
        status.info(
            'This might be a bug in `git cinnabar fsck`. Please open '
            'an issue, with the message above, on\n'
            'https://github.com/glandium/git-cinnabar/issues')
        return 1

    if status('broken'):
        status.info(
            'Your git-cinnabar repository appears to be corrupted.\n'
            'Please open an issue, with the information above, on\n'
            'https://github.com/glandium/git-cinnabar/issues')
        Git.update_ref('refs/cinnabar/broken', metadata_commit)
        if Git.resolve_ref('refs/cinnabar/checked'):
            status.info(
                '\nThen please try to run `git cinnabar rollback --fsck` to '
                'restore last known state, and to update from the mercurial '
                'repository.')
        else:
            status.info('\nThen please try to run `git cinnabar reclone`.')
        status.info(
            '\nPlease note this may affect the commit sha1s of mercurial '
            'changesets, and may require to rebase your local branches.')
        status.info(
            '\nAlternatively, you may start afresh with a new clone. In any '
            'case, please keep this corrupted repository around for further '
            'debugging.')
        return 1

    refresh = []
    if fix_changeset_heads:
        status.fix('Fixing changeset heads metadata order.')
        refresh.append('refs/cinnabar/changesets')
    interval_expired('fsck', 0)
    store.close(refresh=refresh)
    GitHgHelper._helper = False
    metadata_commit = Git.resolve_ref('refs/cinnabar/metadata')
    Git.update_ref('refs/cinnabar/checked', metadata_commit)
    return 0
Example #14
0
def fsck(args):
    '''check cinnabar metadata consistency'''

    if not args.commit and not args.full:
        return fsck_quick()

    status = FsckStatus()

    store = GitHgStore()

    if args.full and args.commit:
        logging.error('Cannot pass both --full and a commit')
        return 1

    if args.commit:
        commits = set()
        all_git_commits = {}

        for c in args.commit:
            cs = store.hg_changeset(c)
            if cs:
                commits.add(c)
                c = cs.node
            commit = GitHgHelper.hg2git(c)
            if commit == NULL_NODE_ID and not cs:
                status.info('Unknown commit or changeset: %s' % c)
                return 1
            if not cs:
                cs = store.hg_changeset(commit)
                commits.add(commit)

        all_git_commits = GitHgHelper.rev_list('--no-walk=unsorted', *commits)
    else:
        all_refs = dict((ref, sha1)
                        for sha1, ref in Git.for_each_ref('refs/cinnabar'))

        if 'refs/cinnabar/metadata' in all_refs:
            git_heads = '%s^^@' % all_refs['refs/cinnabar/metadata']
        else:
            assert False

        all_git_commits = GitHgHelper.rev_list(
            '--topo-order', '--full-history', '--reverse', git_heads)

    dag = gitdag()

    GitHgHelper.reset_heads('manifests')

    full_file_check = FileFindParents.logger.isEnabledFor(logging.DEBUG)

    for node, tree, parents in progress_iter('Checking {} changesets',
                                             all_git_commits):
        node = store._replace.get(node, node)
        hg_node = store.hg_changeset(node)
        if not hg_node:
            status.report('Missing note for git commit: ' + node)
            continue
        GitHgHelper.seen('git2hg', node)

        changeset_data = store.changeset(hg_node)
        changeset = changeset_data.node

        GitHgHelper.seen('hg2git', changeset)
        changeset_ref = store.changeset_ref(changeset)
        if not changeset_ref:
            status.report('Missing changeset in hg2git branch: %s' % changeset)
            continue
        elif str(changeset_ref) != node:
            status.report('Commit mismatch for changeset %s\n'
                          '  hg2git: %s\n  commit: %s'
                          % (changeset, changeset_ref, node))

        hg_changeset = store.changeset(changeset, include_parents=True)
        if hg_changeset.node != hg_changeset.sha1:
            status.report('Sha1 mismatch for changeset %s' % changeset)

        dag.add(hg_changeset.node,
                (hg_changeset.parent1, hg_changeset.parent2),
                changeset_data.branch or 'default')

        raw_changeset = Changeset.from_git_commit(node)
        patcher = ChangesetPatcher.from_diff(raw_changeset, changeset_data)
        if patcher != store.read_changeset_data(node):
            status.fix('Adjusted changeset metadata for %s' % changeset)
            GitHgHelper.set('changeset', changeset, NULL_NODE_ID)
            GitHgHelper.set('changeset', changeset, node)
            GitHgHelper.put_blob(patcher, want_sha1=False)
            GitHgHelper.set('changeset-metadata', changeset, NULL_NODE_ID)
            GitHgHelper.set('changeset-metadata', changeset, ':1')

        manifest = changeset_data.manifest
        if GitHgHelper.seen('hg2git', manifest) or manifest == NULL_NODE_ID:
            continue
        manifest_ref = store.manifest_ref(manifest)
        if not manifest_ref:
            status.report('Missing manifest in hg2git branch: %s' % manifest)

        parents = tuple(
            store.changeset(p).manifest
            for p in hg_changeset.parents
        )
        git_parents = tuple(store.manifest_ref(p) for p in parents
                            if p != NULL_NODE_ID)

        # This doesn't change the value but makes the helper track the manifest
        # dag.
        GitHgHelper.set('manifest', manifest, manifest_ref)

        if not GitHgHelper.check_manifest(manifest):
            status.report('Sha1 mismatch for manifest %s' % manifest)

        manifest_commit_parents = GitCommit(manifest_ref).parents
        if sorted(manifest_commit_parents) != sorted(git_parents):
            # TODO: better error
            status.report('%s(%s) %s != %s' % (manifest, manifest_ref,
                                               manifest_commit_parents,
                                               git_parents))

        # TODO: check that manifest content matches changeset content

        changes = get_changes(manifest_ref, git_parents)
        for path, hg_file, hg_fileparents in changes:
            if hg_file != NULL_NODE_ID and (hg_file == HG_EMPTY_FILE or
                                            GitHgHelper.seen('hg2git',
                                                             hg_file)):
                if full_file_check:
                    file = store.file(hg_file, hg_fileparents, git_parents,
                                      store.manifest_path(path))
                    valid = file.node == file.sha1
                else:
                    valid = GitHgHelper.check_file(hg_file,
                                                   *hg_fileparents)
                if not valid:
                    status.report(
                        'Sha1 mismatch for file %s in manifest %s'
                        % (hg_file, manifest_ref))

    if not args.commit and not status('broken'):
        store_manifest_heads = set(store._manifest_heads_orig)
        manifest_heads = set(GitHgHelper.heads('manifests'))
        if store_manifest_heads != manifest_heads:
            def iter_manifests(a, b):
                for h in a - b:
                    yield h
                for h in b:
                    yield '^%s' % h

            for m, t, p in GitHgHelper.rev_list(
                    '--topo-order', '--full-history', '--reverse',
                    *iter_manifests(manifest_heads, store_manifest_heads)):
                status.fix('Missing manifest commit in manifest branch: %s'
                           % m)

            for m, t, p in GitHgHelper.rev_list(
                    '--topo-order', '--full-history', '--reverse',
                    *iter_manifests(store_manifest_heads, manifest_heads)):
                status.fix('Removing metadata commit %s with no corresponding '
                           'changeset' % (m))

            for h in store_manifest_heads - manifest_heads:
                if GitHgHelper.seen('hg2git', store.hg_manifest(h)):
                    status.fix('Removing non-head reference to %s in manifests'
                               ' metadata.' % h)
    dangling = ()
    if not args.commit and not status('broken'):
        dangling = GitHgHelper.dangling('hg2git')
    for obj in dangling:
        status.fix('Removing dangling metadata for ' + obj)
        # Theoretically, we should figure out if they are files, manifests
        # or changesets and set the right variable accordingly, but in
        # practice, it makes no difference. Reevaluate when GitHgStore.close
        # is modified, though.
        GitHgHelper.set('file', obj, NULL_NODE_ID)
        GitHgHelper.set('file-meta', obj, NULL_NODE_ID)

    if not args.commit and not status('broken'):
        dangling = GitHgHelper.dangling('git2hg')
    for c in dangling:
        status.fix('Removing dangling note for commit ' + c)
        GitHgHelper.set('changeset-metadata', c, NULL_NODE_ID)

    if status('broken'):
        status.info(
            'Your git-cinnabar repository appears to be corrupted. There\n'
            'are known issues in older revisions that have been fixed.\n'
            'Please try running the following command to reset:\n'
            '  git cinnabar reclone\n\n'
            'Please note this command may change the commit sha1s. Your\n'
            'local branches will however stay untouched.\n'
            'Please report any corruption that fsck would detect after a\n'
            'reclone.')

    if not args.commit:
        status.info('Checking head references...')
        computed_heads = defaultdict(set)
        for branch, head in dag.all_heads():
            computed_heads[branch].add(head)

        for branch in sorted(dag.tags()):
            stored_heads = store.heads({branch})
            for head in computed_heads[branch] - stored_heads:
                status.fix('Adding missing head %s in branch %s' %
                           (head, branch))
                store.add_head(head)
            for head in stored_heads - computed_heads[branch]:
                status.fix('Removing non-head reference to %s in branch %s' %
                           (head, branch))
                del store._hgheads[head]

    metadata_commit = Git.resolve_ref('refs/cinnabar/metadata')
    if status('broken'):
        Git.update_ref('refs/cinnabar/broken', metadata_commit)
        return 1

    if args.full:
        Git.update_ref('refs/cinnabar/checked', metadata_commit)
    interval_expired('fsck', 0)
    store.close()

    if status('fixed'):
        return 2
    return 0
Example #15
0
def fsck(args):
    '''check cinnabar metadata consistency'''

    if not args.commit and not args.full:
        return fsck_quick(args.force)

    status = FsckStatus()

    store = GitHgStore()

    if args.full and args.commit:
        logging.error('Cannot pass both --full and a commit')
        return 1

    if args.commit:
        commits = set()
        all_git_commits = {}

        for c in args.commit:
            cs = store.hg_changeset(c)
            if cs:
                commits.add(c)
                c = cs.node
            commit = GitHgHelper.hg2git(c)
            if commit == NULL_NODE_ID and not cs:
                status.info('Unknown commit or changeset: %s' % c)
                return 1
            if not cs:
                cs = store.hg_changeset(commit)
                commits.add(commit)

        all_git_commits = GitHgHelper.rev_list('--no-walk=unsorted', *commits)
    else:
        all_refs = dict(
            (ref, sha1) for sha1, ref in Git.for_each_ref('refs/cinnabar'))

        if 'refs/cinnabar/metadata' in all_refs:
            git_heads = '%s^^@' % all_refs['refs/cinnabar/metadata']
        else:
            assert False

        all_git_commits = GitHgHelper.rev_list('--topo-order',
                                               '--full-history', '--reverse',
                                               git_heads)

    dag = gitdag()

    GitHgHelper.reset_heads('manifests')

    full_file_check = FileFindParents.logger.isEnabledFor(logging.DEBUG)

    for node, tree, parents in progress_iter('Checking {} changesets',
                                             all_git_commits):
        node = store._replace.get(node, node)
        hg_node = store.hg_changeset(node)
        if not hg_node:
            status.report('Missing note for git commit: ' + node)
            continue
        GitHgHelper.seen('git2hg', node)

        changeset_data = store.changeset(hg_node)
        changeset = changeset_data.node

        GitHgHelper.seen('hg2git', changeset)
        changeset_ref = store.changeset_ref(changeset)
        if not changeset_ref:
            status.report('Missing changeset in hg2git branch: %s' % changeset)
            continue
        elif str(changeset_ref) != node:
            status.report('Commit mismatch for changeset %s\n'
                          '  hg2git: %s\n  commit: %s' %
                          (changeset, changeset_ref, node))

        hg_changeset = store.changeset(changeset, include_parents=True)
        if hg_changeset.node != hg_changeset.sha1:
            status.report('Sha1 mismatch for changeset %s' % changeset)

        dag.add(hg_changeset.node,
                (hg_changeset.parent1, hg_changeset.parent2),
                changeset_data.branch or 'default')

        raw_changeset = Changeset.from_git_commit(node)
        patcher = ChangesetPatcher.from_diff(raw_changeset, changeset_data)
        if patcher != store.read_changeset_data(node):
            status.fix('Adjusted changeset metadata for %s' % changeset)
            GitHgHelper.set('changeset', changeset, NULL_NODE_ID)
            GitHgHelper.set('changeset', changeset, node)
            GitHgHelper.put_blob(patcher, want_sha1=False)
            GitHgHelper.set('changeset-metadata', changeset, NULL_NODE_ID)
            GitHgHelper.set('changeset-metadata', changeset, ':1')

        manifest = changeset_data.manifest
        if GitHgHelper.seen('hg2git', manifest) or manifest == NULL_NODE_ID:
            continue
        manifest_ref = store.manifest_ref(manifest)
        if not manifest_ref:
            status.report('Missing manifest in hg2git branch: %s' % manifest)

        parents = tuple(
            store.changeset(p).manifest for p in hg_changeset.parents)
        git_parents = tuple(
            store.manifest_ref(p) for p in parents if p != NULL_NODE_ID)

        # This doesn't change the value but makes the helper track the manifest
        # dag.
        GitHgHelper.set('manifest', manifest, manifest_ref)

        if not GitHgHelper.check_manifest(manifest):
            status.report('Sha1 mismatch for manifest %s' % manifest)

        manifest_commit_parents = GitCommit(manifest_ref).parents
        if sorted(manifest_commit_parents) != sorted(git_parents):
            # TODO: better error
            status.report(
                '%s(%s) %s != %s' %
                (manifest, manifest_ref, manifest_commit_parents, git_parents))

        # TODO: check that manifest content matches changeset content

        changes = get_changes(manifest_ref, git_parents)
        for path, hg_file, hg_fileparents in changes:
            if hg_file != NULL_NODE_ID and (hg_file == HG_EMPTY_FILE
                                            or GitHgHelper.seen(
                                                'hg2git', hg_file)):
                if full_file_check:
                    file = store.file(hg_file, hg_fileparents, git_parents,
                                      store.manifest_path(path))
                    valid = file.node == file.sha1
                else:
                    valid = GitHgHelper.check_file(hg_file, *hg_fileparents)
                if not valid:
                    status.report('Sha1 mismatch for file %s in manifest %s' %
                                  (hg_file, manifest_ref))

    if not args.commit and not status('broken'):
        store_manifest_heads = set(store._manifest_heads_orig)
        manifest_heads = set(GitHgHelper.heads('manifests'))
        if store_manifest_heads != manifest_heads:

            def iter_manifests(a, b):
                for h in a - b:
                    yield h
                for h in b:
                    yield '^%s' % h

            for m, t, p in GitHgHelper.rev_list(
                    '--topo-order', '--full-history', '--reverse',
                    *iter_manifests(manifest_heads, store_manifest_heads)):
                status.fix('Missing manifest commit in manifest branch: %s' %
                           m)

            for m, t, p in GitHgHelper.rev_list(
                    '--topo-order', '--full-history', '--reverse',
                    *iter_manifests(store_manifest_heads, manifest_heads)):
                status.fix('Removing metadata commit %s with no corresponding '
                           'changeset' % (m))

            for h in store_manifest_heads - manifest_heads:
                if GitHgHelper.seen('hg2git', store.hg_manifest(h)):
                    status.fix('Removing non-head reference to %s in manifests'
                               ' metadata.' % h)
    dangling = ()
    if not args.commit and not status('broken'):
        dangling = GitHgHelper.dangling('hg2git')
    for obj in dangling:
        status.fix('Removing dangling metadata for ' + obj)
        # Theoretically, we should figure out if they are files, manifests
        # or changesets and set the right variable accordingly, but in
        # practice, it makes no difference. Reevaluate when GitHgStore.close
        # is modified, though.
        GitHgHelper.set('file', obj, NULL_NODE_ID)
        GitHgHelper.set('file-meta', obj, NULL_NODE_ID)

    if not args.commit and not status('broken'):
        dangling = GitHgHelper.dangling('git2hg')
    for c in dangling:
        status.fix('Removing dangling note for commit ' + c)
        GitHgHelper.set('changeset-metadata', c, NULL_NODE_ID)

    if status('broken'):
        status.info(
            'Your git-cinnabar repository appears to be corrupted. There\n'
            'are known issues in older revisions that have been fixed.\n'
            'Please try running the following command to reset:\n'
            '  git cinnabar reclone\n\n'
            'Please note this command may change the commit sha1s. Your\n'
            'local branches will however stay untouched.\n'
            'Please report any corruption that fsck would detect after a\n'
            'reclone.')

    if not args.commit:
        status.info('Checking head references...')
        computed_heads = defaultdict(set)
        for branch, head in dag.all_heads():
            computed_heads[branch].add(head)

        for branch in sorted(dag.tags()):
            stored_heads = store.heads({branch})
            for head in computed_heads[branch] - stored_heads:
                status.fix('Adding missing head %s in branch %s' %
                           (head, branch))
                store.add_head(head)
            for head in stored_heads - computed_heads[branch]:
                status.fix('Removing non-head reference to %s in branch %s' %
                           (head, branch))
                del store._hgheads[head]

    metadata_commit = Git.resolve_ref('refs/cinnabar/metadata')
    if status('broken'):
        Git.update_ref('refs/cinnabar/broken', metadata_commit)
        return 1

    if args.full:
        Git.update_ref('refs/cinnabar/checked', metadata_commit)
    interval_expired('fsck', 0)
    store.close()

    if status('fixed'):
        return 2
    return 0
Example #16
0
def fsck_quick(force=False):
    status = FsckStatus()
    store = GitHgStore()

    metadata_commit = Git.resolve_ref('refs/cinnabar/metadata')
    if not metadata_commit:
        status.info('There does not seem to be any git-cinnabar metadata.\n'
                    'Is this a git-cinnabar clone?')
        return 1
    broken_metadata = Git.resolve_ref('refs/cinnabar/broken')
    checked_metadata = Git.resolve_ref('refs/cinnabar/checked')
    if checked_metadata == broken_metadata:
        checked_metadata = None
    if metadata_commit == checked_metadata and not force:
        status.info('The git-cinnabar metadata was already checked and is '
                    'presumably clean.\n'
                    'Try `--force` if you want to check anyways.')
        return 0
    elif force:
        checked_metadata = None

    commit = GitCommit(metadata_commit)
    if commit.body != 'files-meta unified-manifests-v2':
        status.info(
            'The git-cinnabar metadata is incompatible with this version.\n'
            'Please use the git-cinnabar version it was used with last.\n')
        return 1
    if len(commit.parents) > 6 or len(commit.parents) < 5:
        status.report('The git-cinnabar metadata seems to be corrupted in '
                      'unexpected ways.\n')
        return 1
    changesets, manifests, hg2git, git2hg, files_meta = commit.parents[:5]

    commit = GitCommit(changesets)
    heads = OrderedDict(
        (node, branch)
        for node, _, branch in (d.partition(' ')
                                for d in commit.body.splitlines()))
    if len(heads) != len(commit.parents):
        status.report('The git-cinnabar metadata seems to be corrupted in '
                      'unexpected ways.\n')
        return 1

    manifest_nodes = []

    parents = None
    fix_changeset_heads = False

    def get_checked_metadata(num):
        if not checked_metadata:
            return None
        commit = Git.resolve_ref('{}^{}'.format(checked_metadata, num))
        if commit:
            return GitCommit(commit)

    checked_commit = get_checked_metadata(1)
    # TODO: Check that the recorded heads are actually dag heads.
    for c, changeset_node in progress_iter(
            'Checking {} changeset heads',
        ((c, node) for c, node in izip(commit.parents, heads)
         if not checked_commit or c not in checked_commit.parents)):
        gitsha1 = GitHgHelper.hg2git(changeset_node)
        if gitsha1 == NULL_NODE_ID:
            status.report('Missing hg2git metadata for changeset %s' %
                          changeset_node)
            continue
        if gitsha1 != c:
            if parents is None:
                parents = set(commit.parents)
            if gitsha1 not in parents:
                status.report('Inconsistent metadata:\n'
                              '  Head metadata says changeset %s maps to %s\n'
                              '  but hg2git metadata says it maps to %s' %
                              (changeset_node, c, gitsha1))
                continue
            fix_changeset_heads = True
        changeset = store._changeset(c, include_parents=True)
        if not changeset:
            status.report('Missing git2hg metadata for git commit %s' % c)
            continue
        if changeset.node != changeset_node:
            if changeset.node not in heads:
                status.report(
                    'Inconsistent metadata:\n'
                    '  Head metadata says %s maps to changeset %s\n'
                    '  but git2hg metadata says it maps to changeset %s' %
                    (c, changeset_node, changeset.node))
                continue
            fix_changeset_heads = True
        if changeset.node != changeset.sha1:
            status.report('Sha1 mismatch for changeset %s' % changeset.node)
            continue
        changeset_branch = changeset.branch or 'default'
        if heads[changeset.node] != changeset_branch:
            status.report(
                'Inconsistent metadata:\n'
                '  Head metadata says changeset %s is in branch %s\n'
                '  but git2hg metadata says it is in branch %s' %
                (changeset.node, heads[changeset.node], changeset_branch))
            continue
        manifest_nodes.append(changeset.manifest)

    if status('broken'):
        return 1

    # Rebuilding manifests benefits from limiting the difference with
    # the last rebuilt manifest. Similarly, building the list of unique
    # files in all manifests benefits from that too.
    # Unfortunately, the manifest heads are not ordered in a topological
    # relevant matter, and the differences between two consecutive manifests
    # can be much larger than they could be. The consequence is spending a
    # large amount of time rebuilding the manifests and gathering the files
    # list. It's actually faster to attempt to reorder them according to
    # some heuristics first, such that the differences are smaller.
    # Here, we use the depth from the root node(s) to reorder the manifests.
    # This doesn't give the most optimal ordering, but it's already much
    # faster. On a clone of multiple mozilla-* repositories with > 1400 heads,
    # it's close to an order of magnitude difference on the "Checking
    # manifests" loop.
    depths = {}
    roots = {}
    manifest_queue = []
    revs = []
    revs.append('{}^@'.format(manifests))
    if checked_metadata:
        revs.append('^{}^2^@'.format(checked_metadata))
    for m, _, parents in progress_iter(
            'Loading {} manifests',
            GitHgHelper.rev_list('--topo-order', '--reverse', '--full-history',
                                 *revs)):
        manifest_queue.append((m, parents))
        if parents:
            depth = {}
            for p in parents:
                for root, num in depths.get(p, {}).iteritems():
                    if root in depth:
                        depth[root] = max(depth[root], num + 1)
                    else:
                        depth[root] = num + 1
            if depth:
                depths[m] = depth
                del depth
                continue
        depths[m] = {m: 0}
        roots[m] = parents

    if status('broken'):
        return 1

    # TODO: check that all manifest_nodes gathered above are available in the
    # manifests dag, and that the dag heads are the recorded heads.
    manifests_commit = GitCommit(manifests)
    checked_commit = get_checked_metadata(2)
    depths = [([depths[p].get(r, 0) for r in roots], p)
              for p in manifests_commit.parents
              if not checked_commit or p not in checked_commit.parents]
    manifests_commit_parents = [p for _, p in sorted(depths)]
    previous = None
    all_interesting = set()
    for m in progress_iter('Checking {} manifest heads',
                           manifests_commit_parents):
        c = GitCommit(m)
        if not SHA1_RE.match(c.body):
            status.report('Invalid manifest metadata in git commit %s' % m)
            continue
        gitsha1 = GitHgHelper.hg2git(c.body)
        if gitsha1 == NULL_NODE_ID:
            status.report('Missing hg2git metadata for manifest %s' % c.body)
            continue
        if not GitHgHelper.check_manifest(c.body):
            status.report('Sha1 mismatch for manifest %s' % c.body)

        files = {}
        if previous:
            for _, _, before, after, d, path in GitHgHelper.diff_tree(
                    previous, m):
                if d in 'AM' and before != after and \
                        (path, after) not in all_interesting:
                    files[path] = after
        else:
            for _, t, sha1, path in GitHgHelper.ls_tree(m, recursive=True):
                if (path, sha1) not in all_interesting:
                    files[path] = sha1
        all_interesting.update(files.iteritems())
        previous = m

    if status('broken'):
        return 1

    # Don't check files that were already there in the previously checked
    # manifests.
    previous = None
    for parents in roots.itervalues():
        for p in parents:
            if previous:
                for _, _, before, after, d, path in GitHgHelper.diff_tree(
                        previous, p):
                    if d in 'AM' and before != after:
                        all_interesting.discard((path, after))
            else:
                for _, t, sha1, path in GitHgHelper.ls_tree(p, recursive=True):
                    all_interesting.discard((path, sha1))
            previous = p

    progress = Progress('Checking {} files')
    while all_interesting and manifest_queue:
        (m, parents) = manifest_queue.pop()
        changes = get_changes(m, parents, all=True)
        for path, hg_file, hg_fileparents in changes:
            if hg_fileparents[1:] == (hg_file, ):
                continue
            elif hg_fileparents[:1] == (hg_file, ):
                continue
            # Reaching here means the file received a modification compared
            # to its parents. If it's a file we're going to check below,
            # it means we don't need to check its parents if somehow they were
            # going to be checked. If it's not a file we're going to check
            # below, it's because it's either a file we weren't interested in
            # in the first place, or it's the parent of a file we have checked.
            # Either way, we aren't interested in the parents.
            for p in hg_fileparents:
                all_interesting.discard((path, p))
            if (path, hg_file) not in all_interesting:
                continue
            all_interesting.remove((path, hg_file))
            if not GitHgHelper.check_file(hg_file, *hg_fileparents):
                p = store.manifest_path(path)
                status.report('Sha1 mismatch for file %s\n'
                              '  revision %s' % (p, hg_file))

                print_parents = ' '.join(p for p in hg_fileparents
                                         if p != NULL_NODE_ID)
                if print_parents:
                    status.report('  with parent%s %s' %
                                  ('s' if len(print_parents) > 41 else '',
                                   print_parents))
            progress.progress()
    progress.finish()
    if all_interesting:
        status.info('Could not find the following files:')
        for path, sha1 in sorted(all_interesting):
            p = store.manifest_path(path)
            status.info('  %s %s' % (sha1, path))
        status.info('This might be a bug in `git cinnabar fsck`. Please open '
                    'an issue, with the message above, on\n'
                    'https://github.com/glandium/git-cinnabar/issues')
        return 1

    if status('broken'):
        status.info('Your git-cinnabar repository appears to be corrupted.\n'
                    'Please open an issue, with the information above, on\n'
                    'https://github.com/glandium/git-cinnabar/issues')
        Git.update_ref('refs/cinnabar/broken', metadata_commit)
        if checked_metadata:
            status.info(
                '\nThen please try to run `git cinnabar rollback --fsck` to '
                'restore last known state, and to update from the mercurial '
                'repository.')
        else:
            status.info('\nThen please try to run `git cinnabar reclone`.')
        status.info(
            '\nPlease note this may affect the commit sha1s of mercurial '
            'changesets, and may require to rebase your local branches.')
        status.info(
            '\nAlternatively, you may start afresh with a new clone. In any '
            'case, please keep this corrupted repository around for further '
            'debugging.')
        return 1

    refresh = []
    if fix_changeset_heads:
        status.fix('Fixing changeset heads metadata order.')
        refresh.append('refs/cinnabar/changesets')
    interval_expired('fsck', 0)
    store.close(refresh=refresh)
    GitHgHelper._helper = False
    metadata_commit = Git.resolve_ref('refs/cinnabar/metadata')
    Git.update_ref('refs/cinnabar/checked', metadata_commit)
    return 0
Example #17
0
def main(args):
    logger = logging.getLogger('-')
    logger.info(args)
    assert len(args) == 2
    remote, url = args
    git_dir = os.environ.get('GIT_DIR')
    if Git.config('core.ignorecase', 'bool') == 'true':
        sys.stderr.write(
            'Your git configuration has core.ignorecase set to "true".\n'
            'Usually, this means git detected the file system is case '
            'insensitive.\n'
            'Git-cinnabar does not support this setup.\n'
            'Either use a case sensitive file system or set '
            'core.ignorecase to "false".\n')
        git_work_tree = os.path.dirname(git_dir)
        if os.path.abspath(os.getcwd() + os.sep).startswith(
                os.path.abspath(git_work_tree) + os.sep) or \
                remote == 'hg::' + url or tuple(
                Git.for_each_ref('refs/remotes/%s' % remote)):
            sys.stderr.write('Use the following command to reclone:\n'
                             '  git cinnabar reclone\n')
        else:
            sys.stderr.write(
                'Use the following command to clone:\n'
                '  git -c core.ignorecase=false clone%(args)s hg::%(url)s '
                '%(dir)s\n' % {
                    'dir': git_work_tree,
                    'url': url,
                    'args': ' -o ' + remote if remote != 'origin' else ''
                })
        return 1
    repo = get_repo(url)
    store = GitHgStore()
    logger.info(LazyString(lambda: '%s' % store.heads()))
    helper = IOLogger(logging.getLogger('remote-helper'), sys.stdin,
                      sys.stdout)
    branchmap = None
    bookmarks = {}
    HEAD = 'branches/default/tip'

    while True:
        cmd, args = read_cmd(helper)
        if not cmd:
            break

        if cmd == 'capabilities':
            assert not args
            helper.write('option\n'
                         'import\n'
                         'bidi-import\n'
                         'push\n'
                         'refspec refs/heads/branches/*:'
                         'refs/cinnabar/refs/heads/branches/*\n'
                         'refspec refs/heads/bookmarks/*:'
                         'refs/cinnabar/refs/heads/bookmarks/*\n'
                         'refspec HEAD:refs/cinnabar/HEAD\n'
                         '\n')
            helper.flush()
        elif cmd == 'list':
            assert not args or args == ['for-push']

            if repo.capable('batch'):
                batch = repo.batch()
                branchmap = batch.branchmap()
                heads = batch.heads()
                bookmarks = batch.listkeys('bookmarks')
                batch.submit()
                branchmap = branchmap.value
                heads = heads.value
                bookmarks = bookmarks.value
            else:
                while True:
                    branchmap = repo.branchmap()
                    heads = repo.heads()
                    if heads == ['\0' * 20]:
                        heads = []
                    # Some branch heads can be non-heads topologically, but if
                    # some heads don't appear in the branchmap, then something
                    # was pushed to the repo between branchmap() and heads()
                    if set(heads).issubset(set(chain(*branchmap.values()))):
                        break
                bookmarks = repo.listkeys('bookmarks')

            branchmap = BranchMap(store, branchmap, heads)
            unknowns = False
            for branch in sorted(branchmap.names()):
                branch_tip = branchmap.tip(branch)
                for head in sorted(branchmap.heads(branch)):
                    sha1 = branchmap.git_sha1(head)
                    if sha1 == '?':
                        unknowns = True
                    if head == branch_tip:
                        continue
                    helper.write('%s refs/heads/branches/%s/%s\n' % (
                        sha1,
                        branch,
                        head,
                    ))
                if branch_tip:
                    helper.write('%s refs/heads/branches/%s/tip\n' % (
                        branchmap.git_sha1(branch_tip),
                        branch,
                    ))
            for name, sha1 in sorted(bookmarks.iteritems()):
                ref = store.changeset_ref(sha1)
                helper.write('%s refs/heads/bookmarks/%s\n' %
                             (ref if ref else '?', name))
            if not unknowns:
                for tag, ref in sorted(store.tags(branchmap.heads())):
                    helper.write('%s refs/tags/%s\n' % (ref, tag))

            if '@' in bookmarks:
                HEAD = 'bookmarks/@'
            helper.write('@refs/heads/%s HEAD\n' '\n' % HEAD)
            helper.flush()
        elif cmd == 'option':
            assert len(args) == 2
            name, value = args
            if name == 'progress':
                if value == 'true':
                    cinnabar.util.progress = True
                    helper.write('ok\n')
                elif value == 'false':
                    cinnabar.util.progress = False
                    helper.write('ok\n')
                else:
                    helper.write('unsupported\n')
            else:
                helper.write('unsupported\n')
            helper.flush()
        elif cmd == 'import':
            try:
                reflog = os.path.join(git_dir, 'logs', 'refs', 'cinnabar')
                mkpath(reflog)
                open(os.path.join(reflog, 'hg2git'), 'a').close()
                open(os.path.join(reflog, 'manifest'), 'a').close()
                assert len(args) == 1
                refs = args
                while cmd:
                    assert cmd == 'import'
                    cmd, args = read_cmd(helper)
                    assert args is None or len(args) == 1
                    if args:
                        refs.extend(args)
            except:
                # If anything wrong happens before we got all the import
                # commands, we risk git picking the existing refs/cinnabar
                # refs. Remove them.
                for line in Git.for_each_ref('refs/cinnabar/refs/heads',
                                             'refs/cinnabar/HEAD',
                                             format='%(refname)'):
                    Git.delete_ref(ref)
                raise

            try:

                def resolve_head(head):
                    if head.startswith('refs/heads/branches/'):
                        head = head[20:]
                        if head[-4:] == '/tip':
                            return branchmap.tip(head[:-4])
                        return head[-40:]
                    if head.startswith('refs/heads/bookmarks/'):
                        head = head[21:]
                        return bookmarks[head]
                    if head == 'HEAD':
                        return bookmarks.get('@') or branchmap.tip('default')
                    return None

                wanted_refs = {
                    k: v
                    for k, v in ((h, resolve_head(h)) for h in refs) if v
                }
                heads = wanted_refs.values()
                if not heads:
                    heads = branchmap.heads()

                # Older versions would create a symbolic ref for
                # refs/remote-hg/HEAD. Newer versions don't, and
                # Git.update_ref doesn't remove the symbolic ref, so it needs
                # to be removed first.
                # Since git symbolic-ref only throws an error when the ref is
                # not symbolic, just try to remove the symbolic ref every time
                # and ignore errors.
                tuple(
                    Git.iter('symbolic-ref',
                             '-d',
                             'refs/remote-hg/HEAD',
                             stderr=open(os.devnull, 'wb')))

                refs_orig = {}
                for line in Git.for_each_ref(
                        'refs/cinnabar/refs/heads',
                        'refs/cinnabar/HEAD',
                        format='%(objectname) %(refname)'):
                    sha1, ref = line.split(' ', 1)
                    refs_orig[ref] = sha1
            except:
                # If anything wrong happens before we actually pull, we risk
                # git pucking the existing refs/cinnabar refs. Remove them.
                # Unlike in the case above, we now have the list of refs git
                # is expected, so we can just remove those.
                for ref in refs:
                    Git.delete_ref('refs/cinnabar/' + ref)
                raise

            try:
                store.init_fast_import(FastImport(sys.stdin, sys.stdout))
                getbundle(repo, store, heads, branchmap)
            except:
                wanted_refs = {}
                raise
            finally:
                for ref, value in wanted_refs.iteritems():
                    ref = 'refs/cinnabar/' + ref
                    if ref not in refs_orig or refs_orig[ref] != value:
                        Git.update_ref(ref, store.changeset_ref(value))
                for ref in refs_orig:
                    if ref[14:] not in wanted_refs:
                        Git.delete_ref(ref)

            store.close()

            if not remote.startswith('hg::'):
                prune = 'remote.%s.prune' % remote
                if (Git.config(prune) != 'true'
                        and Git.config('fetch.prune') != 'true'):
                    sys.stderr.write(
                        'It is recommended that you set "%(conf)s" or '
                        '"fetch.prune" to "true".\n'
                        '  git config %(conf)s true\n'
                        'or\n'
                        '  git config fetch.prune true\n' % {'conf': prune})

            if store.tag_changes:
                sys.stderr.write(
                    '\nRun the following command to update remote tags:\n')
                if not remote.startswith('hg::'):
                    sys.stderr.write('  git remote update %s\n' % remote)
                else:
                    sys.stderr.write('  git fetch --tags %s\n' % remote)

        elif cmd == 'push':
            if not remote.startswith('hg::'):
                data_pref = 'remote.%s.cinnabar-data' % remote
                data = Git.config(data_pref) or 'phase'
            else:
                data = 'phase'

            if data not in ('never', 'phase', 'always'):
                sys.stderr.write('Invalid value for %s: %s\n' %
                                 (data_pref, data))
                return 1

            refspecs = []
            refspecs.extend(args)
            while True:
                cmd, args = read_cmd(helper)
                if not cmd:
                    break
                assert cmd == 'push'
                refspecs.extend(args)
            pushes = {
                s.lstrip('+'): (d, s.startswith('+'))
                for s, d in (r.split(':', 1) for r in refspecs)
            }
            if isinstance(repo, bundlerepo):
                for source, (dest, force) in pushes.iteritems():
                    helper.write('error %s Cannot push to a bundle file\n' %
                                 dest)
                helper.write('\n')
                helper.flush()
            else:
                repo_heads = branchmap.heads()
                PushStore.adopt(store)
                pushed = push(repo, store, pushes, repo_heads,
                              branchmap.names())

                status = {}
                for source, (dest, _) in pushes.iteritems():
                    if dest.startswith('refs/tags/'):
                        if source:
                            status[dest] = 'Pushing tags is unsupported'
                        else:
                            status[dest] = \
                                'Deleting remote tags is unsupported'
                        continue
                    if not dest.startswith('refs/heads/bookmarks/'):
                        if source:
                            status[dest] = bool(len(pushed))
                        else:
                            status[dest] = \
                                'Deleting remote branches is unsupported'
                        continue
                    name = dest[21:]
                    if source:
                        source = store.hg_changeset(Git.resolve_ref(source)) \
                            or ''
                    status[dest] = repo.pushkey('bookmarks', name,
                                                bookmarks.get(name, ''),
                                                source)

                for source, (dest, force) in pushes.iteritems():
                    if status[dest] is True:
                        helper.write('ok %s\n' % dest)
                    elif status[dest]:
                        helper.write('error %s %s\n' % (dest, status[dest]))
                    else:
                        helper.write('error %s nothing changed on remote\n' %
                                     dest)
                helper.write('\n')
                helper.flush()

                if not pushed:
                    data = False
                elif data == 'always':
                    data = True
                elif data == 'phase':
                    phases = repo.listkeys('phases')
                    drafts = {}
                    if not phases.get('publishing', False):
                        drafts = set(p for p, is_draft in phases.iteritems()
                                     if int(is_draft))
                    if not drafts:
                        data = True
                    else:

                        def draft_commits():
                            for d in drafts:
                                c = store.changeset_ref(d)
                                if c:
                                    yield '^%s^@' % c
                            for h in pushed.heads():
                                yield h

                        args = [
                            'rev-list', '--ancestry-path', '--topo-order',
                            '--stdin'
                        ]

                        pushed_drafts = tuple(
                            Git.iter(*args, stdin=draft_commits()))

                        # Theoretically, we could have commits with no
                        # metadata that the remote declares are public, while
                        # the rest of our push is in a draft state. That is
                        # however so unlikely that it's not worth the effort
                        # to support partial metadata storage.
                        data = not bool(pushed_drafts)
                elif data == 'never':
                    data = False

                store.close(rollback=not data)

    store.close()