def __init__(self, store, remote_branchmap, remote_heads):
self._heads = {}
self._all_heads = tuple(autohexlify(h) for h in remote_heads)
self._tips = {}
self._git_sha1s = {}
self._unknown_heads = set()
for branch, heads in util.iteritems(remote_branchmap):
# We can't keep track of tips if the list of heads is not sequenced
sequenced = isinstance(heads, Sequence) or len(heads) == 1
branch_heads = []
for head in heads:
head = autohexlify(head)
branch_heads.append(head)
sha1 = store.changeset_ref(head)
if not sha1:
self._unknown_heads.add(head)
continue
assert head not in self._git_sha1s
self._git_sha1s[head] = sha1
# Use last non-closed head as tip if there's more than one head.
# Caveat: we don't know a head is closed until we've pulled it.
if branch and heads and sequenced:
for head in reversed(branch_heads):
self._tips[branch] = head
if head in self._git_sha1s:
changeset = store.changeset(head)
if changeset.close:
continue
break
if branch:
self._heads[branch] = tuple(branch_heads)
def tags(self):
tags = TagSet()
heads = sorted((n, h) for h, (b, n) in util.iteritems(self._hgheads))
for _, h in heads:
h = self.changeset_ref(h)
tags.update(self._get_hgtags(h))
for tag, node in tags:
if node != NULL_NODE_ID:
yield tag, node
def update(self, other):
if not other:
return
assert isinstance(other, TagSet)
for key, anode in util.iteritems(other._tags):
# derived from mercurial's _updatetags
ahist = other._taghist[key]
if key not in self._tags:
self._tags[key] = anode
self._taghist[key] = set(ahist)
continue
bnode = self._tags[key]
bhist = self._taghist[key]
if (bnode != anode and anode in bhist and
(bnode not in ahist or len(bhist) > len(ahist))):
anode = bnode
self._tags[key] = anode
self._taghist[key] = ahist | set(
n for n in bhist if n not in ahist)
def bundle_data(store, commits):
manifests = OrderedDict()
files = defaultdict(list)
for node, parents in progress_iter('Bundling {} changesets', commits):
if len(parents) > 2:
raise Exception(
'Pushing octopus merges to mercurial is not supported')
changeset_data = store.read_changeset_data(node)
is_new = changeset_data is None or check_enabled('bundle')
if is_new:
store.create_hg_metadata(node, parents)
hg_changeset = store._changeset(node, include_parents=True)
if is_new:
store.add_head(hg_changeset.node, hg_changeset.parent1,
hg_changeset.parent2)
yield hg_changeset
manifest = hg_changeset.manifest
if manifest not in manifests and manifest != NULL_NODE_ID:
if manifest not in (store.changeset(p).manifest
for p in hg_changeset.parents):
manifests[manifest] = hg_changeset.node
yield None
for manifest, changeset in progress_iter('Bundling {} manifests',
iteritems(manifests)):
hg_manifest = store.manifest(manifest, include_parents=True)
hg_manifest.changeset = changeset
yield hg_manifest
manifest_ref = store.manifest_ref(manifest)
parents = tuple(store.manifest_ref(p) for p in hg_manifest.parents)
changes = get_changes(manifest_ref, parents)
for path, hg_file, hg_fileparents in changes:
if hg_file != NULL_NODE_ID:
files[store.manifest_path(path)].append(
(hg_file, hg_fileparents, changeset, parents))
yield None
def iter_files(files):
count_chunks = 0
for count_names, path in enumerate(sorted(files), 1):
yield (count_chunks, count_names), path
nodes = set()
for node, parents, changeset, mn_parents in files[path]:
if node in nodes:
continue
count_chunks += 1
nodes.add(node)
file = store.file(node, parents, mn_parents, path)
file.changeset = changeset
assert file.node == file.sha1
yield (count_chunks, count_names), file
yield (count_chunks, count_names), None
for chunk in progress_enum('Bundling {} revisions of {} files',
iter_files(files)):
yield chunk
yield None
def create_hg_manifest(self, commit, parents):
manifest = GeneratedManifestInfo(NULL_NODE_ID)
changeset_files = []
if parents:
parent_changeset = self.changeset(self.hg_changeset(parents[0]))
parent_manifest = self.manifest(parent_changeset.manifest)
parent_node = parent_manifest.node
if len(parents) == 2:
parent2_changeset = self.changeset(self.hg_changeset(parents[1]))
parent2_manifest = self.manifest(parent2_changeset.manifest)
parent2_node = parent2_manifest.node
if parent_node == parent2_node:
parents = parents[:1]
if not parents:
for line in Git.ls_tree(commit, recursive=True):
mode, typ, sha1, path = line
node = self.create_file(sha1,
git_manifest_parents=(),
path=path)
manifest.add(path, node, self.ATTR[mode], modified=True)
changeset_files.append(path)
manifest.parents = []
manifest.delta_node = NULL_NODE_ID
return manifest, changeset_files
elif len(parents) == 2:
if not experiment('merge'):
raise Exception('Pushing merges is not supported yet')
if not self._merge_warn:
logging.warning('Pushing merges is experimental.')
logging.warning('This may irremediably push bad state to the '
'mercurial server!')
self._merge_warn = 1
git_manifests = (self.manifest_ref(parent_node),
self.manifest_ref(parent2_node))
# TODO: this would benefit from less git queries
changes = list(get_changes(commit, parents))
files = [
(path, mode, sha1)
for mode, _, sha1, path in Git.ls_tree(commit, recursive=True)
]
manifests = sorted_merge(parent_manifest,
parent2_manifest,
key=lambda i: i.path,
non_key=lambda i: i)
for line in sorted_merge(files, sorted_merge(changes, manifests)):
path, f, (change, (manifest_line_p1, manifest_line_p2)) = line
if not f: # File was removed
if manifest_line_p1:
manifest.removed.add(path)
changeset_files.append(path)
continue
mode, sha1 = f
attr = self.ATTR[mode]
if manifest_line_p1 and not manifest_line_p2:
file_parents = (manifest_line_p1.sha1, )
elif manifest_line_p2 and not manifest_line_p1:
file_parents = (manifest_line_p2.sha1, )
elif not manifest_line_p1 and not manifest_line_p2:
file_parents = ()
elif manifest_line_p1.sha1 == manifest_line_p2.sha1:
file_parents = (manifest_line_p1.sha1, )
else:
if self._merge_warn == 1:
logging.warning('This may take a while...')
self._merge_warn = 2
file_parents = (manifest_line_p1.sha1,
manifest_line_p2.sha1)
assert file_parents is not None
f = self._create_file_internal(
sha1,
*file_parents,
git_manifest_parents=git_manifests,
path=path)
file_parents = tuple(p for p in (f.parent1, f.parent2)
if p != NULL_NODE_ID)
merged = len(file_parents) == 2
if not merged and file_parents:
if self.git_file_ref(file_parents[0]) == sha1:
node = file_parents[0]
else:
merged = True
if merged:
node = self._store_file_internal(f)
else:
node = file_parents[0]
attr_change = (manifest_line_p1
and manifest_line_p1.attr != attr)
manifest.add(path, node, attr, modified=merged or attr_change)
if merged or attr_change:
changeset_files.append(path)
if manifest.raw_data == parent_manifest.raw_data:
return parent_manifest, []
manifest.parents = (parent_node, parent2_node)
return manifest, changeset_files
def process_diff(diff):
for (mode_before, mode_after, sha1_before, sha1_after, status,
path) in diff:
if status[:1] == b'R':
yield status[1:], (b'000000', sha1_before, NULL_NODE_ID,
b'D')
yield path, (mode_after, sha1_before, sha1_after, status)
git_diff = sorted(l for l in process_diff(
GitHgHelper.diff_tree(parents[0], commit, detect_copy=True)))
if not git_diff:
return parent_manifest, []
parent_lines = OrderedDict((l.path, l) for l in parent_manifest)
items = manifest.items
for line in sorted_merge(iteritems(parent_lines),
git_diff,
non_key=lambda i: i[1]):
path, manifest_line, change = line
if not change:
items.append(manifest_line)
continue
mode_after, sha1_before, sha1_after, status = change
path2 = status[1:]
status = status[:1]
attr = self.ATTR.get(mode_after)
if status == b'D':
manifest.removed.add(path)
changeset_files.append(path)
continue
if status in b'MT':
if sha1_before == sha1_after:
node = manifest_line.sha1
else:
node = self.create_file(
sha1_after,
manifest_line.sha1,
git_manifest_parents=(
self.manifest_ref(parent_node), ),
path=path)
elif status in b'RC':
if sha1_after != EMPTY_BLOB:
node = self.create_copy(
(path2, parent_lines[path2].sha1),
sha1_after,
git_manifest_parents=(
self.manifest_ref(parent_node), ),
path=path)
else:
node = self.create_file(
sha1_after,
git_manifest_parents=(
self.manifest_ref(parent_node), ),
path=path)
else:
assert status == b'A'
node = self.create_file(
sha1_after,
git_manifest_parents=(self.manifest_ref(parent_node), ),
path=path)
manifest.add(path, node, attr, modified=True)
changeset_files.append(path)
manifest.parents = (parent_node, )
manifest.delta_node = parent_node
return manifest, changeset_files
def heads(self, branches={}):
if not isinstance(branches, (dict, set)):
branches = set(branches)
return set(h for h, (b, _) in util.iteritems(self._hgheads)
if not branches or b in branches)
def merge(self, git_repo_url, hg_repo_url, branch=None):
# Eventually we'll want to handle a full merge, but for now, we only
# handle the case where we don't have metadata to begin with.
# The caller should avoid calling this function otherwise.
assert not self._has_metadata
remote_refs = OrderedDict()
for line in Git.iter('ls-remote', fsdecode(git_repo_url),
stderr=open(os.devnull, 'wb')):
sha1, ref = line.split(None, 1)
remote_refs[ref] = sha1
bundle = None
if not remote_refs and urlparse(git_repo_url).scheme in (b'http',
b'https'):
try:
bundle = HTTPReader(git_repo_url)
except URLError as e:
logging.error(e.reason)
return False
BUNDLE_SIGNATURE = b'# v2 git bundle\n'
signature = bundle.read(len(BUNDLE_SIGNATURE))
if signature != BUNDLE_SIGNATURE:
logging.error('Could not find cinnabar metadata')
return False
bundle = io.BufferedReader(bundle)
while True:
line = bundle.readline().rstrip()
if not line:
break
sha1, ref = line.split(b' ', 1)
remote_refs[ref] = sha1
if branch:
branches = [branch]
else:
branches = self._try_merge_branches(hg_repo_url)
ref = self._find_branch(branches, remote_refs)
if ref is None:
logging.error('Could not find cinnabar metadata')
return False
if bundle:
args = ('-v',) if util.progress else ()
proc = GitProcess('index-pack', '--stdin', '--fix-thin', *args,
stdin=subprocess.PIPE,
stdout=open(os.devnull, 'wb'))
shutil.copyfileobj(bundle, proc.stdin)
else:
fetch = ['fetch', '--no-tags', '--no-recurse-submodules', '-q']
fetch.append('--progress' if util.progress else '--no-progress')
fetch.append(fsdecode(git_repo_url))
cmd = fetch + [fsdecode(ref) + ':refs/cinnabar/fetch']
proc = GitProcess(*cmd, stdout=sys.stdout)
if proc.wait():
logging.error('Failed to fetch cinnabar metadata.')
return False
# Do some basic validation on the metadata we just got.
commit = GitCommit(remote_refs[ref])
if b'cinnabar@git' not in commit.author:
logging.error('Invalid cinnabar metadata.')
return False
flags = set(commit.body.split())
if b'files-meta' not in flags or b'unified-manifests-v2' not in flags \
or len(commit.parents) != len(self.METADATA_REFS):
logging.error('Invalid cinnabar metadata.')
return False
# At this point, we'll just assume this is good enough.
# Get replace refs.
if commit.tree != EMPTY_TREE:
errors = False
by_sha1 = {}
for k, v in util.iteritems(remote_refs):
if v not in by_sha1:
by_sha1[v] = k
needed = []
for line in Git.ls_tree(commit.tree):
mode, typ, sha1, path = line
if sha1 in by_sha1:
ref = b'refs/cinnabar/replace/%s' % path
if bundle:
Git.update_ref(ref, sha1)
else:
needed.append(
fsdecode(b':'.join((by_sha1[sha1], ref))))
else:
logging.error('Missing commit: %s', sha1)
errors = True
if errors:
return False
if not bundle:
cmd = fetch + needed
proc = GitProcess(*cmd, stdout=sys.stdout)
if proc.wait():
logging.error('Failed to fetch cinnabar metadata.')
return False
Git.update_ref(b'refs/cinnabar/metadata', commit.sha1)
self._metadata_sha1 = commit.sha1
GitHgHelper.reload()
Git.delete_ref(b'refs/cinnabar/fetch')
# TODO: avoid the duplication of code with __init__
metadata = self.metadata()
if not metadata:
# This should never happen, but just in case.
logging.warn('Could not find cinnabar metadata')
Git.delete_ref(b'refs/cinnabar/metadata')
GitHgHelper.reload()
return False
metadata, refs = metadata
self._has_metadata = True
self._metadata_refs = refs if metadata else {}
changesets_ref = self._metadata_refs.get(b'refs/cinnabar/changesets')
self._generation = 0
if changesets_ref:
commit = GitCommit(changesets_ref)
for n, head in enumerate(commit.body.splitlines()):
hghead, branch = head.split(b' ', 1)
self._hgheads._previous[hghead] = (branch, 1)
self._generation = n + 1
self._manifest_heads_orig = set(GitHgHelper.heads(b'manifests'))
for line in Git.ls_tree(metadata.tree):
mode, typ, sha1, path = line
self._replace[path] = sha1
return True
def __iter__(self):
return util.iteritems(self._tags)
def close(self, refresh=()):
if self._closed:
return
if self._graft:
self._graft.close()
self._closed = True
# If the helper is not running, we don't have anything to update.
if not GitHgHelper._helper:
return
update_metadata = {}
tree = GitHgHelper.store(b'metadata', b'hg2git')
if tree != NULL_NODE_ID:
hg2git = self._metadata_refs.get(b'refs/cinnabar/hg2git')
with GitHgHelper.commit(
ref=b'refs/cinnabar/hg2git',
) as commit:
commit.write(b'M 040000 %s \n' % tree)
if commit.sha1 != hg2git:
update_metadata[b'refs/cinnabar/hg2git'] = commit.sha1
tree = GitHgHelper.store(b'metadata', b'git2hg')
if tree != NULL_NODE_ID:
notes = self._metadata_refs.get(b'refs/notes/cinnabar')
with GitHgHelper.commit(
ref=b'refs/notes/cinnabar',
) as commit:
commit.write(b'M 040000 %s \n' % tree)
if commit.sha1 != notes:
update_metadata[b'refs/notes/cinnabar'] = commit.sha1
hg_changeset_heads = list(self._hgheads)
changeset_heads = list(self.changeset_ref(h)
for h in hg_changeset_heads)
if (any(self._hgheads.iterchanges()) or
b'refs/cinnabar/changesets' in refresh):
heads = sorted((self._hgheads[h][1], self._hgheads[h][0], h, g)
for h, g in zip(hg_changeset_heads,
changeset_heads))
with GitHgHelper.commit(
ref=b'refs/cinnabar/changesets',
parents=list(h for _, __, ___, h in heads),
message=b'\n'.join(b'%s %s' % (h, b) for _, b, h, __ in heads),
) as commit:
pass
update_metadata[b'refs/cinnabar/changesets'] = commit.sha1
changeset_heads = set(changeset_heads)
manifest_heads = GitHgHelper.heads(b'manifests')
if (set(manifest_heads) != self._manifest_heads_orig or
(b'refs/cinnabar/changesets' in update_metadata and
not manifest_heads) or b'refs/cinnabar/manifests' in refresh):
with GitHgHelper.commit(
ref=b'refs/cinnabar/manifests',
parents=sorted(manifest_heads),
) as commit:
pass
update_metadata[b'refs/cinnabar/manifests'] = commit.sha1
tree = GitHgHelper.store(b'metadata', b'files-meta')
files_meta_ref = self._metadata_refs.get(b'refs/cinnabar/files-meta')
if update_metadata and (tree != NULL_NODE_ID or not files_meta_ref):
with GitHgHelper.commit(
ref=b'refs/cinnabar/files-meta',
) as commit:
if tree != NULL_NODE_ID:
commit.write(b'M 040000 %s \n' % tree)
if commit.sha1 != files_meta_ref:
update_metadata[b'refs/cinnabar/files-meta'] = commit.sha1
replace_changed = False
for status, ref, sha1 in self._replace.iterchanges():
if status == VersionedDict.REMOVED:
Git.delete_ref(b'refs/cinnabar/replace/%s' % ref)
else:
Git.update_ref(b'refs/cinnabar/replace/%s' % ref, sha1)
replace_changed = True
if update_metadata or replace_changed:
parents = list(update_metadata.get(r) or self._metadata_refs[r]
for r in self.METADATA_REFS)
metadata_sha1 = (Git.config('cinnabar.previous-metadata') or
self._metadata_sha1)
if metadata_sha1:
parents.append(metadata_sha1)
with GitHgHelper.commit(
ref=b'refs/cinnabar/metadata',
parents=parents,
message=b' '.join(sorted(self.FLAGS)),
) as commit:
for sha1, target in util.iteritems(self._replace):
commit.filemodify(sha1, target, b'commit')
for c in self._tagcache:
if c not in changeset_heads:
self._tagcache[c] = False
for c in changeset_heads:
if c not in self._tagcache:
tags = self._get_hgtags(c)
files = set(util.itervalues(self._tagcache))
deleted = set()
created = {}
for f in self._tagcache_items:
if (f not in self._tagcache and f not in self._tagfiles or
f not in files and f in self._tagfiles):
deleted.add(f)
def tagset_lines(tags):
for tag, value in tags:
yield b'%s\0%s %s\n' % (tag, value,
b' '.join(tags.hist(tag)))
for f, tags in util.iteritems(self._tags):
if f not in self._tagfiles and f != NULL_NODE_ID:
data = b''.join(tagset_lines(tags))
mark = GitHgHelper.put_blob(data=data)
created[f] = (mark, b'exec')
if created or deleted:
self.tag_changes = True
for c, f in util.iteritems(self._tagcache):
if (f and c not in self._tagcache_items):
if f == NULL_NODE_ID:
created[c] = (f, b'commit')
else:
created[c] = (f, b'regular')
elif f is False and c in self._tagcache_items:
deleted.add(c)
if created or deleted:
with GitHgHelper.commit(
ref=b'refs/cinnabar/tag_cache',
from_commit=self._tagcache_ref,
) as commit:
for f in deleted:
commit.filedelete(f)
for f, (filesha1, typ) in util.iteritems(created):
commit.filemodify(f, filesha1, typ)
# refs/notes/cinnabar is kept for convenience
for ref in update_metadata:
if ref not in (b'refs/notes/cinnabar',):
Git.delete_ref(ref)
if self._metadata_sha1 and update_metadata and not refresh and \
interval_expired('fsck', 86400 * 7):
logging.warn('Have you run `git cinnabar fsck` recently?')
GitHgHelper.close(rollback=False)
def push(self, *refspecs):
try:
default = b'never' if self._graft else b'phase'
values = {
None: default,
b'': default,
b'never': b'never',
b'phase': b'phase',
b'always': b'always',
}
data = Git.config('cinnabar.data', self._remote.name,
values=values)
except InvalidConfig as e:
logging.error(str(e))
return 1
pushes = list((Git.resolve_ref(fsdecode(s.lstrip(b'+'))), d,
s.startswith(b'+'))
for s, d in (r.split(b':', 1) for r in refspecs))
if not self._repo.capable(b'unbundle'):
for source, dest, force in pushes:
self._helper.write(
b'error %s Remote does not support the "unbundle" '
b'capability\n' % dest)
self._helper.write(b'\n')
self._helper.flush()
else:
repo_heads = self._branchmap.heads()
PushStore.adopt(self._store, self._graft)
pushed = push(self._repo, self._store, pushes, repo_heads,
self._branchmap.names(), self._dry_run)
status = {}
for source, dest, _ in pushes:
if dest.startswith(b'refs/tags/'):
if source:
status[dest] = b'Pushing tags is unsupported'
else:
status[dest] = \
b'Deleting remote tags is unsupported'
continue
bookmark_prefix = strip_suffix(
(self._bookmark_template or b''), b'%s')
if not bookmark_prefix or not dest.startswith(bookmark_prefix):
if source:
status[dest] = bool(len(pushed))
else:
status[dest] = \
b'Deleting remote branches is unsupported'
continue
name = unquote_to_bytes(dest[len(bookmark_prefix):])
if source:
source = self._store.hg_changeset(source)
status[dest] = self._repo.pushkey(
b'bookmarks', name, self._bookmarks.get(name, b''),
source or b'')
for source, dest, force in pushes:
if status[dest] is True:
self._helper.write(b'ok %s\n' % dest)
elif status[dest]:
self._helper.write(b'error %s %s\n' % (dest, status[dest]))
else:
self._helper.write(b'error %s nothing changed on remote\n'
% dest)
self._helper.write(b'\n')
self._helper.flush()
if not pushed or self._dry_run:
data = False
elif data == b'always':
data = True
elif data == b'phase':
phases = self._repo.listkeys(b'phases')
drafts = {}
if not phases.get(b'publishing', False):
drafts = set(p for p, is_draft in iteritems(phases)
if int(is_draft))
if not drafts:
data = True
else:
def draft_commits():
for d in drafts:
c = self._store.changeset_ref(d)
if c:
yield b'^%s^@' % c
for h in pushed.heads():
yield h
args = [b'--ancestry-path', b'--topo-order']
args.extend(draft_commits())
pushed_drafts = tuple(
c for c, t, p in GitHgHelper.rev_list(*args))
# Theoretically, we could have commits with no
# metadata that the remote declares are public, while
# the rest of our push is in a draft state. That is
# however so unlikely that it's not worth the effort
# to support partial metadata storage.
data = not bool(pushed_drafts)
elif data == b'never':
data = False
self._store.close(rollback=not data)
def import_(self, *refs):
# If anything wrong happens at any time, we risk git picking
# the existing refs/cinnabar refs, so remove them preventively.
for sha1, ref in Git.for_each_ref('refs/cinnabar/refs/heads',
'refs/cinnabar/hg',
'refs/cinnabar/HEAD'):
Git.delete_ref(ref)
def resolve_head(head):
resolved = self._refs.get(head)
if resolved is None:
return resolved
if resolved.startswith(b'@'):
return self._refs.get(resolved[1:])
return resolved
wanted_refs = {k: v for k, v in (
(h, resolve_head(h)) for h in refs) if v}
heads = wanted_refs.values()
if not heads:
heads = self._branchmap.heads()
try:
# Mercurial can be an order of magnitude slower when creating
# a bundle when not giving topological heads, which some of
# the branch heads might not be.
# http://bz.selenic.com/show_bug.cgi?id=4595
# So, when we're pulling all branch heads, just ask for the
# topological heads instead.
# `heads` might contain known heads, if e.g. the remote has
# never been pulled from, but we happen to have some of its
# heads locally already.
if self._has_unknown_heads:
unknown_heads = self._branchmap.unknown_heads()
if set(heads).issuperset(unknown_heads):
heads = set(self._branchmap.heads()) & unknown_heads
getbundle(self._repo, self._store, heads,
self._branchmap.names())
except Exception:
wanted_refs = {}
raise
finally:
for ref, value in iteritems(wanted_refs):
ref = b'refs/cinnabar/' + ref
Git.update_ref(ref, self._store.changeset_ref(value))
self._store.close()
self._helper.write(b'done\n')
self._helper.flush()
if self._remote.name and self._refs_style('heads'):
if Git.config('fetch.prune', self._remote.name) != b'true':
prune = 'remote.%s.prune' % fsdecode(self._remote.name)
sys.stderr.write(
'It is recommended that you set "%(conf)s" or '
'"fetch.prune" to "true".\n'
' git config %(conf)s true\n'
'or\n'
' git config fetch.prune true\n'
% {'conf': prune}
)
if self._store.tag_changes:
sys.stderr.write(
'\nRun the following command to update tags:\n')
sys.stderr.write(' git fetch --tags hg::tags: tag "*"\n')
def list(self, arg=None):
assert not arg or arg == b'for-push'
fetch = (Git.config('cinnabar.fetch') or b'').split()
if fetch:
heads = [unhexlify(f) for f in fetch]
branchmap = {None: heads}
bookmarks = {}
elif self._repo.capable(b'batch'):
if hasattr(self._repo, 'commandexecutor'):
with self._repo.commandexecutor() as e:
branchmap = e.callcommand(b'branchmap', {})
heads = e.callcommand(b'heads', {})
bookmarks = e.callcommand(b'listkeys', {
b'namespace': b'bookmarks'
})
branchmap = branchmap.result()
heads = heads.result()
bookmarks = bookmarks.result()
elif hasattr(self._repo, b'iterbatch'):
batch = self._repo.iterbatch()
batch.branchmap()
batch.heads()
batch.listkeys(b'bookmarks')
batch.submit()
branchmap, heads, bookmarks = batch.results()
else:
batch = self._repo.batch()
branchmap = batch.branchmap()
heads = batch.heads()
bookmarks = batch.listkeys(b'bookmarks')
batch.submit()
branchmap = branchmap.value
heads = heads.value
bookmarks = bookmarks.value
if heads == [b'\0' * 20]:
heads = []
else:
while True:
branchmap = self._repo.branchmap()
heads = self._repo.heads()
if heads == [b'\0' * 20]:
heads = []
# Some branch heads can be non-heads topologically, but if
# some heads don't appear in the branchmap, then something
# was pushed to the repo between branchmap() and heads()
if set(heads).issubset(
set(chain(*(v for _, v in iteritems(branchmap))))):
break
bookmarks = self._repo.listkeys(b'bookmarks')
self._bookmarks = bookmarks
branchmap = self._branchmap = BranchMap(self._store, branchmap,
heads)
self._has_unknown_heads = bool(self._branchmap.unknown_heads())
if self._graft and self._has_unknown_heads and not arg:
self._store.prepare_graft()
get_heads = set(branchmap.heads()) & branchmap.unknown_heads()
getbundle(self._repo, self._store, get_heads, branchmap.names())
# We may have failed to graft all changesets, in which case we
# skipped them. If that's what happened, we want to create a
# new branchmap containing all we do know about, so that we can
# avoid telling git about things we don't know, because if we
# didn't, it would ask for them, and subsequently fail because
# they are missing.
# Since we can't know for sure what the right tips might be for
# each branch, we won't expose the tips. This means we don't
# need to care about the order of the heads for the new
# branchmap.
self._has_unknown_heads = any(not(self._store.changeset_ref(h))
for h in get_heads)
if self._has_unknown_heads:
new_branchmap = {
branch: set(h for h in branchmap.heads(branch))
for branch in branchmap.names()
}
new_branchmap = {
branch: set(h for h in branchmap.heads(branch)
if h not in branchmap.unknown_heads())
for branch in branchmap.names()
}
new_heads = set(h for h in branchmap.heads()
if h not in branchmap.unknown_heads())
for status, head, branch in self._store._hgheads.iterchanges():
branch_heads = new_branchmap.get(branch)
if status == VersionedDict.REMOVED:
if branch_heads and head in branch_heads:
branch_heads.remove(head)
if head in new_heads:
new_heads.remove(head)
else:
if not branch_heads:
branch_heads = new_branchmap[branch] = set()
branch_heads.add(head)
new_heads.add(head)
branchmap = self._branchmap = BranchMap(
self._store, new_branchmap, list(new_heads))
refs_style = None
refs_styles = ('bookmarks', 'heads', 'tips')
if not fetch and branchmap.heads():
refs_config = 'cinnabar.refs'
if arg == b'for-push':
if Git.config('cinnabar.pushrefs', remote=self._remote.name):
refs_config = 'cinnabar.pushrefs'
refs_style = ConfigSetFunc(refs_config, refs_styles,
remote=self._remote.name,
default='all')
refs_style = refs_style or (lambda x: True)
self._refs_style = refs_style
refs = {}
if refs_style('heads') or refs_style('tips'):
if refs_style('heads') and refs_style('tips'):
self._head_template = b'refs/heads/branches/%s/%s'
self._tip_template = b'refs/heads/branches/%s/tip'
elif refs_style('heads') and refs_style('bookmarks'):
self._head_template = b'refs/heads/branches/%s/%s'
elif refs_style('heads'):
self._head_template = b'refs/heads/%s/%s'
elif refs_style('tips') and refs_style('bookmarks'):
self._tip_template = b'refs/heads/branches/%s'
elif refs_style('tips'):
self._tip_template = b'refs/heads/%s'
for branch in sorted(branchmap.names()):
branch_tip = branchmap.tip(branch)
if refs_style('heads'):
for head in sorted(branchmap.heads(branch)):
if head == branch_tip and refs_style('tips'):
continue
refs[self._head_template % (branch, head)] = head
if branch_tip and refs_style('tips'):
refs[self._tip_template % branch] = branch_tip
if refs_style('bookmarks'):
if refs_style('heads') or refs_style('tips'):
self._bookmark_template = b'refs/heads/bookmarks/%s'
else:
self._bookmark_template = b'refs/heads/%s'
for name, sha1 in sorted(iteritems(bookmarks)):
if sha1 == NULL_NODE_ID:
continue
ref = self._store.changeset_ref(sha1)
if self._graft and not ref:
continue
refs[self._bookmark_template % name] = sha1
for f in fetch:
refs[b'hg/revs/%s' % f] = f
head_ref = None
if refs_style('bookmarks') and b'@' in bookmarks:
head_ref = self._bookmark_template % b'@'
elif refs_style('tips'):
head_ref = self._tip_template % b'default'
elif refs_style('heads'):
head_ref = self._head_template % (
b'default', branchmap.tip(b'default'))
if head_ref:
head = refs.get(head_ref)
if self._graft and head:
head = self._store.changeset_ref(head)
if head:
refs[b'HEAD'] = b'@%s' % head_ref
self._refs = {sanitize_branch_name(k): v
for k, v in iteritems(refs)}
head_prefix = strip_suffix((self._head_template or b''), b'%s/%s')
for k, v in sorted(iteritems(self._refs)):
if head_prefix and k.startswith(head_prefix):
v = self._store.changeset_ref(v) or self._branchmap.git_sha1(v)
elif not v.startswith(b'@'):
v = self._store.changeset_ref(v) or b'?'
if not self._graft or v != b'?':
self._helper.write(b'%s %s\n' % (v, k))
self._helper.write(b'\n')
self._helper.flush()
def fsck_quick(force=False):
status = FsckStatus()
store = GitHgStore()
metadata_commit = Git.resolve_ref('refs/cinnabar/metadata')
if not metadata_commit:
status.info('There does not seem to be any git-cinnabar metadata.\n'
'Is this a git-cinnabar clone?')
return 1
broken_metadata = Git.resolve_ref('refs/cinnabar/broken')
checked_metadata = Git.resolve_ref('refs/cinnabar/checked')
if checked_metadata == broken_metadata:
checked_metadata = None
if metadata_commit == checked_metadata and not force:
status.info('The git-cinnabar metadata was already checked and is '
'presumably clean.\n'
'Try `--force` if you want to check anyways.')
return 0
elif force:
checked_metadata = None
commit = GitCommit(metadata_commit)
if commit.body != b'files-meta unified-manifests-v2':
status.info(
'The git-cinnabar metadata is incompatible with this version.\n'
'Please use the git-cinnabar version it was used with last.\n')
return 1
if len(commit.parents) > 6 or len(commit.parents) < 5:
status.report('The git-cinnabar metadata seems to be corrupted in '
'unexpected ways.\n')
return 1
changesets, manifests, hg2git, git2hg, files_meta = commit.parents[:5]
commit = GitCommit(changesets)
heads = OrderedDict(
(node, branch)
for node, _, branch in (d.partition(b' ')
for d in commit.body.splitlines()))
if len(heads) != len(commit.parents):
status.report('The git-cinnabar metadata seems to be corrupted in '
'unexpected ways.\n')
return 1
manifest_nodes = []
parents = None
fix_changeset_heads = False
def get_checked_metadata(num):
if not checked_metadata:
return None
commit = Git.resolve_ref('{}^{}'.format(
checked_metadata.decode('ascii'), num))
if commit:
return GitCommit(commit)
checked_commit = get_checked_metadata(1)
# TODO: Check that the recorded heads are actually dag heads.
for c, changeset_node in progress_iter(
'Checking {} changeset heads',
((c, node) for c, node in zip(commit.parents, heads)
if not checked_commit or c not in checked_commit.parents)):
gitsha1 = GitHgHelper.hg2git(changeset_node)
if gitsha1 == NULL_NODE_ID:
status.report('Missing hg2git metadata for changeset %s' %
changeset_node.decode('ascii'))
continue
if gitsha1 != c:
if parents is None:
parents = set(commit.parents)
if gitsha1 not in parents:
status.report('Inconsistent metadata:\n'
' Head metadata says changeset %s maps to %s\n'
' but hg2git metadata says it maps to %s' %
(changeset_node.decode('ascii'),
c.decode('ascii'), gitsha1.decode('ascii')))
continue
fix_changeset_heads = True
changeset = store._changeset(c, include_parents=True)
if not changeset:
status.report('Missing git2hg metadata for git commit %s' %
c.decode('ascii'))
continue
if changeset.node != changeset_node:
if changeset.node not in heads:
status.report(
'Inconsistent metadata:\n'
' Head metadata says %s maps to changeset %s\n'
' but git2hg metadata says it maps to changeset %s' %
(c.decode('ascii'), changeset_node.decode('ascii'),
changeset.node.decode('ascii')))
continue
fix_changeset_heads = True
if changeset.node != changeset.sha1:
status.report('Sha1 mismatch for changeset %s' %
changeset.node.decode('ascii'))
continue
changeset_branch = changeset.branch or b'default'
if heads[changeset.node] != changeset_branch:
status.report(
'Inconsistent metadata:\n'
' Head metadata says changeset %s is in branch %s\n'
' but git2hg metadata says it is in branch %s' %
(changeset.node.decode('ascii'), fsdecode(
heads[changeset.node]), fsdecode(changeset_branch)))
continue
manifest_nodes.append(changeset.manifest)
if status('broken'):
return 1
# Rebuilding manifests benefits from limiting the difference with
# the last rebuilt manifest. Similarly, building the list of unique
# files in all manifests benefits from that too.
# Unfortunately, the manifest heads are not ordered in a topological
# relevant matter, and the differences between two consecutive manifests
# can be much larger than they could be. The consequence is spending a
# large amount of time rebuilding the manifests and gathering the files
# list. It's actually faster to attempt to reorder them according to
# some heuristics first, such that the differences are smaller.
# Here, we use the depth from the root node(s) to reorder the manifests.
# This doesn't give the most optimal ordering, but it's already much
# faster. On a clone of multiple mozilla-* repositories with > 1400 heads,
# it's close to an order of magnitude difference on the "Checking
# manifests" loop.
depths = {}
roots = set()
manifest_queue = []
revs = []
revs.append(b'%s^@' % manifests)
if checked_metadata:
revs.append(b'^%s^2^@' % checked_metadata)
for m, _, parents in progress_iter(
'Loading {} manifests',
GitHgHelper.rev_list(b'--topo-order', b'--reverse',
b'--full-history', *revs)):
manifest_queue.append((m, parents))
for p in parents:
if p not in depths:
roots.add(p)
depths[m] = max(depths.get(p, 0) + 1, depths.get(m, 0))
if status('broken'):
return 1
# TODO: check that all manifest_nodes gathered above are available in the
# manifests dag, and that the dag heads are the recorded heads.
manifests_commit = GitCommit(manifests)
checked_commit = get_checked_metadata(2)
depths = [(depths.get(p, 0), p) for p in manifests_commit.parents
if not checked_commit or p not in checked_commit.parents]
manifests_commit_parents = [p for _, p in sorted(depths)]
previous = None
all_interesting = set()
for m in progress_iter('Checking {} manifest heads',
manifests_commit_parents):
c = GitCommit(m)
if not SHA1_RE.match(c.body):
status.report('Invalid manifest metadata in git commit %s' %
m.decode('ascii'))
continue
gitsha1 = GitHgHelper.hg2git(c.body)
if gitsha1 == NULL_NODE_ID:
status.report('Missing hg2git metadata for manifest %s' %
c.body.decode('ascii'))
continue
if not GitHgHelper.check_manifest(c.body):
status.report('Sha1 mismatch for manifest %s' %
c.body.decode('ascii'))
files = {}
if previous:
for _, _, before, after, d, path in GitHgHelper.diff_tree(
previous, m):
if d in b'AM' and before != after and \
(path, after) not in all_interesting:
files[path] = after
else:
for _, t, sha1, path in GitHgHelper.ls_tree(m, recursive=True):
if (path, sha1) not in all_interesting:
files[path] = sha1
all_interesting.update(iteritems(files))
previous = m
if status('broken'):
return 1
# Don't check files that were already there in the previously checked
# manifests.
previous = None
for r in roots:
if previous:
for _, _, before, after, d, path in GitHgHelper.diff_tree(
previous, r):
if d in b'AM' and before != after:
all_interesting.discard((path, after))
else:
for _, t, sha1, path in GitHgHelper.ls_tree(r, recursive=True):
all_interesting.discard((path, sha1))
previous = r
progress = Progress('Checking {} files')
while all_interesting and manifest_queue:
(m, parents) = manifest_queue.pop()
changes = get_changes(m, parents, all=True)
for path, hg_file, hg_fileparents in changes:
if hg_fileparents[1:] == (hg_file, ):
continue
elif hg_fileparents[:1] == (hg_file, ):
continue
# Reaching here means the file received a modification compared
# to its parents. If it's a file we're going to check below,
# it means we don't need to check its parents if somehow they were
# going to be checked. If it's not a file we're going to check
# below, it's because it's either a file we weren't interested in
# in the first place, or it's the parent of a file we have checked.
# Either way, we aren't interested in the parents.
for p in hg_fileparents:
all_interesting.discard((path, p))
if (path, hg_file) not in all_interesting:
continue
all_interesting.remove((path, hg_file))
if not GitHgHelper.check_file(hg_file, *hg_fileparents):
p = store.manifest_path(path)
status.report('Sha1 mismatch for file %s\n'
' revision %s' %
(fsdecode(p), hg_file.decode('ascii')))
print_parents = ' '.join(
p.decode('ascii') for p in hg_fileparents
if p != NULL_NODE_ID)
if print_parents:
status.report(' with parent%s %s' %
('s' if len(print_parents) > 41 else '',
print_parents))
progress.progress()
progress.finish()
if all_interesting:
status.info('Could not find the following files:')
for path, sha1 in sorted(all_interesting):
p = store.manifest_path(path)
status.info(' %s %s' % (sha1.decode('ascii'), fsdecode(p)))
status.info('This might be a bug in `git cinnabar fsck`. Please open '
'an issue, with the message above, on\n'
'https://github.com/glandium/git-cinnabar/issues')
return 1
check_replace(store)
if status('broken'):
status.info('Your git-cinnabar repository appears to be corrupted.\n'
'Please open an issue, with the information above, on\n'
'https://github.com/glandium/git-cinnabar/issues')
Git.update_ref(b'refs/cinnabar/broken', metadata_commit)
if checked_metadata:
status.info(
'\nThen please try to run `git cinnabar rollback --fsck` to '
'restore last known state, and to update from the mercurial '
'repository.')
else:
status.info('\nThen please try to run `git cinnabar reclone`.')
status.info(
'\nPlease note this may affect the commit sha1s of mercurial '
'changesets, and may require to rebase your local branches.')
status.info(
'\nAlternatively, you may start afresh with a new clone. In any '
'case, please keep this corrupted repository around for further '
'debugging.')
return 1
refresh = []
if fix_changeset_heads:
status.fix('Fixing changeset heads metadata order.')
refresh.append('refs/cinnabar/changesets')
interval_expired('fsck', 0)
store.close(refresh=refresh)
GitHgHelper._helper = False
metadata_commit = Git.resolve_ref('refs/cinnabar/metadata')
Git.update_ref(b'refs/cinnabar/checked', metadata_commit)
return 0
def check_replace(store):
self_refs = [r for r, s in iteritems(store._replace) if r == s]
for r in progress_iter('Removing {} self-referencing grafts', self_refs):
del store._replace[r]