def test_sorted_merge(self):
a = [
('a', 1),
('b', 2),
('d', 4),
]
b = [
('b', 'B'),
('c', 'C'),
]
self.assertEquals(list(sorted_merge(a, b)), [
('a', (1,), ()),
('b', (2,), ('B',)),
('c', (), ('C',)),
('d', (4,), ()),
])
self.assertEquals(list(sorted_merge(b, a)), [
('a', (), (1,)),
('b', ('B',), (2,)),
('c', ('C',), ()),
('d', (), (4,)),
])
self.assertEquals(list(sorted_merge(a[:2], b[:1])), [
('a', (1,), ()),
('b', (2,), ('B',)),
])
def test_sorted_merge(self):
a = [
('a', 1),
('b', 2),
('d', 4),
]
b = [
('b', 'B'),
('c', 'C'),
]
self.assertEquals(list(sorted_merge(a, b)), [
('a', (1, ), ()),
('b', (2, ), ('B', )),
('c', (), ('C', )),
('d', (4, ), ()),
])
self.assertEquals(list(sorted_merge(b, a)), [
('a', (), (1, )),
('b', ('B', ), (2, )),
('c', ('C', ), ()),
('d', (), (4, )),
])
self.assertEquals(list(sorted_merge(a[:2], b[:1])), [
('a', (1, ), ()),
('b', (2, ), ('B', )),
])
def create_hg_metadata(self, commit, parents):
if check_enabled('bundle'):
real_changeset = self.changeset(self.hg_changeset(commit))
manifest, changeset_files = self.create_hg_manifest(commit, parents)
commit_data = GitCommit(commit)
if manifest.node == NULL_NODE_ID:
manifest.node = manifest.sha1
if check_enabled('bundle'):
if real_changeset and (manifest.node !=
real_changeset.manifest):
for path, created, real in sorted_merge(
manifest,
self.manifest(real_changeset.manifest),
key=lambda i: i.path,
non_key=lambda i: i):
if bytes(created) != bytes(real):
logging.error('%r != %r', bytes(created),
bytes(real))
self._pushed.add(manifest.node)
self.store_manifest(manifest)
self._manifest_git_tree[manifest.node] = commit_data.tree
changeset = Changeset.from_git_commit(commit_data)
changeset.parents = tuple(self.hg_changeset(p) for p in parents)
changeset.manifest = manifest.node
changeset.files = changeset_files
if parents:
parent_changeset = self.changeset(changeset.parent1)
if parent_changeset.branch:
changeset.branch = parent_changeset.branch
if self._graft is True and parents and changeset.body[-1:] == b'\n':
parent_commit = GitCommit(parents[0])
if (parent_commit.body[-1:] == b'\n'
and parent_commit.body[-2] == parent_changeset.body[-1]):
self._graft = 'true'
if self._graft == 'true' and changeset.body[-1:] == b'\n':
changeset.body = changeset.body[:-1]
changeset.node = changeset.sha1
self._pushed.add(changeset.node)
self.store_changeset(changeset, commit_data)
if check_enabled('bundle') and real_changeset:
error = False
for k in ('files', 'manifest'):
if getattr(real_changeset, k, []) != getattr(changeset, k, []):
logging.error('(%s) %r != %r', k,
getattr(real_changeset, k, None),
getattr(changeset, k, None))
error = True
if error:
raise Exception('Changeset mismatch')
def create_hg_metadata(self, commit, parents):
if check_enabled('bundle'):
real_changeset = self.changeset(self.hg_changeset(commit))
manifest, changeset_files = self.create_hg_manifest(commit, parents)
commit_data = GitCommit(commit)
if manifest.node == NULL_NODE_ID:
manifest.node = manifest.sha1
if check_enabled('bundle'):
if real_changeset and (
manifest.node != real_changeset.manifest):
for path, created, real in sorted_merge(
manifest._lines,
self.manifest(real_changeset.manifest)._lines,
key=lambda i: i.name, non_key=lambda i: i):
if str(created) != str(real):
logging.error('%r != %r', str(created), str(real))
self._pushed.add(manifest.node)
self.store_manifest(manifest)
self._manifest_git_tree[manifest.node] = commit_data.tree
changeset = Changeset.from_git_commit(commit_data)
changeset.parents = tuple(self.hg_changeset(p) for p in parents)
changeset.manifest = manifest.node
changeset.files = changeset_files
if parents:
parent_changeset = self.changeset(changeset.parent1)
if parent_changeset.branch:
changeset.branch = parent_changeset.branch
if self._graft is True and parents and changeset.body[-1] == '\n':
parent_commit = GitCommit(parents[0])
if (parent_commit.body[-1] == '\n' and
parent_commit.body[-2] == parent_changeset.body[-1]):
self._graft = 'true'
if self._graft == 'true' and changeset.body[-1] == '\n':
changeset.body = changeset.body[:-1]
changeset.node = changeset.sha1
self._pushed.add(changeset.node)
self.store_changeset(changeset, commit_data)
if check_enabled('bundle') and real_changeset:
error = False
for k in ('files', 'manifest'):
if getattr(real_changeset, k, []) != getattr(changeset, k, []):
logging.error('(%s) %r != %r', k,
getattr(real_changeset, k, None),
getattr(changeset, k, None))
error = True
if error:
raise Exception('Changeset mismatch')
def create_hg_manifest(self, commit, parents):
manifest = GeneratedManifestInfo(NULL_NODE_ID)
changeset_files = []
if parents:
parent_changeset = self.changeset(self.hg_changeset(parents[0]))
parent_manifest = self.manifest(parent_changeset.manifest)
parent_node = parent_manifest.node
if len(parents) == 2:
parent2_changeset = self.changeset(self.hg_changeset(parents[1]))
parent2_manifest = self.manifest(parent2_changeset.manifest)
parent2_node = parent2_manifest.node
if parent_node == parent2_node:
parents = parents[:1]
if not parents:
for line in Git.ls_tree(commit, recursive=True):
mode, typ, sha1, path = line
node = self.create_file(sha1,
git_manifest_parents=(),
path=path)
manifest.add(path, node, self.ATTR[mode], modified=True)
changeset_files.append(path)
manifest.parents = []
manifest.delta_node = NULL_NODE_ID
return manifest, changeset_files
elif len(parents) == 2:
if not experiment('merge'):
raise Exception('Pushing merges is not supported yet')
if not self._merge_warn:
logging.warning('Pushing merges is experimental.')
logging.warning('This may irremediably push bad state to the '
'mercurial server!')
self._merge_warn = 1
git_manifests = (self.manifest_ref(parent_node),
self.manifest_ref(parent2_node))
# TODO: this would benefit from less git queries
changes = list(get_changes(commit, parents))
files = [
(path, mode, sha1)
for mode, _, sha1, path in Git.ls_tree(commit, recursive=True)
]
manifests = sorted_merge(parent_manifest,
parent2_manifest,
key=lambda i: i.path,
non_key=lambda i: i)
for line in sorted_merge(files, sorted_merge(changes, manifests)):
path, f, (change, (manifest_line_p1, manifest_line_p2)) = line
if not f: # File was removed
if manifest_line_p1:
manifest.removed.add(path)
changeset_files.append(path)
continue
mode, sha1 = f
attr = self.ATTR[mode]
if manifest_line_p1 and not manifest_line_p2:
file_parents = (manifest_line_p1.sha1, )
elif manifest_line_p2 and not manifest_line_p1:
file_parents = (manifest_line_p2.sha1, )
elif not manifest_line_p1 and not manifest_line_p2:
file_parents = ()
elif manifest_line_p1.sha1 == manifest_line_p2.sha1:
file_parents = (manifest_line_p1.sha1, )
else:
if self._merge_warn == 1:
logging.warning('This may take a while...')
self._merge_warn = 2
file_parents = (manifest_line_p1.sha1,
manifest_line_p2.sha1)
assert file_parents is not None
f = self._create_file_internal(
sha1,
*file_parents,
git_manifest_parents=git_manifests,
path=path)
file_parents = tuple(p for p in (f.parent1, f.parent2)
if p != NULL_NODE_ID)
merged = len(file_parents) == 2
if not merged and file_parents:
if self.git_file_ref(file_parents[0]) == sha1:
node = file_parents[0]
else:
merged = True
if merged:
node = self._store_file_internal(f)
else:
node = file_parents[0]
attr_change = (manifest_line_p1
and manifest_line_p1.attr != attr)
manifest.add(path, node, attr, modified=merged or attr_change)
if merged or attr_change:
changeset_files.append(path)
if manifest.raw_data == parent_manifest.raw_data:
return parent_manifest, []
manifest.parents = (parent_node, parent2_node)
return manifest, changeset_files
def process_diff(diff):
for (mode_before, mode_after, sha1_before, sha1_after, status,
path) in diff:
if status[:1] == b'R':
yield status[1:], (b'000000', sha1_before, NULL_NODE_ID,
b'D')
yield path, (mode_after, sha1_before, sha1_after, status)
git_diff = sorted(l for l in process_diff(
GitHgHelper.diff_tree(parents[0], commit, detect_copy=True)))
if not git_diff:
return parent_manifest, []
parent_lines = OrderedDict((l.path, l) for l in parent_manifest)
items = manifest.items
for line in sorted_merge(iteritems(parent_lines),
git_diff,
non_key=lambda i: i[1]):
path, manifest_line, change = line
if not change:
items.append(manifest_line)
continue
mode_after, sha1_before, sha1_after, status = change
path2 = status[1:]
status = status[:1]
attr = self.ATTR.get(mode_after)
if status == b'D':
manifest.removed.add(path)
changeset_files.append(path)
continue
if status in b'MT':
if sha1_before == sha1_after:
node = manifest_line.sha1
else:
node = self.create_file(
sha1_after,
manifest_line.sha1,
git_manifest_parents=(
self.manifest_ref(parent_node), ),
path=path)
elif status in b'RC':
if sha1_after != EMPTY_BLOB:
node = self.create_copy(
(path2, parent_lines[path2].sha1),
sha1_after,
git_manifest_parents=(
self.manifest_ref(parent_node), ),
path=path)
else:
node = self.create_file(
sha1_after,
git_manifest_parents=(
self.manifest_ref(parent_node), ),
path=path)
else:
assert status == b'A'
node = self.create_file(
sha1_after,
git_manifest_parents=(self.manifest_ref(parent_node), ),
path=path)
manifest.add(path, node, attr, modified=True)
changeset_files.append(path)
manifest.parents = (parent_node, )
manifest.delta_node = parent_node
return manifest, changeset_files
def create_hg_manifest(self, commit, parents):
manifest = GeneratedManifestInfo(NULL_NODE_ID)
changeset_files = []
if parents:
parent_changeset = self.changeset(self.hg_changeset(parents[0]))
parent_manifest = self.manifest(parent_changeset.manifest)
parent_node = parent_manifest.node
if len(parents) == 2:
parent2_changeset = self.changeset(self.hg_changeset(parents[1]))
parent2_manifest = self.manifest(parent2_changeset.manifest)
parent2_node = parent2_manifest.node
if parent_node == parent2_node:
parents = parents[:1]
if not parents:
for line in Git.ls_tree(commit, recursive=True):
mode, typ, sha1, path = line
node = self.create_file(sha1, git_manifest_parents=(),
path=path)
manifest.append_line(ManifestLine(path, node, self.ATTR[mode]),
modified=True)
changeset_files.append(path)
manifest.set_parents(NULL_NODE_ID)
manifest.delta_node = NULL_NODE_ID
return manifest, changeset_files
elif len(parents) == 2:
if not experiment('merge'):
raise Exception('Pushing merges is not supported yet')
if not self._merge_warn:
logging.warning('Pushing merges is experimental.')
logging.warning('This may irremediably push bad state to the '
'mercurial server!')
self._merge_warn = 1
git_manifests = (self.manifest_ref(parent_node),
self.manifest_ref(parent2_node))
# TODO: this would benefit from less git queries
changes = list(get_changes(commit, parents))
files = [(path, mode, sha1) for mode, _, sha1, path in
Git.ls_tree(commit, recursive=True)]
manifests = sorted_merge(parent_manifest._lines,
parent2_manifest._lines,
key=lambda i: i.name, non_key=lambda i: i)
for line in sorted_merge(files, sorted_merge(changes, manifests)):
path, f, (change, (manifest_line_p1, manifest_line_p2)) = line
if not f: # File was removed
if manifest_line_p1:
manifest.removed.add(path)
changeset_files.append(path)
continue
mode, sha1 = f
attr = self.ATTR[mode]
if manifest_line_p1 and not manifest_line_p2:
file_parents = (manifest_line_p1.node,)
elif manifest_line_p2 and not manifest_line_p1:
file_parents = (manifest_line_p2.node,)
elif not manifest_line_p1 and not manifest_line_p2:
file_parents = ()
elif manifest_line_p1.node == manifest_line_p2.node:
file_parents = (manifest_line_p1.node,)
else:
if self._merge_warn == 1:
logging.warning('This may take a while...')
self._merge_warn = 2
file_parents = (manifest_line_p1.node,
manifest_line_p2.node)
assert file_parents is not None
f = self._create_file_internal(
sha1, *file_parents,
git_manifest_parents=git_manifests,
path=path
)
file_parents = tuple(p for p in (f.parent1, f.parent2)
if p != NULL_NODE_ID)
merged = len(file_parents) == 2
if not merged and file_parents:
if self.git_file_ref(file_parents[0]) == sha1:
node = file_parents[0]
else:
merged = True
if merged:
node = self._store_file_internal(f)
else:
node = file_parents[0]
attr_change = (manifest_line_p1 and
manifest_line_p1.attr != attr)
manifest.append_line(ManifestLine(path, node, attr),
modified=merged or attr_change)
if merged or attr_change:
changeset_files.append(path)
if manifest.data == parent_manifest.data:
return parent_manifest, []
manifest.set_parents(parent_node, parent2_node)
return manifest, changeset_files
def process_diff(diff):
for (mode_before, mode_after, sha1_before, sha1_after, status,
path) in diff:
if status[0] == 'R':
yield status[1:], (
'000000', sha1_before, NULL_NODE_ID, 'D')
yield path, (mode_after, sha1_before, sha1_after,
status)
git_diff = sorted(
l for l in process_diff(GitHgHelper.diff_tree(
parents[0], commit, detect_copy=True))
)
if not git_diff:
return parent_manifest, []
parent_lines = OrderedDict((l.name, l)
for l in parent_manifest._lines)
for line in sorted_merge(parent_lines.iteritems(), git_diff,
non_key=lambda i: i[1]):
path, manifest_line, change = line
if not change:
manifest.append_line(manifest_line)
continue
mode_after, sha1_before, sha1_after, status = change
path2 = status[1:]
status = status[0]
attr = self.ATTR.get(mode_after)
if status == 'D':
manifest.removed.add(path)
changeset_files.append(path)
continue
if status in 'MT':
if sha1_before == sha1_after:
node = manifest_line.node
else:
node = self.create_file(
sha1_after, str(manifest_line.node),
git_manifest_parents=(
self.manifest_ref(parent_node),),
path=path)
elif status in 'RC':
if sha1_after != EMPTY_BLOB:
node = self.create_copy(
(path2, parent_lines[path2].node), sha1_after,
git_manifest_parents=(
self.manifest_ref(parent_node),),
path=path)
else:
node = self.create_file(
sha1_after,
git_manifest_parents=(
self.manifest_ref(parent_node),),
path=path)
else:
assert status == 'A'
node = self.create_file(
sha1_after,
git_manifest_parents=(
self.manifest_ref(parent_node),),
path=path)
manifest.append_line(ManifestLine(path, node, attr),
modified=True)
changeset_files.append(path)
manifest.set_parents(parent_node)
manifest.delta_node = parent_node
return manifest, changeset_files
def create_hg_metadata(self, commit, parents):
if check_enabled('bundle'):
real_changeset_data = self.read_changeset_data(commit)
manifest = self.create_hg_manifest(commit, parents)
commit_data = GitCommit(commit)
if manifest.node == NULL_NODE_ID:
manifest.node = manifest.sha1
if check_enabled('bundle'):
if real_changeset_data and (manifest.node !=
real_changeset_data['manifest']):
for path, created, real in sorted_merge(
manifest._lines,
self.manifest(
real_changeset_data['manifest'])._lines,
key=lambda i: i.name,
non_key=lambda i: i):
if str(created) != str(real):
logging.error('%r != %r', str(created), str(real))
self._push_manifests[manifest.node] = manifest
self.manifest_ref(manifest.node, hg2git=False, create=True)
self._manifest_git_tree[manifest.node] = commit_data.tree
extra = {}
if commit_data.author != commit_data.committer:
committer = self.hg_author_info(commit_data.committer)
extra['committer'] = '%s %d %d' % committer
if parents:
parent_changeset_data = self.read_changeset_data(parents[0])
branch = parent_changeset_data.get('extra', {}).get('branch')
if branch:
extra['branch'] = branch
changeset_data = self._changeset_data_cache[commit] = {
'files': sorted(chain(manifest.removed, manifest.modified)),
'manifest': manifest.node,
}
if extra:
changeset_data['extra'] = extra
changeset = self._changeset(commit, include_parents=True)
if self._graft is True and parents and changeset.data[-1] == '\n':
parent_cs = self._changeset(parents[0], skip_patch=True)
if 'patch' not in self._changeset_data_cache[parents[0]]:
self._graft = False
else:
patch = self._changeset_data_cache[parents[0]]['patch'][-1]
self._graft = (patch[1] == len(parent_cs.data)
and parent_cs.data[-1] == '\n')
if self._graft:
self._graft = 'true'
if self._graft == 'true' and changeset.data[-1] == '\n':
changeset.data = changeset.data[:-1]
changeset_data['patch'] = ((len(changeset.data),
len(changeset.data) + 1, ''), )
changeset_data['changeset'] = changeset.changeset = changeset.node = \
changeset.sha1
self._push_changesets[changeset.node] = changeset
# This is a horrible way to do this, but this method is not doing much
# better overall anyways.
if extra:
if 'committer' in extra:
del extra['committer']
if not extra:
del changeset_data['extra']
self._changesets[changeset.node] = PseudoString(commit)
if check_enabled('bundle') and real_changeset_data:
error = False
for k in ('files', 'manifest'):
if real_changeset_data.get(k, []) != changeset_data.get(k):
logging.error('(%s) %r != %r', k,
real_changeset_data.get(k),
changeset_data.get(k))
error = True
if error:
raise Exception('Changeset mismatch')
def create_hg_manifest(self, commit, parents):
manifest = GeneratedManifestInfo(NULL_NODE_ID)
if parents:
parent_changeset_data = self.read_changeset_data(parents[0])
parent_manifest = self.manifest(parent_changeset_data['manifest'])
parent_node = parent_manifest.node
if len(parents) == 2:
parent2_changeset_data = self.read_changeset_data(parents[1])
parent2_manifest = self.manifest(
parent2_changeset_data['manifest'])
parent2_node = parent2_manifest.node
if parent_node == parent2_node:
parents = parents[:1]
if not parents:
for line in Git.ls_tree(commit, recursive=True):
mode, typ, sha1, path = line
node = self.create_file(sha1,
git_manifest_parents=(),
path=path)
manifest.append_line(ManifestLine(path, node, self.ATTR[mode]),
modified=True)
manifest.set_parents(NULL_NODE_ID)
manifest.delta_node = NULL_NODE_ID
return manifest
elif len(parents) == 2:
if not experiment('merge'):
raise Exception('Pushing merges is not supported yet')
logging.warning('Pushing merges is experimental.')
logging.warning('This may irremediably push bad state to the '
'mercurial server!')
warned = False
git_manifests = (self.manifest_ref(parent_node),
self.manifest_ref(parent2_node))
# TODO: this would benefit from less git queries
changes = list(get_changes(commit, parents))
files = [
(path, mode, sha1)
for mode, _, sha1, path in Git.ls_tree(commit, recursive=True)
]
manifests = sorted_merge(parent_manifest._lines,
parent2_manifest._lines,
key=lambda i: i.name,
non_key=lambda i: i)
for line in sorted_merge(files, sorted_merge(changes, manifests)):
path, f, (change, (manifest_line_p1, manifest_line_p2)) = line
if not f: # File was removed
if manifest_line_p1:
manifest.removed.add(path)
continue
mode, sha1 = f
attr = self.ATTR[mode]
if manifest_line_p1 and not manifest_line_p2:
file_parents = (manifest_line_p1.node, )
elif manifest_line_p2 and not manifest_line_p1:
file_parents = (manifest_line_p2.node, )
elif not manifest_line_p1 and not manifest_line_p2:
file_parents = ()
elif manifest_line_p1.node == manifest_line_p2.node:
file_parents = (manifest_line_p1.node, )
else:
if (any(isinstance(p, Mark) for p in git_manifests)):
raise Exception(
'Cannot push %s. Please first push %s separately' %
(commit, ' and '.join(
p for i, p in enumerate(parents)
if isinstance(git_manifests[i], Mark))))
if not warned:
logging.warning('This may take a while...')
warned = True
file_parents = (manifest_line_p1.node,
manifest_line_p2.node)
assert file_parents is not None
f = self._create_file_internal(
sha1,
*file_parents,
git_manifest_parents=git_manifests,
path=path)
file_parents = tuple(p for p in (f.parent1, f.parent2)
if p != NULL_NODE_ID)
merged = len(file_parents) == 2
if not merged and file_parents:
if self.git_file_ref(file_parents[0]) == sha1:
node = file_parents[0]
else:
merged = True
if merged:
node = self._store_file_internal(f)
else:
node = PseudoString(file_parents[0])
attr_change = (manifest_line_p1
and manifest_line_p1.attr != attr)
manifest.append_line(ManifestLine(path, node, attr),
modified=merged or attr_change)
if manifest.data == parent_manifest.data:
return parent_manifest
manifest.set_parents(parent_node, parent2_node)
return manifest
def process_diff(diff):
for (mode_before, mode_after, sha1_before, sha1_after, status,
path) in diff:
if status[0] == 'R':
yield status[1:], ('000000', sha1_before, NULL_NODE_ID,
'D')
yield path, (mode_after, sha1_before, sha1_after, status)
git_diff = sorted(l for l in process_diff(
Git.diff_tree(parents[0], commit, detect_copy=True)))
if not git_diff:
return parent_manifest
parent_lines = OrderedDict((l.name, l) for l in parent_manifest._lines)
for line in sorted_merge(parent_lines.iteritems(),
git_diff,
non_key=lambda i: i[1]):
path, manifest_line, change = line
if not change:
manifest.append_line(manifest_line)
continue
mode_after, sha1_before, sha1_after, status = change
path2 = status[1:]
status = status[0]
attr = self.ATTR.get(mode_after)
if status == 'D':
manifest.removed.add(path)
continue
if status in 'MT':
if sha1_before == sha1_after:
node = PseudoString(manifest_line.node)
else:
node = self.create_file(
sha1_after,
str(manifest_line.node),
git_manifest_parents=(
self.manifest_ref(parent_node), ),
path=path)
elif status in 'RC':
if sha1_after != EMPTY_BLOB:
node = self.create_copy(
(path2, parent_lines[path2].node),
sha1_after,
git_manifest_parents=(
self.manifest_ref(parent_node), ),
path=path)
else:
node = self.create_file(
sha1_after,
git_manifest_parents=(
self.manifest_ref(parent_node), ),
path=path)
else:
assert status == 'A'
node = self.create_file(
sha1_after,
git_manifest_parents=(self.manifest_ref(parent_node), ),
path=path)
manifest.append_line(ManifestLine(path, node, attr), modified=True)
manifest.set_parents(parent_node)
manifest.delta_node = parent_node
return manifest