def test_stringPositions(self):
text = \
"Source: foo\n" \
"Build-Depends: aa (> 3),\n" \
" bb,\n" \
"Uploaders: a <*****@*****.**>, <*****@*****.**>\n" \
"\n" \
"Package: foo\n" \
"Suggests: bar\n"
parser = ControlFileParser(text=text)
result = parser.parse()
para0 = result[0]
para1 = result[1]
self.assertEqual(StringPosition(1, 9, 1, 11), para0['Source'].position)
self.assertEqual(StringPosition(2, 16, 3, 4),
para0['Build-Depends'].position)
self.assertEqual(StringPosition(4, 12, 4, 33),
para0['Uploaders'].position)
self.assertEqual(StringPosition(1, 1, 4, 34), para0.position)
self.assertEqual(StringPosition(6, 1, 7, 14), para1.position)
result = parser.parse_depends(None, 'Build-Depends')
self.assertEqual(StringPosition(2, 16, 2, 23), result['aa'].position)
self.assertEqual(StringPosition(3, 2, 3, 3), result['bb'].position)
self.assertEqual(parser.text_at_position(result['aa'].position),
'aa (> 3)')
def test_parseExtraNewline(self):
controlfile = \
"Source: foo\n" \
"Uploaders:\n" \
" a <*****@*****.**>,\n" \
" b <*****@*****.**>\n" \
"\n"
parser = ControlFileParser(text=controlfile)
parser.parse()
def test_nodeChalkBdepsParse(self):
controlfile = \
"Source: node-chalk\n" \
"Build-Depends:\n" \
" debhelper,\n" \
" , nodejs\n" \
" , mocha\n"
parser = ControlFileParser(text=controlfile)
parser.parse_depends(None, 'Build-Depends')
def test_addBuildDependsOneLine(self):
controlfile = \
"Source: ffmpeg\n" \
"Build-Depends: flite1, bz2\n"
parser = ControlFileParser(text=controlfile)
parser.add_depends_entry(None, 'Build-Depends', 'fdk-aac')
self.assertEqual(
parser.get_text(), "Source: ffmpeg\n"
"Build-Depends: flite1, bz2, fdk-aac\n")
def test_removeBuildDependsFinalOneLine(self):
controlfile = \
"Source: gnome-online-accounts\n" \
"Build-Depends: soup, telepathy, webkit,\n"
parser = ControlFileParser(text=controlfile)
parser.remove_depends_entry(None, 'Build-Depends', 'webkit')
self.assertEqual(
parser.get_text(), "Source: gnome-online-accounts\n"
"Build-Depends: soup, telepathy,\n")
def test_removeAllBuildDepends(self):
controlfile = \
"Source: libproxy\n" \
"Build-Depends: llvm\n" \
"Uploaders: a <*****@*****.**>\n"
parser = ControlFileParser(text=controlfile)
parser.remove_depends_entry(None, 'Build-Depends', 'llvm')
new = parser.get_text()
self.assertEqual(new, "Source: libproxy\n" "Uploaders: a <*****@*****.**>\n")
def test_getFieldValue(self):
text = \
"Source: foo\n" \
"\n" \
"Package: one\n" \
"Architecture: amd64\n"
parser = ControlFileParser(text=text)
para0 = parser.get_paragraph(None)
one = parser.get_paragraph("one")
self.assertEqual("foo", para0['Source'])
self.assertEqual("amd64", one['Architecture'])
def test_removeBuildDepends(self):
controlfile = \
"Source: gnome-online-accounts\n" \
"Build-Depends: libsoup2.4-dev,\n" \
" telepathy,\n" \
" libwebkit2gtk-4.0-dev\n"
parser = ControlFileParser(text=controlfile)
parser.remove_depends_entry(None, 'Build-Depends', 'telepathy')
self.assertEqual(
parser.get_text(), "Source: gnome-online-accounts\n"
"Build-Depends: libsoup2.4-dev,\n"
" libwebkit2gtk-4.0-dev\n")
def test_uploadersOneLine(self):
controlfile = \
"Source: cheese\n" \
"Maintainer: GNOME <*****@*****.**>\n" \
"Uploaders: Frog <*****@*****.**>, James <*****@*****.**>\n" \
"Build-Depends: debhelper (>= 11)\n"
parser = ControlFileParser(text=controlfile)
parser.patch(None, 'Uploaders', 'new <*****@*****.**>')
new = parser.get_text()
self.assertEqual(
new, "Source: cheese\n"
"Maintainer: GNOME <*****@*****.**>\n"
"Uploaders: new <*****@*****.**>\n"
"Build-Depends: debhelper (>= 11)\n")
def test_removePackage(self):
controlfile = \
"Source: foo\n" \
"Build-Depends: bar\n" \
"\n" \
"Package: foo1\n" \
"Suggests: something\n" \
"\n" \
"Package: foo2\n" \
"Suggests: something\n" \
"\n" \
"Package: foo3\n" \
"Suggests: something\n" \
"\n" \
"Package: foo4\n" \
"Suggests: something\n"
parser = ControlFileParser(text=controlfile)
parser.remove_package('foo1')
parser.remove_package('foo2')
parser.remove_package('foo4')
self.assertEqual(
parser.get_text(), "Source: foo\n"
"Build-Depends: bar\n"
"\n"
"Package: foo3\n"
"Suggests: something\n")
def test_parseSpaceSeparation(self):
controlfile = \
"Source: foo\n" \
"Uploaders: a <*****@*****.**>\n" \
"\n" \
"Package: foo1\n" \
" \n" \
"Package: foo2\n"
ControlFileParser(text=controlfile).parse()
def test_parseMultiNewline(self):
controlfile = \
"Source: foo\n" \
"Uploaders: a <*****@*****.**>\n" \
"\n" \
"Package: foo1\n" \
"\n" \
"\n" \
"Package: foo2\n"
ControlFileParser(text=controlfile).parse()
def __init__(self, control_path, left_dir, left_name, right_dir,
right_name, base_dir, merged_dir):
# Merge notes recorded here
self.notes = []
# If the merged file was modified (relative to the right version)
self.modified = False
self.left_name = left_name
self.right_name = right_name
self.left_dir = left_dir
self.right_dir = right_dir
self.base_dir = base_dir
self.merged_dir = merged_dir
self.left_control_path = os.path.join(left_dir, control_path)
self.right_control_path = os.path.join(right_dir, control_path)
self.base_control_path = os.path.join(base_dir, control_path)
self.merged_control_path = os.path.join(merged_dir, control_path)
tree.ensure(self.merged_control_path)
self.base_control = ControlFileParser(filename=self.base_control_path)
self.left_control = ControlFileParser(filename=self.left_control_path)
self.right_control = \
ControlFileParser(filename=self.right_control_path)
self.orig_right_md5sum = md5sum(self.right_control_path)
def test_parse(self):
controlfile = \
"# A comment\n" \
"#\n" \
"Source: foo\n" \
"Uploaders:\n" \
" ä <*****@*****.**>,\n" \
" b <*****@*****.**>\n" \
"Build-Depends: a, b (>= 3.0), c | d, e [!armhf],\n" \
" spaceatendofline, \n" \
" f <!nocheck>, g:any, ${my:Variable}\n" \
"Build-Depends-Indep:\n" \
"# comment\n" \
" asdf\n"
parser = ControlFileParser(text=controlfile)
result = parser.parse()
print result[0]['Build-Depends-Indep']
bdeps = parser.parse_depends(None, 'Build-Depends')
self.assertIsNone(bdeps['a'].version_constraint)
self.assertIsNone(bdeps['e'].version_constraint)
self.assertEqual(bdeps['b'].version_constraint, ">= 3.0")
def test_addField(self):
controlfile = \
"Source: foo\n" \
"Build-Depends: bar\n" \
"\n" \
"Package: foo1\n" \
"Suggests: something\n" \
"\n" \
"Package: foo2\n" \
"Suggests: something\n"
parser = ControlFileParser(text=controlfile)
parser.add_field('foo1', 'X-Something', 'asdf')
self.assertEqual(
parser.get_text(), "Source: foo\n"
"Build-Depends: bar\n"
"\n"
"Package: foo1\n"
"Suggests: something\n"
"X-Something: asdf\n"
"\n"
"Package: foo2\n"
"Suggests: something\n")
def test_addParagraph(self):
controlfile = \
"Source: foo\n" \
"Build-Depends: bar\n"
parser = ControlFileParser(text=controlfile)
parser.add_paragraph("Package: mypkg\n" "Suggests: gcc\n")
self.assertIsNotNone(parser.get_paragraph("mypkg"))
self.assertEqual(
parser.get_text(), "Source: foo\n"
"Build-Depends: bar\n"
"\n"
"Package: mypkg\n"
"Suggests: gcc\n")
def test_patchBuildDependsVersionConstraint(self):
controlfile = \
"Source: appstream\n" \
"Build-Depends: something\n" \
" cmake (>= 3.2),\n" \
" debhelper (>= 10),\n"
parser = ControlFileParser(text=controlfile)
result = parser.parse_depends(None, 'Build-Depends')
parser.patch_at_offset(result["cmake"].position, "cmake (>= 4.0)")
new = parser.get_text()
self.assertEqual(
new, "Source: appstream\n"
"Build-Depends: something\n"
" cmake (>= 4.0),\n"
" debhelper (>= 10),\n")
def test_getParagraph(self):
text = \
"Source: foo\n" \
"\n" \
"Package: one\n" \
"Architecture: amd64\n" \
"\n" \
"Package: two\n" \
"Architecture: armhf\n"
parser = ControlFileParser(text=text)
self.assertIsNotNone(parser.get_paragraph(None))
self.assertIsNotNone(parser.get_paragraph("one"))
self.assertIsNotNone(parser.get_paragraph("two"))
self.assertIsNone(parser.get_paragraph("three"))
self.assertEqual(unicode(parser.get_paragraph("one")), "Package: one\n"
"Architecture: amd64\n")
class DebControlMerger(object):
def __init__(self, control_path, left_dir, left_name, right_dir,
right_name, base_dir, merged_dir):
# Merge notes recorded here
self.notes = []
# If the merged file was modified (relative to the right version)
self.modified = False
self.left_name = left_name
self.right_name = right_name
self.left_dir = left_dir
self.right_dir = right_dir
self.base_dir = base_dir
self.merged_dir = merged_dir
self.left_control_path = os.path.join(left_dir, control_path)
self.right_control_path = os.path.join(right_dir, control_path)
self.base_control_path = os.path.join(base_dir, control_path)
self.merged_control_path = os.path.join(merged_dir, control_path)
tree.ensure(self.merged_control_path)
self.base_control = ControlFileParser(filename=self.base_control_path)
self.left_control = ControlFileParser(filename=self.left_control_path)
self.right_control = \
ControlFileParser(filename=self.right_control_path)
self.orig_right_md5sum = md5sum(self.right_control_path)
def record_note(self, note, changelog_worthy=False):
logger.debug(note)
self.notes.append((note, changelog_worthy))
def do_diff3(self):
rc = subprocess.call(
("diff3", "-E", "-m", "-L", self.left_name, self.left_control_path,
"-L", "BASE", self.base_control_path, "-L", self.right_name,
self.right_control_path),
stdout=open(self.merged_control_path, 'w'))
if rc != 0:
return False
self.modified = \
md5sum(self.merged_control_path) != self.orig_right_md5sum
return True
def run(self):
if self.do_diff3():
return True
merge_funcs = (
(self.merge_uploaders, ()),
(self.merge_paragraphs, ()),
(self.merge_recommends, ()),
(self.merge_suggests, ()),
(self.merge_architecture, ()),
(self.merge_depends, (None, 'Build-Depends')),
(self.merge_depends, (None, 'Build-Depends-Indep')),
(self.merge_depends, (None, 'Build-Conflicts')),
(self.merge_package_depends, ('Depends', )),
(self.merge_package_depends, ('Replaces', )),
(self.merge_package_depends, ('Provides', )),
(self.remove_comments, ()),
)
# Try all the merge strategies until diff3 is happy
for func, args in merge_funcs:
func(*args)
if self.do_diff3():
return True
return False
def __restore_original_value(self, package, field):
left_para = self.left_control.get_paragraph(package)
base_para = self.base_control.get_paragraph(package)
if left_para is None or base_para is None:
return False
# See if we've modified the field in our version
base_value = base_para.get(field)
if base_value == left_para.get(field):
return False
# Yes, it's been modified.
logger.debug('Restoring %s to base value', field_name(package, field))
if field not in left_para:
# If we totally removed the field, we should restore it.
# But because removing is easier than readding, we just remove
# the field from the base version.
self.base_control.remove_field(package, field)
self.base_control.write()
elif base_value is None:
# If the field was not present in the base vesion, just remove
# it from the left version
self.left_control.remove_field(package, field)
self.left_control.write()
else:
# Otherwise restore the left field with the value from base
self.left_control.patch(package, field, base_value)
self.left_control.write()
return True
def __restore_original_package_values(self, field):
modified_packages = []
for left_pkg in self.left_control.parse():
if 'Package' not in left_pkg:
continue
pkg = left_pkg['Package']
modified = \
self.__restore_original_value(pkg, field)
if modified:
modified_packages.append(pkg)
return modified_packages
# Endless no longer install recommended packages, so we can eliminate
# merge noise by dropping changes we had made to Recommends.
def merge_recommends(self):
modified_packages = \
self.__restore_original_package_values('Recommends')
for pkg in modified_packages:
self.record_note(
'Dropped uninteresting %s Recommends change' % pkg, True)
# Endless doesn't install suggested packages, so we can eliminate
# merge noise by dropping any changes to Suggests.
def merge_suggests(self):
modified_packages = \
self.__restore_original_package_values('Suggests')
for pkg in modified_packages:
self.record_note('Dropped uninteresting %s Suggests change' % pkg,
True)
# Drop Uploaders changes from the left side as they are irrelevant and
# just generate merge noise.
def merge_uploaders(self):
if self.__restore_original_value(None, 'Uploaders'):
self.record_note('Dropped uninteresting Uploaders change')
# If the left side appended to the Architectures list for a package,
# re-append the same additions to the right side packages
def merge_architecture(self):
for left_pkg in self.left_control.parse():
if 'Package' not in left_pkg or 'Architecture' not in left_pkg:
continue
package = left_pkg['Package']
base_pkg = self.base_control.get_paragraph(package)
if not base_pkg or 'Architecture' not in base_pkg:
continue
base_arch = unicode(base_pkg['Architecture'])
left_arch = unicode(left_pkg['Architecture'])
if base_arch == left_arch or not left_arch.startswith(base_arch):
continue
# Left package appended something onto the Architecture list.
added_part = left_arch[len(base_arch):]
# If the package was removed on the right, just drop our
# local change.
right_pkg = self.right_control.get_paragraph(package)
if right_pkg is None:
self.__restore_original_value(package, 'Architecture')
self.record_note(
'Dropped change to add %s architecture(s) '
'%s because this package was removed '
'upstream' % (package, added_part), True)
continue
if 'Architecture' not in right_pkg:
continue
# Re-apply those changes on the right
right_arch = unicode(right_pkg['Architecture'])
new_value = right_arch + added_part
self.right_control.patch(package, 'Architecture', new_value)
self.right_control.write()
# And drop our local change to prevent it being remerged
self.left_control.patch(package, 'Architecture', base_arch)
self.left_control.write()
self.record_note('Readded architecture(s) %s to %s' %
(added_part.strip(), package))
def merge_paragraph(self, package):
base_para = self.base_control.get_paragraph(package)
left_para = self.left_control.get_paragraph(package)
right_para = self.right_control.get_paragraph(package)
added_fields = set(left_para.keys()) - set(base_para.keys())
if right_para is not None:
added_fields -= set(right_para.keys())
else:
added_fields.clear()
for field in added_fields:
self.right_control.add_field(package, field, left_para[field])
self.right_control.write()
# Remove the field from left version to ease the merge
self.left_control.remove_field(package, field)
self.left_control.write()
self.record_note('Readded %s %s' % (package or '', field))
# Perform simple merge operations on paragraphs and their contents
def merge_paragraphs(self):
self.merge_paragraph(None)
for pkg in self.base_control.get_package_names():
if self.left_control.get_paragraph(pkg):
self.merge_paragraph(pkg)
continue
logger.debug('Carrying forward %s package removal', pkg)
# Package was removed on left, so remove it on the right
self.right_control.remove_package(pkg)
self.right_control.write()
# To ease the merge, also remove it from the base version
self.base_control.remove_package(pkg)
self.base_control.write()
self.record_note('Carried forward removal of %s binary package ' %
pkg)
for pkg in self.left_control.get_package_names():
if self.base_control.get_paragraph(pkg):
continue
logger.debug('Carrying forward %s package addition', pkg)
para = self.left_control.get_paragraph(pkg)
# Package was added on left, so add it on the right
self.right_control.add_paragraph(unicode(para))
self.right_control.write()
# To ease the merge, drop the package addition from the left
self.left_control.remove_package(pkg)
self.left_control.write()
self.record_note('Carried forward addition of %s binary package' %
pkg)
def __merge_modified_version_constraint(self, package, field, entry,
base_dep, left_dep, right_dep):
if right_dep is None:
logging.debug('Drop modified %s %s', field_name(package, field),
entry)
self.left_control.patch_at_offset(left_dep.position, base_dep)
self.left_control.write()
self.record_note(
'Dropped modified %s %s because this '
'dependency disappeared from the upstream '
'version' % (field_name(package, field), entry), True)
return
base_vc = base_dep.version_constraint
left_vc = left_dep.version_constraint
right_vc = right_dep.version_constraint
# If the left version increases the minimum version required,
# but the right version increases it even more, then drop our
# left-side change.
if right_vc is not None and right_vc.startswith(">") \
and base_vc is not None and base_vc.startswith(">") \
and left_vc is not None and left_vc.startswith(">"):
base_min = Version(base_vc.split()[1])
left_min = Version(left_vc.split()[1])
right_min = Version(right_vc.split()[1])
if left_min > base_min and right_min > left_min:
# Restore original version constraint on the left
self.left_control.patch_at_offset(left_vc.position, base_vc)
self.left_control.write()
self.record_note(
'Dropped modified %s %s %s version '
'constraint because upstream increased it '
'further' % (field_name(package, field), left_vc, entry),
True)
return
if right_vc is not None:
logging.debug('Apply %s %s modified version constraint',
field_name(package, field), entry)
self.right_control.patch_at_offset(right_vc.position, left_vc)
self.right_control.write()
# Restore original version constraint on the left to ease the merge
self.left_control.patch_at_offset(left_vc.position, base_vc)
self.left_control.write()
self.record_note('Reapplied modified %s %s %s version '
'constraint' %
(field_name(package, field), left_vc, entry))
def __merge_added_dep(self, package, field, entry, left_dep, right_dep):
if right_dep is None:
# If a dependency was added on the left, but is not present on
# the right, add it there.
self.right_control.add_depends_entry(package, field,
unicode(left_dep))
self.right_control.write()
self.record_note('Carried forward change to add %s %s' %
(field_name(package, field), entry))
else:
# If a dependency was added on the left and is now present
# on the right, we can just drop the change on the left.
self.record_note(
'Dropped %s %s addition since it is now present '
'on the right' % (field_name(package, field), entry), True)
# Drop our change on the left to ease the merge
self.left_control.remove_depends_entry(package, field, entry)
self.left_control.write()
# Merge Dependencies fields
def merge_depends(self, package, field):
base_text = self.base_control.get_paragraph(package).get(field)
left_text = self.left_control.get_paragraph(package).get(field)
if base_text == left_text:
return
base_deps = self.base_control.parse_depends(package, field)
left_deps = self.left_control.parse_depends(package, field)
right_deps = self.right_control.parse_depends(package, field)
logger.debug('Detected %s difference, trying to merge',
field_name(package, field))
# For each build dependency on the left, check if it has a modified
# version constraint compared to the base.
# If so, and if the right version doesn't have that build dependency
# at all, drop our local change by restoring the original version
# constraint.
for pkg in left_deps.keys():
if pkg not in base_deps:
continue
base_bdep = base_deps[pkg]
left_bdep = left_deps[pkg]
if left_bdep.version_constraint == base_bdep.version_constraint:
continue
right_bdep = right_deps.get(pkg)
self.__merge_modified_version_constraint(package, field, pkg,
base_bdep, left_bdep,
right_bdep)
# Reparse after changes
left_deps = self.left_control.parse_depends(package, field)
right_deps = self.right_control.parse_depends(package, field)
# If a package was removed in the left version and also on the right,
# remove it from the base version in order to ease up the merge.
for pkg in base_deps.keys():
if pkg in left_deps or pkg in right_deps:
continue
logging.debug(
'Dropping %s %s from base as it was dropped from '
'left and right', field_name(package, field), pkg)
self.base_control.remove_depends_entry(package, field, pkg)
self.base_control.write()
# Reparse after changes
base_deps = self.base_control.parse_depends(package, field)
self.record_note(
'Dropped removed %s %s because this '
'dependency also disappeared from the upstream '
'version' % (field_name(package, field), pkg), True)
# If a package was removed in the left version and can still be
# removed from the right, remove it from base and right to ease up
# the merge. (The more logical alternative of reintroducing the
# dep on the left side is more complicated than removing...)
for pkg in base_deps.keys():
if pkg in left_deps or pkg not in right_deps:
continue
logging.debug(
'Dropping %s %s from base and right as it was '
'dropped from left', field_name(package, field), pkg)
self.base_control.remove_depends_entry(package, field, pkg)
self.base_control.write()
self.right_control.remove_depends_entry(package, field, pkg)
self.right_control.write()
# Reparse after changes
base_deps = self.base_control.parse_depends(package, field)
right_deps = self.right_control.parse_depends(package, field)
self.record_note('Carried forward removal of %s %s' %
(field_name(package, field), pkg))
# Handle dependencies added on the left
for pkg in left_deps.keys():
if pkg in base_deps:
continue
self.__merge_added_dep(package, field, pkg, left_deps[pkg],
right_deps.get(pkg))
# Reparse after changes
left_deps = self.left_control.parse_depends(package, field)
right_deps = self.right_control.parse_depends(package, field)
# Handle dependencies where an arch list was added on the left,
# where the base did not have any list.
for pkg in base_deps.keys():
if pkg not in left_deps or pkg not in right_deps:
continue
left_bdep = left_deps[pkg]
base_bdep = base_deps[pkg]
if base_bdep.arch_list is not None or left_bdep.arch_list is None:
continue
right_bdep = right_deps[pkg]
logger.debug('Carrying over %s %s added arch list',
field_name(package, field), pkg)
new_val = unicode(right_bdep) + " [" + left_bdep.arch_list + "]"
self.right_control.patch_at_offset(right_bdep.position, new_val)
self.right_control.write()
# Drop the original change on the left side by restoring the
# dependency from the base version, in hope of aiding the merge.
self.left_control.patch_at_offset(left_bdep.position, base_bdep)
self.left_control.write()
self.record_note('Carried forward change to add arch list '
'[%s] to %s %s' %
(left_bdep.arch_list, field, pkg))
# If there's no remaining difference in the actual build-depends list
# and it's just changes in spaces/commas/etc then just drop our
# changes on the left.
base_text = self.base_control.get_paragraph(package).get(field)
left_text = self.left_control.get_paragraph(package).get(field)
if base_text != left_text \
and sorted(left_deps.items()) == sorted(base_deps.items()):
self.__restore_original_value(package, field)
self.record_note('Tweaked %s syntax to ease the merge' %
field_name(package, field))
def merge_package_depends(self, field):
for pkg in self.base_control.get_package_names():
if self.left_control.get_paragraph(pkg) \
and self.right_control.get_paragraph(pkg):
self.merge_depends(pkg, field)
def __write_non_comments(self, filename):
with open(filename, 'r') as fd:
base_text = [line for line in fd if not line.startswith('#')]
with open(filename, 'w') as fd:
fd.writelines(base_text)
# If we couldn't find a more intelligent merge, try to drop any changes
# that were made to the comments in the file. This is done with a heavy
# handed approach of removing all the comments from base and left.
def remove_comments(self):
with open(self.base_control_path, 'r') as fd:
base_comments = [line for line in fd if line.startswith('#')]
with open(self.left_control_path, 'r') as fd:
left_comments = [line for line in fd if line.startswith('#')]
if base_comments == left_comments:
return
logger.debug('Drop all comments from base and left to ease the merge')
self.__write_non_comments(self.base_control_path)
self.__write_non_comments(self.left_control_path)
self.record_note('Dropped comment changes to simplify the merge', True)