def read_changelog(filename):
"""Return a parsed changelog file."""
entries = []
with open(filename) as cl:
(ver, text) = (None, "")
for line in cl:
match = CL_RE.search(line)
if match:
try:
ver = Version(match.group(2))
except ValueError:
ver = None
text += line
elif line.startswith(" -- "):
if ver is None:
ver = Version("0")
text += line
entries.append((ver, text))
(ver, text) = (None, "")
elif len(line.strip()) or ver is not None:
text += line
if len(text):
entries.append((ver, text))
return entries
def read_watermark(distro, source):
"""Read the watermark for a given source."""
mark_file = "%s/%s/watermark" \
% (ROOT, pool_directory(distro, source["Package"]))
if not os.path.isfile(mark_file):
return Version("0")
with open(mark_file) as mark:
return Version(mark.read().strip())
def __merge_modified_version_constraint(self, package, field, entry,
base_dep, left_dep, right_dep):
if right_dep is None:
logging.debug('Drop modified %s %s', field_name(package, field),
entry)
self.left_control.patch_at_offset(left_dep.position, base_dep)
self.left_control.write()
self.record_note(
'Dropped modified %s %s because this '
'dependency disappeared from the upstream '
'version' % (field_name(package, field), entry), True)
return
base_vc = base_dep.version_constraint
left_vc = left_dep.version_constraint
right_vc = right_dep.version_constraint
# If the left version increases the minimum version required,
# but the right version increases it even more, then drop our
# left-side change.
if right_vc is not None and right_vc.startswith(">") \
and base_vc is not None and base_vc.startswith(">") \
and left_vc is not None and left_vc.startswith(">"):
base_min = Version(base_vc.split()[1])
left_min = Version(left_vc.split()[1])
right_min = Version(right_vc.split()[1])
if left_min > base_min and right_min > left_min:
# Restore original version constraint on the left
self.left_control.patch_at_offset(left_vc.position, base_vc)
self.left_control.write()
self.record_note(
'Dropped modified %s %s %s version '
'constraint because upstream increased it '
'further' % (field_name(package, field), left_vc, entry),
True)
return
if right_vc is not None:
logging.debug('Apply %s %s modified version constraint',
field_name(package, field), entry)
self.right_control.patch_at_offset(right_vc.position, left_vc)
self.right_control.write()
# Restore original version constraint on the left to ease the merge
self.left_control.patch_at_offset(left_vc.position, base_vc)
self.left_control.write()
self.record_note('Reapplied modified %s %s %s version '
'constraint' %
(field_name(package, field), left_vc, entry))
def merge(ours, upstream, base, output_dir, force=False):
"""Merge PackageVersion instances @ours (left) and @upstream (right)
using the common ancestor PackageVersion @base, placing the result in
@output_dir.
"""
mergedVersion = Version(upstream.version() + "co1")
base_version = Version(re.sub("build[0-9]+$", "", base.version()))
left_version = Version(re.sub("build[0-9]+$", "", ours.version()))
right_version = Version(re.sub("build[0-9]+$", "", upstream.version()))
if base_version >= left_version:
cleanup(output_dir)
if left_version < right_version:
tree.ensure("%s/%s" % (output_dir, "REPORT"))
def read_basis(filename):
"""Read the basis version of a patch from a file."""
basis_file = filename + "-basis"
if not os.path.isfile(basis_file):
return None
with open(basis_file) as basis:
return Version(basis.read().strip())
def currentVersions(self):
"""Return all available versions of this package in self.distro.
They are in no particular order.
"""
versions = []
for s in self.distro.getSources(self.dist, self.component):
if s['Package'] == self.name:
versions.append(PackageVersion(self, Version(s['Version'])))
return versions
def getPoolVersions(self):
"""Return all available versions of this package in the pool as
PackageVersion objects. They are in no particular order.
"""
versions = []
for f in glob(self.poolPath + '/*.dsc'):
dsc = ControlFile(f, multi_para=False, signed=True).para
versions.append(PackageVersion(self, Version(dsc['Version'])))
return versions
def generate_patch(base_distro,
base_source,
distro,
our_source,
slipped=False,
force=False):
"""Generate a patch file for the given comparison."""
our_version = Version(our_source["Version"])
base_version = Version(base_source["Version"])
if base_version > our_version:
# Allow comparison of source -1 against our -0ubuntuX (slipped)
if not slipped:
return
elif our_version.revision is None:
return
elif not our_version.revision.startswith("0ubuntu"):
return
elif base_version.revision != "1":
return
elif base_version.upstream != our_version.upstream:
return
elif base_version.epoch != our_version.epoch:
return
logging.debug("Allowing comparison of -1 against -0ubuntuX")
elif base_version == our_version:
return
filename = patch_file(distro, our_source, slipped)
if not force:
basis = read_basis(filename)
if basis is not None and basis == base_version:
return
unpack_source(base_distro, base_source)
unpack_source(distro, our_source)
ensure(filename)
save_patch_file(filename, base_source, our_source)
save_basis(filename, base_version)
logging.info("Saved patch file: %s", tree.subdir(ROOT, filename))
def parse(self, file):
"""Parse source control (dsc) file.
Parses the opened source control (dsc) file given, validates it
and stores the most important information in the object. The
rest of the fields can still be accessed through the para
member.
"""
super(SourceControl, self).parse(file, signed=True)
if "Format" in self.para:
try:
self.dsc_format = float(self.para["Format"])
if int(self.dsc_format) != 1:
raise IOError
except ValueError:
raise IOError
if "Source" in self.para:
self.source = self.para["Source"]
if not valid_source.search(self.source):
raise IOError
else:
raise IOError
if "Version" in self.para:
self.version = Version(self.para["Version"])
else:
raise IOError
if "Files" in self.para:
files = self.para["Files"].strip("\n").split("\n")
for f in files:
try:
(md5sum, size, name) = f.split(None, 2)
except ValueError:
raise IOError
sf = SourceFile(name, size, md5sum)
if name.endswith(".tar.gz"):
if self.tar:
raise IOError
self.tar = sf
elif name.endswith(".diff.gz"):
if self.diff:
raise IOError
self.diff = sf
self.files.append(sf)
if not self.tar:
raise IOError
else:
raise IOError
def main(options, args):
logger.info('Producing merges...')
excludes = []
if options.exclude is not None:
for filename in options.exclude:
logger.info('excluding packages from %s', filename)
excludes.extend(read_package_list(filename))
includes = []
if options.include is not None:
for filename in options.include:
logger.info('including packages from %s', filename)
includes.extend(read_package_list(filename))
# For each package in the destination distribution, locate the latest in
# the source distribution; calculate the base from the destination and
# produce a merge combining both sets of changes
for target in config.targets(args):
logger.info('considering target %s', target)
our_dist = target.dist
our_component = target.component
d = target.distro
for pkg in d.packages(target.dist, target.component):
if options.package is not None and pkg.name not in options.package:
continue
if len(includes) and pkg.name not in includes:
logger.info('skipping package %s: not in include list',
pkg.name)
continue
if len(excludes) and pkg.name in excludes:
logger.info('skipping package %s: in exclude list', pkg.name)
continue
if pkg.name in target.blacklist:
logger.info("%s is blacklisted, skipping", pkg.name)
continue
logger.info('considering package %s', pkg.name)
if options.version:
our_version = PackageVersion(pkg, Version(options.version))
logger.debug('our version: %s (from command line)',
our_version)
else:
our_version = pkg.newestVersion()
logger.debug('our version: %s', our_version)
output_dir = result_dir(target.name, pkg.name)
try:
report = handle_package(options, output_dir, target, pkg,
our_version)
if report is not None:
report.write_report(output_dir)
except Exception:
logging.exception('Failed handling merge for %s', pkg)
def getCurrentSources(self):
"""Return a list of Sources stanzas (dictionaries of the form
{ "Field": "value" }) describing versions of this package
available in (self.distro, self.dist, self.component), with
the oldest version first.
"""
sources = self.distro.getSources(self.dist, self.component)
matches = []
for source in sources:
if source['Package'] == self.name:
matches.append(source)
matches.sort(key=lambda x: Version(x['Version']))
return matches
def newestPackageVersions(self, dist, component):
sources = self.getSources(dist, component)
newest = {}
for source in sources:
package = source["Package"]
version = Version(source['Version'])
if package not in newest or version > newest[package]:
newest[package] = version
ret = []
for name in sorted(newest.keys()):
pkg = Package(self, dist, component, name)
ret.append(PackageVersion(pkg, newest[name]))
return ret
def make_patches(our_distro,
our_source,
src_distro,
src_source,
base,
slipped=False,
force=False):
"""Make sets of patches from the given base."""
package = our_source["Package"]
try:
base_source = get_nearest_source(package, base)
base_version = Version(base_source["Version"])
logging.debug("%s: base is %s (%s wanted)", package, base_version,
base)
except IndexError:
return
try:
generate_patch(src_distro, base_source, our_distro, our_source,
slipped, force)
generate_patch(src_distro, base_source, src_distro, src_source,
slipped, force)
finally:
cleanup_source(base_source)
def version(self):
return Version(self.data['version']) if 'version' in self.data \
else None
url = '%s/mr/package/%s/' % (SNAPSHOT_BASE, package_name)
try:
fd = urllib2.urlopen(url)
data = json.load(fd)
except urllib2.HTTPError, e:
if e.code == 404:
return []
raise
except urllib2.URLError, e:
if isinstance(e.reason, OSError) and e.reason.errno == errno.ENOENT:
return []
raise
versions = []
for vdict in data['result']:
versions.append(Version(vdict['version']))
return versions
def debsnap_get_file_hash(data, filename):
# Find the sha1 hash for a given file from the debsnap data
for filehash, files in data['fileinfo'].iteritems():
for fileinfo in files:
if fileinfo['name'] == filename:
return filehash
return None
def debsnap_download_file(url, output_path):
# Download a specific file from debsnap
def __produce_merge(target, base, base_dir, left, left_dir,
upstream, upstream_dir, output_dir):
report = MergeReport(left=left, right=upstream)
report.target = target.name
report.mom_version = str(VERSION)
report.merge_date = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
cleanup(output_dir)
downstream_versions = read_changelog(left_dir + '/debian/changelog')
upstream_versions = read_changelog(upstream_dir + '/debian/changelog')
report.left_changelog = save_changelog(output_dir, downstream_versions,
left, [base.version])
# If the base is a common ancestor, log everything from the ancestor
# to the current version. Otherwise just log the first entry.
limit = 1
for upstream_version, text in upstream_versions:
if upstream_version == base.version:
limit = None
break
report.right_changelog = save_changelog(output_dir, upstream_versions,
upstream, [base.version], limit)
report.set_base(base)
logger.info('base version: %s', base.version)
generate_patch(base, left.package.distro, left, slipped=False, force=False,
unpacked=True)
generate_patch(base, upstream.package.distro, upstream, slipped=False,
force=False, unpacked=True)
report.merged_version = Version(str(upstream.version) +
config.get('LOCAL_SUFFIX') + '1')
if base >= upstream:
logger.info("Nothing to be done: %s >= %s", base, upstream)
report.result = MergeResult.KEEP_OURS
report.merged_version = left.version
report.write_report(output_dir)
return report
# Careful: MergeReport.merged_dir is the output directory for the .dsc or
# tarball, whereas our local variable merged_dir (below) is a temporary
# directory containing unpacked source code. Don't mix them up.
report.merged_dir = output_dir
if base.version == left.version:
logger.info("Syncing %s to %s", left, upstream)
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
report.result = MergeResult.SYNC_THEIRS
report.build_metadata_changed = False
report.right_patch = copy_in(output_dir, upstream)
report.merged_version = upstream.version
report.merged_patch = report.right_patch
report.merged_files = report.right_files
write_report(report, left=left, base=base, right=upstream,
src_file=None,
# this is MergeReport.merged_dir...
output_dir=output_dir,
# ... and for a SYNC_THEIRS merge, we don't need to look
# at the unpacked source code
merged_dir=None)
return report
merged_dir = work_dir(left.package.name, report.merged_version)
logger.info("Merging %s..%s onto %s", upstream, base, left)
try:
merger = DebTreeMerger(left_dir, left.package.name,
left.getDscContents()['Format'],
left.package.distro.name,
upstream_dir, upstream.package.name,
upstream.getDscContents()['Format'],
upstream.package.distro.name,
base_dir, merged_dir)
merger.run()
except Exception as e:
cleanup(merged_dir)
logger.exception("Could not merge %s, probably bad files?", left)
report.result = MergeResult.FAILED
report.message = '%s: %s' % (e.__class__.__name__, e)
report.write_report(output_dir)
return report
for note, changelog_worthy in merger.notes:
report.notes.append(note)
if len(merger.conflicts) == 0 and merger.total_changes_made == 1 \
and len(merger.modified_files) == 1 \
and 'debian/changelog' in merger.modified_files:
# Sync to upstream if the only remaining change is in the changelog
logger.info("Syncing %s to %s since only changes are in changelog",
left, upstream)
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
report.result = MergeResult.SYNC_THEIRS
report.build_metadata_changed = False
report.left_patch = copy_in(output_dir, left)
report.right_patch = copy_in(output_dir, upstream)
report.merged_version = upstream.version
report.merged_patch = report.right_patch
report.merged_files = report.right_files
report.notes.append('Synced to upstream version because the '
'changelog was the only modified file')
write_report(report, left=left, base=base, right=upstream,
src_file=None,
# this is MergeReport.merged_dir...
output_dir=output_dir,
# ... and for a SYNC_THEIRS merge, we don't need to
# look at the unpacked source code
merged_dir=None)
cleanup(merged_dir)
return report
if 'debian/changelog' not in merger.conflicts:
try:
add_changelog(left.package.name, report.merged_version,
left.package.distro.name, left.package.dist,
upstream.package.distro.name, upstream.package.dist,
merger.notes, merged_dir)
except IOError as e:
logger.exception("Could not update changelog for %s!", left)
report.result = MergeResult.FAILED
report.message = 'Could not update changelog: %s' % e
report.write_report(output_dir)
return report
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
copy_in(output_dir, base)
report.left_patch = copy_in(output_dir, left)
report.right_patch = copy_in(output_dir, upstream)
report.build_metadata_changed = False
report.merged_dir = output_dir
if len(merger.conflicts):
src_file = create_tarball(left.package.name, report.merged_version,
output_dir, merged_dir)
report.result = MergeResult.CONFLICTS
report.conflicts = sorted(merger.conflicts)
report.merge_failure_tarball = src_file
report.merged_dir = None
else:
result, message, src_file = create_source(
left.package.name, report.merged_version, left.version,
output_dir, merged_dir)
report.result = result
if result == MergeResult.MERGED:
assert src_file.endswith('.dsc'), src_file
dsc = ControlFile("%s/%s" % (output_dir, src_file),
signed=True).para
report.build_metadata_changed = is_build_metadata_changed(
left.getDscContents(), dsc)
report.merged_files = [src_file] + [f[2] for f in files(dsc)]
# Some of our merge magic may have tweaked the upstream and left
# dirs. Recreate them now so that the diff is accurate.
cleanup_source(upstream)
upstream_dir = unpack_source(upstream)
cleanup_source(left)
left_dir = unpack_source(left)
report.merged_patch = create_patch(
report.merged_version,
"%s/%s_%s_from-theirs.patch" % (output_dir, left.package.name,
report.merged_version),
merged_dir, upstream, upstream_dir)
report.proposed_patch = create_patch(
report.merged_version,
"%s/%s_%s_from-ours.patch" % (output_dir, left.package.name,
report.merged_version),
merged_dir, left, left_dir)
else:
report.result = result
report.message = message
report.merged_dir = ""
report.merge_failure_tarball = src_file
write_report(report, left, base, upstream, src_file=src_file,
output_dir=output_dir, merged_dir=merged_dir)
logger.info("Wrote output to %s", src_file)
cleanup(merged_dir)
return report
def handle_package(options, output_dir, target, pkg, our_version):
update_info = UpdateInfo(pkg)
if update_info.version is None:
logger.error('UpdateInfo version %s does not match our version %s"',
update_info.version, our_version)
report = MergeReport(left=our_version)
report.target = target.name
report.result = MergeResult.FAILED
report.message = 'Could not find update info for version %s' % \
our_version
return report
if update_info.upstream_version is None \
or our_version.version >= update_info.upstream_version:
logger.debug("No updated upstream version available for %s",
our_version)
cleanup(output_dir)
report = MergeReport(left=our_version)
report.target = target.name
report.result = MergeResult.KEEP_OURS
report.merged_version = our_version.version
return report
if update_info.base_version is None:
logger.info("No base version available for %s", our_version)
cleanup(output_dir)
report = MergeReport(left=our_version)
report.target = target.name
report.result = MergeResult.NO_BASE
return report
upstream = target.findSourcePackage(pkg.name, update_info.upstream_version)
if not upstream:
logger.error('Could not find upstream version %s in pool',
update_info.upstream_version)
cleanup(output_dir)
report = MergeReport(left=our_version)
report.target = target.name
report.result = MergeResult.FAILED
report.message = 'Could not find upstream version %s in pool' % \
update_info.upstream_version
return report
upstream = upstream[0]
base = None
pool_versions = target.getAllPoolVersions(pkg.name)
for pv in pool_versions:
if pv.version == update_info.base_version:
base = pv
if base is None:
logger.error('Could not find base version %s in pool',
update_info.base_version)
cleanup(output_dir)
report = MergeReport(left=our_version)
report.target = target.name
report.result = MergeResult.FAILED
report.message = 'Could not find base version %s in pool' % \
update_info.base_version
return report
try:
report = read_report(output_dir)
# See if sync_upstream_packages already set
if not options.force and \
pkg.name in target.sync_upstream_packages and \
Version(report['right_version']) == upstream.version and \
Version(report['left_version']) == our_version.version and \
Version(report['merged_version']) == upstream.version and \
report['result'] == MergeResult.SYNC_THEIRS:
logger.info("sync to upstream for %s [ours=%s, theirs=%s] "
"already produced, skipping run", pkg,
our_version.version, upstream.version)
return None
elif (not options.force and
Version(report['right_version']) == upstream.version and
Version(report['left_version']) == our_version.version and
# we'll retry the merge if there was an unexpected
# failure, a missing base or an unknown result last time
report['result'] in (MergeResult.KEEP_OURS,
MergeResult.SYNC_THEIRS, MergeResult.MERGED,
MergeResult.CONFLICTS)):
logger.info("merge for %s [ours=%s, theirs=%s] already produced, "
"skipping run",
pkg, our_version.version, upstream.version)
return None
except (AttributeError, ValueError, KeyError):
pass
if pkg.name in target.sync_upstream_packages:
logger.info("Syncing to %s per sync_upstream_packages", upstream)
cleanup(output_dir)
report = MergeReport(left=our_version, right=upstream)
report.target = target.name
report.result = MergeResult.SYNC_THEIRS
report.merged_version = upstream.version
report.message = "Using version in upstream distro per " \
"sync_upstream_packages configuration"
if update_info.specific_upstream:
report.notes.append('Synced with specific upstream %s due to '
'runtime parameters'
% update_info.specific_upstream)
return report
if options.sync_to_upstream:
logger.info("Syncing to %s per command line", upstream)
cleanup(output_dir)
report = MergeReport(left=our_version, right=upstream)
report.target = target.name
report.result = MergeResult.SYNC_THEIRS
report.merged_version = upstream.version
if update_info.specific_upstream:
report.notes.append('Synced with specific upstream %s due to '
'runtime parameters'
% update_info.specific_upstream)
else:
report.notes.append('Force-synced to upstream due to runtime '
'parameters')
return report
logger.info("local: %s, upstream: %s", our_version, upstream)
try:
report = produce_merge(target, base, our_version, upstream, output_dir)
if update_info.specific_upstream:
report.notes.append('Merged with specific upstream %s due to '
'runtime parameters'
% update_info.specific_upstream)
return report
except ValueError as e:
logger.exception("Could not produce merge, "
"perhaps %s changed components upstream?", pkg)
report = MergeReport(left=our_version, right=upstream)
report.target = target.name
report.result = MergeResult.FAILED
report.message = 'Could not produce merge: %s' % e
return report
def upstream_version(self):
if 'upstream_version' in self.data:
return Version(self.data['upstream_version'])
else:
return None
def produce_merge(left_source,
left_distro,
left_dist,
base_source,
right_source,
right_distro,
right_dist,
force=False):
"""Produce a merge for the given two packages."""
package = base_source["Package"]
merged_version = Version(right_source["Version"] + "tanglu1")
output_dir = result_dir(package)
if re.search(".*build[0-9]+$", left_source["Version"]):
cleanup(output_dir)
return
base_version = Version(base_source["Version"])
if base_version >= left_source["Version"]:
cleanup(output_dir)
return
elif base_version >= right_source["Version"]:
cleanup(output_dir)
return
if not force:
try:
(prev_base, prev_left, prev_right) \
= read_report(output_dir, left_distro, right_distro)
if prev_base == base_version \
and prev_left == left_source["Version"] \
and prev_right == right_source["Version"]:
return
except ValueError:
pass
logging.info("Trying to merge %s: %s <- %s -> %s", package,
left_source["Version"], base_source["Version"],
right_source["Version"])
left_name = "%s-%s (%s)" % (package, left_source["Version"], left_distro)
right_name = "%s-%s (%s)" % (package, right_source["Version"],
right_distro)
try:
left_dir = unpack_source(left_distro, left_source)
base_dir = unpack_source(right_distro, base_source)
right_dir = unpack_source(right_distro, right_source)
merged_dir = work_dir(package, merged_version)
try:
conflicts = do_merge(left_dir, left_name, left_distro, base_dir,
right_dir, right_name, right_distro,
merged_dir)
add_changelog(package, merged_version, left_distro, left_dist,
right_distro, right_dist, merged_dir)
# Now clean up the output
cleanup(output_dir)
os.makedirs(output_dir)
copy_in(output_dir, base_source)
left_patch = copy_in(output_dir, left_source, left_distro)
right_patch = copy_in(output_dir, right_source, right_distro)
patch_file = None
if len(conflicts):
src_file = create_tarball(package, merged_version, output_dir,
merged_dir)
else:
src_file = create_source(package, merged_version,
Version(left_source["Version"]),
output_dir, merged_dir)
if src_file.endswith(".dsc"):
patch_file = create_patch(package, merged_version,
output_dir, merged_dir,
right_source, right_dir)
write_report(left_source, left_distro, left_patch, base_source,
right_source, right_distro, right_patch,
merged_version, conflicts, src_file, patch_file,
output_dir, merged_dir)
finally:
cleanup(merged_dir)
finally:
cleanup_source(right_source)
cleanup_source(base_source)
cleanup_source(left_source)
def check(self):
try:
self.result = MergeResult(self.result)
except ValueError:
self.message = 'unparsed result %s: %s' % (self.result,
self.message)
self.result = MergeResult.UNKNOWN
if self.source_package is None:
raise AttributeError('Insufficient detail in report: no '
'source package')
if self.left_version is None:
raise AttributeError('Insufficient detail in report: our '
'version is missing')
if self.left_distro is None:
raise AttributeError('Insufficient detail in report: our '
'distro is missing')
if self.result not in (MergeResult.KEEP_OURS, MergeResult.FAILED):
if self.right_version is None:
raise AttributeError('Insufficient detail in report: '
'their version is missing')
if self.right_distro is None:
raise AttributeError('Insufficient detail in report: '
'their distro is missing')
# promote versions to Version objects
for k in ("left_version", "right_version", "base_version",
"merged_version"):
v = getattr(self, k)
if v is not None:
setattr(self, k, Version(v))
if self.result == MergeResult.NO_BASE:
assert self.base_version is None, self.base_version
elif self.result == MergeResult.SYNC_THEIRS:
assert not self.conflicts, self.conflicts
self.merged_dir = ""
self.merged_files = self.right_files
elif self.result == MergeResult.KEEP_OURS:
assert not self.conflicts, self.conflicts
self.merged_dir = ""
self.merged_files = self.left_files
elif self.result == MergeResult.FAILED:
pass
elif self.result == MergeResult.MERGED:
assert not self.conflicts, self.conflicts
elif self.result == MergeResult.CONFLICTS:
assert self.conflicts
if self.result in (MergeResult.CONFLICTS, MergeResult.FAILED,
MergeResult.MERGED):
if (self.merged_version is None or
(self.merged_version.revision is not None and
self.left_version.upstream != self.merged_version.upstream)):
maybe_sa = ' -sa'
else:
maybe_sa = ''
self["genchanges"] = "-S -v%s%s" % (self.left_version, maybe_sa)
def write_report(left_source, left_distro, left_patch, base_source,
right_source, right_distro, right_patch, merged_version,
conflicts, src_file, patch_file, output_dir, merged_dir):
"""Write the merge report."""
filename = "%s/REPORT" % output_dir
with open(filename, "w") as report:
package = base_source["Package"]
# Package and time
print("%s" % package, file=report)
print("%s" % time.ctime(), file=report)
print(file=report)
# General rambling
print(fill("Below now follows the report of the automated "
"merge of the %s changes to the %s source "
"package against the new %s version." %
(left_distro.title(), package, right_distro.title())),
file=report)
print(file=report)
print(fill("This file is designed to be both human readable "
"and machine-parseable. Any line beginning with "
"four spaces is a file that should be downloaded "
"for the complete merge set."),
file=report)
print(file=report)
print(file=report)
print(fill("Here are the particulars of the three versions "
"of %s that were chosen for the merge. The base "
"is the newest version that is a common ancestor "
"of both the %s and %s packages. It may be of "
"a different upstream version, but that's not "
"usually a problem." %
(package, left_distro.title(), right_distro.title())),
file=report)
print(file=report)
print(fill("The files are the source package itself, and "
"the patch from the common base to that version."),
file=report)
print(file=report)
# Base version and files
print("base: %s" % base_source["Version"], file=report)
for md5sum, size, name in files(base_source):
print(" %s" % name, file=report)
print(file=report)
# Left version and files
print("%s: %s" % (left_distro, left_source["Version"]), file=report)
for md5sum, size, name in files(left_source):
print(" %s" % name, file=report)
print(file=report)
if left_patch is not None:
print("base -> %s" % left_distro, file=report)
print(" %s" % left_patch, file=report)
print(file=report)
# Right version and files
print("%s: %s" % (right_distro, right_source["Version"]), file=report)
for md5sum, size, name in files(right_source):
print(" %s" % name, file=report)
print(file=report)
if right_patch is not None:
print("base -> %s" % right_distro, file=report)
print(" %s" % right_patch, file=report)
print(file=report)
# Generated section
print(file=report)
print("Generated Result", file=report)
print("================", file=report)
print(file=report)
if src_file.endswith(".dsc"):
print(fill("No problems were encountered during the "
"merge, so a source package has been "
"produced along with a patch containing "
"the differences from the %s version to the "
"new version." % right_distro.title()),
file=report)
print(file=report)
print(fill("You should compare the generated patch "
"against the patch for the %s version "
"given above and ensure that there are no "
"unexpected changes. You should also "
"sanity check the source package." %
left_distro.title()),
file=report)
print(file=report)
print("generated: %s" % merged_version, file=report)
# Files from the dsc
dsc = ControlFile("%s/%s" % (output_dir, src_file),
multi_para=False,
signed=False).para
print(" %s" % src_file, file=report)
for md5sum, size, name in files(dsc):
print(" %s" % name, file=report)
print(file=report)
if patch_file is not None:
print("%s -> generated" % right_distro, file=report)
print(" %s" % patch_file, file=report)
print(file=report)
else:
print(fill("Due to conflict or error, it was not "
"possible to automatically create a source "
"package. Instead the result of the merge "
"has been placed into the following tar file "
"which you will need to turn into a source "
"package once the problems have been "
"resolved."),
file=report)
print(file=report)
print(" %s" % src_file, file=report)
print(file=report)
if len(conflicts):
print(file=report)
print("Conflicts", file=report)
print("=========", file=report)
print(file=report)
print(fill("In one or more cases, there were different "
"changes made in both %s and %s to the same "
"file; these are known as conflicts." %
(left_distro.title(), right_distro.title())),
file=report)
print(file=report)
print(fill("It is not possible for these to be "
"automatically resolved, so this source "
"needs human attention."),
file=report)
print(file=report)
print(fill("Those files marked with 'C ' contain diff3 "
"conflict markers, which can be resolved "
"using the text editor of your choice. "
"Those marked with 'C*' could not be merged "
"that way, so you will find .%s and .%s "
"files instead and should chose one of them "
"or a combination of both, moving it to the "
"real filename and deleting the other." %
(left_distro.upper(), right_distro.upper())),
file=report)
print(file=report)
conflicts.sort()
for name in conflicts:
if os.path.isfile("%s/%s" % (merged_dir, name)):
print(" C %s" % name, file=report)
else:
print(" C* %s" % name, file=report)
print(file=report)
if merged_version.revision is not None \
and Version(left_source["Version"]).upstream != merged_version.upstream:
sa_arg = " -sa"
else:
sa_arg = ""
print(file=report)
print(fill("Once you have a source package you are happy "
"to upload, you should make sure you include "
"the orig.tar.gz if appropriate and information "
"about all the versions included in the merge."),
file=report)
print(file=report)
print(fill("Use the following command to generate a "
"correct .changes file:"),
file=report)
print(file=report)
print(" $ dpkg-genchanges -S -v%s%s" %
(left_source["Version"], sa_arg),
file=report)
def main(options, args):
src_distro = options.source_distro
src_dist = options.source_suite
our_distro = options.dest_distro
our_dist = options.dest_suite
blacklist = read_blacklist()
# For each package in the destination distribution, locate the latest in
# the source distribution; calculate the base from the destination and
# create patches from that to both
for our_component in DISTROS[our_distro]["components"]:
if options.component is not None \
and our_component not in options.component:
continue
for our_source in get_sources(our_distro, our_dist, our_component):
if options.package is not None \
and our_source["Package"] not in options.package:
continue
if our_source["Package"] in blacklist:
continue
if search(".*build[0-9]+$", our_source["Version"]):
continue
try:
package = our_source["Package"]
our_version = Version(our_source["Version"])
our_pool_source = get_pool_source(our_distro, package,
our_version)
logging.debug("%s: %s is %s", package, our_distro, our_version)
except IndexError:
continue
try:
(src_source, src_version, src_pool_source) \
= get_same_source(src_distro, src_dist, package)
logging.debug("%s: %s is %s", package, src_distro, src_version)
except IndexError:
continue
try:
base = get_base(our_source)
make_patches(our_distro,
our_pool_source,
src_distro,
src_pool_source,
base,
force=options.force)
slip_base = get_base(our_source, slip=True)
if slip_base != base:
make_patches(our_distro,
our_pool_source,
src_distro,
src_pool_source,
slip_base,
True,
force=options.force)
finally:
cleanup_source(our_pool_source)
cleanup_source(src_pool_source)
def test_simple(self):
base = Version('3.0.0-2mom1').base()
self.assertEqual(base.upstream, '3.0.0')
self.assertEqual(base.revision, '2')
self.assertEqual(str(base), '3.0.0-2')
def test_complex(self):
base = Version('2:1.2.3-4ubuntu3mom1').base()
self.assertEqual(base.epoch, 2)
self.assertEqual(base.upstream, '1.2.3')
self.assertEqual(base.revision, '4')
self.assertEqual(str(base), '2:1.2.3-4')
def main(options, args):
src_distro = options.source_distro
src_dist = options.source_suite
our_distro = options.dest_distro
our_dist = options.dest_suite
excludes = []
if options.exclude is not None:
for filename in options.exclude:
excludes.extend(read_package_list(filename))
includes = []
if options.include is not None:
for filename in options.include:
includes.extend(read_package_list(filename))
blacklist = read_blacklist()
# For each package in the destination distribution, locate the latest in
# the source distribution; calculate the base from the destination and
# produce a merge combining both sets of changes
for our_component in DISTROS[our_distro]["components"]:
if options.component is not None \
and our_component not in options.component:
continue
for our_source in get_sources(our_distro, our_dist, our_component):
if options.package is not None \
and our_source["Package"] not in options.package:
continue
if our_source["Package"] in blacklist:
continue
if len(includes) and our_source["Package"] not in includes:
continue
if len(excludes) and our_source["Package"] in excludes:
continue
try:
package = our_source["Package"]
if options.version:
our_version = Version(options.version)
else:
our_version = Version(our_source["Version"])
our_pool_source = get_pool_source(our_distro, package,
our_version)
logging.debug("%s: %s is %s", package, our_distro, our_version)
except IndexError:
continue
try:
(src_source, src_version, src_pool_source) \
= get_same_source(src_distro, src_dist, package)
logging.debug("%s: %s is %s", package, src_distro, src_version)
except IndexError:
continue
try:
base = get_base(our_pool_source)
base_source = get_nearest_source(package, base)
base_version = Version(base_source["Version"])
logging.debug("%s: base is %s (%s wanted)", package,
base_version, base)
except IndexError:
continue
produce_merge(our_pool_source,
our_distro,
our_dist,
base_source,
src_pool_source,
src_distro,
src_dist,
force=options.force)
def test_zeroEpoch(self):
version = Version('0:1.2.3-4')
self.assertEqual(str(version), '0:1.2.3-4')
def _read_report_text(output_dir, filename, report):
"""Read an old-style semi-human-readable REPORT."""
merged_is_right = False
with open(filename) as r:
report["source_package"] = next(r).strip()
in_list = None
for line in r:
if line.startswith(" "):
if in_list == "base":
report["base_files"].append(line.strip())
elif in_list == "left":
report["left_files"].append(line.strip())
elif in_list == "right":
report["right_files"].append(line.strip())
elif in_list == "merged":
report["merged_files"].append(line.strip())
else:
in_list = None
if line.startswith("base:"):
report["base_version"] = Version(line[5:].strip())
in_list = "base"
elif line.startswith("our distro "):
m = re.match("our distro \(([^)]+)\): (.+)", line)
if m:
report["left_distro"] = m.group(1)
report["left_version"] = Version(m.group(2).strip())
in_list = "left"
elif line.startswith("source distro "):
m = re.match("source distro \(([^)]+)\): (.+)", line)
if m:
report["right_distro"] = m.group(1)
report["right_version"] = Version(m.group(2).strip())
in_list = "right"
elif line.startswith("generated:"):
in_list = "merged"
elif line.startswith("Merged without changes: YES"):
merged_is_right = True
elif line.startswith("Build-time metadata changed: NO"):
report["build_metadata_changed"] = False
elif line.startswith("Merge committed: YES"):
report["committed"] = True
elif line.startswith(" C ") or line.startswith(" C* "):
report["conflicts"].append(line[5:].strip())
# Try to synthesize a meaningful result from those fields
if report["base_version"] is None:
report["result"] = MergeResult.NO_BASE
elif merged_is_right:
report["result"] = MergeResult.SYNC_THEIRS
elif report["merged_files"]:
report["result"] = MergeResult.MERGED
elif report["conflicts"]:
report["result"] = MergeResult.CONFLICTS
else:
# doesn't look good... assume FAILED
report["result"] = MergeResult.FAILED
return report
def base_version(self):
if 'base_version' in self.data:
return Version(self.data['base_version'])
else:
return None
def main(options, args):
logger.debug('Committing merges...')
for target in config.targets(args):
d = target.distro
if not isinstance(d, OBSDistro):
logger.debug('Skipping %r distro %r: not an OBSDistro', target, d)
continue
for package in d.packages(target.dist, target.component):
if options.package and package.name not in options.package:
logger.debug('Skipping package %s: not selected', package.name)
continue
if package.name in target.blacklist:
logger.debug('Skipping package %s: blacklisted', package.name)
continue
try:
output_dir = result_dir(target.name, pkg.name)
report = read_report(output_dir)
except ValueError:
logger.debug('Skipping package %s: unable to read report',
package.name)
continue
if report['committed']:
if options.force:
logger.info("Forcing commit of %s", package)
else:
logger.debug("%s already committed, skipping!", package)
continue
if report['result'] not in (MergeResult.MERGED,
MergeResult.SYNC_THEIRS):
logger.debug("%s has nothing to commit: result=%s",
package, report['result'])
continue
filepaths = report['merged_files']
if filepaths == []:
logger.warning("Empty merged file list in %s/REPORT" % output_dir)
continue
if target.committable:
# we can commit directly to the target distribution
# FIXME: is this still a supported configuration? I wouldn't
# want to commit automated merges without some sort of manual
# check on the debdiff...
logger.info("Committing changes to %s", package)
if not options.dry_run:
try:
package.commit('Automatic update by Merge-O-Matic')
except urllib2.HTTPError as e:
logger.exception('Failed to commit %s: HTTP error %s at <%s>:',
package, e.code, e.geturl())
update_report(report, output_dir, False,
"HTTP error %s" % e.code)
except Exception as e:
logger.exception('Failed to commit %s:', package)
# deliberately rather vague, as below
update_report(report, output_dir, False,
"%s" % e.__class__.__name__)
else:
update_report(report, output_dir, True,
committed_to=d.obsProject(target.dist, target.component))
continue
# else we need to branch it and commit to the branch
try:
logger.debug("Branching %s", package)
branchPkg = package.branch("home:%s:branches"%(d.obsUser))
branch = branchPkg.distro
branch.sync(target.dist, target.component, [branchPkg,])
logger.info("Committing changes to %s, and submitting merge request to %s", branchPkg, package)
if report['result'] == MergeResult.SYNC_THEIRS:
srcDistro = Distro.get(report['right_distro'])
version = Version(report['right_version'])
logger.debug('Copying updated upstream version %s from %r into %r',
version,
srcDistro,
target)
for upstream in target.getSourceLists(package.name):
for src in upstream:
srcDistro = src.distro
try:
pkg = srcDistro.findPackage(package.name, searchDist=src.dist,
version=version)[0]
pfx = pkg.poolPath
break
except model.error.PackageNotFound:
pass
else:
logger.debug('Copying merged version from %r into %r',
branch, target)
pfx = result_dir(target.name, package.name)
# this might raise an error
obsFiles = branchPkg.getOBSFiles()
# Get the linked target files since the checkout is expanded
# and may contain them
linkedFiles = package.getOBSFiles()
for f in obsFiles:
if f.endswith(".dsc"):
oldDsc = '%s/%s'%(branchPkg.obsDir(), f)
break
for f in filepaths:
if f.endswith(".dsc"):
newDsc = '%s/%s'%(pfx, f)
break
#logger.debug("Running debdiff on %s and %s", oldDsc, newDsc)
#comment = shell.get(("debdiff", oldDsc, newDsc), okstatus=(0,1))
# FIXME: Debdiff needs implemented in OBS, as large merge descriptions break clucene.
comment = ''
if report['result'] == MergeResult.SYNC_THEIRS:
comment += 'Sync to '
elif report['result'] == MergeResult.MERGED:
comment += 'Merge with '
comment += 'version %s from %s %s' %(report['right_version'],
report['right_distro'],
report['right_suite'])
comment += "\n\nMerge report is available at %s"%('/'.join((config.get('MOM_URL'), subdir(config.get('ROOT'), output_dir), 'REPORT.html')))
# The newlines seem to cause create_submit_request to send
# UTF-32 over the wire, which OBS promptly chokes on. Encode
# the message to UTF-8 first.
comment = comment.encode('utf-8')
if not options.dry_run:
filesUpdated = False
for f in obsFiles + linkedFiles:
if f == "_link":
continue
try:
logger.debug('deleting %s/%s', branchPkg.obsDir(), f)
os.unlink('%s/%s'%(branchPkg.obsDir(), f))
filesUpdated = True
except OSError:
pass
for f in filepaths:
if f == "_link":
continue
logger.debug('copying %s/%s -> %s', pfx, f, branchPkg.obsDir())
shutil.copy2("%s/%s"%(pfx, f), branchPkg.obsDir())
filesUpdated = True
if filesUpdated:
logger.debug('Submitting request to merge %r from %r into %r',
branchPkg, branch, target)
try:
branchPkg.commit('Automatic update by Merge-O-Matic')
obs_project = d.obsProject(target.dist, target.component)
reqid = branchPkg.submitMergeRequest(obs_project, comment)
update_report(report, output_dir, True,
committed_to=obs_project,
request_url=branchPkg.webMergeRequest(reqid))
except xml.etree.cElementTree.ParseError:
logger.exception("Failed to commit %s", branchPkg)
update_report(report, output_dir, False, "OBS API Error")
except urllib2.HTTPError as e:
logger.exception("Failed to commit %s: HTTP error %s at <%s>:",
branchPkg, e.code, e.geturl())
update_report(report, output_dir, False,
"HTTP error %s" % e.code)
except Exception as e:
logger.exception("Failed to commit %s", branchPkg)
# deliberately being a bit vague here in case the exact
# exception leaks internal info
update_report(report, output_dir, False,
"%s" % e.__class__.__name__)
else:
logger.info("Not committing, due to --dry-run")
except urllib2.HTTPError as e:
logger.exception('Failed to branch %s: HTTP error %s at <%s>:',
package, e.code, e.geturl())
update_report(report, output_dir, False,
"Failed to branch: HTTP error %s" % e.code)
except Exception as e:
logger.exception('Failed to branch %s:', package)
# deliberately being a bit vague here in case the exact
# exception leaks internal info
update_report(report, output_dir, False,
"Failed to branch: %s" % e.__class__.__name__)
def version_sort(sources):
"""Sort the source list by version number."""
sources.sort(key=lambda x: Version(x["Version"]))