def __init__(self, distro, dist):
"""Constructor.
@param distro the name of a distribution, e.g. "debian", "ubuntu"
@param dist a release codename, e.g. "wheezy", "wheezy-updates"
"""
assert isinstance(distro, str), distro
assert dist is None or isinstance(dist, str), dist
super(Source, self).__init__()
self._distro = Distro.get(distro)
self._dist = dist
def main(options, args):
logger.info('Comparing target packages with source distros...')
if options.target:
targets = [options.target]
else:
targets = config.get('DISTRO_TARGETS').keys()
# Write to a new list
list_filename = patch_list_file()
tree.ensure(list_filename)
list_file = open(list_filename + ".new", "w")
try:
# For latest version of each package in the distribution, check for a patch for the
# current version; publish if it exists, clean up if not
for target in targets:
our_distro, our_dist, our_component = get_target_distro_dist_component(
target)
d = Distro.get(our_distro)
for pv in d.newestPackageVersions(our_dist, our_component):
if options.package and pv.package.name not in options.package:
continue
if not PackageLists().check_target(target, None,
pv.package.name):
continue
# Publish slipped patches in preference to true-base ones
slip_filename = patch_file(our_distro, pv, True)
filename = patch_file(our_distro, pv, False)
if os.path.isfile(slip_filename):
publish_patch(our_distro, pv, slip_filename, list_file)
elif os.path.isfile(filename):
publish_patch(our_distro, pv, filename, list_file)
else:
unpublish_patch(our_distro, pv)
finally:
list_file.close()
# Move the new list over the old one
os.rename(list_filename + ".new", list_filename)
def main(options, args):
logger.info('Comparing target packages with source distros...')
if options.target:
targets = [options.target]
else:
targets = DISTRO_TARGETS.keys()
# Write to a new list
list_filename = patch_list_file()
tree.ensure(list_filename)
list_file = open(list_filename + ".new", "w")
try:
# For latest version of each package in the distribution, check for a patch for the
# current version; publish if it exists, clean up if not
for target in targets:
our_distro, our_dist, our_component = get_target_distro_dist_component(target)
d = Distro.get(our_distro)
for source in d.newestSources(our_dist, our_component):
package = source["Package"]
if options.package and source['Package'] not in options.package:
continue
if not PACKAGELISTS.check_target(target, None, source["Package"]):
continue
# Publish slipped patches in preference to true-base ones
slip_filename = patch_file(our_distro, source, True)
filename = patch_file(our_distro, source, False)
if os.path.isfile(slip_filename):
publish_patch(our_distro, source, slip_filename, list_file)
elif os.path.isfile(filename):
publish_patch(our_distro, source, filename, list_file)
else:
unpublish_patch(our_distro, source)
finally:
list_file.close()
# Move the new list over the old one
os.rename(list_filename + ".new", list_filename)
def main(options, args):
logger.debug('Committing merges...')
for target in config.targets(args):
d = target.distro
if not isinstance(d, OBSDistro):
logger.debug('Skipping %r distro %r: not an OBSDistro', target, d)
continue
for source in d.newestSources(target.dist, target.component):
if options.package and source['Package'] not in options.package:
logger.debug('Skipping package %s: not selected', source['Package'])
continue
if source['Package'] in target.blacklist:
logger.debug('Skipping package %s: blacklisted', source['Package'])
continue
try:
output_dir = result_dir(target.name, source['Package'])
report = read_report(output_dir)
except ValueError:
logger.debug('Skipping package %s: unable to read report',
source['Package'])
continue
package = d.package(target.dist, target.component,
report.source_package)
if report['committed']:
if options.force:
logger.info("Forcing commit of %s", package)
else:
logger.debug("%s already committed, skipping!", package)
continue
if report['result'] not in (MergeResult.MERGED,
MergeResult.SYNC_THEIRS):
logger.debug("%s has nothing to commit: result=%s",
package, report['result'])
continue
filepaths = report['merged_files']
if filepaths == []:
logger.warning("Empty merged file list in %s/REPORT" % output_dir)
continue
if target.committable:
# we can commit directly to the target distribution
# FIXME: is this still a supported configuration? I wouldn't
# want to commit automated merges without some sort of manual
# check on the debdiff...
logger.info("Committing changes to %s", package)
if not options.dry_run:
try:
package.commit('Automatic update by Merge-O-Matic')
except urllib2.HTTPError as e:
logger.exception('Failed to commit %s: HTTP error %s at <%s>:',
package, e.code, e.geturl())
update_report(report, output_dir, False,
"HTTP error %s" % e.code)
except Exception as e:
logger.exception('Failed to commit %s:', package)
# deliberately rather vague, as below
update_report(report, output_dir, False,
"%s" % e.__class__.__name__)
else:
update_report(report, output_dir, True,
committed_to=d.obsProject(target.dist, target.component))
continue
# else we need to branch it and commit to the branch
try:
logger.debug("Branching %s", package)
branchPkg = package.branch("home:%s:branches"%(d.obsUser))
branch = branchPkg.distro
branch.sync(target.dist, target.component, [branchPkg,])
logger.info("Committing changes to %s, and submitting merge request to %s", branchPkg, package)
if report['result'] == MergeResult.SYNC_THEIRS:
srcDistro = Distro.get(report['right_distro'])
version = Version(report['right_version'])
logger.debug('Copying updated upstream version %s from %r into %r',
version,
srcDistro,
target)
for upstream in target.getSourceLists(package.name):
for src in upstream:
srcDistro = src.distro
try:
pkg = srcDistro.findPackage(package.name, searchDist=src.dist,
version=version)[0]
pfx = pkg.poolDirectory().path
break
except model.error.PackageNotFound:
pass
else:
logger.debug('Copying merged version from %r into %r',
branch, target)
pfx = result_dir(target.name, package.name)
# this might raise an error
obsFiles = branchPkg.getOBSFiles()
# Get the linked target files since the checkout is expanded
# and may contain them
linkedFiles = package.getOBSFiles()
for f in obsFiles:
if f.endswith(".dsc"):
oldDsc = '%s/%s'%(branchPkg.obsDir(), f)
break
for f in filepaths:
if f.endswith(".dsc"):
newDsc = '%s/%s'%(pfx, f)
break
#logger.debug("Running debdiff on %s and %s", oldDsc, newDsc)
#comment = shell.get(("debdiff", oldDsc, newDsc), okstatus=(0,1))
# FIXME: Debdiff needs implemented in OBS, as large merge descriptions break clucene.
comment = ''
if report['result'] == MergeResult.SYNC_THEIRS:
comment += 'Sync to '
elif report['result'] == MergeResult.MERGED:
comment += 'Merge with '
comment += 'version %s from %s %s' %(report['right_version'],
report['right_distro'],
report['right_suite'])
comment += "\n\nMerge report is available at %s"%('/'.join((config.get('MOM_URL'), subdir(config.get('ROOT'), output_dir), 'REPORT.html')))
# The newlines seem to cause create_submit_request to send
# UTF-32 over the wire, which OBS promptly chokes on. Encode
# the message to UTF-8 first.
comment = comment.encode('utf-8')
if not options.dry_run:
filesUpdated = False
for f in obsFiles + linkedFiles:
if f == "_link":
continue
try:
logger.debug('deleting %s/%s', branchPkg.obsDir(), f)
os.unlink('%s/%s'%(branchPkg.obsDir(), f))
filesUpdated = True
except OSError:
pass
for f in filepaths:
if f == "_link":
continue
logger.debug('copying %s/%s -> %s', pfx, f, branchPkg.obsDir())
shutil.copy2("%s/%s"%(pfx, f), branchPkg.obsDir())
filesUpdated = True
if filesUpdated:
logger.debug('Submitting request to merge %r from %r into %r',
branchPkg, branch, target)
try:
branchPkg.commit('Automatic update by Merge-O-Matic')
obs_project = d.obsProject(target.dist, target.component)
reqid = branchPkg.submitMergeRequest(obs_project, comment)
update_report(report, output_dir, True,
committed_to=obs_project,
request_url=branchPkg.webMergeRequest(reqid))
except xml.etree.cElementTree.ParseError:
logger.exception("Failed to commit %s", branchPkg)
update_report(report, output_dir, False, "OBS API Error")
except urllib2.HTTPError as e:
logger.exception("Failed to commit %s: HTTP error %s at <%s>:",
branchPkg, e.code, e.geturl())
update_report(report, output_dir, False,
"HTTP error %s" % e.code)
except Exception as e:
logger.exception("Failed to commit %s", branchPkg)
# deliberately being a bit vague here in case the exact
# exception leaks internal info
update_report(report, output_dir, False,
"%s" % e.__class__.__name__)
else:
logger.info("Not committing, due to --dry-run")
except urllib2.HTTPError as e:
logger.exception('Failed to branch %s: HTTP error %s at <%s>:',
package, e.code, e.geturl())
update_report(report, output_dir, False,
"Failed to branch: HTTP error %s" % e.code)
except Exception as e:
logger.exception('Failed to branch %s:', package)
# deliberately being a bit vague here in case the exact
# exception leaks internal info
update_report(report, output_dir, False,
"Failed to branch: %s" % e.__class__.__name__)
def distro(self):
"""Return the Distro for our "distro" configuration item."""
return Distro.get(self.config('distro'))
def main(options, args):
logger.info('Summarizing merge status...')
if options.target:
targets = [options.target]
else:
targets = DISTRO_TARGETS.keys()
outstanding = []
if os.path.isfile("%s/outstanding-merges.txt" % ROOT):
after_uvf = True
with open("%s/outstanding-merges.txt" % ROOT) as f:
for line in f:
outstanding.append(line.strip())
else:
after_uvf = False
SECTIONS.remove("new")
# For each package in the destination distribution, find out whether
# there's an open merge, and if so add an entry to the table for it.
for target in targets:
logger.info('Considering target %s', target)
our_distro, our_dist, our_component = get_target_distro_dist_component(target)
merges = []
d = Distro.get(our_distro)
for source in d.getSources(our_dist, our_component):
logger.debug('Considering package %s', source["Package"])
try:
output_dir = result_dir(target, source["Package"])
report = read_report(output_dir)
except ValueError:
continue
if report['result'] == MergeResult.KEEP_OURS:
logger.debug('Skipping merge status for %s: result=%s',
source['Package'], report['result'])
continue
try:
priority_idx = PRIORITY.index(source["Priority"])
except (KeyError, ValueError) as e:
# either it has no priority, or the priority is something
# not in our array; Debian packages can end up with
# Priority: source
priority_idx = 0
if report["committed"]:
section = "committed"
elif not after_uvf:
section = "outstanding"
elif source["Package"] in outstanding:
section = "outstanding"
else:
section = "new"
merges.append((section, priority_idx, source["Package"],
source, report["base_version"],
report["left_version"], report["right_version"],
report["right_distro"], output_dir, report))
merges.sort()
if isinstance(d, OBSDistro):
obs_project = d.obsProject(our_dist, our_component)
else:
obs_project = None
write_status_page(target, merges, our_distro, obs_project)
write_status_json(target, merges)
status_file = "%s/merges/tomerge-%s" % (ROOT, target)
remove_old_comments(status_file, merges)
write_status_file(status_file, merges)
def main(options, args):
logger.debug('Committing merges...')
for target in config.targets(args):
d = target.distro
if not isinstance(d, OBSDistro):
logger.debug('Skipping %r distro %r: not an OBSDistro', target, d)
continue
for package in d.packages(target.dist, target.component):
if options.package and package.name not in options.package:
logger.debug('Skipping package %s: not selected', package.name)
continue
if package.name in target.blacklist:
logger.debug('Skipping package %s: blacklisted', package.name)
continue
try:
output_dir = result_dir(target.name, pkg.name)
report = read_report(output_dir)
except ValueError:
logger.debug('Skipping package %s: unable to read report',
package.name)
continue
if report['committed']:
if options.force:
logger.info("Forcing commit of %s", package)
else:
logger.debug("%s already committed, skipping!", package)
continue
if report['result'] not in (MergeResult.MERGED,
MergeResult.SYNC_THEIRS):
logger.debug("%s has nothing to commit: result=%s",
package, report['result'])
continue
filepaths = report['merged_files']
if filepaths == []:
logger.warning("Empty merged file list in %s/REPORT" % output_dir)
continue
if target.committable:
# we can commit directly to the target distribution
# FIXME: is this still a supported configuration? I wouldn't
# want to commit automated merges without some sort of manual
# check on the debdiff...
logger.info("Committing changes to %s", package)
if not options.dry_run:
try:
package.commit('Automatic update by Merge-O-Matic')
except urllib2.HTTPError as e:
logger.exception('Failed to commit %s: HTTP error %s at <%s>:',
package, e.code, e.geturl())
update_report(report, output_dir, False,
"HTTP error %s" % e.code)
except Exception as e:
logger.exception('Failed to commit %s:', package)
# deliberately rather vague, as below
update_report(report, output_dir, False,
"%s" % e.__class__.__name__)
else:
update_report(report, output_dir, True,
committed_to=d.obsProject(target.dist, target.component))
continue
# else we need to branch it and commit to the branch
try:
logger.debug("Branching %s", package)
branchPkg = package.branch("home:%s:branches"%(d.obsUser))
branch = branchPkg.distro
branch.sync(target.dist, target.component, [branchPkg,])
logger.info("Committing changes to %s, and submitting merge request to %s", branchPkg, package)
if report['result'] == MergeResult.SYNC_THEIRS:
srcDistro = Distro.get(report['right_distro'])
version = Version(report['right_version'])
logger.debug('Copying updated upstream version %s from %r into %r',
version,
srcDistro,
target)
for upstream in target.getSourceLists(package.name):
for src in upstream:
srcDistro = src.distro
try:
pkg = srcDistro.findPackage(package.name, searchDist=src.dist,
version=version)[0]
pfx = pkg.poolPath
break
except model.error.PackageNotFound:
pass
else:
logger.debug('Copying merged version from %r into %r',
branch, target)
pfx = result_dir(target.name, package.name)
# this might raise an error
obsFiles = branchPkg.getOBSFiles()
# Get the linked target files since the checkout is expanded
# and may contain them
linkedFiles = package.getOBSFiles()
for f in obsFiles:
if f.endswith(".dsc"):
oldDsc = '%s/%s'%(branchPkg.obsDir(), f)
break
for f in filepaths:
if f.endswith(".dsc"):
newDsc = '%s/%s'%(pfx, f)
break
#logger.debug("Running debdiff on %s and %s", oldDsc, newDsc)
#comment = shell.get(("debdiff", oldDsc, newDsc), okstatus=(0,1))
# FIXME: Debdiff needs implemented in OBS, as large merge descriptions break clucene.
comment = ''
if report['result'] == MergeResult.SYNC_THEIRS:
comment += 'Sync to '
elif report['result'] == MergeResult.MERGED:
comment += 'Merge with '
comment += 'version %s from %s %s' %(report['right_version'],
report['right_distro'],
report['right_suite'])
comment += "\n\nMerge report is available at %s"%('/'.join((config.get('MOM_URL'), subdir(config.get('ROOT'), output_dir), 'REPORT.html')))
# The newlines seem to cause create_submit_request to send
# UTF-32 over the wire, which OBS promptly chokes on. Encode
# the message to UTF-8 first.
comment = comment.encode('utf-8')
if not options.dry_run:
filesUpdated = False
for f in obsFiles + linkedFiles:
if f == "_link":
continue
try:
logger.debug('deleting %s/%s', branchPkg.obsDir(), f)
os.unlink('%s/%s'%(branchPkg.obsDir(), f))
filesUpdated = True
except OSError:
pass
for f in filepaths:
if f == "_link":
continue
logger.debug('copying %s/%s -> %s', pfx, f, branchPkg.obsDir())
shutil.copy2("%s/%s"%(pfx, f), branchPkg.obsDir())
filesUpdated = True
if filesUpdated:
logger.debug('Submitting request to merge %r from %r into %r',
branchPkg, branch, target)
try:
branchPkg.commit('Automatic update by Merge-O-Matic')
obs_project = d.obsProject(target.dist, target.component)
reqid = branchPkg.submitMergeRequest(obs_project, comment)
update_report(report, output_dir, True,
committed_to=obs_project,
request_url=branchPkg.webMergeRequest(reqid))
except xml.etree.cElementTree.ParseError:
logger.exception("Failed to commit %s", branchPkg)
update_report(report, output_dir, False, "OBS API Error")
except urllib2.HTTPError as e:
logger.exception("Failed to commit %s: HTTP error %s at <%s>:",
branchPkg, e.code, e.geturl())
update_report(report, output_dir, False,
"HTTP error %s" % e.code)
except Exception as e:
logger.exception("Failed to commit %s", branchPkg)
# deliberately being a bit vague here in case the exact
# exception leaks internal info
update_report(report, output_dir, False,
"%s" % e.__class__.__name__)
else:
logger.info("Not committing, due to --dry-run")
except urllib2.HTTPError as e:
logger.exception('Failed to branch %s: HTTP error %s at <%s>:',
package, e.code, e.geturl())
update_report(report, output_dir, False,
"Failed to branch: HTTP error %s" % e.code)
except Exception as e:
logger.exception('Failed to branch %s:', package)
# deliberately being a bit vague here in case the exact
# exception leaks internal info
update_report(report, output_dir, False,
"Failed to branch: %s" % e.__class__.__name__)
def main(options, args):
logger.info('Summarizing merge status...')
if options.target:
targets = [options.target]
else:
targets = config.get('DISTRO_TARGETS').keys()
outstanding = []
if os.path.isfile("%s/outstanding-merges.txt" % config.get('ROOT')):
after_uvf = True
with open("%s/outstanding-merges.txt" % config.get('ROOT')) as f:
for line in f:
outstanding.append(line.strip())
else:
after_uvf = False
SECTIONS.remove("new")
# For each package in the destination distribution, find out whether
# there's an open merge, and if so add an entry to the table for it.
for target in targets:
logger.info('Considering target %s', target)
our_distro, our_dist, our_component = get_target_distro_dist_component(
target)
merges = []
d = Distro.get(our_distro)
for source in d.getSources(our_dist, our_component):
logger.debug('Considering package %s', source["Package"])
try:
output_dir = result_dir(target, source["Package"])
report = read_report(output_dir)
except ValueError:
continue
if report['result'] == MergeResult.KEEP_OURS:
logger.debug('Skipping merge status for %s: result=%s',
source['Package'], report['result'])
continue
try:
priority_idx = PRIORITY.index(source["Priority"])
except (KeyError, ValueError) as e:
# either it has no priority, or the priority is something
# not in our array; Debian packages can end up with
# Priority: source
priority_idx = 0
if report["committed"]:
section = "committed"
elif not after_uvf:
section = "outstanding"
elif source["Package"] in outstanding:
section = "outstanding"
else:
section = "new"
merges.append((section, priority_idx, source["Package"], source,
report["base_version"], report["left_version"],
report["right_version"], report["right_distro"],
output_dir, report))
merges.sort()
if isinstance(d, OBSDistro):
obs_project = d.obsProject(our_dist, our_component)
else:
obs_project = None
write_status_page(target, merges, our_distro, obs_project)
write_status_json(target, merges)
status_file = "%s/merges/tomerge-%s" % (config.get('ROOT'), target)
remove_old_comments(status_file, merges)
write_status_file(status_file, merges)