def _set_options(self, auto_mode, send_email, skip_compilation):
self.opts = {}
self.opts['layer_mode'] = settings.get('layer_mode', '')
if self.opts['layer_mode'] == 'yes':
def _layer_settings_error(setting):
E(" In layer mode enable you need to specify %s.\n" % setting)
exit(1)
layer_settings = ('layer_name', 'layer_dir', 'layer_machines')
for s in layer_settings:
self.opts[s] = settings.get(s, '')
if not self.opts[s]:
_layer_settings_error(s)
self.git = Git(self.opts['layer_dir'])
self.poky_git = Git(os.path.dirname(os.getenv('PATH', False).split(':')[0]))
self.opts['machines'] = self.opts['layer_machines'].split()
else:
# XXX: assume that the poky directory is the first entry in the PATH
self.git = Git(os.path.dirname(os.getenv('PATH', False).split(':')[0]))
self.poky_git = None
self.opts['machines'] = settings.get('machines',
'qemux86 qemux86-64 qemuarm qemumips qemuppc').split()
self.opts['interactive'] = not auto_mode
self.opts['send_email'] = send_email
self.opts['author'] = "Upgrade Helper <%s>" % \
settings.get('from', '*****@*****.**')
self.opts['skip_compilation'] = skip_compilation
self.opts['buildhistory'] = self._buildhistory_is_enabled()
self.opts['testimage'] = self._testimage_is_enabled()
def _set_options(self, auto_mode, send_email, skip_compilation):
self.opts = {}
self.opts['layer_mode'] = settings.get('layer_mode', '')
if self.opts['layer_mode'] == 'yes':
def _layer_settings_error(setting):
E(" In layer mode enable you need to specify %s.\n" % setting)
exit(1)
layer_settings = ('layer_name', 'layer_dir', 'layer_machines')
for s in layer_settings:
self.opts[s] = settings.get(s, '')
if not self.opts[s]:
_layer_settings_error(s)
self.git = Git(self.opts['layer_dir'])
self.poky_git = Git(
os.path.dirname(os.getenv('PATH', False).split(':')[0]))
self.opts['machines'] = self.opts['layer_machines'].split()
else:
# XXX: assume that the poky directory is the first entry in the PATH
self.git = Git(
os.path.dirname(os.getenv('PATH', False).split(':')[0]))
self.poky_git = None
self.opts['machines'] = settings.get(
'machines',
'qemux86 qemux86-64 qemuarm qemumips qemuppc').split()
self.opts['interactive'] = not auto_mode
self.opts['send_email'] = send_email
self.opts['author'] = "Upgrade Helper <%s>" % \
settings.get('from', '*****@*****.**')
self.opts['skip_compilation'] = skip_compilation
self.opts['buildhistory'] = self._buildhistory_is_enabled()
self.opts['testimage'] = self._testimage_is_enabled()
def save(name, path, existing_issues):
tags = fetch(path)
artifacts = parse(tags)
print("Tags", tags)
issues = {}
for key, values in artifacts.items():
title, body = create_issue(name, key, values[::-1])
if key in existing_issues:
issues[key] = Git.update_issue(existing_issues[key], title, body)
else:
issues[key] = Git.create_issue(title, body)
sleep(0.34)
return issues
class BuildHistory(object):
def __init__(self, bb, pn, workdir):
self.bb = bb
self.pn = pn
self.workdir = workdir
self.revs = []
self.buildhistory_dir = os.path.join(self.workdir, 'buildhistory')
if not os.path.exists(self.buildhistory_dir):
os.mkdir(self.buildhistory_dir)
self.git = Git(self.buildhistory_dir)
os.environ['BB_ENV_EXTRAWHITE'] = os.environ['BB_ENV_EXTRAWHITE'] + \
" BUILDHISTORY_DIR"
os.environ["BUILDHISTORY_DIR"] = self.buildhistory_dir
def init(self, machines):
self.bb.cleansstate(self.pn)
for machine in machines:
self.bb.complete(self.pn, machine)
self.revs.append(self.git.last_commit("master"))
def add(self):
self.revs.append(self.git.last_commit("master"))
def diff(self):
rev_initial = self.revs[0]
rev_final = self.revs[-1]
try:
cmd = "buildhistory-diff -p %s %s %s" % (self.buildhistory_dir,
rev_initial, rev_final)
stdout, stderr = bb.process.run(cmd)
if stdout and os.path.exists(self.workdir):
with open(os.path.join(self.workdir, "buildhistory-diff.txt"),
"w+") as log:
log.write(stdout)
cmd_full = "buildhistory-diff -a -p %s %s %s" % (
self.buildhistory_dir, rev_initial, rev_final)
stdout, stderr = bb.process.run(cmd_full)
if stdout and os.path.exists(self.workdir):
with open(
os.path.join(self.workdir,
"buildhistory-diff-full.txt"),
"w+") as log:
log.write(stdout)
except bb.process.ExecutionError as e:
W("%s: Buildhistory checking fails\n%s" % (self.pn, e.stdout))
def __init__(self, bb, pn, workdir):
self.bb = bb
self.pn = pn
self.workdir = workdir
self.revs = []
self.buildhistory_dir = os.path.join(self.workdir, 'buildhistory')
if not os.path.exists(self.buildhistory_dir):
os.mkdir(self.buildhistory_dir)
self.git = Git(self.buildhistory_dir)
os.environ['BB_ENV_EXTRAWHITE'] = os.environ['BB_ENV_EXTRAWHITE'] + \
" BUILDHISTORY_DIR"
os.environ["BUILDHISTORY_DIR"] = self.buildhistory_dir
class BuildHistory(object):
def __init__(self, bb, pn, workdir):
self.bb = bb
self.pn = pn
self.workdir = workdir
self.revs = []
self.buildhistory_dir = os.path.join(self.workdir, 'buildhistory')
if not os.path.exists(self.buildhistory_dir):
os.mkdir(self.buildhistory_dir)
self.git = Git(self.buildhistory_dir)
os.environ['BB_ENV_EXTRAWHITE'] = os.environ['BB_ENV_EXTRAWHITE'] + \
" BUILDHISTORY_DIR"
os.environ["BUILDHISTORY_DIR"] = self.buildhistory_dir
def init(self, machines):
self.bb.cleanall(self.pn)
for machine in machines:
self.bb.complete(self.pn, machine)
self.revs.append(self.git.last_commit("master"))
def add(self):
self.revs.append(self.git.last_commit("master"))
def diff(self):
rev_initial = self.revs[0]
rev_final = self.revs[-1]
try:
cmd = "buildhistory-diff -p %s %s %s" % (self.buildhistory_dir,
rev_initial, rev_final)
stdout, stderr = bb.process.run(cmd)
if stdout and os.path.exists(self.workdir):
with open(os.path.join(self.workdir, "buildhistory-diff.txt"),
"w+") as log:
log.write(stdout)
cmd_full = "buildhistory-diff -a -p %s %s %s" % (self.buildhistory_dir,
rev_initial, rev_final)
stdout, stderr = bb.process.run(cmd_full)
if stdout and os.path.exists(self.workdir):
with open(os.path.join(self.workdir, "buildhistory-diff-full.txt"),
"w+") as log:
log.write(stdout)
except bb.process.ExecutionError as e:
W( "%s: Buildhistory checking fails\n%s" % (self.pn, e.stdout))
def __init__(self, bb, pn, workdir):
self.bb = bb
self.pn = pn
self.workdir = workdir
self.revs = []
self.buildhistory_dir = os.path.join(self.workdir, 'buildhistory')
if not os.path.exists(self.buildhistory_dir):
os.mkdir(self.buildhistory_dir)
self.git = Git(self.buildhistory_dir)
os.environ['BB_ENV_EXTRAWHITE'] = os.environ['BB_ENV_EXTRAWHITE'] + \
" BUILDHISTORY_DIR"
os.environ["BUILDHISTORY_DIR"] = self.buildhistory_dir
class Updater(object):
def __init__(self,
auto_mode=False,
send_email=False,
skip_compilation=False):
build_dir = get_build_dir()
self.bb = Bitbake(build_dir)
try:
self.base_env = self.bb.env()
except EmptyEnvError as e:
import traceback
E(" %s\n%s" % (e.message, traceback.format_exc()))
E(" Bitbake output:\n%s" % (e.stdout))
exit(1)
self._set_options(auto_mode, send_email, skip_compilation)
self._make_dirs(build_dir)
self._add_file_logger()
self.email_handler = Email(settings)
self.statistics = Statistics()
def _set_options(self, auto_mode, send_email, skip_compilation):
self.opts = {}
self.opts['layer_mode'] = settings.get('layer_mode', '')
if self.opts['layer_mode'] == 'yes':
def _layer_settings_error(setting):
E(" In layer mode enable you need to specify %s.\n" % setting)
exit(1)
layer_settings = ('layer_name', 'layer_dir', 'layer_machines')
for s in layer_settings:
self.opts[s] = settings.get(s, '')
if not self.opts[s]:
_layer_settings_error(s)
self.git = Git(self.opts['layer_dir'])
self.poky_git = Git(
os.path.dirname(os.getenv('PATH', False).split(':')[0]))
self.opts['machines'] = self.opts['layer_machines'].split()
else:
# XXX: assume that the poky directory is the first entry in the PATH
self.git = Git(
os.path.dirname(os.getenv('PATH', False).split(':')[0]))
self.poky_git = None
self.opts['machines'] = settings.get(
'machines',
'qemux86 qemux86-64 qemuarm qemumips qemuppc').split()
self.opts['interactive'] = not auto_mode
self.opts['send_email'] = send_email
self.opts['author'] = "Upgrade Helper <%s>" % \
settings.get('from', '*****@*****.**')
self.opts['skip_compilation'] = skip_compilation
self.opts['buildhistory'] = self._buildhistory_is_enabled()
self.opts['testimage'] = self._testimage_is_enabled()
def _make_dirs(self, build_dir):
self.uh_dir = os.path.join(build_dir, "upgrade-helper")
if not os.path.exists(self.uh_dir):
os.mkdir(self.uh_dir)
self.uh_base_work_dir = settings.get('workdir', '')
if not self.uh_base_work_dir:
self.uh_base_work_dir = self.uh_dir
if self.opts['layer_mode'] == 'yes':
self.uh_base_work_dir = os.path.join(self.uh_base_work_dir,
self.opts['layer_name'])
if not os.path.exists(self.uh_base_work_dir):
os.mkdir(self.uh_base_work_dir)
self.uh_work_dir = os.path.join(self.uh_base_work_dir, "%s" % \
datetime.now().strftime("%Y%m%d%H%M%S"))
os.mkdir(self.uh_work_dir)
self.uh_recipes_all_dir = os.path.join(self.uh_work_dir, "all")
os.mkdir(self.uh_recipes_all_dir)
self.uh_recipes_succeed_dir = os.path.join(self.uh_work_dir, "succeed")
os.mkdir(self.uh_recipes_succeed_dir)
self.uh_recipes_failed_dir = os.path.join(self.uh_work_dir, "failed")
os.mkdir(self.uh_recipes_failed_dir)
def _add_file_logger(self):
fh = log.FileHandler(
os.path.join(self.uh_work_dir, "upgrade-helper.log"))
logger = log.getLogger()
logger.addHandler(fh)
def _get_status_msg(self, err):
if err:
return str(err)
else:
return "Succeeded"
def _buildhistory_is_enabled(self):
enabled = False
if settings.get("buildhistory", "no") == "yes":
if 'buildhistory' in self.base_env['INHERIT']:
if not 'BUILDHISTORY_COMMIT' in self.base_env:
E(" Buildhistory was INHERIT in conf/local.conf"\
" but need BUILDHISTORY_COMMIT=1 please set.")
exit(1)
if not self.base_env['BUILDHISTORY_COMMIT'] == '1':
E(" Buildhistory was INHERIT in conf/local.conf"\
" but need BUILDHISTORY_COMMIT=1 please set.")
exit(1)
if self.opts['skip_compilation']:
W(" Buildhistory disabled because user" \
" skip compilation!")
else:
enabled = True
else:
E(" Buildhistory was enabled in upgrade-helper.conf"\
" but isn't INHERIT in conf/local.conf, if you want"\
" to enable please set.")
exit(1)
else:
if 'buildhistory' in self.base_env['INHERIT']:
E(" Buildhistory was INHERIT in conf/local.conf"\
" but buildhistory=yes isn't in upgrade-helper.conf,"\
" if you want to enable please set.")
exit(1)
return enabled
def _testimage_is_enabled(self):
enabled = False
if settings.get("testimage", "no") == "yes":
if 'testimage' in self.base_env['INHERIT']:
if not "ptest" in self.base_env["DISTRO_FEATURES"]:
E(" testimage requires ptest in DISTRO_FEATURES please add to"\
" conf/local.conf.")
exit(1)
if not "package-management" in self.base_env[
'EXTRA_IMAGE_FEATURES']:
E(" testimage requires package-management in EXTRA_IMAGE_FEATURES"\
" please add to conf/local.conf.")
exit(1)
if not "ptest-pkgs" in self.base_env['EXTRA_IMAGE_FEATURES']:
E(" testimage/ptest requires ptest-pkgs in EXTRA_IMAGE_FEATURES"\
" please add to conf/local.conf.")
exit(1)
if not "package_rpm" == self.base_env["PACKAGE_CLASSES"]:
E(" testimage/ptest requires PACKAGE_CLASSES set to package_rpm"\
" please add to conf/local.conf.")
exit(1)
enabled = True
else:
E(" testimage was enabled in upgrade-helper.conf"\
" but isn't INHERIT in conf/local.conf, if you want"\
" to enable please set.")
exit(1)
else:
if 'testimage' in self.base_env['INHERIT']:
E(" testimage was INHERIT in conf/local.conf"\
" but testimage=yes isn't in upgrade-helper.conf,"\
" if you want to enable please set.")
exit(1)
return enabled
def _get_packages_to_upgrade(self, packages=None):
if packages is None:
I("Nothing to upgrade")
exit(0)
else:
return packages
# this function will be called at the end of each recipe upgrade
def pkg_upgrade_handler(self, pkg_ctx):
mail_header = \
"Hello,\n\nYou are receiving this email because you are the maintainer\n" \
"of *%s* recipe and this is to let you know that the automatic attempt\n" \
"to upgrade the recipe to *%s* has %s.\n\n"
license_change_info = \
"*LICENSE CHANGED* please review the %s file and update the LICENSE\n" \
"variable in the recipe if is needed.\n\n"
next_steps_info = \
"The recipe has been successfully compiled for machines %s.\n\n" \
"Next steps:\n" \
" - apply the patch: git am %s\n" \
" - check that required upstream patches have not been commented from the recipe,\n" \
" if upstream patches were commented the reason is specified in the commit message.\n" \
" - compile an image that contains the package\n" \
" - perform some basic sanity tests\n" \
" - amend the patch and sign it off: git commit -s --reset-author --amend\n" \
" - send it to the list\n\n" \
testimage_integration_error = \
"The recipe *FAILED* in testimage integration. Attached is the log file.\n\n"
testimage_ptest_info = \
"The recipe has ptest enabled and has been tested with core-image-minimal/ptest \n" \
"with the next machines %s. Attached is the log file.\n\n"
testimage_info = \
"The recipe has been tested using %s testimage and succeeded with \n" \
"the next machines %s. Attached is the log file.\n\n" \
mail_footer = \
"Attached are the patch, license diff (if change) and bitbake log.\n" \
"Any problem please contact Anibal Limon <*****@*****.**>.\n\n" \
"Regards,\nThe Upgrade Helper"
if pkg_ctx['MAINTAINER'] in maintainer_override:
to_addr = maintainer_override[pkg_ctx['MAINTAINER']]
else:
to_addr = pkg_ctx['MAINTAINER']
cc_addr = None
if "status_recipients" in settings:
cc_addr = settings["status_recipients"].split()
subject = "[AUH] " + pkg_ctx['PN'] + ": upgrading to " + pkg_ctx['NPV']
if not pkg_ctx['error']:
subject += " SUCCEEDED"
else:
subject += " FAILED"
msg_body = mail_header % (pkg_ctx['PN'], pkg_ctx['NPV'],
self._get_status_msg(pkg_ctx['error']))
if 'recipe' in pkg_ctx:
license_diff_fn = pkg_ctx['recipe'].get_license_diff_file_name()
if license_diff_fn:
msg_body += license_change_info % license_diff_fn
if not pkg_ctx['error']:
msg_body += next_steps_info % (', '.join(
self.opts['machines']), os.path.basename(
pkg_ctx['patch_file']))
if self.opts['testimage']:
if 'integration_error' in pkg_ctx:
msg_body += testimage_integration_error
else:
if 'ptest' in pkg_ctx:
machines = pkg_ctx['ptest'].keys()
msg_body += testimage_ptest_info % machines
if 'testimage' in pkg_ctx:
machines = pkg_ctx['testimage'].keys()
msg_body += testimage_info % (settings.get('testimage_name', \
DEFAULT_TESTIMAGE), machines)
msg_body += mail_footer
# Add possible attachments to email
attachments = []
for attachment in os.listdir(pkg_ctx['workdir']):
attachment_fullpath = os.path.join(pkg_ctx['workdir'], attachment)
if os.path.isfile(attachment_fullpath):
attachments.append(attachment_fullpath)
# Only send email to Maintainer when recipe upgrade succeed.
if self.opts['send_email'] and not pkg_ctx['error']:
self.email_handler.send_email(to_addr,
subject,
msg_body,
attachments,
cc_addr=cc_addr)
# Preserve email for review purposes.
email_file = os.path.join(pkg_ctx['workdir'], "email_summary")
with open(email_file, "w+") as f:
f.write("To: %s\n" % to_addr)
if isinstance(cc_addr, list):
f.write("To: %s\n" % ' '.join(cc_addr))
else:
f.write("Cc: %s\n" % cc_addr)
f.write("Subject: %s\n" % subject)
f.write("Attachments: %s\n" % ' '.join(attachments))
f.write("\n%s\n" % msg_body)
def commit_changes(self, pkg_ctx):
fail = False
try:
pkg_ctx['patch_file'] = None
if 'recipe' in pkg_ctx:
I(" %s: Auto commit changes ..." % pkg_ctx['PN'])
self.git.commit(pkg_ctx['recipe'].commit_msg,
self.opts['author'])
stdout = self.git.create_patch(pkg_ctx['workdir'])
pkg_ctx['patch_file'] = stdout.strip()
if not pkg_ctx['patch_file']:
msg = "Patch file not generated."
E(" %s: %s\n %s" % (pkg_ctx['PN'], msg, stdout))
pkg_ctx['error'] = Error(msg, stdout)
fail = True
else:
I(" %s: Save patch in directory: %s." %
(pkg_ctx['PN'], pkg_ctx['workdir']))
except Error as e:
msg = ''
for line in e.stdout.split("\n"):
if line.find("nothing to commit") == 0:
msg = "Nothing to commit!"
I(" %s: %s" % (pkg_ctx['PN'], msg))
I(" %s: %s" % (pkg_ctx['PN'], e.stdout))
pkg_ctx['error'] = Error(msg, e.stdout)
fail = True
if fail:
raise pkg_ctx['error']
def send_status_mail(self, statistics_summary):
if "status_recipients" not in settings:
E(" Could not send status email, no recipients set!")
return -1
to_list = settings["status_recipients"].split()
if self.opts['layer_mode'] == 'yes':
subject = "[AUH] Upgrade status %s: %s" \
% (self.opts['layer_name'], date.isoformat(date.today()))
else:
subject = "[AUH] Upgrade status: " + date.isoformat(date.today())
if self.statistics.total_attempted:
self.email_handler.send_email(to_list, subject, statistics_summary)
else:
W("No recipes attempted, not sending status mail!")
def _order_pkgs_to_upgrade(self, pkgs_to_upgrade):
def _get_pn_dep_dic(pn_list, dependency_file):
import re
pn_dep_dic = {}
with open(dependency_file) as dep:
data = dep.read()
dep.close()
for line in data.split('\n'):
m = re.search('^"(.*)" -> "(.*)"$', line)
if not m:
continue
pn = m.group(1)
pn_dep = m.group(2)
if pn == pn_dep:
continue
if pn in pn_list:
if pn_dep in pn_list:
if pn in pn_dep_dic.keys():
pn_dep_dic[pn].append(pn_dep)
else:
pn_dep_dic[pn] = [pn_dep]
elif not pn in pn_dep_dic.keys():
pn_dep_dic[pn] = []
return pn_dep_dic
def _dep_resolve(graph, node, resolved, seen):
seen.append(node)
for edge in graph[node]:
if edge not in resolved:
if edge in seen:
raise RuntimeError("Packages %s and %s have " \
"a circular dependency." \
% (node, edge))
_dep_resolve(graph, edge, resolved, seen)
resolved.append(node)
pn_list = []
for pn, new_ver, maintainer in pkgs_to_upgrade:
pn_list.append(pn)
try:
self.bb.dependency_graph(' '.join(pn_list))
except Error as e:
multiple_providers = False
for l in e.stdout.split('\n'):
if l.find(
"ERROR: Multiple .bb files are due to be built which each provide"
) == 0:
multiple_providers = True
if not multiple_providers:
raise e
dependency_file = os.path.join(get_build_dir(), "pn-depends.dot")
pkgs_to_upgrade_ordered = []
pn_list_ordered = []
pn_dep_dic = _get_pn_dep_dic(pn_list, dependency_file)
if pn_dep_dic:
root = "__root_node__"
pn_dep_dic[root] = pn_dep_dic.keys()
_dep_resolve(pn_dep_dic, root, pn_list_ordered, [])
pn_list_ordered.remove(root)
for pn_ordered in pn_list_ordered:
for pn, new_ver, maintainer in pkgs_to_upgrade:
if pn == pn_ordered:
pkgs_to_upgrade_ordered.append([pn, new_ver, maintainer])
return pkgs_to_upgrade_ordered
def run(self, package_list=None):
#pkgs_to_upgrade = self._order_pkgs_to_upgrade(
# self._get_packages_to_upgrade(package_list))
pkgs_to_upgrade = self._get_packages_to_upgrade(package_list)
total_pkgs = len(pkgs_to_upgrade)
pkgs_ctx = {}
I(" ########### The list of recipes to be upgraded #############")
for p, v, m in pkgs_to_upgrade:
I(" %s, %s, %s" % (p, v, m))
pkgs_ctx[p] = {}
pkgs_ctx[p]['PN'] = p
pkgs_ctx[p]['NPV'] = v
pkgs_ctx[p]['MAINTAINER'] = m
pkgs_ctx[p]['base_dir'] = self.uh_recipes_all_dir
I(" ############################################################")
if pkgs_to_upgrade:
I(" Building gcc runtimes ...")
for machine in self.opts['machines']:
I(" building gcc runtime for %s" % machine)
try:
self.bb.complete("gcc-runtime", machine)
except Exception as e:
E(" Can't build gcc-runtime for %s." % machine)
if isinstance(e, Error):
E(e.stdout)
else:
import traceback
traceback.print_exc(file=sys.stdout)
succeeded_pkgs_ctx = []
failed_pkgs_ctx = []
attempted_pkgs = 0
for pn, _, _ in pkgs_to_upgrade:
pkg_ctx = pkgs_ctx[pn]
pkg_ctx['error'] = None
attempted_pkgs += 1
I(" ATTEMPT PACKAGE %d/%d" % (attempted_pkgs, total_pkgs))
try:
I(" %s: Upgrading to %s" % (pkg_ctx['PN'], pkg_ctx['NPV']))
for step, msg in upgrade_steps:
if msg is not None:
I(" %s: %s" % (pkg_ctx['PN'], msg))
step(self.bb, self.git, self.opts, pkg_ctx)
succeeded_pkgs_ctx.append(pkg_ctx)
I(" %s: Upgrade SUCCESSFUL! Please test!" % pkg_ctx['PN'])
except Exception as e:
if isinstance(e, UpgradeNotNeededError):
I(" %s: %s" % (pkg_ctx['PN'], e.message))
elif isinstance(e, UnsupportedProtocolError):
I(" %s: %s" % (pkg_ctx['PN'], e.message))
else:
if not isinstance(e, Error):
import traceback
msg = "Failed(unknown error)\n" + traceback.format_exc(
)
e = Error(message=msg)
error = e
E(" %s: %s" % (pkg_ctx['PN'], e.message))
if os.listdir(pkg_ctx['workdir']):
E(" %s: Upgrade FAILED! Logs and/or file diffs are available in %s"
% (pkg_ctx['PN'], pkg_ctx['workdir']))
pkg_ctx['error'] = e
failed_pkgs_ctx.append(pkg_ctx)
try:
self.commit_changes(pkg_ctx)
except:
if pkg_ctx in succeeded_pkgs_ctx:
succeeded_pkgs_ctx.remove(pkg_ctx)
failed_pkgs_ctx.append(pkg_ctx)
if self.opts['testimage']:
ctxs = {}
ctxs['succeeded'] = succeeded_pkgs_ctx
ctxs['failed'] = failed_pkgs_ctx
image = settings.get('testimage_name', DEFAULT_TESTIMAGE)
tim = TestImage(self.bb, self.git, self.uh_work_dir, self.opts,
ctxs, image)
tim.run()
for pn in pkgs_ctx.keys():
pkg_ctx = pkgs_ctx[pn]
if pkg_ctx in succeeded_pkgs_ctx:
os.symlink(pkg_ctx['workdir'], os.path.join( \
self.uh_recipes_succeed_dir, pkg_ctx['PN']))
else:
os.symlink(pkg_ctx['workdir'], os.path.join( \
self.uh_recipes_failed_dir, pkg_ctx['PN']))
self.statistics.update(pkg_ctx['PN'], pkg_ctx['NPV'],
pkg_ctx['MAINTAINER'], pkg_ctx['error'])
self.pkg_upgrade_handler(pkg_ctx)
if attempted_pkgs > 0:
publish_work_url = settings.get('publish_work_url', '')
work_tarball = os.path.join(
self.uh_base_work_dir,
os.path.basename(self.uh_work_dir) + '.tar.gz')
if publish_work_url:
I(" Generating work tarball in %s ..." % work_tarball)
import subprocess
if subprocess.call(
["tar", "-chzf", work_tarball, self.uh_work_dir]):
E(" Work tarball (%s) generation failed..." %
(work_tarball))
publish_work_url = ''
statistics_summary = self.statistics.get_summary(
publish_work_url, os.path.basename(self.uh_work_dir))
statistics_file = os.path.join(self.uh_work_dir,
"statistics_summary")
with open(statistics_file, "w+") as f:
f.write(statistics_summary)
I(" %s" % statistics_summary)
if self.opts['send_email']:
self.send_status_mail(statistics_summary)
class Updater(object):
def __init__(self,
auto_mode=False,
send_email=False,
skip_compilation=False):
build_dir = get_build_dir()
self.bb = Bitbake(build_dir)
self.devtool = Devtool()
try:
self.base_env = self.bb.env()
except EmptyEnvError as e:
import traceback
E(" %s\n%s" % (e.message, traceback.format_exc()))
E(" Bitbake output:\n%s" % (e.stdout))
exit(1)
self._set_options(auto_mode, send_email, skip_compilation)
self._make_dirs(build_dir)
self._add_file_logger()
self.email_handler = Email(settings)
self.statistics = Statistics()
def _set_options(self, auto_mode, send_email, skip_compilation):
self.opts = {}
self.opts['layer_mode'] = settings.get('layer_mode', '')
if self.opts['layer_mode'] == 'yes':
def _layer_settings_error(setting):
E(" In layer mode enable you need to specify %s.\n" % setting)
exit(1)
layer_settings = ('layer_name', 'layer_dir', 'layer_machines')
for s in layer_settings:
self.opts[s] = settings.get(s, '')
if not self.opts[s]:
_layer_settings_error(s)
self.git = Git(self.opts['layer_dir'])
self.poky_git = Git(
os.path.dirname(os.getenv('PATH', False).split(':')[0]))
self.opts['machines'] = self.opts['layer_machines'].split()
else:
# XXX: assume that the poky directory is the first entry in the PATH
self.git = Git(
os.path.dirname(os.getenv('PATH', False).split(':')[0]))
self.poky_git = None
self.opts['machines'] = settings.get(
'machines',
'qemux86 qemux86-64 qemuarm qemumips qemuppc').split()
self.opts['interactive'] = not auto_mode
self.opts['send_email'] = send_email
self.opts['author'] = "Upgrade Helper <%s>" % \
settings.get('from', '*****@*****.**')
self.opts['skip_compilation'] = skip_compilation
self.opts['buildhistory'] = self._buildhistory_is_enabled()
self.opts['testimage'] = self._testimage_is_enabled()
def _make_dirs(self, build_dir):
self.uh_dir = os.path.join(build_dir, "upgrade-helper")
if not os.path.exists(self.uh_dir):
os.mkdir(self.uh_dir)
self.uh_base_work_dir = settings.get('workdir', '')
if not self.uh_base_work_dir:
self.uh_base_work_dir = self.uh_dir
if self.opts['layer_mode'] == 'yes':
self.uh_base_work_dir = os.path.join(self.uh_base_work_dir,
self.opts['layer_name'])
if not os.path.exists(self.uh_base_work_dir):
os.mkdir(self.uh_base_work_dir)
self.uh_work_dir = os.path.join(self.uh_base_work_dir, "%s" % \
datetime.now().strftime("%Y%m%d%H%M%S"))
os.mkdir(self.uh_work_dir)
self.uh_recipes_all_dir = os.path.join(self.uh_work_dir, "all")
os.mkdir(self.uh_recipes_all_dir)
self.uh_recipes_succeed_dir = os.path.join(self.uh_work_dir, "succeed")
os.mkdir(self.uh_recipes_succeed_dir)
self.uh_recipes_failed_dir = os.path.join(self.uh_work_dir, "failed")
os.mkdir(self.uh_recipes_failed_dir)
def _add_file_logger(self):
fh = log.FileHandler(
os.path.join(self.uh_work_dir, "upgrade-helper.log"))
logger = log.getLogger()
logger.addHandler(fh)
def _get_status_msg(self, err):
if err:
return str(err)
else:
return "Succeeded"
def _buildhistory_is_enabled(self):
enabled = False
if settings.get("buildhistory", "no") == "yes":
if 'buildhistory' in self.base_env['INHERIT']:
if not 'BUILDHISTORY_COMMIT' in self.base_env:
E(" Buildhistory was INHERIT in conf/local.conf"\
" but need BUILDHISTORY_COMMIT=1 please set.")
exit(1)
if not self.base_env['BUILDHISTORY_COMMIT'] == '1':
E(" Buildhistory was INHERIT in conf/local.conf"\
" but need BUILDHISTORY_COMMIT=1 please set.")
exit(1)
if self.opts['skip_compilation']:
W(" Buildhistory disabled because user" \
" skip compilation!")
else:
enabled = True
else:
E(" Buildhistory was enabled in upgrade-helper.conf"\
" but isn't INHERIT in conf/local.conf, if you want"\
" to enable please set.")
exit(1)
return enabled
def _testimage_is_enabled(self):
enabled = False
if settings.get("testimage", "no") == "yes":
if 'testimage' in self.base_env['INHERIT']:
if not "ptest" in self.base_env["DISTRO_FEATURES"]:
E(" testimage requires ptest in DISTRO_FEATURES please add to"\
" conf/local.conf.")
exit(1)
if not "package-management" in self.base_env[
'EXTRA_IMAGE_FEATURES']:
E(" testimage requires package-management in EXTRA_IMAGE_FEATURES"\
" please add to conf/local.conf.")
exit(1)
if not "ptest-pkgs" in self.base_env['EXTRA_IMAGE_FEATURES']:
E(" testimage/ptest requires ptest-pkgs in EXTRA_IMAGE_FEATURES"\
" please add to conf/local.conf.")
exit(1)
if not "package_rpm" == self.base_env["PACKAGE_CLASSES"]:
E(" testimage/ptest requires PACKAGE_CLASSES set to package_rpm"\
" please add to conf/local.conf.")
exit(1)
enabled = True
else:
E(" testimage was enabled in upgrade-helper.conf"\
" but isn't INHERIT in conf/local.conf, if you want"\
" to enable please set.")
exit(1)
return enabled
def _get_packages_to_upgrade(self, packages=None):
if packages is None:
I("Nothing to upgrade")
exit(0)
else:
return packages
# this function will be called at the end of each recipe upgrade
def pkg_upgrade_handler(self, pkg_ctx):
mail_header = \
"Hello,\n\nYou are receiving this email because you are the maintainer\n" \
"of *%s* recipe and this is to let you know that the automatic attempt\n" \
"to upgrade the recipe to *%s* has %s.\n\n"
license_change_info = \
"*LICENSE CHANGED* please review the %s file, update the LICENSE\n" \
"variable in the recipe and summarize the changes in the commit message.\n\n"
next_steps_info = \
"Next steps:\n" \
" - apply the patch: git am %s\n" \
" - check the changes to upstream patches and summarize them in the commit message,\n" \
" - compile an image that contains the package\n" \
" - perform some basic sanity tests\n" \
" - amend the patch and sign it off: git commit -s --reset-author --amend\n" \
" - send it to the appropriate mailing list\n\n" \
"Alternatively, if you believe the recipe should not be upgraded at this time,\n" \
"you can fill RECIPE_NO_UPDATE_REASON in respective recipe file so that\n" \
"automatic upgrades would no longer be attempted.\n\n"
testimage_integration_error = \
"The recipe *FAILED* in testimage integration. Attached is the log file.\n\n"
testimage_ptest_info = \
"The recipe has ptest enabled and has been tested with core-image-minimal/ptest \n" \
"with the next machines %s. Attached is the log file.\n\n"
testimage_info = \
"The recipe has been tested using %s testimage and succeeded with \n" \
"the next machines %s. Attached is the log file.\n\n" \
mail_footer = \
"Please review the attached files for further information and build/update failures.\n" \
"Any problem please file a bug at https://bugzilla.yoctoproject.org/enter_bug.cgi?product=Automated%20Update%20Handler\n\n" \
"Regards,\nThe Upgrade Helper"
if pkg_ctx['MAINTAINER'] in maintainer_override:
to_addr = maintainer_override[pkg_ctx['MAINTAINER']]
else:
to_addr = pkg_ctx['MAINTAINER']
cc_addr = None
if "status_recipients" in settings:
cc_addr = settings["status_recipients"].split()
newversion = pkg_ctx['NPV'] if not pkg_ctx['NPV'].endswith(
"new-commits-available") else pkg_ctx['NSRCREV']
subject = "[AUH] " + pkg_ctx['PN'] + ": upgrading to " + newversion
if not pkg_ctx['error']:
subject += " SUCCEEDED"
else:
subject += " FAILED"
msg_body = mail_header % (pkg_ctx['PN'], newversion,
self._get_status_msg(pkg_ctx['error']))
if pkg_ctx['error'] is not None:
msg_body += """Detailed error information:
%s
%s
%s
""" % (pkg_ctx['error'].message if pkg_ctx['error'].message else "",
pkg_ctx['error'].stdout if pkg_ctx['error'].stdout else "",
pkg_ctx['error'].stderr if pkg_ctx['error'].stderr else "")
if 'license_diff_fn' in pkg_ctx:
license_diff_fn = pkg_ctx['license_diff_fn']
msg_body += license_change_info % license_diff_fn
if 'patch_file' in pkg_ctx and pkg_ctx['patch_file'] != None:
msg_body += next_steps_info % (os.path.basename(
pkg_ctx['patch_file']))
if self.opts['testimage']:
if 'integration_error' in pkg_ctx:
msg_body += testimage_integration_error
else:
if 'ptest' in pkg_ctx:
machines = pkg_ctx['ptest'].keys()
msg_body += testimage_ptest_info % machines
if 'testimage' in pkg_ctx:
machines = pkg_ctx['testimage'].keys()
msg_body += testimage_info % (settings.get('testimage_name', \
DEFAULT_TESTIMAGE), machines)
msg_body += mail_footer
# Add possible attachments to email
attachments = []
for attachment in os.listdir(pkg_ctx['workdir']):
attachment_fullpath = os.path.join(pkg_ctx['workdir'], attachment)
if os.path.isfile(attachment_fullpath):
attachments.append(attachment_fullpath)
if self.opts['send_email']:
self.email_handler.send_email(to_addr,
subject,
msg_body,
attachments,
cc_addr=cc_addr)
# Preserve email for review purposes.
email_file = os.path.join(pkg_ctx['workdir'], "email_summary")
with open(email_file, "w+") as f:
f.write("To: %s\n" % to_addr)
if isinstance(cc_addr, list):
f.write("To: %s\n" % ' '.join(cc_addr))
else:
f.write("Cc: %s\n" % cc_addr)
f.write("Subject: %s\n" % subject)
f.write("Attachments: %s\n" % ' '.join(attachments))
f.write("\n%s\n" % msg_body)
def commit_changes(self, pkg_ctx):
try:
pkg_ctx['patch_file'] = None
I(" %s: Auto commit changes ..." % pkg_ctx['PN'])
self.git.add(pkg_ctx['recipe_dir'])
self.git.commit(pkg_ctx['commit_msg'], self.opts['author'])
stdout = self.git.create_patch(pkg_ctx['workdir'])
pkg_ctx['patch_file'] = stdout.strip()
if not pkg_ctx['patch_file']:
msg = "Patch file not generated."
E(" %s: %s\n %s" % (pkg_ctx['PN'], msg, stdout))
raise Error(msg, stdout)
else:
I(" %s: Save patch in directory: %s." %
(pkg_ctx['PN'], pkg_ctx['workdir']))
if pkg_ctx['error'] is not None:
I("Due to build errors, the commit will also be reverted to avoid cascading upgrade failures."
)
self.git.revert("HEAD")
except Error as e:
msg = ''
for line in e.stdout.split("\n"):
if line.find("nothing to commit") == 0:
msg = "Nothing to commit!"
I(" %s: %s" % (pkg_ctx['PN'], msg))
I(" %s: %s" % (pkg_ctx['PN'], e.stdout))
raise e
def send_status_mail(self, statistics_summary):
if "status_recipients" not in settings:
E(" Could not send status email, no recipients set!")
return -1
to_list = settings["status_recipients"].split()
if self.opts['layer_mode'] == 'yes':
subject = "[AUH] Upgrade status %s: %s" \
% (self.opts['layer_name'], date.isoformat(date.today()))
else:
subject = "[AUH] Upgrade status: " + date.isoformat(date.today())
if self.statistics.total_attempted:
self.email_handler.send_email(to_list, subject, statistics_summary)
else:
W("No recipes attempted, not sending status mail!")
def run(self, package_list=None):
pkgs_to_upgrade = self._get_packages_to_upgrade(package_list)
total_pkgs = len(pkgs_to_upgrade)
pkgs_ctx = {}
I(" ########### The list of recipes to be upgraded #############")
for p, ov, nv, m, r in pkgs_to_upgrade:
I(" %s, %s, %s, %s, %s" % (p, ov, nv, m, r))
pkgs_ctx[p] = {}
pkgs_ctx[p]['PN'] = p
pkgs_ctx[p]['PV'] = ov
pkgs_ctx[p]['NPV'] = nv
pkgs_ctx[p]['MAINTAINER'] = m
pkgs_ctx[p]['NSRCREV'] = r
pkgs_ctx[p]['base_dir'] = self.uh_recipes_all_dir
I(" ############################################################")
if pkgs_to_upgrade:
I(" Building gcc runtimes ...")
for machine in self.opts['machines']:
I(" building gcc runtime for %s" % machine)
try:
self.bb.complete("gcc-runtime", machine)
except Exception as e:
E(" Can't build gcc-runtime for %s." % machine)
if isinstance(e, Error):
E(e.stdout)
else:
import traceback
traceback.print_exc(file=sys.stdout)
succeeded_pkgs_ctx = []
failed_pkgs_ctx = []
attempted_pkgs = 0
for pn, _, _, _, _ in pkgs_to_upgrade:
pkg_ctx = pkgs_ctx[pn]
pkg_ctx['error'] = None
attempted_pkgs += 1
I(" ATTEMPT PACKAGE %d/%d" % (attempted_pkgs, total_pkgs))
try:
I(" %s: Upgrading to %s" % (pkg_ctx['PN'], pkg_ctx['NPV']))
for step, msg in upgrade_steps:
if msg is not None:
I(" %s: %s" % (pkg_ctx['PN'], msg))
step(self.devtool, self.bb, self.git, self.opts, pkg_ctx)
succeeded_pkgs_ctx.append(pkg_ctx)
I(" %s: Upgrade SUCCESSFUL! Please test!" % pkg_ctx['PN'])
except Exception as e:
if isinstance(e, UpgradeNotNeededError):
I(" %s: %s" % (pkg_ctx['PN'], e.message))
elif isinstance(e, UnsupportedProtocolError):
I(" %s: %s" % (pkg_ctx['PN'], e.message))
else:
if not isinstance(e, Error):
import traceback
msg = "Failed(unknown error)\n" + traceback.format_exc(
)
e = Error(message=msg)
error = e
E(" %s: %s" % (pkg_ctx['PN'], e.message))
if 'workdir' in pkg_ctx and os.listdir(pkg_ctx['workdir']):
E(" %s: Upgrade FAILED! Logs and/or file diffs are available in %s"
% (pkg_ctx['PN'], pkg_ctx['workdir']))
pkg_ctx['error'] = e
failed_pkgs_ctx.append(pkg_ctx)
try:
self.commit_changes(pkg_ctx)
except:
if pkg_ctx in succeeded_pkgs_ctx:
succeeded_pkgs_ctx.remove(pkg_ctx)
failed_pkgs_ctx.append(pkg_ctx)
if self.opts['testimage']:
ctxs = {}
ctxs['succeeded'] = succeeded_pkgs_ctx
ctxs['failed'] = failed_pkgs_ctx
image = settings.get('testimage_name', DEFAULT_TESTIMAGE)
tim = TestImage(self.bb, self.git, self.uh_work_dir, self.opts,
ctxs, image)
tim.run()
for pn in pkgs_ctx.keys():
pkg_ctx = pkgs_ctx[pn]
if pkg_ctx in succeeded_pkgs_ctx:
os.symlink(pkg_ctx['workdir'], os.path.join( \
self.uh_recipes_succeed_dir, pkg_ctx['PN']))
else:
os.symlink(pkg_ctx['workdir'], os.path.join( \
self.uh_recipes_failed_dir, pkg_ctx['PN']))
self.statistics.update(pkg_ctx['PN'], pkg_ctx['NPV'],
pkg_ctx['MAINTAINER'], pkg_ctx['error'])
self.pkg_upgrade_handler(pkg_ctx)
if attempted_pkgs > 0:
publish_work_url = settings.get('publish_work_url', '')
work_tarball = os.path.join(
self.uh_base_work_dir,
os.path.basename(self.uh_work_dir) + '.tar.gz')
if publish_work_url:
I(" Generating work tarball in %s ..." % work_tarball)
import subprocess
if subprocess.call(
["tar", "-chzf", work_tarball, self.uh_work_dir]):
E(" Work tarball (%s) generation failed..." %
(work_tarball))
publish_work_url = ''
statistics_summary = self.statistics.get_summary(
publish_work_url, os.path.basename(self.uh_work_dir))
statistics_file = os.path.join(self.uh_work_dir,
"statistics_summary")
with open(statistics_file, "w+") as f:
f.write(statistics_summary)
I(" %s" % statistics_summary)
if self.opts['send_email']:
self.send_status_mail(statistics_summary)
class Updater(object):
def __init__(self, auto_mode=False, send_email=False, skip_compilation=False):
build_dir = get_build_dir()
self.bb = Bitbake(build_dir)
try:
self.base_env = self.bb.env()
except EmptyEnvError as e:
import traceback
E( " %s\n%s" % (e.message, traceback.format_exc()))
E( " Bitbake output:\n%s" % (e.stdout))
exit(1)
self._set_options(auto_mode, send_email, skip_compilation)
self._make_dirs(build_dir)
self._add_file_logger()
self.email_handler = Email(settings)
self.statistics = Statistics()
def _set_options(self, auto_mode, send_email, skip_compilation):
self.opts = {}
self.opts['layer_mode'] = settings.get('layer_mode', '')
if self.opts['layer_mode'] == 'yes':
def _layer_settings_error(setting):
E(" In layer mode enable you need to specify %s.\n" % setting)
exit(1)
layer_settings = ('layer_name', 'layer_dir', 'layer_machines')
for s in layer_settings:
self.opts[s] = settings.get(s, '')
if not self.opts[s]:
_layer_settings_error(s)
self.git = Git(self.opts['layer_dir'])
self.poky_git = Git(os.path.dirname(os.getenv('PATH', False).split(':')[0]))
self.opts['machines'] = self.opts['layer_machines'].split()
else:
# XXX: assume that the poky directory is the first entry in the PATH
self.git = Git(os.path.dirname(os.getenv('PATH', False).split(':')[0]))
self.poky_git = None
self.opts['machines'] = settings.get('machines',
'qemux86 qemux86-64 qemuarm qemumips qemuppc').split()
self.opts['interactive'] = not auto_mode
self.opts['send_email'] = send_email
self.opts['author'] = "Upgrade Helper <%s>" % \
settings.get('from', '*****@*****.**')
self.opts['skip_compilation'] = skip_compilation
self.opts['buildhistory'] = self._buildhistory_is_enabled()
self.opts['testimage'] = self._testimage_is_enabled()
def _make_dirs(self, build_dir):
self.uh_dir = os.path.join(build_dir, "upgrade-helper")
if not os.path.exists(self.uh_dir):
os.mkdir(self.uh_dir)
self.uh_base_work_dir = settings.get('workdir', '')
if not self.uh_base_work_dir:
self.uh_base_work_dir = self.uh_dir
if self.opts['layer_mode'] == 'yes':
self.uh_base_work_dir = os.path.join(self.uh_base_work_dir,
self.opts['layer_name'])
if not os.path.exists(self.uh_base_work_dir):
os.mkdir(self.uh_base_work_dir)
self.uh_work_dir = os.path.join(self.uh_base_work_dir, "%s" % \
datetime.now().strftime("%Y%m%d%H%M%S"))
os.mkdir(self.uh_work_dir)
self.uh_recipes_all_dir = os.path.join(self.uh_work_dir, "all")
os.mkdir(self.uh_recipes_all_dir)
self.uh_recipes_succeed_dir = os.path.join(self.uh_work_dir, "succeed")
os.mkdir(self.uh_recipes_succeed_dir)
self.uh_recipes_failed_dir = os.path.join(self.uh_work_dir, "failed")
os.mkdir(self.uh_recipes_failed_dir)
def _add_file_logger(self):
fh = log.FileHandler(os.path.join(self.uh_work_dir, "upgrade-helper.log"))
logger = log.getLogger()
logger.addHandler(fh)
def _get_status_msg(self, err):
if err:
return str(err)
else:
return "Succeeded"
def _buildhistory_is_enabled(self):
enabled = False
if settings.get("buildhistory", "no") == "yes":
if 'buildhistory' in self.base_env['INHERIT']:
if not 'BUILDHISTORY_COMMIT' in self.base_env:
E(" Buildhistory was INHERIT in conf/local.conf"\
" but need BUILDHISTORY_COMMIT=1 please set.")
exit(1)
if not self.base_env['BUILDHISTORY_COMMIT'] == '1':
E(" Buildhistory was INHERIT in conf/local.conf"\
" but need BUILDHISTORY_COMMIT=1 please set.")
exit(1)
if self.opts['skip_compilation']:
W(" Buildhistory disabled because user" \
" skip compilation!")
else:
enabled = True
else:
E(" Buildhistory was enabled in upgrade-helper.conf"\
" but isn't INHERIT in conf/local.conf, if you want"\
" to enable please set.")
exit(1)
else:
if 'buildhistory' in self.base_env['INHERIT']:
E(" Buildhistory was INHERIT in conf/local.conf"\
" but buildhistory=yes isn't in upgrade-helper.conf,"\
" if you want to enable please set.")
exit(1)
return enabled
def _testimage_is_enabled(self):
enabled = False
if settings.get("testimage", "no") == "yes":
if 'testimage' in self.base_env['INHERIT']:
if not "ptest" in self.base_env["DISTRO_FEATURES"]:
E(" testimage requires ptest in DISTRO_FEATURES please add to"\
" conf/local.conf.")
exit(1)
if not "package-management" in self.base_env['EXTRA_IMAGE_FEATURES']:
E(" testimage requires package-management in EXTRA_IMAGE_FEATURES"\
" please add to conf/local.conf.")
exit(1)
if not "ptest-pkgs" in self.base_env['EXTRA_IMAGE_FEATURES']:
E(" testimage/ptest requires ptest-pkgs in EXTRA_IMAGE_FEATURES"\
" please add to conf/local.conf.")
exit(1)
if not "package_rpm" == self.base_env["PACKAGE_CLASSES"]:
E(" testimage/ptest requires PACKAGE_CLASSES set to package_rpm"\
" please add to conf/local.conf.")
exit(1)
enabled = True
else:
E(" testimage was enabled in upgrade-helper.conf"\
" but isn't INHERIT in conf/local.conf, if you want"\
" to enable please set.")
exit(1)
else:
if 'testimage' in self.base_env['INHERIT']:
E(" testimage was INHERIT in conf/local.conf"\
" but testimage=yes isn't in upgrade-helper.conf,"\
" if you want to enable please set.")
exit(1)
return enabled
def _get_packages_to_upgrade(self, packages=None):
if packages is None:
I( "Nothing to upgrade")
exit(0)
else:
return packages
# this function will be called at the end of each recipe upgrade
def pkg_upgrade_handler(self, pkg_ctx):
mail_header = \
"Hello,\n\nYou are receiving this email because you are the maintainer\n" \
"of *%s* recipe and this is to let you know that the automatic attempt\n" \
"to upgrade the recipe to *%s* has %s.\n\n"
license_change_info = \
"*LICENSE CHANGED* please review the %s file and update the LICENSE\n" \
"variable in the recipe if is needed.\n\n"
next_steps_info = \
"The recipe has been successfully compiled for machines %s.\n\n" \
"Next steps:\n" \
" - apply the patch: git am %s\n" \
" - check that required upstream patches have not been commented from the recipe,\n" \
" if upstream patches were commented the reason is specified in the commit message.\n" \
" - compile an image that contains the package\n" \
" - perform some basic sanity tests\n" \
" - amend the patch and sign it off: git commit -s --reset-author --amend\n" \
" - send it to the list\n\n" \
testimage_integration_error = \
"The recipe *FAILED* in testimage integration. Attached is the log file.\n\n"
testimage_ptest_info = \
"The recipe has ptest enabled and has been tested with core-image-minimal/ptest \n" \
"with the next machines %s. Attached is the log file.\n\n"
testimage_info = \
"The recipe has been tested using %s testimage and succeeded with \n" \
"the next machines %s. Attached is the log file.\n\n" \
mail_footer = \
"Attached are the patch, license diff (if change) and bitbake log.\n" \
"Any problem please contact Anibal Limon <*****@*****.**>.\n\n" \
"Regards,\nThe Upgrade Helper"
if pkg_ctx['MAINTAINER'] in maintainer_override:
to_addr = maintainer_override[pkg_ctx['MAINTAINER']]
else:
to_addr = pkg_ctx['MAINTAINER']
cc_addr = None
if "status_recipients" in settings:
cc_addr = settings["status_recipients"].split()
subject = "[AUH] " + pkg_ctx['PN'] + ": upgrading to " + pkg_ctx['NPV']
if not pkg_ctx['error']:
subject += " SUCCEEDED"
else:
subject += " FAILED"
msg_body = mail_header % (pkg_ctx['PN'], pkg_ctx['NPV'],
self._get_status_msg(pkg_ctx['error']))
if 'recipe' in pkg_ctx:
license_diff_fn = pkg_ctx['recipe'].get_license_diff_file_name()
if license_diff_fn:
msg_body += license_change_info % license_diff_fn
if not pkg_ctx['error']:
msg_body += next_steps_info % (', '.join(self.opts['machines']),
os.path.basename(pkg_ctx['patch_file']))
if self.opts['testimage']:
if 'integration_error' in pkg_ctx:
msg_body += testimage_integration_error
else:
if 'ptest' in pkg_ctx:
machines = pkg_ctx['ptest'].keys()
msg_body += testimage_ptest_info % machines
if 'testimage' in pkg_ctx:
machines = pkg_ctx['testimage'].keys()
msg_body += testimage_info % (settings.get('testimage_name', \
DEFAULT_TESTIMAGE), machines)
msg_body += mail_footer
# Add possible attachments to email
attachments = []
for attachment in os.listdir(pkg_ctx['workdir']):
attachment_fullpath = os.path.join(pkg_ctx['workdir'], attachment)
if os.path.isfile(attachment_fullpath):
attachments.append(attachment_fullpath)
# Only send email to Maintainer when recipe upgrade succeed.
if self.opts['send_email'] and not pkg_ctx['error']:
self.email_handler.send_email(to_addr, subject, msg_body, attachments, cc_addr=cc_addr)
# Preserve email for review purposes.
email_file = os.path.join(pkg_ctx['workdir'],
"email_summary")
with open(email_file, "w+") as f:
f.write("To: %s\n" % to_addr)
if isinstance(cc_addr, list):
f.write("To: %s\n" % ' '.join(cc_addr))
else:
f.write("Cc: %s\n" % cc_addr)
f.write("Subject: %s\n" % subject)
f.write("Attachments: %s\n" % ' '.join(attachments))
f.write("\n%s\n" % msg_body)
def commit_changes(self, pkg_ctx):
fail = False
try:
pkg_ctx['patch_file'] = None
if 'recipe' in pkg_ctx:
I(" %s: Auto commit changes ..." % pkg_ctx['PN'])
self.git.commit(pkg_ctx['recipe'].commit_msg, self.opts['author'])
stdout = self.git.create_patch(pkg_ctx['workdir'])
pkg_ctx['patch_file'] = stdout.strip()
if not pkg_ctx['patch_file']:
msg = "Patch file not generated."
E(" %s: %s\n %s" % (pkg_ctx['PN'], msg, stdout))
pkg_ctx['error'] = Error(msg, stdout)
fail = True
else:
I(" %s: Save patch in directory: %s." %
(pkg_ctx['PN'], pkg_ctx['workdir']))
except Error as e:
msg = ''
for line in e.stdout.split("\n"):
if line.find("nothing to commit") == 0:
msg = "Nothing to commit!"
I(" %s: %s" % (pkg_ctx['PN'], msg))
I(" %s: %s" % (pkg_ctx['PN'], e.stdout))
pkg_ctx['error'] = Error(msg, e.stdout)
fail = True
if fail:
raise pkg_ctx['error']
def send_status_mail(self, statistics_summary):
if "status_recipients" not in settings:
E(" Could not send status email, no recipients set!")
return -1
to_list = settings["status_recipients"].split()
if self.opts['layer_mode'] == 'yes':
subject = "[AUH] Upgrade status %s: %s" \
% (self.opts['layer_name'], date.isoformat(date.today()))
else:
subject = "[AUH] Upgrade status: " + date.isoformat(date.today())
if self.statistics.total_attempted:
self.email_handler.send_email(to_list, subject, statistics_summary)
else:
W("No recipes attempted, not sending status mail!")
def _order_pkgs_to_upgrade(self, pkgs_to_upgrade):
def _get_pn_dep_dic(pn_list, dependency_file):
import re
pn_dep_dic = {}
with open(dependency_file) as dep:
data = dep.read()
dep.close()
for line in data.split('\n'):
m = re.search('^"(.*)" -> "(.*)"$', line)
if not m:
continue
pn = m.group(1)
pn_dep = m.group(2)
if pn == pn_dep:
continue
if pn in pn_list:
if pn_dep in pn_list:
if pn in pn_dep_dic.keys():
pn_dep_dic[pn].append(pn_dep)
else:
pn_dep_dic[pn] = [pn_dep]
elif not pn in pn_dep_dic.keys():
pn_dep_dic[pn] = []
return pn_dep_dic
def _dep_resolve(graph, node, resolved, seen):
seen.append(node)
for edge in graph[node]:
if edge not in resolved:
if edge in seen:
raise RuntimeError("Packages %s and %s have " \
"a circular dependency." \
% (node, edge))
_dep_resolve(graph, edge, resolved, seen)
resolved.append(node)
pn_list = []
for pn, new_ver, maintainer in pkgs_to_upgrade:
pn_list.append(pn)
try:
self.bb.dependency_graph(' '.join(pn_list))
except Error as e:
multiple_providers = False
for l in e.stdout.split('\n'):
if l.find("ERROR: Multiple .bb files are due to be built which each provide") == 0:
multiple_providers = True
if not multiple_providers:
raise e
dependency_file = os.path.join(get_build_dir(), "pn-depends.dot")
pkgs_to_upgrade_ordered = []
pn_list_ordered = []
pn_dep_dic = _get_pn_dep_dic(pn_list, dependency_file)
if pn_dep_dic:
root = "__root_node__"
pn_dep_dic[root] = pn_dep_dic.keys()
_dep_resolve(pn_dep_dic, root, pn_list_ordered, [])
pn_list_ordered.remove(root)
for pn_ordered in pn_list_ordered:
for pn, new_ver, maintainer in pkgs_to_upgrade:
if pn == pn_ordered:
pkgs_to_upgrade_ordered.append([pn, new_ver, maintainer])
return pkgs_to_upgrade_ordered
def run(self, package_list=None):
#pkgs_to_upgrade = self._order_pkgs_to_upgrade(
# self._get_packages_to_upgrade(package_list))
pkgs_to_upgrade = self._get_packages_to_upgrade(package_list)
total_pkgs = len(pkgs_to_upgrade)
pkgs_ctx = {}
I(" ########### The list of recipes to be upgraded #############")
for p, v, m in pkgs_to_upgrade:
I(" %s, %s, %s" % (p, v, m))
pkgs_ctx[p] = {}
pkgs_ctx[p]['PN'] = p
pkgs_ctx[p]['NPV'] = v
pkgs_ctx[p]['MAINTAINER'] = m
pkgs_ctx[p]['base_dir'] = self.uh_recipes_all_dir
I(" ############################################################")
if pkgs_to_upgrade:
I(" Building gcc runtimes ...")
for machine in self.opts['machines']:
I(" building gcc runtime for %s" % machine)
try:
self.bb.complete("gcc-runtime", machine)
except Exception as e:
E(" Can't build gcc-runtime for %s." % machine)
if isinstance(e, Error):
E(e.stdout)
else:
import traceback
traceback.print_exc(file=sys.stdout)
succeeded_pkgs_ctx = []
failed_pkgs_ctx = []
attempted_pkgs = 0
for pn, _, _ in pkgs_to_upgrade:
pkg_ctx = pkgs_ctx[pn]
pkg_ctx['error'] = None
attempted_pkgs += 1
I(" ATTEMPT PACKAGE %d/%d" % (attempted_pkgs, total_pkgs))
try:
I(" %s: Upgrading to %s" % (pkg_ctx['PN'], pkg_ctx['NPV']))
for step, msg in upgrade_steps:
if msg is not None:
I(" %s: %s" % (pkg_ctx['PN'], msg))
step(self.bb, self.git, self.opts, pkg_ctx)
succeeded_pkgs_ctx.append(pkg_ctx)
I(" %s: Upgrade SUCCESSFUL! Please test!" % pkg_ctx['PN'])
except Exception as e:
if isinstance(e, UpgradeNotNeededError):
I(" %s: %s" % (pkg_ctx['PN'], e.message))
elif isinstance(e, UnsupportedProtocolError):
I(" %s: %s" % (pkg_ctx['PN'], e.message))
else:
if not isinstance(e, Error):
import traceback
msg = "Failed(unknown error)\n" + traceback.format_exc()
e = Error(message=msg)
error = e
E(" %s: %s" % (pkg_ctx['PN'], e.message))
if os.listdir(pkg_ctx['workdir']):
E(" %s: Upgrade FAILED! Logs and/or file diffs are available in %s"
% (pkg_ctx['PN'], pkg_ctx['workdir']))
pkg_ctx['error'] = e
failed_pkgs_ctx.append(pkg_ctx)
try:
self.commit_changes(pkg_ctx)
except:
if pkg_ctx in succeeded_pkgs_ctx:
succeeded_pkgs_ctx.remove(pkg_ctx)
failed_pkgs_ctx.append(pkg_ctx)
if self.opts['testimage']:
ctxs = {}
ctxs['succeeded'] = succeeded_pkgs_ctx
ctxs['failed'] = failed_pkgs_ctx
image = settings.get('testimage_name', DEFAULT_TESTIMAGE)
tim = TestImage(self.bb, self.git, self.uh_work_dir, self.opts,
ctxs, image)
tim.run()
for pn in pkgs_ctx.keys():
pkg_ctx = pkgs_ctx[pn]
if pkg_ctx in succeeded_pkgs_ctx:
os.symlink(pkg_ctx['workdir'], os.path.join( \
self.uh_recipes_succeed_dir, pkg_ctx['PN']))
else:
os.symlink(pkg_ctx['workdir'], os.path.join( \
self.uh_recipes_failed_dir, pkg_ctx['PN']))
self.statistics.update(pkg_ctx['PN'], pkg_ctx['NPV'],
pkg_ctx['MAINTAINER'], pkg_ctx['error'])
self.pkg_upgrade_handler(pkg_ctx)
if attempted_pkgs > 0:
publish_work_url = settings.get('publish_work_url', '')
work_tarball = os.path.join(self.uh_base_work_dir,
os.path.basename(self.uh_work_dir) + '.tar.gz')
if publish_work_url:
I(" Generating work tarball in %s ..." % work_tarball)
import subprocess
if subprocess.call(["tar", "-chzf", work_tarball, self.uh_work_dir]):
E(" Work tarball (%s) generation failed..." % (work_tarball))
publish_work_url = ''
statistics_summary = self.statistics.get_summary(
publish_work_url, os.path.basename(self.uh_work_dir))
statistics_file = os.path.join(self.uh_work_dir,
"statistics_summary")
with open(statistics_file, "w+") as f:
f.write(statistics_summary)
I(" %s" % statistics_summary)
if self.opts['send_email']:
self.send_status_mail(statistics_summary)