def _set_options(self, auto_mode, send_email, skip_compilation): self.opts = {} self.opts['layer_mode'] = settings.get('layer_mode', '') if self.opts['layer_mode'] == 'yes': def _layer_settings_error(setting): E(" In layer mode enable you need to specify %s.\n" % setting) exit(1) layer_settings = ('layer_name', 'layer_dir', 'layer_machines') for s in layer_settings: self.opts[s] = settings.get(s, '') if not self.opts[s]: _layer_settings_error(s) self.git = Git(self.opts['layer_dir']) self.poky_git = Git(os.path.dirname(os.getenv('PATH', False).split(':')[0])) self.opts['machines'] = self.opts['layer_machines'].split() else: # XXX: assume that the poky directory is the first entry in the PATH self.git = Git(os.path.dirname(os.getenv('PATH', False).split(':')[0])) self.poky_git = None self.opts['machines'] = settings.get('machines', 'qemux86 qemux86-64 qemuarm qemumips qemuppc').split() self.opts['interactive'] = not auto_mode self.opts['send_email'] = send_email self.opts['author'] = "Upgrade Helper <%s>" % \ settings.get('from', '*****@*****.**') self.opts['skip_compilation'] = skip_compilation self.opts['buildhistory'] = self._buildhistory_is_enabled() self.opts['testimage'] = self._testimage_is_enabled()
def _set_options(self, auto_mode, send_email, skip_compilation): self.opts = {} self.opts['layer_mode'] = settings.get('layer_mode', '') if self.opts['layer_mode'] == 'yes': def _layer_settings_error(setting): E(" In layer mode enable you need to specify %s.\n" % setting) exit(1) layer_settings = ('layer_name', 'layer_dir', 'layer_machines') for s in layer_settings: self.opts[s] = settings.get(s, '') if not self.opts[s]: _layer_settings_error(s) self.git = Git(self.opts['layer_dir']) self.poky_git = Git( os.path.dirname(os.getenv('PATH', False).split(':')[0])) self.opts['machines'] = self.opts['layer_machines'].split() else: # XXX: assume that the poky directory is the first entry in the PATH self.git = Git( os.path.dirname(os.getenv('PATH', False).split(':')[0])) self.poky_git = None self.opts['machines'] = settings.get( 'machines', 'qemux86 qemux86-64 qemuarm qemumips qemuppc').split() self.opts['interactive'] = not auto_mode self.opts['send_email'] = send_email self.opts['author'] = "Upgrade Helper <%s>" % \ settings.get('from', '*****@*****.**') self.opts['skip_compilation'] = skip_compilation self.opts['buildhistory'] = self._buildhistory_is_enabled() self.opts['testimage'] = self._testimage_is_enabled()
def save(name, path, existing_issues): tags = fetch(path) artifacts = parse(tags) print("Tags", tags) issues = {} for key, values in artifacts.items(): title, body = create_issue(name, key, values[::-1]) if key in existing_issues: issues[key] = Git.update_issue(existing_issues[key], title, body) else: issues[key] = Git.create_issue(title, body) sleep(0.34) return issues
class BuildHistory(object): def __init__(self, bb, pn, workdir): self.bb = bb self.pn = pn self.workdir = workdir self.revs = [] self.buildhistory_dir = os.path.join(self.workdir, 'buildhistory') if not os.path.exists(self.buildhistory_dir): os.mkdir(self.buildhistory_dir) self.git = Git(self.buildhistory_dir) os.environ['BB_ENV_EXTRAWHITE'] = os.environ['BB_ENV_EXTRAWHITE'] + \ " BUILDHISTORY_DIR" os.environ["BUILDHISTORY_DIR"] = self.buildhistory_dir def init(self, machines): self.bb.cleansstate(self.pn) for machine in machines: self.bb.complete(self.pn, machine) self.revs.append(self.git.last_commit("master")) def add(self): self.revs.append(self.git.last_commit("master")) def diff(self): rev_initial = self.revs[0] rev_final = self.revs[-1] try: cmd = "buildhistory-diff -p %s %s %s" % (self.buildhistory_dir, rev_initial, rev_final) stdout, stderr = bb.process.run(cmd) if stdout and os.path.exists(self.workdir): with open(os.path.join(self.workdir, "buildhistory-diff.txt"), "w+") as log: log.write(stdout) cmd_full = "buildhistory-diff -a -p %s %s %s" % ( self.buildhistory_dir, rev_initial, rev_final) stdout, stderr = bb.process.run(cmd_full) if stdout and os.path.exists(self.workdir): with open( os.path.join(self.workdir, "buildhistory-diff-full.txt"), "w+") as log: log.write(stdout) except bb.process.ExecutionError as e: W("%s: Buildhistory checking fails\n%s" % (self.pn, e.stdout))
def __init__(self, bb, pn, workdir): self.bb = bb self.pn = pn self.workdir = workdir self.revs = [] self.buildhistory_dir = os.path.join(self.workdir, 'buildhistory') if not os.path.exists(self.buildhistory_dir): os.mkdir(self.buildhistory_dir) self.git = Git(self.buildhistory_dir) os.environ['BB_ENV_EXTRAWHITE'] = os.environ['BB_ENV_EXTRAWHITE'] + \ " BUILDHISTORY_DIR" os.environ["BUILDHISTORY_DIR"] = self.buildhistory_dir
class BuildHistory(object): def __init__(self, bb, pn, workdir): self.bb = bb self.pn = pn self.workdir = workdir self.revs = [] self.buildhistory_dir = os.path.join(self.workdir, 'buildhistory') if not os.path.exists(self.buildhistory_dir): os.mkdir(self.buildhistory_dir) self.git = Git(self.buildhistory_dir) os.environ['BB_ENV_EXTRAWHITE'] = os.environ['BB_ENV_EXTRAWHITE'] + \ " BUILDHISTORY_DIR" os.environ["BUILDHISTORY_DIR"] = self.buildhistory_dir def init(self, machines): self.bb.cleanall(self.pn) for machine in machines: self.bb.complete(self.pn, machine) self.revs.append(self.git.last_commit("master")) def add(self): self.revs.append(self.git.last_commit("master")) def diff(self): rev_initial = self.revs[0] rev_final = self.revs[-1] try: cmd = "buildhistory-diff -p %s %s %s" % (self.buildhistory_dir, rev_initial, rev_final) stdout, stderr = bb.process.run(cmd) if stdout and os.path.exists(self.workdir): with open(os.path.join(self.workdir, "buildhistory-diff.txt"), "w+") as log: log.write(stdout) cmd_full = "buildhistory-diff -a -p %s %s %s" % (self.buildhistory_dir, rev_initial, rev_final) stdout, stderr = bb.process.run(cmd_full) if stdout and os.path.exists(self.workdir): with open(os.path.join(self.workdir, "buildhistory-diff-full.txt"), "w+") as log: log.write(stdout) except bb.process.ExecutionError as e: W( "%s: Buildhistory checking fails\n%s" % (self.pn, e.stdout))
class Updater(object): def __init__(self, auto_mode=False, send_email=False, skip_compilation=False): build_dir = get_build_dir() self.bb = Bitbake(build_dir) try: self.base_env = self.bb.env() except EmptyEnvError as e: import traceback E(" %s\n%s" % (e.message, traceback.format_exc())) E(" Bitbake output:\n%s" % (e.stdout)) exit(1) self._set_options(auto_mode, send_email, skip_compilation) self._make_dirs(build_dir) self._add_file_logger() self.email_handler = Email(settings) self.statistics = Statistics() def _set_options(self, auto_mode, send_email, skip_compilation): self.opts = {} self.opts['layer_mode'] = settings.get('layer_mode', '') if self.opts['layer_mode'] == 'yes': def _layer_settings_error(setting): E(" In layer mode enable you need to specify %s.\n" % setting) exit(1) layer_settings = ('layer_name', 'layer_dir', 'layer_machines') for s in layer_settings: self.opts[s] = settings.get(s, '') if not self.opts[s]: _layer_settings_error(s) self.git = Git(self.opts['layer_dir']) self.poky_git = Git( os.path.dirname(os.getenv('PATH', False).split(':')[0])) self.opts['machines'] = self.opts['layer_machines'].split() else: # XXX: assume that the poky directory is the first entry in the PATH self.git = Git( os.path.dirname(os.getenv('PATH', False).split(':')[0])) self.poky_git = None self.opts['machines'] = settings.get( 'machines', 'qemux86 qemux86-64 qemuarm qemumips qemuppc').split() self.opts['interactive'] = not auto_mode self.opts['send_email'] = send_email self.opts['author'] = "Upgrade Helper <%s>" % \ settings.get('from', '*****@*****.**') self.opts['skip_compilation'] = skip_compilation self.opts['buildhistory'] = self._buildhistory_is_enabled() self.opts['testimage'] = self._testimage_is_enabled() def _make_dirs(self, build_dir): self.uh_dir = os.path.join(build_dir, "upgrade-helper") if not os.path.exists(self.uh_dir): os.mkdir(self.uh_dir) self.uh_base_work_dir = settings.get('workdir', '') if not self.uh_base_work_dir: self.uh_base_work_dir = self.uh_dir if self.opts['layer_mode'] == 'yes': self.uh_base_work_dir = os.path.join(self.uh_base_work_dir, self.opts['layer_name']) if not os.path.exists(self.uh_base_work_dir): os.mkdir(self.uh_base_work_dir) self.uh_work_dir = os.path.join(self.uh_base_work_dir, "%s" % \ datetime.now().strftime("%Y%m%d%H%M%S")) os.mkdir(self.uh_work_dir) self.uh_recipes_all_dir = os.path.join(self.uh_work_dir, "all") os.mkdir(self.uh_recipes_all_dir) self.uh_recipes_succeed_dir = os.path.join(self.uh_work_dir, "succeed") os.mkdir(self.uh_recipes_succeed_dir) self.uh_recipes_failed_dir = os.path.join(self.uh_work_dir, "failed") os.mkdir(self.uh_recipes_failed_dir) def _add_file_logger(self): fh = log.FileHandler( os.path.join(self.uh_work_dir, "upgrade-helper.log")) logger = log.getLogger() logger.addHandler(fh) def _get_status_msg(self, err): if err: return str(err) else: return "Succeeded" def _buildhistory_is_enabled(self): enabled = False if settings.get("buildhistory", "no") == "yes": if 'buildhistory' in self.base_env['INHERIT']: if not 'BUILDHISTORY_COMMIT' in self.base_env: E(" Buildhistory was INHERIT in conf/local.conf"\ " but need BUILDHISTORY_COMMIT=1 please set.") exit(1) if not self.base_env['BUILDHISTORY_COMMIT'] == '1': E(" Buildhistory was INHERIT in conf/local.conf"\ " but need BUILDHISTORY_COMMIT=1 please set.") exit(1) if self.opts['skip_compilation']: W(" Buildhistory disabled because user" \ " skip compilation!") else: enabled = True else: E(" Buildhistory was enabled in upgrade-helper.conf"\ " but isn't INHERIT in conf/local.conf, if you want"\ " to enable please set.") exit(1) else: if 'buildhistory' in self.base_env['INHERIT']: E(" Buildhistory was INHERIT in conf/local.conf"\ " but buildhistory=yes isn't in upgrade-helper.conf,"\ " if you want to enable please set.") exit(1) return enabled def _testimage_is_enabled(self): enabled = False if settings.get("testimage", "no") == "yes": if 'testimage' in self.base_env['INHERIT']: if not "ptest" in self.base_env["DISTRO_FEATURES"]: E(" testimage requires ptest in DISTRO_FEATURES please add to"\ " conf/local.conf.") exit(1) if not "package-management" in self.base_env[ 'EXTRA_IMAGE_FEATURES']: E(" testimage requires package-management in EXTRA_IMAGE_FEATURES"\ " please add to conf/local.conf.") exit(1) if not "ptest-pkgs" in self.base_env['EXTRA_IMAGE_FEATURES']: E(" testimage/ptest requires ptest-pkgs in EXTRA_IMAGE_FEATURES"\ " please add to conf/local.conf.") exit(1) if not "package_rpm" == self.base_env["PACKAGE_CLASSES"]: E(" testimage/ptest requires PACKAGE_CLASSES set to package_rpm"\ " please add to conf/local.conf.") exit(1) enabled = True else: E(" testimage was enabled in upgrade-helper.conf"\ " but isn't INHERIT in conf/local.conf, if you want"\ " to enable please set.") exit(1) else: if 'testimage' in self.base_env['INHERIT']: E(" testimage was INHERIT in conf/local.conf"\ " but testimage=yes isn't in upgrade-helper.conf,"\ " if you want to enable please set.") exit(1) return enabled def _get_packages_to_upgrade(self, packages=None): if packages is None: I("Nothing to upgrade") exit(0) else: return packages # this function will be called at the end of each recipe upgrade def pkg_upgrade_handler(self, pkg_ctx): mail_header = \ "Hello,\n\nYou are receiving this email because you are the maintainer\n" \ "of *%s* recipe and this is to let you know that the automatic attempt\n" \ "to upgrade the recipe to *%s* has %s.\n\n" license_change_info = \ "*LICENSE CHANGED* please review the %s file and update the LICENSE\n" \ "variable in the recipe if is needed.\n\n" next_steps_info = \ "The recipe has been successfully compiled for machines %s.\n\n" \ "Next steps:\n" \ " - apply the patch: git am %s\n" \ " - check that required upstream patches have not been commented from the recipe,\n" \ " if upstream patches were commented the reason is specified in the commit message.\n" \ " - compile an image that contains the package\n" \ " - perform some basic sanity tests\n" \ " - amend the patch and sign it off: git commit -s --reset-author --amend\n" \ " - send it to the list\n\n" \ testimage_integration_error = \ "The recipe *FAILED* in testimage integration. Attached is the log file.\n\n" testimage_ptest_info = \ "The recipe has ptest enabled and has been tested with core-image-minimal/ptest \n" \ "with the next machines %s. Attached is the log file.\n\n" testimage_info = \ "The recipe has been tested using %s testimage and succeeded with \n" \ "the next machines %s. Attached is the log file.\n\n" \ mail_footer = \ "Attached are the patch, license diff (if change) and bitbake log.\n" \ "Any problem please contact Anibal Limon <*****@*****.**>.\n\n" \ "Regards,\nThe Upgrade Helper" if pkg_ctx['MAINTAINER'] in maintainer_override: to_addr = maintainer_override[pkg_ctx['MAINTAINER']] else: to_addr = pkg_ctx['MAINTAINER'] cc_addr = None if "status_recipients" in settings: cc_addr = settings["status_recipients"].split() subject = "[AUH] " + pkg_ctx['PN'] + ": upgrading to " + pkg_ctx['NPV'] if not pkg_ctx['error']: subject += " SUCCEEDED" else: subject += " FAILED" msg_body = mail_header % (pkg_ctx['PN'], pkg_ctx['NPV'], self._get_status_msg(pkg_ctx['error'])) if 'recipe' in pkg_ctx: license_diff_fn = pkg_ctx['recipe'].get_license_diff_file_name() if license_diff_fn: msg_body += license_change_info % license_diff_fn if not pkg_ctx['error']: msg_body += next_steps_info % (', '.join( self.opts['machines']), os.path.basename( pkg_ctx['patch_file'])) if self.opts['testimage']: if 'integration_error' in pkg_ctx: msg_body += testimage_integration_error else: if 'ptest' in pkg_ctx: machines = pkg_ctx['ptest'].keys() msg_body += testimage_ptest_info % machines if 'testimage' in pkg_ctx: machines = pkg_ctx['testimage'].keys() msg_body += testimage_info % (settings.get('testimage_name', \ DEFAULT_TESTIMAGE), machines) msg_body += mail_footer # Add possible attachments to email attachments = [] for attachment in os.listdir(pkg_ctx['workdir']): attachment_fullpath = os.path.join(pkg_ctx['workdir'], attachment) if os.path.isfile(attachment_fullpath): attachments.append(attachment_fullpath) # Only send email to Maintainer when recipe upgrade succeed. if self.opts['send_email'] and not pkg_ctx['error']: self.email_handler.send_email(to_addr, subject, msg_body, attachments, cc_addr=cc_addr) # Preserve email for review purposes. email_file = os.path.join(pkg_ctx['workdir'], "email_summary") with open(email_file, "w+") as f: f.write("To: %s\n" % to_addr) if isinstance(cc_addr, list): f.write("To: %s\n" % ' '.join(cc_addr)) else: f.write("Cc: %s\n" % cc_addr) f.write("Subject: %s\n" % subject) f.write("Attachments: %s\n" % ' '.join(attachments)) f.write("\n%s\n" % msg_body) def commit_changes(self, pkg_ctx): fail = False try: pkg_ctx['patch_file'] = None if 'recipe' in pkg_ctx: I(" %s: Auto commit changes ..." % pkg_ctx['PN']) self.git.commit(pkg_ctx['recipe'].commit_msg, self.opts['author']) stdout = self.git.create_patch(pkg_ctx['workdir']) pkg_ctx['patch_file'] = stdout.strip() if not pkg_ctx['patch_file']: msg = "Patch file not generated." E(" %s: %s\n %s" % (pkg_ctx['PN'], msg, stdout)) pkg_ctx['error'] = Error(msg, stdout) fail = True else: I(" %s: Save patch in directory: %s." % (pkg_ctx['PN'], pkg_ctx['workdir'])) except Error as e: msg = '' for line in e.stdout.split("\n"): if line.find("nothing to commit") == 0: msg = "Nothing to commit!" I(" %s: %s" % (pkg_ctx['PN'], msg)) I(" %s: %s" % (pkg_ctx['PN'], e.stdout)) pkg_ctx['error'] = Error(msg, e.stdout) fail = True if fail: raise pkg_ctx['error'] def send_status_mail(self, statistics_summary): if "status_recipients" not in settings: E(" Could not send status email, no recipients set!") return -1 to_list = settings["status_recipients"].split() if self.opts['layer_mode'] == 'yes': subject = "[AUH] Upgrade status %s: %s" \ % (self.opts['layer_name'], date.isoformat(date.today())) else: subject = "[AUH] Upgrade status: " + date.isoformat(date.today()) if self.statistics.total_attempted: self.email_handler.send_email(to_list, subject, statistics_summary) else: W("No recipes attempted, not sending status mail!") def _order_pkgs_to_upgrade(self, pkgs_to_upgrade): def _get_pn_dep_dic(pn_list, dependency_file): import re pn_dep_dic = {} with open(dependency_file) as dep: data = dep.read() dep.close() for line in data.split('\n'): m = re.search('^"(.*)" -> "(.*)"$', line) if not m: continue pn = m.group(1) pn_dep = m.group(2) if pn == pn_dep: continue if pn in pn_list: if pn_dep in pn_list: if pn in pn_dep_dic.keys(): pn_dep_dic[pn].append(pn_dep) else: pn_dep_dic[pn] = [pn_dep] elif not pn in pn_dep_dic.keys(): pn_dep_dic[pn] = [] return pn_dep_dic def _dep_resolve(graph, node, resolved, seen): seen.append(node) for edge in graph[node]: if edge not in resolved: if edge in seen: raise RuntimeError("Packages %s and %s have " \ "a circular dependency." \ % (node, edge)) _dep_resolve(graph, edge, resolved, seen) resolved.append(node) pn_list = [] for pn, new_ver, maintainer in pkgs_to_upgrade: pn_list.append(pn) try: self.bb.dependency_graph(' '.join(pn_list)) except Error as e: multiple_providers = False for l in e.stdout.split('\n'): if l.find( "ERROR: Multiple .bb files are due to be built which each provide" ) == 0: multiple_providers = True if not multiple_providers: raise e dependency_file = os.path.join(get_build_dir(), "pn-depends.dot") pkgs_to_upgrade_ordered = [] pn_list_ordered = [] pn_dep_dic = _get_pn_dep_dic(pn_list, dependency_file) if pn_dep_dic: root = "__root_node__" pn_dep_dic[root] = pn_dep_dic.keys() _dep_resolve(pn_dep_dic, root, pn_list_ordered, []) pn_list_ordered.remove(root) for pn_ordered in pn_list_ordered: for pn, new_ver, maintainer in pkgs_to_upgrade: if pn == pn_ordered: pkgs_to_upgrade_ordered.append([pn, new_ver, maintainer]) return pkgs_to_upgrade_ordered def run(self, package_list=None): #pkgs_to_upgrade = self._order_pkgs_to_upgrade( # self._get_packages_to_upgrade(package_list)) pkgs_to_upgrade = self._get_packages_to_upgrade(package_list) total_pkgs = len(pkgs_to_upgrade) pkgs_ctx = {} I(" ########### The list of recipes to be upgraded #############") for p, v, m in pkgs_to_upgrade: I(" %s, %s, %s" % (p, v, m)) pkgs_ctx[p] = {} pkgs_ctx[p]['PN'] = p pkgs_ctx[p]['NPV'] = v pkgs_ctx[p]['MAINTAINER'] = m pkgs_ctx[p]['base_dir'] = self.uh_recipes_all_dir I(" ############################################################") if pkgs_to_upgrade: I(" Building gcc runtimes ...") for machine in self.opts['machines']: I(" building gcc runtime for %s" % machine) try: self.bb.complete("gcc-runtime", machine) except Exception as e: E(" Can't build gcc-runtime for %s." % machine) if isinstance(e, Error): E(e.stdout) else: import traceback traceback.print_exc(file=sys.stdout) succeeded_pkgs_ctx = [] failed_pkgs_ctx = [] attempted_pkgs = 0 for pn, _, _ in pkgs_to_upgrade: pkg_ctx = pkgs_ctx[pn] pkg_ctx['error'] = None attempted_pkgs += 1 I(" ATTEMPT PACKAGE %d/%d" % (attempted_pkgs, total_pkgs)) try: I(" %s: Upgrading to %s" % (pkg_ctx['PN'], pkg_ctx['NPV'])) for step, msg in upgrade_steps: if msg is not None: I(" %s: %s" % (pkg_ctx['PN'], msg)) step(self.bb, self.git, self.opts, pkg_ctx) succeeded_pkgs_ctx.append(pkg_ctx) I(" %s: Upgrade SUCCESSFUL! Please test!" % pkg_ctx['PN']) except Exception as e: if isinstance(e, UpgradeNotNeededError): I(" %s: %s" % (pkg_ctx['PN'], e.message)) elif isinstance(e, UnsupportedProtocolError): I(" %s: %s" % (pkg_ctx['PN'], e.message)) else: if not isinstance(e, Error): import traceback msg = "Failed(unknown error)\n" + traceback.format_exc( ) e = Error(message=msg) error = e E(" %s: %s" % (pkg_ctx['PN'], e.message)) if os.listdir(pkg_ctx['workdir']): E(" %s: Upgrade FAILED! Logs and/or file diffs are available in %s" % (pkg_ctx['PN'], pkg_ctx['workdir'])) pkg_ctx['error'] = e failed_pkgs_ctx.append(pkg_ctx) try: self.commit_changes(pkg_ctx) except: if pkg_ctx in succeeded_pkgs_ctx: succeeded_pkgs_ctx.remove(pkg_ctx) failed_pkgs_ctx.append(pkg_ctx) if self.opts['testimage']: ctxs = {} ctxs['succeeded'] = succeeded_pkgs_ctx ctxs['failed'] = failed_pkgs_ctx image = settings.get('testimage_name', DEFAULT_TESTIMAGE) tim = TestImage(self.bb, self.git, self.uh_work_dir, self.opts, ctxs, image) tim.run() for pn in pkgs_ctx.keys(): pkg_ctx = pkgs_ctx[pn] if pkg_ctx in succeeded_pkgs_ctx: os.symlink(pkg_ctx['workdir'], os.path.join( \ self.uh_recipes_succeed_dir, pkg_ctx['PN'])) else: os.symlink(pkg_ctx['workdir'], os.path.join( \ self.uh_recipes_failed_dir, pkg_ctx['PN'])) self.statistics.update(pkg_ctx['PN'], pkg_ctx['NPV'], pkg_ctx['MAINTAINER'], pkg_ctx['error']) self.pkg_upgrade_handler(pkg_ctx) if attempted_pkgs > 0: publish_work_url = settings.get('publish_work_url', '') work_tarball = os.path.join( self.uh_base_work_dir, os.path.basename(self.uh_work_dir) + '.tar.gz') if publish_work_url: I(" Generating work tarball in %s ..." % work_tarball) import subprocess if subprocess.call( ["tar", "-chzf", work_tarball, self.uh_work_dir]): E(" Work tarball (%s) generation failed..." % (work_tarball)) publish_work_url = '' statistics_summary = self.statistics.get_summary( publish_work_url, os.path.basename(self.uh_work_dir)) statistics_file = os.path.join(self.uh_work_dir, "statistics_summary") with open(statistics_file, "w+") as f: f.write(statistics_summary) I(" %s" % statistics_summary) if self.opts['send_email']: self.send_status_mail(statistics_summary)
class Updater(object): def __init__(self, auto_mode=False, send_email=False, skip_compilation=False): build_dir = get_build_dir() self.bb = Bitbake(build_dir) self.devtool = Devtool() try: self.base_env = self.bb.env() except EmptyEnvError as e: import traceback E(" %s\n%s" % (e.message, traceback.format_exc())) E(" Bitbake output:\n%s" % (e.stdout)) exit(1) self._set_options(auto_mode, send_email, skip_compilation) self._make_dirs(build_dir) self._add_file_logger() self.email_handler = Email(settings) self.statistics = Statistics() def _set_options(self, auto_mode, send_email, skip_compilation): self.opts = {} self.opts['layer_mode'] = settings.get('layer_mode', '') if self.opts['layer_mode'] == 'yes': def _layer_settings_error(setting): E(" In layer mode enable you need to specify %s.\n" % setting) exit(1) layer_settings = ('layer_name', 'layer_dir', 'layer_machines') for s in layer_settings: self.opts[s] = settings.get(s, '') if not self.opts[s]: _layer_settings_error(s) self.git = Git(self.opts['layer_dir']) self.poky_git = Git( os.path.dirname(os.getenv('PATH', False).split(':')[0])) self.opts['machines'] = self.opts['layer_machines'].split() else: # XXX: assume that the poky directory is the first entry in the PATH self.git = Git( os.path.dirname(os.getenv('PATH', False).split(':')[0])) self.poky_git = None self.opts['machines'] = settings.get( 'machines', 'qemux86 qemux86-64 qemuarm qemumips qemuppc').split() self.opts['interactive'] = not auto_mode self.opts['send_email'] = send_email self.opts['author'] = "Upgrade Helper <%s>" % \ settings.get('from', '*****@*****.**') self.opts['skip_compilation'] = skip_compilation self.opts['buildhistory'] = self._buildhistory_is_enabled() self.opts['testimage'] = self._testimage_is_enabled() def _make_dirs(self, build_dir): self.uh_dir = os.path.join(build_dir, "upgrade-helper") if not os.path.exists(self.uh_dir): os.mkdir(self.uh_dir) self.uh_base_work_dir = settings.get('workdir', '') if not self.uh_base_work_dir: self.uh_base_work_dir = self.uh_dir if self.opts['layer_mode'] == 'yes': self.uh_base_work_dir = os.path.join(self.uh_base_work_dir, self.opts['layer_name']) if not os.path.exists(self.uh_base_work_dir): os.mkdir(self.uh_base_work_dir) self.uh_work_dir = os.path.join(self.uh_base_work_dir, "%s" % \ datetime.now().strftime("%Y%m%d%H%M%S")) os.mkdir(self.uh_work_dir) self.uh_recipes_all_dir = os.path.join(self.uh_work_dir, "all") os.mkdir(self.uh_recipes_all_dir) self.uh_recipes_succeed_dir = os.path.join(self.uh_work_dir, "succeed") os.mkdir(self.uh_recipes_succeed_dir) self.uh_recipes_failed_dir = os.path.join(self.uh_work_dir, "failed") os.mkdir(self.uh_recipes_failed_dir) def _add_file_logger(self): fh = log.FileHandler( os.path.join(self.uh_work_dir, "upgrade-helper.log")) logger = log.getLogger() logger.addHandler(fh) def _get_status_msg(self, err): if err: return str(err) else: return "Succeeded" def _buildhistory_is_enabled(self): enabled = False if settings.get("buildhistory", "no") == "yes": if 'buildhistory' in self.base_env['INHERIT']: if not 'BUILDHISTORY_COMMIT' in self.base_env: E(" Buildhistory was INHERIT in conf/local.conf"\ " but need BUILDHISTORY_COMMIT=1 please set.") exit(1) if not self.base_env['BUILDHISTORY_COMMIT'] == '1': E(" Buildhistory was INHERIT in conf/local.conf"\ " but need BUILDHISTORY_COMMIT=1 please set.") exit(1) if self.opts['skip_compilation']: W(" Buildhistory disabled because user" \ " skip compilation!") else: enabled = True else: E(" Buildhistory was enabled in upgrade-helper.conf"\ " but isn't INHERIT in conf/local.conf, if you want"\ " to enable please set.") exit(1) return enabled def _testimage_is_enabled(self): enabled = False if settings.get("testimage", "no") == "yes": if 'testimage' in self.base_env['INHERIT']: if not "ptest" in self.base_env["DISTRO_FEATURES"]: E(" testimage requires ptest in DISTRO_FEATURES please add to"\ " conf/local.conf.") exit(1) if not "package-management" in self.base_env[ 'EXTRA_IMAGE_FEATURES']: E(" testimage requires package-management in EXTRA_IMAGE_FEATURES"\ " please add to conf/local.conf.") exit(1) if not "ptest-pkgs" in self.base_env['EXTRA_IMAGE_FEATURES']: E(" testimage/ptest requires ptest-pkgs in EXTRA_IMAGE_FEATURES"\ " please add to conf/local.conf.") exit(1) if not "package_rpm" == self.base_env["PACKAGE_CLASSES"]: E(" testimage/ptest requires PACKAGE_CLASSES set to package_rpm"\ " please add to conf/local.conf.") exit(1) enabled = True else: E(" testimage was enabled in upgrade-helper.conf"\ " but isn't INHERIT in conf/local.conf, if you want"\ " to enable please set.") exit(1) return enabled def _get_packages_to_upgrade(self, packages=None): if packages is None: I("Nothing to upgrade") exit(0) else: return packages # this function will be called at the end of each recipe upgrade def pkg_upgrade_handler(self, pkg_ctx): mail_header = \ "Hello,\n\nYou are receiving this email because you are the maintainer\n" \ "of *%s* recipe and this is to let you know that the automatic attempt\n" \ "to upgrade the recipe to *%s* has %s.\n\n" license_change_info = \ "*LICENSE CHANGED* please review the %s file, update the LICENSE\n" \ "variable in the recipe and summarize the changes in the commit message.\n\n" next_steps_info = \ "Next steps:\n" \ " - apply the patch: git am %s\n" \ " - check the changes to upstream patches and summarize them in the commit message,\n" \ " - compile an image that contains the package\n" \ " - perform some basic sanity tests\n" \ " - amend the patch and sign it off: git commit -s --reset-author --amend\n" \ " - send it to the appropriate mailing list\n\n" \ "Alternatively, if you believe the recipe should not be upgraded at this time,\n" \ "you can fill RECIPE_NO_UPDATE_REASON in respective recipe file so that\n" \ "automatic upgrades would no longer be attempted.\n\n" testimage_integration_error = \ "The recipe *FAILED* in testimage integration. Attached is the log file.\n\n" testimage_ptest_info = \ "The recipe has ptest enabled and has been tested with core-image-minimal/ptest \n" \ "with the next machines %s. Attached is the log file.\n\n" testimage_info = \ "The recipe has been tested using %s testimage and succeeded with \n" \ "the next machines %s. Attached is the log file.\n\n" \ mail_footer = \ "Please review the attached files for further information and build/update failures.\n" \ "Any problem please file a bug at https://bugzilla.yoctoproject.org/enter_bug.cgi?product=Automated%20Update%20Handler\n\n" \ "Regards,\nThe Upgrade Helper" if pkg_ctx['MAINTAINER'] in maintainer_override: to_addr = maintainer_override[pkg_ctx['MAINTAINER']] else: to_addr = pkg_ctx['MAINTAINER'] cc_addr = None if "status_recipients" in settings: cc_addr = settings["status_recipients"].split() newversion = pkg_ctx['NPV'] if not pkg_ctx['NPV'].endswith( "new-commits-available") else pkg_ctx['NSRCREV'] subject = "[AUH] " + pkg_ctx['PN'] + ": upgrading to " + newversion if not pkg_ctx['error']: subject += " SUCCEEDED" else: subject += " FAILED" msg_body = mail_header % (pkg_ctx['PN'], newversion, self._get_status_msg(pkg_ctx['error'])) if pkg_ctx['error'] is not None: msg_body += """Detailed error information: %s %s %s """ % (pkg_ctx['error'].message if pkg_ctx['error'].message else "", pkg_ctx['error'].stdout if pkg_ctx['error'].stdout else "", pkg_ctx['error'].stderr if pkg_ctx['error'].stderr else "") if 'license_diff_fn' in pkg_ctx: license_diff_fn = pkg_ctx['license_diff_fn'] msg_body += license_change_info % license_diff_fn if 'patch_file' in pkg_ctx and pkg_ctx['patch_file'] != None: msg_body += next_steps_info % (os.path.basename( pkg_ctx['patch_file'])) if self.opts['testimage']: if 'integration_error' in pkg_ctx: msg_body += testimage_integration_error else: if 'ptest' in pkg_ctx: machines = pkg_ctx['ptest'].keys() msg_body += testimage_ptest_info % machines if 'testimage' in pkg_ctx: machines = pkg_ctx['testimage'].keys() msg_body += testimage_info % (settings.get('testimage_name', \ DEFAULT_TESTIMAGE), machines) msg_body += mail_footer # Add possible attachments to email attachments = [] for attachment in os.listdir(pkg_ctx['workdir']): attachment_fullpath = os.path.join(pkg_ctx['workdir'], attachment) if os.path.isfile(attachment_fullpath): attachments.append(attachment_fullpath) if self.opts['send_email']: self.email_handler.send_email(to_addr, subject, msg_body, attachments, cc_addr=cc_addr) # Preserve email for review purposes. email_file = os.path.join(pkg_ctx['workdir'], "email_summary") with open(email_file, "w+") as f: f.write("To: %s\n" % to_addr) if isinstance(cc_addr, list): f.write("To: %s\n" % ' '.join(cc_addr)) else: f.write("Cc: %s\n" % cc_addr) f.write("Subject: %s\n" % subject) f.write("Attachments: %s\n" % ' '.join(attachments)) f.write("\n%s\n" % msg_body) def commit_changes(self, pkg_ctx): try: pkg_ctx['patch_file'] = None I(" %s: Auto commit changes ..." % pkg_ctx['PN']) self.git.add(pkg_ctx['recipe_dir']) self.git.commit(pkg_ctx['commit_msg'], self.opts['author']) stdout = self.git.create_patch(pkg_ctx['workdir']) pkg_ctx['patch_file'] = stdout.strip() if not pkg_ctx['patch_file']: msg = "Patch file not generated." E(" %s: %s\n %s" % (pkg_ctx['PN'], msg, stdout)) raise Error(msg, stdout) else: I(" %s: Save patch in directory: %s." % (pkg_ctx['PN'], pkg_ctx['workdir'])) if pkg_ctx['error'] is not None: I("Due to build errors, the commit will also be reverted to avoid cascading upgrade failures." ) self.git.revert("HEAD") except Error as e: msg = '' for line in e.stdout.split("\n"): if line.find("nothing to commit") == 0: msg = "Nothing to commit!" I(" %s: %s" % (pkg_ctx['PN'], msg)) I(" %s: %s" % (pkg_ctx['PN'], e.stdout)) raise e def send_status_mail(self, statistics_summary): if "status_recipients" not in settings: E(" Could not send status email, no recipients set!") return -1 to_list = settings["status_recipients"].split() if self.opts['layer_mode'] == 'yes': subject = "[AUH] Upgrade status %s: %s" \ % (self.opts['layer_name'], date.isoformat(date.today())) else: subject = "[AUH] Upgrade status: " + date.isoformat(date.today()) if self.statistics.total_attempted: self.email_handler.send_email(to_list, subject, statistics_summary) else: W("No recipes attempted, not sending status mail!") def run(self, package_list=None): pkgs_to_upgrade = self._get_packages_to_upgrade(package_list) total_pkgs = len(pkgs_to_upgrade) pkgs_ctx = {} I(" ########### The list of recipes to be upgraded #############") for p, ov, nv, m, r in pkgs_to_upgrade: I(" %s, %s, %s, %s, %s" % (p, ov, nv, m, r)) pkgs_ctx[p] = {} pkgs_ctx[p]['PN'] = p pkgs_ctx[p]['PV'] = ov pkgs_ctx[p]['NPV'] = nv pkgs_ctx[p]['MAINTAINER'] = m pkgs_ctx[p]['NSRCREV'] = r pkgs_ctx[p]['base_dir'] = self.uh_recipes_all_dir I(" ############################################################") if pkgs_to_upgrade: I(" Building gcc runtimes ...") for machine in self.opts['machines']: I(" building gcc runtime for %s" % machine) try: self.bb.complete("gcc-runtime", machine) except Exception as e: E(" Can't build gcc-runtime for %s." % machine) if isinstance(e, Error): E(e.stdout) else: import traceback traceback.print_exc(file=sys.stdout) succeeded_pkgs_ctx = [] failed_pkgs_ctx = [] attempted_pkgs = 0 for pn, _, _, _, _ in pkgs_to_upgrade: pkg_ctx = pkgs_ctx[pn] pkg_ctx['error'] = None attempted_pkgs += 1 I(" ATTEMPT PACKAGE %d/%d" % (attempted_pkgs, total_pkgs)) try: I(" %s: Upgrading to %s" % (pkg_ctx['PN'], pkg_ctx['NPV'])) for step, msg in upgrade_steps: if msg is not None: I(" %s: %s" % (pkg_ctx['PN'], msg)) step(self.devtool, self.bb, self.git, self.opts, pkg_ctx) succeeded_pkgs_ctx.append(pkg_ctx) I(" %s: Upgrade SUCCESSFUL! Please test!" % pkg_ctx['PN']) except Exception as e: if isinstance(e, UpgradeNotNeededError): I(" %s: %s" % (pkg_ctx['PN'], e.message)) elif isinstance(e, UnsupportedProtocolError): I(" %s: %s" % (pkg_ctx['PN'], e.message)) else: if not isinstance(e, Error): import traceback msg = "Failed(unknown error)\n" + traceback.format_exc( ) e = Error(message=msg) error = e E(" %s: %s" % (pkg_ctx['PN'], e.message)) if 'workdir' in pkg_ctx and os.listdir(pkg_ctx['workdir']): E(" %s: Upgrade FAILED! Logs and/or file diffs are available in %s" % (pkg_ctx['PN'], pkg_ctx['workdir'])) pkg_ctx['error'] = e failed_pkgs_ctx.append(pkg_ctx) try: self.commit_changes(pkg_ctx) except: if pkg_ctx in succeeded_pkgs_ctx: succeeded_pkgs_ctx.remove(pkg_ctx) failed_pkgs_ctx.append(pkg_ctx) if self.opts['testimage']: ctxs = {} ctxs['succeeded'] = succeeded_pkgs_ctx ctxs['failed'] = failed_pkgs_ctx image = settings.get('testimage_name', DEFAULT_TESTIMAGE) tim = TestImage(self.bb, self.git, self.uh_work_dir, self.opts, ctxs, image) tim.run() for pn in pkgs_ctx.keys(): pkg_ctx = pkgs_ctx[pn] if pkg_ctx in succeeded_pkgs_ctx: os.symlink(pkg_ctx['workdir'], os.path.join( \ self.uh_recipes_succeed_dir, pkg_ctx['PN'])) else: os.symlink(pkg_ctx['workdir'], os.path.join( \ self.uh_recipes_failed_dir, pkg_ctx['PN'])) self.statistics.update(pkg_ctx['PN'], pkg_ctx['NPV'], pkg_ctx['MAINTAINER'], pkg_ctx['error']) self.pkg_upgrade_handler(pkg_ctx) if attempted_pkgs > 0: publish_work_url = settings.get('publish_work_url', '') work_tarball = os.path.join( self.uh_base_work_dir, os.path.basename(self.uh_work_dir) + '.tar.gz') if publish_work_url: I(" Generating work tarball in %s ..." % work_tarball) import subprocess if subprocess.call( ["tar", "-chzf", work_tarball, self.uh_work_dir]): E(" Work tarball (%s) generation failed..." % (work_tarball)) publish_work_url = '' statistics_summary = self.statistics.get_summary( publish_work_url, os.path.basename(self.uh_work_dir)) statistics_file = os.path.join(self.uh_work_dir, "statistics_summary") with open(statistics_file, "w+") as f: f.write(statistics_summary) I(" %s" % statistics_summary) if self.opts['send_email']: self.send_status_mail(statistics_summary)
class Updater(object): def __init__(self, auto_mode=False, send_email=False, skip_compilation=False): build_dir = get_build_dir() self.bb = Bitbake(build_dir) try: self.base_env = self.bb.env() except EmptyEnvError as e: import traceback E( " %s\n%s" % (e.message, traceback.format_exc())) E( " Bitbake output:\n%s" % (e.stdout)) exit(1) self._set_options(auto_mode, send_email, skip_compilation) self._make_dirs(build_dir) self._add_file_logger() self.email_handler = Email(settings) self.statistics = Statistics() def _set_options(self, auto_mode, send_email, skip_compilation): self.opts = {} self.opts['layer_mode'] = settings.get('layer_mode', '') if self.opts['layer_mode'] == 'yes': def _layer_settings_error(setting): E(" In layer mode enable you need to specify %s.\n" % setting) exit(1) layer_settings = ('layer_name', 'layer_dir', 'layer_machines') for s in layer_settings: self.opts[s] = settings.get(s, '') if not self.opts[s]: _layer_settings_error(s) self.git = Git(self.opts['layer_dir']) self.poky_git = Git(os.path.dirname(os.getenv('PATH', False).split(':')[0])) self.opts['machines'] = self.opts['layer_machines'].split() else: # XXX: assume that the poky directory is the first entry in the PATH self.git = Git(os.path.dirname(os.getenv('PATH', False).split(':')[0])) self.poky_git = None self.opts['machines'] = settings.get('machines', 'qemux86 qemux86-64 qemuarm qemumips qemuppc').split() self.opts['interactive'] = not auto_mode self.opts['send_email'] = send_email self.opts['author'] = "Upgrade Helper <%s>" % \ settings.get('from', '*****@*****.**') self.opts['skip_compilation'] = skip_compilation self.opts['buildhistory'] = self._buildhistory_is_enabled() self.opts['testimage'] = self._testimage_is_enabled() def _make_dirs(self, build_dir): self.uh_dir = os.path.join(build_dir, "upgrade-helper") if not os.path.exists(self.uh_dir): os.mkdir(self.uh_dir) self.uh_base_work_dir = settings.get('workdir', '') if not self.uh_base_work_dir: self.uh_base_work_dir = self.uh_dir if self.opts['layer_mode'] == 'yes': self.uh_base_work_dir = os.path.join(self.uh_base_work_dir, self.opts['layer_name']) if not os.path.exists(self.uh_base_work_dir): os.mkdir(self.uh_base_work_dir) self.uh_work_dir = os.path.join(self.uh_base_work_dir, "%s" % \ datetime.now().strftime("%Y%m%d%H%M%S")) os.mkdir(self.uh_work_dir) self.uh_recipes_all_dir = os.path.join(self.uh_work_dir, "all") os.mkdir(self.uh_recipes_all_dir) self.uh_recipes_succeed_dir = os.path.join(self.uh_work_dir, "succeed") os.mkdir(self.uh_recipes_succeed_dir) self.uh_recipes_failed_dir = os.path.join(self.uh_work_dir, "failed") os.mkdir(self.uh_recipes_failed_dir) def _add_file_logger(self): fh = log.FileHandler(os.path.join(self.uh_work_dir, "upgrade-helper.log")) logger = log.getLogger() logger.addHandler(fh) def _get_status_msg(self, err): if err: return str(err) else: return "Succeeded" def _buildhistory_is_enabled(self): enabled = False if settings.get("buildhistory", "no") == "yes": if 'buildhistory' in self.base_env['INHERIT']: if not 'BUILDHISTORY_COMMIT' in self.base_env: E(" Buildhistory was INHERIT in conf/local.conf"\ " but need BUILDHISTORY_COMMIT=1 please set.") exit(1) if not self.base_env['BUILDHISTORY_COMMIT'] == '1': E(" Buildhistory was INHERIT in conf/local.conf"\ " but need BUILDHISTORY_COMMIT=1 please set.") exit(1) if self.opts['skip_compilation']: W(" Buildhistory disabled because user" \ " skip compilation!") else: enabled = True else: E(" Buildhistory was enabled in upgrade-helper.conf"\ " but isn't INHERIT in conf/local.conf, if you want"\ " to enable please set.") exit(1) else: if 'buildhistory' in self.base_env['INHERIT']: E(" Buildhistory was INHERIT in conf/local.conf"\ " but buildhistory=yes isn't in upgrade-helper.conf,"\ " if you want to enable please set.") exit(1) return enabled def _testimage_is_enabled(self): enabled = False if settings.get("testimage", "no") == "yes": if 'testimage' in self.base_env['INHERIT']: if not "ptest" in self.base_env["DISTRO_FEATURES"]: E(" testimage requires ptest in DISTRO_FEATURES please add to"\ " conf/local.conf.") exit(1) if not "package-management" in self.base_env['EXTRA_IMAGE_FEATURES']: E(" testimage requires package-management in EXTRA_IMAGE_FEATURES"\ " please add to conf/local.conf.") exit(1) if not "ptest-pkgs" in self.base_env['EXTRA_IMAGE_FEATURES']: E(" testimage/ptest requires ptest-pkgs in EXTRA_IMAGE_FEATURES"\ " please add to conf/local.conf.") exit(1) if not "package_rpm" == self.base_env["PACKAGE_CLASSES"]: E(" testimage/ptest requires PACKAGE_CLASSES set to package_rpm"\ " please add to conf/local.conf.") exit(1) enabled = True else: E(" testimage was enabled in upgrade-helper.conf"\ " but isn't INHERIT in conf/local.conf, if you want"\ " to enable please set.") exit(1) else: if 'testimage' in self.base_env['INHERIT']: E(" testimage was INHERIT in conf/local.conf"\ " but testimage=yes isn't in upgrade-helper.conf,"\ " if you want to enable please set.") exit(1) return enabled def _get_packages_to_upgrade(self, packages=None): if packages is None: I( "Nothing to upgrade") exit(0) else: return packages # this function will be called at the end of each recipe upgrade def pkg_upgrade_handler(self, pkg_ctx): mail_header = \ "Hello,\n\nYou are receiving this email because you are the maintainer\n" \ "of *%s* recipe and this is to let you know that the automatic attempt\n" \ "to upgrade the recipe to *%s* has %s.\n\n" license_change_info = \ "*LICENSE CHANGED* please review the %s file and update the LICENSE\n" \ "variable in the recipe if is needed.\n\n" next_steps_info = \ "The recipe has been successfully compiled for machines %s.\n\n" \ "Next steps:\n" \ " - apply the patch: git am %s\n" \ " - check that required upstream patches have not been commented from the recipe,\n" \ " if upstream patches were commented the reason is specified in the commit message.\n" \ " - compile an image that contains the package\n" \ " - perform some basic sanity tests\n" \ " - amend the patch and sign it off: git commit -s --reset-author --amend\n" \ " - send it to the list\n\n" \ testimage_integration_error = \ "The recipe *FAILED* in testimage integration. Attached is the log file.\n\n" testimage_ptest_info = \ "The recipe has ptest enabled and has been tested with core-image-minimal/ptest \n" \ "with the next machines %s. Attached is the log file.\n\n" testimage_info = \ "The recipe has been tested using %s testimage and succeeded with \n" \ "the next machines %s. Attached is the log file.\n\n" \ mail_footer = \ "Attached are the patch, license diff (if change) and bitbake log.\n" \ "Any problem please contact Anibal Limon <*****@*****.**>.\n\n" \ "Regards,\nThe Upgrade Helper" if pkg_ctx['MAINTAINER'] in maintainer_override: to_addr = maintainer_override[pkg_ctx['MAINTAINER']] else: to_addr = pkg_ctx['MAINTAINER'] cc_addr = None if "status_recipients" in settings: cc_addr = settings["status_recipients"].split() subject = "[AUH] " + pkg_ctx['PN'] + ": upgrading to " + pkg_ctx['NPV'] if not pkg_ctx['error']: subject += " SUCCEEDED" else: subject += " FAILED" msg_body = mail_header % (pkg_ctx['PN'], pkg_ctx['NPV'], self._get_status_msg(pkg_ctx['error'])) if 'recipe' in pkg_ctx: license_diff_fn = pkg_ctx['recipe'].get_license_diff_file_name() if license_diff_fn: msg_body += license_change_info % license_diff_fn if not pkg_ctx['error']: msg_body += next_steps_info % (', '.join(self.opts['machines']), os.path.basename(pkg_ctx['patch_file'])) if self.opts['testimage']: if 'integration_error' in pkg_ctx: msg_body += testimage_integration_error else: if 'ptest' in pkg_ctx: machines = pkg_ctx['ptest'].keys() msg_body += testimage_ptest_info % machines if 'testimage' in pkg_ctx: machines = pkg_ctx['testimage'].keys() msg_body += testimage_info % (settings.get('testimage_name', \ DEFAULT_TESTIMAGE), machines) msg_body += mail_footer # Add possible attachments to email attachments = [] for attachment in os.listdir(pkg_ctx['workdir']): attachment_fullpath = os.path.join(pkg_ctx['workdir'], attachment) if os.path.isfile(attachment_fullpath): attachments.append(attachment_fullpath) # Only send email to Maintainer when recipe upgrade succeed. if self.opts['send_email'] and not pkg_ctx['error']: self.email_handler.send_email(to_addr, subject, msg_body, attachments, cc_addr=cc_addr) # Preserve email for review purposes. email_file = os.path.join(pkg_ctx['workdir'], "email_summary") with open(email_file, "w+") as f: f.write("To: %s\n" % to_addr) if isinstance(cc_addr, list): f.write("To: %s\n" % ' '.join(cc_addr)) else: f.write("Cc: %s\n" % cc_addr) f.write("Subject: %s\n" % subject) f.write("Attachments: %s\n" % ' '.join(attachments)) f.write("\n%s\n" % msg_body) def commit_changes(self, pkg_ctx): fail = False try: pkg_ctx['patch_file'] = None if 'recipe' in pkg_ctx: I(" %s: Auto commit changes ..." % pkg_ctx['PN']) self.git.commit(pkg_ctx['recipe'].commit_msg, self.opts['author']) stdout = self.git.create_patch(pkg_ctx['workdir']) pkg_ctx['patch_file'] = stdout.strip() if not pkg_ctx['patch_file']: msg = "Patch file not generated." E(" %s: %s\n %s" % (pkg_ctx['PN'], msg, stdout)) pkg_ctx['error'] = Error(msg, stdout) fail = True else: I(" %s: Save patch in directory: %s." % (pkg_ctx['PN'], pkg_ctx['workdir'])) except Error as e: msg = '' for line in e.stdout.split("\n"): if line.find("nothing to commit") == 0: msg = "Nothing to commit!" I(" %s: %s" % (pkg_ctx['PN'], msg)) I(" %s: %s" % (pkg_ctx['PN'], e.stdout)) pkg_ctx['error'] = Error(msg, e.stdout) fail = True if fail: raise pkg_ctx['error'] def send_status_mail(self, statistics_summary): if "status_recipients" not in settings: E(" Could not send status email, no recipients set!") return -1 to_list = settings["status_recipients"].split() if self.opts['layer_mode'] == 'yes': subject = "[AUH] Upgrade status %s: %s" \ % (self.opts['layer_name'], date.isoformat(date.today())) else: subject = "[AUH] Upgrade status: " + date.isoformat(date.today()) if self.statistics.total_attempted: self.email_handler.send_email(to_list, subject, statistics_summary) else: W("No recipes attempted, not sending status mail!") def _order_pkgs_to_upgrade(self, pkgs_to_upgrade): def _get_pn_dep_dic(pn_list, dependency_file): import re pn_dep_dic = {} with open(dependency_file) as dep: data = dep.read() dep.close() for line in data.split('\n'): m = re.search('^"(.*)" -> "(.*)"$', line) if not m: continue pn = m.group(1) pn_dep = m.group(2) if pn == pn_dep: continue if pn in pn_list: if pn_dep in pn_list: if pn in pn_dep_dic.keys(): pn_dep_dic[pn].append(pn_dep) else: pn_dep_dic[pn] = [pn_dep] elif not pn in pn_dep_dic.keys(): pn_dep_dic[pn] = [] return pn_dep_dic def _dep_resolve(graph, node, resolved, seen): seen.append(node) for edge in graph[node]: if edge not in resolved: if edge in seen: raise RuntimeError("Packages %s and %s have " \ "a circular dependency." \ % (node, edge)) _dep_resolve(graph, edge, resolved, seen) resolved.append(node) pn_list = [] for pn, new_ver, maintainer in pkgs_to_upgrade: pn_list.append(pn) try: self.bb.dependency_graph(' '.join(pn_list)) except Error as e: multiple_providers = False for l in e.stdout.split('\n'): if l.find("ERROR: Multiple .bb files are due to be built which each provide") == 0: multiple_providers = True if not multiple_providers: raise e dependency_file = os.path.join(get_build_dir(), "pn-depends.dot") pkgs_to_upgrade_ordered = [] pn_list_ordered = [] pn_dep_dic = _get_pn_dep_dic(pn_list, dependency_file) if pn_dep_dic: root = "__root_node__" pn_dep_dic[root] = pn_dep_dic.keys() _dep_resolve(pn_dep_dic, root, pn_list_ordered, []) pn_list_ordered.remove(root) for pn_ordered in pn_list_ordered: for pn, new_ver, maintainer in pkgs_to_upgrade: if pn == pn_ordered: pkgs_to_upgrade_ordered.append([pn, new_ver, maintainer]) return pkgs_to_upgrade_ordered def run(self, package_list=None): #pkgs_to_upgrade = self._order_pkgs_to_upgrade( # self._get_packages_to_upgrade(package_list)) pkgs_to_upgrade = self._get_packages_to_upgrade(package_list) total_pkgs = len(pkgs_to_upgrade) pkgs_ctx = {} I(" ########### The list of recipes to be upgraded #############") for p, v, m in pkgs_to_upgrade: I(" %s, %s, %s" % (p, v, m)) pkgs_ctx[p] = {} pkgs_ctx[p]['PN'] = p pkgs_ctx[p]['NPV'] = v pkgs_ctx[p]['MAINTAINER'] = m pkgs_ctx[p]['base_dir'] = self.uh_recipes_all_dir I(" ############################################################") if pkgs_to_upgrade: I(" Building gcc runtimes ...") for machine in self.opts['machines']: I(" building gcc runtime for %s" % machine) try: self.bb.complete("gcc-runtime", machine) except Exception as e: E(" Can't build gcc-runtime for %s." % machine) if isinstance(e, Error): E(e.stdout) else: import traceback traceback.print_exc(file=sys.stdout) succeeded_pkgs_ctx = [] failed_pkgs_ctx = [] attempted_pkgs = 0 for pn, _, _ in pkgs_to_upgrade: pkg_ctx = pkgs_ctx[pn] pkg_ctx['error'] = None attempted_pkgs += 1 I(" ATTEMPT PACKAGE %d/%d" % (attempted_pkgs, total_pkgs)) try: I(" %s: Upgrading to %s" % (pkg_ctx['PN'], pkg_ctx['NPV'])) for step, msg in upgrade_steps: if msg is not None: I(" %s: %s" % (pkg_ctx['PN'], msg)) step(self.bb, self.git, self.opts, pkg_ctx) succeeded_pkgs_ctx.append(pkg_ctx) I(" %s: Upgrade SUCCESSFUL! Please test!" % pkg_ctx['PN']) except Exception as e: if isinstance(e, UpgradeNotNeededError): I(" %s: %s" % (pkg_ctx['PN'], e.message)) elif isinstance(e, UnsupportedProtocolError): I(" %s: %s" % (pkg_ctx['PN'], e.message)) else: if not isinstance(e, Error): import traceback msg = "Failed(unknown error)\n" + traceback.format_exc() e = Error(message=msg) error = e E(" %s: %s" % (pkg_ctx['PN'], e.message)) if os.listdir(pkg_ctx['workdir']): E(" %s: Upgrade FAILED! Logs and/or file diffs are available in %s" % (pkg_ctx['PN'], pkg_ctx['workdir'])) pkg_ctx['error'] = e failed_pkgs_ctx.append(pkg_ctx) try: self.commit_changes(pkg_ctx) except: if pkg_ctx in succeeded_pkgs_ctx: succeeded_pkgs_ctx.remove(pkg_ctx) failed_pkgs_ctx.append(pkg_ctx) if self.opts['testimage']: ctxs = {} ctxs['succeeded'] = succeeded_pkgs_ctx ctxs['failed'] = failed_pkgs_ctx image = settings.get('testimage_name', DEFAULT_TESTIMAGE) tim = TestImage(self.bb, self.git, self.uh_work_dir, self.opts, ctxs, image) tim.run() for pn in pkgs_ctx.keys(): pkg_ctx = pkgs_ctx[pn] if pkg_ctx in succeeded_pkgs_ctx: os.symlink(pkg_ctx['workdir'], os.path.join( \ self.uh_recipes_succeed_dir, pkg_ctx['PN'])) else: os.symlink(pkg_ctx['workdir'], os.path.join( \ self.uh_recipes_failed_dir, pkg_ctx['PN'])) self.statistics.update(pkg_ctx['PN'], pkg_ctx['NPV'], pkg_ctx['MAINTAINER'], pkg_ctx['error']) self.pkg_upgrade_handler(pkg_ctx) if attempted_pkgs > 0: publish_work_url = settings.get('publish_work_url', '') work_tarball = os.path.join(self.uh_base_work_dir, os.path.basename(self.uh_work_dir) + '.tar.gz') if publish_work_url: I(" Generating work tarball in %s ..." % work_tarball) import subprocess if subprocess.call(["tar", "-chzf", work_tarball, self.uh_work_dir]): E(" Work tarball (%s) generation failed..." % (work_tarball)) publish_work_url = '' statistics_summary = self.statistics.get_summary( publish_work_url, os.path.basename(self.uh_work_dir)) statistics_file = os.path.join(self.uh_work_dir, "statistics_summary") with open(statistics_file, "w+") as f: f.write(statistics_summary) I(" %s" % statistics_summary) if self.opts['send_email']: self.send_status_mail(statistics_summary)