def create_duplicate_email_report(data): """Create the email report for duplicated trigger emails. When we receive multiple triggers for the same job-kernel, send an error email. Only a TXT one will be created. :param data: The data for the email. :type dict :return The txt and html body, and the subject string. """ subject_str = MULTIPLE_EMAILS_SUBJECT.format(**data) txt_body = rcommon.create_txt_email("multiple_emails.txt", **data) return txt_body, None, subject_str
def _create_build_email(**kwargs): """Parse the results and create the email text body to send. :param job: The name of the job. :type job: str :param kernel: The name of the kernel. :type kernel: str :param git_commit: The git commit. :type git_commit: str :param git_url: The git url. :type git_url: str :param git_branch: The git branch. :type git_branch: str :param failed_data: The parsed failed results. :type failed_data: dict :param fail_count: The total number of failed results. :type fail_count: int :param total_count: The total number of results. :type total_count: int :param total_unique_data: The unique values data structure. :type total_unique_data: dictionary :param pass_count: The total number of passed results. :type pass_count: int :param base_url: The base URL to build the dashboard links. :type base_url: string :param boot_url: The base URL for the boot section of the dashboard. :type boot_url: string :param build_url: The base URL for the build section of the dashboard. :type build_url: string :param info_email: The email address for the footer note. :type info_email: string :return A tuple with the email body and subject as strings. """ txt_body = None html_body = None subject_str = None k_get = kwargs.get email_format = k_get("email_format") total_unique_data = k_get("total_unique_data", None) failed_data = k_get("failed_data", None) error_data = k_get("error_data", None) subject_str = _get_build_subject_string(**kwargs) built_unique_one = G_(u"Built: {:s}") built_unique_string = None if total_unique_data: unique_archs = rcommon.count_unique(total_unique_data.get( "arch", None)) kwargs["unique_archs"] = unique_archs arch_str = P_(u"{unique_archs:d} unique architecture", u"{unique_archs:d} unique architectures", unique_archs) if unique_archs > 0: built_unique_string = built_unique_one.format(arch_str) if built_unique_string: built_unique_string = built_unique_string.format(**kwargs) build_summary_url = BUILD_SUMMARY_URL.format(**kwargs) kwargs["built_unique_string"] = built_unique_string kwargs["tree_string"] = G_(u"Tree: {job:s}").format(**kwargs) kwargs["branch_string"] = G_(u"Branch: {git_branch:s}").format(**kwargs) kwargs["git_describe_string"] = G_(u"Git Describe: {kernel:s}").format( **kwargs) kwargs["subject_str"] = subject_str git_url = k_get("git_url") git_commit = k_get("git_commit") translated_git_url = \ rcommon.translate_git_url(git_url, git_commit) or git_url git_txt_string = G_(u"Git URL: {:s}").format(git_url) git_html_string = G_(u"Git URL: <a href=\"{:s}\">{:s}</a>").format( translated_git_url, git_url) kwargs["git_commit_string"] = G_(u"Git Commit: {:s}").format(git_commit) kwargs["git_url_string"] = (git_txt_string, git_html_string) if failed_data or error_data: kwargs["platforms"] = _parse_and_structure_results(**kwargs) if models.EMAIL_TXT_FORMAT_KEY in email_format: kwargs["full_build_summary"] = ( G_(u"Full Build Summary: {:s}").format(build_summary_url)) txt_body = rcommon.create_txt_email("build.txt", **kwargs) if models.EMAIL_HTML_FORMAT_KEY in email_format: # Fix the summary URLs for the HTML email. kwargs["full_build_summary"] = ( G_(u"Full Build Summary: <a href=\"{url:s}\">{url:s}</a>").format( **{"url": build_summary_url})) html_body = rcommon.create_html_email("build.html", **kwargs) return txt_body, html_body, subject_str
def _create_boot_email(boot_data): """Parse the results and create the email text body to send. :param boot_data: Details about the boot results :type boot_data: dict :return A tuple with the email body and subject as strings. """ txt_body = None html_body = None subject_str = None b_get = boot_data.get total_unique_data = b_get("total_unique_data", None) info_email = b_get("info_email", None) email_format = b_get("email_format") subject_str = get_boot_subject_string(boot_data) tested_one = G_(u"Tested: {:s}") tested_two = G_(u"Tested: {:s}, {:s}") tested_three = G_(u"Tested: {:s}, {:s}, {:s}") tested_string = None if total_unique_data: unique_boards = rcommon.count_unique( total_unique_data.get(models.BOARD_KEY, None)) unique_socs = rcommon.count_unique( total_unique_data.get(models.MACH_KEY, None)) unique_builds = rcommon.count_unique( total_unique_data[models.DEFCONFIG_FULL_KEY]) boot_data["unique_boards"] = unique_boards boot_data["unique_socs"] = unique_socs boot_data["unique_builds"] = unique_builds boards_str = P_( u"{unique_boards:d} unique board", u"{unique_boards:d} unique boards", unique_boards ) soc_str = P_( u"{unique_socs:d} SoC family", u"{unique_socs:d} SoC families", unique_socs ) builds_str = P_( u"{unique_builds:d} build out of {total_builds:d}", u"{unique_builds:d} builds out of {total_builds:d}", unique_builds ) if unique_boards > 0 and unique_socs > 0 and unique_builds > 0: tested_string = tested_three.format( boards_str, soc_str, builds_str) elif unique_boards > 0 and unique_socs > 0 and unique_builds == 0: tested_string = tested_two.format(boards_str, soc_str) elif unique_boards > 0 and unique_socs == 0 and unique_builds > 0: tested_string = tested_two.format(boards_str, builds_str) elif unique_boards == 0 and unique_socs > 0 and unique_builds > 0: tested_string = tested_two.format(soc_str, builds_str) elif unique_boards > 0 and unique_socs == 0 and unique_builds == 0: tested_string = tested_one.format(boards_str) elif unique_boards == 0 and unique_socs > 0 and unique_builds == 0: tested_string = tested_one.format(soc_str) elif unique_boards == 0 and unique_socs == 0 and unique_builds > 0: tested_string = tested_one.format(builds_str) if tested_string: tested_string = tested_string.format(**boot_data) boot_summary_url = rcommon.BOOT_SUMMARY_URL.format(**boot_data) build_summary_url = rcommon.BUILD_SUMMARY_URL.format(**boot_data) boot_data["tree_string"] = G_(u"Tree: {job:s}").format(**boot_data) boot_data["branch_string"] = G_(u"Branch: {git_branch:s}").format( **boot_data) boot_data["git_describe_string"] = G_(u"Git Describe: {kernel:s}").format( **boot_data) boot_data["info_email"] = info_email boot_data["tested_string"] = tested_string boot_data["subject_str"] = subject_str git_url = b_get("git_url") git_commit = b_get("git_commit") translated_git_url = \ rcommon.translate_git_url(git_url, git_commit) or git_url git_txt_string = G_(u"Git URL: {:s}").format(git_url) git_html_string = G_(u"Git URL: <a href=\"{:s}\">{:s}</a>").format( translated_git_url, git_url) boot_data["git_commit_string"] = G_(u"Git Commit: {:s}").format(git_commit) boot_data["git_url_string"] = (git_txt_string, git_html_string) boot_data["platforms"] = _parse_and_structure_results(boot_data) if models.EMAIL_TXT_FORMAT_KEY in email_format: boot_data["full_boot_summary"] = ( G_(u"Full Boot Summary: {:s}").format(boot_summary_url)) boot_data["full_build_summary"] = ( G_(u"Full Build Summary: {:s}").format(build_summary_url)) txt_body = rcommon.create_txt_email("boot.txt", **boot_data) if models.EMAIL_HTML_FORMAT_KEY in email_format: # Fix the summary URLs for the HTML email. boot_data["full_boot_summary"] = ( G_(u"Full Boot Summary: <a href=\"{url:s}\">{url:s}</a>").format( **{"url": boot_summary_url}) ) boot_data["full_build_summary"] = ( G_(u"Full Build Summary: <a href=\"{url:s}\">{url:s}</a>").format( **{"url": build_summary_url}) ) html_body = rcommon.create_html_email("boot.html", **boot_data) return txt_body, html_body, subject_str
def create_bisect_report(data, email_options, db_options, base_path=utils.BASE_PATH): """Create the bisection report email to be sent. :param data: The meta-data for the bisection job. :type data: dictionary :param email_options: The email options. :type email_options: dict :param db_options: The mongodb database connection parameters. :type db_options: dict :param base_path: Path to the top-level storage directory. :type base_path: string :return A tuple with the TXT email body, the HTML email body and the headers as dictionary. If an error occured, None. """ database = utils.db.get_db_connection(db_options) job, branch, kernel, test_suite, lab, target = (data[k] for k in [ models.JOB_KEY, models.GIT_BRANCH_KEY, models.KERNEL_KEY, models.TYPE_KEY, models.LAB_NAME_KEY, models.DEVICE_TYPE_KEY, ]) email_format, email_subject = (email_options[k] for k in [ "format", "subject", ]) specs = { x: data[x] for x in [ models.TYPE_KEY, models.ARCHITECTURE_KEY, models.DEFCONFIG_FULL_KEY, models.JOB_KEY, models.GIT_BRANCH_KEY, models.LAB_NAME_KEY, models.DEVICE_TYPE_KEY, models.BISECT_GOOD_COMMIT_KEY, models.BISECT_BAD_COMMIT_KEY, ] } doc = utils.db.find_one2(database[models.BISECT_COLLECTION], specs) if not doc: utils.LOG.warning("Failed to find bisection document") return None headers = { rcommon.X_REPORT: rcommon.BISECT_REPORT_TYPE, rcommon.X_BRANCH: branch, rcommon.X_TREE: job, rcommon.X_KERNEL: kernel, rcommon.X_LAB: lab, } rel_path = '/'.join((job, branch, kernel) + tuple(data[k] for k in [ models.ARCHITECTURE_KEY, models.DEFCONFIG_FULL_KEY, models.LAB_NAME_KEY, ])) log_path = os.path.join(base_path, rel_path, data[models.BISECT_LOG_KEY]) with open(log_path) as log_file: log_data = json.load(log_file) url_params = { 'boot_url': rcommon.DEFAULT_BOOT_URL, 'job': job, 'git_branch': branch, } boot_data = {b["status"]: b for b in doc[models.BISECT_DATA_KEY]} bad_describe = boot_data["FAIL"]["git_describe"] bad_details_url = '/'.join([ rcommon.DEFAULT_BASE_URL, "boot", "id", str(boot_data["FAIL"]["_id"]) ]) log_url_txt, log_url_html = ('/'.join([ rcommon.DEFAULT_STORAGE_URL, rel_path, boot_data["FAIL"][k] ]) for k in [models.BOOT_LOG_KEY, models.BOOT_LOG_HTML_KEY]) template_data = { "subject_str": email_subject, "bad": doc[models.BISECT_BAD_SUMMARY_KEY], "bad_details_url": bad_details_url, "bad_describe": bad_describe, "log_url_txt": log_url_txt, "log_url_html": log_url_html, "found": doc[models.BISECT_FOUND_SUMMARY_KEY], "checks": doc[models.BISECT_CHECKS_KEY], "tree": job, "git_url": doc[models.GIT_URL_KEY], "branch": branch, "target": doc[models.DEVICE_TYPE_KEY], "arch": doc[models.ARCHITECTURE_KEY], "lab_name": lab, "defconfig": doc[models.DEFCONFIG_FULL_KEY], "test_suite": test_suite, "show": log_data["show"], "log": log_data["log"], } if models.EMAIL_TXT_FORMAT_KEY in email_format: txt_body = rcommon.create_txt_email("bisect.txt", **template_data) else: txt_body = None if models.EMAIL_HTML_FORMAT_KEY in email_format: html_body = rcommon.create_html_email("bisect.html", **template_data) else: html_body = None return txt_body, html_body, headers
def create_test_report(db_options, data, email_format, email_template=None, base_path=utils.BASE_PATH): """Create the tests report email to be sent. :param db_options: The mongodb database connection parameters. :type db_options: dict :param data: The meta-data for the test job. :type data: dictionary :param email_format: The email format to send. :type email_format: list :param email_template: A specific email template to use. :type email_template: str :param base_path: Path to the top-level storage directory. :type base_path: string :return A tuple with the email body, the email subject and the headers as dictionary. If an error occured, None. """ database = utils.db.get_db_connection(db_options) job, branch, kernel, plan = (data[k] for k in [ models.JOB_KEY, models.GIT_BRANCH_KEY, models.KERNEL_KEY, models.PLAN_KEY, ]) template = TEMPLATES.get(email_template or plan, {}) spec = {x: y for x, y in data.iteritems() if x != models.PLAN_KEY} group_spec = dict(spec) group_spec.update({ models.NAME_KEY: plan, models.PARENT_ID_KEY: None, }) groups = list( utils.db.find(database[models.TEST_GROUP_COLLECTION], spec=group_spec, fields=TEST_REPORT_FIELDS, sort=[ (models.BOARD_KEY, pymongo.ASCENDING), (models.BUILD_ENVIRONMENT_KEY, pymongo.ASCENDING), (models.DEFCONFIG_KEY, pymongo.ASCENDING), (models.LAB_NAME_KEY, pymongo.ASCENDING), (models.ARCHITECTURE_KEY, pymongo.ASCENDING), ])) if not groups: utils.LOG.warning("Failed to find test group documents") return None for group in groups: group_spec = dict(spec) group_spec.update({ k: group[k] for k in [ models.DEVICE_TYPE_KEY, models.ARCHITECTURE_KEY, models.BUILD_ENVIRONMENT_KEY, models.DEFCONFIG_FULL_KEY, ] }) _add_test_group_data(group, database, group_spec) tests_total = sum(group["total_tests"] for group in groups) regr_total = sum(group["total_regr"] for group in groups) plan_subject = template.get("subject", plan) subject_str = "{}/{} {}: {} runs, {} regressions ({})".format( job, branch, plan_subject, len(groups), regr_total, kernel) git_url, git_commit = ( groups[0][k] for k in [models.GIT_URL_KEY, models.GIT_COMMIT_KEY]) # Add test suites info if it's the same for all the groups (typical case) keys = set() last_test_suites = None for g in groups: info = g[models.INITRD_INFO_KEY] if not info: continue last_test_suites = info.get('tests_suites') if not last_test_suites: continue keys.add( tuple((ts['name'], ts['git_commit']) for ts in last_test_suites)) test_suites = list(last_test_suites) if len(keys) == 1 else None totals = { status: sum(g['total_results'][status] for g in groups) for status in ["PASS", "FAIL", "SKIP"] } headers = { rcommon.X_REPORT: rcommon.TEST_REPORT_TYPE, rcommon.X_BRANCH: branch, rcommon.X_TREE: job, rcommon.X_KERNEL: kernel, } summary_headers, summaries = _create_summaries(groups) for group, summary in zip(groups, summaries): group['summary'] = summary template_data = { "subject_str": subject_str, "summary_headers": summary_headers, "tree": job, "branch": branch, "git_url": git_url, "kernel": kernel, "git_commit": git_commit, "plan": plan, "boot_log": models.BOOT_LOG_KEY, "boot_log_html": models.BOOT_LOG_HTML_KEY, "storage_url": rcommon.DEFAULT_STORAGE_URL, "test_groups": groups, "test_suites": test_suites, "totals": totals, } template_file = template.get("file", "test.txt") template_data.update(template.get("params", {})) body = rcommon.create_txt_email(template_file, **template_data) return body, subject_str, headers
def create_bisect_report(data, email_options, db_options, base_path=utils.BASE_PATH): """Create the bisection report email to be sent. :param data: The meta-data for the bisection job. :type data: dictionary :param email_options: The email options. :type email_options: dict :param db_options: The mongodb database connection parameters. :type db_options: dict :param base_path: Path to the top-level storage directory. :type base_path: string :return A tuple with the TXT email body and the headers as dictionary. If an error occured, None. """ db = utils.db.get_db_connection(db_options) job, branch, kernel, test_case_path, lab, target = (data[k] for k in [ models.JOB_KEY, models.GIT_BRANCH_KEY, models.KERNEL_KEY, models.TEST_CASE_PATH_KEY, models.LAB_NAME_KEY, models.DEVICE_TYPE_KEY, ]) email_format, email_subject = (email_options[k] for k in [ "format", "subject", ]) specs = { x: data[x] for x in [ models.TYPE_KEY, models.ARCHITECTURE_KEY, models.DEFCONFIG_FULL_KEY, models.BUILD_ENVIRONMENT_KEY, models.JOB_KEY, models.KERNEL_KEY, models.GIT_BRANCH_KEY, models.LAB_NAME_KEY, models.DEVICE_TYPE_KEY, models.BISECT_GOOD_COMMIT_KEY, models.BISECT_BAD_COMMIT_KEY, models.TEST_CASE_PATH_KEY, ] } doc = utils.db.find_one2(db[models.BISECT_COLLECTION], specs) if not doc: utils.LOG.warning("Failed to find bisection document") return None report_hashable_str = "-".join( str(x) for x in [ doc[models.BISECT_FOUND_SUMMARY_KEY], doc[models.KERNEL_KEY], ]) report_hash = hashlib.sha1(report_hashable_str).hexdigest() redisdb_conn = redisdb.get_db_connection(db_options) if redisdb_conn.exists(report_hash): utils.LOG.info("Bisection report already sent for {}: {}".format( doc[models.KERNEL_KEY], doc[models.BISECT_FOUND_SUMMARY_KEY])) return None redisdb_conn.set(report_hash, "bisection-report", ex=86400) headers = { rcommon.X_REPORT: rcommon.BISECT_REPORT_TYPE, rcommon.X_BRANCH: branch, rcommon.X_TREE: job, rcommon.X_KERNEL: kernel, rcommon.X_LAB: lab, } rel_path = '/'.join((job, branch, kernel) + tuple(data[k] for k in [ models.ARCHITECTURE_KEY, models.DEFCONFIG_FULL_KEY, models.BUILD_ENVIRONMENT_KEY, models.LAB_NAME_KEY, ])) log_path = os.path.join(base_path, rel_path, data[models.BISECT_LOG_KEY]) with open(log_path) as log_file: log_data = json.load(log_file) regr = utils.db.find_one2(db[models.TEST_REGRESSION_COLLECTION], doc[models.REGRESSION_ID_KEY]) test_case = utils.db.find_one2( db[models.TEST_CASE_COLLECTION], regr[models.REGRESSIONS_KEY][-1][models.TEST_CASE_ID_KEY]) test_group = utils.db.find_one2(db[models.TEST_GROUP_COLLECTION], test_case[models.TEST_GROUP_ID_KEY]) # Disabled until we have a working Tests view on the frontend # bad_details_url = '/'.join([ # rcommon.DEFAULT_BASE_URL, "boot", "id", str(boot_data["FAIL"]["_id"])]) log_url_txt, log_url_html = ('/'.join([ rcommon.DEFAULT_STORAGE_URL, rel_path, test_group[k] ]) for k in [models.BOOT_LOG_KEY, models.BOOT_LOG_HTML_KEY]) cc = doc[models.COMPILER_KEY] cc_ver = doc[models.COMPILER_VERSION_KEY] compiler_str = "-".join([cc, cc_ver]) if cc_ver else cc template_data = { "subject_str": email_subject, "bad": doc[models.BISECT_BAD_SUMMARY_KEY], # "bad_details_url": bad_details_url, "log_url_txt": log_url_txt, "log_url_html": log_url_html, "found": doc[models.BISECT_FOUND_SUMMARY_KEY], "checks": doc[models.BISECT_CHECKS_KEY], "tree": job, "git_url": doc[models.GIT_URL_KEY], "branch": branch, "target": doc[models.DEVICE_TYPE_KEY], "arch": doc[models.ARCHITECTURE_KEY], "lab_name": lab, "defconfig": doc[models.DEFCONFIG_FULL_KEY], "compiler": compiler_str, "test_case_path": doc[models.TEST_CASE_PATH_KEY], "show": log_data["show"], "log": log_data["log"], } body = rcommon.create_txt_email("bisect.txt", **template_data) return body, headers
def create_test_report(data, email_format, db_options, base_path=utils.BASE_PATH): """Create the tests report email to be sent. :param data: The meta-data for the test job. :type data: dictionary :param email_format: The email format to send. :type email_format: list :param db_options: The mongodb database connection parameters. :type db_options: dict :param base_path: Path to the top-level storage directory. :type base_path: string :return A tuple with the email body, the email subject and the headers as dictionary. If an error occured, None. """ database = utils.db.get_db_connection(db_options) job, branch, kernel, plan = (data[k] for k in [ models.JOB_KEY, models.GIT_BRANCH_KEY, models.KERNEL_KEY, models.PLAN_KEY, ]) plan_options = TEST_PLAN_OPTIONS.get(plan, {}) spec = {x: y for x, y in data.iteritems() if x != models.PLAN_KEY} group_spec = dict(spec) group_spec.update({ models.NAME_KEY: plan, models.PARENT_ID_KEY: None, }) groups = list( utils.db.find(database[models.TEST_GROUP_COLLECTION], spec=group_spec, fields=TEST_REPORT_FIELDS)) if not groups: utils.LOG.warning("Failed to find test group documents") return None for group in groups: group_spec = dict(spec) group_spec.update({ k: group[k] for k in [ models.DEVICE_TYPE_KEY, models.ARCHITECTURE_KEY, models.BUILD_ENVIRONMENT_KEY, models.DEFCONFIG_FULL_KEY, ] }) _add_test_group_data(group, database, group_spec) tests_total = sum(group["total_tests"] for group in groups) regr_total = sum(group["regressions"] for group in groups) plan_subject = plan_options.get("subject", plan) subject_str = "{}/{} {}: {} tests, {} regressions ({})".format( job, branch, plan_subject, tests_total, regr_total, kernel) git_url, git_commit = ( groups[0][k] for k in [models.GIT_URL_KEY, models.GIT_COMMIT_KEY]) headers = { rcommon.X_REPORT: rcommon.TEST_REPORT_TYPE, rcommon.X_BRANCH: branch, rcommon.X_TREE: job, rcommon.X_KERNEL: kernel, } template_data = { "subject_str": subject_str, "tree": job, "branch": branch, "git_url": git_url, "kernel": kernel, "git_commit": git_commit, "plan": plan, "boot_log": models.BOOT_LOG_KEY, "boot_log_html": models.BOOT_LOG_HTML_KEY, "storage_url": rcommon.DEFAULT_STORAGE_URL, "test_groups": groups, } template = plan_options.get("template", "test.txt") template_data.update(plan_options.get("params", {})) body = rcommon.create_txt_email(template, **template_data) return body, subject_str, headers
def create_test_report(data, email_format, db_options, base_path=utils.BASE_PATH): """Create the tests report email to be sent. :param data: The meta-data for the test job. :type data: dictionary :param email_format: The email format to send. :type email_format: list :param db_options: The mongodb database connection parameters. :type db_options: dict :param base_path: Path to the top-level storage directory. :type base_path: string :return A tuple with the email body, the email subject and the headers as dictionary. If an error occured, None. """ database = utils.db.get_db_connection(db_options) job, branch, kernel, plan = (data[k] for k in [ models.JOB_KEY, models.GIT_BRANCH_KEY, models.KERNEL_KEY, models.PLAN_KEY, ]) plan_options = TEST_PLAN_OPTIONS.get(plan, {}) spec = {x: y for x, y in data.iteritems() if x != models.PLAN_KEY} group_spec = dict(spec) group_spec.update({ models.NAME_KEY: plan, models.PARENT_ID_KEY: None, }) groups = list(utils.db.find( database[models.TEST_GROUP_COLLECTION], spec=group_spec, fields=TEST_REPORT_FIELDS)) if not groups: utils.LOG.warning("Failed to find test group documents") return None for group in groups: group_spec = dict(spec) group_spec.update({ k: group[k] for k in [ models.DEVICE_TYPE_KEY, models.ARCHITECTURE_KEY, models.BUILD_ENVIRONMENT_KEY, models.DEFCONFIG_FULL_KEY, ] }) _add_test_group_data(group, database, group_spec) tests_total = sum(group["total_tests"] for group in groups) regr_total = sum(group["regressions"] for group in groups) plan_subject = plan_options.get("subject", plan) subject_str = "{}/{} {}: {} tests, {} regressions ({})".format( job, branch, plan_subject, tests_total, regr_total, kernel) git_url, git_commit = (groups[0][k] for k in [ models.GIT_URL_KEY, models.GIT_COMMIT_KEY]) headers = { rcommon.X_REPORT: rcommon.TEST_REPORT_TYPE, rcommon.X_BRANCH: branch, rcommon.X_TREE: job, rcommon.X_KERNEL: kernel, } template_data = { "subject_str": subject_str, "tree": job, "branch": branch, "git_url": git_url, "kernel": kernel, "git_commit": git_commit, "plan": plan, "boot_log": models.BOOT_LOG_KEY, "boot_log_html": models.BOOT_LOG_HTML_KEY, "storage_url": rcommon.DEFAULT_STORAGE_URL, "test_groups": groups, } template = plan_options.get("template", "test.txt") template_data.update(plan_options.get("params", {})) body = rcommon.create_txt_email(template, **template_data) return body, subject_str, headers
def create_bisect_report(data, email_format, db_options, base_path=utils.BASE_PATH): """Create the bisection report email to be sent. :param data: The meta-data for the bisection job. :type data: dictionary :param email_format: The email format to send. :type email_format: list :param db_options: The mongodb database connection parameters. :type db_options: dict :param base_path: Path to the top-level storage directory. :type base_path: string :return A tuple with the TXT email body, the HTML email body and the headers as dictionary. If an error occured, None. """ database = utils.db.get_db_connection(db_options) job, branch, kernel, lab, target = (data[k] for k in [ models.JOB_KEY, models.GIT_BRANCH_KEY, models.KERNEL_KEY, models.LAB_NAME_KEY, models.DEVICE_TYPE_KEY, ]) specs = {x: data[x] for x in [ models.TYPE_KEY, models.ARCHITECTURE_KEY, models.DEFCONFIG_FULL_KEY, models.JOB_KEY, models.GIT_BRANCH_KEY, models.LAB_NAME_KEY, models.DEVICE_TYPE_KEY, models.BISECT_GOOD_COMMIT_KEY, models.BISECT_BAD_COMMIT_KEY, ]} doc = utils.db.find_one2(database[models.BISECT_COLLECTION], specs) if not doc: utils.LOG.warning("Failed to find bisection document") return None headers = { rcommon.X_REPORT: rcommon.BISECT_REPORT_TYPE, rcommon.X_BRANCH: branch, rcommon.X_TREE: job, rcommon.X_KERNEL: kernel, rcommon.X_LAB: lab, } subject_str = "Bisection result for {}/{} ({}) on {}".format( job, branch, kernel, target) log_path_elements = (base_path, job, branch, kernel) + tuple( data[k] for k in [ models.ARCHITECTURE_KEY, models.DEFCONFIG_FULL_KEY, models.LAB_NAME_KEY, models.BISECT_LOG_KEY, ] ) log_path = os.path.join(*log_path_elements) with open(log_path) as log_file: log_data = json.load(log_file) template_data = { "subject_str": subject_str, "good": doc[models.BISECT_GOOD_SUMMARY_KEY], "bad": doc[models.BISECT_BAD_SUMMARY_KEY], "found": doc[models.BISECT_FOUND_SUMMARY_KEY], "checks": doc[models.BISECT_CHECKS_KEY], "tree": job, "git_url": doc[models.GIT_URL_KEY], "branch": branch, "target": doc[models.DEVICE_TYPE_KEY], "lab_name": lab, "defconfig": doc[models.DEFCONFIG_FULL_KEY], "plan": "boot", "show": log_data["show"], "log": log_data["log"], } if models.EMAIL_TXT_FORMAT_KEY in email_format: txt_body = rcommon.create_txt_email("bisect.txt", **template_data) else: txt_body = None if models.EMAIL_HTML_FORMAT_KEY in email_format: html_body = rcommon.create_html_email("bisect.html", **template_data) else: html_body = None return txt_body, html_body, headers
def create_test_report(data, email_format, db_options, base_path=utils.BASE_PATH): """Create the tests report email to be sent. :param data: The meta-data for the test job. :type data: dictionary :param email_format: The email format to send. :type email_format: list :param db_options: The mongodb database connection parameters. :type db_options: dict :param base_path: Path to the top-level storage directory. :type base_path: string :return A tuple with the TXT email body, the HTML email body and the headers as dictionary. If an error occured, None. """ database = utils.db.get_db_connection(db_options) job, branch, kernel, plans = (data[k] for k in [ models.JOB_KEY, models.GIT_BRANCH_KEY, models.KERNEL_KEY, models.PLANS_KEY ]) # Avoid using the field "plans" when fetching the documents # from mongodb del data['plans'] specs = {x: data[x] for x in data.keys() if data[x]} test_group_docs = list( utils.db.find(database[models.TEST_GROUP_COLLECTION], spec=specs, fields=TEST_REPORT_FIELDS)) top_groups = [] sub_group_ids = [] for group in test_group_docs: sub_group_ids.extend(group[models.SUB_GROUPS_KEY]) top_groups = [] for group in test_group_docs: if group["_id"] not in sub_group_ids and \ group["name"] != "lava" and \ not plans: top_groups.append(group) elif plans and group["name"] in plans: top_groups.append(group) if not top_groups: utils.LOG.warning("Failed to find test group documents") return None for group in top_groups: _add_test_group_data(group, database) if not plans: plans_string = "All the results are included" subject_str = "Test results for {}/{} - {}".format(job, branch, kernel) else: plans_string = ", ".join(plans) subject_str = "Test results ({}) for {}/{} - {}".format( plans_string, job, branch, kernel) git_url, git_commit = ( top_groups[0][k] for k in [models.GIT_URL_KEY, models.GIT_COMMIT_KEY]) headers = { rcommon.X_REPORT: rcommon.TEST_REPORT_TYPE, rcommon.X_BRANCH: branch, rcommon.X_TREE: job, rcommon.X_KERNEL: kernel, } template_data = { "subject_str": subject_str, "tree": job, "branch": branch, "git_url": git_url, "kernel": kernel, "git_commit": git_commit, "plans_string": plans_string, "boot_log": models.BOOT_LOG_KEY, "boot_log_html": models.BOOT_LOG_HTML_KEY, "storage_url": rcommon.DEFAULT_STORAGE_URL, "test_groups": top_groups, } if models.EMAIL_TXT_FORMAT_KEY in email_format: txt_body = rcommon.create_txt_email("test.txt", **template_data) else: txt_body = None if models.EMAIL_HTML_FORMAT_KEY in email_format: html_body = rcommon.create_html_email("test.html", **template_data) else: html_body = None return txt_body, html_body, subject_str, headers
def _create_boot_email(**kwargs): """Parse the results and create the email text body to send. :param job: The name of the job. :type job: str :param kernel: The name of the kernel. :type kernel: str :param git_commit: The git commit. :type git_commit: str :param git_url: The git url. :type git_url: str :param git_branch: The git branch. :type git_branch: str :param lab_name: The name of the lab. :type lab_name: str :param failed_data: The parsed failed results. :type failed_data: dict :param fail_count: The total number of failed results. :type fail_count: int :param offline_data: The parsed offline results. :type offline_data: dict :param offline_count: The total number of offline results. :type offline_count: int :param total_count: The total number of results. :type total_count: int :param total_unique_data: The unique values data structure. :type total_unique_data: dictionary :param pass_count: The total number of passed results. :type pass_count: int :param conflict_data: The parsed conflicting results. :type conflict_data: dict :param conflict_count: The number of conflicting results. :type conflict_count: int :param total_builds: The total number of defconfig built. :type total_builds: int :param base_url: The base URL to build the dashboard links. :type base_url: string :param boot_url: The base URL for the boot section of the dashboard. :type boot_url: string :param build_url: The base URL for the build section of the dashboard. :type build_url: string :param info_email: The email address for the footer note. :type info_email: string :return A tuple with the email body and subject as strings. """ txt_body = None html_body = None subject_str = None k_get = kwargs.get total_unique_data = k_get("total_unique_data", None) info_email = k_get("info_email", None) email_format = k_get("email_format") subject_str = _get_boot_subject_string(**kwargs) tested_one = G_(u"Tested: {:s}") tested_two = G_(u"Tested: {:s}, {:s}") tested_three = G_(u"Tested: {:s}, {:s}, {:s}") tested_string = None if total_unique_data: unique_boards = rcommon.count_unique( total_unique_data.get(models.BOARD_KEY, None)) unique_socs = rcommon.count_unique( total_unique_data.get(models.MACH_KEY, None)) unique_builds = rcommon.count_unique( total_unique_data[models.DEFCONFIG_FULL_KEY]) kwargs["unique_boards"] = unique_boards kwargs["unique_socs"] = unique_socs kwargs["unique_builds"] = unique_builds boards_str = P_( u"{unique_boards:d} unique board", u"{unique_boards:d} unique boards", unique_boards ) soc_str = P_( u"{unique_socs:d} SoC family", u"{unique_socs:d} SoC families", unique_socs ) builds_str = P_( u"{unique_builds:d} build out of {total_builds:d}", u"{unique_builds:d} builds out of {total_builds:d}", unique_builds ) if all([unique_boards > 0, unique_socs > 0, unique_builds > 0]): tested_string = tested_three.format( boards_str, soc_str, builds_str) elif all([unique_boards > 0, unique_socs > 0, unique_builds == 0]): tested_string = tested_two.format(boards_str, soc_str) elif all([unique_boards > 0, unique_socs == 0, unique_builds > 0]): tested_string = tested_two.format(boards_str, builds_str) elif all([unique_boards == 0, unique_socs > 0, unique_builds > 0]): tested_string = tested_two.format(soc_str, builds_str) elif all([unique_boards > 0, unique_socs == 0, unique_builds == 0]): tested_string = tested_one.format(boards_str) elif all([unique_boards == 0, unique_socs > 0, unique_builds == 0]): tested_string = tested_one.format(soc_str) elif all([unique_boards == 0, unique_socs == 0, unique_builds > 0]): tested_string = tested_one.format(builds_str) if tested_string: tested_string = tested_string.format(**kwargs) boot_summary_url = BOOT_SUMMARY_URL.format(**kwargs) build_summary_url = BUILD_SUMMARY_URL.format(**kwargs) kwargs["tree_string"] = G_(u"Tree: {job:s}").format(**kwargs) kwargs["branch_string"] = G_(u"Branch: {git_branch:s}").format(**kwargs) kwargs["git_describe_string"] = G_(u"Git Describe: {kernel:s}").format( **kwargs) kwargs["info_email"] = info_email kwargs["tested_string"] = tested_string kwargs["subject_str"] = subject_str git_url = k_get("git_url") git_commit = k_get("git_commit") translated_git_url = \ rcommon.translate_git_url(git_url, git_commit) or git_url git_txt_string = G_(u"Git URL: {:s}").format(git_url) git_html_string = G_(u"Git URL: <a href=\"{:s}\">{:s}</a>").format( translated_git_url, git_url) kwargs["git_commit_string"] = G_(u"Git Commit: {:s}").format(git_commit) kwargs["git_url_string"] = (git_txt_string, git_html_string) kwargs["platforms"] = _parse_and_structure_results(**kwargs) if models.EMAIL_TXT_FORMAT_KEY in email_format: kwargs["full_boot_summary"] = ( G_(u"Full Boot Summary: {:s}").format(boot_summary_url)) kwargs["full_build_summary"] = ( G_(u"Full Build Summary: {:s}").format(build_summary_url)) txt_body = rcommon.create_txt_email("boot.txt", **kwargs) if models.EMAIL_HTML_FORMAT_KEY in email_format: # Fix the summary URLs for the HTML email. kwargs["full_boot_summary"] = ( G_(u"Full Boot Summary: <a href=\"{url:s}\">{url:s}</a>").format( **{"url": boot_summary_url}) ) kwargs["full_build_summary"] = ( G_(u"Full Build Summary: <a href=\"{url:s}\">{url:s}</a>").format( **{"url": build_summary_url}) ) html_body = rcommon.create_html_email("boot.html", **kwargs) return txt_body, html_body, subject_str
def _create_build_email(**kwargs): """Parse the results and create the email text body to send. :param job: The name of the job. :type job: str :param kernel: The name of the kernel. :type kernel: str :param git_commit: The git commit. :type git_commit: str :param git_url: The git url. :type git_url: str :param git_branch: The git branch. :type git_branch: str :param failed_data: The parsed failed results. :type failed_data: dict :param fail_count: The total number of failed results. :type fail_count: int :param total_count: The total number of results. :type total_count: int :param total_unique_data: The unique values data structure. :type total_unique_data: dictionary :param pass_count: The total number of passed results. :type pass_count: int :param base_url: The base URL to build the dashboard links. :type base_url: string :param boot_url: The base URL for the boot section of the dashboard. :type boot_url: string :param build_url: The base URL for the build section of the dashboard. :type build_url: string :param info_email: The email address for the footer note. :type info_email: string :return A tuple with the email body and subject as strings. """ txt_body = None html_body = None subject_str = None k_get = kwargs.get email_format = k_get("email_format") total_unique_data = k_get("total_unique_data", None) failed_data = k_get("failed_data", None) error_data = k_get("error_data", None) subject_str = _get_build_subject_string(**kwargs) built_unique_one = G_(u"Built: {:s}") built_unique_string = None if total_unique_data: unique_archs = rcommon.count_unique( total_unique_data.get("arch", None)) kwargs["unique_archs"] = unique_archs arch_str = P_( u"{unique_archs:d} unique architecture", u"{unique_archs:d} unique architectures", unique_archs ) if unique_archs > 0: built_unique_string = built_unique_one.format(arch_str) if built_unique_string: built_unique_string = built_unique_string.format(**kwargs) build_summary_url = u"{build_url:s}/{job:s}/kernel/{kernel:s}/".format( **kwargs) kwargs["built_unique_string"] = built_unique_string kwargs["tree_string"] = G_(u"Tree: {job:s}").format(**kwargs) kwargs["branch_string"] = G_(u"Branch: {git_branch:s}").format(**kwargs) kwargs["git_describe_string"] = G_(u"Git Describe: {kernel:s}").format( **kwargs) kwargs["subject_str"] = subject_str git_url = k_get("git_url") git_commit = k_get("git_commit") translated_git_url = \ rcommon.translate_git_url(git_url, git_commit) or git_url git_txt_string = G_(u"Git URL: {:s}").format(git_url) git_html_string = G_(u"Git URL: <a href=\"{:s}\">{:s}</a>").format( translated_git_url, git_url) kwargs["git_commit_string"] = G_(u"Git Commit: {:s}").format(git_commit) kwargs["git_url_string"] = (git_txt_string, git_html_string) if any([failed_data, error_data]): kwargs["platforms"] = _parse_and_structure_results(**kwargs) if models.EMAIL_TXT_FORMAT_KEY in email_format: kwargs["full_build_summary"] = ( G_(u"Full Build Summary: {:s}").format(build_summary_url)) txt_body = rcommon.create_txt_email("build.txt", **kwargs) if models.EMAIL_HTML_FORMAT_KEY in email_format: # Fix the summary URLs for the HTML email. kwargs["full_build_summary"] = ( G_(u"Full Build Summary: <a href=\"{url:s}\">{url:s}</a>").format( **{"url": build_summary_url})) html_body = rcommon.create_html_email("build.html", **kwargs) # utils.LOG.info(kwargs) return txt_body, html_body, subject_str
def _create_boot_email(**kwargs): """Parse the results and create the email text body to send. :param job: The name of the job. :type job: str :param kernel: The name of the kernel. :type kernel: str :param git_commit: The git commit. :type git_commit: str :param git_url: The git url. :type git_url: str :param git_branch: The git branch. :type git_branch: str :param lab_name: The name of the lab. :type lab_name: str :param failed_data: The parsed failed results. :type failed_data: dict :param fail_count: The total number of failed results. :type fail_count: int :param offline_data: The parsed offline results. :type offline_data: dict :param offline_count: The total number of offline results. :type offline_count: int :param total_count: The total number of results. :type total_count: int :param total_unique_data: The unique values data structure. :type total_unique_data: dictionary :param pass_count: The total number of passed results. :type pass_count: int :param conflict_data: The parsed conflicting results. :type conflict_data: dict :param conflict_count: The number of conflicting results. :type conflict_count: int :param total_builds: The total number of defconfig built. :type total_builds: int :param base_url: The base URL to build the dashboard links. :type base_url: string :param boot_url: The base URL for the boot section of the dashboard. :type boot_url: string :param build_url: The base URL for the build section of the dashboard. :type build_url: string :param info_email: The email address for the footer note. :type info_email: string :return A tuple with the email body and subject as strings. """ txt_body = None html_body = None subject_str = None k_get = kwargs.get total_unique_data = k_get("total_unique_data", None) info_email = k_get("info_email", None) email_format = k_get("email_format") subject_str = get_boot_subject_string(**kwargs) tested_one = G_(u"Tested: {:s}") tested_two = G_(u"Tested: {:s}, {:s}") tested_three = G_(u"Tested: {:s}, {:s}, {:s}") tested_string = None if total_unique_data: unique_boards = rcommon.count_unique( total_unique_data.get(models.BOARD_KEY, None)) unique_socs = rcommon.count_unique( total_unique_data.get(models.MACH_KEY, None)) unique_builds = rcommon.count_unique( total_unique_data[models.DEFCONFIG_FULL_KEY]) kwargs["unique_boards"] = unique_boards kwargs["unique_socs"] = unique_socs kwargs["unique_builds"] = unique_builds boards_str = P_(u"{unique_boards:d} unique board", u"{unique_boards:d} unique boards", unique_boards) soc_str = P_(u"{unique_socs:d} SoC family", u"{unique_socs:d} SoC families", unique_socs) builds_str = P_(u"{unique_builds:d} build out of {total_builds:d}", u"{unique_builds:d} builds out of {total_builds:d}", unique_builds) if unique_boards > 0 and unique_socs > 0 and unique_builds > 0: tested_string = tested_three.format(boards_str, soc_str, builds_str) elif unique_boards > 0 and unique_socs > 0 and unique_builds == 0: tested_string = tested_two.format(boards_str, soc_str) elif unique_boards > 0 and unique_socs == 0 and unique_builds > 0: tested_string = tested_two.format(boards_str, builds_str) elif unique_boards == 0 and unique_socs > 0 and unique_builds > 0: tested_string = tested_two.format(soc_str, builds_str) elif unique_boards > 0 and unique_socs == 0 and unique_builds == 0: tested_string = tested_one.format(boards_str) elif unique_boards == 0 and unique_socs > 0 and unique_builds == 0: tested_string = tested_one.format(soc_str) elif unique_boards == 0 and unique_socs == 0 and unique_builds > 0: tested_string = tested_one.format(builds_str) if tested_string: tested_string = tested_string.format(**kwargs) boot_summary_url = BOOT_SUMMARY_URL.format(**kwargs) build_summary_url = BUILD_SUMMARY_URL.format(**kwargs) kwargs["tree_string"] = G_(u"Tree: {job:s}").format(**kwargs) kwargs["branch_string"] = G_(u"Branch: {git_branch:s}").format(**kwargs) kwargs["git_describe_string"] = G_(u"Git Describe: {kernel:s}").format( **kwargs) kwargs["info_email"] = info_email kwargs["tested_string"] = tested_string kwargs["subject_str"] = subject_str git_url = k_get("git_url") git_commit = k_get("git_commit") translated_git_url = \ rcommon.translate_git_url(git_url, git_commit) or git_url git_txt_string = G_(u"Git URL: {:s}").format(git_url) git_html_string = G_(u"Git URL: <a href=\"{:s}\">{:s}</a>").format( translated_git_url, git_url) kwargs["git_commit_string"] = G_(u"Git Commit: {:s}").format(git_commit) kwargs["git_url_string"] = (git_txt_string, git_html_string) kwargs["platforms"] = _parse_and_structure_results(**kwargs) if kwargs["regressions"]: kwargs["regressions"] = \ parse_regressions( kwargs["regressions"][models.REGRESSIONS_KEY], **kwargs) else: kwargs["regressions"] = None if models.EMAIL_TXT_FORMAT_KEY in email_format: kwargs["full_boot_summary"] = ( G_(u"Full Boot Summary: {:s}").format(boot_summary_url)) kwargs["full_build_summary"] = ( G_(u"Full Build Summary: {:s}").format(build_summary_url)) txt_body = rcommon.create_txt_email("boot.txt", **kwargs) if models.EMAIL_HTML_FORMAT_KEY in email_format: # Fix the summary URLs for the HTML email. kwargs["full_boot_summary"] = ( G_(u"Full Boot Summary: <a href=\"{url:s}\">{url:s}</a>").format( **{"url": boot_summary_url})) kwargs["full_build_summary"] = ( G_(u"Full Build Summary: <a href=\"{url:s}\">{url:s}</a>").format( **{"url": build_summary_url})) html_body = rcommon.create_html_email("boot.html", **kwargs) return txt_body, html_body, subject_str