def create_build_report(job,
branch,
kernel,
email_format,
db_options,
mail_options=None):
"""Create the build report email to be sent.
:param job: The name of the job.
:type job: str
:param kernel: The name of the kernel.
:type kernel: str
:param email_format: The email format to send.
:type email_format: list
:param db_options: The mongodb database connection parameters.
:type db_options: dict
:param mail_options: The options necessary to connect to the SMTP server.
:type mail_options: dict
:return A tuple with the email body and subject as strings or None.
"""
kwargs = {}
txt_body = None
html_body = None
subject = None
# This is used to provide a footer note in the email report.
info_email = None
fail_count = total_count = 0
errors_count = warnings_count = 0
fail_results = []
if mail_options:
info_email = mail_options.get("info_email", None)
spec = {
models.JOB_KEY: job,
models.GIT_BRANCH_KEY: branch,
models.KERNEL_KEY: kernel
}
database = utils.db.get_db_connection(db_options)
total_results, total_count = utils.db.find_and_count(
database[models.BUILD_COLLECTION],
0,
0,
spec=spec,
fields=BUILD_SEARCH_FIELDS)
total_unique_data = rcommon.get_unique_data(
total_results.clone(), unique_keys=[models.ARCHITECTURE_KEY])
spec[models.STATUS_KEY] = models.FAIL_STATUS
fail_results, fail_count = utils.db.find_and_count(
database[models.BUILD_COLLECTION],
0,
0,
spec=spec,
fields=BUILD_SEARCH_FIELDS,
sort=BUILD_SEARCH_SORT)
failed_data = _parse_build_data(fail_results.clone())
# Retrieve the parsed errors/warnings/mismatches summary and then
# the details.
errors_spec = {
models.JOB_KEY: job,
models.GIT_BRANCH_KEY: branch,
models.KERNEL_KEY: kernel
}
errors_summary = utils.db.find_one2(
database[models.ERRORS_SUMMARY_COLLECTION],
errors_spec,
fields=[models.ERRORS_KEY, models.WARNINGS_KEY, models.MISMATCHES_KEY])
error_details = utils.db.find(database[models.ERROR_LOGS_COLLECTION],
0,
0,
spec=errors_spec,
sort=[(models.DEFCONFIG_FULL_KEY, 1)])
error_details = [d for d in error_details.clone()]
err_data, errors_count, warnings_count = _get_errors_count(error_details)
kwargs = {
"base_url": rcommon.DEFAULT_BASE_URL,
"build_url": rcommon.DEFAULT_BUILD_URL,
"email_format": email_format,
"error_data": err_data,
"error_details": error_details,
"errors_count": errors_count,
"errors_summary": errors_summary,
"fail_count": fail_count,
"failed_data": failed_data,
"info_email": info_email,
"pass_count": total_count - fail_count,
"storage_url": rcommon.DEFAULT_STORAGE_URL,
"total_count": total_count,
"total_unique_data": total_unique_data,
"warnings_count": warnings_count,
"git_branch": branch,
models.JOB_KEY: job,
models.KERNEL_KEY: kernel,
}
kwargs["git_commit"], kwargs["git_url"] = \
rcommon.get_git_data(job, branch, kernel, db_options)
custom_headers = {
rcommon.X_REPORT: rcommon.BUILD_REPORT_TYPE,
rcommon.X_BRANCH: branch,
rcommon.X_TREE: job,
rcommon.X_KERNEL: kernel,
}
if all([fail_count == 0, total_count == 0]):
utils.LOG.warn(
"Nothing found for '%s-%s-%s': no build email report sent", job,
branch, kernel)
else:
txt_body, html_body, subject = _create_build_email(**kwargs)
return txt_body, html_body, subject, custom_headers
def _parse_boot_results(results, intersect_results=None, get_unique=False):
"""Parse the boot results from the database creating a new data structure.
This is done to provide a simpler data structure to create the email
body.
If `get_unique` is True, it will return also a dictionary with unique
values found in the passed `results` for `arch`, `board` and
`defconfig_full` keys.
:param results: The boot results to parse.
:type results: `pymongo.cursor.Cursor` or a list of dict
:param get_unique: Return the unique values in the data structure. Default
to False.
:type get_unique: bool
:param intersect_results: The boot results to remove intersecting items.
:type intersect_results: dict
:return A tuple with the parsed data as dictionary, a tuple of data as
a dictionary that has had intersecting entries removed or None, the number
of intersections found or 0, and the unique data or None.
"""
parsed_data = {}
unique_data = None
intersections = 0
if get_unique:
unique_data = rcommon.get_unique_data(results)
for result in results:
res_get = result.get
lab_name = res_get(models.LAB_NAME_KEY)
board = res_get(models.BOARD_KEY)
arch = res_get(models.ARCHITECTURE_KEY)
defconfig = res_get(models.DEFCONFIG_FULL_KEY)
status = res_get(models.STATUS_KEY)
build_env = res_get(models.BUILD_ENVIRONMENT_KEY)
result_struct = {
arch: {
defconfig: {
build_env: {
board: {
lab_name: status
}
}
}
}
}
# Check if the current result intersects the other interect_results
if intersect_results:
arch_view = intersect_results.viewkeys()
if arch in arch_view:
defconfig_view = intersect_results[arch].viewkeys()
if defconfig in defconfig_view:
build_env_view = \
intersect_results[arch][defconfig].viewkeys()
irad = intersect_results[arch][defconfig]
if build_env in build_env_view:
if irad[build_env].get(board, None):
intersections += 1
del irad[build_env][board]
# Clean up also the remainder of the data structure
# so that we really have cleaned up data.
if not irad[build_env]:
del irad[build_env]
if not intersect_results[arch][defconfig]:
del intersect_results[arch][defconfig]
if not intersect_results[arch]:
del intersect_results[arch]
if arch in parsed_data:
if defconfig in parsed_data[arch]:
if build_env in parsed_data[arch][defconfig]:
if board in parsed_data[arch][defconfig][build_env]:
pdad = parsed_data[arch][defconfig]
pdadb = pdad[build_env]
rsad = result_struct[arch][defconfig]
if lab_name not in pdadb[board]:
pdadb[board][lab_name] = \
rsad[build_env][board][lab_name]
else:
parsed_data[arch][defconfig][build_env][board] = \
result_struct[arch][defconfig][build_env][board]
else:
parsed_data[arch][defconfig][build_env] = \
result_struct[arch][defconfig][build_env]
else:
parsed_data[arch][defconfig] = result_struct[arch][defconfig]
else:
parsed_data[arch] = result_struct[arch]
return parsed_data, intersect_results, intersections, unique_data
def _parse_boot_results(results, intersect_results=None, get_unique=False):
"""Parse the boot results from the database creating a new data structure.
This is done to provide a simpler data structure to create the email
body.
If `get_unique` is True, it will return also a dictionary with unique
values found in the passed `results` for `arch`, `board` and
`defconfig_full` keys.
:param results: The boot results to parse.
:type results: `pymongo.cursor.Cursor` or a list of dict
:param get_unique: Return the unique values in the data structure. Default
to False.
:type get_unique: bool
:param intersect_results: The boot results to remove intersecting items.
:type intersect_results: dict
:return A tuple with the parsed data as dictionary, a tuple of data as
a dictionary that has had intersecting entries removed or None, the number
of intersections found or 0, and the unique data or None.
"""
parsed_data = {}
unique_data = None
intersections = 0
if get_unique:
unique_data = rcommon.get_unique_data(results)
for result in results:
res_get = result.get
lab_name = res_get(models.LAB_NAME_KEY)
board = res_get(models.BOARD_KEY)
arch = res_get(models.ARCHITECTURE_KEY)
defconfig = res_get(models.DEFCONFIG_FULL_KEY)
status = res_get(models.STATUS_KEY)
build_env = res_get(models.BUILD_ENVIRONMENT_KEY)
result_struct = {
arch: {
defconfig: {
build_env: {
board: {
lab_name: status
}
}
}
}
}
# Check if the current result intersects the other interect_results
if intersect_results:
arch_view = intersect_results.viewkeys()
if arch in arch_view:
defconfig_view = intersect_results[arch].viewkeys()
if defconfig in defconfig_view:
build_env_view = \
intersect_results[arch][defconfig].viewkeys()
irad = intersect_results[arch][defconfig]
if build_env in build_env_view:
if irad[build_env].get(board, None):
intersections += 1
del irad[build_env][board]
# Clean up also the remainder of the data structure
# so that we really have cleaned up data.
if not irad[build_env]:
del irad[build_env]
if not intersect_results[arch]:
del intersect_results[arch]
if arch in parsed_data:
if defconfig in parsed_data[arch]:
if build_env in parsed_data[arch][defconfig]:
if board in parsed_data[arch][defconfig][build_env]:
pgad = parsed_data[arch][defconfig]
pgadb = pgad[build_env]
rsad = result_struct[arch][defconfig]
if build_env in pgadb[board]:
pgadb[board][lab_name] = \
rsad[build_env][board][lab_name]
else:
parsed_data[arch][defconfig][build_env][board] = \
result_struct[arch][defconfig][build_env][board]
else:
parsed_data[arch][defconfig][build_env] = \
result_struct[arch][defconfig][build_env]
else:
parsed_data[arch][defconfig] = result_struct[arch][defconfig]
else:
parsed_data[arch] = result_struct[arch]
return parsed_data, intersect_results, intersections, unique_data
def create_build_report(job,
kernel, email_format, db_options, mail_options=None):
"""Create the build report email to be sent.
:param job: The name of the job.
:type job: str
:param kernel: The name of the kernel.
:type kernel: str
:param email_format: The email format to send.
:type email_format: list
:param db_options: The mongodb database connection parameters.
:type db_options: dict
:param mail_options: The options necessary to connect to the SMTP server.
:type mail_options: dict
:return A tuple with the email body and subject as strings or None.
"""
kwargs = {}
txt_body = None
html_body = None
subject = None
# This is used to provide a footer note in the email report.
info_email = None
fail_count = total_count = 0
errors_count = warnings_count = 0
fail_results = []
if mail_options:
info_email = mail_options.get("info_email", None)
spec = {
models.JOB_KEY: job,
models.KERNEL_KEY: kernel
}
database = utils.db.get_db_connection(db_options)
total_results, total_count = utils.db.find_and_count(
database[models.DEFCONFIG_COLLECTION],
0,
0,
spec=spec,
fields=BUILD_SEARCH_FIELDS
)
err_data, errors_count, warnings_count = _get_errors_count(
total_results.clone())
unique_keys = [models.ARCHITECTURE_KEY]
total_unique_data = rcommon.get_unique_data(
total_results.clone(), unique_keys=unique_keys)
git_commit, git_url, git_branch = rcommon.get_git_data(
job, kernel, db_options)
spec[models.STATUS_KEY] = models.FAIL_STATUS
fail_results, fail_count = utils.db.find_and_count(
database[models.DEFCONFIG_COLLECTION],
0,
0,
spec=spec,
fields=BUILD_SEARCH_FIELDS,
sort=BUILD_SEARCH_SORT)
failed_data = _parse_build_data(fail_results.clone())
# Retrieve the parsed errors/warnings/mismatches summary and then
# the details.
errors_spec = {
models.JOB_KEY: job,
models.KERNEL_KEY: kernel
}
summary_fields = [
models.ERRORS_KEY, models.WARNINGS_KEY, models.MISMATCHES_KEY
]
errors_summary = utils.db.find_one2(
database[models.ERRORS_SUMMARY_COLLECTION],
errors_spec,
summary_fields
)
error_details = utils.db.find(
database[models.ERROR_LOGS_COLLECTION],
0,
0,
spec=errors_spec,
sort=[(models.DEFCONFIG_FULL_KEY, 1)]
)
error_details = [d for d in error_details.clone()]
kwargs = {
"base_url": rcommon.DEFAULT_BASE_URL,
"build_url": rcommon.DEFAULT_BUILD_URL,
"email_format": email_format,
"error_data": err_data,
"error_details": error_details,
"errors_count": errors_count,
"errors_summary": errors_summary,
"fail_count": fail_count,
"failed_data": failed_data,
"git_branch": git_branch,
"git_commit": git_commit,
"git_url": git_url,
"info_email": info_email,
"pass_count": total_count - fail_count,
"storage_url": rcommon.DEFAULT_STORAGE_URL,
"total_count": total_count,
"total_unique_data": total_unique_data,
"warnings_count": warnings_count,
models.JOB_KEY: job,
models.KERNEL_KEY: kernel,
}
custom_headers = {
rcommon.X_REPORT: rcommon.BUILD_REPORT_TYPE,
rcommon.X_BRANCH: git_branch,
rcommon.X_TREE: job,
rcommon.X_KERNEL: kernel,
}
if all([fail_count == 0, total_count == 0]):
utils.LOG.warn(
"Nothing found for '%s-%s': no build email report sent",
job, kernel)
else:
txt_body, html_body, subject = _create_build_email(**kwargs)
return txt_body, html_body, subject, custom_headers