def boot_report(config):
connection, jobs, duration = parse_yaml(config.get("boot"))
# TODO: Fix this when multi-lab sync is working
results_directory = os.getcwd() + '/results'
results = {}
utils.mkdir(results_directory)
test_plan = None
if config.get("lab"):
report_directory = os.path.join(results_directory, config.get("lab"))
else:
report_directory = results_directory
if os.path.exists(report_directory):
shutil.rmtree(report_directory)
utils.mkdir(report_directory)
for job_id in jobs:
print 'Job ID: %s' % job_id
# Init
boot_meta = {}
arch = None
board_instance = None
boot_retries = 0
kernel_defconfig_full = None
kernel_defconfig = None
kernel_defconfig_base = None
kernel_version = None
device_tree = None
kernel_tree = None
kernel_addr = None
initrd_addr = None
dtb_addr = None
dtb_append = None
job_file = ''
board_offline = False
kernel_boot_time = None
boot_failure_reason = None
efi_rtc = False
# Retrieve job details
device_type = ''
job_details = connection.scheduler.job_details(job_id)
if job_details['requested_device_type_id']:
device_type = job_details['requested_device_type_id']
if job_details['description']:
job_name = job_details['description']
try:
job_short_name = re.search(".*?([A-Z]+.*)", job_name).group(1)
except Exception:
job_short_name = 'boot-test'
try:
device_name = job_details['_actual_device_cache']['hostname']
except Exception:
continue
result = jobs[job_id]['result']
bundle = jobs[job_id]['bundle']
if not device_type:
device_type = job_details['_actual_device_cache']['device_type_id']
try:
binary_job_file = connection.scheduler.job_output(job_id)
except xmlrpclib.Fault:
print 'Job output not found for %s' % device_type
continue
# Parse LAVA messages out of log
raw_job_file = str(binary_job_file)
for line in raw_job_file.splitlines():
if 'Infrastructure Error:' in line:
print 'Infrastructure Error detected!'
index = line.find('Infrastructure Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Bootloader Error:' in line:
print 'Bootloader Error detected!'
index = line.find('Bootloader Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Kernel Error:' in line:
print 'Kernel Error detected!'
index = line.find('Kernel Error:')
boot_failure_reason = line[index:]
if 'Userspace Error:' in line:
print 'Userspace Error detected!'
index = line.find('Userspace Error:')
boot_failure_reason = line[index:]
if '<LAVA_DISPATCHER>' not in line:
if len(line) != 0:
job_file += line + '\n'
if 'rtc-efi rtc-efi: setting system clock to' in line:
if device_type == 'dynamic-vm':
efi_rtc = True
if not kernel_defconfig or not kernel_version or not kernel_tree:
try:
job_metadata_info = connection.results.get_testjob_metadata(
job_id)
kernel_defconfig = utils.get_value_by_key(
job_metadata_info, 'kernel_defconfig')
kernel_version = utils.get_value_by_key(
job_metadata_info, 'kernel_version')
kernel_tree = utils.get_value_by_key(job_metadata_info,
'kernel_tree')
device_tree = utils.get_value_by_key(job_metadata_info,
'device_tree')
except Exception:
continue
# Record the boot log and result
# TODO: Will need to map device_types to dashboard device types
if kernel_defconfig and device_type and result:
if ('arm' == arch or 'arm64' == arch) and device_tree is None:
platform_name = device_map[device_type][0] + ',legacy'
else:
if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
platform_name = device_map[device_type][0] + '_rootfs:nfs'
else:
platform_name = device_map[device_type][0]
# Create txt format boot metadata
print 'Creating boot log for %s' % (platform_name + job_name +
'_' + job_id)
log = 'boot-%s.txt' % (platform_name + job_name + '_' + job_id)
if config.get("lab"):
directory = os.path.join(
results_directory,
kernel_defconfig + '/' + config.get("lab"))
else:
directory = os.path.join(results_directory, kernel_defconfig)
utils.ensure_dir(directory)
utils.write_file(job_file, log, directory)
if kernel_boot_time is None:
kernel_boot_time = '0.0'
if results.has_key(kernel_defconfig):
results[kernel_defconfig].append({
'device_type': platform_name,
'job_id': job_id,
'job_name': job_short_name,
'kernel_boot_time': kernel_boot_time,
'result': result,
'device_name': device_name
})
else:
results[kernel_defconfig] = [{
'device_type': platform_name,
'job_id': job_id,
'job_name': job_short_name,
'kernel_boot_time': kernel_boot_time,
'result': result,
'device_name': device_name
}]
# Create JSON format boot metadata
print 'Creating JSON format boot metadata'
if config.get("lab"):
boot_meta['lab_name'] = config.get("lab")
else:
boot_meta['lab_name'] = None
if board_instance:
boot_meta['board_instance'] = board_instance
boot_meta['retries'] = boot_retries
boot_meta['boot_log'] = log
# TODO: Fix this
boot_meta['version'] = '1.0'
boot_meta['arch'] = arch
boot_meta['defconfig'] = kernel_defconfig_base
if kernel_defconfig_full is not None:
boot_meta['defconfig_full'] = kernel_defconfig_full
if device_map[device_type][1]:
boot_meta['mach'] = device_map[device_type][1]
boot_meta['kernel'] = kernel_version
boot_meta['job'] = kernel_tree
boot_meta['board'] = platform_name
if board_offline and result == 'FAIL':
boot_meta['boot_result'] = 'OFFLINE'
#results[kernel_defconfig]['result'] = 'OFFLINE'
else:
boot_meta['boot_result'] = result
if result == 'FAIL' or result == 'OFFLINE':
if boot_failure_reason:
boot_meta['boot_result_description'] = boot_failure_reason
else:
boot_meta[
'boot_result_description'] = 'Unknown Error: platform failed to boot'
boot_meta['boot_time'] = kernel_boot_time
# TODO: Fix this
boot_meta['boot_warnings'] = None
if device_tree:
if arch == 'arm64':
boot_meta['dtb'] = 'dtbs/' + device_map[device_type][
1] + '/' + device_tree
else:
boot_meta['dtb'] = 'dtbs/' + device_tree
else:
boot_meta['dtb'] = device_tree
boot_meta['dtb_addr'] = dtb_addr
boot_meta['dtb_append'] = dtb_append
# TODO: Fix this
boot_meta['initrd'] = None
boot_meta['initrd_addr'] = initrd_addr
if arch == 'arm':
boot_meta['kernel_image'] = 'zImage'
elif arch == 'arm64':
boot_meta['kernel_image'] = 'Image'
else:
boot_meta['kernel_image'] = 'bzImage'
boot_meta['loadaddr'] = kernel_addr
json_file = 'boot-%s.json' % (platform_name + job_name + '_' +
job_id)
utils.write_json(json_file, directory, boot_meta)
# add by wuyanjun
parser_and_get_result(job_file, log, directory, report_directory,
connection)
#try to generate test_summary
generate_test_report(job_id, connection)
if results and kernel_tree and kernel_version:
print 'Creating summary for %s' % (kernel_version)
boot = '%s-boot-report.txt' % (kernel_version)
if test_plan and ('boot' in test_plan or 'BOOT' in test_plan):
boot = boot.replace('boot', test_plan)
passed = 0
failed = 0
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'PASS':
passed += 1
else:
failed += 1
total = passed + failed
with open(os.path.join(report_directory, boot), 'a') as f:
f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' %
(kernel_tree, str(total), str(passed), str(failed),
kernel_version))
f.write('\n')
f.write('Total Duration: %.2f minutes\n' % (duration / 60))
f.write('Tree/Branch: %s\n' % kernel_tree)
f.write('Git Describe: %s\n' % kernel_version)
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'OFFLINE':
if first:
f.write('\n')
f.write('Boards Offline:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'OFFLINE':
f.write(
' %s %s %s %ss %s: %s\n' %
(result['job_id'], result['device_type'],
result['device_name'], result['kernel_boot_time'],
result['job_name'], result['result']))
f.write('\n')
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'FAIL':
if first:
f.write('\n')
f.write('Failed Boot Tests:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'FAIL':
f.write(
' %s %s %s %ss %s: %s\n' %
(result['job_id'], result['device_type'],
result['device_name'], result['kernel_boot_time'],
result['job_name'], result['result']))
f.write('\n')
f.write('Full Boot Report:\n')
for defconfig, results_list in results.items():
f.write('\n')
f.write(defconfig)
f.write('\n')
for result in results_list:
f.write(' %s %s %s %ss %s: %s\n' %
(result['job_id'], result['device_type'],
result['device_name'], result['kernel_boot_time'],
result['job_name'], result['result']))
def boot_report(config):
connection, jobs, duration = parse_yaml(config.get("boot"))
# TODO: Fix this when multi-lab sync is working
#download_log2html(log2html)
results_directory = os.getcwd() + '/results'
results = {}
utils.mkdir(results_directory)
test_plan = None
if config.get("lab"):
report_directory = os.path.join(results_directory, config.get("lab"))
else:
report_directory = results_directory
if os.path.exists(report_directory):
shutil.rmtree(report_directory)
utils.mkdir(report_directory)
for job_id in jobs:
print 'Job ID: %s' % job_id
# Init
boot_meta = {}
api_url = None
arch = None
board_instance = None
boot_retries = 0
kernel_defconfig_full = None
kernel_defconfig = None
kernel_defconfig_base = None
kernel_version = None
device_tree = None
kernel_endian = None
kernel_tree = None
kernel_addr = None
initrd_addr = None
dtb_addr = None
dtb_append = None
fastboot = None
fastboot_cmd = None
job_file = ''
board_offline = False
kernel_boot_time = None
boot_failure_reason = None
efi_rtc = False
# Retrieve job details
device_type = ''
job_details = connection.scheduler.job_details(job_id)
if job_details['requested_device_type_id']:
device_type = job_details['requested_device_type_id']
if job_details['description']:
job_name = job_details['description']
try:
job_short_name = re.search(".*?([A-Z]+.*)", job_name).group(1)
except Exception:
job_short_name = 'boot-test'
try:
device_name = job_details['_actual_device_cache']['hostname']
except Exception:
continue
result = jobs[job_id]['result']
bundle = jobs[job_id]['bundle']
if not device_type:
device_type = job_details['_actual_device_cache']['device_type_id']
if bundle is None and device_type == 'dynamic-vm':
host_job_id = job_id.replace('.1', '.0')
bundle = jobs[host_job_id]['bundle']
if bundle is None:
print '%s bundle is empty, skipping...' % device_type
continue
# Retrieve the log file
try:
binary_job_file = connection.scheduler.job_output(job_id)
except xmlrpclib.Fault:
print 'Job output not found for %s' % device_type
continue
# Parse LAVA messages out of log
raw_job_file = str(binary_job_file)
for line in raw_job_file.splitlines():
if 'Infrastructure Error:' in line:
print 'Infrastructure Error detected!'
index = line.find('Infrastructure Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Bootloader Error:' in line:
print 'Bootloader Error detected!'
index = line.find('Bootloader Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Kernel Error:' in line:
print 'Kernel Error detected!'
index = line.find('Kernel Error:')
boot_failure_reason = line[index:]
if 'Userspace Error:' in line:
print 'Userspace Error detected!'
index = line.find('Userspace Error:')
boot_failure_reason = line[index:]
if '<LAVA_DISPATCHER>' not in line:
if len(line) != 0:
job_file += line + '\n'
if 'rtc-efi rtc-efi: setting system clock to' in line:
if device_type == 'dynamic-vm':
efi_rtc = True
# Retrieve bundle
if bundle is not None:
json_bundle = connection.dashboard.get(bundle)
bundle_data = json.loads(json_bundle['content'])
# Get the boot data from LAVA
for test_results in bundle_data['test_runs']:
# Check for the LAVA self boot test
if test_results['test_id'] == 'lava':
for test in test_results['test_results']:
# TODO for compat :(
if test['test_case_id'] == 'kernel_boot_time':
kernel_boot_time = test['measurement']
if test['test_case_id'] == 'test_kernel_boot_time':
kernel_boot_time = test['measurement']
bundle_attributes = bundle_data['test_runs'][-1][
'attributes']
if utils.in_bundle_attributes(bundle_attributes,
'kernel.defconfig'):
print bundle_attributes['kernel.defconfig']
if utils.in_bundle_attributes(bundle_attributes, 'target'):
board_instance = bundle_attributes['target']
if utils.in_bundle_attributes(bundle_attributes,
'kernel.defconfig'):
kernel_defconfig = bundle_attributes['kernel.defconfig']
defconfig_list = kernel_defconfig.split('-')
#arch = defconfig_list[0]
arch = defconfig_list[-1]
# Remove arch
defconfig_list.pop(0)
kernel_defconfig_full = '-'.join(defconfig_list)
kernel_defconfig_base = ''.join(
kernel_defconfig_full.split('+')[:1])
if kernel_defconfig_full == kernel_defconfig_base:
kernel_defconfig_full = None
if utils.in_bundle_attributes(bundle_attributes, 'kernel.version'):
kernel_version = bundle_attributes['kernel.version']
if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
device_tree = bundle_attributes['device.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
kernel_endian = bundle_attributes['kernel.endian']
if utils.in_bundle_attributes(bundle_attributes,
'platform.fastboot'):
fastboot = bundle_attributes['platform.fastboot']
if kernel_boot_time is None:
if utils.in_bundle_attributes(bundle_attributes,
'kernel-boot-time'):
kernel_boot_time = bundle_attributes['kernel-boot-time']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
kernel_tree = bundle_attributes['kernel.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
kernel_addr = bundle_attributes['kernel-addr']
if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
initrd_addr = bundle_attributes['initrd-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
dtb_addr = bundle_attributes['dtb-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
dtb_append = bundle_attributes['dtb-append']
if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
boot_retries = int(bundle_attributes['boot_retries'])
if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
test_tmp = bundle_attributes['test.plan']
if test_tmp:
test_plan = test_tmp
else:
if not kernel_defconfig or not kernel_version or not kernel_tree:
try:
job_metadata_info = connection.results.get_testjob_metadata(
job_id)
kernel_defconfig = utils.get_value_by_key(
job_metadata_info, 'kernel_defconfig')
kernel_version = utils.get_value_by_key(
job_metadata_info, 'kernel_version')
kernel_tree = utils.get_value_by_key(
job_metadata_info, 'kernel_tree')
kernel_endian = utils.get_value_by_key(
job_metadata_info, 'kernel_endian')
device_tree = utils.get_value_by_key(
job_metadata_info, 'device_tree')
except Exception:
continue
# Check if we found efi-rtc
if test_plan == 'boot-kvm-uefi' and not efi_rtc:
if device_type == 'dynamic-vm':
boot_failure_reason = 'Unable to read EFI rtc'
result = 'FAIL'
# Record the boot log and result
# TODO: Will need to map device_types to dashboard device types
if kernel_defconfig and device_type and result:
if ('arm' == arch or 'arm64' == arch) and device_tree is None:
platform_name = device_map[device_type][0] + ',legacy'
else:
if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
platform_name = device_map[device_type][0] + '_rootfs:nfs'
else:
platform_name = device_map[device_type][0]
# Create txt format boot metadata
print 'Creating boot log for %s' % (platform_name + job_name +
'_' + job_id)
log = 'boot-%s.txt' % (platform_name + job_name + '_' + job_id)
html = 'boot-%s.html' % (platform_name + job_name + '_' + job_id)
if config.get("lab"):
directory = os.path.join(
results_directory,
kernel_defconfig + '/' + config.get("lab"))
else:
directory = os.path.join(results_directory, kernel_defconfig)
utils.ensure_dir(directory)
utils.write_file(job_file, log, directory)
if kernel_boot_time is None:
kernel_boot_time = '0.0'
if results.has_key(kernel_defconfig):
results[kernel_defconfig].append({
'device_type': platform_name,
'job_id': job_id,
'job_name': job_short_name,
'kernel_boot_time': kernel_boot_time,
'result': result,
'device_name': device_name
})
else:
results[kernel_defconfig] = [{
'device_type': platform_name,
'job_id': job_id,
'job_name': job_short_name,
'kernel_boot_time': kernel_boot_time,
'result': result,
'device_name': device_name
}]
# Create JSON format boot metadata
print 'Creating JSON format boot metadata'
if config.get("lab"):
boot_meta['lab_name'] = config.get("lab")
else:
boot_meta['lab_name'] = None
if board_instance:
boot_meta['board_instance'] = board_instance
boot_meta['retries'] = boot_retries
boot_meta['boot_log'] = log
boot_meta['boot_log_html'] = html
# TODO: Fix this
boot_meta['version'] = '1.0'
boot_meta['arch'] = arch
boot_meta['defconfig'] = kernel_defconfig_base
if kernel_defconfig_full is not None:
boot_meta['defconfig_full'] = kernel_defconfig_full
if device_map[device_type][1]:
boot_meta['mach'] = device_map[device_type][1]
boot_meta['kernel'] = kernel_version
boot_meta['job'] = kernel_tree
boot_meta['board'] = platform_name
if board_offline and result == 'FAIL':
boot_meta['boot_result'] = 'OFFLINE'
#results[kernel_defconfig]['result'] = 'OFFLINE'
else:
boot_meta['boot_result'] = result
if result == 'FAIL' or result == 'OFFLINE':
if boot_failure_reason:
boot_meta['boot_result_description'] = boot_failure_reason
else:
boot_meta[
'boot_result_description'] = 'Unknown Error: platform failed to boot'
boot_meta['boot_time'] = kernel_boot_time
# TODO: Fix this
boot_meta['boot_warnings'] = None
if device_tree:
if arch == 'arm64':
boot_meta['dtb'] = 'dtbs/' + device_map[device_type][
1] + '/' + device_tree
else:
boot_meta['dtb'] = 'dtbs/' + device_tree
else:
boot_meta['dtb'] = device_tree
boot_meta['dtb_addr'] = dtb_addr
boot_meta['dtb_append'] = dtb_append
boot_meta['endian'] = kernel_endian
boot_meta['fastboot'] = fastboot
# TODO: Fix this
boot_meta['initrd'] = None
boot_meta['initrd_addr'] = initrd_addr
if arch == 'arm':
boot_meta['kernel_image'] = 'zImage'
elif arch == 'arm64':
boot_meta['kernel_image'] = 'Image'
else:
boot_meta['kernel_image'] = 'bzImage'
boot_meta['loadaddr'] = kernel_addr
json_file = 'boot-%s.json' % (platform_name + job_name + '_' +
job_id)
utils.write_json(json_file, directory, boot_meta)
# add by wuyanjun
# add the ip device mapping
get_ip_board_mapping(job_file, log, directory, report_directory)
parser_and_get_result(job_file, log, directory, report_directory,
connection)
if results and kernel_tree and kernel_version:
print 'Creating summary for %s' % (kernel_version)
boot = '%s-boot-report.txt' % (kernel_version)
if test_plan and ('boot' in test_plan or 'BOOT' in test_plan):
boot = boot.replace('boot', test_plan)
passed = 0
failed = 0
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'PASS':
passed += 1
else:
failed += 1
total = passed + failed
with open(os.path.join(report_directory, boot), 'a') as f:
f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' %
(kernel_tree, str(total), str(passed), str(failed),
kernel_version))
f.write('\n')
f.write('Total Duration: %.2f minutes\n' % (duration / 60))
f.write('Tree/Branch: %s\n' % kernel_tree)
f.write('Git Describe: %s\n' % kernel_version)
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'OFFLINE':
if first:
f.write('\n')
f.write('Boards Offline:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'OFFLINE':
f.write(
' %s %s %s %ss %s: %s\n' %
(result['job_id'], result['device_type'],
result['device_name'], result['kernel_boot_time'],
result['job_name'], result['result']))
f.write('\n')
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'FAIL':
if first:
f.write('\n')
f.write('Failed Boot Tests:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'FAIL':
f.write(
' %s %s %s %ss %s: %s\n' %
(result['job_id'], result['device_type'],
result['device_name'], result['kernel_boot_time'],
result['job_name'], result['result']))
f.write('\n')
f.write('Full Boot Report:\n')
for defconfig, results_list in results.items():
f.write('\n')
f.write(defconfig)
f.write('\n')
for result in results_list:
f.write(' %s %s %s %ss %s: %s\n' %
(result['job_id'], result['device_type'],
result['device_name'], result['kernel_boot_time'],
result['job_name'], result['result']))