def boot_report(config): connection, jobs, duration = parse_yaml(config.get("boot")) # TODO: Fix this when multi-lab sync is working results_directory = os.getcwd() + '/results' results = {} utils.mkdir(results_directory) test_plan = None if config.get("lab"): report_directory = os.path.join(results_directory, config.get("lab")) else: report_directory = results_directory if os.path.exists(report_directory): shutil.rmtree(report_directory) utils.mkdir(report_directory) for job_id in jobs: print 'Job ID: %s' % job_id # Init boot_meta = {} arch = None board_instance = None boot_retries = 0 kernel_defconfig_full = None kernel_defconfig = None kernel_defconfig_base = None kernel_version = None device_tree = None kernel_tree = None kernel_addr = None initrd_addr = None dtb_addr = None dtb_append = None job_file = '' board_offline = False kernel_boot_time = None boot_failure_reason = None efi_rtc = False # Retrieve job details device_type = '' job_details = connection.scheduler.job_details(job_id) if job_details['requested_device_type_id']: device_type = job_details['requested_device_type_id'] if job_details['description']: job_name = job_details['description'] try: job_short_name = re.search(".*?([A-Z]+.*)", job_name).group(1) except Exception: job_short_name = 'boot-test' try: device_name = job_details['_actual_device_cache']['hostname'] except Exception: continue result = jobs[job_id]['result'] bundle = jobs[job_id]['bundle'] if not device_type: device_type = job_details['_actual_device_cache']['device_type_id'] try: binary_job_file = connection.scheduler.job_output(job_id) except xmlrpclib.Fault: print 'Job output not found for %s' % device_type continue # Parse LAVA messages out of log raw_job_file = str(binary_job_file) for line in raw_job_file.splitlines(): if 'Infrastructure Error:' in line: print 'Infrastructure Error detected!' index = line.find('Infrastructure Error:') boot_failure_reason = line[index:] board_offline = True if 'Bootloader Error:' in line: print 'Bootloader Error detected!' index = line.find('Bootloader Error:') boot_failure_reason = line[index:] board_offline = True if 'Kernel Error:' in line: print 'Kernel Error detected!' index = line.find('Kernel Error:') boot_failure_reason = line[index:] if 'Userspace Error:' in line: print 'Userspace Error detected!' index = line.find('Userspace Error:') boot_failure_reason = line[index:] if '<LAVA_DISPATCHER>' not in line: if len(line) != 0: job_file += line + '\n' if 'rtc-efi rtc-efi: setting system clock to' in line: if device_type == 'dynamic-vm': efi_rtc = True if not kernel_defconfig or not kernel_version or not kernel_tree: try: job_metadata_info = connection.results.get_testjob_metadata( job_id) kernel_defconfig = utils.get_value_by_key( job_metadata_info, 'kernel_defconfig') kernel_version = utils.get_value_by_key( job_metadata_info, 'kernel_version') kernel_tree = utils.get_value_by_key(job_metadata_info, 'kernel_tree') device_tree = utils.get_value_by_key(job_metadata_info, 'device_tree') except Exception: continue # Record the boot log and result # TODO: Will need to map device_types to dashboard device types if kernel_defconfig and device_type and result: if ('arm' == arch or 'arm64' == arch) and device_tree is None: platform_name = device_map[device_type][0] + ',legacy' else: if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp': platform_name = device_map[device_type][0] + '_rootfs:nfs' else: platform_name = device_map[device_type][0] # Create txt format boot metadata print 'Creating boot log for %s' % (platform_name + job_name + '_' + job_id) log = 'boot-%s.txt' % (platform_name + job_name + '_' + job_id) if config.get("lab"): directory = os.path.join( results_directory, kernel_defconfig + '/' + config.get("lab")) else: directory = os.path.join(results_directory, kernel_defconfig) utils.ensure_dir(directory) utils.write_file(job_file, log, directory) if kernel_boot_time is None: kernel_boot_time = '0.0' if results.has_key(kernel_defconfig): results[kernel_defconfig].append({ 'device_type': platform_name, 'job_id': job_id, 'job_name': job_short_name, 'kernel_boot_time': kernel_boot_time, 'result': result, 'device_name': device_name }) else: results[kernel_defconfig] = [{ 'device_type': platform_name, 'job_id': job_id, 'job_name': job_short_name, 'kernel_boot_time': kernel_boot_time, 'result': result, 'device_name': device_name }] # Create JSON format boot metadata print 'Creating JSON format boot metadata' if config.get("lab"): boot_meta['lab_name'] = config.get("lab") else: boot_meta['lab_name'] = None if board_instance: boot_meta['board_instance'] = board_instance boot_meta['retries'] = boot_retries boot_meta['boot_log'] = log # TODO: Fix this boot_meta['version'] = '1.0' boot_meta['arch'] = arch boot_meta['defconfig'] = kernel_defconfig_base if kernel_defconfig_full is not None: boot_meta['defconfig_full'] = kernel_defconfig_full if device_map[device_type][1]: boot_meta['mach'] = device_map[device_type][1] boot_meta['kernel'] = kernel_version boot_meta['job'] = kernel_tree boot_meta['board'] = platform_name if board_offline and result == 'FAIL': boot_meta['boot_result'] = 'OFFLINE' #results[kernel_defconfig]['result'] = 'OFFLINE' else: boot_meta['boot_result'] = result if result == 'FAIL' or result == 'OFFLINE': if boot_failure_reason: boot_meta['boot_result_description'] = boot_failure_reason else: boot_meta[ 'boot_result_description'] = 'Unknown Error: platform failed to boot' boot_meta['boot_time'] = kernel_boot_time # TODO: Fix this boot_meta['boot_warnings'] = None if device_tree: if arch == 'arm64': boot_meta['dtb'] = 'dtbs/' + device_map[device_type][ 1] + '/' + device_tree else: boot_meta['dtb'] = 'dtbs/' + device_tree else: boot_meta['dtb'] = device_tree boot_meta['dtb_addr'] = dtb_addr boot_meta['dtb_append'] = dtb_append # TODO: Fix this boot_meta['initrd'] = None boot_meta['initrd_addr'] = initrd_addr if arch == 'arm': boot_meta['kernel_image'] = 'zImage' elif arch == 'arm64': boot_meta['kernel_image'] = 'Image' else: boot_meta['kernel_image'] = 'bzImage' boot_meta['loadaddr'] = kernel_addr json_file = 'boot-%s.json' % (platform_name + job_name + '_' + job_id) utils.write_json(json_file, directory, boot_meta) # add by wuyanjun parser_and_get_result(job_file, log, directory, report_directory, connection) #try to generate test_summary generate_test_report(job_id, connection) if results and kernel_tree and kernel_version: print 'Creating summary for %s' % (kernel_version) boot = '%s-boot-report.txt' % (kernel_version) if test_plan and ('boot' in test_plan or 'BOOT' in test_plan): boot = boot.replace('boot', test_plan) passed = 0 failed = 0 for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'PASS': passed += 1 else: failed += 1 total = passed + failed with open(os.path.join(report_directory, boot), 'a') as f: f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' % (kernel_tree, str(total), str(passed), str(failed), kernel_version)) f.write('\n') f.write('Total Duration: %.2f minutes\n' % (duration / 60)) f.write('Tree/Branch: %s\n' % kernel_tree) f.write('Git Describe: %s\n' % kernel_version) first = True for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'OFFLINE': if first: f.write('\n') f.write('Boards Offline:\n') first = False f.write('\n') f.write(defconfig) f.write('\n') break for result in results_list: if result['result'] == 'OFFLINE': f.write( ' %s %s %s %ss %s: %s\n' % (result['job_id'], result['device_type'], result['device_name'], result['kernel_boot_time'], result['job_name'], result['result'])) f.write('\n') first = True for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'FAIL': if first: f.write('\n') f.write('Failed Boot Tests:\n') first = False f.write('\n') f.write(defconfig) f.write('\n') break for result in results_list: if result['result'] == 'FAIL': f.write( ' %s %s %s %ss %s: %s\n' % (result['job_id'], result['device_type'], result['device_name'], result['kernel_boot_time'], result['job_name'], result['result'])) f.write('\n') f.write('Full Boot Report:\n') for defconfig, results_list in results.items(): f.write('\n') f.write(defconfig) f.write('\n') for result in results_list: f.write(' %s %s %s %ss %s: %s\n' % (result['job_id'], result['device_type'], result['device_name'], result['kernel_boot_time'], result['job_name'], result['result']))
def boot_report(config): connection, jobs, duration = parse_yaml(config.get("boot")) # TODO: Fix this when multi-lab sync is working #download_log2html(log2html) results_directory = os.getcwd() + '/results' results = {} utils.mkdir(results_directory) test_plan = None if config.get("lab"): report_directory = os.path.join(results_directory, config.get("lab")) else: report_directory = results_directory if os.path.exists(report_directory): shutil.rmtree(report_directory) utils.mkdir(report_directory) for job_id in jobs: print 'Job ID: %s' % job_id # Init boot_meta = {} api_url = None arch = None board_instance = None boot_retries = 0 kernel_defconfig_full = None kernel_defconfig = None kernel_defconfig_base = None kernel_version = None device_tree = None kernel_endian = None kernel_tree = None kernel_addr = None initrd_addr = None dtb_addr = None dtb_append = None fastboot = None fastboot_cmd = None job_file = '' board_offline = False kernel_boot_time = None boot_failure_reason = None efi_rtc = False # Retrieve job details device_type = '' job_details = connection.scheduler.job_details(job_id) if job_details['requested_device_type_id']: device_type = job_details['requested_device_type_id'] if job_details['description']: job_name = job_details['description'] try: job_short_name = re.search(".*?([A-Z]+.*)", job_name).group(1) except Exception: job_short_name = 'boot-test' try: device_name = job_details['_actual_device_cache']['hostname'] except Exception: continue result = jobs[job_id]['result'] bundle = jobs[job_id]['bundle'] if not device_type: device_type = job_details['_actual_device_cache']['device_type_id'] if bundle is None and device_type == 'dynamic-vm': host_job_id = job_id.replace('.1', '.0') bundle = jobs[host_job_id]['bundle'] if bundle is None: print '%s bundle is empty, skipping...' % device_type continue # Retrieve the log file try: binary_job_file = connection.scheduler.job_output(job_id) except xmlrpclib.Fault: print 'Job output not found for %s' % device_type continue # Parse LAVA messages out of log raw_job_file = str(binary_job_file) for line in raw_job_file.splitlines(): if 'Infrastructure Error:' in line: print 'Infrastructure Error detected!' index = line.find('Infrastructure Error:') boot_failure_reason = line[index:] board_offline = True if 'Bootloader Error:' in line: print 'Bootloader Error detected!' index = line.find('Bootloader Error:') boot_failure_reason = line[index:] board_offline = True if 'Kernel Error:' in line: print 'Kernel Error detected!' index = line.find('Kernel Error:') boot_failure_reason = line[index:] if 'Userspace Error:' in line: print 'Userspace Error detected!' index = line.find('Userspace Error:') boot_failure_reason = line[index:] if '<LAVA_DISPATCHER>' not in line: if len(line) != 0: job_file += line + '\n' if 'rtc-efi rtc-efi: setting system clock to' in line: if device_type == 'dynamic-vm': efi_rtc = True # Retrieve bundle if bundle is not None: json_bundle = connection.dashboard.get(bundle) bundle_data = json.loads(json_bundle['content']) # Get the boot data from LAVA for test_results in bundle_data['test_runs']: # Check for the LAVA self boot test if test_results['test_id'] == 'lava': for test in test_results['test_results']: # TODO for compat :( if test['test_case_id'] == 'kernel_boot_time': kernel_boot_time = test['measurement'] if test['test_case_id'] == 'test_kernel_boot_time': kernel_boot_time = test['measurement'] bundle_attributes = bundle_data['test_runs'][-1][ 'attributes'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'): print bundle_attributes['kernel.defconfig'] if utils.in_bundle_attributes(bundle_attributes, 'target'): board_instance = bundle_attributes['target'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'): kernel_defconfig = bundle_attributes['kernel.defconfig'] defconfig_list = kernel_defconfig.split('-') #arch = defconfig_list[0] arch = defconfig_list[-1] # Remove arch defconfig_list.pop(0) kernel_defconfig_full = '-'.join(defconfig_list) kernel_defconfig_base = ''.join( kernel_defconfig_full.split('+')[:1]) if kernel_defconfig_full == kernel_defconfig_base: kernel_defconfig_full = None if utils.in_bundle_attributes(bundle_attributes, 'kernel.version'): kernel_version = bundle_attributes['kernel.version'] if utils.in_bundle_attributes(bundle_attributes, 'device.tree'): device_tree = bundle_attributes['device.tree'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'): kernel_endian = bundle_attributes['kernel.endian'] if utils.in_bundle_attributes(bundle_attributes, 'platform.fastboot'): fastboot = bundle_attributes['platform.fastboot'] if kernel_boot_time is None: if utils.in_bundle_attributes(bundle_attributes, 'kernel-boot-time'): kernel_boot_time = bundle_attributes['kernel-boot-time'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'): kernel_tree = bundle_attributes['kernel.tree'] if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'): kernel_addr = bundle_attributes['kernel-addr'] if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'): initrd_addr = bundle_attributes['initrd-addr'] if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'): dtb_addr = bundle_attributes['dtb-addr'] if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'): dtb_append = bundle_attributes['dtb-append'] if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'): boot_retries = int(bundle_attributes['boot_retries']) if utils.in_bundle_attributes(bundle_attributes, 'test.plan'): test_tmp = bundle_attributes['test.plan'] if test_tmp: test_plan = test_tmp else: if not kernel_defconfig or not kernel_version or not kernel_tree: try: job_metadata_info = connection.results.get_testjob_metadata( job_id) kernel_defconfig = utils.get_value_by_key( job_metadata_info, 'kernel_defconfig') kernel_version = utils.get_value_by_key( job_metadata_info, 'kernel_version') kernel_tree = utils.get_value_by_key( job_metadata_info, 'kernel_tree') kernel_endian = utils.get_value_by_key( job_metadata_info, 'kernel_endian') device_tree = utils.get_value_by_key( job_metadata_info, 'device_tree') except Exception: continue # Check if we found efi-rtc if test_plan == 'boot-kvm-uefi' and not efi_rtc: if device_type == 'dynamic-vm': boot_failure_reason = 'Unable to read EFI rtc' result = 'FAIL' # Record the boot log and result # TODO: Will need to map device_types to dashboard device types if kernel_defconfig and device_type and result: if ('arm' == arch or 'arm64' == arch) and device_tree is None: platform_name = device_map[device_type][0] + ',legacy' else: if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp': platform_name = device_map[device_type][0] + '_rootfs:nfs' else: platform_name = device_map[device_type][0] # Create txt format boot metadata print 'Creating boot log for %s' % (platform_name + job_name + '_' + job_id) log = 'boot-%s.txt' % (platform_name + job_name + '_' + job_id) html = 'boot-%s.html' % (platform_name + job_name + '_' + job_id) if config.get("lab"): directory = os.path.join( results_directory, kernel_defconfig + '/' + config.get("lab")) else: directory = os.path.join(results_directory, kernel_defconfig) utils.ensure_dir(directory) utils.write_file(job_file, log, directory) if kernel_boot_time is None: kernel_boot_time = '0.0' if results.has_key(kernel_defconfig): results[kernel_defconfig].append({ 'device_type': platform_name, 'job_id': job_id, 'job_name': job_short_name, 'kernel_boot_time': kernel_boot_time, 'result': result, 'device_name': device_name }) else: results[kernel_defconfig] = [{ 'device_type': platform_name, 'job_id': job_id, 'job_name': job_short_name, 'kernel_boot_time': kernel_boot_time, 'result': result, 'device_name': device_name }] # Create JSON format boot metadata print 'Creating JSON format boot metadata' if config.get("lab"): boot_meta['lab_name'] = config.get("lab") else: boot_meta['lab_name'] = None if board_instance: boot_meta['board_instance'] = board_instance boot_meta['retries'] = boot_retries boot_meta['boot_log'] = log boot_meta['boot_log_html'] = html # TODO: Fix this boot_meta['version'] = '1.0' boot_meta['arch'] = arch boot_meta['defconfig'] = kernel_defconfig_base if kernel_defconfig_full is not None: boot_meta['defconfig_full'] = kernel_defconfig_full if device_map[device_type][1]: boot_meta['mach'] = device_map[device_type][1] boot_meta['kernel'] = kernel_version boot_meta['job'] = kernel_tree boot_meta['board'] = platform_name if board_offline and result == 'FAIL': boot_meta['boot_result'] = 'OFFLINE' #results[kernel_defconfig]['result'] = 'OFFLINE' else: boot_meta['boot_result'] = result if result == 'FAIL' or result == 'OFFLINE': if boot_failure_reason: boot_meta['boot_result_description'] = boot_failure_reason else: boot_meta[ 'boot_result_description'] = 'Unknown Error: platform failed to boot' boot_meta['boot_time'] = kernel_boot_time # TODO: Fix this boot_meta['boot_warnings'] = None if device_tree: if arch == 'arm64': boot_meta['dtb'] = 'dtbs/' + device_map[device_type][ 1] + '/' + device_tree else: boot_meta['dtb'] = 'dtbs/' + device_tree else: boot_meta['dtb'] = device_tree boot_meta['dtb_addr'] = dtb_addr boot_meta['dtb_append'] = dtb_append boot_meta['endian'] = kernel_endian boot_meta['fastboot'] = fastboot # TODO: Fix this boot_meta['initrd'] = None boot_meta['initrd_addr'] = initrd_addr if arch == 'arm': boot_meta['kernel_image'] = 'zImage' elif arch == 'arm64': boot_meta['kernel_image'] = 'Image' else: boot_meta['kernel_image'] = 'bzImage' boot_meta['loadaddr'] = kernel_addr json_file = 'boot-%s.json' % (platform_name + job_name + '_' + job_id) utils.write_json(json_file, directory, boot_meta) # add by wuyanjun # add the ip device mapping get_ip_board_mapping(job_file, log, directory, report_directory) parser_and_get_result(job_file, log, directory, report_directory, connection) if results and kernel_tree and kernel_version: print 'Creating summary for %s' % (kernel_version) boot = '%s-boot-report.txt' % (kernel_version) if test_plan and ('boot' in test_plan or 'BOOT' in test_plan): boot = boot.replace('boot', test_plan) passed = 0 failed = 0 for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'PASS': passed += 1 else: failed += 1 total = passed + failed with open(os.path.join(report_directory, boot), 'a') as f: f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' % (kernel_tree, str(total), str(passed), str(failed), kernel_version)) f.write('\n') f.write('Total Duration: %.2f minutes\n' % (duration / 60)) f.write('Tree/Branch: %s\n' % kernel_tree) f.write('Git Describe: %s\n' % kernel_version) first = True for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'OFFLINE': if first: f.write('\n') f.write('Boards Offline:\n') first = False f.write('\n') f.write(defconfig) f.write('\n') break for result in results_list: if result['result'] == 'OFFLINE': f.write( ' %s %s %s %ss %s: %s\n' % (result['job_id'], result['device_type'], result['device_name'], result['kernel_boot_time'], result['job_name'], result['result'])) f.write('\n') first = True for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'FAIL': if first: f.write('\n') f.write('Failed Boot Tests:\n') first = False f.write('\n') f.write(defconfig) f.write('\n') break for result in results_list: if result['result'] == 'FAIL': f.write( ' %s %s %s %ss %s: %s\n' % (result['job_id'], result['device_type'], result['device_name'], result['kernel_boot_time'], result['job_name'], result['result'])) f.write('\n') f.write('Full Boot Report:\n') for defconfig, results_list in results.items(): f.write('\n') f.write(defconfig) f.write('\n') for result in results_list: f.write(' %s %s %s %ss %s: %s\n' % (result['job_id'], result['device_type'], result['device_name'], result['kernel_boot_time'], result['job_name'], result['result']))