def start(self): '''Starting the daemon.''' self.daemon.start() write_file(self.daemon_log, 'Daemon started: PID {}'.format(self.pid), 'at') time.sleep(1) return self.status()
def joblist(self, euid): '''Getting the current joblist of euid as raw dict.''' # Sending one item dict with key `joblist` content = {'joblist': euid} write_file(self.runner_in, content, nodate=True, json_dumps=True) # Getting jobs: `{id: {}, id: {}, ...}` jobs = read_file(self.runner_out, whole=True, json_loads=True) return json.dumps(sorted(jobs.items()), indent=4) # Prettify-ing
def send_to_daemon(self): write_file( DAEMON_IN, self.out_dict, mode='wt', nodate=True, json_dumps=True )
def add_job(self, euid, command, time_, use_shell=False, job_id=None): '''Adds a new job.''' # Sending `job` dict to fifo with required params job = { 'euid': euid, 'command': command, 'time_': time_, 'use_shell': use_shell, 'job_id': job_id } write_file(self.runner_in, job, nodate=True, json_dumps=True)
def run(src): subdirs = get_files_to_process(src) # print(subdirs) for dir in subdirs: for file in dir["files"]: image = read_file(file) backdrop = detect_backdrop(image) crop = crop_left_of_blue_line_hsv(image, backdrop=backdrop, visualize=True) filename = file.split("/")[-1] write_file(crop, "/Users/creimers/Downloads", filename)
def remove_job(self, euid, *job_ids): '''Remove a job.''' # Sending a dict with `remove` as key # and [(euid, job_id),...] as value remove_job = {'remove': [(euid, id) for id in job_ids]} write_file(self.runner_in, remove_job, nodate=True, json_dumps=True) write_file( self.daemon_out, {"msg": "Queued"}, nodate=True, json_dumps=True, )
def binary_mask_parallel( dir, smoothen, old, clear, out_dir_name=None, no_black_tape=False ): """ create binary masks in parallel Args: dir <dict> : dictionary that holds the files to be processed index_threshold <float>: threshold for the mask by index function index_threshold <int>: threshold for the mask by threshold function smoothen <bool>: whether or not to smoothen the contour old <bool>: is this an "old" picture clear <bool>: should the output dir be cleared? out_dir_name <str>: alternative name for output dir no_black_tape <bool>: no black tape arround carrot """ method = "binary-masks" if out_dir_name is not None: dir_name = "__".join([method, out_dir_name]) else: dir_name = method target = get_target_dir(dir["path"], dir_name, clear) for file in dir["files"]: log_activity(file, method, False) try: image = read_file(file) filename = file.split("/")[-1] minimize = True binary_mask = create_binary_mask( image, smoothen=smoothen, minimize=minimize, old=old, no_black_tape=no_black_tape, ) write_file(binary_mask, target, filename) except Exception as error: click.secho(file, fg="red") click.secho(repr(error), fg="red")
def combine_doulist_albums(): data_file = os.path.join(DOUBAN_ROOT, 'albums.json') id_file = os.path.join(DOUBAN_ROOT, 'album_ids.json') all_albums = [] all_ids = [] output = _get_doulist_output() for name in os.listdir(output): if not name.endswith('.dat'): continue filename = os.path.join(output, name) albums = read_dict(filename) if albums: print('Combine %s (%s)' % (filename, len(albums))) all_albums.extend(albums) all_ids.extend([doubanweb.get_id_from_album_url(a[0]) for a in albums]) all_albums = sorted(all_albums, key=itemgetter(0)) all_ids = distinct_list(all_ids, sort=True) write_dict(data_file, all_albums) write_file(id_file, ','.join(all_ids)) print('Combined file: %s (%s)' % (data_file, len(all_albums)))
def combine_doulist_albums(): data_file = os.path.join(DOUBAN_ROOT, 'albums.json') id_file = os.path.join(DOUBAN_ROOT, 'album_ids.json') all_albums = [] all_ids = [] output = _get_doulist_output() for name in os.listdir(output): if not name.endswith('.dat'): continue filename = os.path.join(output, name) albums = read_dict(filename) if albums: print('Combine %s (%s)' % (filename, len(albums))) all_albums.extend(albums) all_ids.extend( [doubanweb.get_id_from_album_url(a[0]) for a in albums]) all_albums = sorted(all_albums, key=itemgetter(0)) all_ids = distinct_list(all_ids, sort=True) write_dict(data_file, all_albums) write_file(id_file, ','.join(all_ids)) print('Combined file: %s (%s)' % (data_file, len(all_albums)))
def run(self): '''Runs continuously, gets input from fifo, sends those for processing to parse_check_forward. ''' with open(DAEMON_IN, 'rt') as d_in: while True: for line in d_in: try: content = json.loads(line) except json.JSONDecodeError: pass else: if 'stop' in content: write_file(DAEMON_LOG, 'Daemon stopped', mode='at') self.parse_check_forward(content) # This is redundant, just making sure if 'stop' in content: break else: time.sleep(0.1) continue break
def create_html_from_text(text_file, dst=None): if not isinstance(text_file, unicode): text_file = text_file.decode('utf-8') output = dst or os.path.dirname(text_file) # print('create_chapter from %s' % text_file) if not os.path.exists(output): os.makedirs(output) filename = os.path.basename(text_file) name, ext = os.path.splitext(filename) html_file = os.path.join(output, '%s.html' % name) if os.path.exists(html_file): return html_file, name else: text_lines = read_list(text_file) text_lines = ['<p>%s</p>' % line for line in text_lines] # first line as title, h2 body_str = '\n'.join(text_lines) html_tpl = read_file(CHAPTER_TEMPLATE) html_str = html_tpl.format(name, name, body_str) write_file(html_file, html_str) print('create_chapter to %s' % html_file) return html_file, name
def login(): print('Please login before operations:') username = raw_input('Username: '******'Password: '******'login-before.html', text) data = { 'source': 'index_nav', 'redir': DOUBAN_URL, 'remember': 'on', 'login': '******', 'form_email': username, 'form_password': password } t = parse_captcha(text) if t: captcha_image, captcha_id = t else: captcha_image = None captcha_id = None captcha_value = None if captcha_image and captcha_id: print('login need captcha %s' % captcha_image) urllib.urlretrieve(captcha_image, filename='captcha.png') subprocess.call(['open', 'captcha.png']) captcha_value = raw_input('Captcha: ') if captcha_value: data['captcha-solution'] = captcha_value data['captcha-id'] = captcha_id r = _post(LOGIN_URL, data=data) root = etree.HTML(r.text) if root.xpath('//div[@id="statuses"]'): save_cookies() print('Login successful!!!') print(r.headers) print(requests.utils.dict_from_cookiejar(_session.cookies)) write_file('login-after.html', r.text)
def mask_overlay_parallel( dir, smoothen, old, clear, out_dir_name=None, no_black_tape=False ): """ create minary mask overlays in parallel """ method = MASK_OVERLAYS_DIR if out_dir_name is not None: dir_name = "__".join([method, out_dir_name]) else: dir_name = method target = get_target_dir(dir["path"], dir_name, clear) for file in dir["files"]: try: log_activity(file, method) image = read_file(file) masked_overlay = create_mask_overlay( image, smoothen=smoothen, old=old, no_black_tape=no_black_tape ) filename = file.split("/")[-1] write_file(masked_overlay, target, filename) except: click.secho(file, fg="red")
def main(): dry_run = data['dry-run'] dtds = get_dtds(data['dtd'], data['mozilla-central']) print('======== DTDs ========') print(json.dumps(dtds, sort_keys=True, indent=2)) s = read_file(data['xul'], data['mozilla-central']) print('======== INPUT ========') print(s) print('======== OUTPUT ========') (new_xul, messages) = collect_messages(s, data['prefix']) print(new_xul) if not dry_run: write_file(data['xul'], new_xul, data['mozilla-central']) print('======== L10N ========') print(json.dumps(messages, sort_keys=True, indent=2)) migration = build_migration(messages, dtds, data) print('======== MIGRATION ========') print(migration) recipe_path = "{}/{}".format(data['migration'], data['recipe']) if not dry_run: write_file(recipe_path, migration, data['mozilla-central']) ftl = build_ftl(messages, dtds, data) print('======== Fluent ========') print(ftl.encode("utf-8")) if not dry_run: write_file(data['ftl'], ftl.encode("utf-8"), data['mozilla-central'], append=True)
def boot_report(config): connection, jobs, duration = parse_yaml(config.get("boot")) # TODO: Fix this when multi-lab sync is working results_directory = os.getcwd() + '/results' results = {} utils.mkdir(results_directory) test_plan = None if config.get("lab"): report_directory = os.path.join(results_directory, config.get("lab")) else: report_directory = results_directory if os.path.exists(report_directory): shutil.rmtree(report_directory) utils.mkdir(report_directory) for job_id in jobs: print 'Job ID: %s' % job_id # Init boot_meta = {} arch = None board_instance = None boot_retries = 0 kernel_defconfig_full = None kernel_defconfig = None kernel_defconfig_base = None kernel_version = None device_tree = None kernel_tree = None kernel_addr = None initrd_addr = None dtb_addr = None dtb_append = None job_file = '' board_offline = False kernel_boot_time = None boot_failure_reason = None efi_rtc = False # Retrieve job details device_type = '' job_details = connection.scheduler.job_details(job_id) if job_details['requested_device_type_id']: device_type = job_details['requested_device_type_id'] if job_details['description']: job_name = job_details['description'] try: job_short_name = re.search(".*?([A-Z]+.*)", job_name).group(1) except Exception: job_short_name = 'boot-test' try: device_name = job_details['_actual_device_cache']['hostname'] except Exception: continue result = jobs[job_id]['result'] bundle = jobs[job_id]['bundle'] if not device_type: device_type = job_details['_actual_device_cache']['device_type_id'] try: binary_job_file = connection.scheduler.job_output(job_id) except xmlrpclib.Fault: print 'Job output not found for %s' % device_type continue # Parse LAVA messages out of log raw_job_file = str(binary_job_file) for line in raw_job_file.splitlines(): if 'Infrastructure Error:' in line: print 'Infrastructure Error detected!' index = line.find('Infrastructure Error:') boot_failure_reason = line[index:] board_offline = True if 'Bootloader Error:' in line: print 'Bootloader Error detected!' index = line.find('Bootloader Error:') boot_failure_reason = line[index:] board_offline = True if 'Kernel Error:' in line: print 'Kernel Error detected!' index = line.find('Kernel Error:') boot_failure_reason = line[index:] if 'Userspace Error:' in line: print 'Userspace Error detected!' index = line.find('Userspace Error:') boot_failure_reason = line[index:] if '<LAVA_DISPATCHER>' not in line: if len(line) != 0: job_file += line + '\n' if 'rtc-efi rtc-efi: setting system clock to' in line: if device_type == 'dynamic-vm': efi_rtc = True if not kernel_defconfig or not kernel_version or not kernel_tree: try: job_metadata_info = connection.results.get_testjob_metadata( job_id) kernel_defconfig = utils.get_value_by_key( job_metadata_info, 'kernel_defconfig') kernel_version = utils.get_value_by_key( job_metadata_info, 'kernel_version') kernel_tree = utils.get_value_by_key(job_metadata_info, 'kernel_tree') device_tree = utils.get_value_by_key(job_metadata_info, 'device_tree') except Exception: continue # Record the boot log and result # TODO: Will need to map device_types to dashboard device types if kernel_defconfig and device_type and result: if ('arm' == arch or 'arm64' == arch) and device_tree is None: platform_name = device_map[device_type][0] + ',legacy' else: if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp': platform_name = device_map[device_type][0] + '_rootfs:nfs' else: platform_name = device_map[device_type][0] # Create txt format boot metadata print 'Creating boot log for %s' % (platform_name + job_name + '_' + job_id) log = 'boot-%s.txt' % (platform_name + job_name + '_' + job_id) if config.get("lab"): directory = os.path.join( results_directory, kernel_defconfig + '/' + config.get("lab")) else: directory = os.path.join(results_directory, kernel_defconfig) utils.ensure_dir(directory) utils.write_file(job_file, log, directory) if kernel_boot_time is None: kernel_boot_time = '0.0' if results.has_key(kernel_defconfig): results[kernel_defconfig].append({ 'device_type': platform_name, 'job_id': job_id, 'job_name': job_short_name, 'kernel_boot_time': kernel_boot_time, 'result': result, 'device_name': device_name }) else: results[kernel_defconfig] = [{ 'device_type': platform_name, 'job_id': job_id, 'job_name': job_short_name, 'kernel_boot_time': kernel_boot_time, 'result': result, 'device_name': device_name }] # Create JSON format boot metadata print 'Creating JSON format boot metadata' if config.get("lab"): boot_meta['lab_name'] = config.get("lab") else: boot_meta['lab_name'] = None if board_instance: boot_meta['board_instance'] = board_instance boot_meta['retries'] = boot_retries boot_meta['boot_log'] = log # TODO: Fix this boot_meta['version'] = '1.0' boot_meta['arch'] = arch boot_meta['defconfig'] = kernel_defconfig_base if kernel_defconfig_full is not None: boot_meta['defconfig_full'] = kernel_defconfig_full if device_map[device_type][1]: boot_meta['mach'] = device_map[device_type][1] boot_meta['kernel'] = kernel_version boot_meta['job'] = kernel_tree boot_meta['board'] = platform_name if board_offline and result == 'FAIL': boot_meta['boot_result'] = 'OFFLINE' #results[kernel_defconfig]['result'] = 'OFFLINE' else: boot_meta['boot_result'] = result if result == 'FAIL' or result == 'OFFLINE': if boot_failure_reason: boot_meta['boot_result_description'] = boot_failure_reason else: boot_meta[ 'boot_result_description'] = 'Unknown Error: platform failed to boot' boot_meta['boot_time'] = kernel_boot_time # TODO: Fix this boot_meta['boot_warnings'] = None if device_tree: if arch == 'arm64': boot_meta['dtb'] = 'dtbs/' + device_map[device_type][ 1] + '/' + device_tree else: boot_meta['dtb'] = 'dtbs/' + device_tree else: boot_meta['dtb'] = device_tree boot_meta['dtb_addr'] = dtb_addr boot_meta['dtb_append'] = dtb_append # TODO: Fix this boot_meta['initrd'] = None boot_meta['initrd_addr'] = initrd_addr if arch == 'arm': boot_meta['kernel_image'] = 'zImage' elif arch == 'arm64': boot_meta['kernel_image'] = 'Image' else: boot_meta['kernel_image'] = 'bzImage' boot_meta['loadaddr'] = kernel_addr json_file = 'boot-%s.json' % (platform_name + job_name + '_' + job_id) utils.write_json(json_file, directory, boot_meta) # add by wuyanjun parser_and_get_result(job_file, log, directory, report_directory, connection) #try to generate test_summary generate_test_report(job_id, connection) if results and kernel_tree and kernel_version: print 'Creating summary for %s' % (kernel_version) boot = '%s-boot-report.txt' % (kernel_version) if test_plan and ('boot' in test_plan or 'BOOT' in test_plan): boot = boot.replace('boot', test_plan) passed = 0 failed = 0 for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'PASS': passed += 1 else: failed += 1 total = passed + failed with open(os.path.join(report_directory, boot), 'a') as f: f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' % (kernel_tree, str(total), str(passed), str(failed), kernel_version)) f.write('\n') f.write('Total Duration: %.2f minutes\n' % (duration / 60)) f.write('Tree/Branch: %s\n' % kernel_tree) f.write('Git Describe: %s\n' % kernel_version) first = True for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'OFFLINE': if first: f.write('\n') f.write('Boards Offline:\n') first = False f.write('\n') f.write(defconfig) f.write('\n') break for result in results_list: if result['result'] == 'OFFLINE': f.write( ' %s %s %s %ss %s: %s\n' % (result['job_id'], result['device_type'], result['device_name'], result['kernel_boot_time'], result['job_name'], result['result'])) f.write('\n') first = True for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'FAIL': if first: f.write('\n') f.write('Failed Boot Tests:\n') first = False f.write('\n') f.write(defconfig) f.write('\n') break for result in results_list: if result['result'] == 'FAIL': f.write( ' %s %s %s %ss %s: %s\n' % (result['job_id'], result['device_type'], result['device_name'], result['kernel_boot_time'], result['job_name'], result['result'])) f.write('\n') f.write('Full Boot Report:\n') for defconfig, results_list in results.items(): f.write('\n') f.write(defconfig) f.write('\n') for result in results_list: f.write(' %s %s %s %ss %s: %s\n' % (result['job_id'], result['device_type'], result['device_name'], result['kernel_boot_time'], result['job_name'], result['result']))
'xul': 'browser/components/preferences/in-content/{0}.xul'.format(pane), 'dtd': [ 'browser/locales/en-US/chrome/browser/preferences/{0}.dtd'.format(pane) ], 'migration': './migration.py', 'ftl': 'browser/locales/en-US/browser/preferences/{0}.ftl'.format(pane), } if __name__ == '__main__': dtds = get_dtds(data['dtd'], data['mozilla-central']) s = read_file(data['xul'], data['mozilla-central']) print('======== INPUT ========') print(s) print('======== OUTPUT ========') (new_xul, messages) = collect_messages(s) print(new_xul) write_file(data['xul'], new_xul, data['mozilla-central']) print('======== L10N ========') print(json.dumps(messages, sort_keys=True, indent=2)) migration = build_migration(messages, dtds, data) print('======== MIGRATION ========') print(migration)
tmp = "".join(lines[snippet_start:snippet_end+1]) #print tmp utils.mapping_macros_comments(tmp, macro_comment_dict) #for k,v in macro_comment_dict.iteritems(): # print "dict[%s]=" % k, v lines[snippet_start:snippet_end+1] = \ self.expand_snippet(lines[snippet_start:snippet_end+1], macro_command, macro_comment_dict) lines_expanded += lines[:snippet_end+1] lines[:] = lines[snippet_end+1:] except StopIteration: print 'Iterator stop Normally' if len(lines_expanded) == 0: lines_expanded[:] = lines[:] else: lines_expanded += lines[:] pass break except ParseBaseException, e: print 'Iterator stop Due to ParseBaseException' logging.exception(e) pass break except GetOutOfLoop: print 'Iterator stop Due to Nested macro' pass except AttributeError: print 'Iterator stop Due to AttributeError, check macro_command' utils.comment_out(lines, match_start, match_end) pass utils.write_file(self.file_name, lines_expanded)
def boot_report(config): connection, jobs, duration = parse_json(config.get("boot")) # TODO: Fix this when multi-lab sync is working #download_log2html(log2html) results_directory = os.getcwd() + '/results' results = {} dt_tests = False utils.mkdir(results_directory) for job_id in jobs: print 'Job ID: %s' % job_id # Init boot_meta = {} api_url = None arch = None board_instance = None boot_retries = 0 kernel_defconfig = None kernel_defconfig_base = None kernel_version = None device_tree = None kernel_endian = None kernel_tree = None git_branch = None kernel_addr = None initrd_addr = None dtb_addr = None dtb_append = None fastboot = None fastboot_cmd = None test_plan = None job_file = '' dt_test = None dt_test_result = None dt_tests_passed = None dt_tests_failed = None board_offline = False kernel_boot_time = None boot_failure_reason = None efi_rtc = False # Retrieve job details job_details = connection.scheduler.job_details(job_id) if job_details['requested_device_type_id']: device_type = job_details['requested_device_type_id'] if job_details['description']: job_name = job_details['description'] result = jobs[job_id]['result'] bundle = jobs[job_id]['bundle'] if bundle is None and device_type == 'dynamic-vm': host_job_id = job_id.replace('.1', '.0') bundle = jobs[host_job_id]['bundle'] if bundle is None: print '%s bundle is empty, skipping...' % device_type continue # Retrieve the log file try: binary_job_file = connection.scheduler.job_output(job_id) except xmlrpclib.Fault: print 'Job output not found for %s' % device_type continue # Parse LAVA messages out of log raw_job_file = str(binary_job_file) for line in raw_job_file.splitlines(): if 'Infrastructure Error:' in line: print 'Infrastructure Error detected!' index = line.find('Infrastructure Error:') boot_failure_reason = line[index:] board_offline = True if 'Bootloader Error:' in line: print 'Bootloader Error detected!' index = line.find('Bootloader Error:') boot_failure_reason = line[index:] board_offline = True if 'Kernel Error:' in line: print 'Kernel Error detected!' index = line.find('Kernel Error:') boot_failure_reason = line[index:] if 'Userspace Error:' in line: print 'Userspace Error detected!' index = line.find('Userspace Error:') boot_failure_reason = line[index:] if '<LAVA_DISPATCHER>' not in line: if len(line) != 0: job_file += line + '\n' if '### dt-test ### end of selftest' in line: dt_tests = True regex = re.compile("(?P<test>\d+\*?)") dt_test_results = regex.findall(line) if len(dt_test_results) > 2: dt_tests_passed = dt_test_results[2] dt_tests_failed = dt_test_results[3] else: dt_tests_passed = dt_test_results[0] dt_tests_failed = dt_test_results[1] if int(dt_tests_failed) > 0: dt_test_result = 'FAIL' else: dt_test_result = 'PASS' if 'rtc-efi rtc-efi: setting system clock to' in line: if device_type == 'dynamic-vm': efi_rtc = True # Retrieve bundle if bundle is not None: json_bundle = connection.dashboard.get(bundle) bundle_data = json.loads(json_bundle['content']) # Get the boot data from LAVA for test_results in bundle_data['test_runs']: # Check for the LAVA self boot test if test_results['test_id'] == 'lava': for test in test_results['test_results']: # TODO for compat :( if test['test_case_id'] == 'kernel_boot_time': kernel_boot_time = test['measurement'] if test['test_case_id'] == 'test_kernel_boot_time': kernel_boot_time = test['measurement'] bundle_attributes = bundle_data['test_runs'][-1]['attributes'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'): print bundle_attributes['kernel.defconfig'] if utils.in_bundle_attributes(bundle_attributes, 'target'): board_instance = bundle_attributes['target'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'): kernel_defconfig = bundle_attributes['kernel.defconfig'] kernel_defconfig_base = ''.join(kernel_defconfig.split('+')[:1]) if utils.in_bundle_attributes(bundle_attributes, 'arch'): arch = bundle_attributes['arch'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.describe'): kernel_version = bundle_attributes['kernel.describe'] if utils.in_bundle_attributes(bundle_attributes, 'device.tree'): device_tree = bundle_attributes['device.tree'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'): kernel_endian = bundle_attributes['kernel.endian'] if utils.in_bundle_attributes(bundle_attributes, 'platform.fastboot'): fastboot = bundle_attributes['platform.fastboot'] if kernel_boot_time is None: if utils.in_bundle_attributes(bundle_attributes, 'kernel-boot-time'): kernel_boot_time = bundle_attributes['kernel-boot-time'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'): kernel_tree = bundle_attributes['kernel.tree'] if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'): kernel_addr = bundle_attributes['kernel-addr'] if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'): initrd_addr = bundle_attributes['initrd-addr'] if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'): dtb_addr = bundle_attributes['dtb-addr'] if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'): dtb_append = bundle_attributes['dtb-append'] if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'): boot_retries = int(bundle_attributes['boot_retries']) if utils.in_bundle_attributes(bundle_attributes, 'test.plan'): test_plan = bundle_attributes['test.plan'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.branch'): git_branch = bundle_attributes['kernel.branch'] # Check if we found efi-rtc if test_plan == 'boot-kvm-uefi' and not efi_rtc: if device_type == 'dynamic-vm': boot_failure_reason = 'Unable to read EFI rtc' result = 'FAIL' # Record the boot log and result # TODO: Will need to map device_types to dashboard device types if kernel_defconfig and device_type and result: if (arch == 'arm' or arch =='arm64') and device_tree is None: platform_name = device_map[device_type][0] + ',legacy' else: if device_tree == 'vexpress-v2p-ca15_a7.dtb': platform_name = 'vexpress-v2p-ca15_a7' elif device_tree == 'fsl-ls2080a-simu.dtb': platform_name = 'fsl-ls2080a-simu' elif test_plan == 'boot-kvm' or test_plan == 'boot-kvm-uefi': if device_tree == 'sun7i-a20-cubietruck.dtb': if device_type == 'dynamic-vm': device_type = 'cubieboard3-kvm-guest' platform_name = device_map[device_type][0] else: device_type = 'cubieboard3-kvm-host' platform_name = device_map[device_type][0] elif device_tree == 'apm-mustang.dtb': if device_type == 'dynamic-vm': if test_plan == 'boot-kvm-uefi': device_type = 'mustang-kvm-uefi-guest' else: device_type = 'mustang-kvm-guest' platform_name = device_map[device_type][0] else: if test_plan == 'boot-kvm-uefi': device_type = 'mustang-kvm-uefi-host' else: device_type = 'mustang-kvm-host' platform_name = device_map[device_type][0] elif device_tree == 'juno.dtb': if device_type == 'dynamic-vm': if test_plan == 'boot-kvm-uefi': device_type = 'juno-kvm-uefi-guest' else: device_type = 'juno-kvm-guest' platform_name = device_map[device_type][0] else: if test_plan == 'boot-kvm-uefi': device_type = 'juno-kvm-uefi-host' else: device_type = 'juno-kvm-host' platform_name = device_map[device_type][0] elif test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp': platform_name = device_map[device_type][0] + '_rootfs:nfs' else: platform_name = device_map[device_type][0] print 'Creating boot log for %s' % platform_name log = 'boot-%s.txt' % platform_name html = 'boot-%s.html' % platform_name if config.get("lab"): directory = os.path.join(results_directory, kernel_defconfig + '/' + config.get("lab")) else: directory = os.path.join(results_directory, kernel_defconfig) utils.ensure_dir(directory) utils.write_file(job_file, log, directory) if kernel_boot_time is None: kernel_boot_time = '0.0' if results.has_key(kernel_defconfig): results[kernel_defconfig].append({'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result}) else: results[kernel_defconfig] = [{'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result}] # Create JSON format boot metadata print 'Creating JSON format boot metadata' if config.get("lab"): boot_meta['lab_name'] = config.get("lab") else: boot_meta['lab_name'] = None if board_instance: boot_meta['board_instance'] = board_instance boot_meta['retries'] = boot_retries boot_meta['boot_log'] = log boot_meta['boot_log_html'] = html # TODO: Fix this boot_meta['version'] = '1.1' boot_meta['arch'] = arch boot_meta['defconfig'] = kernel_defconfig_base boot_meta['defconfig_full'] = kernel_defconfig if device_map[device_type][1]: boot_meta['mach'] = device_map[device_type][1] boot_meta['kernel'] = kernel_version boot_meta['git_branch'] = git_branch boot_meta['job'] = kernel_tree boot_meta['board'] = platform_name if board_offline and result == 'FAIL': boot_meta['boot_result'] = 'OFFLINE' #results[kernel_defconfig]['result'] = 'OFFLINE' else: boot_meta['boot_result'] = result if result == 'FAIL' or result == 'OFFLINE': if boot_failure_reason: boot_meta['boot_result_description'] = boot_failure_reason else: boot_meta['boot_result_description'] = 'Unknown Error: platform failed to boot' boot_meta['boot_time'] = kernel_boot_time # TODO: Fix this boot_meta['boot_warnings'] = None if device_tree: if arch == 'arm64': boot_meta['dtb'] = 'dtbs/' + device_map[device_type][1] + '/' + device_tree else: boot_meta['dtb'] = 'dtbs/' + device_tree else: boot_meta['dtb'] = device_tree boot_meta['dtb_addr'] = dtb_addr boot_meta['dtb_append'] = dtb_append boot_meta['dt_test'] = dt_test boot_meta['endian'] = kernel_endian boot_meta['fastboot'] = fastboot # TODO: Fix this boot_meta['initrd'] = None boot_meta['initrd_addr'] = initrd_addr if arch == 'arm': boot_meta['kernel_image'] = 'zImage' elif arch == 'arm64': boot_meta['kernel_image'] = 'Image' else: boot_meta['kernel_image'] = 'bzImage' boot_meta['loadaddr'] = kernel_addr json_file = 'boot-%s.json' % platform_name utils.write_json(json_file, directory, boot_meta) print 'Creating html version of boot log for %s' % platform_name cmd = 'python log2html.py %s' % os.path.join(directory, log) subprocess.check_output(cmd, shell=True) if config.get("lab") and config.get("api") and config.get("token"): print 'Sending boot result to %s for %s' % (config.get("api"), platform_name) headers = { 'Authorization': config.get("token"), 'Content-Type': 'application/json' } api_url = urlparse.urljoin(config.get("api"), '/boot') push('POST', api_url, data=json.dumps(boot_meta), headers=headers) headers = { 'Authorization': config.get("token"), } print 'Uploading text version of boot log' with open(os.path.join(directory, log)) as lh: data = lh.read() api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s/%s/%s' % (kernel_tree, git_branch, kernel_version, arch, kernel_defconfig, config.get("lab"), log)) push('PUT', api_url, data=data, headers=headers) print 'Uploading html version of boot log' with open(os.path.join(directory, html)) as lh: data = lh.read() api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s/%s/%s' % (kernel_tree, git_branch, kernel_version, arch, kernel_defconfig, config.get("lab"), html)) push('PUT', api_url, data=data, headers=headers)
def boot_report(config): connection, jobs, duration = parse_json(config.get("boot")) # TODO: Fix this when multi-lab sync is working #download_log2html(log2html) results_directory = os.getcwd() + '/results' results = {} dt_tests = False utils.mkdir(results_directory) for job_id in jobs: print 'Job ID: %s' % job_id # Init boot_meta = {} api_url = None arch = None board_instance = None boot_retries = 0 kernel_defconfig_full = None kernel_defconfig = None kernel_defconfig_base = None kernel_version = None device_tree = None kernel_endian = None kernel_tree = None kernel_addr = None initrd_addr = None dtb_addr = None dtb_append = None fastboot = None fastboot_cmd = None test_plan = None job_file = '' dt_test = None dt_test_result = None dt_tests_passed = None dt_tests_failed = None board_offline = False kernel_boot_time = None boot_failure_reason = None efi_rtc = False # Retrieve job details device_type = '' job_details = connection.scheduler.job_details(job_id) if job_details['requested_device_type_id']: device_type = job_details['requested_device_type_id'] if job_details['description']: job_name = job_details['description'] device_name = job_details['_actual_device_cache']['hostname'] result = jobs[job_id]['result'] bundle = jobs[job_id]['bundle'] if not device_type: device_type = job_details['_actual_device_cache']['device_type_id'] if bundle is None and device_type == 'dynamic-vm': host_job_id = job_id.replace('.1', '.0') bundle = jobs[host_job_id]['bundle'] if bundle is None: print '%s bundle is empty, skipping...' % device_type continue # Retrieve the log file try: binary_job_file = connection.scheduler.job_output(job_id) except xmlrpclib.Fault: print 'Job output not found for %s' % device_type continue # Parse LAVA messages out of log raw_job_file = str(binary_job_file) for line in raw_job_file.splitlines(): if 'Infrastructure Error:' in line: print 'Infrastructure Error detected!' index = line.find('Infrastructure Error:') boot_failure_reason = line[index:] board_offline = True if 'Bootloader Error:' in line: print 'Bootloader Error detected!' index = line.find('Bootloader Error:') boot_failure_reason = line[index:] board_offline = True if 'Kernel Error:' in line: print 'Kernel Error detected!' index = line.find('Kernel Error:') boot_failure_reason = line[index:] if 'Userspace Error:' in line: print 'Userspace Error detected!' index = line.find('Userspace Error:') boot_failure_reason = line[index:] if '<LAVA_DISPATCHER>' not in line: if len(line) != 0: job_file += line + '\n' if '### dt-test ### end of selftest' in line: dt_tests = True regex = re.compile("(?P<test>\d+\*?)") dt_test_results = regex.findall(line) if len(dt_test_results) > 2: dt_tests_passed = dt_test_results[2] dt_tests_failed = dt_test_results[3] else: dt_tests_passed = dt_test_results[0] dt_tests_failed = dt_test_results[1] if int(dt_tests_failed) > 0: dt_test_result = 'FAIL' else: dt_test_result = 'PASS' if 'rtc-efi rtc-efi: setting system clock to' in line: if device_type == 'dynamic-vm': efi_rtc = True # Retrieve bundle if bundle is not None: json_bundle = connection.dashboard.get(bundle) bundle_data = json.loads(json_bundle['content']) # Get the boot data from LAVA for test_results in bundle_data['test_runs']: # Check for the LAVA self boot test if test_results['test_id'] == 'lava': for test in test_results['test_results']: # TODO for compat :( if test['test_case_id'] == 'kernel_boot_time': kernel_boot_time = test['measurement'] if test['test_case_id'] == 'test_kernel_boot_time': kernel_boot_time = test['measurement'] bundle_attributes = bundle_data['test_runs'][-1][ 'attributes'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'): print bundle_attributes['kernel.defconfig'] if utils.in_bundle_attributes(bundle_attributes, 'target'): board_instance = bundle_attributes['target'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'): kernel_defconfig = bundle_attributes['kernel.defconfig'] defconfig_list = kernel_defconfig.split('-') #arch = defconfig_list[0] arch = defconfig_list[-1] # Remove arch defconfig_list.pop(0) kernel_defconfig_full = '-'.join(defconfig_list) kernel_defconfig_base = ''.join( kernel_defconfig_full.split('+')[:1]) if kernel_defconfig_full == kernel_defconfig_base: kernel_defconfig_full = None if utils.in_bundle_attributes(bundle_attributes, 'kernel.version'): kernel_version = bundle_attributes['kernel.version'] if utils.in_bundle_attributes(bundle_attributes, 'device.tree'): device_tree = bundle_attributes['device.tree'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'): kernel_endian = bundle_attributes['kernel.endian'] if utils.in_bundle_attributes(bundle_attributes, 'platform.fastboot'): fastboot = bundle_attributes['platform.fastboot'] if kernel_boot_time is None: if utils.in_bundle_attributes(bundle_attributes, 'kernel-boot-time'): kernel_boot_time = bundle_attributes['kernel-boot-time'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'): kernel_tree = bundle_attributes['kernel.tree'] if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'): kernel_addr = bundle_attributes['kernel-addr'] if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'): initrd_addr = bundle_attributes['initrd-addr'] if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'): dtb_addr = bundle_attributes['dtb-addr'] if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'): dtb_append = bundle_attributes['dtb-append'] if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'): boot_retries = int(bundle_attributes['boot_retries']) if utils.in_bundle_attributes(bundle_attributes, 'test.plan'): test_plan = bundle_attributes['test.plan'] # Check if we found efi-rtc if test_plan == 'boot-kvm-uefi' and not efi_rtc: if device_type == 'dynamic-vm': boot_failure_reason = 'Unable to read EFI rtc' result = 'FAIL' # Record the boot log and result # TODO: Will need to map device_types to dashboard device types if kernel_defconfig and device_type and result: if ('arm' == arch or 'arm64' == arch) and device_tree is None: platform_name = device_map[device_type][0] + ',legacy' else: if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp': platform_name = device_map[device_type][0] + '_rootfs:nfs' else: platform_name = device_map[device_type][0] print 'Creating boot log for %s' % platform_name log = 'boot-%s.txt' % (platform_name + job_name) html = 'boot-%s.html' % (platform_name + job_name) if config.get("lab"): directory = os.path.join( results_directory, kernel_defconfig + '/' + config.get("lab")) else: directory = os.path.join(results_directory, kernel_defconfig) utils.ensure_dir(directory) utils.write_file(job_file, log, directory) if kernel_boot_time is None: kernel_boot_time = '0.0' if results.has_key(kernel_defconfig): results[kernel_defconfig].append({ 'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result, 'device_name': device_name }) else: results[kernel_defconfig] = [{ 'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result, 'device_name': device_name }] # Create JSON format boot metadata print 'Creating JSON format boot metadata' if config.get("lab"): boot_meta['lab_name'] = config.get("lab") else: boot_meta['lab_name'] = None if board_instance: boot_meta['board_instance'] = board_instance boot_meta['retries'] = boot_retries boot_meta['boot_log'] = log boot_meta['boot_log_html'] = html # TODO: Fix this boot_meta['version'] = '1.0' boot_meta['arch'] = arch boot_meta['defconfig'] = kernel_defconfig_base if kernel_defconfig_full is not None: boot_meta['defconfig_full'] = kernel_defconfig_full if device_map[device_type][1]: boot_meta['mach'] = device_map[device_type][1] boot_meta['kernel'] = kernel_version boot_meta['job'] = kernel_tree boot_meta['board'] = platform_name if board_offline and result == 'FAIL': boot_meta['boot_result'] = 'OFFLINE' #results[kernel_defconfig]['result'] = 'OFFLINE' else: boot_meta['boot_result'] = result if result == 'FAIL' or result == 'OFFLINE': if boot_failure_reason: boot_meta['boot_result_description'] = boot_failure_reason else: boot_meta[ 'boot_result_description'] = 'Unknown Error: platform failed to boot' boot_meta['boot_time'] = kernel_boot_time # TODO: Fix this boot_meta['boot_warnings'] = None if device_tree: if arch == 'arm64': boot_meta['dtb'] = 'dtbs/' + device_map[device_type][ 1] + '/' + device_tree else: boot_meta['dtb'] = 'dtbs/' + device_tree else: boot_meta['dtb'] = device_tree boot_meta['dtb_addr'] = dtb_addr boot_meta['dtb_append'] = dtb_append boot_meta['dt_test'] = dt_test boot_meta['endian'] = kernel_endian boot_meta['fastboot'] = fastboot # TODO: Fix this boot_meta['initrd'] = None boot_meta['initrd_addr'] = initrd_addr if arch == 'arm': boot_meta['kernel_image'] = 'zImage' elif arch == 'arm64': boot_meta['kernel_image'] = 'Image' else: boot_meta['kernel_image'] = 'bzImage' boot_meta['loadaddr'] = kernel_addr json_file = 'boot-%s.json' % (platform_name + job_name) utils.write_json(json_file, directory, boot_meta) #print 'Creating html version of boot log for %s' % platform_name #cmd = 'python log2html.py %s' % os.path.join(directory, log) #subprocess.check_output(cmd, shell=True) #if config.get("lab") and config.get("api") and config.get("token"): # print 'Sending boot result to %s for %s' % (config.get("api"), platform_name) # headers = { # 'Authorization': config.get("token"), # 'Content-Type': 'application/json' # } # api_url = urlparse.urljoin(config.get("api"), '/boot') # push('POST', api_url, data=json.dumps(boot_meta), headers=headers) # headers = { # 'Authorization': config.get("token"), # } # print 'Uploading text version of boot log' # with open(os.path.join(directory, log)) as lh: # data = lh.read() # api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s' % (kernel_tree, # kernel_version, # kernel_defconfig, # config.get("lab"), # log)) # push('PUT', api_url, data=data, headers=headers) # print 'Uploading html version of boot log' # with open(os.path.join(directory, html)) as lh: # data = lh.read() # api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s' % (kernel_tree, # kernel_version, # kernel_defconfig, # config.get("lab"), # html)) # push('PUT', api_url, data=data, headers=headers) if config.get("lab"): report_directory = os.path.join(results_directory, config.get("lab")) utils.mkdir(report_directory) else: report_directory = results_directory if results and kernel_tree and kernel_version and 'boot' in test_plan or 'BOOT' in test_plan: print 'Creating summary for %s' % (kernel_version) boot = '%s-boot-report.txt' % (kernel_version) boot = boot.replace('boot', test_plan) passed = 0 failed = 0 for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'PASS': passed += 1 else: failed += 1 total = passed + failed with open(os.path.join(report_directory, boot), 'a') as f: #f.write('To: %s\n' % config.get("email")) #f.write('From: [email protected]\n') f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' % (kernel_tree, str(total), str(passed), str(failed), kernel_version)) f.write('\n') #f.write('Full Build Report: http://192.168.1.108:5000/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version)) #f.write('Full Boot Report: http://192.168.1.108:5000/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version)) #f.write('\n') f.write('Total Duration: %.2f minutes\n' % (duration / 60)) f.write('Tree/Branch: %s\n' % kernel_tree) f.write('Git Describe: %s\n' % kernel_version) first = True for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'OFFLINE': if first: f.write('\n') f.write('Boards Offline:\n') first = False f.write('\n') f.write(defconfig) f.write('\n') break for result in results_list: if result['result'] == 'OFFLINE': f.write(' %s %s %ss boot-test: %s\n' % (result['device_type'], result['device_name'], result['kernel_boot_time'], result['result'])) f.write('\n') first = True for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'FAIL': if first: f.write('\n') f.write('Failed Boot Tests:\n') first = False f.write('\n') f.write(defconfig) f.write('\n') break for result in results_list: if result['result'] == 'FAIL': f.write(' %s %s %ss boot-test: %s\n' % (result['device_type'], result['device_name'], result['kernel_boot_time'], result['result'])) if config.get("lab"): f.write( ' http://192.168.1.108:8083/kernel-ci/%s/%s/%s/%s/boot-%s.html' % (kernel_tree, kernel_version, defconfig, config.get("lab"), result['device_type'])) else: f.write( ' http://192.168.1.108:8083/kernel-ci/%s/%s/%s/boot-%s.html' % (kernel_tree, kernel_version, defconfig, result['device_type'])) f.write('\n') f.write('\n') f.write('Full Boot Report:\n') for defconfig, results_list in results.items(): f.write('\n') f.write(defconfig) f.write('\n') for result in results_list: f.write(' %s %s %ss boot-test: %s\n' % (result['device_type'], result['device_name'], result['kernel_boot_time'], result['result'])) # add by wuyanjun if results and directory: parser_and_get_result(results, directory, report_directory) #get_ip_board_mapping(results, directory, report_directory) # dt-self-test if results and kernel_tree and kernel_version and dt_tests: print 'Creating device tree runtime self test summary for %s' % kernel_version dt_self_test = '%s-dt-runtime-self-test-report.txt' % kernel_version passed = 0 failed = 0 for defconfig, results_list in results.items(): for result in results_list: if result['dt_test_result'] == 'PASS': passed += 1 elif result['dt_test_result'] == 'FAIL': failed += 1 total = passed + failed with open(os.path.join(report_directory, dt_self_test), 'a') as f: #f.write('To: %s\n' % config.get("email")) #f.write('From: [email protected]\n') f.write( 'Subject: %s dt-runtime-unit-tests: %s boards tested: %s passed, %s failed (%s)\n' % (kernel_tree, str(total), str(passed), str(failed), kernel_version)) f.write('\n') #f.write('Full Build Report: http://192.168.1.108:5000/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version)) #f.write('Full Boot Report: http://192.168.1.108:5000/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version)) #f.write('Full Test Report: http://192.168.1.108:5000/test/%s/kernel/%s/\n' % (kernel_tree, kernel_version)) #f.write('\n') f.write('Tree/Branch: %s\n' % kernel_tree) f.write('Git Describe: %s\n' % kernel_version) first = True for defconfig, results_list in results.items(): for result in results_list: if result['dt_test_result'] == 'FAIL': if first: f.write('\n') f.write('Failed Device Tree Unit Tests:\n') first = False f.write('\n') f.write(defconfig) f.write('\n') break for result in results_list: if result['dt_test_result'] == "FAIL": f.write( ' %s %s passed: %s / failed: %s dt-runtime-unit-tests: %s\n' % (result['device_type'], result['device_name'], result['dt_tests_passed'], result['dt_tests_failed'], result['dt_test_result'])) if config.get("lab"): f.write( ' http://192.168.1.108:8083/kernel-ci/%s/%s/%s/%s/boot-%s.html' % (kernel_tree, kernel_version, defconfig, config.get("lab"), result['device_type'])) else: f.write( ' http://192.168.1.108:8083/kernel-ci/%s/%s/%s/boot-%s.html' % (kernel_tree, kernel_version, defconfig, result['device_type'])) f.write('\n') f.write('\n') f.write('Full Unit Test Report:\n') for defconfig, results_list in results.items(): first = True for result in results_list: if result['dt_test_result']: if first: f.write('\n') f.write(defconfig) f.write('\n') first = False f.write( ' %s %s passed: %s / failed: %s dt-runtime-unit-tests: %s\n' % (result['device_type'], result['device_name'], result['dt_tests_passed'], result['dt_tests_failed'], result['dt_test_result'])) # sendmail if config.get("email"): print 'Sending e-mail summary to %s' % config.get("email") if os.path.exists(report_directory): cmd = 'cat %s | sendmail -t' % os.path.join(report_directory, boot) subprocess.check_output(cmd, shell=True) if dt_tests: if os.path.exists(report_directory): cmd = 'cat %s | sendmail -t' % os.path.join( report_directory, dt_self_test) subprocess.check_output(cmd, shell=True)
def tip_mask(src, model, visualize=False): """ mask the tips of the straightened carrots Args: src (str) - absolute path to the binary mask visualize (bool) - only visualize the masking """ # if not dest: dest = src.split(STRAIGHTENED_MASKS_DIR)[0] dest = os.path.join(dest, DETIPPED_MASKS_DIR) if os.path.exists(dest): shutil.rmtree(dest) if not os.path.exists(dest): os.makedirs(dest) for file in os.listdir(src): print(file) src_filepath = os.path.join(src, file) dest_filepath = os.path.join(dest, file) mask = cv2.imread(src_filepath, cv2.IMREAD_GRAYSCALE) attributes = get_attributes_from_filename(src_filepath) scale = attributes.get("Scale", None) mm_per_px = pixel_to_mm(scale) if mask is None: msg = "File %s is empty!" % src_filepath click.secho(msg, fg="red") continue # get index from ml model try: tip_index = tip_mask_ml(mask, model, mm_per_px) except Exception as e: click.secho(file, fg="red") print(e) tip_index = [0] tip_index = int(tip_index[0]) # print(tip_index) # print(mask.shape[1]) tip_index = mask.shape[1] - tip_index # get index based on threshold # tip_index = find_tip_pseudo_dynamic(mask, pure=True) # tip_index_advanced = find_tip_pseudo_dynamic(mask, pure=False) # if tip_index_advanced > 0: # crop_index = tip_index_advanced # else: # crop_index = tip_index crop_index = tip_index if visualize: # paint only tip = mark_start_of_tail(mask.copy(), tip_index, [0, 0, 255]) # tip = mark_start_of_tail(tip, tip_index_advanced, [0, 255, 0]) # print(dest) write_file(tip, dest, file) continue else: # crop + buffer + wirte mask = mask[:, crop_index:] black_col = np.zeros((mask.shape[0], 10), dtype=np.uint8) mask = np.hstack([black_col, mask]) # another round of contour reduction to remove dangling white pixels mask = reduce_to_contour(mask, minimize=False) cv2.imwrite(dest_filepath, mask) old_tip_index = get_index_of_tip(mask.T) tip_length = crop_index - old_tip_index if tip_length < 0: tip_length = 0 tip_biomass = get_biomass(mask[:, old_tip_index:crop_index]) new_filepath = append_or_change_filename(dest_filepath, "TipLength", None, tip_length) append_or_change_filename(new_filepath, "TipBiomass", None, tip_biomass)
'dummy-ssh': ['dummy-ssh', None], 'dummy_ssh_d02': ['dummy_ssh_d02', None], 'dummy_ssh_d01': ['dummy_ssh_d01', None], 'kvm': ['x86-kvm', None] } def download_log2html(url): print 'Fetching latest log2html script' try: response = urllib2.urlopen(url, timeout=30) except IOError, e: print 'error fetching %s: %s' % (url, e) exit(1) script = response.read() utils.write_file(script, 'log2html.py', os.getcwd()) def parse_json(json): jobs = utils.load_json(json) url = utils.validate_input(jobs['username'], jobs['token'], jobs['server']) connection = utils.connect(url) duration = jobs['duration'] # Remove unused data jobs.pop('duration') jobs.pop('username') jobs.pop('token') jobs.pop('server') return connection, jobs, duration
'dummy_ssh_d02': ['dummy_ssh_d02', None], 'dummy_ssh_d01': ['dummy_ssh_d01', None], 'dummy_ssh_d03': ['dummy_ssh_d03', None], 'dummy_ssh_d05': ['dummy_ssh_d05', None], 'kvm': ['x86-kvm', None]} def download_log2html(url): print 'Fetching latest log2html script' try: response = urllib2.urlopen(url, timeout=30) except IOError, e: print 'error fetching %s: %s' % (url, e) exit(1) script = response.read() utils.write_file(script, 'log2html.py', os.getcwd()) def parse_json(json): jobs = utils.load_json(json) url = utils.validate_input(jobs['username'], jobs['token'], jobs['server']) connection = utils.connect(url) duration = jobs['duration'] # Remove unused data jobs.pop('duration') jobs.pop('username') jobs.pop('token') jobs.pop('server') return connection, jobs, duration def push(method, url, data, headers):
def boot_report(config): connection, jobs, duration = parse_json(config.get("boot")) # TODO: Fix this when multi-lab sync is working #download_log2html(log2html) results_directory = os.getcwd() + '/results' results = {} dt_tests = False utils.mkdir(results_directory) for job_id in jobs: print 'Job ID: %s' % job_id # Init boot_meta = {} api_url = None arch = None board_instance = None boot_retries = 0 kernel_defconfig = None kernel_defconfig_base = None kernel_version = None device_tree = None kernel_endian = None kernel_tree = None git_branch = None kernel_addr = None initrd_addr = None dtb_addr = None dtb_append = None fastboot = None fastboot_cmd = None test_plan = None job_file = '' dt_test = None dt_test_result = None dt_tests_passed = None dt_tests_failed = None board_offline = False kernel_boot_time = None boot_failure_reason = None efi_rtc = False # Retrieve job details job_details = connection.scheduler.job_details(job_id) if job_details['requested_device_type_id']: device_type = job_details['requested_device_type_id'] if job_details['description']: job_name = job_details['description'] result = jobs[job_id]['result'] bundle = jobs[job_id]['bundle'] if bundle is None and device_type == 'dynamic-vm': host_job_id = job_id.replace('.1', '.0') bundle = jobs[host_job_id]['bundle'] if bundle is None: print '%s bundle is empty, skipping...' % device_type continue # Retrieve the log file try: binary_job_file = connection.scheduler.job_output(job_id) except xmlrpclib.Fault: print 'Job output not found for %s' % device_type continue # Parse LAVA messages out of log raw_job_file = str(binary_job_file) for line in raw_job_file.splitlines(): if 'Infrastructure Error:' in line: print 'Infrastructure Error detected!' index = line.find('Infrastructure Error:') boot_failure_reason = line[index:] board_offline = True if 'Bootloader Error:' in line: print 'Bootloader Error detected!' index = line.find('Bootloader Error:') boot_failure_reason = line[index:] board_offline = True if 'Kernel Error:' in line: print 'Kernel Error detected!' index = line.find('Kernel Error:') boot_failure_reason = line[index:] if 'Userspace Error:' in line: print 'Userspace Error detected!' index = line.find('Userspace Error:') boot_failure_reason = line[index:] if '<LAVA_DISPATCHER>' not in line: if len(line) != 0: job_file += line + '\n' if '### dt-test ### end of selftest' in line: dt_tests = True regex = re.compile("(?P<test>\d+\*?)") dt_test_results = regex.findall(line) if len(dt_test_results) > 2: dt_tests_passed = dt_test_results[2] dt_tests_failed = dt_test_results[3] else: dt_tests_passed = dt_test_results[0] dt_tests_failed = dt_test_results[1] if int(dt_tests_failed) > 0: dt_test_result = 'FAIL' else: dt_test_result = 'PASS' if 'rtc-efi rtc-efi: setting system clock to' in line: if device_type == 'dynamic-vm': efi_rtc = True # Retrieve bundle if bundle is not None: json_bundle = connection.dashboard.get(bundle) bundle_data = json.loads(json_bundle['content']) # Get the boot data from LAVA for test_results in bundle_data['test_runs']: # Check for the LAVA self boot test if test_results['test_id'] == 'lava': for test in test_results['test_results']: # TODO for compat :( if test['test_case_id'] == 'kernel_boot_time': kernel_boot_time = test['measurement'] if test['test_case_id'] == 'test_kernel_boot_time': kernel_boot_time = test['measurement'] bundle_attributes = bundle_data['test_runs'][-1][ 'attributes'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'): print bundle_attributes['kernel.defconfig'] if utils.in_bundle_attributes(bundle_attributes, 'target'): board_instance = bundle_attributes['target'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'): kernel_defconfig = bundle_attributes['kernel.defconfig'] kernel_defconfig_base = ''.join( kernel_defconfig.split('+')[:1]) if utils.in_bundle_attributes(bundle_attributes, 'arch'): arch = bundle_attributes['arch'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.describe'): kernel_version = bundle_attributes['kernel.describe'] if utils.in_bundle_attributes(bundle_attributes, 'device.tree'): device_tree = bundle_attributes['device.tree'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'): kernel_endian = bundle_attributes['kernel.endian'] if utils.in_bundle_attributes(bundle_attributes, 'platform.fastboot'): fastboot = bundle_attributes['platform.fastboot'] if kernel_boot_time is None: if utils.in_bundle_attributes(bundle_attributes, 'kernel-boot-time'): kernel_boot_time = bundle_attributes['kernel-boot-time'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'): kernel_tree = bundle_attributes['kernel.tree'] if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'): kernel_addr = bundle_attributes['kernel-addr'] if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'): initrd_addr = bundle_attributes['initrd-addr'] if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'): dtb_addr = bundle_attributes['dtb-addr'] if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'): dtb_append = bundle_attributes['dtb-append'] if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'): boot_retries = int(bundle_attributes['boot_retries']) if utils.in_bundle_attributes(bundle_attributes, 'test.plan'): test_plan = bundle_attributes['test.plan'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.branch'): git_branch = bundle_attributes['kernel.branch'] # Check if we found efi-rtc if test_plan == 'boot-kvm-uefi' and not efi_rtc: if device_type == 'dynamic-vm': boot_failure_reason = 'Unable to read EFI rtc' result = 'FAIL' # Record the boot log and result # TODO: Will need to map device_types to dashboard device types if kernel_defconfig and device_type and result: if (arch == 'arm' or arch == 'arm64') and device_tree is None: platform_name = device_map[device_type][0] + ',legacy' else: if device_tree == 'vexpress-v2p-ca15_a7.dtb': platform_name = 'vexpress-v2p-ca15_a7' elif device_tree == 'fsl-ls2080a-simu.dtb': platform_name = 'fsl-ls2080a-simu' elif test_plan == 'boot-kvm' or test_plan == 'boot-kvm-uefi': if device_tree == 'sun7i-a20-cubietruck.dtb': if device_type == 'dynamic-vm': device_type = 'cubieboard3-kvm-guest' platform_name = device_map[device_type][0] else: device_type = 'cubieboard3-kvm-host' platform_name = device_map[device_type][0] elif device_tree == 'apm-mustang.dtb': if device_type == 'dynamic-vm': if test_plan == 'boot-kvm-uefi': device_type = 'mustang-kvm-uefi-guest' else: device_type = 'mustang-kvm-guest' platform_name = device_map[device_type][0] else: if test_plan == 'boot-kvm-uefi': device_type = 'mustang-kvm-uefi-host' else: device_type = 'mustang-kvm-host' platform_name = device_map[device_type][0] elif device_tree == 'juno.dtb': if device_type == 'dynamic-vm': if test_plan == 'boot-kvm-uefi': device_type = 'juno-kvm-uefi-guest' else: device_type = 'juno-kvm-guest' platform_name = device_map[device_type][0] else: if test_plan == 'boot-kvm-uefi': device_type = 'juno-kvm-uefi-host' else: device_type = 'juno-kvm-host' platform_name = device_map[device_type][0] elif test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp': platform_name = device_map[device_type][0] + '_rootfs:nfs' else: platform_name = device_map[device_type][0] print 'Creating boot log for %s' % platform_name log = 'boot-%s.txt' % platform_name html = 'boot-%s.html' % platform_name if config.get("lab"): directory = os.path.join( results_directory, kernel_defconfig + '/' + config.get("lab")) else: directory = os.path.join(results_directory, kernel_defconfig) utils.ensure_dir(directory) utils.write_file(job_file, log, directory) if kernel_boot_time is None: kernel_boot_time = '0.0' if results.has_key(kernel_defconfig): results[kernel_defconfig].append({ 'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result }) else: results[kernel_defconfig] = [{ 'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result }] # Create JSON format boot metadata print 'Creating JSON format boot metadata' if config.get("lab"): boot_meta['lab_name'] = config.get("lab") else: boot_meta['lab_name'] = None if board_instance: boot_meta['board_instance'] = board_instance boot_meta['retries'] = boot_retries boot_meta['boot_log'] = log boot_meta['boot_log_html'] = html # TODO: Fix this boot_meta['version'] = '1.1' boot_meta['arch'] = arch boot_meta['defconfig'] = kernel_defconfig_base boot_meta['defconfig_full'] = kernel_defconfig if device_map[device_type][1]: boot_meta['mach'] = device_map[device_type][1] boot_meta['kernel'] = kernel_version boot_meta['git_branch'] = git_branch boot_meta['job'] = kernel_tree boot_meta['board'] = platform_name if board_offline and result == 'FAIL': boot_meta['boot_result'] = 'OFFLINE' #results[kernel_defconfig]['result'] = 'OFFLINE' else: boot_meta['boot_result'] = result if result == 'FAIL' or result == 'OFFLINE': if boot_failure_reason: boot_meta['boot_result_description'] = boot_failure_reason else: boot_meta[ 'boot_result_description'] = 'Unknown Error: platform failed to boot' boot_meta['boot_time'] = kernel_boot_time # TODO: Fix this boot_meta['boot_warnings'] = None if device_tree: if arch == 'arm64': boot_meta['dtb'] = 'dtbs/' + device_map[device_type][ 1] + '/' + device_tree else: boot_meta['dtb'] = 'dtbs/' + device_tree else: boot_meta['dtb'] = device_tree boot_meta['dtb_addr'] = dtb_addr boot_meta['dtb_append'] = dtb_append boot_meta['dt_test'] = dt_test boot_meta['endian'] = kernel_endian boot_meta['fastboot'] = fastboot # TODO: Fix this boot_meta['initrd'] = None boot_meta['initrd_addr'] = initrd_addr if arch == 'arm': boot_meta['kernel_image'] = 'zImage' elif arch == 'arm64': boot_meta['kernel_image'] = 'Image' else: boot_meta['kernel_image'] = 'bzImage' boot_meta['loadaddr'] = kernel_addr json_file = 'boot-%s.json' % platform_name utils.write_json(json_file, directory, boot_meta) print 'Creating html version of boot log for %s' % platform_name cmd = 'python log2html.py %s' % os.path.join(directory, log) subprocess.check_output(cmd, shell=True) if config.get("lab") and config.get("api") and config.get("token"): print 'Sending boot result to %s for %s' % (config.get("api"), platform_name) headers = { 'Authorization': config.get("token"), 'Content-Type': 'application/json' } api_url = urlparse.urljoin(config.get("api"), '/boot') push('POST', api_url, data=json.dumps(boot_meta), headers=headers) headers = { 'Authorization': config.get("token"), } print 'Uploading text version of boot log' with open(os.path.join(directory, log)) as lh: data = lh.read() api_url = urlparse.urljoin( config.get("api"), '/upload/%s/%s/%s/%s/%s/%s/%s' % (kernel_tree, git_branch, kernel_version, arch, kernel_defconfig, config.get("lab"), log)) push('PUT', api_url, data=data, headers=headers) print 'Uploading html version of boot log' with open(os.path.join(directory, html)) as lh: data = lh.read() api_url = urlparse.urljoin( config.get("api"), '/upload/%s/%s/%s/%s/%s/%s/%s' % (kernel_tree, git_branch, kernel_version, arch, kernel_defconfig, config.get("lab"), html)) push('PUT', api_url, data=data, headers=headers)
# This is redundant, just making sure if 'stop' in content: break else: time.sleep(0.1) continue break if __name__ == '__main__': daemon = HatDaemon() # Start-stop if len(sys.argv) == 2: action_map = {'start': daemon.start, 'stop': daemon.stop} if sys.argv[1] in action_map: action_map[sys.argv[1]]() # if daemon is started, starting to `run` if daemon.status(): write_file(PID_FILE, daemon.pid, nodate=True) daemon_wrapper = DaemonWrapper(daemon) # To make first iteration of runner, sending a no-op to the # runner_in fifo; runner will start with previous jobs now write_file( RUNNER_IN, {'noop': True}, json_dumps=True, nodate=True, ) # Perpetual running daemon_wrapper.run()
def boot_report(config): connection, jobs, duration = parse_json(config.get("boot")) # TODO: Fix this when multi-lab sync is working #download_log2html(log2html) results_directory = os.getcwd() + '/results' results = {} utils.mkdir(results_directory) test_plan = None if config.get("lab"): report_directory = os.path.join(results_directory, config.get("lab")) else: report_directory = results_directory if os.path.exists(report_directory): shutil.rmtree(report_directory) utils.mkdir(report_directory) for job_id in jobs: print 'Job ID: %s' % job_id # Init boot_meta = {} api_url = None arch = None board_instance = None boot_retries = 0 kernel_defconfig_full = None kernel_defconfig = None kernel_defconfig_base = None kernel_version = None device_tree = None kernel_endian = None kernel_tree = None kernel_addr = None initrd_addr = None dtb_addr = None dtb_append = None fastboot = None fastboot_cmd = None job_file = '' board_offline = False kernel_boot_time = None boot_failure_reason = None efi_rtc = False # Retrieve job details device_type = '' job_details = connection.scheduler.job_details(job_id) if job_details['requested_device_type_id']: device_type = job_details['requested_device_type_id'] if job_details['description']: job_name = job_details['description'] try: job_short_name = re.search(".*?([A-Z]+.*)", job_name).group(1) except Exception: job_short_name = 'boot-test' try: device_name = job_details['_actual_device_cache']['hostname'] except Exception: continue result = jobs[job_id]['result'] bundle = jobs[job_id]['bundle'] if not device_type: device_type = job_details['_actual_device_cache']['device_type_id'] if bundle is None and device_type == 'dynamic-vm': host_job_id = job_id.replace('.1', '.0') bundle = jobs[host_job_id]['bundle'] if bundle is None: print '%s bundle is empty, skipping...' % device_type continue # Retrieve the log file try: binary_job_file = connection.scheduler.job_output(job_id) except xmlrpclib.Fault: print 'Job output not found for %s' % device_type continue # Parse LAVA messages out of log raw_job_file = str(binary_job_file) for line in raw_job_file.splitlines(): if 'Infrastructure Error:' in line: print 'Infrastructure Error detected!' index = line.find('Infrastructure Error:') boot_failure_reason = line[index:] board_offline = True if 'Bootloader Error:' in line: print 'Bootloader Error detected!' index = line.find('Bootloader Error:') boot_failure_reason = line[index:] board_offline = True if 'Kernel Error:' in line: print 'Kernel Error detected!' index = line.find('Kernel Error:') boot_failure_reason = line[index:] if 'Userspace Error:' in line: print 'Userspace Error detected!' index = line.find('Userspace Error:') boot_failure_reason = line[index:] if '<LAVA_DISPATCHER>' not in line: if len(line) != 0: job_file += line + '\n' if 'rtc-efi rtc-efi: setting system clock to' in line: if device_type == 'dynamic-vm': efi_rtc = True # Retrieve bundle if bundle is not None: json_bundle = connection.dashboard.get(bundle) bundle_data = json.loads(json_bundle['content']) # Get the boot data from LAVA for test_results in bundle_data['test_runs']: # Check for the LAVA self boot test if test_results['test_id'] == 'lava': for test in test_results['test_results']: # TODO for compat :( if test['test_case_id'] == 'kernel_boot_time': kernel_boot_time = test['measurement'] if test['test_case_id'] == 'test_kernel_boot_time': kernel_boot_time = test['measurement'] bundle_attributes = bundle_data['test_runs'][-1]['attributes'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'): print bundle_attributes['kernel.defconfig'] if utils.in_bundle_attributes(bundle_attributes, 'target'): board_instance = bundle_attributes['target'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'): kernel_defconfig = bundle_attributes['kernel.defconfig'] defconfig_list = kernel_defconfig.split('-') #arch = defconfig_list[0] arch = defconfig_list[-1] # Remove arch defconfig_list.pop(0) kernel_defconfig_full = '-'.join(defconfig_list) kernel_defconfig_base = ''.join(kernel_defconfig_full.split('+')[:1]) if kernel_defconfig_full == kernel_defconfig_base: kernel_defconfig_full = None if utils.in_bundle_attributes(bundle_attributes, 'kernel.version'): kernel_version = bundle_attributes['kernel.version'] if utils.in_bundle_attributes(bundle_attributes, 'device.tree'): device_tree = bundle_attributes['device.tree'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'): kernel_endian = bundle_attributes['kernel.endian'] if utils.in_bundle_attributes(bundle_attributes, 'platform.fastboot'): fastboot = bundle_attributes['platform.fastboot'] if kernel_boot_time is None: if utils.in_bundle_attributes(bundle_attributes, 'kernel-boot-time'): kernel_boot_time = bundle_attributes['kernel-boot-time'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'): kernel_tree = bundle_attributes['kernel.tree'] if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'): kernel_addr = bundle_attributes['kernel-addr'] if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'): initrd_addr = bundle_attributes['initrd-addr'] if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'): dtb_addr = bundle_attributes['dtb-addr'] if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'): dtb_append = bundle_attributes['dtb-append'] if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'): boot_retries = int(bundle_attributes['boot_retries']) if utils.in_bundle_attributes(bundle_attributes, 'test.plan'): test_tmp = bundle_attributes['test.plan'] if test_tmp: test_plan = test_tmp else: if not kernel_defconfig or not kernel_version or not kernel_tree: job_defnition = {} if 'original_definition' in job_details.keys(): job_definition = job_details['original_definition'] try: job_dictionary = eval(job_definition) except Exception: pass if job_dictionary: if 'actions' in job_dictionary.keys(): actions = job_dictionary['actions'] for i in range(0, len(actions)): try: kernel_defconfig = actions[i]['metadata']['kernel.defconfig'] kernel_version = actions[i]['metadata']['kernel.version'] kernel_tree = actions[i]['metadata']['kernel.tree'] kernel_endian = actions[i]['metadata']['kernel.endian'] platform_fastboot = actions[i]['metadata']['platform.fastboot'] device_tree = actions[i]['metadata']['kernel.tree'] break except KeyError: continue if 'target' in job_details.keys(): print job_details.keys() # Check if we found efi-rtc if test_plan == 'boot-kvm-uefi' and not efi_rtc: if device_type == 'dynamic-vm': boot_failure_reason = 'Unable to read EFI rtc' result = 'FAIL' # Record the boot log and result # TODO: Will need to map device_types to dashboard device types if kernel_defconfig and device_type and result: if ( 'arm' == arch or 'arm64' == arch ) and device_tree is None: platform_name = device_map[device_type][0] + ',legacy' else: if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp': platform_name = device_map[device_type][0] + '_rootfs:nfs' else: platform_name = device_map[device_type][0] # Create txt format boot metadata print 'Creating boot log for %s' % (platform_name + job_name + '_' + job_id) log = 'boot-%s.txt' % (platform_name + job_name + '_' + job_id) html = 'boot-%s.html' % (platform_name + job_name + '_' + job_id) if config.get("lab"): directory = os.path.join(results_directory, kernel_defconfig + '/' + config.get("lab")) else: directory = os.path.join(results_directory, kernel_defconfig) utils.ensure_dir(directory) utils.write_file(job_file, log, directory) if kernel_boot_time is None: kernel_boot_time = '0.0' if results.has_key(kernel_defconfig): results[kernel_defconfig].append({'device_type': platform_name, 'job_id': job_id, 'job_name': job_short_name, 'kernel_boot_time': kernel_boot_time, 'result': result, 'device_name': device_name}) else: results[kernel_defconfig] = [{'device_type': platform_name, 'job_id': job_id, 'job_name': job_short_name, 'kernel_boot_time': kernel_boot_time, 'result': result, 'device_name': device_name}] # Create JSON format boot metadata print 'Creating JSON format boot metadata' if config.get("lab"): boot_meta['lab_name'] = config.get("lab") else: boot_meta['lab_name'] = None if board_instance: boot_meta['board_instance'] = board_instance boot_meta['retries'] = boot_retries boot_meta['boot_log'] = log boot_meta['boot_log_html'] = html # TODO: Fix this boot_meta['version'] = '1.0' boot_meta['arch'] = arch boot_meta['defconfig'] = kernel_defconfig_base if kernel_defconfig_full is not None: boot_meta['defconfig_full'] = kernel_defconfig_full if device_map[device_type][1]: boot_meta['mach'] = device_map[device_type][1] boot_meta['kernel'] = kernel_version boot_meta['job'] = kernel_tree boot_meta['board'] = platform_name if board_offline and result == 'FAIL': boot_meta['boot_result'] = 'OFFLINE' #results[kernel_defconfig]['result'] = 'OFFLINE' else: boot_meta['boot_result'] = result if result == 'FAIL' or result == 'OFFLINE': if boot_failure_reason: boot_meta['boot_result_description'] = boot_failure_reason else: boot_meta['boot_result_description'] = 'Unknown Error: platform failed to boot' boot_meta['boot_time'] = kernel_boot_time # TODO: Fix this boot_meta['boot_warnings'] = None if device_tree: if arch == 'arm64': boot_meta['dtb'] = 'dtbs/' + device_map[device_type][1] + '/' + device_tree else: boot_meta['dtb'] = 'dtbs/' + device_tree else: boot_meta['dtb'] = device_tree boot_meta['dtb_addr'] = dtb_addr boot_meta['dtb_append'] = dtb_append boot_meta['endian'] = kernel_endian boot_meta['fastboot'] = fastboot # TODO: Fix this boot_meta['initrd'] = None boot_meta['initrd_addr'] = initrd_addr if arch == 'arm': boot_meta['kernel_image'] = 'zImage' elif arch == 'arm64': boot_meta['kernel_image'] = 'Image' else: boot_meta['kernel_image'] = 'bzImage' boot_meta['loadaddr'] = kernel_addr json_file = 'boot-%s.json' % (platform_name + job_name + '_' + job_id) utils.write_json(json_file, directory, boot_meta) # add by wuyanjun # add the ip device mapping get_ip_board_mapping(job_file, log, directory, report_directory) parser_and_get_result(job_file, log, directory, report_directory, connection) if results and kernel_tree and kernel_version: print 'Creating summary for %s' % (kernel_version) boot = '%s-boot-report.txt' % (kernel_version) if test_plan and ('boot' in test_plan or 'BOOT' in test_plan): boot = boot.replace('boot', test_plan) passed = 0 failed = 0 for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'PASS': passed += 1 else: failed += 1 total = passed + failed with open(os.path.join(report_directory, boot), 'a') as f: f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' % (kernel_tree, str(total), str(passed), str(failed), kernel_version)) f.write('\n') f.write('Total Duration: %.2f minutes\n' % (duration / 60)) f.write('Tree/Branch: %s\n' % kernel_tree) f.write('Git Describe: %s\n' % kernel_version) first = True for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'OFFLINE': if first: f.write('\n') f.write('Boards Offline:\n') first = False f.write('\n') f.write(defconfig) f.write('\n') break for result in results_list: if result['result'] == 'OFFLINE': f.write(' %s %s %s %ss %s: %s\n' % (result['job_id'], result['device_type'], result['device_name'], result['kernel_boot_time'], result['job_name'], result['result'])) f.write('\n') first = True for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'FAIL': if first: f.write('\n') f.write('Failed Boot Tests:\n') first = False f.write('\n') f.write(defconfig) f.write('\n') break for result in results_list: if result['result'] == 'FAIL': f.write(' %s %s %s %ss %s: %s\n' % (result['job_id'], result['device_type'], result['device_name'], result['kernel_boot_time'], result['job_name'], result['result'])) f.write('\n') f.write('Full Boot Report:\n') for defconfig, results_list in results.items(): f.write('\n') f.write(defconfig) f.write('\n') for result in results_list: f.write(' %s %s %s %ss %s: %s\n' % (result['job_id'], result['device_type'], result['device_name'], result['kernel_boot_time'], result['job_name'], result['result']))
def _write_daemon_out(self, output): '''Writes the output to the daemon_out fifo.''' write_file(DAEMON_OUT, output, nodate=True, json_dumps=True)
def boot_report(config): connection, jobs, duration = parse_json(config.get("boot")) # TODO: Fix this when multi-lab sync is working #download_log2html(log2html) results_directory = os.getcwd() + '/results' results = {} utils.mkdir(results_directory) test_plan = None for job_id in jobs: print 'Job ID: %s' % job_id # Init boot_meta = {} api_url = None arch = None board_instance = None boot_retries = 0 kernel_defconfig_full = None kernel_defconfig = None kernel_defconfig_base = None kernel_version = None device_tree = None kernel_endian = None kernel_tree = None kernel_addr = None initrd_addr = None dtb_addr = None dtb_append = None fastboot = None fastboot_cmd = None job_file = '' board_offline = False kernel_boot_time = None boot_failure_reason = None efi_rtc = False # Retrieve job details device_type = '' job_details = connection.scheduler.job_details(job_id) if job_details['requested_device_type_id']: device_type = job_details['requested_device_type_id'] if job_details['description']: job_name = job_details['description'] try: job_short_name = re.search(".*?([A-Z]+.*)", job_name).group(1) except Exception: job_short_name = 'boot-test' device_name = job_details['_actual_device_cache']['hostname'] result = jobs[job_id]['result'] bundle = jobs[job_id]['bundle'] if not device_type: device_type = job_details['_actual_device_cache']['device_type_id'] if bundle is None and device_type == 'dynamic-vm': host_job_id = job_id.replace('.1', '.0') bundle = jobs[host_job_id]['bundle'] if bundle is None: print '%s bundle is empty, skipping...' % device_type continue # Retrieve the log file try: binary_job_file = connection.scheduler.job_output(job_id) except xmlrpclib.Fault: print 'Job output not found for %s' % device_type continue # Parse LAVA messages out of log raw_job_file = str(binary_job_file) for line in raw_job_file.splitlines(): if 'Infrastructure Error:' in line: print 'Infrastructure Error detected!' index = line.find('Infrastructure Error:') boot_failure_reason = line[index:] board_offline = True if 'Bootloader Error:' in line: print 'Bootloader Error detected!' index = line.find('Bootloader Error:') boot_failure_reason = line[index:] board_offline = True if 'Kernel Error:' in line: print 'Kernel Error detected!' index = line.find('Kernel Error:') boot_failure_reason = line[index:] if 'Userspace Error:' in line: print 'Userspace Error detected!' index = line.find('Userspace Error:') boot_failure_reason = line[index:] if '<LAVA_DISPATCHER>' not in line: if len(line) != 0: job_file += line + '\n' if 'rtc-efi rtc-efi: setting system clock to' in line: if device_type == 'dynamic-vm': efi_rtc = True # Retrieve bundle if bundle is not None: json_bundle = connection.dashboard.get(bundle) bundle_data = json.loads(json_bundle['content']) # Get the boot data from LAVA for test_results in bundle_data['test_runs']: # Check for the LAVA self boot test if test_results['test_id'] == 'lava': for test in test_results['test_results']: # TODO for compat :( if test['test_case_id'] == 'kernel_boot_time': kernel_boot_time = test['measurement'] if test['test_case_id'] == 'test_kernel_boot_time': kernel_boot_time = test['measurement'] bundle_attributes = bundle_data['test_runs'][-1][ 'attributes'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'): print bundle_attributes['kernel.defconfig'] if utils.in_bundle_attributes(bundle_attributes, 'target'): board_instance = bundle_attributes['target'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'): kernel_defconfig = bundle_attributes['kernel.defconfig'] defconfig_list = kernel_defconfig.split('-') #arch = defconfig_list[0] arch = defconfig_list[-1] # Remove arch defconfig_list.pop(0) kernel_defconfig_full = '-'.join(defconfig_list) kernel_defconfig_base = ''.join( kernel_defconfig_full.split('+')[:1]) if kernel_defconfig_full == kernel_defconfig_base: kernel_defconfig_full = None if utils.in_bundle_attributes(bundle_attributes, 'kernel.version'): kernel_version = bundle_attributes['kernel.version'] if utils.in_bundle_attributes(bundle_attributes, 'device.tree'): device_tree = bundle_attributes['device.tree'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'): kernel_endian = bundle_attributes['kernel.endian'] if utils.in_bundle_attributes(bundle_attributes, 'platform.fastboot'): fastboot = bundle_attributes['platform.fastboot'] if kernel_boot_time is None: if utils.in_bundle_attributes(bundle_attributes, 'kernel-boot-time'): kernel_boot_time = bundle_attributes['kernel-boot-time'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'): kernel_tree = bundle_attributes['kernel.tree'] if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'): kernel_addr = bundle_attributes['kernel-addr'] if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'): initrd_addr = bundle_attributes['initrd-addr'] if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'): dtb_addr = bundle_attributes['dtb-addr'] if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'): dtb_append = bundle_attributes['dtb-append'] if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'): boot_retries = int(bundle_attributes['boot_retries']) if utils.in_bundle_attributes(bundle_attributes, 'test.plan'): test_tmp = bundle_attributes['test.plan'] if test_tmp: test_plan = test_tmp else: if not kernel_defconfig or not kernel_version or not kernel_tree: job_defnition = {} if 'original_definition' in job_details.keys(): job_definition = job_details['original_definition'] try: job_dictionary = eval(job_definition) except Exception: pass if job_dictionary: if 'actions' in job_dictionary.keys(): actions = job_dictionary['actions'] for i in range(0, len(actions)): try: kernel_defconfig = actions[i]['metadata'][ 'kernel.defconfig'] kernel_version = actions[i]['metadata'][ 'kernel.version'] kernel_tree = actions[i]['metadata'][ 'kernel.tree'] kernel_endian = actions[i]['metadata'][ 'kernel.endian'] platform_fastboot = actions[i]['metadata'][ 'platform.fastboot'] device_tree = actions[i]['metadata'][ 'kernel.tree'] break except KeyError: continue if 'target' in job_details.keys(): print job_details.keys() # Check if we found efi-rtc if test_plan == 'boot-kvm-uefi' and not efi_rtc: if device_type == 'dynamic-vm': boot_failure_reason = 'Unable to read EFI rtc' result = 'FAIL' # Record the boot log and result # TODO: Will need to map device_types to dashboard device types if kernel_defconfig and device_type and result: if ('arm' == arch or 'arm64' == arch) and device_tree is None: platform_name = device_map[device_type][0] + ',legacy' else: if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp': platform_name = device_map[device_type][0] + '_rootfs:nfs' else: platform_name = device_map[device_type][0] print 'Creating boot log for %s' % (platform_name + job_name + '_' + job_id) log = 'boot-%s.txt' % (platform_name + job_name + '_' + job_id) html = 'boot-%s.html' % (platform_name + job_name + '_' + job_id) if config.get("lab"): directory = os.path.join( results_directory, kernel_defconfig + '/' + config.get("lab")) else: directory = os.path.join(results_directory, kernel_defconfig) utils.ensure_dir(directory) utils.write_file(job_file, log, directory) if kernel_boot_time is None: kernel_boot_time = '0.0' if results.has_key(kernel_defconfig): results[kernel_defconfig].append({ 'device_type': platform_name, 'job_id': job_id, 'job_name': job_short_name, 'kernel_boot_time': kernel_boot_time, 'result': result, 'device_name': device_name }) else: results[kernel_defconfig] = [{ 'device_type': platform_name, 'job_id': job_id, 'job_name': job_short_name, 'kernel_boot_time': kernel_boot_time, 'result': result, 'device_name': device_name }] # Create JSON format boot metadata print 'Creating JSON format boot metadata' if config.get("lab"): boot_meta['lab_name'] = config.get("lab") else: boot_meta['lab_name'] = None if board_instance: boot_meta['board_instance'] = board_instance boot_meta['retries'] = boot_retries boot_meta['boot_log'] = log boot_meta['boot_log_html'] = html # TODO: Fix this boot_meta['version'] = '1.0' boot_meta['arch'] = arch boot_meta['defconfig'] = kernel_defconfig_base if kernel_defconfig_full is not None: boot_meta['defconfig_full'] = kernel_defconfig_full if device_map[device_type][1]: boot_meta['mach'] = device_map[device_type][1] boot_meta['kernel'] = kernel_version boot_meta['job'] = kernel_tree boot_meta['board'] = platform_name if board_offline and result == 'FAIL': boot_meta['boot_result'] = 'OFFLINE' #results[kernel_defconfig]['result'] = 'OFFLINE' else: boot_meta['boot_result'] = result if result == 'FAIL' or result == 'OFFLINE': if boot_failure_reason: boot_meta['boot_result_description'] = boot_failure_reason else: boot_meta[ 'boot_result_description'] = 'Unknown Error: platform failed to boot' boot_meta['boot_time'] = kernel_boot_time # TODO: Fix this boot_meta['boot_warnings'] = None if device_tree: if arch == 'arm64': boot_meta['dtb'] = 'dtbs/' + device_map[device_type][ 1] + '/' + device_tree else: boot_meta['dtb'] = 'dtbs/' + device_tree else: boot_meta['dtb'] = device_tree boot_meta['dtb_addr'] = dtb_addr boot_meta['dtb_append'] = dtb_append boot_meta['endian'] = kernel_endian boot_meta['fastboot'] = fastboot # TODO: Fix this boot_meta['initrd'] = None boot_meta['initrd_addr'] = initrd_addr if arch == 'arm': boot_meta['kernel_image'] = 'zImage' elif arch == 'arm64': boot_meta['kernel_image'] = 'Image' else: boot_meta['kernel_image'] = 'bzImage' boot_meta['loadaddr'] = kernel_addr json_file = 'boot-%s.json' % (platform_name + job_name + '_' + job_id) utils.write_json(json_file, directory, boot_meta) if config.get("lab"): report_directory = os.path.join(results_directory, config.get("lab")) utils.mkdir(report_directory) else: report_directory = results_directory if results and kernel_tree and kernel_version: print 'Creating summary for %s' % (kernel_version) boot = '%s-boot-report.txt' % (kernel_version) if test_plan and ('boot' in test_plan or 'BOOT' in test_plan): boot = boot.replace('boot', test_plan) passed = 0 failed = 0 for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'PASS': passed += 1 else: failed += 1 total = passed + failed with open(os.path.join(report_directory, boot), 'a') as f: f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' % (kernel_tree, str(total), str(passed), str(failed), kernel_version)) f.write('\n') f.write('Total Duration: %.2f minutes\n' % (duration / 60)) f.write('Tree/Branch: %s\n' % kernel_tree) f.write('Git Describe: %s\n' % kernel_version) first = True for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'OFFLINE': if first: f.write('\n') f.write('Boards Offline:\n') first = False f.write('\n') f.write(defconfig) f.write('\n') break for result in results_list: if result['result'] == 'OFFLINE': f.write( ' %s %s %s %ss %s: %s\n' % (result['job_id'], result['device_type'], result['device_name'], result['kernel_boot_time'], result['job_name'], result['result'])) f.write('\n') first = True for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'FAIL': if first: f.write('\n') f.write('Failed Boot Tests:\n') first = False f.write('\n') f.write(defconfig) f.write('\n') break for result in results_list: if result['result'] == 'FAIL': f.write( ' %s %s %s %ss %s: %s\n' % (result['job_id'], result['device_type'], result['device_name'], result['kernel_boot_time'], result['job_name'], result['result'])) f.write('\n') f.write('Full Boot Report:\n') for defconfig, results_list in results.items(): f.write('\n') f.write(defconfig) f.write('\n') for result in results_list: f.write(' %s %s %s %ss %s: %s\n' % (result['job_id'], result['device_type'], result['device_name'], result['kernel_boot_time'], result['job_name'], result['result'])) # add by wuyanjun if results and directory: parser_and_get_result(results, directory, report_directory) get_ip_board_mapping(results, directory, report_directory)
def boot_report(config): connection, jobs, duration = parse_json(config.get("boot")) # TODO: Fix this when multi-lab sync is working #download_log2html(log2html) results_directory = os.getcwd() + '/results' results = {} dt_tests = False utils.mkdir(results_directory) for job_id in jobs: print 'Job ID: %s' % job_id # Init boot_meta = {} api_url = None arch = None board_instance = None boot_retries = 0 kernel_defconfig_full = None kernel_defconfig = None kernel_defconfig_base = None kernel_version = None device_tree = None kernel_endian = None kernel_tree = None kernel_addr = None initrd_addr = None dtb_addr = None dtb_append = None fastboot = None fastboot_cmd = None test_plan = None job_file = '' dt_test = None dt_test_result = None dt_tests_passed = None dt_tests_failed = None board_offline = False kernel_boot_time = None boot_failure_reason = None efi_rtc = False # Retrieve job details job_details = connection.scheduler.job_details(job_id) if job_details['requested_device_type_id']: device_type = job_details['requested_device_type_id'] if job_details['description']: job_name = job_details['description'] result = jobs[job_id]['result'] bundle = jobs[job_id]['bundle'] if bundle is None and device_type == 'dynamic-vm': host_job_id = job_id.replace('.1', '.0') bundle = jobs[host_job_id]['bundle'] if bundle is None: print '%s bundle is empty, skipping...' % device_type continue # Retrieve the log file try: binary_job_file = connection.scheduler.job_output(job_id) except xmlrpclib.Fault: print 'Job output not found for %s' % device_type continue # Parse LAVA messages out of log raw_job_file = str(binary_job_file) for line in raw_job_file.splitlines(): if 'Infrastructure Error:' in line: print 'Infrastructure Error detected!' index = line.find('Infrastructure Error:') boot_failure_reason = line[index:] board_offline = True if 'Bootloader Error:' in line: print 'Bootloader Error detected!' index = line.find('Bootloader Error:') boot_failure_reason = line[index:] board_offline = True if 'Kernel Error:' in line: print 'Kernel Error detected!' index = line.find('Kernel Error:') boot_failure_reason = line[index:] if 'Userspace Error:' in line: print 'Userspace Error detected!' index = line.find('Userspace Error:') boot_failure_reason = line[index:] if '<LAVA_DISPATCHER>' not in line: if len(line) != 0: job_file += line + '\n' if '### dt-test ### end of selftest' in line: dt_tests = True regex = re.compile("(?P<test>\d+\*?)") dt_test_results = regex.findall(line) if len(dt_test_results) > 2: dt_tests_passed = dt_test_results[2] dt_tests_failed = dt_test_results[3] else: dt_tests_passed = dt_test_results[0] dt_tests_failed = dt_test_results[1] if int(dt_tests_failed) > 0: dt_test_result = 'FAIL' else: dt_test_result = 'PASS' if 'rtc-efi rtc-efi: setting system clock to' in line: if device_type == 'dynamic-vm': efi_rtc = True # Retrieve bundle if bundle is not None: json_bundle = connection.dashboard.get(bundle) bundle_data = json.loads(json_bundle['content']) # Get the boot data from LAVA for test_results in bundle_data['test_runs']: # Check for the LAVA self boot test if test_results['test_id'] == 'lava': for test in test_results['test_results']: # TODO for compat :( if test['test_case_id'] == 'kernel_boot_time': kernel_boot_time = test['measurement'] if test['test_case_id'] == 'test_kernel_boot_time': kernel_boot_time = test['measurement'] bundle_attributes = bundle_data['test_runs'][-1]['attributes'] if test_results['test_id'] == 'lava-command': # Post stuff with the test API. build_id = "56b9648659b514b7f6e41fac" test_suite = test_api_post(config, build_id, test_results) if test_suite: print json.dumps(test_suite) print 'Sending test suite to %s' % config.get("api") headers = { 'Authorization': config.get("token"), 'Content-Type': 'application/json' } api_url = urlparse.urljoin(config.get("api"), '/test/suite') push('POST', api_url, json.dumps(test_suite), headers) if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'): print bundle_attributes['kernel.defconfig'] if utils.in_bundle_attributes(bundle_attributes, 'target'): board_instance = bundle_attributes['target'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'): kernel_defconfig = bundle_attributes['kernel.defconfig'] defconfig_list = kernel_defconfig.split('-') arch = defconfig_list[0] # Remove arch defconfig_list.pop(0) kernel_defconfig_full = '-'.join(defconfig_list) kernel_defconfig_base = ''.join(kernel_defconfig_full.split('+')[:1]) if kernel_defconfig_full == kernel_defconfig_base: kernel_defconfig_full = None if utils.in_bundle_attributes(bundle_attributes, 'kernel.version'): kernel_version = bundle_attributes['kernel.version'] if utils.in_bundle_attributes(bundle_attributes, 'device.tree'): device_tree = bundle_attributes['device.tree'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'): kernel_endian = bundle_attributes['kernel.endian'] if utils.in_bundle_attributes(bundle_attributes, 'platform.fastboot'): fastboot = bundle_attributes['platform.fastboot'] if kernel_boot_time is None: if utils.in_bundle_attributes(bundle_attributes, 'kernel-boot-time'): kernel_boot_time = bundle_attributes['kernel-boot-time'] if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'): kernel_tree = bundle_attributes['kernel.tree'] if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'): kernel_addr = bundle_attributes['kernel-addr'] if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'): initrd_addr = bundle_attributes['initrd-addr'] if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'): dtb_addr = bundle_attributes['dtb-addr'] if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'): dtb_append = bundle_attributes['dtb-append'] if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'): boot_retries = int(bundle_attributes['boot_retries']) if utils.in_bundle_attributes(bundle_attributes, 'test.plan'): test_plan = bundle_attributes['test.plan'] # Check if we found efi-rtc if test_plan == 'boot-kvm-uefi' and not efi_rtc: if device_type == 'dynamic-vm': boot_failure_reason = 'Unable to read EFI rtc' result = 'FAIL' # Record the boot log and result # TODO: Will need to map device_types to dashboard device types if kernel_defconfig and device_type and result: if (arch == 'arm' or arch =='arm64') and device_tree is None: platform_name = device_map[device_type][0] + ',legacy' else: if device_tree == 'vexpress-v2p-ca15_a7.dtb': platform_name = 'vexpress-v2p-ca15_a7' elif device_tree == 'fsl-ls2080a-simu.dtb': platform_name = 'fsl-ls2080a-simu' elif test_plan == 'boot-kvm' or test_plan == 'boot-kvm-uefi': if device_tree == 'sun7i-a20-cubietruck.dtb': if device_type == 'dynamic-vm': device_type = 'cubieboard3-kvm-guest' platform_name = device_map[device_type][0] else: device_type = 'cubieboard3-kvm-host' platform_name = device_map[device_type][0] elif device_tree == 'apm-mustang.dtb': if device_type == 'dynamic-vm': if test_plan == 'boot-kvm-uefi': device_type = 'mustang-kvm-uefi-guest' else: device_type = 'mustang-kvm-guest' platform_name = device_map[device_type][0] else: if test_plan == 'boot-kvm-uefi': device_type = 'mustang-kvm-uefi-host' else: device_type = 'mustang-kvm-host' platform_name = device_map[device_type][0] elif device_tree == 'juno.dtb': if device_type == 'dynamic-vm': if test_plan == 'boot-kvm-uefi': device_type = 'juno-kvm-uefi-guest' else: device_type = 'juno-kvm-guest' platform_name = device_map[device_type][0] else: if test_plan == 'boot-kvm-uefi': device_type = 'juno-kvm-uefi-host' else: device_type = 'juno-kvm-host' platform_name = device_map[device_type][0] elif test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp': platform_name = device_map[device_type][0] + '_rootfs:nfs' else: platform_name = device_map[device_type][0] print 'Creating boot log for %s' % platform_name log = 'boot-%s.txt' % platform_name html = 'boot-%s.html' % platform_name if config.get("lab"): directory = os.path.join(results_directory, kernel_defconfig + '/' + config.get("lab")) else: directory = os.path.join(results_directory, kernel_defconfig) utils.ensure_dir(directory) utils.write_file(job_file, log, directory) if kernel_boot_time is None: kernel_boot_time = '0.0' if results.has_key(kernel_defconfig): results[kernel_defconfig].append({'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result}) else: results[kernel_defconfig] = [{'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result}] # Create JSON format boot metadata print 'Creating JSON format boot metadata' if config.get("lab"): boot_meta['lab_name'] = config.get("lab") else: boot_meta['lab_name'] = None if board_instance: boot_meta['board_instance'] = board_instance boot_meta['retries'] = boot_retries boot_meta['boot_log'] = log boot_meta['boot_log_html'] = html # TODO: Fix this boot_meta['version'] = '1.0' boot_meta['arch'] = arch boot_meta['defconfig'] = kernel_defconfig_base if kernel_defconfig_full is not None: boot_meta['defconfig_full'] = kernel_defconfig_full if device_map[device_type][1]: boot_meta['mach'] = device_map[device_type][1] boot_meta['kernel'] = kernel_version boot_meta['job'] = kernel_tree boot_meta['board'] = platform_name if board_offline and result == 'FAIL': boot_meta['boot_result'] = 'OFFLINE' #results[kernel_defconfig]['result'] = 'OFFLINE' else: boot_meta['boot_result'] = result if result == 'FAIL' or result == 'OFFLINE': if boot_failure_reason: boot_meta['boot_result_description'] = boot_failure_reason else: boot_meta['boot_result_description'] = 'Unknown Error: platform failed to boot' boot_meta['boot_time'] = kernel_boot_time # TODO: Fix this boot_meta['boot_warnings'] = None if device_tree: if arch == 'arm64': boot_meta['dtb'] = 'dtbs/' + device_map[device_type][1] + '/' + device_tree else: boot_meta['dtb'] = 'dtbs/' + device_tree else: boot_meta['dtb'] = device_tree boot_meta['dtb_addr'] = dtb_addr boot_meta['dtb_append'] = dtb_append boot_meta['dt_test'] = dt_test boot_meta['endian'] = kernel_endian boot_meta['fastboot'] = fastboot # TODO: Fix this boot_meta['initrd'] = None boot_meta['initrd_addr'] = initrd_addr if arch == 'arm': boot_meta['kernel_image'] = 'zImage' elif arch == 'arm64': boot_meta['kernel_image'] = 'Image' else: boot_meta['kernel_image'] = 'bzImage' boot_meta['loadaddr'] = kernel_addr json_file = 'boot-%s.json' % platform_name utils.write_json(json_file, directory, boot_meta) print 'Creating html version of boot log for %s' % platform_name cmd = 'python log2html.py %s' % os.path.join(directory, log) subprocess.check_output(cmd, shell=True) if config.get("lab") and config.get("api") and config.get("token"): print 'Sending boot result to %s for %s' % (config.get("api"), platform_name) headers = { 'Authorization': config.get("token"), 'Content-Type': 'application/json' } api_url = urlparse.urljoin(config.get("api"), '/boot') push('POST', api_url, data=json.dumps(boot_meta), headers=headers) headers = { 'Authorization': config.get("token"), } print 'Uploading text version of boot log' with open(os.path.join(directory, log)) as lh: data = lh.read() api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s' % (kernel_tree, kernel_version, kernel_defconfig, config.get("lab"), log)) push('PUT', api_url, data=data, headers=headers) print 'Uploading html version of boot log' with open(os.path.join(directory, html)) as lh: data = lh.read() api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s' % (kernel_tree, kernel_version, kernel_defconfig, config.get("lab"), html)) push('PUT', api_url, data=data, headers=headers) if results and kernel_tree and kernel_version: print 'Creating boot summary for %s' % kernel_version boot = '%s-boot-report.txt' % kernel_version passed = 0 failed = 0 for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'PASS': passed += 1 else: failed += 1 total = passed + failed if config.get("lab"): report_directory = os.path.join(results_directory, config.get("lab")) utils.mkdir(report_directory) else: report_directory = results_directory with open(os.path.join(report_directory, boot), 'a') as f: f.write('To: %s\n' % config.get("email")) f.write('From: [email protected]\n') f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' % (kernel_tree, str(total), str(passed), str(failed), kernel_version)) f.write('\n') f.write('Full Build Report: http://kernelci.org/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version)) f.write('Full Boot Report: http://kernelci.org/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version)) f.write('\n') f.write('Total Duration: %.2f minutes\n' % (duration / 60)) f.write('Tree/Branch: %s\n' % kernel_tree) f.write('Git Describe: %s\n' % kernel_version) first = True for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'OFFLINE': if first: f.write('\n') f.write('Boards Offline:\n') first = False f.write('\n') f.write(defconfig) f.write('\n') break for result in results_list: if result['result'] == 'OFFLINE': f.write(' %s %ss boot-test: %s\n' % (result['device_type'], result['kernel_boot_time'], result['result'])) f.write('\n') first = True for defconfig, results_list in results.items(): for result in results_list: if result['result'] == 'FAIL': if first: f.write('\n') f.write('Failed Boot Tests:\n') first = False f.write('\n') f.write(defconfig) f.write('\n') break for result in results_list: if result['result'] == 'FAIL': f.write(' %s %ss boot-test: %s\n' % (result['device_type'], result['kernel_boot_time'], result['result'])) if config.get("lab"): f.write(' http://storage.kernelci.org/kernel-ci/%s/%s/%s/%s/boot-%s.html' % (kernel_tree, kernel_version, defconfig, config.get("lab"), result['device_type'])) else: f.write(' http://storage.kernelci.org/kernel-ci/%s/%s/%s/boot-%s.html' % (kernel_tree, kernel_version, defconfig, result['device_type'])) f.write('\n') f.write('\n') f.write('Full Boot Report:\n') for defconfig, results_list in results.items(): f.write('\n') f.write(defconfig) f.write('\n') for result in results_list: f.write(' %s %ss boot-test: %s\n' % (result['device_type'], result['kernel_boot_time'], result['result'])) # dt-self-test if results and kernel_tree and kernel_version and dt_tests: print 'Creating device tree runtime self test summary for %s' % kernel_version dt_self_test = '%s-dt-runtime-self-test-report.txt' % kernel_version passed = 0 failed = 0 for defconfig, results_list in results.items(): for result in results_list: if result['dt_test_result'] == 'PASS': passed += 1 elif result['dt_test_result'] == 'FAIL': failed += 1 total = passed + failed with open(os.path.join(report_directory, dt_self_test), 'a') as f: f.write('To: %s\n' % config.get("email")) f.write('From: [email protected]\n') f.write('Subject: %s dt-runtime-unit-tests: %s boards tested: %s passed, %s failed (%s)\n' % (kernel_tree, str(total), str(passed), str(failed), kernel_version)) f.write('\n') f.write('Full Build Report: http://kernelci.org/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version)) f.write('Full Boot Report: http://kernelci.org/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version)) f.write('Full Test Report: http://kernelci.org/test/%s/kernel/%s/\n' % (kernel_tree, kernel_version)) f.write('\n') f.write('Tree/Branch: %s\n' % kernel_tree) f.write('Git Describe: %s\n' % kernel_version) first = True for defconfig, results_list in results.items(): for result in results_list: if result['dt_test_result'] == 'FAIL': if first: f.write('\n') f.write('Failed Device Tree Unit Tests:\n') first = False f.write('\n') f.write(defconfig) f.write('\n') break for result in results_list: if result['dt_test_result'] == "FAIL": f.write(' %s passed: %s / failed: %s dt-runtime-unit-tests: %s\n' % (result['device_type'], result['dt_tests_passed'], result['dt_tests_failed'], result['dt_test_result'])) if config.get("lab"): f.write(' http://storage.kernelci.org/kernel-ci/%s/%s/%s/%s/boot-%s.html' % (kernel_tree, kernel_version, defconfig, config.get("lab"), result['device_type'])) else: f.write(' http://storage.kernelci.org/kernel-ci/%s/%s/%s/boot-%s.html' % (kernel_tree, kernel_version, defconfig, result['device_type'])) f.write('\n') f.write('\n') f.write('Full Unit Test Report:\n') for defconfig, results_list in results.items(): first = True for result in results_list: if result['dt_test_result']: if first: f.write('\n') f.write(defconfig) f.write('\n') first = False f.write(' %s passed: %s / failed: %s dt-runtime-unit-tests: %s\n' % (result['device_type'], result['dt_tests_passed'], result['dt_tests_failed'], result['dt_test_result'])) # sendmail if config.get("email"): print 'Sending e-mail summary to %s' % config.get("email") if os.path.exists(report_directory): cmd = 'cat %s | sendmail -t' % os.path.join(report_directory, boot) subprocess.check_output(cmd, shell=True) if dt_tests: if os.path.exists(report_directory): cmd = 'cat %s | sendmail -t' % os.path.join(report_directory, dt_self_test) subprocess.check_output(cmd, shell=True)