Esempio n. 1
0
def create_models(num_networks, hyperparameter_ranges_file):
    identifier = datetime.datetime.now().strftime('%Y%m%d%H%M%S')

    data_is_target_list = [0]
    num_scat_list = [1, 2, 3]
    batch_size_list = [32]
    # data_noise_gaussian_list = [0, 1]
    data_noise_gaussian_list = [1]  # Decided on 11/22/2018 b/c better models
    #dropout_input_list = [0, 0.1, 0.2]
    #dropout_list = [0, 0.1, 0.2, 0.3, 0.4, 0.5]
    weight_decay_list = [0]

    for count in range(num_networks):
        data_is_target = choice(data_is_target_list)
        n_scat = choice(num_scat_list)
        bs = choice(batch_size_list)
        data_noise_gaussian = choice(data_noise_gaussian_list)
        #dropout_input = choice(dropout_input_list)
        weight_decay = choice(weight_decay_list)

        # get params
        model_params = choose_hyperparameters_from_file(
            hyperparameter_ranges_file)

        # set other params
        model_params['data_is_target'] = data_is_target
        home = os.path.expanduser('~')
        model_params['data_train'] = os.path.join(
            home, 'Downloads', '20180402_L74_70mm',
            'train_' + str(n_scat) + '.h5')
        model_params['data_val'] = os.path.join(home, 'Downloads',
                                                '20180402_L74_70mm',
                                                'val_' + str(n_scat) + '.h5')
        model_params['batch_size'] = bs
        model_params['data_noise_gaussian'] = data_noise_gaussian
        #model_params['dropout_input'] = dropout_input
        model_params['weight_decay'] = weight_decay
        model_params['patience'] = 20
        model_params['cuda'] = 1
        model_params['save_initial'] = 0

        k_list = [3, 4, 5]

        for k in k_list:
            model_params['k'] = k
            model_params['save_dir'] = os.path.join(
                'DNNs', identifier + '_' + str(count + 1) + '_created',
                'k_' + str(k))

            # print(model_params['save_dir'])
            ensure_dir(model_params['save_dir'])
            save_model_params(
                os.path.join(model_params['save_dir'], model_params_fname),
                model_params)

        print('create_models: created model {}_{}'.format(identifier, count))

    return identifier
Esempio n. 2
0
 def get_user_dir(self):
     """
     Get a data folder for the current user.
     Creates folder if it does not exist.
     """
     user_dir = '%s/icfhome/%s' % (os.path.dirname(__file__),
                                   self.current_user)
     ensure_dir(user_dir)
     return user_dir
Esempio n. 3
0
def get_and_save_model_dict(kernel_width, num_kernels, num_layers, index):
    model_dict = get_model_dict(kernel_width, num_kernels, num_layers)
    for k in [3, 4, 5]:
        model_dict['k'] = k
        identifier = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
        model_dict['save_dir'] = os_path_join(
            'DNNs', 'fcnn_v1.6.7_{}_{}_created'.format(identifier, index),
            'k_{}'.format(k))

        # print(model_params['save_dir'])
        ensure_dir(model_dict['save_dir'])
        save_model_params(
            os_path_join(model_dict['save_dir'], MODEL_PARAMS_FNAME),
            model_dict)
        print('created ', model_dict['save_dir'])
Esempio n. 4
0
def create_models(num_networks, hyperparameter_ranges_file):
    identifier = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')

    for count in range(num_networks):
        model_params = choose_hyperparameters_from_file(hyperparameter_ranges_file)

        model_params['train_features'] = os.path.join('data', 'train-images-idx3-ubyte.gz')
        model_params['train_labels'] = os.path.join('data', 'train-labels-idx3-ubyte.gz')
        model_params['valid_features'] = os.path.join('data', 't10k-images-idx3-ubyte.gz')
        model_params['valid_labels'] = os.path.join('data', 't10k-labels-idx3-ubyte.gz')

        model_params['save_dir'] = os.path.join(trained_models_folder, identifier + '_' + str(count+1) + '_created')
        ensure_dir(model_params['save_dir'])
        save_model_params(os.path.join(model_params['save_dir'], 'model_params.txt'), model_params)

    return identifier
Esempio n. 5
0
def boot_report(config):
    connection, jobs, duration = parse_json(config.get("boot"))
    # TODO: Fix this when multi-lab sync is working
    #download_log2html(log2html)
    results_directory = os.getcwd() + '/results'
    results = {}
    utils.mkdir(results_directory)
    test_plan = None
    for job_id in jobs:
        print 'Job ID: %s' % job_id
        # Init
        boot_meta = {}
        api_url = None
        arch = None
        board_instance = None
        boot_retries = 0
        kernel_defconfig_full = None
        kernel_defconfig = None
        kernel_defconfig_base = None
        kernel_version = None
        device_tree = None
        kernel_endian = None
        kernel_tree = None
        kernel_addr = None
        initrd_addr = None
        dtb_addr = None
        dtb_append = None
        fastboot = None
        fastboot_cmd = None
        job_file = ''
        board_offline = False
        kernel_boot_time = None
        boot_failure_reason = None
        efi_rtc = False
        # Retrieve job details
        device_type = ''
        job_details = connection.scheduler.job_details(job_id)
        if job_details['requested_device_type_id']:
            device_type = job_details['requested_device_type_id']
        if job_details['description']:
            job_name = job_details['description']
            try:
                job_short_name = re.search(".*?([A-Z]+.*)", job_name).group(1)
            except Exception:
                job_short_name = 'boot-test'
        device_name = job_details['_actual_device_cache']['hostname']
        result = jobs[job_id]['result']
        bundle = jobs[job_id]['bundle']
        if not device_type:
            device_type = job_details['_actual_device_cache']['device_type_id']
        if bundle is None and device_type == 'dynamic-vm':
            host_job_id = job_id.replace('.1', '.0')
            bundle = jobs[host_job_id]['bundle']
            if bundle is None:
                print '%s bundle is empty, skipping...' % device_type
                continue
        # Retrieve the log file
        try:
            binary_job_file = connection.scheduler.job_output(job_id)
        except xmlrpclib.Fault:
            print 'Job output not found for %s' % device_type
            continue
        # Parse LAVA messages out of log
        raw_job_file = str(binary_job_file)
        for line in raw_job_file.splitlines():
            if 'Infrastructure Error:' in line:
                print 'Infrastructure Error detected!'
                index = line.find('Infrastructure Error:')
                boot_failure_reason = line[index:]
                board_offline = True
            if 'Bootloader Error:' in line:
                print 'Bootloader Error detected!'
                index = line.find('Bootloader Error:')
                boot_failure_reason = line[index:]
                board_offline = True
            if 'Kernel Error:' in line:
                print 'Kernel Error detected!'
                index = line.find('Kernel Error:')
                boot_failure_reason = line[index:]
            if 'Userspace Error:' in line:
                print 'Userspace Error detected!'
                index = line.find('Userspace Error:')
                boot_failure_reason = line[index:]
            if '<LAVA_DISPATCHER>' not in line:
                if len(line) != 0:
                    job_file += line + '\n'
            if 'rtc-efi rtc-efi: setting system clock to' in line:
                if device_type == 'dynamic-vm':
                    efi_rtc = True

        # Retrieve bundle
        if bundle is not None:
            json_bundle = connection.dashboard.get(bundle)
            bundle_data = json.loads(json_bundle['content'])
            # Get the boot data from LAVA
            for test_results in bundle_data['test_runs']:
                # Check for the LAVA self boot test
                if test_results['test_id'] == 'lava':
                    for test in test_results['test_results']:
                        # TODO for compat :(
                        if test['test_case_id'] == 'kernel_boot_time':
                            kernel_boot_time = test['measurement']
                        if test['test_case_id'] == 'test_kernel_boot_time':
                            kernel_boot_time = test['measurement']
                    bundle_attributes = bundle_data['test_runs'][-1][
                        'attributes']
            if utils.in_bundle_attributes(bundle_attributes,
                                          'kernel.defconfig'):
                print bundle_attributes['kernel.defconfig']
            if utils.in_bundle_attributes(bundle_attributes, 'target'):
                board_instance = bundle_attributes['target']
            if utils.in_bundle_attributes(bundle_attributes,
                                          'kernel.defconfig'):
                kernel_defconfig = bundle_attributes['kernel.defconfig']
                defconfig_list = kernel_defconfig.split('-')
                #arch = defconfig_list[0]
                arch = defconfig_list[-1]
                # Remove arch
                defconfig_list.pop(0)
                kernel_defconfig_full = '-'.join(defconfig_list)
                kernel_defconfig_base = ''.join(
                    kernel_defconfig_full.split('+')[:1])
                if kernel_defconfig_full == kernel_defconfig_base:
                    kernel_defconfig_full = None
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.version'):
                kernel_version = bundle_attributes['kernel.version']
            if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
                device_tree = bundle_attributes['device.tree']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
                kernel_endian = bundle_attributes['kernel.endian']
            if utils.in_bundle_attributes(bundle_attributes,
                                          'platform.fastboot'):
                fastboot = bundle_attributes['platform.fastboot']
            if kernel_boot_time is None:
                if utils.in_bundle_attributes(bundle_attributes,
                                              'kernel-boot-time'):
                    kernel_boot_time = bundle_attributes['kernel-boot-time']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
                kernel_tree = bundle_attributes['kernel.tree']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
                kernel_addr = bundle_attributes['kernel-addr']
            if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
                initrd_addr = bundle_attributes['initrd-addr']
            if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
                dtb_addr = bundle_attributes['dtb-addr']
            if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
                dtb_append = bundle_attributes['dtb-append']
            if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
                boot_retries = int(bundle_attributes['boot_retries'])
            if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
                test_tmp = bundle_attributes['test.plan']
                if test_tmp:
                    test_plan = test_tmp
        else:
            if not kernel_defconfig or not kernel_version or not kernel_tree:
                job_defnition = {}
                if 'original_definition' in job_details.keys():
                    job_definition = job_details['original_definition']
                    try:
                        job_dictionary = eval(job_definition)
                    except Exception:
                        pass
                    if job_dictionary:
                        if 'actions' in job_dictionary.keys():
                            actions = job_dictionary['actions']
                            for i in range(0, len(actions)):
                                try:
                                    kernel_defconfig = actions[i]['metadata'][
                                        'kernel.defconfig']
                                    kernel_version = actions[i]['metadata'][
                                        'kernel.version']
                                    kernel_tree = actions[i]['metadata'][
                                        'kernel.tree']
                                    kernel_endian = actions[i]['metadata'][
                                        'kernel.endian']
                                    platform_fastboot = actions[i]['metadata'][
                                        'platform.fastboot']
                                    device_tree = actions[i]['metadata'][
                                        'kernel.tree']
                                    break
                                except KeyError:
                                    continue
                if 'target' in job_details.keys():
                    print job_details.keys()
        # Check if we found efi-rtc
        if test_plan == 'boot-kvm-uefi' and not efi_rtc:
            if device_type == 'dynamic-vm':
                boot_failure_reason = 'Unable to read EFI rtc'
                result = 'FAIL'

        # Record the boot log and result
        # TODO: Will need to map device_types to dashboard device types
        if kernel_defconfig and device_type and result:
            if ('arm' == arch or 'arm64' == arch) and device_tree is None:
                platform_name = device_map[device_type][0] + ',legacy'
            else:
                if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
                    platform_name = device_map[device_type][0] + '_rootfs:nfs'
                else:
                    platform_name = device_map[device_type][0]

            print 'Creating boot log for %s' % (platform_name + job_name +
                                                '_' + job_id)
            log = 'boot-%s.txt' % (platform_name + job_name + '_' + job_id)
            html = 'boot-%s.html' % (platform_name + job_name + '_' + job_id)
            if config.get("lab"):
                directory = os.path.join(
                    results_directory,
                    kernel_defconfig + '/' + config.get("lab"))
            else:
                directory = os.path.join(results_directory, kernel_defconfig)
            utils.ensure_dir(directory)
            utils.write_file(job_file, log, directory)
            if kernel_boot_time is None:
                kernel_boot_time = '0.0'
            if results.has_key(kernel_defconfig):
                results[kernel_defconfig].append({
                    'device_type': platform_name,
                    'job_id': job_id,
                    'job_name': job_short_name,
                    'kernel_boot_time': kernel_boot_time,
                    'result': result,
                    'device_name': device_name
                })
            else:
                results[kernel_defconfig] = [{
                    'device_type': platform_name,
                    'job_id': job_id,
                    'job_name': job_short_name,
                    'kernel_boot_time': kernel_boot_time,
                    'result': result,
                    'device_name': device_name
                }]
            # Create JSON format boot metadata
            print 'Creating JSON format boot metadata'
            if config.get("lab"):
                boot_meta['lab_name'] = config.get("lab")
            else:
                boot_meta['lab_name'] = None
            if board_instance:
                boot_meta['board_instance'] = board_instance
            boot_meta['retries'] = boot_retries
            boot_meta['boot_log'] = log
            boot_meta['boot_log_html'] = html
            # TODO: Fix this
            boot_meta['version'] = '1.0'
            boot_meta['arch'] = arch
            boot_meta['defconfig'] = kernel_defconfig_base
            if kernel_defconfig_full is not None:
                boot_meta['defconfig_full'] = kernel_defconfig_full
            if device_map[device_type][1]:
                boot_meta['mach'] = device_map[device_type][1]
            boot_meta['kernel'] = kernel_version

            boot_meta['job'] = kernel_tree
            boot_meta['board'] = platform_name
            if board_offline and result == 'FAIL':
                boot_meta['boot_result'] = 'OFFLINE'
                #results[kernel_defconfig]['result'] = 'OFFLINE'
            else:
                boot_meta['boot_result'] = result
            if result == 'FAIL' or result == 'OFFLINE':
                if boot_failure_reason:
                    boot_meta['boot_result_description'] = boot_failure_reason
                else:
                    boot_meta[
                        'boot_result_description'] = 'Unknown Error: platform failed to boot'
            boot_meta['boot_time'] = kernel_boot_time
            # TODO: Fix this
            boot_meta['boot_warnings'] = None
            if device_tree:
                if arch == 'arm64':
                    boot_meta['dtb'] = 'dtbs/' + device_map[device_type][
                        1] + '/' + device_tree
                else:
                    boot_meta['dtb'] = 'dtbs/' + device_tree
            else:
                boot_meta['dtb'] = device_tree
            boot_meta['dtb_addr'] = dtb_addr
            boot_meta['dtb_append'] = dtb_append
            boot_meta['endian'] = kernel_endian
            boot_meta['fastboot'] = fastboot
            # TODO: Fix this
            boot_meta['initrd'] = None
            boot_meta['initrd_addr'] = initrd_addr
            if arch == 'arm':
                boot_meta['kernel_image'] = 'zImage'
            elif arch == 'arm64':
                boot_meta['kernel_image'] = 'Image'
            else:
                boot_meta['kernel_image'] = 'bzImage'
            boot_meta['loadaddr'] = kernel_addr
            json_file = 'boot-%s.json' % (platform_name + job_name + '_' +
                                          job_id)
            utils.write_json(json_file, directory, boot_meta)

    if config.get("lab"):
        report_directory = os.path.join(results_directory, config.get("lab"))
        utils.mkdir(report_directory)
    else:
        report_directory = results_directory

    if results and kernel_tree and kernel_version:
        print 'Creating summary for %s' % (kernel_version)
        boot = '%s-boot-report.txt' % (kernel_version)
        if test_plan and ('boot' in test_plan or 'BOOT' in test_plan):
            boot = boot.replace('boot', test_plan)
        passed = 0
        failed = 0
        for defconfig, results_list in results.items():
            for result in results_list:
                if result['result'] == 'PASS':
                    passed += 1
                else:
                    failed += 1
        total = passed + failed
        with open(os.path.join(report_directory, boot), 'a') as f:
            f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' %
                    (kernel_tree, str(total), str(passed), str(failed),
                     kernel_version))
            f.write('\n')
            f.write('Total Duration: %.2f minutes\n' % (duration / 60))
            f.write('Tree/Branch: %s\n' % kernel_tree)
            f.write('Git Describe: %s\n' % kernel_version)
            first = True
            for defconfig, results_list in results.items():
                for result in results_list:
                    if result['result'] == 'OFFLINE':
                        if first:
                            f.write('\n')
                            f.write('Boards Offline:\n')
                            first = False
                        f.write('\n')
                        f.write(defconfig)
                        f.write('\n')
                        break
                for result in results_list:
                    if result['result'] == 'OFFLINE':
                        f.write(
                            '    %s   %s   %s   %ss   %s: %s\n' %
                            (result['job_id'], result['device_type'],
                             result['device_name'], result['kernel_boot_time'],
                             result['job_name'], result['result']))
                        f.write('\n')
            first = True
            for defconfig, results_list in results.items():
                for result in results_list:
                    if result['result'] == 'FAIL':
                        if first:
                            f.write('\n')
                            f.write('Failed Boot Tests:\n')
                            first = False
                        f.write('\n')
                        f.write(defconfig)
                        f.write('\n')
                        break
                for result in results_list:
                    if result['result'] == 'FAIL':
                        f.write(
                            '    %s   %s   %s   %ss   %s: %s\n' %
                            (result['job_id'], result['device_type'],
                             result['device_name'], result['kernel_boot_time'],
                             result['job_name'], result['result']))
            f.write('\n')
            f.write('Full Boot Report:\n')
            for defconfig, results_list in results.items():
                f.write('\n')
                f.write(defconfig)
                f.write('\n')
                for result in results_list:
                    f.write('    %s   %s   %s   %ss   %s: %s\n' %
                            (result['job_id'], result['device_type'],
                             result['device_name'], result['kernel_boot_time'],
                             result['job_name'], result['result']))
    # add by wuyanjun
    if results and directory:
        parser_and_get_result(results, directory, report_directory)
        get_ip_board_mapping(results, directory, report_directory)
Esempio n. 6
0
def boot_report(config):
    connection, jobs, duration =  parse_json(config.get("boot"))
    # TODO: Fix this when multi-lab sync is working
    #download_log2html(log2html)
    results_directory = os.getcwd() + '/results'
    results = {}
    utils.mkdir(results_directory)
    test_plan = None

    if config.get("lab"):
        report_directory = os.path.join(results_directory, config.get("lab"))
    else:
        report_directory = results_directory
    if os.path.exists(report_directory):
        shutil.rmtree(report_directory)
    utils.mkdir(report_directory)

    for job_id in jobs:
        print 'Job ID: %s' % job_id
        # Init
        boot_meta = {}
        api_url = None
        arch = None
        board_instance = None
        boot_retries = 0
        kernel_defconfig_full = None
        kernel_defconfig = None
        kernel_defconfig_base = None
        kernel_version = None
        device_tree = None
        kernel_endian = None
        kernel_tree = None
        kernel_addr = None
        initrd_addr = None
        dtb_addr = None
        dtb_append = None
        fastboot = None
        fastboot_cmd = None
        job_file = ''
        board_offline = False
        kernel_boot_time = None
        boot_failure_reason = None
        efi_rtc = False
        # Retrieve job details
        device_type = ''
        job_details = connection.scheduler.job_details(job_id)
        if job_details['requested_device_type_id']:
            device_type = job_details['requested_device_type_id']
        if job_details['description']:
            job_name = job_details['description']
            try:
                job_short_name = re.search(".*?([A-Z]+.*)", job_name).group(1)
            except Exception:
                job_short_name = 'boot-test'
        try:
            device_name = job_details['_actual_device_cache']['hostname']
        except Exception:
            continue
        result = jobs[job_id]['result']
        bundle = jobs[job_id]['bundle']
        if not device_type:
            device_type = job_details['_actual_device_cache']['device_type_id']
        if bundle is None and device_type == 'dynamic-vm':
            host_job_id = job_id.replace('.1', '.0')
            bundle = jobs[host_job_id]['bundle']
            if bundle is None:
                print '%s bundle is empty, skipping...' % device_type
                continue
        # Retrieve the log file
        try:
            binary_job_file = connection.scheduler.job_output(job_id)
        except xmlrpclib.Fault:
            print 'Job output not found for %s' % device_type
            continue
        # Parse LAVA messages out of log
        raw_job_file = str(binary_job_file)
        for line in raw_job_file.splitlines():
            if 'Infrastructure Error:' in line:
                print 'Infrastructure Error detected!'
                index = line.find('Infrastructure Error:')
                boot_failure_reason = line[index:]
                board_offline = True
            if 'Bootloader Error:' in line:
                print 'Bootloader Error detected!'
                index = line.find('Bootloader Error:')
                boot_failure_reason = line[index:]
                board_offline = True
            if 'Kernel Error:' in line:
                print 'Kernel Error detected!'
                index = line.find('Kernel Error:')
                boot_failure_reason = line[index:]
            if 'Userspace Error:' in line:
                print 'Userspace Error detected!'
                index = line.find('Userspace Error:')
                boot_failure_reason = line[index:]
            if '<LAVA_DISPATCHER>' not in line:
                if len(line) != 0:
                    job_file += line + '\n'
            if 'rtc-efi rtc-efi: setting system clock to' in line:
                if device_type == 'dynamic-vm':
                    efi_rtc = True

        # Retrieve bundle
        if bundle is not None:
            json_bundle = connection.dashboard.get(bundle)
            bundle_data = json.loads(json_bundle['content'])
            # Get the boot data from LAVA
            for test_results in bundle_data['test_runs']:
                # Check for the LAVA self boot test
                if test_results['test_id'] == 'lava':
                    for test in test_results['test_results']:
                        # TODO for compat :(
                        if test['test_case_id'] == 'kernel_boot_time':
                            kernel_boot_time = test['measurement']
                        if test['test_case_id'] == 'test_kernel_boot_time':
                            kernel_boot_time = test['measurement']
                    bundle_attributes = bundle_data['test_runs'][-1]['attributes']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
                print bundle_attributes['kernel.defconfig']
            if utils.in_bundle_attributes(bundle_attributes, 'target'):
                board_instance = bundle_attributes['target']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
                kernel_defconfig = bundle_attributes['kernel.defconfig']
                defconfig_list = kernel_defconfig.split('-')
                #arch = defconfig_list[0]
                arch = defconfig_list[-1]
                # Remove arch
                defconfig_list.pop(0)
                kernel_defconfig_full = '-'.join(defconfig_list)
                kernel_defconfig_base = ''.join(kernel_defconfig_full.split('+')[:1])
                if kernel_defconfig_full == kernel_defconfig_base:
                    kernel_defconfig_full = None
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.version'):
                kernel_version = bundle_attributes['kernel.version']
            if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
                device_tree = bundle_attributes['device.tree']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
                kernel_endian = bundle_attributes['kernel.endian']
            if utils.in_bundle_attributes(bundle_attributes, 'platform.fastboot'):
                fastboot = bundle_attributes['platform.fastboot']
            if kernel_boot_time is None:
                if utils.in_bundle_attributes(bundle_attributes, 'kernel-boot-time'):
                    kernel_boot_time = bundle_attributes['kernel-boot-time']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
                kernel_tree = bundle_attributes['kernel.tree']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
                kernel_addr = bundle_attributes['kernel-addr']
            if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
                initrd_addr = bundle_attributes['initrd-addr']
            if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
                dtb_addr = bundle_attributes['dtb-addr']
            if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
                dtb_append = bundle_attributes['dtb-append']
            if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
                boot_retries = int(bundle_attributes['boot_retries'])
            if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
                test_tmp = bundle_attributes['test.plan']
                if test_tmp:
                    test_plan = test_tmp
        else:
            if not kernel_defconfig or not kernel_version or not kernel_tree:
                job_defnition = {}
                if 'original_definition' in job_details.keys():
                    job_definition = job_details['original_definition']
                    try:
                        job_dictionary = eval(job_definition)
                    except Exception:
                        pass
                    if job_dictionary:
                        if 'actions' in job_dictionary.keys():
                            actions = job_dictionary['actions']
                            for i in range(0, len(actions)):
                                try:
                                    kernel_defconfig = actions[i]['metadata']['kernel.defconfig']
                                    kernel_version = actions[i]['metadata']['kernel.version']
                                    kernel_tree = actions[i]['metadata']['kernel.tree']
                                    kernel_endian = actions[i]['metadata']['kernel.endian']
                                    platform_fastboot = actions[i]['metadata']['platform.fastboot']
                                    device_tree = actions[i]['metadata']['kernel.tree']
                                    break
                                except KeyError:
                                    continue
                if 'target' in job_details.keys():
                    print job_details.keys()
        # Check if we found efi-rtc
        if test_plan == 'boot-kvm-uefi' and not efi_rtc:
            if device_type == 'dynamic-vm':
                boot_failure_reason = 'Unable to read EFI rtc'
                result = 'FAIL'

        # Record the boot log and result
        # TODO: Will need to map device_types to dashboard device types
        if kernel_defconfig and device_type and result:
            if ( 'arm' == arch or 'arm64' == arch ) and device_tree is None:
                platform_name = device_map[device_type][0] + ',legacy'
            else:
                if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
                    platform_name = device_map[device_type][0] + '_rootfs:nfs'
                else:
                    platform_name = device_map[device_type][0]

            # Create txt format boot metadata
            print 'Creating boot log for %s' % (platform_name + job_name + '_' + job_id)
            log = 'boot-%s.txt' % (platform_name + job_name + '_' + job_id)
            html = 'boot-%s.html' % (platform_name + job_name + '_' + job_id)
            if config.get("lab"):
                directory = os.path.join(results_directory, kernel_defconfig + '/' + config.get("lab"))
            else:
                directory = os.path.join(results_directory, kernel_defconfig)
            utils.ensure_dir(directory)

            utils.write_file(job_file, log, directory)

            if kernel_boot_time is None:
                kernel_boot_time = '0.0'
            if results.has_key(kernel_defconfig):
                results[kernel_defconfig].append({'device_type': platform_name,
                    'job_id': job_id, 'job_name': job_short_name,
                    'kernel_boot_time': kernel_boot_time, 'result': result,
                    'device_name': device_name})
            else:
                results[kernel_defconfig] = [{'device_type': platform_name,
                    'job_id': job_id, 'job_name': job_short_name,
                    'kernel_boot_time': kernel_boot_time, 'result': result,
                    'device_name': device_name}]

            # Create JSON format boot metadata
            print 'Creating JSON format boot metadata'
            if config.get("lab"):
                boot_meta['lab_name'] = config.get("lab")
            else:
                boot_meta['lab_name'] = None
            if board_instance:
                boot_meta['board_instance'] = board_instance
            boot_meta['retries'] = boot_retries
            boot_meta['boot_log'] = log
            boot_meta['boot_log_html'] = html
            # TODO: Fix this
            boot_meta['version'] = '1.0'
            boot_meta['arch'] = arch
            boot_meta['defconfig'] = kernel_defconfig_base
            if kernel_defconfig_full is not None:
                boot_meta['defconfig_full'] = kernel_defconfig_full
            if device_map[device_type][1]:
                boot_meta['mach'] = device_map[device_type][1]
            boot_meta['kernel'] = kernel_version

            boot_meta['job'] = kernel_tree
            boot_meta['board'] = platform_name
            if board_offline and result == 'FAIL':
                boot_meta['boot_result'] = 'OFFLINE'
                #results[kernel_defconfig]['result'] = 'OFFLINE'
            else:
                boot_meta['boot_result'] = result
            if result == 'FAIL' or result == 'OFFLINE':
                if boot_failure_reason:
                    boot_meta['boot_result_description'] = boot_failure_reason
                else:
                    boot_meta['boot_result_description'] = 'Unknown Error: platform failed to boot'
            boot_meta['boot_time'] = kernel_boot_time
            # TODO: Fix this
            boot_meta['boot_warnings'] = None
            if device_tree:
                if arch == 'arm64':
                    boot_meta['dtb'] = 'dtbs/' + device_map[device_type][1] + '/' + device_tree
                else:
                    boot_meta['dtb'] = 'dtbs/' + device_tree
            else:
                boot_meta['dtb'] = device_tree
            boot_meta['dtb_addr'] = dtb_addr
            boot_meta['dtb_append'] = dtb_append
            boot_meta['endian'] = kernel_endian
            boot_meta['fastboot'] = fastboot
            # TODO: Fix this
            boot_meta['initrd'] = None
            boot_meta['initrd_addr'] = initrd_addr
            if arch == 'arm':
                boot_meta['kernel_image'] = 'zImage'
            elif arch == 'arm64':
                boot_meta['kernel_image'] = 'Image'
            else:
                boot_meta['kernel_image'] = 'bzImage'
            boot_meta['loadaddr'] = kernel_addr
            json_file = 'boot-%s.json' % (platform_name + job_name + '_' + job_id)
            utils.write_json(json_file, directory, boot_meta)
            # add by wuyanjun
            # add the ip device mapping
            get_ip_board_mapping(job_file, log, directory, report_directory)
            parser_and_get_result(job_file, log, directory, report_directory, connection)

    if results and kernel_tree and kernel_version:
        print 'Creating summary for %s' % (kernel_version)
        boot = '%s-boot-report.txt' % (kernel_version)
        if test_plan and ('boot' in test_plan or 'BOOT' in test_plan):
            boot = boot.replace('boot', test_plan)
        passed = 0
        failed = 0
        for defconfig, results_list in results.items():
            for result in results_list:
                if result['result'] == 'PASS':
                    passed += 1
                else:
                    failed += 1
        total = passed + failed
        with open(os.path.join(report_directory, boot), 'a') as f:
            f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' % (kernel_tree,
                                                                                str(total),
                                                                                str(passed),
                                                                                str(failed),
                                                                                kernel_version))
            f.write('\n')
            f.write('Total Duration: %.2f minutes\n' % (duration / 60))
            f.write('Tree/Branch: %s\n' % kernel_tree)
            f.write('Git Describe: %s\n' % kernel_version)
            first = True
            for defconfig, results_list in results.items():
                for result in results_list:
                    if result['result'] == 'OFFLINE':
                        if first:
                            f.write('\n')
                            f.write('Boards Offline:\n')
                            first = False
                        f.write('\n')
                        f.write(defconfig)
                        f.write('\n')
                        break
                for result in results_list:
                    if result['result'] == 'OFFLINE':
                        f.write('    %s   %s   %s   %ss   %s: %s\n' % (result['job_id'],
                                                                    result['device_type'],
                                                                    result['device_name'],
                                                                    result['kernel_boot_time'],
                                                                    result['job_name'],
                                                                    result['result']))
                        f.write('\n')
            first = True
            for defconfig, results_list in results.items():
                for result in results_list:
                    if result['result'] == 'FAIL':
                        if first:
                            f.write('\n')
                            f.write('Failed Boot Tests:\n')
                            first = False
                        f.write('\n')
                        f.write(defconfig)
                        f.write('\n')
                        break
                for result in results_list:
                    if result['result'] == 'FAIL':
                        f.write('    %s   %s   %s   %ss   %s: %s\n' % (result['job_id'],
                                                                    result['device_type'],
                                                                    result['device_name'],
                                                                    result['kernel_boot_time'],
                                                                    result['job_name'],
                                                                    result['result']))
            f.write('\n')
            f.write('Full Boot Report:\n')
            for defconfig, results_list in results.items():
                f.write('\n')
                f.write(defconfig)
                f.write('\n')
                for result in results_list:
                    f.write('    %s   %s   %s   %ss   %s: %s\n' %
                                                                    (result['job_id'],
                                                                        result['device_type'],
                                                                        result['device_name'],
                                                                        result['kernel_boot_time'],
                                                                        result['job_name'],
                                                                        result['result']))
Esempio n. 7
0
def boot_report(config):
    connection, jobs, duration = parse_json(config.get("boot"))
    # TODO: Fix this when multi-lab sync is working
    #download_log2html(log2html)
    results_directory = os.getcwd() + '/results'
    results = {}
    dt_tests = False
    utils.mkdir(results_directory)
    for job_id in jobs:
        print 'Job ID: %s' % job_id
        # Init
        boot_meta = {}
        api_url = None
        arch = None
        board_instance = None
        boot_retries = 0
        kernel_defconfig = None
        kernel_defconfig_base = None
        kernel_version = None
        device_tree = None
        kernel_endian = None
        kernel_tree = None
        git_branch = None
        kernel_addr = None
        initrd_addr = None
        dtb_addr = None
        dtb_append = None
        fastboot = None
        fastboot_cmd = None
        test_plan = None
        job_file = ''
        dt_test = None
        dt_test_result = None
        dt_tests_passed = None
        dt_tests_failed = None
        board_offline = False
        kernel_boot_time = None
        boot_failure_reason = None
        efi_rtc = False
        # Retrieve job details
        job_details = connection.scheduler.job_details(job_id)
        if job_details['requested_device_type_id']:
            device_type = job_details['requested_device_type_id']
        if job_details['description']:
            job_name = job_details['description']
        result = jobs[job_id]['result']
        bundle = jobs[job_id]['bundle']
        if bundle is None and device_type == 'dynamic-vm':
            host_job_id = job_id.replace('.1', '.0')
            bundle = jobs[host_job_id]['bundle']
            if bundle is None:
                print '%s bundle is empty, skipping...' % device_type
                continue
        # Retrieve the log file
        try:
            binary_job_file = connection.scheduler.job_output(job_id)
        except xmlrpclib.Fault:
            print 'Job output not found for %s' % device_type
            continue
        # Parse LAVA messages out of log
        raw_job_file = str(binary_job_file)
        for line in raw_job_file.splitlines():
            if 'Infrastructure Error:' in line:
                print 'Infrastructure Error detected!'
                index = line.find('Infrastructure Error:')
                boot_failure_reason = line[index:]
                board_offline = True
            if 'Bootloader Error:' in line:
                print 'Bootloader Error detected!'
                index = line.find('Bootloader Error:')
                boot_failure_reason = line[index:]
                board_offline = True
            if 'Kernel Error:' in line:
                print 'Kernel Error detected!'
                index = line.find('Kernel Error:')
                boot_failure_reason = line[index:]
            if 'Userspace Error:' in line:
                print 'Userspace Error detected!'
                index = line.find('Userspace Error:')
                boot_failure_reason = line[index:]
            if '<LAVA_DISPATCHER>' not in line:
                if len(line) != 0:
                    job_file += line + '\n'
            if '### dt-test ### end of selftest' in line:
                dt_tests = True
                regex = re.compile("(?P<test>\d+\*?)")
                dt_test_results = regex.findall(line)
                if len(dt_test_results) > 2:
                    dt_tests_passed = dt_test_results[2]
                    dt_tests_failed = dt_test_results[3]
                else:
                    dt_tests_passed = dt_test_results[0]
                    dt_tests_failed = dt_test_results[1]
                if int(dt_tests_failed) > 0:
                    dt_test_result = 'FAIL'
                else:
                    dt_test_result = 'PASS'
            if 'rtc-efi rtc-efi: setting system clock to' in line:
                if device_type == 'dynamic-vm':
                    efi_rtc = True
        # Retrieve bundle
        if bundle is not None:
            json_bundle = connection.dashboard.get(bundle)
            bundle_data = json.loads(json_bundle['content'])
            # Get the boot data from LAVA
            for test_results in bundle_data['test_runs']:
                # Check for the LAVA self boot test
                if test_results['test_id'] == 'lava':
                    for test in test_results['test_results']:
                        # TODO for compat :(
                        if test['test_case_id'] == 'kernel_boot_time':
                            kernel_boot_time = test['measurement']
                        if test['test_case_id'] == 'test_kernel_boot_time':
                            kernel_boot_time = test['measurement']
                    bundle_attributes = bundle_data['test_runs'][-1][
                        'attributes']
            if utils.in_bundle_attributes(bundle_attributes,
                                          'kernel.defconfig'):
                print bundle_attributes['kernel.defconfig']
            if utils.in_bundle_attributes(bundle_attributes, 'target'):
                board_instance = bundle_attributes['target']
            if utils.in_bundle_attributes(bundle_attributes,
                                          'kernel.defconfig'):
                kernel_defconfig = bundle_attributes['kernel.defconfig']
                kernel_defconfig_base = ''.join(
                    kernel_defconfig.split('+')[:1])
            if utils.in_bundle_attributes(bundle_attributes, 'arch'):
                arch = bundle_attributes['arch']
            if utils.in_bundle_attributes(bundle_attributes,
                                          'kernel.describe'):
                kernel_version = bundle_attributes['kernel.describe']
            if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
                device_tree = bundle_attributes['device.tree']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
                kernel_endian = bundle_attributes['kernel.endian']
            if utils.in_bundle_attributes(bundle_attributes,
                                          'platform.fastboot'):
                fastboot = bundle_attributes['platform.fastboot']
            if kernel_boot_time is None:
                if utils.in_bundle_attributes(bundle_attributes,
                                              'kernel-boot-time'):
                    kernel_boot_time = bundle_attributes['kernel-boot-time']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
                kernel_tree = bundle_attributes['kernel.tree']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
                kernel_addr = bundle_attributes['kernel-addr']
            if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
                initrd_addr = bundle_attributes['initrd-addr']
            if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
                dtb_addr = bundle_attributes['dtb-addr']
            if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
                dtb_append = bundle_attributes['dtb-append']
            if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
                boot_retries = int(bundle_attributes['boot_retries'])
            if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
                test_plan = bundle_attributes['test.plan']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.branch'):
                git_branch = bundle_attributes['kernel.branch']

        # Check if we found efi-rtc
        if test_plan == 'boot-kvm-uefi' and not efi_rtc:
            if device_type == 'dynamic-vm':
                boot_failure_reason = 'Unable to read EFI rtc'
                result = 'FAIL'
        # Record the boot log and result
        # TODO: Will need to map device_types to dashboard device types
        if kernel_defconfig and device_type and result:
            if (arch == 'arm' or arch == 'arm64') and device_tree is None:
                platform_name = device_map[device_type][0] + ',legacy'
            else:
                if device_tree == 'vexpress-v2p-ca15_a7.dtb':
                    platform_name = 'vexpress-v2p-ca15_a7'
                elif device_tree == 'fsl-ls2080a-simu.dtb':
                    platform_name = 'fsl-ls2080a-simu'
                elif test_plan == 'boot-kvm' or test_plan == 'boot-kvm-uefi':
                    if device_tree == 'sun7i-a20-cubietruck.dtb':
                        if device_type == 'dynamic-vm':
                            device_type = 'cubieboard3-kvm-guest'
                            platform_name = device_map[device_type][0]
                        else:
                            device_type = 'cubieboard3-kvm-host'
                            platform_name = device_map[device_type][0]
                    elif device_tree == 'apm-mustang.dtb':
                        if device_type == 'dynamic-vm':
                            if test_plan == 'boot-kvm-uefi':
                                device_type = 'mustang-kvm-uefi-guest'
                            else:
                                device_type = 'mustang-kvm-guest'
                            platform_name = device_map[device_type][0]
                        else:
                            if test_plan == 'boot-kvm-uefi':
                                device_type = 'mustang-kvm-uefi-host'
                            else:
                                device_type = 'mustang-kvm-host'
                            platform_name = device_map[device_type][0]
                    elif device_tree == 'juno.dtb':
                        if device_type == 'dynamic-vm':
                            if test_plan == 'boot-kvm-uefi':
                                device_type = 'juno-kvm-uefi-guest'
                            else:
                                device_type = 'juno-kvm-guest'
                            platform_name = device_map[device_type][0]
                        else:
                            if test_plan == 'boot-kvm-uefi':
                                device_type = 'juno-kvm-uefi-host'
                            else:
                                device_type = 'juno-kvm-host'
                            platform_name = device_map[device_type][0]
                elif test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
                    platform_name = device_map[device_type][0] + '_rootfs:nfs'
                else:
                    platform_name = device_map[device_type][0]
            print 'Creating boot log for %s' % platform_name
            log = 'boot-%s.txt' % platform_name
            html = 'boot-%s.html' % platform_name
            if config.get("lab"):
                directory = os.path.join(
                    results_directory,
                    kernel_defconfig + '/' + config.get("lab"))
            else:
                directory = os.path.join(results_directory, kernel_defconfig)
            utils.ensure_dir(directory)
            utils.write_file(job_file, log, directory)
            if kernel_boot_time is None:
                kernel_boot_time = '0.0'
            if results.has_key(kernel_defconfig):
                results[kernel_defconfig].append({
                    'device_type': platform_name,
                    'dt_test_result': dt_test_result,
                    'dt_tests_passed': dt_tests_passed,
                    'dt_tests_failed': dt_tests_failed,
                    'kernel_boot_time': kernel_boot_time,
                    'result': result
                })
            else:
                results[kernel_defconfig] = [{
                    'device_type': platform_name,
                    'dt_test_result': dt_test_result,
                    'dt_tests_passed': dt_tests_passed,
                    'dt_tests_failed': dt_tests_failed,
                    'kernel_boot_time': kernel_boot_time,
                    'result': result
                }]
            # Create JSON format boot metadata
            print 'Creating JSON format boot metadata'
            if config.get("lab"):
                boot_meta['lab_name'] = config.get("lab")
            else:
                boot_meta['lab_name'] = None
            if board_instance:
                boot_meta['board_instance'] = board_instance
            boot_meta['retries'] = boot_retries
            boot_meta['boot_log'] = log
            boot_meta['boot_log_html'] = html
            # TODO: Fix this
            boot_meta['version'] = '1.1'
            boot_meta['arch'] = arch
            boot_meta['defconfig'] = kernel_defconfig_base
            boot_meta['defconfig_full'] = kernel_defconfig
            if device_map[device_type][1]:
                boot_meta['mach'] = device_map[device_type][1]
            boot_meta['kernel'] = kernel_version
            boot_meta['git_branch'] = git_branch
            boot_meta['job'] = kernel_tree
            boot_meta['board'] = platform_name
            if board_offline and result == 'FAIL':
                boot_meta['boot_result'] = 'OFFLINE'
                #results[kernel_defconfig]['result'] = 'OFFLINE'
            else:
                boot_meta['boot_result'] = result
            if result == 'FAIL' or result == 'OFFLINE':
                if boot_failure_reason:
                    boot_meta['boot_result_description'] = boot_failure_reason
                else:
                    boot_meta[
                        'boot_result_description'] = 'Unknown Error: platform failed to boot'
            boot_meta['boot_time'] = kernel_boot_time
            # TODO: Fix this
            boot_meta['boot_warnings'] = None
            if device_tree:
                if arch == 'arm64':
                    boot_meta['dtb'] = 'dtbs/' + device_map[device_type][
                        1] + '/' + device_tree
                else:
                    boot_meta['dtb'] = 'dtbs/' + device_tree
            else:
                boot_meta['dtb'] = device_tree
            boot_meta['dtb_addr'] = dtb_addr
            boot_meta['dtb_append'] = dtb_append
            boot_meta['dt_test'] = dt_test
            boot_meta['endian'] = kernel_endian
            boot_meta['fastboot'] = fastboot
            # TODO: Fix this
            boot_meta['initrd'] = None
            boot_meta['initrd_addr'] = initrd_addr
            if arch == 'arm':
                boot_meta['kernel_image'] = 'zImage'
            elif arch == 'arm64':
                boot_meta['kernel_image'] = 'Image'
            else:
                boot_meta['kernel_image'] = 'bzImage'
            boot_meta['loadaddr'] = kernel_addr
            json_file = 'boot-%s.json' % platform_name
            utils.write_json(json_file, directory, boot_meta)
            print 'Creating html version of boot log for %s' % platform_name
            cmd = 'python log2html.py %s' % os.path.join(directory, log)
            subprocess.check_output(cmd, shell=True)
            if config.get("lab") and config.get("api") and config.get("token"):
                print 'Sending boot result to %s for %s' % (config.get("api"),
                                                            platform_name)
                headers = {
                    'Authorization': config.get("token"),
                    'Content-Type': 'application/json'
                }
                api_url = urlparse.urljoin(config.get("api"), '/boot')
                push('POST',
                     api_url,
                     data=json.dumps(boot_meta),
                     headers=headers)
                headers = {
                    'Authorization': config.get("token"),
                }
                print 'Uploading text version of boot log'
                with open(os.path.join(directory, log)) as lh:
                    data = lh.read()
                api_url = urlparse.urljoin(
                    config.get("api"), '/upload/%s/%s/%s/%s/%s/%s/%s' %
                    (kernel_tree, git_branch, kernel_version, arch,
                     kernel_defconfig, config.get("lab"), log))
                push('PUT', api_url, data=data, headers=headers)
                print 'Uploading html version of boot log'
                with open(os.path.join(directory, html)) as lh:
                    data = lh.read()
                api_url = urlparse.urljoin(
                    config.get("api"), '/upload/%s/%s/%s/%s/%s/%s/%s' %
                    (kernel_tree, git_branch, kernel_version, arch,
                     kernel_defconfig, config.get("lab"), html))
                push('PUT', api_url, data=data, headers=headers)
Esempio n. 8
0
def boot_report(config):
    connection, jobs, duration = parse_yaml(config.get("boot"))
    # TODO: Fix this when multi-lab sync is working
    results_directory = os.getcwd() + '/results'
    results = {}
    utils.mkdir(results_directory)
    test_plan = None

    if config.get("lab"):
        report_directory = os.path.join(results_directory, config.get("lab"))
    else:
        report_directory = results_directory

    if os.path.exists(report_directory):
        shutil.rmtree(report_directory)
    utils.mkdir(report_directory)

    for job_id in jobs:
        print 'Job ID: %s' % job_id
        # Init
        boot_meta = {}
        arch = None
        board_instance = None
        boot_retries = 0
        kernel_defconfig_full = None
        kernel_defconfig = None
        kernel_defconfig_base = None
        kernel_version = None
        device_tree = None
        kernel_tree = None
        kernel_addr = None
        initrd_addr = None
        dtb_addr = None
        dtb_append = None
        job_file = ''
        board_offline = False
        kernel_boot_time = None
        boot_failure_reason = None
        efi_rtc = False
        # Retrieve job details
        device_type = ''
        job_details = connection.scheduler.job_details(job_id)
        if job_details['requested_device_type_id']:
            device_type = job_details['requested_device_type_id']
        if job_details['description']:
            job_name = job_details['description']
            try:
                job_short_name = re.search(".*?([A-Z]+.*)", job_name).group(1)
            except Exception:
                job_short_name = 'boot-test'
        try:
            device_name = job_details['_actual_device_cache']['hostname']
        except Exception:
            continue
        result = jobs[job_id]['result']
        bundle = jobs[job_id]['bundle']
        if not device_type:
            device_type = job_details['_actual_device_cache']['device_type_id']
        try:
            binary_job_file = connection.scheduler.job_output(job_id)
        except xmlrpclib.Fault:
            print 'Job output not found for %s' % device_type
            continue
        # Parse LAVA messages out of log
        raw_job_file = str(binary_job_file)
        for line in raw_job_file.splitlines():
            if 'Infrastructure Error:' in line:
                print 'Infrastructure Error detected!'
                index = line.find('Infrastructure Error:')
                boot_failure_reason = line[index:]
                board_offline = True
            if 'Bootloader Error:' in line:
                print 'Bootloader Error detected!'
                index = line.find('Bootloader Error:')
                boot_failure_reason = line[index:]
                board_offline = True
            if 'Kernel Error:' in line:
                print 'Kernel Error detected!'
                index = line.find('Kernel Error:')
                boot_failure_reason = line[index:]
            if 'Userspace Error:' in line:
                print 'Userspace Error detected!'
                index = line.find('Userspace Error:')
                boot_failure_reason = line[index:]
            if '<LAVA_DISPATCHER>' not in line:
                if len(line) != 0:
                    job_file += line + '\n'
            if 'rtc-efi rtc-efi: setting system clock to' in line:
                if device_type == 'dynamic-vm':
                    efi_rtc = True
        if not kernel_defconfig or not kernel_version or not kernel_tree:
            try:
                job_metadata_info = connection.results.get_testjob_metadata(
                    job_id)
                kernel_defconfig = utils.get_value_by_key(
                    job_metadata_info, 'kernel_defconfig')
                kernel_version = utils.get_value_by_key(
                    job_metadata_info, 'kernel_version')
                kernel_tree = utils.get_value_by_key(job_metadata_info,
                                                     'kernel_tree')
                device_tree = utils.get_value_by_key(job_metadata_info,
                                                     'device_tree')
            except Exception:
                continue

        # Record the boot log and result
        # TODO: Will need to map device_types to dashboard device types
        if kernel_defconfig and device_type and result:
            if ('arm' == arch or 'arm64' == arch) and device_tree is None:
                platform_name = device_map[device_type][0] + ',legacy'
            else:
                if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
                    platform_name = device_map[device_type][0] + '_rootfs:nfs'
                else:
                    platform_name = device_map[device_type][0]

            # Create txt format boot metadata
            print 'Creating boot log for %s' % (platform_name + job_name +
                                                '_' + job_id)
            log = 'boot-%s.txt' % (platform_name + job_name + '_' + job_id)
            if config.get("lab"):
                directory = os.path.join(
                    results_directory,
                    kernel_defconfig + '/' + config.get("lab"))
            else:
                directory = os.path.join(results_directory, kernel_defconfig)
            utils.ensure_dir(directory)

            utils.write_file(job_file, log, directory)

            if kernel_boot_time is None:
                kernel_boot_time = '0.0'
            if results.has_key(kernel_defconfig):
                results[kernel_defconfig].append({
                    'device_type': platform_name,
                    'job_id': job_id,
                    'job_name': job_short_name,
                    'kernel_boot_time': kernel_boot_time,
                    'result': result,
                    'device_name': device_name
                })
            else:
                results[kernel_defconfig] = [{
                    'device_type': platform_name,
                    'job_id': job_id,
                    'job_name': job_short_name,
                    'kernel_boot_time': kernel_boot_time,
                    'result': result,
                    'device_name': device_name
                }]

            # Create JSON format boot metadata
            print 'Creating JSON format boot metadata'
            if config.get("lab"):
                boot_meta['lab_name'] = config.get("lab")
            else:
                boot_meta['lab_name'] = None
            if board_instance:
                boot_meta['board_instance'] = board_instance
            boot_meta['retries'] = boot_retries
            boot_meta['boot_log'] = log
            # TODO: Fix this
            boot_meta['version'] = '1.0'
            boot_meta['arch'] = arch
            boot_meta['defconfig'] = kernel_defconfig_base
            if kernel_defconfig_full is not None:
                boot_meta['defconfig_full'] = kernel_defconfig_full
            if device_map[device_type][1]:
                boot_meta['mach'] = device_map[device_type][1]
            boot_meta['kernel'] = kernel_version

            boot_meta['job'] = kernel_tree
            boot_meta['board'] = platform_name
            if board_offline and result == 'FAIL':
                boot_meta['boot_result'] = 'OFFLINE'
                #results[kernel_defconfig]['result'] = 'OFFLINE'
            else:
                boot_meta['boot_result'] = result
            if result == 'FAIL' or result == 'OFFLINE':
                if boot_failure_reason:
                    boot_meta['boot_result_description'] = boot_failure_reason
                else:
                    boot_meta[
                        'boot_result_description'] = 'Unknown Error: platform failed to boot'
            boot_meta['boot_time'] = kernel_boot_time
            # TODO: Fix this
            boot_meta['boot_warnings'] = None
            if device_tree:
                if arch == 'arm64':
                    boot_meta['dtb'] = 'dtbs/' + device_map[device_type][
                        1] + '/' + device_tree
                else:
                    boot_meta['dtb'] = 'dtbs/' + device_tree
            else:
                boot_meta['dtb'] = device_tree
            boot_meta['dtb_addr'] = dtb_addr
            boot_meta['dtb_append'] = dtb_append
            # TODO: Fix this
            boot_meta['initrd'] = None
            boot_meta['initrd_addr'] = initrd_addr
            if arch == 'arm':
                boot_meta['kernel_image'] = 'zImage'
            elif arch == 'arm64':
                boot_meta['kernel_image'] = 'Image'
            else:
                boot_meta['kernel_image'] = 'bzImage'
            boot_meta['loadaddr'] = kernel_addr
            json_file = 'boot-%s.json' % (platform_name + job_name + '_' +
                                          job_id)
            utils.write_json(json_file, directory, boot_meta)
            # add by wuyanjun
            parser_and_get_result(job_file, log, directory, report_directory,
                                  connection)

            #try to generate test_summary
            generate_test_report(job_id, connection)

    if results and kernel_tree and kernel_version:
        print 'Creating summary for %s' % (kernel_version)
        boot = '%s-boot-report.txt' % (kernel_version)
        if test_plan and ('boot' in test_plan or 'BOOT' in test_plan):
            boot = boot.replace('boot', test_plan)
        passed = 0
        failed = 0
        for defconfig, results_list in results.items():
            for result in results_list:
                if result['result'] == 'PASS':
                    passed += 1
                else:
                    failed += 1
        total = passed + failed
        with open(os.path.join(report_directory, boot), 'a') as f:
            f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' %
                    (kernel_tree, str(total), str(passed), str(failed),
                     kernel_version))
            f.write('\n')
            f.write('Total Duration: %.2f minutes\n' % (duration / 60))
            f.write('Tree/Branch: %s\n' % kernel_tree)
            f.write('Git Describe: %s\n' % kernel_version)
            first = True
            for defconfig, results_list in results.items():
                for result in results_list:
                    if result['result'] == 'OFFLINE':
                        if first:
                            f.write('\n')
                            f.write('Boards Offline:\n')
                            first = False
                        f.write('\n')
                        f.write(defconfig)
                        f.write('\n')
                        break
                for result in results_list:
                    if result['result'] == 'OFFLINE':
                        f.write(
                            '    %s   %s   %s   %ss   %s: %s\n' %
                            (result['job_id'], result['device_type'],
                             result['device_name'], result['kernel_boot_time'],
                             result['job_name'], result['result']))
                        f.write('\n')
            first = True
            for defconfig, results_list in results.items():
                for result in results_list:
                    if result['result'] == 'FAIL':
                        if first:
                            f.write('\n')
                            f.write('Failed Boot Tests:\n')
                            first = False
                        f.write('\n')
                        f.write(defconfig)
                        f.write('\n')
                        break
                for result in results_list:
                    if result['result'] == 'FAIL':
                        f.write(
                            '    %s   %s   %s   %ss   %s: %s\n' %
                            (result['job_id'], result['device_type'],
                             result['device_name'], result['kernel_boot_time'],
                             result['job_name'], result['result']))
            f.write('\n')
            f.write('Full Boot Report:\n')
            for defconfig, results_list in results.items():
                f.write('\n')
                f.write(defconfig)
                f.write('\n')
                for result in results_list:
                    f.write('    %s   %s   %s   %ss   %s: %s\n' %
                            (result['job_id'], result['device_type'],
                             result['device_name'], result['kernel_boot_time'],
                             result['job_name'], result['result']))
Esempio n. 9
0
def boot_report(config):
    connection, jobs, duration =  parse_json(config.get("boot"))
    # TODO: Fix this when multi-lab sync is working
    #download_log2html(log2html)
    results_directory = os.getcwd() + '/results'
    results = {}
    dt_tests = False
    utils.mkdir(results_directory)
    for job_id in jobs:
        print 'Job ID: %s' % job_id
        # Init
        boot_meta = {}
        api_url = None
        arch = None
        board_instance = None
        boot_retries = 0
        kernel_defconfig = None
        kernel_defconfig_base = None
        kernel_version = None
        device_tree = None
        kernel_endian = None
        kernel_tree = None
        git_branch = None
        kernel_addr = None
        initrd_addr = None
        dtb_addr = None
        dtb_append = None
        fastboot = None
        fastboot_cmd = None
        test_plan = None
        job_file = ''
        dt_test = None
        dt_test_result = None
        dt_tests_passed = None
        dt_tests_failed = None
        board_offline = False
        kernel_boot_time = None
        boot_failure_reason = None
        efi_rtc = False
        # Retrieve job details
        job_details = connection.scheduler.job_details(job_id)
        if job_details['requested_device_type_id']:
            device_type = job_details['requested_device_type_id']
        if job_details['description']:
            job_name = job_details['description']
        result = jobs[job_id]['result']
        bundle = jobs[job_id]['bundle']
        if bundle is None and device_type == 'dynamic-vm':
            host_job_id = job_id.replace('.1', '.0')
            bundle = jobs[host_job_id]['bundle']
            if bundle is None:
                print '%s bundle is empty, skipping...' % device_type
                continue
        # Retrieve the log file
        try:
            binary_job_file = connection.scheduler.job_output(job_id)
        except xmlrpclib.Fault:
            print 'Job output not found for %s' % device_type
            continue
        # Parse LAVA messages out of log
        raw_job_file = str(binary_job_file)
        for line in raw_job_file.splitlines():
            if 'Infrastructure Error:' in line:
                print 'Infrastructure Error detected!'
                index = line.find('Infrastructure Error:')
                boot_failure_reason = line[index:]
                board_offline = True
            if 'Bootloader Error:' in line:
                print 'Bootloader Error detected!'
                index = line.find('Bootloader Error:')
                boot_failure_reason = line[index:]
                board_offline = True
            if 'Kernel Error:' in line:
                print 'Kernel Error detected!'
                index = line.find('Kernel Error:')
                boot_failure_reason = line[index:]
            if 'Userspace Error:' in line:
                print 'Userspace Error detected!'
                index = line.find('Userspace Error:')
                boot_failure_reason = line[index:]
            if '<LAVA_DISPATCHER>' not in line:
                if len(line) != 0:
                    job_file += line + '\n'
            if '### dt-test ### end of selftest' in line:
                dt_tests = True
                regex = re.compile("(?P<test>\d+\*?)")
                dt_test_results = regex.findall(line)
                if len(dt_test_results) > 2:
                    dt_tests_passed = dt_test_results[2]
                    dt_tests_failed = dt_test_results[3]
                else:
                    dt_tests_passed = dt_test_results[0]
                    dt_tests_failed = dt_test_results[1]
                if int(dt_tests_failed) > 0:
                    dt_test_result = 'FAIL'
                else:
                    dt_test_result = 'PASS'
            if 'rtc-efi rtc-efi: setting system clock to' in line:
                if device_type == 'dynamic-vm':
                    efi_rtc = True
        # Retrieve bundle
        if bundle is not None:
            json_bundle = connection.dashboard.get(bundle)
            bundle_data = json.loads(json_bundle['content'])
            # Get the boot data from LAVA
            for test_results in bundle_data['test_runs']:
                # Check for the LAVA self boot test
                if test_results['test_id'] == 'lava':
                    for test in test_results['test_results']:
                        # TODO for compat :(
                        if test['test_case_id'] == 'kernel_boot_time':
                            kernel_boot_time = test['measurement']
                        if test['test_case_id'] == 'test_kernel_boot_time':
                            kernel_boot_time = test['measurement']
                    bundle_attributes = bundle_data['test_runs'][-1]['attributes']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
                print bundle_attributes['kernel.defconfig']
            if utils.in_bundle_attributes(bundle_attributes, 'target'):
                board_instance = bundle_attributes['target']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
                kernel_defconfig = bundle_attributes['kernel.defconfig']
                kernel_defconfig_base = ''.join(kernel_defconfig.split('+')[:1])
            if utils.in_bundle_attributes(bundle_attributes, 'arch'):
                arch = bundle_attributes['arch']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.describe'):
                kernel_version = bundle_attributes['kernel.describe']
            if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
                device_tree = bundle_attributes['device.tree']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
                kernel_endian = bundle_attributes['kernel.endian']
            if utils.in_bundle_attributes(bundle_attributes, 'platform.fastboot'):
                fastboot = bundle_attributes['platform.fastboot']
            if kernel_boot_time is None:
                if utils.in_bundle_attributes(bundle_attributes, 'kernel-boot-time'):
                    kernel_boot_time = bundle_attributes['kernel-boot-time']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
                kernel_tree = bundle_attributes['kernel.tree']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
                kernel_addr = bundle_attributes['kernel-addr']
            if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
                initrd_addr = bundle_attributes['initrd-addr']
            if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
                dtb_addr = bundle_attributes['dtb-addr']
            if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
                dtb_append = bundle_attributes['dtb-append']
            if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
                boot_retries = int(bundle_attributes['boot_retries'])
            if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
                test_plan = bundle_attributes['test.plan']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.branch'):
                git_branch = bundle_attributes['kernel.branch']

        # Check if we found efi-rtc
        if test_plan == 'boot-kvm-uefi' and not efi_rtc:
            if device_type == 'dynamic-vm':
                boot_failure_reason = 'Unable to read EFI rtc'
                result = 'FAIL'
        # Record the boot log and result
        # TODO: Will need to map device_types to dashboard device types
        if kernel_defconfig and device_type and result:
            if (arch == 'arm' or arch =='arm64') and device_tree is None:
                platform_name = device_map[device_type][0] + ',legacy'
            else:
                if device_tree == 'vexpress-v2p-ca15_a7.dtb':
                    platform_name = 'vexpress-v2p-ca15_a7'
                elif device_tree == 'fsl-ls2080a-simu.dtb':
                    platform_name = 'fsl-ls2080a-simu'
                elif test_plan == 'boot-kvm' or test_plan == 'boot-kvm-uefi':
                    if device_tree == 'sun7i-a20-cubietruck.dtb':
                        if device_type == 'dynamic-vm':
                            device_type = 'cubieboard3-kvm-guest'
                            platform_name = device_map[device_type][0]
                        else:
                            device_type = 'cubieboard3-kvm-host'
                            platform_name = device_map[device_type][0]
                    elif device_tree == 'apm-mustang.dtb':
                        if device_type == 'dynamic-vm':
                            if test_plan == 'boot-kvm-uefi':
                                device_type = 'mustang-kvm-uefi-guest'
                            else:
                                device_type = 'mustang-kvm-guest'
                            platform_name = device_map[device_type][0]
                        else:
                            if test_plan == 'boot-kvm-uefi':
                                device_type = 'mustang-kvm-uefi-host'
                            else:
                                device_type = 'mustang-kvm-host'
                            platform_name = device_map[device_type][0]
                    elif device_tree == 'juno.dtb':
                        if device_type == 'dynamic-vm':
                            if test_plan == 'boot-kvm-uefi':
                                device_type = 'juno-kvm-uefi-guest'
                            else:
                                device_type = 'juno-kvm-guest'
                            platform_name = device_map[device_type][0]
                        else:
                            if test_plan == 'boot-kvm-uefi':
                                device_type = 'juno-kvm-uefi-host'
                            else:
                                device_type = 'juno-kvm-host'
                            platform_name = device_map[device_type][0]
                elif test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
                    platform_name = device_map[device_type][0] + '_rootfs:nfs'
                else:
                    platform_name = device_map[device_type][0]
            print 'Creating boot log for %s' % platform_name
            log = 'boot-%s.txt' % platform_name
            html = 'boot-%s.html' % platform_name
            if config.get("lab"):
                directory = os.path.join(results_directory, kernel_defconfig + '/' + config.get("lab"))
            else:
                directory = os.path.join(results_directory, kernel_defconfig)
            utils.ensure_dir(directory)
            utils.write_file(job_file, log, directory)
            if kernel_boot_time is None:
                kernel_boot_time = '0.0'
            if results.has_key(kernel_defconfig):
                results[kernel_defconfig].append({'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result})
            else:
                results[kernel_defconfig] = [{'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result}]
            # Create JSON format boot metadata
            print 'Creating JSON format boot metadata'
            if config.get("lab"):
                boot_meta['lab_name'] = config.get("lab")
            else:
                boot_meta['lab_name'] = None
            if board_instance:
                boot_meta['board_instance'] = board_instance
            boot_meta['retries'] = boot_retries
            boot_meta['boot_log'] = log
            boot_meta['boot_log_html'] = html
            # TODO: Fix this
            boot_meta['version'] = '1.1'
            boot_meta['arch'] = arch
            boot_meta['defconfig'] = kernel_defconfig_base
            boot_meta['defconfig_full'] = kernel_defconfig
            if device_map[device_type][1]:
                boot_meta['mach'] = device_map[device_type][1]
            boot_meta['kernel'] = kernel_version
            boot_meta['git_branch'] = git_branch
            boot_meta['job'] = kernel_tree
            boot_meta['board'] = platform_name
            if board_offline and result == 'FAIL':
                boot_meta['boot_result'] = 'OFFLINE'
                #results[kernel_defconfig]['result'] = 'OFFLINE'
            else:
                boot_meta['boot_result'] = result
            if result == 'FAIL' or result == 'OFFLINE':
                if boot_failure_reason:
                    boot_meta['boot_result_description'] = boot_failure_reason
                else:
                    boot_meta['boot_result_description'] = 'Unknown Error: platform failed to boot'
            boot_meta['boot_time'] = kernel_boot_time
            # TODO: Fix this
            boot_meta['boot_warnings'] = None
            if device_tree:
                if arch == 'arm64':
                    boot_meta['dtb'] = 'dtbs/' + device_map[device_type][1] + '/' + device_tree
                else:
                    boot_meta['dtb'] = 'dtbs/' + device_tree
            else:
                boot_meta['dtb'] = device_tree
            boot_meta['dtb_addr'] = dtb_addr
            boot_meta['dtb_append'] = dtb_append
            boot_meta['dt_test'] = dt_test
            boot_meta['endian'] = kernel_endian
            boot_meta['fastboot'] = fastboot
            # TODO: Fix this
            boot_meta['initrd'] = None
            boot_meta['initrd_addr'] = initrd_addr
            if arch == 'arm':
                boot_meta['kernel_image'] = 'zImage'
            elif arch == 'arm64':
                boot_meta['kernel_image'] = 'Image'
            else:
                boot_meta['kernel_image'] = 'bzImage'
            boot_meta['loadaddr'] = kernel_addr
            json_file = 'boot-%s.json' % platform_name
            utils.write_json(json_file, directory, boot_meta)
            print 'Creating html version of boot log for %s' % platform_name
            cmd = 'python log2html.py %s' % os.path.join(directory, log)
            subprocess.check_output(cmd, shell=True)
            if config.get("lab") and config.get("api") and config.get("token"):
                print 'Sending boot result to %s for %s' % (config.get("api"), platform_name)
                headers = {
                    'Authorization': config.get("token"),
                    'Content-Type': 'application/json'
                }
                api_url = urlparse.urljoin(config.get("api"), '/boot')
                push('POST', api_url, data=json.dumps(boot_meta), headers=headers)
                headers = {
                    'Authorization': config.get("token"),
                }
                print 'Uploading text version of boot log'
                with open(os.path.join(directory, log)) as lh:
                    data = lh.read()
                api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s/%s/%s' % (kernel_tree,
                                                                                 git_branch,
                                                                                 kernel_version,
                                                                                 arch,
                                                                                 kernel_defconfig,
                                                                                 config.get("lab"),
                                                                                 log))
                push('PUT', api_url, data=data, headers=headers)
                print 'Uploading html version of boot log'
                with open(os.path.join(directory, html)) as lh:
                    data = lh.read()
                api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s/%s/%s' % (kernel_tree,
                                                                                 git_branch,
                                                                                 kernel_version,
                                                                                 arch,
                                                                                 kernel_defconfig,
                                                                                 config.get("lab"),
                                                                                 html))
                push('PUT', api_url, data=data, headers=headers)
def boot_report(config):
    connection, jobs, duration = parse_json(config.get("boot"))
    # TODO: Fix this when multi-lab sync is working
    #download_log2html(log2html)
    results_directory = os.getcwd() + '/results'
    results = {}
    dt_tests = False
    utils.mkdir(results_directory)
    for job_id in jobs:
        print 'Job ID: %s' % job_id
        # Init
        boot_meta = {}
        api_url = None
        arch = None
        board_instance = None
        boot_retries = 0
        kernel_defconfig_full = None
        kernel_defconfig = None
        kernel_defconfig_base = None
        kernel_version = None
        device_tree = None
        kernel_endian = None
        kernel_tree = None
        kernel_addr = None
        initrd_addr = None
        dtb_addr = None
        dtb_append = None
        fastboot = None
        fastboot_cmd = None
        test_plan = None
        job_file = ''
        dt_test = None
        dt_test_result = None
        dt_tests_passed = None
        dt_tests_failed = None
        board_offline = False
        kernel_boot_time = None
        boot_failure_reason = None
        efi_rtc = False
        # Retrieve job details
        device_type = ''
        job_details = connection.scheduler.job_details(job_id)
        if job_details['requested_device_type_id']:
            device_type = job_details['requested_device_type_id']
        if job_details['description']:
            job_name = job_details['description']
        device_name = job_details['_actual_device_cache']['hostname']
        result = jobs[job_id]['result']
        bundle = jobs[job_id]['bundle']
        if not device_type:
            device_type = job_details['_actual_device_cache']['device_type_id']
        if bundle is None and device_type == 'dynamic-vm':
            host_job_id = job_id.replace('.1', '.0')
            bundle = jobs[host_job_id]['bundle']
            if bundle is None:
                print '%s bundle is empty, skipping...' % device_type
                continue
        # Retrieve the log file
        try:
            binary_job_file = connection.scheduler.job_output(job_id)
        except xmlrpclib.Fault:
            print 'Job output not found for %s' % device_type
            continue
        # Parse LAVA messages out of log
        raw_job_file = str(binary_job_file)
        for line in raw_job_file.splitlines():
            if 'Infrastructure Error:' in line:
                print 'Infrastructure Error detected!'
                index = line.find('Infrastructure Error:')
                boot_failure_reason = line[index:]
                board_offline = True
            if 'Bootloader Error:' in line:
                print 'Bootloader Error detected!'
                index = line.find('Bootloader Error:')
                boot_failure_reason = line[index:]
                board_offline = True
            if 'Kernel Error:' in line:
                print 'Kernel Error detected!'
                index = line.find('Kernel Error:')
                boot_failure_reason = line[index:]
            if 'Userspace Error:' in line:
                print 'Userspace Error detected!'
                index = line.find('Userspace Error:')
                boot_failure_reason = line[index:]
            if '<LAVA_DISPATCHER>' not in line:
                if len(line) != 0:
                    job_file += line + '\n'
            if '### dt-test ### end of selftest' in line:
                dt_tests = True
                regex = re.compile("(?P<test>\d+\*?)")
                dt_test_results = regex.findall(line)
                if len(dt_test_results) > 2:
                    dt_tests_passed = dt_test_results[2]
                    dt_tests_failed = dt_test_results[3]
                else:
                    dt_tests_passed = dt_test_results[0]
                    dt_tests_failed = dt_test_results[1]
                if int(dt_tests_failed) > 0:
                    dt_test_result = 'FAIL'
                else:
                    dt_test_result = 'PASS'
            if 'rtc-efi rtc-efi: setting system clock to' in line:
                if device_type == 'dynamic-vm':
                    efi_rtc = True

        # Retrieve bundle
        if bundle is not None:
            json_bundle = connection.dashboard.get(bundle)
            bundle_data = json.loads(json_bundle['content'])
            # Get the boot data from LAVA
            for test_results in bundle_data['test_runs']:
                # Check for the LAVA self boot test
                if test_results['test_id'] == 'lava':
                    for test in test_results['test_results']:
                        # TODO for compat :(
                        if test['test_case_id'] == 'kernel_boot_time':
                            kernel_boot_time = test['measurement']
                        if test['test_case_id'] == 'test_kernel_boot_time':
                            kernel_boot_time = test['measurement']
                    bundle_attributes = bundle_data['test_runs'][-1][
                        'attributes']
            if utils.in_bundle_attributes(bundle_attributes,
                                          'kernel.defconfig'):
                print bundle_attributes['kernel.defconfig']
            if utils.in_bundle_attributes(bundle_attributes, 'target'):
                board_instance = bundle_attributes['target']
            if utils.in_bundle_attributes(bundle_attributes,
                                          'kernel.defconfig'):
                kernel_defconfig = bundle_attributes['kernel.defconfig']
                defconfig_list = kernel_defconfig.split('-')
                #arch = defconfig_list[0]
                arch = defconfig_list[-1]
                # Remove arch
                defconfig_list.pop(0)
                kernel_defconfig_full = '-'.join(defconfig_list)
                kernel_defconfig_base = ''.join(
                    kernel_defconfig_full.split('+')[:1])
                if kernel_defconfig_full == kernel_defconfig_base:
                    kernel_defconfig_full = None
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.version'):
                kernel_version = bundle_attributes['kernel.version']
            if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
                device_tree = bundle_attributes['device.tree']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
                kernel_endian = bundle_attributes['kernel.endian']
            if utils.in_bundle_attributes(bundle_attributes,
                                          'platform.fastboot'):
                fastboot = bundle_attributes['platform.fastboot']
            if kernel_boot_time is None:
                if utils.in_bundle_attributes(bundle_attributes,
                                              'kernel-boot-time'):
                    kernel_boot_time = bundle_attributes['kernel-boot-time']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
                kernel_tree = bundle_attributes['kernel.tree']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
                kernel_addr = bundle_attributes['kernel-addr']
            if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
                initrd_addr = bundle_attributes['initrd-addr']
            if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
                dtb_addr = bundle_attributes['dtb-addr']
            if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
                dtb_append = bundle_attributes['dtb-append']
            if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
                boot_retries = int(bundle_attributes['boot_retries'])
            if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
                test_plan = bundle_attributes['test.plan']

        # Check if we found efi-rtc
        if test_plan == 'boot-kvm-uefi' and not efi_rtc:
            if device_type == 'dynamic-vm':
                boot_failure_reason = 'Unable to read EFI rtc'
                result = 'FAIL'
        # Record the boot log and result
        # TODO: Will need to map device_types to dashboard device types
        if kernel_defconfig and device_type and result:
            if ('arm' == arch or 'arm64' == arch) and device_tree is None:
                platform_name = device_map[device_type][0] + ',legacy'
            else:
                if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
                    platform_name = device_map[device_type][0] + '_rootfs:nfs'
                else:
                    platform_name = device_map[device_type][0]

            print 'Creating boot log for %s' % platform_name
            log = 'boot-%s.txt' % (platform_name + job_name)
            html = 'boot-%s.html' % (platform_name + job_name)
            if config.get("lab"):
                directory = os.path.join(
                    results_directory,
                    kernel_defconfig + '/' + config.get("lab"))
            else:
                directory = os.path.join(results_directory, kernel_defconfig)
            utils.ensure_dir(directory)
            utils.write_file(job_file, log, directory)
            if kernel_boot_time is None:
                kernel_boot_time = '0.0'
            if results.has_key(kernel_defconfig):
                results[kernel_defconfig].append({
                    'device_type': platform_name,
                    'dt_test_result': dt_test_result,
                    'dt_tests_passed': dt_tests_passed,
                    'dt_tests_failed': dt_tests_failed,
                    'kernel_boot_time': kernel_boot_time,
                    'result': result,
                    'device_name': device_name
                })
            else:
                results[kernel_defconfig] = [{
                    'device_type': platform_name,
                    'dt_test_result': dt_test_result,
                    'dt_tests_passed': dt_tests_passed,
                    'dt_tests_failed': dt_tests_failed,
                    'kernel_boot_time': kernel_boot_time,
                    'result': result,
                    'device_name': device_name
                }]
            # Create JSON format boot metadata
            print 'Creating JSON format boot metadata'
            if config.get("lab"):
                boot_meta['lab_name'] = config.get("lab")
            else:
                boot_meta['lab_name'] = None
            if board_instance:
                boot_meta['board_instance'] = board_instance
            boot_meta['retries'] = boot_retries
            boot_meta['boot_log'] = log
            boot_meta['boot_log_html'] = html
            # TODO: Fix this
            boot_meta['version'] = '1.0'
            boot_meta['arch'] = arch
            boot_meta['defconfig'] = kernel_defconfig_base
            if kernel_defconfig_full is not None:
                boot_meta['defconfig_full'] = kernel_defconfig_full
            if device_map[device_type][1]:
                boot_meta['mach'] = device_map[device_type][1]
            boot_meta['kernel'] = kernel_version
            boot_meta['job'] = kernel_tree
            boot_meta['board'] = platform_name
            if board_offline and result == 'FAIL':
                boot_meta['boot_result'] = 'OFFLINE'
                #results[kernel_defconfig]['result'] = 'OFFLINE'
            else:
                boot_meta['boot_result'] = result
            if result == 'FAIL' or result == 'OFFLINE':
                if boot_failure_reason:
                    boot_meta['boot_result_description'] = boot_failure_reason
                else:
                    boot_meta[
                        'boot_result_description'] = 'Unknown Error: platform failed to boot'
            boot_meta['boot_time'] = kernel_boot_time
            # TODO: Fix this
            boot_meta['boot_warnings'] = None
            if device_tree:
                if arch == 'arm64':
                    boot_meta['dtb'] = 'dtbs/' + device_map[device_type][
                        1] + '/' + device_tree
                else:
                    boot_meta['dtb'] = 'dtbs/' + device_tree
            else:
                boot_meta['dtb'] = device_tree
            boot_meta['dtb_addr'] = dtb_addr
            boot_meta['dtb_append'] = dtb_append
            boot_meta['dt_test'] = dt_test
            boot_meta['endian'] = kernel_endian
            boot_meta['fastboot'] = fastboot
            # TODO: Fix this
            boot_meta['initrd'] = None
            boot_meta['initrd_addr'] = initrd_addr
            if arch == 'arm':
                boot_meta['kernel_image'] = 'zImage'
            elif arch == 'arm64':
                boot_meta['kernel_image'] = 'Image'
            else:
                boot_meta['kernel_image'] = 'bzImage'
            boot_meta['loadaddr'] = kernel_addr
            json_file = 'boot-%s.json' % (platform_name + job_name)
            utils.write_json(json_file, directory, boot_meta)
            #print 'Creating html version of boot log for %s' % platform_name
            #cmd = 'python log2html.py %s' % os.path.join(directory, log)
            #subprocess.check_output(cmd, shell=True)
            #if config.get("lab") and config.get("api") and config.get("token"):
            #    print 'Sending boot result to %s for %s' % (config.get("api"), platform_name)
            #    headers = {
            #        'Authorization': config.get("token"),
            #        'Content-Type': 'application/json'
            #    }
            #    api_url = urlparse.urljoin(config.get("api"), '/boot')
            #    push('POST', api_url, data=json.dumps(boot_meta), headers=headers)
            #    headers = {
            #        'Authorization': config.get("token"),
            #    }
            #    print 'Uploading text version of boot log'
            #    with open(os.path.join(directory, log)) as lh:
            #        data = lh.read()
            #    api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s' % (kernel_tree,
            #                                                                     kernel_version,
            #                                                                     kernel_defconfig,
            #                                                                     config.get("lab"),
            #                                                                     log))
            #    push('PUT', api_url, data=data, headers=headers)
            #    print 'Uploading html version of boot log'
            #    with open(os.path.join(directory, html)) as lh:
            #        data = lh.read()
            #    api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s' % (kernel_tree,
            #                                                                     kernel_version,
            #                                                                     kernel_defconfig,
            #                                                                     config.get("lab"),
            #                                                                     html))
            #    push('PUT', api_url, data=data, headers=headers)
    if config.get("lab"):
        report_directory = os.path.join(results_directory, config.get("lab"))
        utils.mkdir(report_directory)
    else:
        report_directory = results_directory

    if results and kernel_tree and kernel_version and 'boot' in test_plan or 'BOOT' in test_plan:
        print 'Creating summary for %s' % (kernel_version)
        boot = '%s-boot-report.txt' % (kernel_version)
        boot = boot.replace('boot', test_plan)
        passed = 0
        failed = 0
        for defconfig, results_list in results.items():
            for result in results_list:
                if result['result'] == 'PASS':
                    passed += 1
                else:
                    failed += 1
        total = passed + failed
        with open(os.path.join(report_directory, boot), 'a') as f:
            #f.write('To: %s\n' % config.get("email"))
            #f.write('From: [email protected]\n')
            f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' %
                    (kernel_tree, str(total), str(passed), str(failed),
                     kernel_version))
            f.write('\n')
            #f.write('Full Build Report: http://192.168.1.108:5000/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            #f.write('Full Boot Report: http://192.168.1.108:5000/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            #f.write('\n')
            f.write('Total Duration: %.2f minutes\n' % (duration / 60))
            f.write('Tree/Branch: %s\n' % kernel_tree)
            f.write('Git Describe: %s\n' % kernel_version)
            first = True
            for defconfig, results_list in results.items():
                for result in results_list:
                    if result['result'] == 'OFFLINE':
                        if first:
                            f.write('\n')
                            f.write('Boards Offline:\n')
                            first = False
                        f.write('\n')
                        f.write(defconfig)
                        f.write('\n')
                        break
                for result in results_list:
                    if result['result'] == 'OFFLINE':
                        f.write('    %s   %s   %ss   boot-test: %s\n' %
                                (result['device_type'], result['device_name'],
                                 result['kernel_boot_time'], result['result']))
                        f.write('\n')
            first = True
            for defconfig, results_list in results.items():
                for result in results_list:
                    if result['result'] == 'FAIL':
                        if first:
                            f.write('\n')
                            f.write('Failed Boot Tests:\n')
                            first = False
                        f.write('\n')
                        f.write(defconfig)
                        f.write('\n')
                        break
                for result in results_list:
                    if result['result'] == 'FAIL':
                        f.write('    %s   %s   %ss   boot-test: %s\n' %
                                (result['device_type'], result['device_name'],
                                 result['kernel_boot_time'], result['result']))
                        if config.get("lab"):
                            f.write(
                                '    http://192.168.1.108:8083/kernel-ci/%s/%s/%s/%s/boot-%s.html'
                                % (kernel_tree, kernel_version, defconfig,
                                   config.get("lab"), result['device_type']))
                        else:
                            f.write(
                                '    http://192.168.1.108:8083/kernel-ci/%s/%s/%s/boot-%s.html'
                                % (kernel_tree, kernel_version, defconfig,
                                   result['device_type']))
                        f.write('\n')
            f.write('\n')
            f.write('Full Boot Report:\n')
            for defconfig, results_list in results.items():
                f.write('\n')
                f.write(defconfig)
                f.write('\n')
                for result in results_list:
                    f.write('    %s   %s   %ss   boot-test: %s\n' %
                            (result['device_type'], result['device_name'],
                             result['kernel_boot_time'], result['result']))
    # add by wuyanjun
    if results and directory:
        parser_and_get_result(results, directory, report_directory)
        #get_ip_board_mapping(results, directory, report_directory)

    # dt-self-test
    if results and kernel_tree and kernel_version and dt_tests:
        print 'Creating device tree runtime self test summary for %s' % kernel_version
        dt_self_test = '%s-dt-runtime-self-test-report.txt' % kernel_version
        passed = 0
        failed = 0
        for defconfig, results_list in results.items():
            for result in results_list:
                if result['dt_test_result'] == 'PASS':
                    passed += 1
                elif result['dt_test_result'] == 'FAIL':
                    failed += 1
        total = passed + failed
        with open(os.path.join(report_directory, dt_self_test), 'a') as f:
            #f.write('To: %s\n' % config.get("email"))
            #f.write('From: [email protected]\n')
            f.write(
                'Subject: %s dt-runtime-unit-tests: %s boards tested: %s passed, %s failed (%s)\n'
                % (kernel_tree, str(total), str(passed), str(failed),
                   kernel_version))
            f.write('\n')
            #f.write('Full Build Report: http://192.168.1.108:5000/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            #f.write('Full Boot Report: http://192.168.1.108:5000/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            #f.write('Full Test Report: http://192.168.1.108:5000/test/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            #f.write('\n')
            f.write('Tree/Branch: %s\n' % kernel_tree)
            f.write('Git Describe: %s\n' % kernel_version)
            first = True
            for defconfig, results_list in results.items():
                for result in results_list:
                    if result['dt_test_result'] == 'FAIL':
                        if first:
                            f.write('\n')
                            f.write('Failed Device Tree Unit Tests:\n')
                            first = False
                        f.write('\n')
                        f.write(defconfig)
                        f.write('\n')
                        break
                for result in results_list:
                    if result['dt_test_result'] == "FAIL":
                        f.write(
                            '    %s   %s   passed: %s / failed: %s   dt-runtime-unit-tests: %s\n'
                            % (result['device_type'], result['device_name'],
                               result['dt_tests_passed'],
                               result['dt_tests_failed'],
                               result['dt_test_result']))
                        if config.get("lab"):
                            f.write(
                                '    http://192.168.1.108:8083/kernel-ci/%s/%s/%s/%s/boot-%s.html'
                                % (kernel_tree, kernel_version, defconfig,
                                   config.get("lab"), result['device_type']))
                        else:
                            f.write(
                                '    http://192.168.1.108:8083/kernel-ci/%s/%s/%s/boot-%s.html'
                                % (kernel_tree, kernel_version, defconfig,
                                   result['device_type']))
            f.write('\n')
            f.write('\n')
            f.write('Full Unit Test Report:\n')
            for defconfig, results_list in results.items():
                first = True
                for result in results_list:
                    if result['dt_test_result']:
                        if first:
                            f.write('\n')
                            f.write(defconfig)
                            f.write('\n')
                            first = False
                        f.write(
                            '    %s   %s   passed: %s / failed: %s   dt-runtime-unit-tests: %s\n'
                            % (result['device_type'], result['device_name'],
                               result['dt_tests_passed'],
                               result['dt_tests_failed'],
                               result['dt_test_result']))

    # sendmail
    if config.get("email"):
        print 'Sending e-mail summary to %s' % config.get("email")
        if os.path.exists(report_directory):
            cmd = 'cat %s | sendmail -t' % os.path.join(report_directory, boot)
            subprocess.check_output(cmd, shell=True)
        if dt_tests:
            if os.path.exists(report_directory):
                cmd = 'cat %s | sendmail -t' % os.path.join(
                    report_directory, dt_self_test)
                subprocess.check_output(cmd, shell=True)
Esempio n. 11
0
def train_one(model_folder):
    new_model_folder_name = model_folder.replace('_created', '_training')
    os_rename(model_folder, new_model_folder_name)
    frequencies = glob(os_path_join(new_model_folder_name, 'k_*'))
    for frequency in frequencies:
        # Load model
        print('train.py: training {}'.format(frequency))
        model_params_path = os_path_join(frequency, model_params_fname)

        # create model
        model, model_params = get_which_model_from_params_fname(
            model_params_path, return_params=True)
        if 'cuda' in model_params:
            using_cuda = model_params['cuda'] and torch_cuda_is_available()
        else:
            using_cuda = torch_cuda_is_available()

        if using_cuda is True:
            model.cuda()

        # save initial weights
        if 'save_initial' in model_params and model_params[
                'save_initial'] and model_params['save_dir']:
            suffix = '_initial'
            path = add_suffix_to_path(model_params_fname['save_dir'], suffix)  # pylint: disable=E1126
            ensure_dir(path)
            torch_save(model.state_dict(),
                       os_path_join(path, MODEL_DATA_FNAME))
            save_model_params(os_path_join(path, model_params_fname),
                              model_params)

        # loss
        if 'cost_function' in model_params:
            loss = model_params['cost_function']
        elif 'loss_function' in model_params:
            loss = model_params['loss_function']
        else:
            raise ValueError(
                'model_params missing key cost_function or loss_function')

        if loss not in ['MSE', 'L1', 'SmoothL1']:
            raise TypeError('Error must be MSE, L1, or SmoothL1. You gave ' +
                            str(loss))
        if loss == 'MSE':
            from torch.nn import MSELoss
            loss = MSELoss()
        elif loss == 'L1':
            from torch.nn import L1Loss
            loss = L1Loss()
        elif loss == 'SmoothL1':
            from torch.nn import SmoothL1Loss
            loss = SmoothL1Loss()

        # optimizer
        if model_params['optimizer'] == 'Adam':
            from torch.optim import Adam
            optimizer = Adam(model.parameters(),
                             lr=model_params['learning_rate'],
                             weight_decay=model_params['weight_decay'])
        elif model_params['optimizer'] == 'SGD':
            from torch.optim import SGD
            optimizer = SGD(model.parameters(),
                            lr=model_params['learning_rate'],
                            momentum=model_params['momentum'],
                            weight_decay=model_params['weight_decay'])
        else:
            raise ValueError(
                'model_params[\'optimizer\'] must be either Adam or SGD. Got '
                + model_params['optimizer'])

        logger = Logger()

        # Load training, validation, and test data
        # Load primary training data
        dat_train = ApertureDataset(
            model_params['data_train'],
            NUM_SAMPLES_TRAIN,
            k=model_params['k'],
            target_is_data=model_params['data_is_target'])
        loader_train = DataLoader(dat_train,
                                  batch_size=model_params['batch_size'],
                                  shuffle=True,
                                  num_workers=DATALOADER_NUM_WORKERS,
                                  pin_memory=using_cuda)

        # Load secondary training data - used to evaluate training loss after every epoch
        dat_train2 = ApertureDataset(
            model_params['data_train'],
            NUM_SAMPLES_TRAIN_EVAL,
            k=model_params['k'],
            target_is_data=model_params['data_is_target'])
        loader_train_eval = DataLoader(dat_train2,
                                       batch_size=model_params['batch_size'],
                                       shuffle=False,
                                       num_workers=DATALOADER_NUM_WORKERS,
                                       pin_memory=using_cuda)

        # Load validation data - used to evaluate validation loss after every epoch
        dat_val = ApertureDataset(
            model_params['data_val'],
            NUM_SAMPLES_VALID,
            k=model_params['k'],
            target_is_data=model_params['data_is_target'])
        loader_val = DataLoader(dat_val,
                                batch_size=model_params['batch_size'],
                                shuffle=False,
                                num_workers=DATALOADER_NUM_WORKERS,
                                pin_memory=using_cuda)

        trainer = Trainer(
            model=model,
            loss=loss,
            optimizer=optimizer,
            patience=model_params['patience'],
            loader_train=loader_train,
            loader_train_eval=loader_train_eval,
            loader_val=loader_val,
            cuda=using_cuda,
            logger=logger,
            data_noise_gaussian=model_params['data_noise_gaussian'],
            save_dir=frequency)

        # run training
        trainer.train()

    os_rename(new_model_folder_name,
              new_model_folder_name.replace('_training', '_trained'))
Esempio n. 12
0
def extract_features_batch(model, source_path, target_path, dataset,
                           voxel_size, device):
    """
    Extracts the per point features in the FCGF feature space and saves them to the predefined path

    Args:
        model (FCGF model instance): model used to inferr the descriptors
        source_path (str): path to the raw files
        target path (str): where to save the extracted data
        dataset (float): name of the dataset
        voxel_size (float): voxel sized used to create the sparse tensor
        device (pytorch device): cuda or cpu
    """

    source_path = os.path.join(source_path, dataset, 'raw_data')
    target_path = os.path.join(target_path, dataset, 'features')

    ensure_dir(target_path)

    folders = get_folder_list(source_path)

    assert len(folders) > 0, 'Could not find {} folders under {}'.format(
        dataset, source_path)

    logging.info(folders)
    list_file = os.path.join(target_path, 'list.txt')
    f = open(list_file, 'w')
    model.eval()

    for fo in folders:
        scene_name = fo.split()
        files = get_file_list(fo, '.ply')
        fo_base = os.path.basename(fo)
        ensure_dir(os.path.join(target_path, fo_base))

        f.write('%s %d\n' % (fo_base, len(files)))
        for i, fi in enumerate(files):
            save_fn = '%s_%03d' % (fo_base, i)
            if os.path.exists(
                    os.path.join(target_path, fo_base, save_fn + '.npz')):
                print(
                    'Correspondence file already exits moving to the next example.'
                )
            else:
                # Extract features from a file
                pcd = o3d.io.read_point_cloud(fi)

                if i % 100 == 0:
                    logging.info(f'{i} / {len(files)}: {save_fn}')

                xyz_down, feature = extract_features(model,
                                                     xyz=np.array(pcd.points),
                                                     rgb=None,
                                                     normal=None,
                                                     voxel_size=voxel_size,
                                                     device=device,
                                                     skip_check=True)

                np.savez_compressed(os.path.join(target_path, fo_base,
                                                 save_fn),
                                    points=np.array(pcd.points),
                                    xyz=xyz_down,
                                    feature=feature.detach().cpu().numpy())

    f.close()
Esempio n. 13
0
def extract_precomputed_training_data(dataset, source_path, target_path,
                                      voxel_size, inlier_threshold):
    """
    Prepares the data for training the filtering networks with precomputed correspondences (without FCGF descriptor) 

    Args:
        dataset (str): name of the dataset
        source_path (str): path to the raw data
        target_path (str): path to where the extracted data will be saved
        voxel_size (float): voxel size that was used to compute the features
        inlier_threshold (float): threshold to determine if a correspondence is an inlier or outlier
    """
    source_path = os.path.join(source_path, dataset, 'raw_data')
    features_path = os.path.join(target_path, dataset, 'features')
    correspondence_path = os.path.join(target_path, dataset, 'correspondences')
    target_path = os.path.join(target_path, dataset, 'training_data')

    ensure_dir(target_path)

    # Check that the GT global transformation matrices are available and that the FCGF features are computed
    folders = get_folder_list(source_path)

    assert len(folders) > 0, 'Could not find {} folders under {}'.format(
        dataset, source_path)

    logging.info('Found {} scenes from the {} dataset!'.format(
        len(folders), dataset))

    for fo in folders:

        scene_name = fo.split()
        fo_base = os.path.basename(fo)
        ensure_dir(os.path.join(target_path, fo_base))

        pc_files = get_file_list(fo, '.ply')
        trans_files = get_file_list(fo, '.txt')
        assert len(pc_files) <= len(
            trans_files
        ), 'The number of point cloud files does not equal the number of GT trans parameters!'

        feat_files = get_file_list(os.path.join(features_path, fo_base),
                                   '.npz')
        assert len(pc_files) == len(
            feat_files
        ), 'Features for scene {} are either not computed or some are missing!'.format(
            fo_base)

        coor_files = get_file_list(os.path.join(correspondence_path, fo_base),
                                   '.npz')

        assert len(coor_files) == int(
            (len(feat_files) * (len(feat_files) - 1)) / 2
        ), 'Correspondence files for the scene {} are missing. First run the correspondence extraction!'.format(
            fo_base)

        # Loop over all fragment pairs and compute the training data
        for idx_1 in range(len(pc_files)):
            for idx_2 in range(idx_1 + 1, len(pc_files)):
                if os.path.exists(
                        os.path.join(
                            target_path, fo_base,
                            '{}_{}_{}.npz'.format(fo_base,
                                                  str(idx_1).zfill(3),
                                                  str(idx_2).zfill(3)))):
                    logging.info(
                        'Training file already exits moving to the next example.'
                    )

                data = np.load(
                    os.path.join(
                        correspondence_path, fo_base,
                        '{}_{}_{}.npz'.format(fo_base,
                                              str(idx_1).zfill(3),
                                              str(idx_2).zfill(3))))
                xs = data['xs']
                mutuals = data['mutuals']
                ratios = data['ratios']

                # Get the GT transformation parameters
                t_1 = np.genfromtxt(os.path.join(
                    source_path, fo_base,
                    'cloud_bin_{}.info.txt'.format(idx_1)),
                                    skip_header=1)
                t_2 = np.genfromtxt(os.path.join(
                    source_path, fo_base,
                    'cloud_bin_{}.info.txt'.format(idx_2)),
                                    skip_header=1)

                # Get the GT transformation parameters
                pc_1 = load_point_cloud(os.path.join(
                    source_path, fo_base, 'cloud_bin_{}.ply'.format(idx_1)),
                                        data_type='numpy')
                pc_2 = load_point_cloud(os.path.join(
                    source_path, fo_base, 'cloud_bin_{}.ply'.format(idx_2)),
                                        data_type='numpy')

                pc_1_tr = transform_point_cloud(pc_1, t_1[0:3, 0:3],
                                                t_1[0:3, 3].reshape(-1, 1))
                pc_2_tr = transform_point_cloud(pc_2, t_2[0:3, 0:3],
                                                t_2[0:3, 3].reshape(-1, 1))

                overlap_ratio = compute_overlap_ratio(pc_1_tr,
                                                      pc_2_tr,
                                                      np.eye(4),
                                                      method='FCGF',
                                                      voxel_size=voxel_size)

                # Estimate pairwise transformation parameters
                t_3 = np.matmul(np.linalg.inv(t_2), t_1)

                r_matrix = t_3[0:3, 0:3]
                t_vector = t_3[0:3, 3]

                # Transform the keypoints of the first point cloud
                pc_1_key_tr = transform_point_cloud(xs[:, 0:3], r_matrix,
                                                    t_vector.reshape(-1, 1))

                # Compute the residuals after the transformation
                y_s = np.sqrt(
                    np.sum(np.square(pc_1_key_tr - xs[:, 3:6]), axis=1))

                # Inlier percentage
                inlier_ratio = np.where(
                    y_s < inlier_threshold)[0].shape[0] / y_s.shape[0]
                inlier_ratio_mutuals = np.where(
                    y_s[mutuals.astype(bool).reshape(-1)] < inlier_threshold
                )[0].shape[0] / np.sum(mutuals)

                np.savez_compressed(os.path.join(
                    target_path, fo_base, 'cloud_{}_{}.npz'.format(
                        str(idx_1).zfill(3),
                        str(idx_2).zfill(3))),
                                    R=r_matrix,
                                    t=t_vector,
                                    x=xs,
                                    y=y_s,
                                    mutuals=mutuals,
                                    inlier_ratio=inlier_ratio,
                                    inlier_ratio_mutuals=inlier_ratio_mutuals,
                                    ratios=ratios,
                                    overlap=overlap_ratio)
Esempio n. 14
0
def run_correspondence_extraction(dataset, source_path, target_path,
                                  n_correspondences, idx):
    """
    Computes the correspondences in the FCGF space together with the mutuals and ratios side information

    Args:
        dataset (str): name of the dataset
        source_path (str): path to the raw data
        target_path (str): path to where the extracted data will be saved
        n_correspondences (int): number of points to sample
        idx (int): index of the scene, used for parallel processing

    """

    # Initialize all the paths
    features_path = os.path.join(target_path, dataset, 'features')
    target_path = os.path.join(target_path, dataset, 'correspondences')

    fo = get_folder_list(source_path)[idx]
    fo_base = os.path.basename(fo)
    files = get_file_list(os.path.join(features_path, fo_base), '.npz')

    ensure_dir(os.path.join(target_path, fo_base))

    # Loop over all fragment pairs and compute the training data
    for idx_1 in range(len(files)):
        for idx_2 in range(idx_1 + 1, len(files)):
            if os.path.exists(
                    os.path.join(
                        target_path, fo_base,
                        '{}_{}_{}.npz'.format(fo_base,
                                              str(idx_1).zfill(3),
                                              str(idx_2).zfill(3)))):
                logging.info(
                    'Correspondence file already exits moving to the next example.'
                )

            else:
                pc_1_data = np.load(
                    os.path.join(
                        features_path, fo_base,
                        fo_base + '_{}.npz'.format(str(idx_1).zfill(3))))
                pc_1_features = pc_1_data['feature']
                pc_1_keypoints = pc_1_data['xyz']

                pc_2_data = np.load(
                    os.path.join(
                        features_path, fo_base,
                        fo_base + '_{}.npz'.format(str(idx_2).zfill(3))))
                pc_2_features = pc_2_data['feature']
                pc_2_keypoints = pc_2_data['xyz']

                # Sample with replacement if less then n_correspondences points are in the point cloud
                if pc_1_features.shape[0] >= n_correspondences:
                    inds_1 = np.random.choice(pc_1_features.shape[0],
                                              n_correspondences,
                                              replace=False)
                else:
                    inds_1 = np.random.choice(pc_1_features.shape[0],
                                              n_correspondences,
                                              replace=True)

                if pc_2_features.shape[0] >= n_correspondences:
                    inds_2 = np.random.choice(pc_2_features.shape[0],
                                              n_correspondences,
                                              replace=False)
                else:
                    inds_2 = np.random.choice(pc_2_features.shape[0],
                                              n_correspondences,
                                              replace=True)

                pc_1_features = pc_1_features[inds_1, :]
                pc_2_features = pc_2_features[inds_2, :]
                pc_1_key = pc_1_keypoints[inds_1, :]
                pc_2_key = pc_2_keypoints[inds_2, :]

                # find the correspondence using nearest neighbor search in the feature space (two way)
                nn_search = NearestNeighbors(n_neighbors=1,
                                             metric='minkowski',
                                             p=2)
                nn_search.fit(pc_2_features)
                nn_dists, nn_indices = nn_search.kneighbors(
                    X=pc_1_features, n_neighbors=2, return_distance=True)

                nn_search.fit(pc_1_features)
                nn_dists_1, nn_indices_1 = nn_search.kneighbors(
                    X=pc_2_features, n_neighbors=2, return_distance=True)

                ol_nn_ids = np.where(
                    (nn_indices[nn_indices_1[:, 0], 0] -
                     np.arange(pc_1_features.shape[0])) == 0)[0]

                # Initialize mutuals and ratios
                mutuals = np.zeros((n_correspondences, 1))
                mutuals[ol_nn_ids] = 1
                ratios = nn_dists[:, 0] / nn_dists[:, 1]

                # Concatenate the correspondence coordinates
                xs = np.concatenate(
                    (pc_1_key[nn_indices_1[:, 0], :], pc_2_key), axis=1)

                np.savez_compressed(os.path.join(
                    target_path, fo_base,
                    '{}_{}_{}.npz'.format(fo_base,
                                          str(idx_1).zfill(3),
                                          str(idx_2).zfill(3))),
                                    x=xs,
                                    mutuals=mutuals,
                                    ratios=ratios)
Esempio n. 15
0
def boot_report(config):
    connection, jobs, duration =  parse_json(config.get("boot"))
    # TODO: Fix this when multi-lab sync is working
    #download_log2html(log2html)
    results_directory = os.getcwd() + '/results'
    results = {}
    dt_tests = False
    utils.mkdir(results_directory)
    for job_id in jobs:
        print 'Job ID: %s' % job_id
        # Init
        boot_meta = {}
        api_url = None
        arch = None
        board_instance = None
        boot_retries = 0
        kernel_defconfig_full = None
        kernel_defconfig = None
        kernel_defconfig_base = None
        kernel_version = None
        device_tree = None
        kernel_endian = None
        kernel_tree = None
        kernel_addr = None
        initrd_addr = None
        dtb_addr = None
        dtb_append = None
        fastboot = None
        fastboot_cmd = None
        test_plan = None
        job_file = ''
        dt_test = None
        dt_test_result = None
        dt_tests_passed = None
        dt_tests_failed = None
        board_offline = False
        kernel_boot_time = None
        boot_failure_reason = None
        efi_rtc = False
        # Retrieve job details
        job_details = connection.scheduler.job_details(job_id)
        if job_details['requested_device_type_id']:
            device_type = job_details['requested_device_type_id']
        if job_details['description']:
            job_name = job_details['description']
        result = jobs[job_id]['result']
        bundle = jobs[job_id]['bundle']
        if bundle is None and device_type == 'dynamic-vm':
            host_job_id = job_id.replace('.1', '.0')
            bundle = jobs[host_job_id]['bundle']
            if bundle is None:
                print '%s bundle is empty, skipping...' % device_type
                continue
        # Retrieve the log file
        try:
            binary_job_file = connection.scheduler.job_output(job_id)
        except xmlrpclib.Fault:
            print 'Job output not found for %s' % device_type
            continue
        # Parse LAVA messages out of log
        raw_job_file = str(binary_job_file)
        for line in raw_job_file.splitlines():
            if 'Infrastructure Error:' in line:
                print 'Infrastructure Error detected!'
                index = line.find('Infrastructure Error:')
                boot_failure_reason = line[index:]
                board_offline = True
            if 'Bootloader Error:' in line:
                print 'Bootloader Error detected!'
                index = line.find('Bootloader Error:')
                boot_failure_reason = line[index:]
                board_offline = True
            if 'Kernel Error:' in line:
                print 'Kernel Error detected!'
                index = line.find('Kernel Error:')
                boot_failure_reason = line[index:]
            if 'Userspace Error:' in line:
                print 'Userspace Error detected!'
                index = line.find('Userspace Error:')
                boot_failure_reason = line[index:]
            if '<LAVA_DISPATCHER>' not in line:
                if len(line) != 0:
                    job_file += line + '\n'
            if '### dt-test ### end of selftest' in line:
                dt_tests = True
                regex = re.compile("(?P<test>\d+\*?)")
                dt_test_results = regex.findall(line)
                if len(dt_test_results) > 2:
                    dt_tests_passed = dt_test_results[2]
                    dt_tests_failed = dt_test_results[3]
                else:
                    dt_tests_passed = dt_test_results[0]
                    dt_tests_failed = dt_test_results[1]
                if int(dt_tests_failed) > 0:
                    dt_test_result = 'FAIL'
                else:
                    dt_test_result = 'PASS'
            if 'rtc-efi rtc-efi: setting system clock to' in line:
                if device_type == 'dynamic-vm':
                    efi_rtc = True
        # Retrieve bundle
        if bundle is not None:
            json_bundle = connection.dashboard.get(bundle)
            bundle_data = json.loads(json_bundle['content'])
            # Get the boot data from LAVA
            for test_results in bundle_data['test_runs']:
                # Check for the LAVA self boot test
                if test_results['test_id'] == 'lava':
                    for test in test_results['test_results']:
                        # TODO for compat :(
                        if test['test_case_id'] == 'kernel_boot_time':
                            kernel_boot_time = test['measurement']
                        if test['test_case_id'] == 'test_kernel_boot_time':
                            kernel_boot_time = test['measurement']
                    bundle_attributes = bundle_data['test_runs'][-1]['attributes']
		if test_results['test_id'] == 'lava-command':
                    # Post stuff with the test API.
                    build_id = "56b9648659b514b7f6e41fac"
                    test_suite = test_api_post(config, build_id, test_results)
                    if test_suite:
                        print json.dumps(test_suite)
                        print 'Sending test suite to %s' % config.get("api")
                        headers = {
                                   'Authorization': config.get("token"),
                                   'Content-Type': 'application/json'
                                  }
                        api_url = urlparse.urljoin(config.get("api"), '/test/suite')
                        push('POST', api_url, json.dumps(test_suite), headers)
                    
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
                print bundle_attributes['kernel.defconfig']
            if utils.in_bundle_attributes(bundle_attributes, 'target'):
                board_instance = bundle_attributes['target']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
                kernel_defconfig = bundle_attributes['kernel.defconfig']
                defconfig_list = kernel_defconfig.split('-')
                arch = defconfig_list[0]
                # Remove arch
                defconfig_list.pop(0)
                kernel_defconfig_full = '-'.join(defconfig_list)
                kernel_defconfig_base = ''.join(kernel_defconfig_full.split('+')[:1])
                if kernel_defconfig_full == kernel_defconfig_base:
                    kernel_defconfig_full = None
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.version'):
                kernel_version = bundle_attributes['kernel.version']
            if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
                device_tree = bundle_attributes['device.tree']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
                kernel_endian = bundle_attributes['kernel.endian']
            if utils.in_bundle_attributes(bundle_attributes, 'platform.fastboot'):
                fastboot = bundle_attributes['platform.fastboot']
            if kernel_boot_time is None:
                if utils.in_bundle_attributes(bundle_attributes, 'kernel-boot-time'):
                    kernel_boot_time = bundle_attributes['kernel-boot-time']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
                kernel_tree = bundle_attributes['kernel.tree']
            if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
                kernel_addr = bundle_attributes['kernel-addr']
            if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
                initrd_addr = bundle_attributes['initrd-addr']
            if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
                dtb_addr = bundle_attributes['dtb-addr']
            if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
                dtb_append = bundle_attributes['dtb-append']
            if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
                boot_retries = int(bundle_attributes['boot_retries'])
            if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
                test_plan = bundle_attributes['test.plan']

        # Check if we found efi-rtc
        if test_plan == 'boot-kvm-uefi' and not efi_rtc:
            if device_type == 'dynamic-vm':
                boot_failure_reason = 'Unable to read EFI rtc'
                result = 'FAIL'
        # Record the boot log and result
        # TODO: Will need to map device_types to dashboard device types
        if kernel_defconfig and device_type and result:
            if (arch == 'arm' or arch =='arm64') and device_tree is None:
                platform_name = device_map[device_type][0] + ',legacy'
            else:
                if device_tree == 'vexpress-v2p-ca15_a7.dtb':
                    platform_name = 'vexpress-v2p-ca15_a7'
                elif device_tree == 'fsl-ls2080a-simu.dtb':
                    platform_name = 'fsl-ls2080a-simu'
                elif test_plan == 'boot-kvm' or test_plan == 'boot-kvm-uefi':
                    if device_tree == 'sun7i-a20-cubietruck.dtb':
                        if device_type == 'dynamic-vm':
                            device_type = 'cubieboard3-kvm-guest'
                            platform_name = device_map[device_type][0]
                        else:
                            device_type = 'cubieboard3-kvm-host'
                            platform_name = device_map[device_type][0]
                    elif device_tree == 'apm-mustang.dtb':
                        if device_type == 'dynamic-vm':
                            if test_plan == 'boot-kvm-uefi':
                                device_type = 'mustang-kvm-uefi-guest'
                            else:
                                device_type = 'mustang-kvm-guest'
                            platform_name = device_map[device_type][0]
                        else:
                            if test_plan == 'boot-kvm-uefi':
                                device_type = 'mustang-kvm-uefi-host'
                            else:
                                device_type = 'mustang-kvm-host'
                            platform_name = device_map[device_type][0]
                    elif device_tree == 'juno.dtb':
                        if device_type == 'dynamic-vm':
                            if test_plan == 'boot-kvm-uefi':
                                device_type = 'juno-kvm-uefi-guest'
                            else:
                                device_type = 'juno-kvm-guest'
                            platform_name = device_map[device_type][0]
                        else:
                            if test_plan == 'boot-kvm-uefi':
                                device_type = 'juno-kvm-uefi-host'
                            else:
                                device_type = 'juno-kvm-host'
                            platform_name = device_map[device_type][0]
                elif test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
                    platform_name = device_map[device_type][0] + '_rootfs:nfs'
                else:
                    platform_name = device_map[device_type][0]
            print 'Creating boot log for %s' % platform_name
            log = 'boot-%s.txt' % platform_name
            html = 'boot-%s.html' % platform_name
            if config.get("lab"):
                directory = os.path.join(results_directory, kernel_defconfig + '/' + config.get("lab"))
            else:
                directory = os.path.join(results_directory, kernel_defconfig)
            utils.ensure_dir(directory)
            utils.write_file(job_file, log, directory)
            if kernel_boot_time is None:
                kernel_boot_time = '0.0'
            if results.has_key(kernel_defconfig):
                results[kernel_defconfig].append({'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result})
            else:
                results[kernel_defconfig] = [{'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result}]
            # Create JSON format boot metadata
            print 'Creating JSON format boot metadata'
            if config.get("lab"):
                boot_meta['lab_name'] = config.get("lab")
            else:
                boot_meta['lab_name'] = None
            if board_instance:
                boot_meta['board_instance'] = board_instance
            boot_meta['retries'] = boot_retries
            boot_meta['boot_log'] = log
            boot_meta['boot_log_html'] = html
            # TODO: Fix this
            boot_meta['version'] = '1.0'
            boot_meta['arch'] = arch
            boot_meta['defconfig'] = kernel_defconfig_base
            if kernel_defconfig_full is not None:
                boot_meta['defconfig_full'] = kernel_defconfig_full
            if device_map[device_type][1]:
                boot_meta['mach'] = device_map[device_type][1]
            boot_meta['kernel'] = kernel_version
            boot_meta['job'] = kernel_tree
            boot_meta['board'] = platform_name
            if board_offline and result == 'FAIL':
                boot_meta['boot_result'] = 'OFFLINE'
                #results[kernel_defconfig]['result'] = 'OFFLINE'
            else:
                boot_meta['boot_result'] = result
            if result == 'FAIL' or result == 'OFFLINE':
                if boot_failure_reason:
                    boot_meta['boot_result_description'] = boot_failure_reason
                else:
                    boot_meta['boot_result_description'] = 'Unknown Error: platform failed to boot'
            boot_meta['boot_time'] = kernel_boot_time
            # TODO: Fix this
            boot_meta['boot_warnings'] = None
            if device_tree:
                if arch == 'arm64':
                    boot_meta['dtb'] = 'dtbs/' + device_map[device_type][1] + '/' + device_tree
                else:
                    boot_meta['dtb'] = 'dtbs/' + device_tree
            else:
                boot_meta['dtb'] = device_tree
            boot_meta['dtb_addr'] = dtb_addr
            boot_meta['dtb_append'] = dtb_append
            boot_meta['dt_test'] = dt_test
            boot_meta['endian'] = kernel_endian
            boot_meta['fastboot'] = fastboot
            # TODO: Fix this
            boot_meta['initrd'] = None
            boot_meta['initrd_addr'] = initrd_addr
            if arch == 'arm':
                boot_meta['kernel_image'] = 'zImage'
            elif arch == 'arm64':
                boot_meta['kernel_image'] = 'Image'
            else:
                boot_meta['kernel_image'] = 'bzImage'
            boot_meta['loadaddr'] = kernel_addr
            json_file = 'boot-%s.json' % platform_name
            utils.write_json(json_file, directory, boot_meta)
            print 'Creating html version of boot log for %s' % platform_name
            cmd = 'python log2html.py %s' % os.path.join(directory, log)
            subprocess.check_output(cmd, shell=True)
            if config.get("lab") and config.get("api") and config.get("token"):
                print 'Sending boot result to %s for %s' % (config.get("api"), platform_name)
                headers = {
                    'Authorization': config.get("token"),
                    'Content-Type': 'application/json'
                }
                api_url = urlparse.urljoin(config.get("api"), '/boot')
                push('POST', api_url, data=json.dumps(boot_meta), headers=headers)
                headers = {
                    'Authorization': config.get("token"),
                }
                print 'Uploading text version of boot log'
                with open(os.path.join(directory, log)) as lh:
                    data = lh.read()
                api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s' % (kernel_tree,
                                                                                 kernel_version,
                                                                                 kernel_defconfig,
                                                                                 config.get("lab"),
                                                                                 log))
                push('PUT', api_url, data=data, headers=headers)
                print 'Uploading html version of boot log'
                with open(os.path.join(directory, html)) as lh:
                    data = lh.read()
                api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s' % (kernel_tree,
                                                                                 kernel_version,
                                                                                 kernel_defconfig,
                                                                                 config.get("lab"),
                                                                                 html))
                push('PUT', api_url, data=data, headers=headers)

    if results and kernel_tree and kernel_version:
        print 'Creating boot summary for %s' % kernel_version
        boot = '%s-boot-report.txt' % kernel_version
        passed = 0
        failed = 0
        for defconfig, results_list in results.items():
            for result in results_list:
                if result['result'] == 'PASS':
                    passed += 1
                else:
                    failed += 1
        total = passed + failed
        if config.get("lab"):
            report_directory = os.path.join(results_directory, config.get("lab"))
            utils.mkdir(report_directory)
        else:
            report_directory = results_directory
        with open(os.path.join(report_directory, boot), 'a') as f:
            f.write('To: %s\n' % config.get("email"))
            f.write('From: [email protected]\n')
            f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' % (kernel_tree,
                                                                                str(total),
                                                                                str(passed),
                                                                                str(failed),
                                                                                kernel_version))
            f.write('\n')
            f.write('Full Build Report: http://kernelci.org/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            f.write('Full Boot Report: http://kernelci.org/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            f.write('\n')
            f.write('Total Duration: %.2f minutes\n' % (duration / 60))
            f.write('Tree/Branch: %s\n' % kernel_tree)
            f.write('Git Describe: %s\n' % kernel_version)
            first = True
            for defconfig, results_list in results.items():
                for result in results_list:
                    if result['result'] == 'OFFLINE':
                        if first:
                            f.write('\n')
                            f.write('Boards Offline:\n')
                            first = False
                        f.write('\n')
                        f.write(defconfig)
                        f.write('\n')
                        break
                for result in results_list:
                    if result['result'] == 'OFFLINE':
                        f.write('    %s   %ss   boot-test: %s\n' % (result['device_type'],
                                                                    result['kernel_boot_time'],
                                                                    result['result']))
                        f.write('\n')
            first = True
            for defconfig, results_list in results.items():
                for result in results_list:
                    if result['result'] == 'FAIL':
                        if first:
                            f.write('\n')
                            f.write('Failed Boot Tests:\n')
                            first = False
                        f.write('\n')
                        f.write(defconfig)
                        f.write('\n')
                        break
                for result in results_list:
                    if result['result'] == 'FAIL':
                        f.write('    %s   %ss   boot-test: %s\n' % (result['device_type'],
                                                                    result['kernel_boot_time'],
                                                                    result['result']))
                        if config.get("lab"):
                            f.write('    http://storage.kernelci.org/kernel-ci/%s/%s/%s/%s/boot-%s.html' % (kernel_tree,
                                                                                                            kernel_version,
                                                                                                            defconfig,
                                                                                                            config.get("lab"),
                                                                                                            result['device_type']))
                        else:
                            f.write('    http://storage.kernelci.org/kernel-ci/%s/%s/%s/boot-%s.html' % (kernel_tree,
                                                                                                         kernel_version,
                                                                                                         defconfig,
                                                                                                         result['device_type']))
                        f.write('\n')
            f.write('\n')
            f.write('Full Boot Report:\n')
            for defconfig, results_list in results.items():
                f.write('\n')
                f.write(defconfig)
                f.write('\n')
                for result in results_list:
                    f.write('    %s   %ss   boot-test: %s\n' % (result['device_type'], result['kernel_boot_time'], result['result']))

    # dt-self-test
    if results and kernel_tree and kernel_version and dt_tests:
        print 'Creating device tree runtime self test summary for %s' % kernel_version
        dt_self_test = '%s-dt-runtime-self-test-report.txt' % kernel_version
        passed = 0
        failed = 0
        for defconfig, results_list in results.items():
            for result in results_list:
                if result['dt_test_result'] == 'PASS':
                    passed += 1
                elif result['dt_test_result'] == 'FAIL':
                    failed += 1
        total = passed + failed
        with open(os.path.join(report_directory, dt_self_test), 'a') as f:
            f.write('To: %s\n' % config.get("email"))
            f.write('From: [email protected]\n')
            f.write('Subject: %s dt-runtime-unit-tests: %s boards tested: %s passed, %s failed (%s)\n' % (kernel_tree,
                                                                                                           str(total),
                                                                                                           str(passed),
                                                                                                           str(failed),
                                                                                                           kernel_version))
            f.write('\n')
            f.write('Full Build Report: http://kernelci.org/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            f.write('Full Boot Report: http://kernelci.org/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            f.write('Full Test Report: http://kernelci.org/test/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            f.write('\n')
            f.write('Tree/Branch: %s\n' % kernel_tree)
            f.write('Git Describe: %s\n' % kernel_version)
            first = True
            for defconfig, results_list in results.items():
                for result in results_list:
                    if result['dt_test_result'] == 'FAIL':
                        if first:
                            f.write('\n')
                            f.write('Failed Device Tree Unit Tests:\n')
                            first = False
                        f.write('\n')
                        f.write(defconfig)
                        f.write('\n')
                        break
                for result in results_list:
                    if result['dt_test_result'] == "FAIL":
                        f.write('    %s   passed: %s / failed: %s   dt-runtime-unit-tests: %s\n' % (result['device_type'],
                                                                                                    result['dt_tests_passed'],
                                                                                                    result['dt_tests_failed'],
                                                                                                    result['dt_test_result']))
                        if config.get("lab"):
                            f.write('    http://storage.kernelci.org/kernel-ci/%s/%s/%s/%s/boot-%s.html' % (kernel_tree,
                                                                                                        kernel_version,
                                                                                                        defconfig,
                                                                                                        config.get("lab"),
                                                                                                        result['device_type']))
                        else:
                            f.write('    http://storage.kernelci.org/kernel-ci/%s/%s/%s/boot-%s.html' % (kernel_tree,
                                                                                                         kernel_version,
                                                                                                         defconfig,
                                                                                                         result['device_type']))
            f.write('\n')
            f.write('\n')
            f.write('Full Unit Test Report:\n')
            for defconfig, results_list in results.items():
                first = True
                for result in results_list:
                    if result['dt_test_result']:
                        if first:
                            f.write('\n')
                            f.write(defconfig)
                            f.write('\n')
                            first = False
                        f.write('    %s   passed: %s / failed: %s   dt-runtime-unit-tests: %s\n' % (result['device_type'],
                                                                                                    result['dt_tests_passed'],
                                                                                                    result['dt_tests_failed'],
                                                                                                    result['dt_test_result']))

    # sendmail
    if config.get("email"):
        print 'Sending e-mail summary to %s' % config.get("email")
        if os.path.exists(report_directory):
            cmd = 'cat %s | sendmail -t' % os.path.join(report_directory, boot)
            subprocess.check_output(cmd, shell=True)
        if dt_tests:
            if os.path.exists(report_directory):
                cmd = 'cat %s | sendmail -t' % os.path.join(report_directory, dt_self_test)
                subprocess.check_output(cmd, shell=True)
Esempio n. 16
0
def process(file_path, new_screen_name):
    if not os.path.isfile(file_path):
        sys.exit('File not found')

    data = read_json(file_path)

    new_data = dict()
    new_data['layout'] = data['layout']
    new_data['info'] = {
        'versionCode': data['info']['versionCode'],
        'versionName': data['info']['versionName'],
    }
    new_data['preferences'] = {
        'Display.Theme': 'WhiteHCTheme',
        'EarthModel': 'WGS84',
    }

    if '-por2lan' in sys.argv:
        new_data['layout']['landscape'] = new_data['layout']['portrait']

    if '-lan2por' in sys.argv:
        new_data['layout']['landscape'] = new_data['layout']['portrait']

    # nocomp and onlycomp split
    for orient in ['portrait', 'landscape']:
        if orient not in new_data['layout']:
            continue

        if '-nocomp' in sys.argv:
            new_data['layout'][orient] = [
                i for i in new_data['layout'][orient]
                if 'org.xcontest.XCTrack.navig.TaskCompetition' not in
                i['navigations']
            ]

        if '-onlycomp' in sys.argv:
            new_data['layout'][orient] = [
                i for i in new_data['layout'][orient]
                if 'org.xcontest.XCTrack.navig.TaskCompetition' in
                i['navigations']
            ]

    # portrait and landscape modes
    if '-por' in sys.argv:
        if 'landscape' in new_data['layout']:
            del new_data['layout']['landscape']
        new_data['preferences']['Display.Orientation'] = 'PORTRAIT'

    elif '-lan' in sys.argv:
        if 'portrait' in new_data['layout']:
            del new_data['layout']['portrait']
        new_data['preferences']['Display.Orientation'] = 'LANDSCAPE'

    else:
        sys.exit('Please specify either -por (portrait) or -lan (landscape)')

    sort_widgets_by_name(new_data['layout'])

    ensure_dir('screens')
    target_file = os.path.join('screens', new_screen_name + '.xcfg')
    write_json(target_file, new_data)