def create_models(num_networks, hyperparameter_ranges_file):
identifier = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
data_is_target_list = [0]
num_scat_list = [1, 2, 3]
batch_size_list = [32]
# data_noise_gaussian_list = [0, 1]
data_noise_gaussian_list = [1] # Decided on 11/22/2018 b/c better models
#dropout_input_list = [0, 0.1, 0.2]
#dropout_list = [0, 0.1, 0.2, 0.3, 0.4, 0.5]
weight_decay_list = [0]
for count in range(num_networks):
data_is_target = choice(data_is_target_list)
n_scat = choice(num_scat_list)
bs = choice(batch_size_list)
data_noise_gaussian = choice(data_noise_gaussian_list)
#dropout_input = choice(dropout_input_list)
weight_decay = choice(weight_decay_list)
# get params
model_params = choose_hyperparameters_from_file(
hyperparameter_ranges_file)
# set other params
model_params['data_is_target'] = data_is_target
home = os.path.expanduser('~')
model_params['data_train'] = os.path.join(
home, 'Downloads', '20180402_L74_70mm',
'train_' + str(n_scat) + '.h5')
model_params['data_val'] = os.path.join(home, 'Downloads',
'20180402_L74_70mm',
'val_' + str(n_scat) + '.h5')
model_params['batch_size'] = bs
model_params['data_noise_gaussian'] = data_noise_gaussian
#model_params['dropout_input'] = dropout_input
model_params['weight_decay'] = weight_decay
model_params['patience'] = 20
model_params['cuda'] = 1
model_params['save_initial'] = 0
k_list = [3, 4, 5]
for k in k_list:
model_params['k'] = k
model_params['save_dir'] = os.path.join(
'DNNs', identifier + '_' + str(count + 1) + '_created',
'k_' + str(k))
# print(model_params['save_dir'])
ensure_dir(model_params['save_dir'])
save_model_params(
os.path.join(model_params['save_dir'], model_params_fname),
model_params)
print('create_models: created model {}_{}'.format(identifier, count))
return identifier
def get_user_dir(self):
"""
Get a data folder for the current user.
Creates folder if it does not exist.
"""
user_dir = '%s/icfhome/%s' % (os.path.dirname(__file__),
self.current_user)
ensure_dir(user_dir)
return user_dir
def get_and_save_model_dict(kernel_width, num_kernels, num_layers, index):
model_dict = get_model_dict(kernel_width, num_kernels, num_layers)
for k in [3, 4, 5]:
model_dict['k'] = k
identifier = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
model_dict['save_dir'] = os_path_join(
'DNNs', 'fcnn_v1.6.7_{}_{}_created'.format(identifier, index),
'k_{}'.format(k))
# print(model_params['save_dir'])
ensure_dir(model_dict['save_dir'])
save_model_params(
os_path_join(model_dict['save_dir'], MODEL_PARAMS_FNAME),
model_dict)
print('created ', model_dict['save_dir'])
def create_models(num_networks, hyperparameter_ranges_file):
identifier = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
for count in range(num_networks):
model_params = choose_hyperparameters_from_file(hyperparameter_ranges_file)
model_params['train_features'] = os.path.join('data', 'train-images-idx3-ubyte.gz')
model_params['train_labels'] = os.path.join('data', 'train-labels-idx3-ubyte.gz')
model_params['valid_features'] = os.path.join('data', 't10k-images-idx3-ubyte.gz')
model_params['valid_labels'] = os.path.join('data', 't10k-labels-idx3-ubyte.gz')
model_params['save_dir'] = os.path.join(trained_models_folder, identifier + '_' + str(count+1) + '_created')
ensure_dir(model_params['save_dir'])
save_model_params(os.path.join(model_params['save_dir'], 'model_params.txt'), model_params)
return identifier
def boot_report(config):
connection, jobs, duration = parse_json(config.get("boot"))
# TODO: Fix this when multi-lab sync is working
#download_log2html(log2html)
results_directory = os.getcwd() + '/results'
results = {}
utils.mkdir(results_directory)
test_plan = None
for job_id in jobs:
print 'Job ID: %s' % job_id
# Init
boot_meta = {}
api_url = None
arch = None
board_instance = None
boot_retries = 0
kernel_defconfig_full = None
kernel_defconfig = None
kernel_defconfig_base = None
kernel_version = None
device_tree = None
kernel_endian = None
kernel_tree = None
kernel_addr = None
initrd_addr = None
dtb_addr = None
dtb_append = None
fastboot = None
fastboot_cmd = None
job_file = ''
board_offline = False
kernel_boot_time = None
boot_failure_reason = None
efi_rtc = False
# Retrieve job details
device_type = ''
job_details = connection.scheduler.job_details(job_id)
if job_details['requested_device_type_id']:
device_type = job_details['requested_device_type_id']
if job_details['description']:
job_name = job_details['description']
try:
job_short_name = re.search(".*?([A-Z]+.*)", job_name).group(1)
except Exception:
job_short_name = 'boot-test'
device_name = job_details['_actual_device_cache']['hostname']
result = jobs[job_id]['result']
bundle = jobs[job_id]['bundle']
if not device_type:
device_type = job_details['_actual_device_cache']['device_type_id']
if bundle is None and device_type == 'dynamic-vm':
host_job_id = job_id.replace('.1', '.0')
bundle = jobs[host_job_id]['bundle']
if bundle is None:
print '%s bundle is empty, skipping...' % device_type
continue
# Retrieve the log file
try:
binary_job_file = connection.scheduler.job_output(job_id)
except xmlrpclib.Fault:
print 'Job output not found for %s' % device_type
continue
# Parse LAVA messages out of log
raw_job_file = str(binary_job_file)
for line in raw_job_file.splitlines():
if 'Infrastructure Error:' in line:
print 'Infrastructure Error detected!'
index = line.find('Infrastructure Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Bootloader Error:' in line:
print 'Bootloader Error detected!'
index = line.find('Bootloader Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Kernel Error:' in line:
print 'Kernel Error detected!'
index = line.find('Kernel Error:')
boot_failure_reason = line[index:]
if 'Userspace Error:' in line:
print 'Userspace Error detected!'
index = line.find('Userspace Error:')
boot_failure_reason = line[index:]
if '<LAVA_DISPATCHER>' not in line:
if len(line) != 0:
job_file += line + '\n'
if 'rtc-efi rtc-efi: setting system clock to' in line:
if device_type == 'dynamic-vm':
efi_rtc = True
# Retrieve bundle
if bundle is not None:
json_bundle = connection.dashboard.get(bundle)
bundle_data = json.loads(json_bundle['content'])
# Get the boot data from LAVA
for test_results in bundle_data['test_runs']:
# Check for the LAVA self boot test
if test_results['test_id'] == 'lava':
for test in test_results['test_results']:
# TODO for compat :(
if test['test_case_id'] == 'kernel_boot_time':
kernel_boot_time = test['measurement']
if test['test_case_id'] == 'test_kernel_boot_time':
kernel_boot_time = test['measurement']
bundle_attributes = bundle_data['test_runs'][-1][
'attributes']
if utils.in_bundle_attributes(bundle_attributes,
'kernel.defconfig'):
print bundle_attributes['kernel.defconfig']
if utils.in_bundle_attributes(bundle_attributes, 'target'):
board_instance = bundle_attributes['target']
if utils.in_bundle_attributes(bundle_attributes,
'kernel.defconfig'):
kernel_defconfig = bundle_attributes['kernel.defconfig']
defconfig_list = kernel_defconfig.split('-')
#arch = defconfig_list[0]
arch = defconfig_list[-1]
# Remove arch
defconfig_list.pop(0)
kernel_defconfig_full = '-'.join(defconfig_list)
kernel_defconfig_base = ''.join(
kernel_defconfig_full.split('+')[:1])
if kernel_defconfig_full == kernel_defconfig_base:
kernel_defconfig_full = None
if utils.in_bundle_attributes(bundle_attributes, 'kernel.version'):
kernel_version = bundle_attributes['kernel.version']
if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
device_tree = bundle_attributes['device.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
kernel_endian = bundle_attributes['kernel.endian']
if utils.in_bundle_attributes(bundle_attributes,
'platform.fastboot'):
fastboot = bundle_attributes['platform.fastboot']
if kernel_boot_time is None:
if utils.in_bundle_attributes(bundle_attributes,
'kernel-boot-time'):
kernel_boot_time = bundle_attributes['kernel-boot-time']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
kernel_tree = bundle_attributes['kernel.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
kernel_addr = bundle_attributes['kernel-addr']
if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
initrd_addr = bundle_attributes['initrd-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
dtb_addr = bundle_attributes['dtb-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
dtb_append = bundle_attributes['dtb-append']
if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
boot_retries = int(bundle_attributes['boot_retries'])
if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
test_tmp = bundle_attributes['test.plan']
if test_tmp:
test_plan = test_tmp
else:
if not kernel_defconfig or not kernel_version or not kernel_tree:
job_defnition = {}
if 'original_definition' in job_details.keys():
job_definition = job_details['original_definition']
try:
job_dictionary = eval(job_definition)
except Exception:
pass
if job_dictionary:
if 'actions' in job_dictionary.keys():
actions = job_dictionary['actions']
for i in range(0, len(actions)):
try:
kernel_defconfig = actions[i]['metadata'][
'kernel.defconfig']
kernel_version = actions[i]['metadata'][
'kernel.version']
kernel_tree = actions[i]['metadata'][
'kernel.tree']
kernel_endian = actions[i]['metadata'][
'kernel.endian']
platform_fastboot = actions[i]['metadata'][
'platform.fastboot']
device_tree = actions[i]['metadata'][
'kernel.tree']
break
except KeyError:
continue
if 'target' in job_details.keys():
print job_details.keys()
# Check if we found efi-rtc
if test_plan == 'boot-kvm-uefi' and not efi_rtc:
if device_type == 'dynamic-vm':
boot_failure_reason = 'Unable to read EFI rtc'
result = 'FAIL'
# Record the boot log and result
# TODO: Will need to map device_types to dashboard device types
if kernel_defconfig and device_type and result:
if ('arm' == arch or 'arm64' == arch) and device_tree is None:
platform_name = device_map[device_type][0] + ',legacy'
else:
if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
platform_name = device_map[device_type][0] + '_rootfs:nfs'
else:
platform_name = device_map[device_type][0]
print 'Creating boot log for %s' % (platform_name + job_name +
'_' + job_id)
log = 'boot-%s.txt' % (platform_name + job_name + '_' + job_id)
html = 'boot-%s.html' % (platform_name + job_name + '_' + job_id)
if config.get("lab"):
directory = os.path.join(
results_directory,
kernel_defconfig + '/' + config.get("lab"))
else:
directory = os.path.join(results_directory, kernel_defconfig)
utils.ensure_dir(directory)
utils.write_file(job_file, log, directory)
if kernel_boot_time is None:
kernel_boot_time = '0.0'
if results.has_key(kernel_defconfig):
results[kernel_defconfig].append({
'device_type': platform_name,
'job_id': job_id,
'job_name': job_short_name,
'kernel_boot_time': kernel_boot_time,
'result': result,
'device_name': device_name
})
else:
results[kernel_defconfig] = [{
'device_type': platform_name,
'job_id': job_id,
'job_name': job_short_name,
'kernel_boot_time': kernel_boot_time,
'result': result,
'device_name': device_name
}]
# Create JSON format boot metadata
print 'Creating JSON format boot metadata'
if config.get("lab"):
boot_meta['lab_name'] = config.get("lab")
else:
boot_meta['lab_name'] = None
if board_instance:
boot_meta['board_instance'] = board_instance
boot_meta['retries'] = boot_retries
boot_meta['boot_log'] = log
boot_meta['boot_log_html'] = html
# TODO: Fix this
boot_meta['version'] = '1.0'
boot_meta['arch'] = arch
boot_meta['defconfig'] = kernel_defconfig_base
if kernel_defconfig_full is not None:
boot_meta['defconfig_full'] = kernel_defconfig_full
if device_map[device_type][1]:
boot_meta['mach'] = device_map[device_type][1]
boot_meta['kernel'] = kernel_version
boot_meta['job'] = kernel_tree
boot_meta['board'] = platform_name
if board_offline and result == 'FAIL':
boot_meta['boot_result'] = 'OFFLINE'
#results[kernel_defconfig]['result'] = 'OFFLINE'
else:
boot_meta['boot_result'] = result
if result == 'FAIL' or result == 'OFFLINE':
if boot_failure_reason:
boot_meta['boot_result_description'] = boot_failure_reason
else:
boot_meta[
'boot_result_description'] = 'Unknown Error: platform failed to boot'
boot_meta['boot_time'] = kernel_boot_time
# TODO: Fix this
boot_meta['boot_warnings'] = None
if device_tree:
if arch == 'arm64':
boot_meta['dtb'] = 'dtbs/' + device_map[device_type][
1] + '/' + device_tree
else:
boot_meta['dtb'] = 'dtbs/' + device_tree
else:
boot_meta['dtb'] = device_tree
boot_meta['dtb_addr'] = dtb_addr
boot_meta['dtb_append'] = dtb_append
boot_meta['endian'] = kernel_endian
boot_meta['fastboot'] = fastboot
# TODO: Fix this
boot_meta['initrd'] = None
boot_meta['initrd_addr'] = initrd_addr
if arch == 'arm':
boot_meta['kernel_image'] = 'zImage'
elif arch == 'arm64':
boot_meta['kernel_image'] = 'Image'
else:
boot_meta['kernel_image'] = 'bzImage'
boot_meta['loadaddr'] = kernel_addr
json_file = 'boot-%s.json' % (platform_name + job_name + '_' +
job_id)
utils.write_json(json_file, directory, boot_meta)
if config.get("lab"):
report_directory = os.path.join(results_directory, config.get("lab"))
utils.mkdir(report_directory)
else:
report_directory = results_directory
if results and kernel_tree and kernel_version:
print 'Creating summary for %s' % (kernel_version)
boot = '%s-boot-report.txt' % (kernel_version)
if test_plan and ('boot' in test_plan or 'BOOT' in test_plan):
boot = boot.replace('boot', test_plan)
passed = 0
failed = 0
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'PASS':
passed += 1
else:
failed += 1
total = passed + failed
with open(os.path.join(report_directory, boot), 'a') as f:
f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' %
(kernel_tree, str(total), str(passed), str(failed),
kernel_version))
f.write('\n')
f.write('Total Duration: %.2f minutes\n' % (duration / 60))
f.write('Tree/Branch: %s\n' % kernel_tree)
f.write('Git Describe: %s\n' % kernel_version)
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'OFFLINE':
if first:
f.write('\n')
f.write('Boards Offline:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'OFFLINE':
f.write(
' %s %s %s %ss %s: %s\n' %
(result['job_id'], result['device_type'],
result['device_name'], result['kernel_boot_time'],
result['job_name'], result['result']))
f.write('\n')
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'FAIL':
if first:
f.write('\n')
f.write('Failed Boot Tests:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'FAIL':
f.write(
' %s %s %s %ss %s: %s\n' %
(result['job_id'], result['device_type'],
result['device_name'], result['kernel_boot_time'],
result['job_name'], result['result']))
f.write('\n')
f.write('Full Boot Report:\n')
for defconfig, results_list in results.items():
f.write('\n')
f.write(defconfig)
f.write('\n')
for result in results_list:
f.write(' %s %s %s %ss %s: %s\n' %
(result['job_id'], result['device_type'],
result['device_name'], result['kernel_boot_time'],
result['job_name'], result['result']))
# add by wuyanjun
if results and directory:
parser_and_get_result(results, directory, report_directory)
get_ip_board_mapping(results, directory, report_directory)
def boot_report(config):
connection, jobs, duration = parse_json(config.get("boot"))
# TODO: Fix this when multi-lab sync is working
#download_log2html(log2html)
results_directory = os.getcwd() + '/results'
results = {}
utils.mkdir(results_directory)
test_plan = None
if config.get("lab"):
report_directory = os.path.join(results_directory, config.get("lab"))
else:
report_directory = results_directory
if os.path.exists(report_directory):
shutil.rmtree(report_directory)
utils.mkdir(report_directory)
for job_id in jobs:
print 'Job ID: %s' % job_id
# Init
boot_meta = {}
api_url = None
arch = None
board_instance = None
boot_retries = 0
kernel_defconfig_full = None
kernel_defconfig = None
kernel_defconfig_base = None
kernel_version = None
device_tree = None
kernel_endian = None
kernel_tree = None
kernel_addr = None
initrd_addr = None
dtb_addr = None
dtb_append = None
fastboot = None
fastboot_cmd = None
job_file = ''
board_offline = False
kernel_boot_time = None
boot_failure_reason = None
efi_rtc = False
# Retrieve job details
device_type = ''
job_details = connection.scheduler.job_details(job_id)
if job_details['requested_device_type_id']:
device_type = job_details['requested_device_type_id']
if job_details['description']:
job_name = job_details['description']
try:
job_short_name = re.search(".*?([A-Z]+.*)", job_name).group(1)
except Exception:
job_short_name = 'boot-test'
try:
device_name = job_details['_actual_device_cache']['hostname']
except Exception:
continue
result = jobs[job_id]['result']
bundle = jobs[job_id]['bundle']
if not device_type:
device_type = job_details['_actual_device_cache']['device_type_id']
if bundle is None and device_type == 'dynamic-vm':
host_job_id = job_id.replace('.1', '.0')
bundle = jobs[host_job_id]['bundle']
if bundle is None:
print '%s bundle is empty, skipping...' % device_type
continue
# Retrieve the log file
try:
binary_job_file = connection.scheduler.job_output(job_id)
except xmlrpclib.Fault:
print 'Job output not found for %s' % device_type
continue
# Parse LAVA messages out of log
raw_job_file = str(binary_job_file)
for line in raw_job_file.splitlines():
if 'Infrastructure Error:' in line:
print 'Infrastructure Error detected!'
index = line.find('Infrastructure Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Bootloader Error:' in line:
print 'Bootloader Error detected!'
index = line.find('Bootloader Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Kernel Error:' in line:
print 'Kernel Error detected!'
index = line.find('Kernel Error:')
boot_failure_reason = line[index:]
if 'Userspace Error:' in line:
print 'Userspace Error detected!'
index = line.find('Userspace Error:')
boot_failure_reason = line[index:]
if '<LAVA_DISPATCHER>' not in line:
if len(line) != 0:
job_file += line + '\n'
if 'rtc-efi rtc-efi: setting system clock to' in line:
if device_type == 'dynamic-vm':
efi_rtc = True
# Retrieve bundle
if bundle is not None:
json_bundle = connection.dashboard.get(bundle)
bundle_data = json.loads(json_bundle['content'])
# Get the boot data from LAVA
for test_results in bundle_data['test_runs']:
# Check for the LAVA self boot test
if test_results['test_id'] == 'lava':
for test in test_results['test_results']:
# TODO for compat :(
if test['test_case_id'] == 'kernel_boot_time':
kernel_boot_time = test['measurement']
if test['test_case_id'] == 'test_kernel_boot_time':
kernel_boot_time = test['measurement']
bundle_attributes = bundle_data['test_runs'][-1]['attributes']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
print bundle_attributes['kernel.defconfig']
if utils.in_bundle_attributes(bundle_attributes, 'target'):
board_instance = bundle_attributes['target']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
kernel_defconfig = bundle_attributes['kernel.defconfig']
defconfig_list = kernel_defconfig.split('-')
#arch = defconfig_list[0]
arch = defconfig_list[-1]
# Remove arch
defconfig_list.pop(0)
kernel_defconfig_full = '-'.join(defconfig_list)
kernel_defconfig_base = ''.join(kernel_defconfig_full.split('+')[:1])
if kernel_defconfig_full == kernel_defconfig_base:
kernel_defconfig_full = None
if utils.in_bundle_attributes(bundle_attributes, 'kernel.version'):
kernel_version = bundle_attributes['kernel.version']
if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
device_tree = bundle_attributes['device.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
kernel_endian = bundle_attributes['kernel.endian']
if utils.in_bundle_attributes(bundle_attributes, 'platform.fastboot'):
fastboot = bundle_attributes['platform.fastboot']
if kernel_boot_time is None:
if utils.in_bundle_attributes(bundle_attributes, 'kernel-boot-time'):
kernel_boot_time = bundle_attributes['kernel-boot-time']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
kernel_tree = bundle_attributes['kernel.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
kernel_addr = bundle_attributes['kernel-addr']
if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
initrd_addr = bundle_attributes['initrd-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
dtb_addr = bundle_attributes['dtb-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
dtb_append = bundle_attributes['dtb-append']
if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
boot_retries = int(bundle_attributes['boot_retries'])
if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
test_tmp = bundle_attributes['test.plan']
if test_tmp:
test_plan = test_tmp
else:
if not kernel_defconfig or not kernel_version or not kernel_tree:
job_defnition = {}
if 'original_definition' in job_details.keys():
job_definition = job_details['original_definition']
try:
job_dictionary = eval(job_definition)
except Exception:
pass
if job_dictionary:
if 'actions' in job_dictionary.keys():
actions = job_dictionary['actions']
for i in range(0, len(actions)):
try:
kernel_defconfig = actions[i]['metadata']['kernel.defconfig']
kernel_version = actions[i]['metadata']['kernel.version']
kernel_tree = actions[i]['metadata']['kernel.tree']
kernel_endian = actions[i]['metadata']['kernel.endian']
platform_fastboot = actions[i]['metadata']['platform.fastboot']
device_tree = actions[i]['metadata']['kernel.tree']
break
except KeyError:
continue
if 'target' in job_details.keys():
print job_details.keys()
# Check if we found efi-rtc
if test_plan == 'boot-kvm-uefi' and not efi_rtc:
if device_type == 'dynamic-vm':
boot_failure_reason = 'Unable to read EFI rtc'
result = 'FAIL'
# Record the boot log and result
# TODO: Will need to map device_types to dashboard device types
if kernel_defconfig and device_type and result:
if ( 'arm' == arch or 'arm64' == arch ) and device_tree is None:
platform_name = device_map[device_type][0] + ',legacy'
else:
if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
platform_name = device_map[device_type][0] + '_rootfs:nfs'
else:
platform_name = device_map[device_type][0]
# Create txt format boot metadata
print 'Creating boot log for %s' % (platform_name + job_name + '_' + job_id)
log = 'boot-%s.txt' % (platform_name + job_name + '_' + job_id)
html = 'boot-%s.html' % (platform_name + job_name + '_' + job_id)
if config.get("lab"):
directory = os.path.join(results_directory, kernel_defconfig + '/' + config.get("lab"))
else:
directory = os.path.join(results_directory, kernel_defconfig)
utils.ensure_dir(directory)
utils.write_file(job_file, log, directory)
if kernel_boot_time is None:
kernel_boot_time = '0.0'
if results.has_key(kernel_defconfig):
results[kernel_defconfig].append({'device_type': platform_name,
'job_id': job_id, 'job_name': job_short_name,
'kernel_boot_time': kernel_boot_time, 'result': result,
'device_name': device_name})
else:
results[kernel_defconfig] = [{'device_type': platform_name,
'job_id': job_id, 'job_name': job_short_name,
'kernel_boot_time': kernel_boot_time, 'result': result,
'device_name': device_name}]
# Create JSON format boot metadata
print 'Creating JSON format boot metadata'
if config.get("lab"):
boot_meta['lab_name'] = config.get("lab")
else:
boot_meta['lab_name'] = None
if board_instance:
boot_meta['board_instance'] = board_instance
boot_meta['retries'] = boot_retries
boot_meta['boot_log'] = log
boot_meta['boot_log_html'] = html
# TODO: Fix this
boot_meta['version'] = '1.0'
boot_meta['arch'] = arch
boot_meta['defconfig'] = kernel_defconfig_base
if kernel_defconfig_full is not None:
boot_meta['defconfig_full'] = kernel_defconfig_full
if device_map[device_type][1]:
boot_meta['mach'] = device_map[device_type][1]
boot_meta['kernel'] = kernel_version
boot_meta['job'] = kernel_tree
boot_meta['board'] = platform_name
if board_offline and result == 'FAIL':
boot_meta['boot_result'] = 'OFFLINE'
#results[kernel_defconfig]['result'] = 'OFFLINE'
else:
boot_meta['boot_result'] = result
if result == 'FAIL' or result == 'OFFLINE':
if boot_failure_reason:
boot_meta['boot_result_description'] = boot_failure_reason
else:
boot_meta['boot_result_description'] = 'Unknown Error: platform failed to boot'
boot_meta['boot_time'] = kernel_boot_time
# TODO: Fix this
boot_meta['boot_warnings'] = None
if device_tree:
if arch == 'arm64':
boot_meta['dtb'] = 'dtbs/' + device_map[device_type][1] + '/' + device_tree
else:
boot_meta['dtb'] = 'dtbs/' + device_tree
else:
boot_meta['dtb'] = device_tree
boot_meta['dtb_addr'] = dtb_addr
boot_meta['dtb_append'] = dtb_append
boot_meta['endian'] = kernel_endian
boot_meta['fastboot'] = fastboot
# TODO: Fix this
boot_meta['initrd'] = None
boot_meta['initrd_addr'] = initrd_addr
if arch == 'arm':
boot_meta['kernel_image'] = 'zImage'
elif arch == 'arm64':
boot_meta['kernel_image'] = 'Image'
else:
boot_meta['kernel_image'] = 'bzImage'
boot_meta['loadaddr'] = kernel_addr
json_file = 'boot-%s.json' % (platform_name + job_name + '_' + job_id)
utils.write_json(json_file, directory, boot_meta)
# add by wuyanjun
# add the ip device mapping
get_ip_board_mapping(job_file, log, directory, report_directory)
parser_and_get_result(job_file, log, directory, report_directory, connection)
if results and kernel_tree and kernel_version:
print 'Creating summary for %s' % (kernel_version)
boot = '%s-boot-report.txt' % (kernel_version)
if test_plan and ('boot' in test_plan or 'BOOT' in test_plan):
boot = boot.replace('boot', test_plan)
passed = 0
failed = 0
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'PASS':
passed += 1
else:
failed += 1
total = passed + failed
with open(os.path.join(report_directory, boot), 'a') as f:
f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' % (kernel_tree,
str(total),
str(passed),
str(failed),
kernel_version))
f.write('\n')
f.write('Total Duration: %.2f minutes\n' % (duration / 60))
f.write('Tree/Branch: %s\n' % kernel_tree)
f.write('Git Describe: %s\n' % kernel_version)
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'OFFLINE':
if first:
f.write('\n')
f.write('Boards Offline:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'OFFLINE':
f.write(' %s %s %s %ss %s: %s\n' % (result['job_id'],
result['device_type'],
result['device_name'],
result['kernel_boot_time'],
result['job_name'],
result['result']))
f.write('\n')
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'FAIL':
if first:
f.write('\n')
f.write('Failed Boot Tests:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'FAIL':
f.write(' %s %s %s %ss %s: %s\n' % (result['job_id'],
result['device_type'],
result['device_name'],
result['kernel_boot_time'],
result['job_name'],
result['result']))
f.write('\n')
f.write('Full Boot Report:\n')
for defconfig, results_list in results.items():
f.write('\n')
f.write(defconfig)
f.write('\n')
for result in results_list:
f.write(' %s %s %s %ss %s: %s\n' %
(result['job_id'],
result['device_type'],
result['device_name'],
result['kernel_boot_time'],
result['job_name'],
result['result']))
def boot_report(config):
connection, jobs, duration = parse_json(config.get("boot"))
# TODO: Fix this when multi-lab sync is working
#download_log2html(log2html)
results_directory = os.getcwd() + '/results'
results = {}
dt_tests = False
utils.mkdir(results_directory)
for job_id in jobs:
print 'Job ID: %s' % job_id
# Init
boot_meta = {}
api_url = None
arch = None
board_instance = None
boot_retries = 0
kernel_defconfig = None
kernel_defconfig_base = None
kernel_version = None
device_tree = None
kernel_endian = None
kernel_tree = None
git_branch = None
kernel_addr = None
initrd_addr = None
dtb_addr = None
dtb_append = None
fastboot = None
fastboot_cmd = None
test_plan = None
job_file = ''
dt_test = None
dt_test_result = None
dt_tests_passed = None
dt_tests_failed = None
board_offline = False
kernel_boot_time = None
boot_failure_reason = None
efi_rtc = False
# Retrieve job details
job_details = connection.scheduler.job_details(job_id)
if job_details['requested_device_type_id']:
device_type = job_details['requested_device_type_id']
if job_details['description']:
job_name = job_details['description']
result = jobs[job_id]['result']
bundle = jobs[job_id]['bundle']
if bundle is None and device_type == 'dynamic-vm':
host_job_id = job_id.replace('.1', '.0')
bundle = jobs[host_job_id]['bundle']
if bundle is None:
print '%s bundle is empty, skipping...' % device_type
continue
# Retrieve the log file
try:
binary_job_file = connection.scheduler.job_output(job_id)
except xmlrpclib.Fault:
print 'Job output not found for %s' % device_type
continue
# Parse LAVA messages out of log
raw_job_file = str(binary_job_file)
for line in raw_job_file.splitlines():
if 'Infrastructure Error:' in line:
print 'Infrastructure Error detected!'
index = line.find('Infrastructure Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Bootloader Error:' in line:
print 'Bootloader Error detected!'
index = line.find('Bootloader Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Kernel Error:' in line:
print 'Kernel Error detected!'
index = line.find('Kernel Error:')
boot_failure_reason = line[index:]
if 'Userspace Error:' in line:
print 'Userspace Error detected!'
index = line.find('Userspace Error:')
boot_failure_reason = line[index:]
if '<LAVA_DISPATCHER>' not in line:
if len(line) != 0:
job_file += line + '\n'
if '### dt-test ### end of selftest' in line:
dt_tests = True
regex = re.compile("(?P<test>\d+\*?)")
dt_test_results = regex.findall(line)
if len(dt_test_results) > 2:
dt_tests_passed = dt_test_results[2]
dt_tests_failed = dt_test_results[3]
else:
dt_tests_passed = dt_test_results[0]
dt_tests_failed = dt_test_results[1]
if int(dt_tests_failed) > 0:
dt_test_result = 'FAIL'
else:
dt_test_result = 'PASS'
if 'rtc-efi rtc-efi: setting system clock to' in line:
if device_type == 'dynamic-vm':
efi_rtc = True
# Retrieve bundle
if bundle is not None:
json_bundle = connection.dashboard.get(bundle)
bundle_data = json.loads(json_bundle['content'])
# Get the boot data from LAVA
for test_results in bundle_data['test_runs']:
# Check for the LAVA self boot test
if test_results['test_id'] == 'lava':
for test in test_results['test_results']:
# TODO for compat :(
if test['test_case_id'] == 'kernel_boot_time':
kernel_boot_time = test['measurement']
if test['test_case_id'] == 'test_kernel_boot_time':
kernel_boot_time = test['measurement']
bundle_attributes = bundle_data['test_runs'][-1][
'attributes']
if utils.in_bundle_attributes(bundle_attributes,
'kernel.defconfig'):
print bundle_attributes['kernel.defconfig']
if utils.in_bundle_attributes(bundle_attributes, 'target'):
board_instance = bundle_attributes['target']
if utils.in_bundle_attributes(bundle_attributes,
'kernel.defconfig'):
kernel_defconfig = bundle_attributes['kernel.defconfig']
kernel_defconfig_base = ''.join(
kernel_defconfig.split('+')[:1])
if utils.in_bundle_attributes(bundle_attributes, 'arch'):
arch = bundle_attributes['arch']
if utils.in_bundle_attributes(bundle_attributes,
'kernel.describe'):
kernel_version = bundle_attributes['kernel.describe']
if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
device_tree = bundle_attributes['device.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
kernel_endian = bundle_attributes['kernel.endian']
if utils.in_bundle_attributes(bundle_attributes,
'platform.fastboot'):
fastboot = bundle_attributes['platform.fastboot']
if kernel_boot_time is None:
if utils.in_bundle_attributes(bundle_attributes,
'kernel-boot-time'):
kernel_boot_time = bundle_attributes['kernel-boot-time']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
kernel_tree = bundle_attributes['kernel.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
kernel_addr = bundle_attributes['kernel-addr']
if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
initrd_addr = bundle_attributes['initrd-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
dtb_addr = bundle_attributes['dtb-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
dtb_append = bundle_attributes['dtb-append']
if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
boot_retries = int(bundle_attributes['boot_retries'])
if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
test_plan = bundle_attributes['test.plan']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.branch'):
git_branch = bundle_attributes['kernel.branch']
# Check if we found efi-rtc
if test_plan == 'boot-kvm-uefi' and not efi_rtc:
if device_type == 'dynamic-vm':
boot_failure_reason = 'Unable to read EFI rtc'
result = 'FAIL'
# Record the boot log and result
# TODO: Will need to map device_types to dashboard device types
if kernel_defconfig and device_type and result:
if (arch == 'arm' or arch == 'arm64') and device_tree is None:
platform_name = device_map[device_type][0] + ',legacy'
else:
if device_tree == 'vexpress-v2p-ca15_a7.dtb':
platform_name = 'vexpress-v2p-ca15_a7'
elif device_tree == 'fsl-ls2080a-simu.dtb':
platform_name = 'fsl-ls2080a-simu'
elif test_plan == 'boot-kvm' or test_plan == 'boot-kvm-uefi':
if device_tree == 'sun7i-a20-cubietruck.dtb':
if device_type == 'dynamic-vm':
device_type = 'cubieboard3-kvm-guest'
platform_name = device_map[device_type][0]
else:
device_type = 'cubieboard3-kvm-host'
platform_name = device_map[device_type][0]
elif device_tree == 'apm-mustang.dtb':
if device_type == 'dynamic-vm':
if test_plan == 'boot-kvm-uefi':
device_type = 'mustang-kvm-uefi-guest'
else:
device_type = 'mustang-kvm-guest'
platform_name = device_map[device_type][0]
else:
if test_plan == 'boot-kvm-uefi':
device_type = 'mustang-kvm-uefi-host'
else:
device_type = 'mustang-kvm-host'
platform_name = device_map[device_type][0]
elif device_tree == 'juno.dtb':
if device_type == 'dynamic-vm':
if test_plan == 'boot-kvm-uefi':
device_type = 'juno-kvm-uefi-guest'
else:
device_type = 'juno-kvm-guest'
platform_name = device_map[device_type][0]
else:
if test_plan == 'boot-kvm-uefi':
device_type = 'juno-kvm-uefi-host'
else:
device_type = 'juno-kvm-host'
platform_name = device_map[device_type][0]
elif test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
platform_name = device_map[device_type][0] + '_rootfs:nfs'
else:
platform_name = device_map[device_type][0]
print 'Creating boot log for %s' % platform_name
log = 'boot-%s.txt' % platform_name
html = 'boot-%s.html' % platform_name
if config.get("lab"):
directory = os.path.join(
results_directory,
kernel_defconfig + '/' + config.get("lab"))
else:
directory = os.path.join(results_directory, kernel_defconfig)
utils.ensure_dir(directory)
utils.write_file(job_file, log, directory)
if kernel_boot_time is None:
kernel_boot_time = '0.0'
if results.has_key(kernel_defconfig):
results[kernel_defconfig].append({
'device_type': platform_name,
'dt_test_result': dt_test_result,
'dt_tests_passed': dt_tests_passed,
'dt_tests_failed': dt_tests_failed,
'kernel_boot_time': kernel_boot_time,
'result': result
})
else:
results[kernel_defconfig] = [{
'device_type': platform_name,
'dt_test_result': dt_test_result,
'dt_tests_passed': dt_tests_passed,
'dt_tests_failed': dt_tests_failed,
'kernel_boot_time': kernel_boot_time,
'result': result
}]
# Create JSON format boot metadata
print 'Creating JSON format boot metadata'
if config.get("lab"):
boot_meta['lab_name'] = config.get("lab")
else:
boot_meta['lab_name'] = None
if board_instance:
boot_meta['board_instance'] = board_instance
boot_meta['retries'] = boot_retries
boot_meta['boot_log'] = log
boot_meta['boot_log_html'] = html
# TODO: Fix this
boot_meta['version'] = '1.1'
boot_meta['arch'] = arch
boot_meta['defconfig'] = kernel_defconfig_base
boot_meta['defconfig_full'] = kernel_defconfig
if device_map[device_type][1]:
boot_meta['mach'] = device_map[device_type][1]
boot_meta['kernel'] = kernel_version
boot_meta['git_branch'] = git_branch
boot_meta['job'] = kernel_tree
boot_meta['board'] = platform_name
if board_offline and result == 'FAIL':
boot_meta['boot_result'] = 'OFFLINE'
#results[kernel_defconfig]['result'] = 'OFFLINE'
else:
boot_meta['boot_result'] = result
if result == 'FAIL' or result == 'OFFLINE':
if boot_failure_reason:
boot_meta['boot_result_description'] = boot_failure_reason
else:
boot_meta[
'boot_result_description'] = 'Unknown Error: platform failed to boot'
boot_meta['boot_time'] = kernel_boot_time
# TODO: Fix this
boot_meta['boot_warnings'] = None
if device_tree:
if arch == 'arm64':
boot_meta['dtb'] = 'dtbs/' + device_map[device_type][
1] + '/' + device_tree
else:
boot_meta['dtb'] = 'dtbs/' + device_tree
else:
boot_meta['dtb'] = device_tree
boot_meta['dtb_addr'] = dtb_addr
boot_meta['dtb_append'] = dtb_append
boot_meta['dt_test'] = dt_test
boot_meta['endian'] = kernel_endian
boot_meta['fastboot'] = fastboot
# TODO: Fix this
boot_meta['initrd'] = None
boot_meta['initrd_addr'] = initrd_addr
if arch == 'arm':
boot_meta['kernel_image'] = 'zImage'
elif arch == 'arm64':
boot_meta['kernel_image'] = 'Image'
else:
boot_meta['kernel_image'] = 'bzImage'
boot_meta['loadaddr'] = kernel_addr
json_file = 'boot-%s.json' % platform_name
utils.write_json(json_file, directory, boot_meta)
print 'Creating html version of boot log for %s' % platform_name
cmd = 'python log2html.py %s' % os.path.join(directory, log)
subprocess.check_output(cmd, shell=True)
if config.get("lab") and config.get("api") and config.get("token"):
print 'Sending boot result to %s for %s' % (config.get("api"),
platform_name)
headers = {
'Authorization': config.get("token"),
'Content-Type': 'application/json'
}
api_url = urlparse.urljoin(config.get("api"), '/boot')
push('POST',
api_url,
data=json.dumps(boot_meta),
headers=headers)
headers = {
'Authorization': config.get("token"),
}
print 'Uploading text version of boot log'
with open(os.path.join(directory, log)) as lh:
data = lh.read()
api_url = urlparse.urljoin(
config.get("api"), '/upload/%s/%s/%s/%s/%s/%s/%s' %
(kernel_tree, git_branch, kernel_version, arch,
kernel_defconfig, config.get("lab"), log))
push('PUT', api_url, data=data, headers=headers)
print 'Uploading html version of boot log'
with open(os.path.join(directory, html)) as lh:
data = lh.read()
api_url = urlparse.urljoin(
config.get("api"), '/upload/%s/%s/%s/%s/%s/%s/%s' %
(kernel_tree, git_branch, kernel_version, arch,
kernel_defconfig, config.get("lab"), html))
push('PUT', api_url, data=data, headers=headers)
def boot_report(config):
connection, jobs, duration = parse_yaml(config.get("boot"))
# TODO: Fix this when multi-lab sync is working
results_directory = os.getcwd() + '/results'
results = {}
utils.mkdir(results_directory)
test_plan = None
if config.get("lab"):
report_directory = os.path.join(results_directory, config.get("lab"))
else:
report_directory = results_directory
if os.path.exists(report_directory):
shutil.rmtree(report_directory)
utils.mkdir(report_directory)
for job_id in jobs:
print 'Job ID: %s' % job_id
# Init
boot_meta = {}
arch = None
board_instance = None
boot_retries = 0
kernel_defconfig_full = None
kernel_defconfig = None
kernel_defconfig_base = None
kernel_version = None
device_tree = None
kernel_tree = None
kernel_addr = None
initrd_addr = None
dtb_addr = None
dtb_append = None
job_file = ''
board_offline = False
kernel_boot_time = None
boot_failure_reason = None
efi_rtc = False
# Retrieve job details
device_type = ''
job_details = connection.scheduler.job_details(job_id)
if job_details['requested_device_type_id']:
device_type = job_details['requested_device_type_id']
if job_details['description']:
job_name = job_details['description']
try:
job_short_name = re.search(".*?([A-Z]+.*)", job_name).group(1)
except Exception:
job_short_name = 'boot-test'
try:
device_name = job_details['_actual_device_cache']['hostname']
except Exception:
continue
result = jobs[job_id]['result']
bundle = jobs[job_id]['bundle']
if not device_type:
device_type = job_details['_actual_device_cache']['device_type_id']
try:
binary_job_file = connection.scheduler.job_output(job_id)
except xmlrpclib.Fault:
print 'Job output not found for %s' % device_type
continue
# Parse LAVA messages out of log
raw_job_file = str(binary_job_file)
for line in raw_job_file.splitlines():
if 'Infrastructure Error:' in line:
print 'Infrastructure Error detected!'
index = line.find('Infrastructure Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Bootloader Error:' in line:
print 'Bootloader Error detected!'
index = line.find('Bootloader Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Kernel Error:' in line:
print 'Kernel Error detected!'
index = line.find('Kernel Error:')
boot_failure_reason = line[index:]
if 'Userspace Error:' in line:
print 'Userspace Error detected!'
index = line.find('Userspace Error:')
boot_failure_reason = line[index:]
if '<LAVA_DISPATCHER>' not in line:
if len(line) != 0:
job_file += line + '\n'
if 'rtc-efi rtc-efi: setting system clock to' in line:
if device_type == 'dynamic-vm':
efi_rtc = True
if not kernel_defconfig or not kernel_version or not kernel_tree:
try:
job_metadata_info = connection.results.get_testjob_metadata(
job_id)
kernel_defconfig = utils.get_value_by_key(
job_metadata_info, 'kernel_defconfig')
kernel_version = utils.get_value_by_key(
job_metadata_info, 'kernel_version')
kernel_tree = utils.get_value_by_key(job_metadata_info,
'kernel_tree')
device_tree = utils.get_value_by_key(job_metadata_info,
'device_tree')
except Exception:
continue
# Record the boot log and result
# TODO: Will need to map device_types to dashboard device types
if kernel_defconfig and device_type and result:
if ('arm' == arch or 'arm64' == arch) and device_tree is None:
platform_name = device_map[device_type][0] + ',legacy'
else:
if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
platform_name = device_map[device_type][0] + '_rootfs:nfs'
else:
platform_name = device_map[device_type][0]
# Create txt format boot metadata
print 'Creating boot log for %s' % (platform_name + job_name +
'_' + job_id)
log = 'boot-%s.txt' % (platform_name + job_name + '_' + job_id)
if config.get("lab"):
directory = os.path.join(
results_directory,
kernel_defconfig + '/' + config.get("lab"))
else:
directory = os.path.join(results_directory, kernel_defconfig)
utils.ensure_dir(directory)
utils.write_file(job_file, log, directory)
if kernel_boot_time is None:
kernel_boot_time = '0.0'
if results.has_key(kernel_defconfig):
results[kernel_defconfig].append({
'device_type': platform_name,
'job_id': job_id,
'job_name': job_short_name,
'kernel_boot_time': kernel_boot_time,
'result': result,
'device_name': device_name
})
else:
results[kernel_defconfig] = [{
'device_type': platform_name,
'job_id': job_id,
'job_name': job_short_name,
'kernel_boot_time': kernel_boot_time,
'result': result,
'device_name': device_name
}]
# Create JSON format boot metadata
print 'Creating JSON format boot metadata'
if config.get("lab"):
boot_meta['lab_name'] = config.get("lab")
else:
boot_meta['lab_name'] = None
if board_instance:
boot_meta['board_instance'] = board_instance
boot_meta['retries'] = boot_retries
boot_meta['boot_log'] = log
# TODO: Fix this
boot_meta['version'] = '1.0'
boot_meta['arch'] = arch
boot_meta['defconfig'] = kernel_defconfig_base
if kernel_defconfig_full is not None:
boot_meta['defconfig_full'] = kernel_defconfig_full
if device_map[device_type][1]:
boot_meta['mach'] = device_map[device_type][1]
boot_meta['kernel'] = kernel_version
boot_meta['job'] = kernel_tree
boot_meta['board'] = platform_name
if board_offline and result == 'FAIL':
boot_meta['boot_result'] = 'OFFLINE'
#results[kernel_defconfig]['result'] = 'OFFLINE'
else:
boot_meta['boot_result'] = result
if result == 'FAIL' or result == 'OFFLINE':
if boot_failure_reason:
boot_meta['boot_result_description'] = boot_failure_reason
else:
boot_meta[
'boot_result_description'] = 'Unknown Error: platform failed to boot'
boot_meta['boot_time'] = kernel_boot_time
# TODO: Fix this
boot_meta['boot_warnings'] = None
if device_tree:
if arch == 'arm64':
boot_meta['dtb'] = 'dtbs/' + device_map[device_type][
1] + '/' + device_tree
else:
boot_meta['dtb'] = 'dtbs/' + device_tree
else:
boot_meta['dtb'] = device_tree
boot_meta['dtb_addr'] = dtb_addr
boot_meta['dtb_append'] = dtb_append
# TODO: Fix this
boot_meta['initrd'] = None
boot_meta['initrd_addr'] = initrd_addr
if arch == 'arm':
boot_meta['kernel_image'] = 'zImage'
elif arch == 'arm64':
boot_meta['kernel_image'] = 'Image'
else:
boot_meta['kernel_image'] = 'bzImage'
boot_meta['loadaddr'] = kernel_addr
json_file = 'boot-%s.json' % (platform_name + job_name + '_' +
job_id)
utils.write_json(json_file, directory, boot_meta)
# add by wuyanjun
parser_and_get_result(job_file, log, directory, report_directory,
connection)
#try to generate test_summary
generate_test_report(job_id, connection)
if results and kernel_tree and kernel_version:
print 'Creating summary for %s' % (kernel_version)
boot = '%s-boot-report.txt' % (kernel_version)
if test_plan and ('boot' in test_plan or 'BOOT' in test_plan):
boot = boot.replace('boot', test_plan)
passed = 0
failed = 0
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'PASS':
passed += 1
else:
failed += 1
total = passed + failed
with open(os.path.join(report_directory, boot), 'a') as f:
f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' %
(kernel_tree, str(total), str(passed), str(failed),
kernel_version))
f.write('\n')
f.write('Total Duration: %.2f minutes\n' % (duration / 60))
f.write('Tree/Branch: %s\n' % kernel_tree)
f.write('Git Describe: %s\n' % kernel_version)
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'OFFLINE':
if first:
f.write('\n')
f.write('Boards Offline:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'OFFLINE':
f.write(
' %s %s %s %ss %s: %s\n' %
(result['job_id'], result['device_type'],
result['device_name'], result['kernel_boot_time'],
result['job_name'], result['result']))
f.write('\n')
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'FAIL':
if first:
f.write('\n')
f.write('Failed Boot Tests:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'FAIL':
f.write(
' %s %s %s %ss %s: %s\n' %
(result['job_id'], result['device_type'],
result['device_name'], result['kernel_boot_time'],
result['job_name'], result['result']))
f.write('\n')
f.write('Full Boot Report:\n')
for defconfig, results_list in results.items():
f.write('\n')
f.write(defconfig)
f.write('\n')
for result in results_list:
f.write(' %s %s %s %ss %s: %s\n' %
(result['job_id'], result['device_type'],
result['device_name'], result['kernel_boot_time'],
result['job_name'], result['result']))
def boot_report(config):
connection, jobs, duration = parse_json(config.get("boot"))
# TODO: Fix this when multi-lab sync is working
#download_log2html(log2html)
results_directory = os.getcwd() + '/results'
results = {}
dt_tests = False
utils.mkdir(results_directory)
for job_id in jobs:
print 'Job ID: %s' % job_id
# Init
boot_meta = {}
api_url = None
arch = None
board_instance = None
boot_retries = 0
kernel_defconfig = None
kernel_defconfig_base = None
kernel_version = None
device_tree = None
kernel_endian = None
kernel_tree = None
git_branch = None
kernel_addr = None
initrd_addr = None
dtb_addr = None
dtb_append = None
fastboot = None
fastboot_cmd = None
test_plan = None
job_file = ''
dt_test = None
dt_test_result = None
dt_tests_passed = None
dt_tests_failed = None
board_offline = False
kernel_boot_time = None
boot_failure_reason = None
efi_rtc = False
# Retrieve job details
job_details = connection.scheduler.job_details(job_id)
if job_details['requested_device_type_id']:
device_type = job_details['requested_device_type_id']
if job_details['description']:
job_name = job_details['description']
result = jobs[job_id]['result']
bundle = jobs[job_id]['bundle']
if bundle is None and device_type == 'dynamic-vm':
host_job_id = job_id.replace('.1', '.0')
bundle = jobs[host_job_id]['bundle']
if bundle is None:
print '%s bundle is empty, skipping...' % device_type
continue
# Retrieve the log file
try:
binary_job_file = connection.scheduler.job_output(job_id)
except xmlrpclib.Fault:
print 'Job output not found for %s' % device_type
continue
# Parse LAVA messages out of log
raw_job_file = str(binary_job_file)
for line in raw_job_file.splitlines():
if 'Infrastructure Error:' in line:
print 'Infrastructure Error detected!'
index = line.find('Infrastructure Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Bootloader Error:' in line:
print 'Bootloader Error detected!'
index = line.find('Bootloader Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Kernel Error:' in line:
print 'Kernel Error detected!'
index = line.find('Kernel Error:')
boot_failure_reason = line[index:]
if 'Userspace Error:' in line:
print 'Userspace Error detected!'
index = line.find('Userspace Error:')
boot_failure_reason = line[index:]
if '<LAVA_DISPATCHER>' not in line:
if len(line) != 0:
job_file += line + '\n'
if '### dt-test ### end of selftest' in line:
dt_tests = True
regex = re.compile("(?P<test>\d+\*?)")
dt_test_results = regex.findall(line)
if len(dt_test_results) > 2:
dt_tests_passed = dt_test_results[2]
dt_tests_failed = dt_test_results[3]
else:
dt_tests_passed = dt_test_results[0]
dt_tests_failed = dt_test_results[1]
if int(dt_tests_failed) > 0:
dt_test_result = 'FAIL'
else:
dt_test_result = 'PASS'
if 'rtc-efi rtc-efi: setting system clock to' in line:
if device_type == 'dynamic-vm':
efi_rtc = True
# Retrieve bundle
if bundle is not None:
json_bundle = connection.dashboard.get(bundle)
bundle_data = json.loads(json_bundle['content'])
# Get the boot data from LAVA
for test_results in bundle_data['test_runs']:
# Check for the LAVA self boot test
if test_results['test_id'] == 'lava':
for test in test_results['test_results']:
# TODO for compat :(
if test['test_case_id'] == 'kernel_boot_time':
kernel_boot_time = test['measurement']
if test['test_case_id'] == 'test_kernel_boot_time':
kernel_boot_time = test['measurement']
bundle_attributes = bundle_data['test_runs'][-1]['attributes']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
print bundle_attributes['kernel.defconfig']
if utils.in_bundle_attributes(bundle_attributes, 'target'):
board_instance = bundle_attributes['target']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
kernel_defconfig = bundle_attributes['kernel.defconfig']
kernel_defconfig_base = ''.join(kernel_defconfig.split('+')[:1])
if utils.in_bundle_attributes(bundle_attributes, 'arch'):
arch = bundle_attributes['arch']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.describe'):
kernel_version = bundle_attributes['kernel.describe']
if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
device_tree = bundle_attributes['device.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
kernel_endian = bundle_attributes['kernel.endian']
if utils.in_bundle_attributes(bundle_attributes, 'platform.fastboot'):
fastboot = bundle_attributes['platform.fastboot']
if kernel_boot_time is None:
if utils.in_bundle_attributes(bundle_attributes, 'kernel-boot-time'):
kernel_boot_time = bundle_attributes['kernel-boot-time']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
kernel_tree = bundle_attributes['kernel.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
kernel_addr = bundle_attributes['kernel-addr']
if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
initrd_addr = bundle_attributes['initrd-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
dtb_addr = bundle_attributes['dtb-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
dtb_append = bundle_attributes['dtb-append']
if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
boot_retries = int(bundle_attributes['boot_retries'])
if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
test_plan = bundle_attributes['test.plan']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.branch'):
git_branch = bundle_attributes['kernel.branch']
# Check if we found efi-rtc
if test_plan == 'boot-kvm-uefi' and not efi_rtc:
if device_type == 'dynamic-vm':
boot_failure_reason = 'Unable to read EFI rtc'
result = 'FAIL'
# Record the boot log and result
# TODO: Will need to map device_types to dashboard device types
if kernel_defconfig and device_type and result:
if (arch == 'arm' or arch =='arm64') and device_tree is None:
platform_name = device_map[device_type][0] + ',legacy'
else:
if device_tree == 'vexpress-v2p-ca15_a7.dtb':
platform_name = 'vexpress-v2p-ca15_a7'
elif device_tree == 'fsl-ls2080a-simu.dtb':
platform_name = 'fsl-ls2080a-simu'
elif test_plan == 'boot-kvm' or test_plan == 'boot-kvm-uefi':
if device_tree == 'sun7i-a20-cubietruck.dtb':
if device_type == 'dynamic-vm':
device_type = 'cubieboard3-kvm-guest'
platform_name = device_map[device_type][0]
else:
device_type = 'cubieboard3-kvm-host'
platform_name = device_map[device_type][0]
elif device_tree == 'apm-mustang.dtb':
if device_type == 'dynamic-vm':
if test_plan == 'boot-kvm-uefi':
device_type = 'mustang-kvm-uefi-guest'
else:
device_type = 'mustang-kvm-guest'
platform_name = device_map[device_type][0]
else:
if test_plan == 'boot-kvm-uefi':
device_type = 'mustang-kvm-uefi-host'
else:
device_type = 'mustang-kvm-host'
platform_name = device_map[device_type][0]
elif device_tree == 'juno.dtb':
if device_type == 'dynamic-vm':
if test_plan == 'boot-kvm-uefi':
device_type = 'juno-kvm-uefi-guest'
else:
device_type = 'juno-kvm-guest'
platform_name = device_map[device_type][0]
else:
if test_plan == 'boot-kvm-uefi':
device_type = 'juno-kvm-uefi-host'
else:
device_type = 'juno-kvm-host'
platform_name = device_map[device_type][0]
elif test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
platform_name = device_map[device_type][0] + '_rootfs:nfs'
else:
platform_name = device_map[device_type][0]
print 'Creating boot log for %s' % platform_name
log = 'boot-%s.txt' % platform_name
html = 'boot-%s.html' % platform_name
if config.get("lab"):
directory = os.path.join(results_directory, kernel_defconfig + '/' + config.get("lab"))
else:
directory = os.path.join(results_directory, kernel_defconfig)
utils.ensure_dir(directory)
utils.write_file(job_file, log, directory)
if kernel_boot_time is None:
kernel_boot_time = '0.0'
if results.has_key(kernel_defconfig):
results[kernel_defconfig].append({'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result})
else:
results[kernel_defconfig] = [{'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result}]
# Create JSON format boot metadata
print 'Creating JSON format boot metadata'
if config.get("lab"):
boot_meta['lab_name'] = config.get("lab")
else:
boot_meta['lab_name'] = None
if board_instance:
boot_meta['board_instance'] = board_instance
boot_meta['retries'] = boot_retries
boot_meta['boot_log'] = log
boot_meta['boot_log_html'] = html
# TODO: Fix this
boot_meta['version'] = '1.1'
boot_meta['arch'] = arch
boot_meta['defconfig'] = kernel_defconfig_base
boot_meta['defconfig_full'] = kernel_defconfig
if device_map[device_type][1]:
boot_meta['mach'] = device_map[device_type][1]
boot_meta['kernel'] = kernel_version
boot_meta['git_branch'] = git_branch
boot_meta['job'] = kernel_tree
boot_meta['board'] = platform_name
if board_offline and result == 'FAIL':
boot_meta['boot_result'] = 'OFFLINE'
#results[kernel_defconfig]['result'] = 'OFFLINE'
else:
boot_meta['boot_result'] = result
if result == 'FAIL' or result == 'OFFLINE':
if boot_failure_reason:
boot_meta['boot_result_description'] = boot_failure_reason
else:
boot_meta['boot_result_description'] = 'Unknown Error: platform failed to boot'
boot_meta['boot_time'] = kernel_boot_time
# TODO: Fix this
boot_meta['boot_warnings'] = None
if device_tree:
if arch == 'arm64':
boot_meta['dtb'] = 'dtbs/' + device_map[device_type][1] + '/' + device_tree
else:
boot_meta['dtb'] = 'dtbs/' + device_tree
else:
boot_meta['dtb'] = device_tree
boot_meta['dtb_addr'] = dtb_addr
boot_meta['dtb_append'] = dtb_append
boot_meta['dt_test'] = dt_test
boot_meta['endian'] = kernel_endian
boot_meta['fastboot'] = fastboot
# TODO: Fix this
boot_meta['initrd'] = None
boot_meta['initrd_addr'] = initrd_addr
if arch == 'arm':
boot_meta['kernel_image'] = 'zImage'
elif arch == 'arm64':
boot_meta['kernel_image'] = 'Image'
else:
boot_meta['kernel_image'] = 'bzImage'
boot_meta['loadaddr'] = kernel_addr
json_file = 'boot-%s.json' % platform_name
utils.write_json(json_file, directory, boot_meta)
print 'Creating html version of boot log for %s' % platform_name
cmd = 'python log2html.py %s' % os.path.join(directory, log)
subprocess.check_output(cmd, shell=True)
if config.get("lab") and config.get("api") and config.get("token"):
print 'Sending boot result to %s for %s' % (config.get("api"), platform_name)
headers = {
'Authorization': config.get("token"),
'Content-Type': 'application/json'
}
api_url = urlparse.urljoin(config.get("api"), '/boot')
push('POST', api_url, data=json.dumps(boot_meta), headers=headers)
headers = {
'Authorization': config.get("token"),
}
print 'Uploading text version of boot log'
with open(os.path.join(directory, log)) as lh:
data = lh.read()
api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s/%s/%s' % (kernel_tree,
git_branch,
kernel_version,
arch,
kernel_defconfig,
config.get("lab"),
log))
push('PUT', api_url, data=data, headers=headers)
print 'Uploading html version of boot log'
with open(os.path.join(directory, html)) as lh:
data = lh.read()
api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s/%s/%s' % (kernel_tree,
git_branch,
kernel_version,
arch,
kernel_defconfig,
config.get("lab"),
html))
push('PUT', api_url, data=data, headers=headers)
def boot_report(config):
connection, jobs, duration = parse_json(config.get("boot"))
# TODO: Fix this when multi-lab sync is working
#download_log2html(log2html)
results_directory = os.getcwd() + '/results'
results = {}
dt_tests = False
utils.mkdir(results_directory)
for job_id in jobs:
print 'Job ID: %s' % job_id
# Init
boot_meta = {}
api_url = None
arch = None
board_instance = None
boot_retries = 0
kernel_defconfig_full = None
kernel_defconfig = None
kernel_defconfig_base = None
kernel_version = None
device_tree = None
kernel_endian = None
kernel_tree = None
kernel_addr = None
initrd_addr = None
dtb_addr = None
dtb_append = None
fastboot = None
fastboot_cmd = None
test_plan = None
job_file = ''
dt_test = None
dt_test_result = None
dt_tests_passed = None
dt_tests_failed = None
board_offline = False
kernel_boot_time = None
boot_failure_reason = None
efi_rtc = False
# Retrieve job details
device_type = ''
job_details = connection.scheduler.job_details(job_id)
if job_details['requested_device_type_id']:
device_type = job_details['requested_device_type_id']
if job_details['description']:
job_name = job_details['description']
device_name = job_details['_actual_device_cache']['hostname']
result = jobs[job_id]['result']
bundle = jobs[job_id]['bundle']
if not device_type:
device_type = job_details['_actual_device_cache']['device_type_id']
if bundle is None and device_type == 'dynamic-vm':
host_job_id = job_id.replace('.1', '.0')
bundle = jobs[host_job_id]['bundle']
if bundle is None:
print '%s bundle is empty, skipping...' % device_type
continue
# Retrieve the log file
try:
binary_job_file = connection.scheduler.job_output(job_id)
except xmlrpclib.Fault:
print 'Job output not found for %s' % device_type
continue
# Parse LAVA messages out of log
raw_job_file = str(binary_job_file)
for line in raw_job_file.splitlines():
if 'Infrastructure Error:' in line:
print 'Infrastructure Error detected!'
index = line.find('Infrastructure Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Bootloader Error:' in line:
print 'Bootloader Error detected!'
index = line.find('Bootloader Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Kernel Error:' in line:
print 'Kernel Error detected!'
index = line.find('Kernel Error:')
boot_failure_reason = line[index:]
if 'Userspace Error:' in line:
print 'Userspace Error detected!'
index = line.find('Userspace Error:')
boot_failure_reason = line[index:]
if '<LAVA_DISPATCHER>' not in line:
if len(line) != 0:
job_file += line + '\n'
if '### dt-test ### end of selftest' in line:
dt_tests = True
regex = re.compile("(?P<test>\d+\*?)")
dt_test_results = regex.findall(line)
if len(dt_test_results) > 2:
dt_tests_passed = dt_test_results[2]
dt_tests_failed = dt_test_results[3]
else:
dt_tests_passed = dt_test_results[0]
dt_tests_failed = dt_test_results[1]
if int(dt_tests_failed) > 0:
dt_test_result = 'FAIL'
else:
dt_test_result = 'PASS'
if 'rtc-efi rtc-efi: setting system clock to' in line:
if device_type == 'dynamic-vm':
efi_rtc = True
# Retrieve bundle
if bundle is not None:
json_bundle = connection.dashboard.get(bundle)
bundle_data = json.loads(json_bundle['content'])
# Get the boot data from LAVA
for test_results in bundle_data['test_runs']:
# Check for the LAVA self boot test
if test_results['test_id'] == 'lava':
for test in test_results['test_results']:
# TODO for compat :(
if test['test_case_id'] == 'kernel_boot_time':
kernel_boot_time = test['measurement']
if test['test_case_id'] == 'test_kernel_boot_time':
kernel_boot_time = test['measurement']
bundle_attributes = bundle_data['test_runs'][-1][
'attributes']
if utils.in_bundle_attributes(bundle_attributes,
'kernel.defconfig'):
print bundle_attributes['kernel.defconfig']
if utils.in_bundle_attributes(bundle_attributes, 'target'):
board_instance = bundle_attributes['target']
if utils.in_bundle_attributes(bundle_attributes,
'kernel.defconfig'):
kernel_defconfig = bundle_attributes['kernel.defconfig']
defconfig_list = kernel_defconfig.split('-')
#arch = defconfig_list[0]
arch = defconfig_list[-1]
# Remove arch
defconfig_list.pop(0)
kernel_defconfig_full = '-'.join(defconfig_list)
kernel_defconfig_base = ''.join(
kernel_defconfig_full.split('+')[:1])
if kernel_defconfig_full == kernel_defconfig_base:
kernel_defconfig_full = None
if utils.in_bundle_attributes(bundle_attributes, 'kernel.version'):
kernel_version = bundle_attributes['kernel.version']
if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
device_tree = bundle_attributes['device.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
kernel_endian = bundle_attributes['kernel.endian']
if utils.in_bundle_attributes(bundle_attributes,
'platform.fastboot'):
fastboot = bundle_attributes['platform.fastboot']
if kernel_boot_time is None:
if utils.in_bundle_attributes(bundle_attributes,
'kernel-boot-time'):
kernel_boot_time = bundle_attributes['kernel-boot-time']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
kernel_tree = bundle_attributes['kernel.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
kernel_addr = bundle_attributes['kernel-addr']
if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
initrd_addr = bundle_attributes['initrd-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
dtb_addr = bundle_attributes['dtb-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
dtb_append = bundle_attributes['dtb-append']
if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
boot_retries = int(bundle_attributes['boot_retries'])
if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
test_plan = bundle_attributes['test.plan']
# Check if we found efi-rtc
if test_plan == 'boot-kvm-uefi' and not efi_rtc:
if device_type == 'dynamic-vm':
boot_failure_reason = 'Unable to read EFI rtc'
result = 'FAIL'
# Record the boot log and result
# TODO: Will need to map device_types to dashboard device types
if kernel_defconfig and device_type and result:
if ('arm' == arch or 'arm64' == arch) and device_tree is None:
platform_name = device_map[device_type][0] + ',legacy'
else:
if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
platform_name = device_map[device_type][0] + '_rootfs:nfs'
else:
platform_name = device_map[device_type][0]
print 'Creating boot log for %s' % platform_name
log = 'boot-%s.txt' % (platform_name + job_name)
html = 'boot-%s.html' % (platform_name + job_name)
if config.get("lab"):
directory = os.path.join(
results_directory,
kernel_defconfig + '/' + config.get("lab"))
else:
directory = os.path.join(results_directory, kernel_defconfig)
utils.ensure_dir(directory)
utils.write_file(job_file, log, directory)
if kernel_boot_time is None:
kernel_boot_time = '0.0'
if results.has_key(kernel_defconfig):
results[kernel_defconfig].append({
'device_type': platform_name,
'dt_test_result': dt_test_result,
'dt_tests_passed': dt_tests_passed,
'dt_tests_failed': dt_tests_failed,
'kernel_boot_time': kernel_boot_time,
'result': result,
'device_name': device_name
})
else:
results[kernel_defconfig] = [{
'device_type': platform_name,
'dt_test_result': dt_test_result,
'dt_tests_passed': dt_tests_passed,
'dt_tests_failed': dt_tests_failed,
'kernel_boot_time': kernel_boot_time,
'result': result,
'device_name': device_name
}]
# Create JSON format boot metadata
print 'Creating JSON format boot metadata'
if config.get("lab"):
boot_meta['lab_name'] = config.get("lab")
else:
boot_meta['lab_name'] = None
if board_instance:
boot_meta['board_instance'] = board_instance
boot_meta['retries'] = boot_retries
boot_meta['boot_log'] = log
boot_meta['boot_log_html'] = html
# TODO: Fix this
boot_meta['version'] = '1.0'
boot_meta['arch'] = arch
boot_meta['defconfig'] = kernel_defconfig_base
if kernel_defconfig_full is not None:
boot_meta['defconfig_full'] = kernel_defconfig_full
if device_map[device_type][1]:
boot_meta['mach'] = device_map[device_type][1]
boot_meta['kernel'] = kernel_version
boot_meta['job'] = kernel_tree
boot_meta['board'] = platform_name
if board_offline and result == 'FAIL':
boot_meta['boot_result'] = 'OFFLINE'
#results[kernel_defconfig]['result'] = 'OFFLINE'
else:
boot_meta['boot_result'] = result
if result == 'FAIL' or result == 'OFFLINE':
if boot_failure_reason:
boot_meta['boot_result_description'] = boot_failure_reason
else:
boot_meta[
'boot_result_description'] = 'Unknown Error: platform failed to boot'
boot_meta['boot_time'] = kernel_boot_time
# TODO: Fix this
boot_meta['boot_warnings'] = None
if device_tree:
if arch == 'arm64':
boot_meta['dtb'] = 'dtbs/' + device_map[device_type][
1] + '/' + device_tree
else:
boot_meta['dtb'] = 'dtbs/' + device_tree
else:
boot_meta['dtb'] = device_tree
boot_meta['dtb_addr'] = dtb_addr
boot_meta['dtb_append'] = dtb_append
boot_meta['dt_test'] = dt_test
boot_meta['endian'] = kernel_endian
boot_meta['fastboot'] = fastboot
# TODO: Fix this
boot_meta['initrd'] = None
boot_meta['initrd_addr'] = initrd_addr
if arch == 'arm':
boot_meta['kernel_image'] = 'zImage'
elif arch == 'arm64':
boot_meta['kernel_image'] = 'Image'
else:
boot_meta['kernel_image'] = 'bzImage'
boot_meta['loadaddr'] = kernel_addr
json_file = 'boot-%s.json' % (platform_name + job_name)
utils.write_json(json_file, directory, boot_meta)
#print 'Creating html version of boot log for %s' % platform_name
#cmd = 'python log2html.py %s' % os.path.join(directory, log)
#subprocess.check_output(cmd, shell=True)
#if config.get("lab") and config.get("api") and config.get("token"):
# print 'Sending boot result to %s for %s' % (config.get("api"), platform_name)
# headers = {
# 'Authorization': config.get("token"),
# 'Content-Type': 'application/json'
# }
# api_url = urlparse.urljoin(config.get("api"), '/boot')
# push('POST', api_url, data=json.dumps(boot_meta), headers=headers)
# headers = {
# 'Authorization': config.get("token"),
# }
# print 'Uploading text version of boot log'
# with open(os.path.join(directory, log)) as lh:
# data = lh.read()
# api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s' % (kernel_tree,
# kernel_version,
# kernel_defconfig,
# config.get("lab"),
# log))
# push('PUT', api_url, data=data, headers=headers)
# print 'Uploading html version of boot log'
# with open(os.path.join(directory, html)) as lh:
# data = lh.read()
# api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s' % (kernel_tree,
# kernel_version,
# kernel_defconfig,
# config.get("lab"),
# html))
# push('PUT', api_url, data=data, headers=headers)
if config.get("lab"):
report_directory = os.path.join(results_directory, config.get("lab"))
utils.mkdir(report_directory)
else:
report_directory = results_directory
if results and kernel_tree and kernel_version and 'boot' in test_plan or 'BOOT' in test_plan:
print 'Creating summary for %s' % (kernel_version)
boot = '%s-boot-report.txt' % (kernel_version)
boot = boot.replace('boot', test_plan)
passed = 0
failed = 0
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'PASS':
passed += 1
else:
failed += 1
total = passed + failed
with open(os.path.join(report_directory, boot), 'a') as f:
#f.write('To: %s\n' % config.get("email"))
#f.write('From: [email protected]\n')
f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' %
(kernel_tree, str(total), str(passed), str(failed),
kernel_version))
f.write('\n')
#f.write('Full Build Report: http://192.168.1.108:5000/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
#f.write('Full Boot Report: http://192.168.1.108:5000/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
#f.write('\n')
f.write('Total Duration: %.2f minutes\n' % (duration / 60))
f.write('Tree/Branch: %s\n' % kernel_tree)
f.write('Git Describe: %s\n' % kernel_version)
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'OFFLINE':
if first:
f.write('\n')
f.write('Boards Offline:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'OFFLINE':
f.write(' %s %s %ss boot-test: %s\n' %
(result['device_type'], result['device_name'],
result['kernel_boot_time'], result['result']))
f.write('\n')
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'FAIL':
if first:
f.write('\n')
f.write('Failed Boot Tests:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'FAIL':
f.write(' %s %s %ss boot-test: %s\n' %
(result['device_type'], result['device_name'],
result['kernel_boot_time'], result['result']))
if config.get("lab"):
f.write(
' http://192.168.1.108:8083/kernel-ci/%s/%s/%s/%s/boot-%s.html'
% (kernel_tree, kernel_version, defconfig,
config.get("lab"), result['device_type']))
else:
f.write(
' http://192.168.1.108:8083/kernel-ci/%s/%s/%s/boot-%s.html'
% (kernel_tree, kernel_version, defconfig,
result['device_type']))
f.write('\n')
f.write('\n')
f.write('Full Boot Report:\n')
for defconfig, results_list in results.items():
f.write('\n')
f.write(defconfig)
f.write('\n')
for result in results_list:
f.write(' %s %s %ss boot-test: %s\n' %
(result['device_type'], result['device_name'],
result['kernel_boot_time'], result['result']))
# add by wuyanjun
if results and directory:
parser_and_get_result(results, directory, report_directory)
#get_ip_board_mapping(results, directory, report_directory)
# dt-self-test
if results and kernel_tree and kernel_version and dt_tests:
print 'Creating device tree runtime self test summary for %s' % kernel_version
dt_self_test = '%s-dt-runtime-self-test-report.txt' % kernel_version
passed = 0
failed = 0
for defconfig, results_list in results.items():
for result in results_list:
if result['dt_test_result'] == 'PASS':
passed += 1
elif result['dt_test_result'] == 'FAIL':
failed += 1
total = passed + failed
with open(os.path.join(report_directory, dt_self_test), 'a') as f:
#f.write('To: %s\n' % config.get("email"))
#f.write('From: [email protected]\n')
f.write(
'Subject: %s dt-runtime-unit-tests: %s boards tested: %s passed, %s failed (%s)\n'
% (kernel_tree, str(total), str(passed), str(failed),
kernel_version))
f.write('\n')
#f.write('Full Build Report: http://192.168.1.108:5000/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
#f.write('Full Boot Report: http://192.168.1.108:5000/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
#f.write('Full Test Report: http://192.168.1.108:5000/test/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
#f.write('\n')
f.write('Tree/Branch: %s\n' % kernel_tree)
f.write('Git Describe: %s\n' % kernel_version)
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['dt_test_result'] == 'FAIL':
if first:
f.write('\n')
f.write('Failed Device Tree Unit Tests:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['dt_test_result'] == "FAIL":
f.write(
' %s %s passed: %s / failed: %s dt-runtime-unit-tests: %s\n'
% (result['device_type'], result['device_name'],
result['dt_tests_passed'],
result['dt_tests_failed'],
result['dt_test_result']))
if config.get("lab"):
f.write(
' http://192.168.1.108:8083/kernel-ci/%s/%s/%s/%s/boot-%s.html'
% (kernel_tree, kernel_version, defconfig,
config.get("lab"), result['device_type']))
else:
f.write(
' http://192.168.1.108:8083/kernel-ci/%s/%s/%s/boot-%s.html'
% (kernel_tree, kernel_version, defconfig,
result['device_type']))
f.write('\n')
f.write('\n')
f.write('Full Unit Test Report:\n')
for defconfig, results_list in results.items():
first = True
for result in results_list:
if result['dt_test_result']:
if first:
f.write('\n')
f.write(defconfig)
f.write('\n')
first = False
f.write(
' %s %s passed: %s / failed: %s dt-runtime-unit-tests: %s\n'
% (result['device_type'], result['device_name'],
result['dt_tests_passed'],
result['dt_tests_failed'],
result['dt_test_result']))
# sendmail
if config.get("email"):
print 'Sending e-mail summary to %s' % config.get("email")
if os.path.exists(report_directory):
cmd = 'cat %s | sendmail -t' % os.path.join(report_directory, boot)
subprocess.check_output(cmd, shell=True)
if dt_tests:
if os.path.exists(report_directory):
cmd = 'cat %s | sendmail -t' % os.path.join(
report_directory, dt_self_test)
subprocess.check_output(cmd, shell=True)
def train_one(model_folder):
new_model_folder_name = model_folder.replace('_created', '_training')
os_rename(model_folder, new_model_folder_name)
frequencies = glob(os_path_join(new_model_folder_name, 'k_*'))
for frequency in frequencies:
# Load model
print('train.py: training {}'.format(frequency))
model_params_path = os_path_join(frequency, model_params_fname)
# create model
model, model_params = get_which_model_from_params_fname(
model_params_path, return_params=True)
if 'cuda' in model_params:
using_cuda = model_params['cuda'] and torch_cuda_is_available()
else:
using_cuda = torch_cuda_is_available()
if using_cuda is True:
model.cuda()
# save initial weights
if 'save_initial' in model_params and model_params[
'save_initial'] and model_params['save_dir']:
suffix = '_initial'
path = add_suffix_to_path(model_params_fname['save_dir'], suffix) # pylint: disable=E1126
ensure_dir(path)
torch_save(model.state_dict(),
os_path_join(path, MODEL_DATA_FNAME))
save_model_params(os_path_join(path, model_params_fname),
model_params)
# loss
if 'cost_function' in model_params:
loss = model_params['cost_function']
elif 'loss_function' in model_params:
loss = model_params['loss_function']
else:
raise ValueError(
'model_params missing key cost_function or loss_function')
if loss not in ['MSE', 'L1', 'SmoothL1']:
raise TypeError('Error must be MSE, L1, or SmoothL1. You gave ' +
str(loss))
if loss == 'MSE':
from torch.nn import MSELoss
loss = MSELoss()
elif loss == 'L1':
from torch.nn import L1Loss
loss = L1Loss()
elif loss == 'SmoothL1':
from torch.nn import SmoothL1Loss
loss = SmoothL1Loss()
# optimizer
if model_params['optimizer'] == 'Adam':
from torch.optim import Adam
optimizer = Adam(model.parameters(),
lr=model_params['learning_rate'],
weight_decay=model_params['weight_decay'])
elif model_params['optimizer'] == 'SGD':
from torch.optim import SGD
optimizer = SGD(model.parameters(),
lr=model_params['learning_rate'],
momentum=model_params['momentum'],
weight_decay=model_params['weight_decay'])
else:
raise ValueError(
'model_params[\'optimizer\'] must be either Adam or SGD. Got '
+ model_params['optimizer'])
logger = Logger()
# Load training, validation, and test data
# Load primary training data
dat_train = ApertureDataset(
model_params['data_train'],
NUM_SAMPLES_TRAIN,
k=model_params['k'],
target_is_data=model_params['data_is_target'])
loader_train = DataLoader(dat_train,
batch_size=model_params['batch_size'],
shuffle=True,
num_workers=DATALOADER_NUM_WORKERS,
pin_memory=using_cuda)
# Load secondary training data - used to evaluate training loss after every epoch
dat_train2 = ApertureDataset(
model_params['data_train'],
NUM_SAMPLES_TRAIN_EVAL,
k=model_params['k'],
target_is_data=model_params['data_is_target'])
loader_train_eval = DataLoader(dat_train2,
batch_size=model_params['batch_size'],
shuffle=False,
num_workers=DATALOADER_NUM_WORKERS,
pin_memory=using_cuda)
# Load validation data - used to evaluate validation loss after every epoch
dat_val = ApertureDataset(
model_params['data_val'],
NUM_SAMPLES_VALID,
k=model_params['k'],
target_is_data=model_params['data_is_target'])
loader_val = DataLoader(dat_val,
batch_size=model_params['batch_size'],
shuffle=False,
num_workers=DATALOADER_NUM_WORKERS,
pin_memory=using_cuda)
trainer = Trainer(
model=model,
loss=loss,
optimizer=optimizer,
patience=model_params['patience'],
loader_train=loader_train,
loader_train_eval=loader_train_eval,
loader_val=loader_val,
cuda=using_cuda,
logger=logger,
data_noise_gaussian=model_params['data_noise_gaussian'],
save_dir=frequency)
# run training
trainer.train()
os_rename(new_model_folder_name,
new_model_folder_name.replace('_training', '_trained'))
def extract_features_batch(model, source_path, target_path, dataset,
voxel_size, device):
"""
Extracts the per point features in the FCGF feature space and saves them to the predefined path
Args:
model (FCGF model instance): model used to inferr the descriptors
source_path (str): path to the raw files
target path (str): where to save the extracted data
dataset (float): name of the dataset
voxel_size (float): voxel sized used to create the sparse tensor
device (pytorch device): cuda or cpu
"""
source_path = os.path.join(source_path, dataset, 'raw_data')
target_path = os.path.join(target_path, dataset, 'features')
ensure_dir(target_path)
folders = get_folder_list(source_path)
assert len(folders) > 0, 'Could not find {} folders under {}'.format(
dataset, source_path)
logging.info(folders)
list_file = os.path.join(target_path, 'list.txt')
f = open(list_file, 'w')
model.eval()
for fo in folders:
scene_name = fo.split()
files = get_file_list(fo, '.ply')
fo_base = os.path.basename(fo)
ensure_dir(os.path.join(target_path, fo_base))
f.write('%s %d\n' % (fo_base, len(files)))
for i, fi in enumerate(files):
save_fn = '%s_%03d' % (fo_base, i)
if os.path.exists(
os.path.join(target_path, fo_base, save_fn + '.npz')):
print(
'Correspondence file already exits moving to the next example.'
)
else:
# Extract features from a file
pcd = o3d.io.read_point_cloud(fi)
if i % 100 == 0:
logging.info(f'{i} / {len(files)}: {save_fn}')
xyz_down, feature = extract_features(model,
xyz=np.array(pcd.points),
rgb=None,
normal=None,
voxel_size=voxel_size,
device=device,
skip_check=True)
np.savez_compressed(os.path.join(target_path, fo_base,
save_fn),
points=np.array(pcd.points),
xyz=xyz_down,
feature=feature.detach().cpu().numpy())
f.close()
def extract_precomputed_training_data(dataset, source_path, target_path,
voxel_size, inlier_threshold):
"""
Prepares the data for training the filtering networks with precomputed correspondences (without FCGF descriptor)
Args:
dataset (str): name of the dataset
source_path (str): path to the raw data
target_path (str): path to where the extracted data will be saved
voxel_size (float): voxel size that was used to compute the features
inlier_threshold (float): threshold to determine if a correspondence is an inlier or outlier
"""
source_path = os.path.join(source_path, dataset, 'raw_data')
features_path = os.path.join(target_path, dataset, 'features')
correspondence_path = os.path.join(target_path, dataset, 'correspondences')
target_path = os.path.join(target_path, dataset, 'training_data')
ensure_dir(target_path)
# Check that the GT global transformation matrices are available and that the FCGF features are computed
folders = get_folder_list(source_path)
assert len(folders) > 0, 'Could not find {} folders under {}'.format(
dataset, source_path)
logging.info('Found {} scenes from the {} dataset!'.format(
len(folders), dataset))
for fo in folders:
scene_name = fo.split()
fo_base = os.path.basename(fo)
ensure_dir(os.path.join(target_path, fo_base))
pc_files = get_file_list(fo, '.ply')
trans_files = get_file_list(fo, '.txt')
assert len(pc_files) <= len(
trans_files
), 'The number of point cloud files does not equal the number of GT trans parameters!'
feat_files = get_file_list(os.path.join(features_path, fo_base),
'.npz')
assert len(pc_files) == len(
feat_files
), 'Features for scene {} are either not computed or some are missing!'.format(
fo_base)
coor_files = get_file_list(os.path.join(correspondence_path, fo_base),
'.npz')
assert len(coor_files) == int(
(len(feat_files) * (len(feat_files) - 1)) / 2
), 'Correspondence files for the scene {} are missing. First run the correspondence extraction!'.format(
fo_base)
# Loop over all fragment pairs and compute the training data
for idx_1 in range(len(pc_files)):
for idx_2 in range(idx_1 + 1, len(pc_files)):
if os.path.exists(
os.path.join(
target_path, fo_base,
'{}_{}_{}.npz'.format(fo_base,
str(idx_1).zfill(3),
str(idx_2).zfill(3)))):
logging.info(
'Training file already exits moving to the next example.'
)
data = np.load(
os.path.join(
correspondence_path, fo_base,
'{}_{}_{}.npz'.format(fo_base,
str(idx_1).zfill(3),
str(idx_2).zfill(3))))
xs = data['xs']
mutuals = data['mutuals']
ratios = data['ratios']
# Get the GT transformation parameters
t_1 = np.genfromtxt(os.path.join(
source_path, fo_base,
'cloud_bin_{}.info.txt'.format(idx_1)),
skip_header=1)
t_2 = np.genfromtxt(os.path.join(
source_path, fo_base,
'cloud_bin_{}.info.txt'.format(idx_2)),
skip_header=1)
# Get the GT transformation parameters
pc_1 = load_point_cloud(os.path.join(
source_path, fo_base, 'cloud_bin_{}.ply'.format(idx_1)),
data_type='numpy')
pc_2 = load_point_cloud(os.path.join(
source_path, fo_base, 'cloud_bin_{}.ply'.format(idx_2)),
data_type='numpy')
pc_1_tr = transform_point_cloud(pc_1, t_1[0:3, 0:3],
t_1[0:3, 3].reshape(-1, 1))
pc_2_tr = transform_point_cloud(pc_2, t_2[0:3, 0:3],
t_2[0:3, 3].reshape(-1, 1))
overlap_ratio = compute_overlap_ratio(pc_1_tr,
pc_2_tr,
np.eye(4),
method='FCGF',
voxel_size=voxel_size)
# Estimate pairwise transformation parameters
t_3 = np.matmul(np.linalg.inv(t_2), t_1)
r_matrix = t_3[0:3, 0:3]
t_vector = t_3[0:3, 3]
# Transform the keypoints of the first point cloud
pc_1_key_tr = transform_point_cloud(xs[:, 0:3], r_matrix,
t_vector.reshape(-1, 1))
# Compute the residuals after the transformation
y_s = np.sqrt(
np.sum(np.square(pc_1_key_tr - xs[:, 3:6]), axis=1))
# Inlier percentage
inlier_ratio = np.where(
y_s < inlier_threshold)[0].shape[0] / y_s.shape[0]
inlier_ratio_mutuals = np.where(
y_s[mutuals.astype(bool).reshape(-1)] < inlier_threshold
)[0].shape[0] / np.sum(mutuals)
np.savez_compressed(os.path.join(
target_path, fo_base, 'cloud_{}_{}.npz'.format(
str(idx_1).zfill(3),
str(idx_2).zfill(3))),
R=r_matrix,
t=t_vector,
x=xs,
y=y_s,
mutuals=mutuals,
inlier_ratio=inlier_ratio,
inlier_ratio_mutuals=inlier_ratio_mutuals,
ratios=ratios,
overlap=overlap_ratio)
def run_correspondence_extraction(dataset, source_path, target_path,
n_correspondences, idx):
"""
Computes the correspondences in the FCGF space together with the mutuals and ratios side information
Args:
dataset (str): name of the dataset
source_path (str): path to the raw data
target_path (str): path to where the extracted data will be saved
n_correspondences (int): number of points to sample
idx (int): index of the scene, used for parallel processing
"""
# Initialize all the paths
features_path = os.path.join(target_path, dataset, 'features')
target_path = os.path.join(target_path, dataset, 'correspondences')
fo = get_folder_list(source_path)[idx]
fo_base = os.path.basename(fo)
files = get_file_list(os.path.join(features_path, fo_base), '.npz')
ensure_dir(os.path.join(target_path, fo_base))
# Loop over all fragment pairs and compute the training data
for idx_1 in range(len(files)):
for idx_2 in range(idx_1 + 1, len(files)):
if os.path.exists(
os.path.join(
target_path, fo_base,
'{}_{}_{}.npz'.format(fo_base,
str(idx_1).zfill(3),
str(idx_2).zfill(3)))):
logging.info(
'Correspondence file already exits moving to the next example.'
)
else:
pc_1_data = np.load(
os.path.join(
features_path, fo_base,
fo_base + '_{}.npz'.format(str(idx_1).zfill(3))))
pc_1_features = pc_1_data['feature']
pc_1_keypoints = pc_1_data['xyz']
pc_2_data = np.load(
os.path.join(
features_path, fo_base,
fo_base + '_{}.npz'.format(str(idx_2).zfill(3))))
pc_2_features = pc_2_data['feature']
pc_2_keypoints = pc_2_data['xyz']
# Sample with replacement if less then n_correspondences points are in the point cloud
if pc_1_features.shape[0] >= n_correspondences:
inds_1 = np.random.choice(pc_1_features.shape[0],
n_correspondences,
replace=False)
else:
inds_1 = np.random.choice(pc_1_features.shape[0],
n_correspondences,
replace=True)
if pc_2_features.shape[0] >= n_correspondences:
inds_2 = np.random.choice(pc_2_features.shape[0],
n_correspondences,
replace=False)
else:
inds_2 = np.random.choice(pc_2_features.shape[0],
n_correspondences,
replace=True)
pc_1_features = pc_1_features[inds_1, :]
pc_2_features = pc_2_features[inds_2, :]
pc_1_key = pc_1_keypoints[inds_1, :]
pc_2_key = pc_2_keypoints[inds_2, :]
# find the correspondence using nearest neighbor search in the feature space (two way)
nn_search = NearestNeighbors(n_neighbors=1,
metric='minkowski',
p=2)
nn_search.fit(pc_2_features)
nn_dists, nn_indices = nn_search.kneighbors(
X=pc_1_features, n_neighbors=2, return_distance=True)
nn_search.fit(pc_1_features)
nn_dists_1, nn_indices_1 = nn_search.kneighbors(
X=pc_2_features, n_neighbors=2, return_distance=True)
ol_nn_ids = np.where(
(nn_indices[nn_indices_1[:, 0], 0] -
np.arange(pc_1_features.shape[0])) == 0)[0]
# Initialize mutuals and ratios
mutuals = np.zeros((n_correspondences, 1))
mutuals[ol_nn_ids] = 1
ratios = nn_dists[:, 0] / nn_dists[:, 1]
# Concatenate the correspondence coordinates
xs = np.concatenate(
(pc_1_key[nn_indices_1[:, 0], :], pc_2_key), axis=1)
np.savez_compressed(os.path.join(
target_path, fo_base,
'{}_{}_{}.npz'.format(fo_base,
str(idx_1).zfill(3),
str(idx_2).zfill(3))),
x=xs,
mutuals=mutuals,
ratios=ratios)
def boot_report(config):
connection, jobs, duration = parse_json(config.get("boot"))
# TODO: Fix this when multi-lab sync is working
#download_log2html(log2html)
results_directory = os.getcwd() + '/results'
results = {}
dt_tests = False
utils.mkdir(results_directory)
for job_id in jobs:
print 'Job ID: %s' % job_id
# Init
boot_meta = {}
api_url = None
arch = None
board_instance = None
boot_retries = 0
kernel_defconfig_full = None
kernel_defconfig = None
kernel_defconfig_base = None
kernel_version = None
device_tree = None
kernel_endian = None
kernel_tree = None
kernel_addr = None
initrd_addr = None
dtb_addr = None
dtb_append = None
fastboot = None
fastboot_cmd = None
test_plan = None
job_file = ''
dt_test = None
dt_test_result = None
dt_tests_passed = None
dt_tests_failed = None
board_offline = False
kernel_boot_time = None
boot_failure_reason = None
efi_rtc = False
# Retrieve job details
job_details = connection.scheduler.job_details(job_id)
if job_details['requested_device_type_id']:
device_type = job_details['requested_device_type_id']
if job_details['description']:
job_name = job_details['description']
result = jobs[job_id]['result']
bundle = jobs[job_id]['bundle']
if bundle is None and device_type == 'dynamic-vm':
host_job_id = job_id.replace('.1', '.0')
bundle = jobs[host_job_id]['bundle']
if bundle is None:
print '%s bundle is empty, skipping...' % device_type
continue
# Retrieve the log file
try:
binary_job_file = connection.scheduler.job_output(job_id)
except xmlrpclib.Fault:
print 'Job output not found for %s' % device_type
continue
# Parse LAVA messages out of log
raw_job_file = str(binary_job_file)
for line in raw_job_file.splitlines():
if 'Infrastructure Error:' in line:
print 'Infrastructure Error detected!'
index = line.find('Infrastructure Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Bootloader Error:' in line:
print 'Bootloader Error detected!'
index = line.find('Bootloader Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Kernel Error:' in line:
print 'Kernel Error detected!'
index = line.find('Kernel Error:')
boot_failure_reason = line[index:]
if 'Userspace Error:' in line:
print 'Userspace Error detected!'
index = line.find('Userspace Error:')
boot_failure_reason = line[index:]
if '<LAVA_DISPATCHER>' not in line:
if len(line) != 0:
job_file += line + '\n'
if '### dt-test ### end of selftest' in line:
dt_tests = True
regex = re.compile("(?P<test>\d+\*?)")
dt_test_results = regex.findall(line)
if len(dt_test_results) > 2:
dt_tests_passed = dt_test_results[2]
dt_tests_failed = dt_test_results[3]
else:
dt_tests_passed = dt_test_results[0]
dt_tests_failed = dt_test_results[1]
if int(dt_tests_failed) > 0:
dt_test_result = 'FAIL'
else:
dt_test_result = 'PASS'
if 'rtc-efi rtc-efi: setting system clock to' in line:
if device_type == 'dynamic-vm':
efi_rtc = True
# Retrieve bundle
if bundle is not None:
json_bundle = connection.dashboard.get(bundle)
bundle_data = json.loads(json_bundle['content'])
# Get the boot data from LAVA
for test_results in bundle_data['test_runs']:
# Check for the LAVA self boot test
if test_results['test_id'] == 'lava':
for test in test_results['test_results']:
# TODO for compat :(
if test['test_case_id'] == 'kernel_boot_time':
kernel_boot_time = test['measurement']
if test['test_case_id'] == 'test_kernel_boot_time':
kernel_boot_time = test['measurement']
bundle_attributes = bundle_data['test_runs'][-1]['attributes']
if test_results['test_id'] == 'lava-command':
# Post stuff with the test API.
build_id = "56b9648659b514b7f6e41fac"
test_suite = test_api_post(config, build_id, test_results)
if test_suite:
print json.dumps(test_suite)
print 'Sending test suite to %s' % config.get("api")
headers = {
'Authorization': config.get("token"),
'Content-Type': 'application/json'
}
api_url = urlparse.urljoin(config.get("api"), '/test/suite')
push('POST', api_url, json.dumps(test_suite), headers)
if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
print bundle_attributes['kernel.defconfig']
if utils.in_bundle_attributes(bundle_attributes, 'target'):
board_instance = bundle_attributes['target']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
kernel_defconfig = bundle_attributes['kernel.defconfig']
defconfig_list = kernel_defconfig.split('-')
arch = defconfig_list[0]
# Remove arch
defconfig_list.pop(0)
kernel_defconfig_full = '-'.join(defconfig_list)
kernel_defconfig_base = ''.join(kernel_defconfig_full.split('+')[:1])
if kernel_defconfig_full == kernel_defconfig_base:
kernel_defconfig_full = None
if utils.in_bundle_attributes(bundle_attributes, 'kernel.version'):
kernel_version = bundle_attributes['kernel.version']
if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
device_tree = bundle_attributes['device.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
kernel_endian = bundle_attributes['kernel.endian']
if utils.in_bundle_attributes(bundle_attributes, 'platform.fastboot'):
fastboot = bundle_attributes['platform.fastboot']
if kernel_boot_time is None:
if utils.in_bundle_attributes(bundle_attributes, 'kernel-boot-time'):
kernel_boot_time = bundle_attributes['kernel-boot-time']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
kernel_tree = bundle_attributes['kernel.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
kernel_addr = bundle_attributes['kernel-addr']
if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
initrd_addr = bundle_attributes['initrd-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
dtb_addr = bundle_attributes['dtb-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
dtb_append = bundle_attributes['dtb-append']
if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
boot_retries = int(bundle_attributes['boot_retries'])
if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
test_plan = bundle_attributes['test.plan']
# Check if we found efi-rtc
if test_plan == 'boot-kvm-uefi' and not efi_rtc:
if device_type == 'dynamic-vm':
boot_failure_reason = 'Unable to read EFI rtc'
result = 'FAIL'
# Record the boot log and result
# TODO: Will need to map device_types to dashboard device types
if kernel_defconfig and device_type and result:
if (arch == 'arm' or arch =='arm64') and device_tree is None:
platform_name = device_map[device_type][0] + ',legacy'
else:
if device_tree == 'vexpress-v2p-ca15_a7.dtb':
platform_name = 'vexpress-v2p-ca15_a7'
elif device_tree == 'fsl-ls2080a-simu.dtb':
platform_name = 'fsl-ls2080a-simu'
elif test_plan == 'boot-kvm' or test_plan == 'boot-kvm-uefi':
if device_tree == 'sun7i-a20-cubietruck.dtb':
if device_type == 'dynamic-vm':
device_type = 'cubieboard3-kvm-guest'
platform_name = device_map[device_type][0]
else:
device_type = 'cubieboard3-kvm-host'
platform_name = device_map[device_type][0]
elif device_tree == 'apm-mustang.dtb':
if device_type == 'dynamic-vm':
if test_plan == 'boot-kvm-uefi':
device_type = 'mustang-kvm-uefi-guest'
else:
device_type = 'mustang-kvm-guest'
platform_name = device_map[device_type][0]
else:
if test_plan == 'boot-kvm-uefi':
device_type = 'mustang-kvm-uefi-host'
else:
device_type = 'mustang-kvm-host'
platform_name = device_map[device_type][0]
elif device_tree == 'juno.dtb':
if device_type == 'dynamic-vm':
if test_plan == 'boot-kvm-uefi':
device_type = 'juno-kvm-uefi-guest'
else:
device_type = 'juno-kvm-guest'
platform_name = device_map[device_type][0]
else:
if test_plan == 'boot-kvm-uefi':
device_type = 'juno-kvm-uefi-host'
else:
device_type = 'juno-kvm-host'
platform_name = device_map[device_type][0]
elif test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
platform_name = device_map[device_type][0] + '_rootfs:nfs'
else:
platform_name = device_map[device_type][0]
print 'Creating boot log for %s' % platform_name
log = 'boot-%s.txt' % platform_name
html = 'boot-%s.html' % platform_name
if config.get("lab"):
directory = os.path.join(results_directory, kernel_defconfig + '/' + config.get("lab"))
else:
directory = os.path.join(results_directory, kernel_defconfig)
utils.ensure_dir(directory)
utils.write_file(job_file, log, directory)
if kernel_boot_time is None:
kernel_boot_time = '0.0'
if results.has_key(kernel_defconfig):
results[kernel_defconfig].append({'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result})
else:
results[kernel_defconfig] = [{'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result}]
# Create JSON format boot metadata
print 'Creating JSON format boot metadata'
if config.get("lab"):
boot_meta['lab_name'] = config.get("lab")
else:
boot_meta['lab_name'] = None
if board_instance:
boot_meta['board_instance'] = board_instance
boot_meta['retries'] = boot_retries
boot_meta['boot_log'] = log
boot_meta['boot_log_html'] = html
# TODO: Fix this
boot_meta['version'] = '1.0'
boot_meta['arch'] = arch
boot_meta['defconfig'] = kernel_defconfig_base
if kernel_defconfig_full is not None:
boot_meta['defconfig_full'] = kernel_defconfig_full
if device_map[device_type][1]:
boot_meta['mach'] = device_map[device_type][1]
boot_meta['kernel'] = kernel_version
boot_meta['job'] = kernel_tree
boot_meta['board'] = platform_name
if board_offline and result == 'FAIL':
boot_meta['boot_result'] = 'OFFLINE'
#results[kernel_defconfig]['result'] = 'OFFLINE'
else:
boot_meta['boot_result'] = result
if result == 'FAIL' or result == 'OFFLINE':
if boot_failure_reason:
boot_meta['boot_result_description'] = boot_failure_reason
else:
boot_meta['boot_result_description'] = 'Unknown Error: platform failed to boot'
boot_meta['boot_time'] = kernel_boot_time
# TODO: Fix this
boot_meta['boot_warnings'] = None
if device_tree:
if arch == 'arm64':
boot_meta['dtb'] = 'dtbs/' + device_map[device_type][1] + '/' + device_tree
else:
boot_meta['dtb'] = 'dtbs/' + device_tree
else:
boot_meta['dtb'] = device_tree
boot_meta['dtb_addr'] = dtb_addr
boot_meta['dtb_append'] = dtb_append
boot_meta['dt_test'] = dt_test
boot_meta['endian'] = kernel_endian
boot_meta['fastboot'] = fastboot
# TODO: Fix this
boot_meta['initrd'] = None
boot_meta['initrd_addr'] = initrd_addr
if arch == 'arm':
boot_meta['kernel_image'] = 'zImage'
elif arch == 'arm64':
boot_meta['kernel_image'] = 'Image'
else:
boot_meta['kernel_image'] = 'bzImage'
boot_meta['loadaddr'] = kernel_addr
json_file = 'boot-%s.json' % platform_name
utils.write_json(json_file, directory, boot_meta)
print 'Creating html version of boot log for %s' % platform_name
cmd = 'python log2html.py %s' % os.path.join(directory, log)
subprocess.check_output(cmd, shell=True)
if config.get("lab") and config.get("api") and config.get("token"):
print 'Sending boot result to %s for %s' % (config.get("api"), platform_name)
headers = {
'Authorization': config.get("token"),
'Content-Type': 'application/json'
}
api_url = urlparse.urljoin(config.get("api"), '/boot')
push('POST', api_url, data=json.dumps(boot_meta), headers=headers)
headers = {
'Authorization': config.get("token"),
}
print 'Uploading text version of boot log'
with open(os.path.join(directory, log)) as lh:
data = lh.read()
api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s' % (kernel_tree,
kernel_version,
kernel_defconfig,
config.get("lab"),
log))
push('PUT', api_url, data=data, headers=headers)
print 'Uploading html version of boot log'
with open(os.path.join(directory, html)) as lh:
data = lh.read()
api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s' % (kernel_tree,
kernel_version,
kernel_defconfig,
config.get("lab"),
html))
push('PUT', api_url, data=data, headers=headers)
if results and kernel_tree and kernel_version:
print 'Creating boot summary for %s' % kernel_version
boot = '%s-boot-report.txt' % kernel_version
passed = 0
failed = 0
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'PASS':
passed += 1
else:
failed += 1
total = passed + failed
if config.get("lab"):
report_directory = os.path.join(results_directory, config.get("lab"))
utils.mkdir(report_directory)
else:
report_directory = results_directory
with open(os.path.join(report_directory, boot), 'a') as f:
f.write('To: %s\n' % config.get("email"))
f.write('From: [email protected]\n')
f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' % (kernel_tree,
str(total),
str(passed),
str(failed),
kernel_version))
f.write('\n')
f.write('Full Build Report: http://kernelci.org/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
f.write('Full Boot Report: http://kernelci.org/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
f.write('\n')
f.write('Total Duration: %.2f minutes\n' % (duration / 60))
f.write('Tree/Branch: %s\n' % kernel_tree)
f.write('Git Describe: %s\n' % kernel_version)
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'OFFLINE':
if first:
f.write('\n')
f.write('Boards Offline:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'OFFLINE':
f.write(' %s %ss boot-test: %s\n' % (result['device_type'],
result['kernel_boot_time'],
result['result']))
f.write('\n')
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'FAIL':
if first:
f.write('\n')
f.write('Failed Boot Tests:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'FAIL':
f.write(' %s %ss boot-test: %s\n' % (result['device_type'],
result['kernel_boot_time'],
result['result']))
if config.get("lab"):
f.write(' http://storage.kernelci.org/kernel-ci/%s/%s/%s/%s/boot-%s.html' % (kernel_tree,
kernel_version,
defconfig,
config.get("lab"),
result['device_type']))
else:
f.write(' http://storage.kernelci.org/kernel-ci/%s/%s/%s/boot-%s.html' % (kernel_tree,
kernel_version,
defconfig,
result['device_type']))
f.write('\n')
f.write('\n')
f.write('Full Boot Report:\n')
for defconfig, results_list in results.items():
f.write('\n')
f.write(defconfig)
f.write('\n')
for result in results_list:
f.write(' %s %ss boot-test: %s\n' % (result['device_type'], result['kernel_boot_time'], result['result']))
# dt-self-test
if results and kernel_tree and kernel_version and dt_tests:
print 'Creating device tree runtime self test summary for %s' % kernel_version
dt_self_test = '%s-dt-runtime-self-test-report.txt' % kernel_version
passed = 0
failed = 0
for defconfig, results_list in results.items():
for result in results_list:
if result['dt_test_result'] == 'PASS':
passed += 1
elif result['dt_test_result'] == 'FAIL':
failed += 1
total = passed + failed
with open(os.path.join(report_directory, dt_self_test), 'a') as f:
f.write('To: %s\n' % config.get("email"))
f.write('From: [email protected]\n')
f.write('Subject: %s dt-runtime-unit-tests: %s boards tested: %s passed, %s failed (%s)\n' % (kernel_tree,
str(total),
str(passed),
str(failed),
kernel_version))
f.write('\n')
f.write('Full Build Report: http://kernelci.org/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
f.write('Full Boot Report: http://kernelci.org/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
f.write('Full Test Report: http://kernelci.org/test/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
f.write('\n')
f.write('Tree/Branch: %s\n' % kernel_tree)
f.write('Git Describe: %s\n' % kernel_version)
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['dt_test_result'] == 'FAIL':
if first:
f.write('\n')
f.write('Failed Device Tree Unit Tests:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['dt_test_result'] == "FAIL":
f.write(' %s passed: %s / failed: %s dt-runtime-unit-tests: %s\n' % (result['device_type'],
result['dt_tests_passed'],
result['dt_tests_failed'],
result['dt_test_result']))
if config.get("lab"):
f.write(' http://storage.kernelci.org/kernel-ci/%s/%s/%s/%s/boot-%s.html' % (kernel_tree,
kernel_version,
defconfig,
config.get("lab"),
result['device_type']))
else:
f.write(' http://storage.kernelci.org/kernel-ci/%s/%s/%s/boot-%s.html' % (kernel_tree,
kernel_version,
defconfig,
result['device_type']))
f.write('\n')
f.write('\n')
f.write('Full Unit Test Report:\n')
for defconfig, results_list in results.items():
first = True
for result in results_list:
if result['dt_test_result']:
if first:
f.write('\n')
f.write(defconfig)
f.write('\n')
first = False
f.write(' %s passed: %s / failed: %s dt-runtime-unit-tests: %s\n' % (result['device_type'],
result['dt_tests_passed'],
result['dt_tests_failed'],
result['dt_test_result']))
# sendmail
if config.get("email"):
print 'Sending e-mail summary to %s' % config.get("email")
if os.path.exists(report_directory):
cmd = 'cat %s | sendmail -t' % os.path.join(report_directory, boot)
subprocess.check_output(cmd, shell=True)
if dt_tests:
if os.path.exists(report_directory):
cmd = 'cat %s | sendmail -t' % os.path.join(report_directory, dt_self_test)
subprocess.check_output(cmd, shell=True)
def process(file_path, new_screen_name):
if not os.path.isfile(file_path):
sys.exit('File not found')
data = read_json(file_path)
new_data = dict()
new_data['layout'] = data['layout']
new_data['info'] = {
'versionCode': data['info']['versionCode'],
'versionName': data['info']['versionName'],
}
new_data['preferences'] = {
'Display.Theme': 'WhiteHCTheme',
'EarthModel': 'WGS84',
}
if '-por2lan' in sys.argv:
new_data['layout']['landscape'] = new_data['layout']['portrait']
if '-lan2por' in sys.argv:
new_data['layout']['landscape'] = new_data['layout']['portrait']
# nocomp and onlycomp split
for orient in ['portrait', 'landscape']:
if orient not in new_data['layout']:
continue
if '-nocomp' in sys.argv:
new_data['layout'][orient] = [
i for i in new_data['layout'][orient]
if 'org.xcontest.XCTrack.navig.TaskCompetition' not in
i['navigations']
]
if '-onlycomp' in sys.argv:
new_data['layout'][orient] = [
i for i in new_data['layout'][orient]
if 'org.xcontest.XCTrack.navig.TaskCompetition' in
i['navigations']
]
# portrait and landscape modes
if '-por' in sys.argv:
if 'landscape' in new_data['layout']:
del new_data['layout']['landscape']
new_data['preferences']['Display.Orientation'] = 'PORTRAIT'
elif '-lan' in sys.argv:
if 'portrait' in new_data['layout']:
del new_data['layout']['portrait']
new_data['preferences']['Display.Orientation'] = 'LANDSCAPE'
else:
sys.exit('Please specify either -por (portrait) or -lan (landscape)')
sort_widgets_by_name(new_data['layout'])
ensure_dir('screens')
target_file = os.path.join('screens', new_screen_name + '.xcfg')
write_json(target_file, new_data)
