def main(args):
config = configuration.get_config(args)
url = utils.validate_input(config.get("username"), config.get("token"),
config.get("server"))
connection = utils.connect(url)
if config.get("repo"):
retrieve_jobs(config.get("repo"))
load_jobs()
start_time = time.time()
bundle_stream = None
if config.get("stream"):
bundle_stream = config.get("stream")
submit_jobs(connection, config.get("server"), bundle_stream=bundle_stream)
if config.get("poll"):
jobs = poll_jobs(connection, config.get("timeout"))
end_time = time.time()
if config.get("bisect"):
for job_id in jobs:
if 'result' in jobs[job_id]:
if jobs[job_id]['result'] == 'FAIL':
exit(1)
jobs['duration'] = end_time - start_time
jobs['username'] = config.get("username")
jobs['token'] = config.get("token")
jobs['server'] = config.get("server")
results_directory = os.getcwd() + '/results'
utils.mkdir(results_directory)
utils.write_json(config.get("poll"), results_directory, jobs)
exit(0)
def main(args):
config = configuration.get_config(args)
url = utils.validate_input(config.get("username"), config.get("token"), config.get("server"))
connection = utils.connect(url)
if config.get("repo"):
retrieve_jobs(config.get("repo"))
load_jobs()
start_time = time.time()
bundle_stream = None
if config.get("stream"):
bundle_stream = config.get("stream")
submit_jobs(connection, config.get("server"), bundle_stream=bundle_stream)
if config.get("poll"):
jobs = poll_jobs(connection, config.get("timeout"))
end_time = time.time()
if config.get("bisect"):
for job_id in jobs:
if 'result' in jobs[job_id]:
if jobs[job_id]['result'] == 'FAIL':
exit(1)
jobs['duration'] = end_time - start_time
jobs['username'] = config.get("username")
jobs['token'] = config.get("token")
jobs['server'] = config.get("server")
results_directory = os.getcwd() + '/results'
utils.mkdir(results_directory)
utils.write_json(config.get("poll"), results_directory, jobs)
exit(0)
def main():
# Print out the docstring at top of file
print(__doc__)
# Create the path for the JSON that will be output
outfile = os.path.join(DATA_DIR, 'processed/sample.json')
# Use the built-in write_json function that we imported from lib.utils
write_json(outfile, {"foo": 1, "bar": 2})
print("Wrote data to {}\n".format(outfile))
def __init__(self, args, options='', timestamp=True):
# parse default and custom cli options
for opt in options:
args.add_argument(*opt.flags, default=None, type=opt.type)
args = args.parse_args()
if args.device:
os.environ["CUDA_VISIBLE_DEVICES"] = args.device
if args.resume:
self.resume = Path(args.resume)
self.cfg_fname = self.resume.parent / 'config.json'
else:
msg_no_cfg = "Configuration file need to be specified. Add '-c config.json', for example."
assert args.config is not None, msg_no_cfg
self.resume = None
self.cfg_fname = Path(args.config)
# load config file and apply custom cli options
config = read_json(self.cfg_fname)
self._config = _update_config(config, options, args)
# set save_dir where trained model and log will be saved.
save_dir = Path(self.config['trainer']['save_dir'])
timestamp = datetime.now().strftime(
r'%m%d_%H%M%S') if timestamp else ''
exper_name = self.config['name']
self._save_dir = save_dir / 'models' / exper_name / timestamp
self._log_dir = save_dir / 'log' / exper_name / timestamp
self.save_dir.mkdir(parents=True, exist_ok=True)
self.log_dir.mkdir(parents=True, exist_ok=True)
# save updated config file to the checkpoint dir
write_json(self.config, self.save_dir / 'config.json')
# configure logging module
setup_logging(self.log_dir)
self.log_levels = {
0: logging.WARNING,
1: logging.INFO,
2: logging.DEBUG
}
def main(args):
config = configuration.get_config(args)
url = utils.validate_input(config.get("username"), config.get("token"), config.get("server"))
connection = utils.connect(url)
if config.get("repo"):
retrieve_jobs(config.get("repo"))
if config.get("jobs"):
load_jobs(config.get("jobs"))
print "Loading jobs from top folder " + str(config.get("jobs"))
else:
load_jobs(os.getcwd())
start_time = time.time()
bundle_stream = None
if config.get("stream"):
bundle_stream = config.get("stream")
submit_jobs(connection, config.get("server"), bundle_stream=bundle_stream)
if config.get("poll"):
jobs = poll_jobs(connection, config.get("timeout"))
end_time = time.time()
if config.get("bisect"):
for job_id in jobs:
if "result" in jobs[job_id]:
if jobs[job_id]["result"] == "FAIL":
exit(1)
jobs["duration"] = end_time - start_time
jobs["username"] = config.get("username")
jobs["token"] = config.get("token")
jobs["server"] = config.get("server")
results_directory = os.getcwd() + "/results"
utils.mkdir(results_directory)
utils.write_json(config.get("poll"), results_directory, jobs)
exit(0)
def __load_docs(path_list, emb_json_path):
""" load all the data from the path list """
docs = []
length = len(path_list)
# traverse the path list to load all the data
for i, _path in enumerate(path_list):
# show progress
if i % 5 == 0:
progress = float(i + 1) / length * 100.
print('\rprogress: %.2f%% ' % progress, end='')
# remove nan in doc
tmp_doc = list(
map(lambda x: x
if isinstance(x, str) else '', load_json(_path)))
while '' in tmp_doc:
tmp_doc.remove('')
docs.append(tmp_doc)
# cache data for faster processing next time
write_json(emb_json_path, docs)
return docs
def gen_group_according_to(file_path):
print('loading data ...')
dict_dealer_index_2_group = utils.load_json(file_path)
data, d_dealers, total_volume, total_transaction_count, bound_timestamp, d_new_bonds = utils.load_pkl(
os.path.join(path.ROOT_DIR, 'runtime', 'tmp123.pkl'))
utils.write_pkl(
os.path.join(path.ROOT_DIR, 'runtime', 'tmp_d_dealers.pkl'), d_dealers)
# d_dealers = utils.load_pkl(os.path.join(path.ROOT_DIR, 'runtime', 'tmp_d_dealers.pkl'))
labels = set(list(map(lambda x: x[1], dict_dealer_index_2_group.items())))
group_list = [{} for i in range(len(labels))]
print('traversing data ...')
length = len(d_dealers)
cur = 0
for dealer_index, trace_list in d_dealers.items():
# show progress
if cur % 5 == 0:
progress = float(cur + 1) / length * 100.
print('\rprogress: %.2f%% ' % progress, end='')
cur += 1
if dealer_index not in dict_dealer_index_2_group:
continue
group_index = dict_dealer_index_2_group[dealer_index]
group_list[group_index][dealer_index] = trace_list
print('\rprogress: 100.0% \nsaving data ...')
plan_name = os.path.splitext(os.path.split(file_path)[1])[0] + '.json'
group_path = os.path.join(path.DATA_ROOT_DIR, 'groups', plan_name)
utils.write_json(group_path, group_list)
if not val['pos']:
del val['pos']
if not val:
delete_zhs.append(zh)
for en, val in __en_zh_dict.items():
if 'pos' not in val or not val['pos']:
continue
val['pos'] = clean_pos(val['pos'])
if not val['pos']:
del val['pos']
if not val:
delete_ens.append(en)
for k in delete_zhs:
del __zh_en_dict[k]
for k in delete_ens:
del __en_zh_dict[k]
__en_zh_dict = filter_duplicate(__en_zh_dict)
__zh_en_dict = filter_duplicate(__zh_en_dict)
write_json(filtered_pos_union_en_zh_dict_path, __en_zh_dict)
write_json(filtered_pos_union_zh_en_dict_path, __zh_en_dict)
print('done')
def boot_report(config):
connection, jobs, duration = parse_json(config.get("boot"))
# TODO: Fix this when multi-lab sync is working
#download_log2html(log2html)
results_directory = os.getcwd() + '/results'
results = {}
dt_tests = False
utils.mkdir(results_directory)
for job_id in jobs:
print 'Job ID: %s' % job_id
# Init
boot_meta = {}
api_url = None
arch = None
board_instance = None
boot_retries = 0
kernel_defconfig = None
kernel_defconfig_base = None
kernel_version = None
device_tree = None
kernel_endian = None
kernel_tree = None
git_branch = None
kernel_addr = None
initrd_addr = None
dtb_addr = None
dtb_append = None
fastboot = None
fastboot_cmd = None
test_plan = None
job_file = ''
dt_test = None
dt_test_result = None
dt_tests_passed = None
dt_tests_failed = None
board_offline = False
kernel_boot_time = None
boot_failure_reason = None
efi_rtc = False
# Retrieve job details
job_details = connection.scheduler.job_details(job_id)
if job_details['requested_device_type_id']:
device_type = job_details['requested_device_type_id']
if job_details['description']:
job_name = job_details['description']
result = jobs[job_id]['result']
bundle = jobs[job_id]['bundle']
if bundle is None and device_type == 'dynamic-vm':
host_job_id = job_id.replace('.1', '.0')
bundle = jobs[host_job_id]['bundle']
if bundle is None:
print '%s bundle is empty, skipping...' % device_type
continue
# Retrieve the log file
try:
binary_job_file = connection.scheduler.job_output(job_id)
except xmlrpclib.Fault:
print 'Job output not found for %s' % device_type
continue
# Parse LAVA messages out of log
raw_job_file = str(binary_job_file)
for line in raw_job_file.splitlines():
if 'Infrastructure Error:' in line:
print 'Infrastructure Error detected!'
index = line.find('Infrastructure Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Bootloader Error:' in line:
print 'Bootloader Error detected!'
index = line.find('Bootloader Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Kernel Error:' in line:
print 'Kernel Error detected!'
index = line.find('Kernel Error:')
boot_failure_reason = line[index:]
if 'Userspace Error:' in line:
print 'Userspace Error detected!'
index = line.find('Userspace Error:')
boot_failure_reason = line[index:]
if '<LAVA_DISPATCHER>' not in line:
if len(line) != 0:
job_file += line + '\n'
if '### dt-test ### end of selftest' in line:
dt_tests = True
regex = re.compile("(?P<test>\d+\*?)")
dt_test_results = regex.findall(line)
if len(dt_test_results) > 2:
dt_tests_passed = dt_test_results[2]
dt_tests_failed = dt_test_results[3]
else:
dt_tests_passed = dt_test_results[0]
dt_tests_failed = dt_test_results[1]
if int(dt_tests_failed) > 0:
dt_test_result = 'FAIL'
else:
dt_test_result = 'PASS'
if 'rtc-efi rtc-efi: setting system clock to' in line:
if device_type == 'dynamic-vm':
efi_rtc = True
# Retrieve bundle
if bundle is not None:
json_bundle = connection.dashboard.get(bundle)
bundle_data = json.loads(json_bundle['content'])
# Get the boot data from LAVA
for test_results in bundle_data['test_runs']:
# Check for the LAVA self boot test
if test_results['test_id'] == 'lava':
for test in test_results['test_results']:
# TODO for compat :(
if test['test_case_id'] == 'kernel_boot_time':
kernel_boot_time = test['measurement']
if test['test_case_id'] == 'test_kernel_boot_time':
kernel_boot_time = test['measurement']
bundle_attributes = bundle_data['test_runs'][-1]['attributes']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
print bundle_attributes['kernel.defconfig']
if utils.in_bundle_attributes(bundle_attributes, 'target'):
board_instance = bundle_attributes['target']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
kernel_defconfig = bundle_attributes['kernel.defconfig']
kernel_defconfig_base = ''.join(kernel_defconfig.split('+')[:1])
if utils.in_bundle_attributes(bundle_attributes, 'arch'):
arch = bundle_attributes['arch']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.describe'):
kernel_version = bundle_attributes['kernel.describe']
if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
device_tree = bundle_attributes['device.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
kernel_endian = bundle_attributes['kernel.endian']
if utils.in_bundle_attributes(bundle_attributes, 'platform.fastboot'):
fastboot = bundle_attributes['platform.fastboot']
if kernel_boot_time is None:
if utils.in_bundle_attributes(bundle_attributes, 'kernel-boot-time'):
kernel_boot_time = bundle_attributes['kernel-boot-time']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
kernel_tree = bundle_attributes['kernel.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
kernel_addr = bundle_attributes['kernel-addr']
if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
initrd_addr = bundle_attributes['initrd-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
dtb_addr = bundle_attributes['dtb-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
dtb_append = bundle_attributes['dtb-append']
if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
boot_retries = int(bundle_attributes['boot_retries'])
if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
test_plan = bundle_attributes['test.plan']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.branch'):
git_branch = bundle_attributes['kernel.branch']
# Check if we found efi-rtc
if test_plan == 'boot-kvm-uefi' and not efi_rtc:
if device_type == 'dynamic-vm':
boot_failure_reason = 'Unable to read EFI rtc'
result = 'FAIL'
# Record the boot log and result
# TODO: Will need to map device_types to dashboard device types
if kernel_defconfig and device_type and result:
if (arch == 'arm' or arch =='arm64') and device_tree is None:
platform_name = device_map[device_type][0] + ',legacy'
else:
if device_tree == 'vexpress-v2p-ca15_a7.dtb':
platform_name = 'vexpress-v2p-ca15_a7'
elif device_tree == 'fsl-ls2080a-simu.dtb':
platform_name = 'fsl-ls2080a-simu'
elif test_plan == 'boot-kvm' or test_plan == 'boot-kvm-uefi':
if device_tree == 'sun7i-a20-cubietruck.dtb':
if device_type == 'dynamic-vm':
device_type = 'cubieboard3-kvm-guest'
platform_name = device_map[device_type][0]
else:
device_type = 'cubieboard3-kvm-host'
platform_name = device_map[device_type][0]
elif device_tree == 'apm-mustang.dtb':
if device_type == 'dynamic-vm':
if test_plan == 'boot-kvm-uefi':
device_type = 'mustang-kvm-uefi-guest'
else:
device_type = 'mustang-kvm-guest'
platform_name = device_map[device_type][0]
else:
if test_plan == 'boot-kvm-uefi':
device_type = 'mustang-kvm-uefi-host'
else:
device_type = 'mustang-kvm-host'
platform_name = device_map[device_type][0]
elif device_tree == 'juno.dtb':
if device_type == 'dynamic-vm':
if test_plan == 'boot-kvm-uefi':
device_type = 'juno-kvm-uefi-guest'
else:
device_type = 'juno-kvm-guest'
platform_name = device_map[device_type][0]
else:
if test_plan == 'boot-kvm-uefi':
device_type = 'juno-kvm-uefi-host'
else:
device_type = 'juno-kvm-host'
platform_name = device_map[device_type][0]
elif test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
platform_name = device_map[device_type][0] + '_rootfs:nfs'
else:
platform_name = device_map[device_type][0]
print 'Creating boot log for %s' % platform_name
log = 'boot-%s.txt' % platform_name
html = 'boot-%s.html' % platform_name
if config.get("lab"):
directory = os.path.join(results_directory, kernel_defconfig + '/' + config.get("lab"))
else:
directory = os.path.join(results_directory, kernel_defconfig)
utils.ensure_dir(directory)
utils.write_file(job_file, log, directory)
if kernel_boot_time is None:
kernel_boot_time = '0.0'
if results.has_key(kernel_defconfig):
results[kernel_defconfig].append({'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result})
else:
results[kernel_defconfig] = [{'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result}]
# Create JSON format boot metadata
print 'Creating JSON format boot metadata'
if config.get("lab"):
boot_meta['lab_name'] = config.get("lab")
else:
boot_meta['lab_name'] = None
if board_instance:
boot_meta['board_instance'] = board_instance
boot_meta['retries'] = boot_retries
boot_meta['boot_log'] = log
boot_meta['boot_log_html'] = html
# TODO: Fix this
boot_meta['version'] = '1.1'
boot_meta['arch'] = arch
boot_meta['defconfig'] = kernel_defconfig_base
boot_meta['defconfig_full'] = kernel_defconfig
if device_map[device_type][1]:
boot_meta['mach'] = device_map[device_type][1]
boot_meta['kernel'] = kernel_version
boot_meta['git_branch'] = git_branch
boot_meta['job'] = kernel_tree
boot_meta['board'] = platform_name
if board_offline and result == 'FAIL':
boot_meta['boot_result'] = 'OFFLINE'
#results[kernel_defconfig]['result'] = 'OFFLINE'
else:
boot_meta['boot_result'] = result
if result == 'FAIL' or result == 'OFFLINE':
if boot_failure_reason:
boot_meta['boot_result_description'] = boot_failure_reason
else:
boot_meta['boot_result_description'] = 'Unknown Error: platform failed to boot'
boot_meta['boot_time'] = kernel_boot_time
# TODO: Fix this
boot_meta['boot_warnings'] = None
if device_tree:
if arch == 'arm64':
boot_meta['dtb'] = 'dtbs/' + device_map[device_type][1] + '/' + device_tree
else:
boot_meta['dtb'] = 'dtbs/' + device_tree
else:
boot_meta['dtb'] = device_tree
boot_meta['dtb_addr'] = dtb_addr
boot_meta['dtb_append'] = dtb_append
boot_meta['dt_test'] = dt_test
boot_meta['endian'] = kernel_endian
boot_meta['fastboot'] = fastboot
# TODO: Fix this
boot_meta['initrd'] = None
boot_meta['initrd_addr'] = initrd_addr
if arch == 'arm':
boot_meta['kernel_image'] = 'zImage'
elif arch == 'arm64':
boot_meta['kernel_image'] = 'Image'
else:
boot_meta['kernel_image'] = 'bzImage'
boot_meta['loadaddr'] = kernel_addr
json_file = 'boot-%s.json' % platform_name
utils.write_json(json_file, directory, boot_meta)
print 'Creating html version of boot log for %s' % platform_name
cmd = 'python log2html.py %s' % os.path.join(directory, log)
subprocess.check_output(cmd, shell=True)
if config.get("lab") and config.get("api") and config.get("token"):
print 'Sending boot result to %s for %s' % (config.get("api"), platform_name)
headers = {
'Authorization': config.get("token"),
'Content-Type': 'application/json'
}
api_url = urlparse.urljoin(config.get("api"), '/boot')
push('POST', api_url, data=json.dumps(boot_meta), headers=headers)
headers = {
'Authorization': config.get("token"),
}
print 'Uploading text version of boot log'
with open(os.path.join(directory, log)) as lh:
data = lh.read()
api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s/%s/%s' % (kernel_tree,
git_branch,
kernel_version,
arch,
kernel_defconfig,
config.get("lab"),
log))
push('PUT', api_url, data=data, headers=headers)
print 'Uploading html version of boot log'
with open(os.path.join(directory, html)) as lh:
data = lh.read()
api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s/%s/%s' % (kernel_tree,
git_branch,
kernel_version,
arch,
kernel_defconfig,
config.get("lab"),
html))
push('PUT', api_url, data=data, headers=headers)
# add data to zh_en_dict
for zh_word in zh_translation:
if zh_word not in zh_en_dict:
zh_en_dict[zh_word] = initialize_dict_element()
zh_en_dict[zh_word]['translation'].append(en_word)
zh_en_dict[zh_word]['pos'] += pos_list
zh_en_dict[zh_word]['tar_meanings'] += en_meanings
print('\n\nfiltering duplicate elements ...')
en_zh_dict = utils.filter_duplicate(en_zh_dict)
zh_en_dict = utils.filter_duplicate(zh_en_dict)
print('\nsaving data ... ')
write_json(os.path.join(dict_dir, 'en_zh_dict_from_ecdict_v_all.json'),
en_zh_dict)
write_json(os.path.join(dict_dir, 'zh_en_dict_from_ecdict_v_all.json'),
zh_en_dict)
print('\nanalyzing ...')
print(f'len en_zh_dict: {len(en_zh_dict)}')
print(f'len zh_en_dict: {len(zh_en_dict)}')
print('\ndone')
# len en_zh_dict: 3322644
# len zh_en_dict: 2827802
path_pkl_2015 = os.path.join(path.TRACE_DIR, 'finra_trace_2015.pkl')
data = utils.load_pkl(path_pkl_2015)
print('\nstart converting data ...')
data = np.array(data)
data = list(map(lambda x: {'bond_id': x[0], 'issue_id': x[16]}, data))
print('finish converting ')
dict_skip_bond = {}
dict_bond_id_offering_date = {}
for v in data:
bond_id = v['bond_id']
issue_id = str(int(v['issue_id']))
if issue_id not in dict_issue_id_offering_date:
dict_skip_bond[bond_id] = True
print('------------------------')
print(bond_id, issue_id)
continue
offering_date = dict_issue_id_offering_date[issue_id]
dict_bond_id_offering_date[bond_id] = offering_date
save_path = os.path.join(path.DATA_ROOT_DIR, 'dict_bond_id_offering_date.json')
utils.write_json(save_path, dict_bond_id_offering_date)
print('done')
print(f'skip bonds: {len(dict_skip_bond)}')
def boot_report(config):
connection, jobs, duration = parse_json(config.get("boot"))
# TODO: Fix this when multi-lab sync is working
#download_log2html(log2html)
results_directory = os.getcwd() + '/results'
results = {}
utils.mkdir(results_directory)
test_plan = None
if config.get("lab"):
report_directory = os.path.join(results_directory, config.get("lab"))
else:
report_directory = results_directory
if os.path.exists(report_directory):
shutil.rmtree(report_directory)
utils.mkdir(report_directory)
for job_id in jobs:
print 'Job ID: %s' % job_id
# Init
boot_meta = {}
api_url = None
arch = None
board_instance = None
boot_retries = 0
kernel_defconfig_full = None
kernel_defconfig = None
kernel_defconfig_base = None
kernel_version = None
device_tree = None
kernel_endian = None
kernel_tree = None
kernel_addr = None
initrd_addr = None
dtb_addr = None
dtb_append = None
fastboot = None
fastboot_cmd = None
job_file = ''
board_offline = False
kernel_boot_time = None
boot_failure_reason = None
efi_rtc = False
# Retrieve job details
device_type = ''
job_details = connection.scheduler.job_details(job_id)
if job_details['requested_device_type_id']:
device_type = job_details['requested_device_type_id']
if job_details['description']:
job_name = job_details['description']
try:
job_short_name = re.search(".*?([A-Z]+.*)", job_name).group(1)
except Exception:
job_short_name = 'boot-test'
try:
device_name = job_details['_actual_device_cache']['hostname']
except Exception:
continue
result = jobs[job_id]['result']
bundle = jobs[job_id]['bundle']
if not device_type:
device_type = job_details['_actual_device_cache']['device_type_id']
if bundle is None and device_type == 'dynamic-vm':
host_job_id = job_id.replace('.1', '.0')
bundle = jobs[host_job_id]['bundle']
if bundle is None:
print '%s bundle is empty, skipping...' % device_type
continue
# Retrieve the log file
try:
binary_job_file = connection.scheduler.job_output(job_id)
except xmlrpclib.Fault:
print 'Job output not found for %s' % device_type
continue
# Parse LAVA messages out of log
raw_job_file = str(binary_job_file)
for line in raw_job_file.splitlines():
if 'Infrastructure Error:' in line:
print 'Infrastructure Error detected!'
index = line.find('Infrastructure Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Bootloader Error:' in line:
print 'Bootloader Error detected!'
index = line.find('Bootloader Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Kernel Error:' in line:
print 'Kernel Error detected!'
index = line.find('Kernel Error:')
boot_failure_reason = line[index:]
if 'Userspace Error:' in line:
print 'Userspace Error detected!'
index = line.find('Userspace Error:')
boot_failure_reason = line[index:]
if '<LAVA_DISPATCHER>' not in line:
if len(line) != 0:
job_file += line + '\n'
if 'rtc-efi rtc-efi: setting system clock to' in line:
if device_type == 'dynamic-vm':
efi_rtc = True
# Retrieve bundle
if bundle is not None:
json_bundle = connection.dashboard.get(bundle)
bundle_data = json.loads(json_bundle['content'])
# Get the boot data from LAVA
for test_results in bundle_data['test_runs']:
# Check for the LAVA self boot test
if test_results['test_id'] == 'lava':
for test in test_results['test_results']:
# TODO for compat :(
if test['test_case_id'] == 'kernel_boot_time':
kernel_boot_time = test['measurement']
if test['test_case_id'] == 'test_kernel_boot_time':
kernel_boot_time = test['measurement']
bundle_attributes = bundle_data['test_runs'][-1]['attributes']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
print bundle_attributes['kernel.defconfig']
if utils.in_bundle_attributes(bundle_attributes, 'target'):
board_instance = bundle_attributes['target']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
kernel_defconfig = bundle_attributes['kernel.defconfig']
defconfig_list = kernel_defconfig.split('-')
#arch = defconfig_list[0]
arch = defconfig_list[-1]
# Remove arch
defconfig_list.pop(0)
kernel_defconfig_full = '-'.join(defconfig_list)
kernel_defconfig_base = ''.join(kernel_defconfig_full.split('+')[:1])
if kernel_defconfig_full == kernel_defconfig_base:
kernel_defconfig_full = None
if utils.in_bundle_attributes(bundle_attributes, 'kernel.version'):
kernel_version = bundle_attributes['kernel.version']
if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
device_tree = bundle_attributes['device.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
kernel_endian = bundle_attributes['kernel.endian']
if utils.in_bundle_attributes(bundle_attributes, 'platform.fastboot'):
fastboot = bundle_attributes['platform.fastboot']
if kernel_boot_time is None:
if utils.in_bundle_attributes(bundle_attributes, 'kernel-boot-time'):
kernel_boot_time = bundle_attributes['kernel-boot-time']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
kernel_tree = bundle_attributes['kernel.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
kernel_addr = bundle_attributes['kernel-addr']
if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
initrd_addr = bundle_attributes['initrd-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
dtb_addr = bundle_attributes['dtb-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
dtb_append = bundle_attributes['dtb-append']
if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
boot_retries = int(bundle_attributes['boot_retries'])
if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
test_tmp = bundle_attributes['test.plan']
if test_tmp:
test_plan = test_tmp
else:
if not kernel_defconfig or not kernel_version or not kernel_tree:
job_defnition = {}
if 'original_definition' in job_details.keys():
job_definition = job_details['original_definition']
try:
job_dictionary = eval(job_definition)
except Exception:
pass
if job_dictionary:
if 'actions' in job_dictionary.keys():
actions = job_dictionary['actions']
for i in range(0, len(actions)):
try:
kernel_defconfig = actions[i]['metadata']['kernel.defconfig']
kernel_version = actions[i]['metadata']['kernel.version']
kernel_tree = actions[i]['metadata']['kernel.tree']
kernel_endian = actions[i]['metadata']['kernel.endian']
platform_fastboot = actions[i]['metadata']['platform.fastboot']
device_tree = actions[i]['metadata']['kernel.tree']
break
except KeyError:
continue
if 'target' in job_details.keys():
print job_details.keys()
# Check if we found efi-rtc
if test_plan == 'boot-kvm-uefi' and not efi_rtc:
if device_type == 'dynamic-vm':
boot_failure_reason = 'Unable to read EFI rtc'
result = 'FAIL'
# Record the boot log and result
# TODO: Will need to map device_types to dashboard device types
if kernel_defconfig and device_type and result:
if ( 'arm' == arch or 'arm64' == arch ) and device_tree is None:
platform_name = device_map[device_type][0] + ',legacy'
else:
if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
platform_name = device_map[device_type][0] + '_rootfs:nfs'
else:
platform_name = device_map[device_type][0]
# Create txt format boot metadata
print 'Creating boot log for %s' % (platform_name + job_name + '_' + job_id)
log = 'boot-%s.txt' % (platform_name + job_name + '_' + job_id)
html = 'boot-%s.html' % (platform_name + job_name + '_' + job_id)
if config.get("lab"):
directory = os.path.join(results_directory, kernel_defconfig + '/' + config.get("lab"))
else:
directory = os.path.join(results_directory, kernel_defconfig)
utils.ensure_dir(directory)
utils.write_file(job_file, log, directory)
if kernel_boot_time is None:
kernel_boot_time = '0.0'
if results.has_key(kernel_defconfig):
results[kernel_defconfig].append({'device_type': platform_name,
'job_id': job_id, 'job_name': job_short_name,
'kernel_boot_time': kernel_boot_time, 'result': result,
'device_name': device_name})
else:
results[kernel_defconfig] = [{'device_type': platform_name,
'job_id': job_id, 'job_name': job_short_name,
'kernel_boot_time': kernel_boot_time, 'result': result,
'device_name': device_name}]
# Create JSON format boot metadata
print 'Creating JSON format boot metadata'
if config.get("lab"):
boot_meta['lab_name'] = config.get("lab")
else:
boot_meta['lab_name'] = None
if board_instance:
boot_meta['board_instance'] = board_instance
boot_meta['retries'] = boot_retries
boot_meta['boot_log'] = log
boot_meta['boot_log_html'] = html
# TODO: Fix this
boot_meta['version'] = '1.0'
boot_meta['arch'] = arch
boot_meta['defconfig'] = kernel_defconfig_base
if kernel_defconfig_full is not None:
boot_meta['defconfig_full'] = kernel_defconfig_full
if device_map[device_type][1]:
boot_meta['mach'] = device_map[device_type][1]
boot_meta['kernel'] = kernel_version
boot_meta['job'] = kernel_tree
boot_meta['board'] = platform_name
if board_offline and result == 'FAIL':
boot_meta['boot_result'] = 'OFFLINE'
#results[kernel_defconfig]['result'] = 'OFFLINE'
else:
boot_meta['boot_result'] = result
if result == 'FAIL' or result == 'OFFLINE':
if boot_failure_reason:
boot_meta['boot_result_description'] = boot_failure_reason
else:
boot_meta['boot_result_description'] = 'Unknown Error: platform failed to boot'
boot_meta['boot_time'] = kernel_boot_time
# TODO: Fix this
boot_meta['boot_warnings'] = None
if device_tree:
if arch == 'arm64':
boot_meta['dtb'] = 'dtbs/' + device_map[device_type][1] + '/' + device_tree
else:
boot_meta['dtb'] = 'dtbs/' + device_tree
else:
boot_meta['dtb'] = device_tree
boot_meta['dtb_addr'] = dtb_addr
boot_meta['dtb_append'] = dtb_append
boot_meta['endian'] = kernel_endian
boot_meta['fastboot'] = fastboot
# TODO: Fix this
boot_meta['initrd'] = None
boot_meta['initrd_addr'] = initrd_addr
if arch == 'arm':
boot_meta['kernel_image'] = 'zImage'
elif arch == 'arm64':
boot_meta['kernel_image'] = 'Image'
else:
boot_meta['kernel_image'] = 'bzImage'
boot_meta['loadaddr'] = kernel_addr
json_file = 'boot-%s.json' % (platform_name + job_name + '_' + job_id)
utils.write_json(json_file, directory, boot_meta)
# add by wuyanjun
# add the ip device mapping
get_ip_board_mapping(job_file, log, directory, report_directory)
parser_and_get_result(job_file, log, directory, report_directory, connection)
if results and kernel_tree and kernel_version:
print 'Creating summary for %s' % (kernel_version)
boot = '%s-boot-report.txt' % (kernel_version)
if test_plan and ('boot' in test_plan or 'BOOT' in test_plan):
boot = boot.replace('boot', test_plan)
passed = 0
failed = 0
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'PASS':
passed += 1
else:
failed += 1
total = passed + failed
with open(os.path.join(report_directory, boot), 'a') as f:
f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' % (kernel_tree,
str(total),
str(passed),
str(failed),
kernel_version))
f.write('\n')
f.write('Total Duration: %.2f minutes\n' % (duration / 60))
f.write('Tree/Branch: %s\n' % kernel_tree)
f.write('Git Describe: %s\n' % kernel_version)
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'OFFLINE':
if first:
f.write('\n')
f.write('Boards Offline:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'OFFLINE':
f.write(' %s %s %s %ss %s: %s\n' % (result['job_id'],
result['device_type'],
result['device_name'],
result['kernel_boot_time'],
result['job_name'],
result['result']))
f.write('\n')
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'FAIL':
if first:
f.write('\n')
f.write('Failed Boot Tests:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'FAIL':
f.write(' %s %s %s %ss %s: %s\n' % (result['job_id'],
result['device_type'],
result['device_name'],
result['kernel_boot_time'],
result['job_name'],
result['result']))
f.write('\n')
f.write('Full Boot Report:\n')
for defconfig, results_list in results.items():
f.write('\n')
f.write(defconfig)
f.write('\n')
for result in results_list:
f.write(' %s %s %s %ss %s: %s\n' %
(result['job_id'],
result['device_type'],
result['device_name'],
result['kernel_boot_time'],
result['job_name'],
result['result']))
def boot_report(config):
connection, jobs, duration = parse_json(config.get("boot"))
# TODO: Fix this when multi-lab sync is working
#download_log2html(log2html)
results_directory = os.getcwd() + '/results'
results = {}
dt_tests = False
utils.mkdir(results_directory)
for job_id in jobs:
print 'Job ID: %s' % job_id
# Init
boot_meta = {}
api_url = None
arch = None
board_instance = None
boot_retries = 0
kernel_defconfig_full = None
kernel_defconfig = None
kernel_defconfig_base = None
kernel_version = None
device_tree = None
kernel_endian = None
kernel_tree = None
kernel_addr = None
initrd_addr = None
dtb_addr = None
dtb_append = None
fastboot = None
fastboot_cmd = None
test_plan = None
job_file = ''
dt_test = None
dt_test_result = None
dt_tests_passed = None
dt_tests_failed = None
board_offline = False
kernel_boot_time = None
boot_failure_reason = None
efi_rtc = False
# Retrieve job details
job_details = connection.scheduler.job_details(job_id)
if job_details['requested_device_type_id']:
device_type = job_details['requested_device_type_id']
if job_details['description']:
job_name = job_details['description']
result = jobs[job_id]['result']
bundle = jobs[job_id]['bundle']
if bundle is None and device_type == 'dynamic-vm':
host_job_id = job_id.replace('.1', '.0')
bundle = jobs[host_job_id]['bundle']
if bundle is None:
print '%s bundle is empty, skipping...' % device_type
continue
# Retrieve the log file
try:
binary_job_file = connection.scheduler.job_output(job_id)
except xmlrpclib.Fault:
print 'Job output not found for %s' % device_type
continue
# Parse LAVA messages out of log
raw_job_file = str(binary_job_file)
for line in raw_job_file.splitlines():
if 'Infrastructure Error:' in line:
print 'Infrastructure Error detected!'
index = line.find('Infrastructure Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Bootloader Error:' in line:
print 'Bootloader Error detected!'
index = line.find('Bootloader Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Kernel Error:' in line:
print 'Kernel Error detected!'
index = line.find('Kernel Error:')
boot_failure_reason = line[index:]
if 'Userspace Error:' in line:
print 'Userspace Error detected!'
index = line.find('Userspace Error:')
boot_failure_reason = line[index:]
if '<LAVA_DISPATCHER>' not in line:
if len(line) != 0:
job_file += line + '\n'
if '### dt-test ### end of selftest' in line:
dt_tests = True
regex = re.compile("(?P<test>\d+\*?)")
dt_test_results = regex.findall(line)
if len(dt_test_results) > 2:
dt_tests_passed = dt_test_results[2]
dt_tests_failed = dt_test_results[3]
else:
dt_tests_passed = dt_test_results[0]
dt_tests_failed = dt_test_results[1]
if int(dt_tests_failed) > 0:
dt_test_result = 'FAIL'
else:
dt_test_result = 'PASS'
if 'rtc-efi rtc-efi: setting system clock to' in line:
if device_type == 'dynamic-vm':
efi_rtc = True
# Retrieve bundle
if bundle is not None:
json_bundle = connection.dashboard.get(bundle)
bundle_data = json.loads(json_bundle['content'])
# Get the boot data from LAVA
for test_results in bundle_data['test_runs']:
# Check for the LAVA self boot test
if test_results['test_id'] == 'lava':
for test in test_results['test_results']:
# TODO for compat :(
if test['test_case_id'] == 'kernel_boot_time':
kernel_boot_time = test['measurement']
if test['test_case_id'] == 'test_kernel_boot_time':
kernel_boot_time = test['measurement']
bundle_attributes = bundle_data['test_runs'][-1]['attributes']
if test_results['test_id'] == 'lava-command':
# Post stuff with the test API.
build_id = "56b9648659b514b7f6e41fac"
test_suite = test_api_post(config, build_id, test_results)
if test_suite:
print json.dumps(test_suite)
print 'Sending test suite to %s' % config.get("api")
headers = {
'Authorization': config.get("token"),
'Content-Type': 'application/json'
}
api_url = urlparse.urljoin(config.get("api"), '/test/suite')
push('POST', api_url, json.dumps(test_suite), headers)
if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
print bundle_attributes['kernel.defconfig']
if utils.in_bundle_attributes(bundle_attributes, 'target'):
board_instance = bundle_attributes['target']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
kernel_defconfig = bundle_attributes['kernel.defconfig']
defconfig_list = kernel_defconfig.split('-')
arch = defconfig_list[0]
# Remove arch
defconfig_list.pop(0)
kernel_defconfig_full = '-'.join(defconfig_list)
kernel_defconfig_base = ''.join(kernel_defconfig_full.split('+')[:1])
if kernel_defconfig_full == kernel_defconfig_base:
kernel_defconfig_full = None
if utils.in_bundle_attributes(bundle_attributes, 'kernel.version'):
kernel_version = bundle_attributes['kernel.version']
if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
device_tree = bundle_attributes['device.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
kernel_endian = bundle_attributes['kernel.endian']
if utils.in_bundle_attributes(bundle_attributes, 'platform.fastboot'):
fastboot = bundle_attributes['platform.fastboot']
if kernel_boot_time is None:
if utils.in_bundle_attributes(bundle_attributes, 'kernel-boot-time'):
kernel_boot_time = bundle_attributes['kernel-boot-time']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
kernel_tree = bundle_attributes['kernel.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
kernel_addr = bundle_attributes['kernel-addr']
if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
initrd_addr = bundle_attributes['initrd-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
dtb_addr = bundle_attributes['dtb-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
dtb_append = bundle_attributes['dtb-append']
if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
boot_retries = int(bundle_attributes['boot_retries'])
if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
test_plan = bundle_attributes['test.plan']
# Check if we found efi-rtc
if test_plan == 'boot-kvm-uefi' and not efi_rtc:
if device_type == 'dynamic-vm':
boot_failure_reason = 'Unable to read EFI rtc'
result = 'FAIL'
# Record the boot log and result
# TODO: Will need to map device_types to dashboard device types
if kernel_defconfig and device_type and result:
if (arch == 'arm' or arch =='arm64') and device_tree is None:
platform_name = device_map[device_type][0] + ',legacy'
else:
if device_tree == 'vexpress-v2p-ca15_a7.dtb':
platform_name = 'vexpress-v2p-ca15_a7'
elif device_tree == 'fsl-ls2080a-simu.dtb':
platform_name = 'fsl-ls2080a-simu'
elif test_plan == 'boot-kvm' or test_plan == 'boot-kvm-uefi':
if device_tree == 'sun7i-a20-cubietruck.dtb':
if device_type == 'dynamic-vm':
device_type = 'cubieboard3-kvm-guest'
platform_name = device_map[device_type][0]
else:
device_type = 'cubieboard3-kvm-host'
platform_name = device_map[device_type][0]
elif device_tree == 'apm-mustang.dtb':
if device_type == 'dynamic-vm':
if test_plan == 'boot-kvm-uefi':
device_type = 'mustang-kvm-uefi-guest'
else:
device_type = 'mustang-kvm-guest'
platform_name = device_map[device_type][0]
else:
if test_plan == 'boot-kvm-uefi':
device_type = 'mustang-kvm-uefi-host'
else:
device_type = 'mustang-kvm-host'
platform_name = device_map[device_type][0]
elif device_tree == 'juno.dtb':
if device_type == 'dynamic-vm':
if test_plan == 'boot-kvm-uefi':
device_type = 'juno-kvm-uefi-guest'
else:
device_type = 'juno-kvm-guest'
platform_name = device_map[device_type][0]
else:
if test_plan == 'boot-kvm-uefi':
device_type = 'juno-kvm-uefi-host'
else:
device_type = 'juno-kvm-host'
platform_name = device_map[device_type][0]
elif test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
platform_name = device_map[device_type][0] + '_rootfs:nfs'
else:
platform_name = device_map[device_type][0]
print 'Creating boot log for %s' % platform_name
log = 'boot-%s.txt' % platform_name
html = 'boot-%s.html' % platform_name
if config.get("lab"):
directory = os.path.join(results_directory, kernel_defconfig + '/' + config.get("lab"))
else:
directory = os.path.join(results_directory, kernel_defconfig)
utils.ensure_dir(directory)
utils.write_file(job_file, log, directory)
if kernel_boot_time is None:
kernel_boot_time = '0.0'
if results.has_key(kernel_defconfig):
results[kernel_defconfig].append({'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result})
else:
results[kernel_defconfig] = [{'device_type': platform_name, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result}]
# Create JSON format boot metadata
print 'Creating JSON format boot metadata'
if config.get("lab"):
boot_meta['lab_name'] = config.get("lab")
else:
boot_meta['lab_name'] = None
if board_instance:
boot_meta['board_instance'] = board_instance
boot_meta['retries'] = boot_retries
boot_meta['boot_log'] = log
boot_meta['boot_log_html'] = html
# TODO: Fix this
boot_meta['version'] = '1.0'
boot_meta['arch'] = arch
boot_meta['defconfig'] = kernel_defconfig_base
if kernel_defconfig_full is not None:
boot_meta['defconfig_full'] = kernel_defconfig_full
if device_map[device_type][1]:
boot_meta['mach'] = device_map[device_type][1]
boot_meta['kernel'] = kernel_version
boot_meta['job'] = kernel_tree
boot_meta['board'] = platform_name
if board_offline and result == 'FAIL':
boot_meta['boot_result'] = 'OFFLINE'
#results[kernel_defconfig]['result'] = 'OFFLINE'
else:
boot_meta['boot_result'] = result
if result == 'FAIL' or result == 'OFFLINE':
if boot_failure_reason:
boot_meta['boot_result_description'] = boot_failure_reason
else:
boot_meta['boot_result_description'] = 'Unknown Error: platform failed to boot'
boot_meta['boot_time'] = kernel_boot_time
# TODO: Fix this
boot_meta['boot_warnings'] = None
if device_tree:
if arch == 'arm64':
boot_meta['dtb'] = 'dtbs/' + device_map[device_type][1] + '/' + device_tree
else:
boot_meta['dtb'] = 'dtbs/' + device_tree
else:
boot_meta['dtb'] = device_tree
boot_meta['dtb_addr'] = dtb_addr
boot_meta['dtb_append'] = dtb_append
boot_meta['dt_test'] = dt_test
boot_meta['endian'] = kernel_endian
boot_meta['fastboot'] = fastboot
# TODO: Fix this
boot_meta['initrd'] = None
boot_meta['initrd_addr'] = initrd_addr
if arch == 'arm':
boot_meta['kernel_image'] = 'zImage'
elif arch == 'arm64':
boot_meta['kernel_image'] = 'Image'
else:
boot_meta['kernel_image'] = 'bzImage'
boot_meta['loadaddr'] = kernel_addr
json_file = 'boot-%s.json' % platform_name
utils.write_json(json_file, directory, boot_meta)
print 'Creating html version of boot log for %s' % platform_name
cmd = 'python log2html.py %s' % os.path.join(directory, log)
subprocess.check_output(cmd, shell=True)
if config.get("lab") and config.get("api") and config.get("token"):
print 'Sending boot result to %s for %s' % (config.get("api"), platform_name)
headers = {
'Authorization': config.get("token"),
'Content-Type': 'application/json'
}
api_url = urlparse.urljoin(config.get("api"), '/boot')
push('POST', api_url, data=json.dumps(boot_meta), headers=headers)
headers = {
'Authorization': config.get("token"),
}
print 'Uploading text version of boot log'
with open(os.path.join(directory, log)) as lh:
data = lh.read()
api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s' % (kernel_tree,
kernel_version,
kernel_defconfig,
config.get("lab"),
log))
push('PUT', api_url, data=data, headers=headers)
print 'Uploading html version of boot log'
with open(os.path.join(directory, html)) as lh:
data = lh.read()
api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s' % (kernel_tree,
kernel_version,
kernel_defconfig,
config.get("lab"),
html))
push('PUT', api_url, data=data, headers=headers)
if results and kernel_tree and kernel_version:
print 'Creating boot summary for %s' % kernel_version
boot = '%s-boot-report.txt' % kernel_version
passed = 0
failed = 0
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'PASS':
passed += 1
else:
failed += 1
total = passed + failed
if config.get("lab"):
report_directory = os.path.join(results_directory, config.get("lab"))
utils.mkdir(report_directory)
else:
report_directory = results_directory
with open(os.path.join(report_directory, boot), 'a') as f:
f.write('To: %s\n' % config.get("email"))
f.write('From: [email protected]\n')
f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' % (kernel_tree,
str(total),
str(passed),
str(failed),
kernel_version))
f.write('\n')
f.write('Full Build Report: http://kernelci.org/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
f.write('Full Boot Report: http://kernelci.org/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
f.write('\n')
f.write('Total Duration: %.2f minutes\n' % (duration / 60))
f.write('Tree/Branch: %s\n' % kernel_tree)
f.write('Git Describe: %s\n' % kernel_version)
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'OFFLINE':
if first:
f.write('\n')
f.write('Boards Offline:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'OFFLINE':
f.write(' %s %ss boot-test: %s\n' % (result['device_type'],
result['kernel_boot_time'],
result['result']))
f.write('\n')
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'FAIL':
if first:
f.write('\n')
f.write('Failed Boot Tests:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'FAIL':
f.write(' %s %ss boot-test: %s\n' % (result['device_type'],
result['kernel_boot_time'],
result['result']))
if config.get("lab"):
f.write(' http://storage.kernelci.org/kernel-ci/%s/%s/%s/%s/boot-%s.html' % (kernel_tree,
kernel_version,
defconfig,
config.get("lab"),
result['device_type']))
else:
f.write(' http://storage.kernelci.org/kernel-ci/%s/%s/%s/boot-%s.html' % (kernel_tree,
kernel_version,
defconfig,
result['device_type']))
f.write('\n')
f.write('\n')
f.write('Full Boot Report:\n')
for defconfig, results_list in results.items():
f.write('\n')
f.write(defconfig)
f.write('\n')
for result in results_list:
f.write(' %s %ss boot-test: %s\n' % (result['device_type'], result['kernel_boot_time'], result['result']))
# dt-self-test
if results and kernel_tree and kernel_version and dt_tests:
print 'Creating device tree runtime self test summary for %s' % kernel_version
dt_self_test = '%s-dt-runtime-self-test-report.txt' % kernel_version
passed = 0
failed = 0
for defconfig, results_list in results.items():
for result in results_list:
if result['dt_test_result'] == 'PASS':
passed += 1
elif result['dt_test_result'] == 'FAIL':
failed += 1
total = passed + failed
with open(os.path.join(report_directory, dt_self_test), 'a') as f:
f.write('To: %s\n' % config.get("email"))
f.write('From: [email protected]\n')
f.write('Subject: %s dt-runtime-unit-tests: %s boards tested: %s passed, %s failed (%s)\n' % (kernel_tree,
str(total),
str(passed),
str(failed),
kernel_version))
f.write('\n')
f.write('Full Build Report: http://kernelci.org/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
f.write('Full Boot Report: http://kernelci.org/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
f.write('Full Test Report: http://kernelci.org/test/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
f.write('\n')
f.write('Tree/Branch: %s\n' % kernel_tree)
f.write('Git Describe: %s\n' % kernel_version)
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['dt_test_result'] == 'FAIL':
if first:
f.write('\n')
f.write('Failed Device Tree Unit Tests:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['dt_test_result'] == "FAIL":
f.write(' %s passed: %s / failed: %s dt-runtime-unit-tests: %s\n' % (result['device_type'],
result['dt_tests_passed'],
result['dt_tests_failed'],
result['dt_test_result']))
if config.get("lab"):
f.write(' http://storage.kernelci.org/kernel-ci/%s/%s/%s/%s/boot-%s.html' % (kernel_tree,
kernel_version,
defconfig,
config.get("lab"),
result['device_type']))
else:
f.write(' http://storage.kernelci.org/kernel-ci/%s/%s/%s/boot-%s.html' % (kernel_tree,
kernel_version,
defconfig,
result['device_type']))
f.write('\n')
f.write('\n')
f.write('Full Unit Test Report:\n')
for defconfig, results_list in results.items():
first = True
for result in results_list:
if result['dt_test_result']:
if first:
f.write('\n')
f.write(defconfig)
f.write('\n')
first = False
f.write(' %s passed: %s / failed: %s dt-runtime-unit-tests: %s\n' % (result['device_type'],
result['dt_tests_passed'],
result['dt_tests_failed'],
result['dt_test_result']))
# sendmail
if config.get("email"):
print 'Sending e-mail summary to %s' % config.get("email")
if os.path.exists(report_directory):
cmd = 'cat %s | sendmail -t' % os.path.join(report_directory, boot)
subprocess.check_output(cmd, shell=True)
if dt_tests:
if os.path.exists(report_directory):
cmd = 'cat %s | sendmail -t' % os.path.join(report_directory, dt_self_test)
subprocess.check_output(cmd, shell=True)
# labels = spectral_clustering(A, n_clusters=4)
# ret = SpectralClustering(n_clusters=4).fit(points)
ret = KMeans(n_clusters=4).fit(points)
labels = ret.labels_
print('Reducing dimensions ...')
d_dealers_2_group = {}
for i, v in enumerate(points):
d_dealers_2_group[origin_l_dealers[i][0]] = int(labels[i])
print(f'label: {labels[i]}, points: {origin_l_dealers[i][:-2]}')
group_type = 'L-means_filter_lower_5'
utils.write_json(
os.path.join(path.ROOT_DIR, 'group', f'group_{group_type}.json'),
d_dealers_2_group)
visual_points = ReduceDim.tsne(points, len(points), n_components=2)
# visual_points = ReduceDim.pca(points, 2)
print('Plotting ...')
X, Y = list(zip(*visual_points))
Visual.spots(
X,
Y,
labels,
f'{group_type} group dealers (t-sne visualization)',
spot_size=3,
save_path=f'D:\Github\\bond_prediction\group\group_{group_type}.png')
def boot_report(config):
connection, jobs, duration = parse_json(config.get("boot"))
# TODO: Fix this when multi-lab sync is working
#download_log2html(log2html)
results_directory = os.getcwd() + '/results'
results = {}
dt_tests = False
utils.mkdir(results_directory)
for job_id in jobs:
print 'Job ID: %s' % job_id
# Init
boot_meta = {}
api_url = None
arch = None
board_instance = None
boot_retries = 0
kernel_defconfig = None
kernel_defconfig_base = None
kernel_version = None
device_tree = None
kernel_endian = None
kernel_tree = None
git_branch = None
kernel_addr = None
initrd_addr = None
dtb_addr = None
dtb_append = None
fastboot = None
fastboot_cmd = None
test_plan = None
job_file = ''
dt_test = None
dt_test_result = None
dt_tests_passed = None
dt_tests_failed = None
board_offline = False
kernel_boot_time = None
boot_failure_reason = None
efi_rtc = False
# Retrieve job details
job_details = connection.scheduler.job_details(job_id)
if job_details['requested_device_type_id']:
device_type = job_details['requested_device_type_id']
if job_details['description']:
job_name = job_details['description']
result = jobs[job_id]['result']
bundle = jobs[job_id]['bundle']
if bundle is None and device_type == 'dynamic-vm':
host_job_id = job_id.replace('.1', '.0')
bundle = jobs[host_job_id]['bundle']
if bundle is None:
print '%s bundle is empty, skipping...' % device_type
continue
# Retrieve the log file
try:
binary_job_file = connection.scheduler.job_output(job_id)
except xmlrpclib.Fault:
print 'Job output not found for %s' % device_type
continue
# Parse LAVA messages out of log
raw_job_file = str(binary_job_file)
for line in raw_job_file.splitlines():
if 'Infrastructure Error:' in line:
print 'Infrastructure Error detected!'
index = line.find('Infrastructure Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Bootloader Error:' in line:
print 'Bootloader Error detected!'
index = line.find('Bootloader Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Kernel Error:' in line:
print 'Kernel Error detected!'
index = line.find('Kernel Error:')
boot_failure_reason = line[index:]
if 'Userspace Error:' in line:
print 'Userspace Error detected!'
index = line.find('Userspace Error:')
boot_failure_reason = line[index:]
if '<LAVA_DISPATCHER>' not in line:
if len(line) != 0:
job_file += line + '\n'
if '### dt-test ### end of selftest' in line:
dt_tests = True
regex = re.compile("(?P<test>\d+\*?)")
dt_test_results = regex.findall(line)
if len(dt_test_results) > 2:
dt_tests_passed = dt_test_results[2]
dt_tests_failed = dt_test_results[3]
else:
dt_tests_passed = dt_test_results[0]
dt_tests_failed = dt_test_results[1]
if int(dt_tests_failed) > 0:
dt_test_result = 'FAIL'
else:
dt_test_result = 'PASS'
if 'rtc-efi rtc-efi: setting system clock to' in line:
if device_type == 'dynamic-vm':
efi_rtc = True
# Retrieve bundle
if bundle is not None:
json_bundle = connection.dashboard.get(bundle)
bundle_data = json.loads(json_bundle['content'])
# Get the boot data from LAVA
for test_results in bundle_data['test_runs']:
# Check for the LAVA self boot test
if test_results['test_id'] == 'lava':
for test in test_results['test_results']:
# TODO for compat :(
if test['test_case_id'] == 'kernel_boot_time':
kernel_boot_time = test['measurement']
if test['test_case_id'] == 'test_kernel_boot_time':
kernel_boot_time = test['measurement']
bundle_attributes = bundle_data['test_runs'][-1][
'attributes']
if utils.in_bundle_attributes(bundle_attributes,
'kernel.defconfig'):
print bundle_attributes['kernel.defconfig']
if utils.in_bundle_attributes(bundle_attributes, 'target'):
board_instance = bundle_attributes['target']
if utils.in_bundle_attributes(bundle_attributes,
'kernel.defconfig'):
kernel_defconfig = bundle_attributes['kernel.defconfig']
kernel_defconfig_base = ''.join(
kernel_defconfig.split('+')[:1])
if utils.in_bundle_attributes(bundle_attributes, 'arch'):
arch = bundle_attributes['arch']
if utils.in_bundle_attributes(bundle_attributes,
'kernel.describe'):
kernel_version = bundle_attributes['kernel.describe']
if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
device_tree = bundle_attributes['device.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
kernel_endian = bundle_attributes['kernel.endian']
if utils.in_bundle_attributes(bundle_attributes,
'platform.fastboot'):
fastboot = bundle_attributes['platform.fastboot']
if kernel_boot_time is None:
if utils.in_bundle_attributes(bundle_attributes,
'kernel-boot-time'):
kernel_boot_time = bundle_attributes['kernel-boot-time']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
kernel_tree = bundle_attributes['kernel.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
kernel_addr = bundle_attributes['kernel-addr']
if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
initrd_addr = bundle_attributes['initrd-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
dtb_addr = bundle_attributes['dtb-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
dtb_append = bundle_attributes['dtb-append']
if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
boot_retries = int(bundle_attributes['boot_retries'])
if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
test_plan = bundle_attributes['test.plan']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.branch'):
git_branch = bundle_attributes['kernel.branch']
# Check if we found efi-rtc
if test_plan == 'boot-kvm-uefi' and not efi_rtc:
if device_type == 'dynamic-vm':
boot_failure_reason = 'Unable to read EFI rtc'
result = 'FAIL'
# Record the boot log and result
# TODO: Will need to map device_types to dashboard device types
if kernel_defconfig and device_type and result:
if (arch == 'arm' or arch == 'arm64') and device_tree is None:
platform_name = device_map[device_type][0] + ',legacy'
else:
if device_tree == 'vexpress-v2p-ca15_a7.dtb':
platform_name = 'vexpress-v2p-ca15_a7'
elif device_tree == 'fsl-ls2080a-simu.dtb':
platform_name = 'fsl-ls2080a-simu'
elif test_plan == 'boot-kvm' or test_plan == 'boot-kvm-uefi':
if device_tree == 'sun7i-a20-cubietruck.dtb':
if device_type == 'dynamic-vm':
device_type = 'cubieboard3-kvm-guest'
platform_name = device_map[device_type][0]
else:
device_type = 'cubieboard3-kvm-host'
platform_name = device_map[device_type][0]
elif device_tree == 'apm-mustang.dtb':
if device_type == 'dynamic-vm':
if test_plan == 'boot-kvm-uefi':
device_type = 'mustang-kvm-uefi-guest'
else:
device_type = 'mustang-kvm-guest'
platform_name = device_map[device_type][0]
else:
if test_plan == 'boot-kvm-uefi':
device_type = 'mustang-kvm-uefi-host'
else:
device_type = 'mustang-kvm-host'
platform_name = device_map[device_type][0]
elif device_tree == 'juno.dtb':
if device_type == 'dynamic-vm':
if test_plan == 'boot-kvm-uefi':
device_type = 'juno-kvm-uefi-guest'
else:
device_type = 'juno-kvm-guest'
platform_name = device_map[device_type][0]
else:
if test_plan == 'boot-kvm-uefi':
device_type = 'juno-kvm-uefi-host'
else:
device_type = 'juno-kvm-host'
platform_name = device_map[device_type][0]
elif test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
platform_name = device_map[device_type][0] + '_rootfs:nfs'
else:
platform_name = device_map[device_type][0]
print 'Creating boot log for %s' % platform_name
log = 'boot-%s.txt' % platform_name
html = 'boot-%s.html' % platform_name
if config.get("lab"):
directory = os.path.join(
results_directory,
kernel_defconfig + '/' + config.get("lab"))
else:
directory = os.path.join(results_directory, kernel_defconfig)
utils.ensure_dir(directory)
utils.write_file(job_file, log, directory)
if kernel_boot_time is None:
kernel_boot_time = '0.0'
if results.has_key(kernel_defconfig):
results[kernel_defconfig].append({
'device_type': platform_name,
'dt_test_result': dt_test_result,
'dt_tests_passed': dt_tests_passed,
'dt_tests_failed': dt_tests_failed,
'kernel_boot_time': kernel_boot_time,
'result': result
})
else:
results[kernel_defconfig] = [{
'device_type': platform_name,
'dt_test_result': dt_test_result,
'dt_tests_passed': dt_tests_passed,
'dt_tests_failed': dt_tests_failed,
'kernel_boot_time': kernel_boot_time,
'result': result
}]
# Create JSON format boot metadata
print 'Creating JSON format boot metadata'
if config.get("lab"):
boot_meta['lab_name'] = config.get("lab")
else:
boot_meta['lab_name'] = None
if board_instance:
boot_meta['board_instance'] = board_instance
boot_meta['retries'] = boot_retries
boot_meta['boot_log'] = log
boot_meta['boot_log_html'] = html
# TODO: Fix this
boot_meta['version'] = '1.1'
boot_meta['arch'] = arch
boot_meta['defconfig'] = kernel_defconfig_base
boot_meta['defconfig_full'] = kernel_defconfig
if device_map[device_type][1]:
boot_meta['mach'] = device_map[device_type][1]
boot_meta['kernel'] = kernel_version
boot_meta['git_branch'] = git_branch
boot_meta['job'] = kernel_tree
boot_meta['board'] = platform_name
if board_offline and result == 'FAIL':
boot_meta['boot_result'] = 'OFFLINE'
#results[kernel_defconfig]['result'] = 'OFFLINE'
else:
boot_meta['boot_result'] = result
if result == 'FAIL' or result == 'OFFLINE':
if boot_failure_reason:
boot_meta['boot_result_description'] = boot_failure_reason
else:
boot_meta[
'boot_result_description'] = 'Unknown Error: platform failed to boot'
boot_meta['boot_time'] = kernel_boot_time
# TODO: Fix this
boot_meta['boot_warnings'] = None
if device_tree:
if arch == 'arm64':
boot_meta['dtb'] = 'dtbs/' + device_map[device_type][
1] + '/' + device_tree
else:
boot_meta['dtb'] = 'dtbs/' + device_tree
else:
boot_meta['dtb'] = device_tree
boot_meta['dtb_addr'] = dtb_addr
boot_meta['dtb_append'] = dtb_append
boot_meta['dt_test'] = dt_test
boot_meta['endian'] = kernel_endian
boot_meta['fastboot'] = fastboot
# TODO: Fix this
boot_meta['initrd'] = None
boot_meta['initrd_addr'] = initrd_addr
if arch == 'arm':
boot_meta['kernel_image'] = 'zImage'
elif arch == 'arm64':
boot_meta['kernel_image'] = 'Image'
else:
boot_meta['kernel_image'] = 'bzImage'
boot_meta['loadaddr'] = kernel_addr
json_file = 'boot-%s.json' % platform_name
utils.write_json(json_file, directory, boot_meta)
print 'Creating html version of boot log for %s' % platform_name
cmd = 'python log2html.py %s' % os.path.join(directory, log)
subprocess.check_output(cmd, shell=True)
if config.get("lab") and config.get("api") and config.get("token"):
print 'Sending boot result to %s for %s' % (config.get("api"),
platform_name)
headers = {
'Authorization': config.get("token"),
'Content-Type': 'application/json'
}
api_url = urlparse.urljoin(config.get("api"), '/boot')
push('POST',
api_url,
data=json.dumps(boot_meta),
headers=headers)
headers = {
'Authorization': config.get("token"),
}
print 'Uploading text version of boot log'
with open(os.path.join(directory, log)) as lh:
data = lh.read()
api_url = urlparse.urljoin(
config.get("api"), '/upload/%s/%s/%s/%s/%s/%s/%s' %
(kernel_tree, git_branch, kernel_version, arch,
kernel_defconfig, config.get("lab"), log))
push('PUT', api_url, data=data, headers=headers)
print 'Uploading html version of boot log'
with open(os.path.join(directory, html)) as lh:
data = lh.read()
api_url = urlparse.urljoin(
config.get("api"), '/upload/%s/%s/%s/%s/%s/%s/%s' %
(kernel_tree, git_branch, kernel_version, arch,
kernel_defconfig, config.get("lab"), html))
push('PUT', api_url, data=data, headers=headers)
from lib.utils import load_json, write_json
l = load_json(path.TOPIC_BONDS_LIST_JSON)
topic_bonds = set()
threshold = 0.3
for bonds in l:
percentage = 0.
for i, v in enumerate(bonds):
bond_id = v[0]
weight = v[1]
if weight < 0.1:
break
percentage += weight
topic_bonds.add(bond_id)
if percentage >= 0.5:
break
topic_bonds = list(topic_bonds)
topic_bonds.sort()
write_json(path.TOPIC_BONDS_JSON, topic_bonds)
print(len(topic_bonds))
print(topic_bonds)
print('done')
def boot_report(config):
connection, jobs, duration = parse_yaml(config.get("boot"))
# TODO: Fix this when multi-lab sync is working
results_directory = os.getcwd() + '/results'
results = {}
utils.mkdir(results_directory)
test_plan = None
if config.get("lab"):
report_directory = os.path.join(results_directory, config.get("lab"))
else:
report_directory = results_directory
if os.path.exists(report_directory):
shutil.rmtree(report_directory)
utils.mkdir(report_directory)
for job_id in jobs:
print 'Job ID: %s' % job_id
# Init
boot_meta = {}
arch = None
board_instance = None
boot_retries = 0
kernel_defconfig_full = None
kernel_defconfig = None
kernel_defconfig_base = None
kernel_version = None
device_tree = None
kernel_tree = None
kernel_addr = None
initrd_addr = None
dtb_addr = None
dtb_append = None
job_file = ''
board_offline = False
kernel_boot_time = None
boot_failure_reason = None
efi_rtc = False
# Retrieve job details
device_type = ''
job_details = connection.scheduler.job_details(job_id)
if job_details['requested_device_type_id']:
device_type = job_details['requested_device_type_id']
if job_details['description']:
job_name = job_details['description']
try:
job_short_name = re.search(".*?([A-Z]+.*)", job_name).group(1)
except Exception:
job_short_name = 'boot-test'
try:
device_name = job_details['_actual_device_cache']['hostname']
except Exception:
continue
result = jobs[job_id]['result']
bundle = jobs[job_id]['bundle']
if not device_type:
device_type = job_details['_actual_device_cache']['device_type_id']
try:
binary_job_file = connection.scheduler.job_output(job_id)
except xmlrpclib.Fault:
print 'Job output not found for %s' % device_type
continue
# Parse LAVA messages out of log
raw_job_file = str(binary_job_file)
for line in raw_job_file.splitlines():
if 'Infrastructure Error:' in line:
print 'Infrastructure Error detected!'
index = line.find('Infrastructure Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Bootloader Error:' in line:
print 'Bootloader Error detected!'
index = line.find('Bootloader Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Kernel Error:' in line:
print 'Kernel Error detected!'
index = line.find('Kernel Error:')
boot_failure_reason = line[index:]
if 'Userspace Error:' in line:
print 'Userspace Error detected!'
index = line.find('Userspace Error:')
boot_failure_reason = line[index:]
if '<LAVA_DISPATCHER>' not in line:
if len(line) != 0:
job_file += line + '\n'
if 'rtc-efi rtc-efi: setting system clock to' in line:
if device_type == 'dynamic-vm':
efi_rtc = True
if not kernel_defconfig or not kernel_version or not kernel_tree:
try:
job_metadata_info = connection.results.get_testjob_metadata(
job_id)
kernel_defconfig = utils.get_value_by_key(
job_metadata_info, 'kernel_defconfig')
kernel_version = utils.get_value_by_key(
job_metadata_info, 'kernel_version')
kernel_tree = utils.get_value_by_key(job_metadata_info,
'kernel_tree')
device_tree = utils.get_value_by_key(job_metadata_info,
'device_tree')
except Exception:
continue
# Record the boot log and result
# TODO: Will need to map device_types to dashboard device types
if kernel_defconfig and device_type and result:
if ('arm' == arch or 'arm64' == arch) and device_tree is None:
platform_name = device_map[device_type][0] + ',legacy'
else:
if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
platform_name = device_map[device_type][0] + '_rootfs:nfs'
else:
platform_name = device_map[device_type][0]
# Create txt format boot metadata
print 'Creating boot log for %s' % (platform_name + job_name +
'_' + job_id)
log = 'boot-%s.txt' % (platform_name + job_name + '_' + job_id)
if config.get("lab"):
directory = os.path.join(
results_directory,
kernel_defconfig + '/' + config.get("lab"))
else:
directory = os.path.join(results_directory, kernel_defconfig)
utils.ensure_dir(directory)
utils.write_file(job_file, log, directory)
if kernel_boot_time is None:
kernel_boot_time = '0.0'
if results.has_key(kernel_defconfig):
results[kernel_defconfig].append({
'device_type': platform_name,
'job_id': job_id,
'job_name': job_short_name,
'kernel_boot_time': kernel_boot_time,
'result': result,
'device_name': device_name
})
else:
results[kernel_defconfig] = [{
'device_type': platform_name,
'job_id': job_id,
'job_name': job_short_name,
'kernel_boot_time': kernel_boot_time,
'result': result,
'device_name': device_name
}]
# Create JSON format boot metadata
print 'Creating JSON format boot metadata'
if config.get("lab"):
boot_meta['lab_name'] = config.get("lab")
else:
boot_meta['lab_name'] = None
if board_instance:
boot_meta['board_instance'] = board_instance
boot_meta['retries'] = boot_retries
boot_meta['boot_log'] = log
# TODO: Fix this
boot_meta['version'] = '1.0'
boot_meta['arch'] = arch
boot_meta['defconfig'] = kernel_defconfig_base
if kernel_defconfig_full is not None:
boot_meta['defconfig_full'] = kernel_defconfig_full
if device_map[device_type][1]:
boot_meta['mach'] = device_map[device_type][1]
boot_meta['kernel'] = kernel_version
boot_meta['job'] = kernel_tree
boot_meta['board'] = platform_name
if board_offline and result == 'FAIL':
boot_meta['boot_result'] = 'OFFLINE'
#results[kernel_defconfig]['result'] = 'OFFLINE'
else:
boot_meta['boot_result'] = result
if result == 'FAIL' or result == 'OFFLINE':
if boot_failure_reason:
boot_meta['boot_result_description'] = boot_failure_reason
else:
boot_meta[
'boot_result_description'] = 'Unknown Error: platform failed to boot'
boot_meta['boot_time'] = kernel_boot_time
# TODO: Fix this
boot_meta['boot_warnings'] = None
if device_tree:
if arch == 'arm64':
boot_meta['dtb'] = 'dtbs/' + device_map[device_type][
1] + '/' + device_tree
else:
boot_meta['dtb'] = 'dtbs/' + device_tree
else:
boot_meta['dtb'] = device_tree
boot_meta['dtb_addr'] = dtb_addr
boot_meta['dtb_append'] = dtb_append
# TODO: Fix this
boot_meta['initrd'] = None
boot_meta['initrd_addr'] = initrd_addr
if arch == 'arm':
boot_meta['kernel_image'] = 'zImage'
elif arch == 'arm64':
boot_meta['kernel_image'] = 'Image'
else:
boot_meta['kernel_image'] = 'bzImage'
boot_meta['loadaddr'] = kernel_addr
json_file = 'boot-%s.json' % (platform_name + job_name + '_' +
job_id)
utils.write_json(json_file, directory, boot_meta)
# add by wuyanjun
parser_and_get_result(job_file, log, directory, report_directory,
connection)
#try to generate test_summary
generate_test_report(job_id, connection)
if results and kernel_tree and kernel_version:
print 'Creating summary for %s' % (kernel_version)
boot = '%s-boot-report.txt' % (kernel_version)
if test_plan and ('boot' in test_plan or 'BOOT' in test_plan):
boot = boot.replace('boot', test_plan)
passed = 0
failed = 0
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'PASS':
passed += 1
else:
failed += 1
total = passed + failed
with open(os.path.join(report_directory, boot), 'a') as f:
f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' %
(kernel_tree, str(total), str(passed), str(failed),
kernel_version))
f.write('\n')
f.write('Total Duration: %.2f minutes\n' % (duration / 60))
f.write('Tree/Branch: %s\n' % kernel_tree)
f.write('Git Describe: %s\n' % kernel_version)
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'OFFLINE':
if first:
f.write('\n')
f.write('Boards Offline:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'OFFLINE':
f.write(
' %s %s %s %ss %s: %s\n' %
(result['job_id'], result['device_type'],
result['device_name'], result['kernel_boot_time'],
result['job_name'], result['result']))
f.write('\n')
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'FAIL':
if first:
f.write('\n')
f.write('Failed Boot Tests:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'FAIL':
f.write(
' %s %s %s %ss %s: %s\n' %
(result['job_id'], result['device_type'],
result['device_name'], result['kernel_boot_time'],
result['job_name'], result['result']))
f.write('\n')
f.write('Full Boot Report:\n')
for defconfig, results_list in results.items():
f.write('\n')
f.write(defconfig)
f.write('\n')
for result in results_list:
f.write(' %s %s %s %ss %s: %s\n' %
(result['job_id'], result['device_type'],
result['device_name'], result['kernel_boot_time'],
result['job_name'], result['result']))
data = list(map(decode_2_utf8, content))
data = list(map(lambda x: x.strip().split(','), data))
columns = data[0]
data = data[1:]
# for i, v in enumerate(columns):
# print(i, v, data[0][i], data[1][i])
index_issue_id = 0
index_offering_date = 32
dict_issue_id_offering_date = {}
for v in data:
issue_id = v[index_issue_id]
offering_date = v[index_offering_date]
if issue_id in dict_issue_id_offering_date:
print('------------------------')
print(issue_id, offering_date)
offering_date = f'{offering_date[:4]}-{offering_date[4:6]}-{offering_date[-2:]}'
dict_issue_id_offering_date[issue_id] = offering_date
save_path = os.path.join(path.DATA_ROOT_DIR, 'dict_issue_id_offering_date.json')
write_json(save_path, dict_issue_id_offering_date)
print('done')
mode = 0 if '_v_all' not in file_name else 1
# if mode == 1:
# continue
length = len(tmp_dict)
i = 0
for key, val in tmp_dict.items():
if i % 50 == 0:
progress = float(i + 1) / length * 100.
print('\rprogress: %.2f%% ' % progress, end='')
_merged_dict = __merge_dict(_merged_dict, key, val, mode)
i += 1
return _merged_dict
ro_en_dict = {}
en_ro_dict = {}
traverse_dict_and_merge(ro_en_dir, ro_en_dict)
traverse_dict_and_merge(en_ro_dir, en_ro_dict)
print('\nwriting data to files ...')
write_json(merged_ro_en_dict_path, ro_en_dict)
write_json(merged_en_ro_dict_path, en_ro_dict)
print('\ndone')
s = 0
for k, v in dictionary.items():
print(k, v['translation'])
s += 1
if s > 1000:
break
print('\n-----------------------------')
print(len(dictionary))
exit()
new_dict = {}
def __add_dict(_dict, k, v):
if k not in _dict:
_dict[k] = {'translation': []}
_dict[k]['translation'].append(v)
for key, val in dictionary.items():
translations = val['translation']
for translation in translations:
if translation not in new_dict:
__add_dict(new_dict, translation, key)
write_json(new_dict_path, new_dict)
print('done')
del zh_en_dict[zh]['translation']
if not __check_has_val(zh_en_dict[zh]):
delete_zh_keys.append(zh)
else:
zh_en_dict[zh]['translation'] = translations
print('filtering deleted keys for zh_en_dict ...')
for k in delete_zh_keys:
del zh_en_dict[k]
print('writing data to file for zh_en_dict ... ')
write_json(filtered_zh_en_dict_path, zh_en_dict)
del zh_en_dict
print('\nloading en_zh_dict ...')
en_zh_dict = load_json(merged_en_zh_dict_path)
en_zh_dict = filter_duplicate(en_zh_dict)
print('filtering en_zh_dict ...')
for en, val in en_zh_dict.items():
if 'translation' not in val:
continue
translations = val['translation']
# with open(path.ner_test_path, 'rb') as f:
with open(path.ner_train_path, 'rb') as f:
data = f.readlines()
data = list(map(lambda x: json.loads(x), data))
new_data = []
for i, val in enumerate(data):
clean_val = check_entities(val)
split_val = split_sentence(clean_val)
new_data.append(split_val)
print('saving data ... ')
utils.write_json(
utils.get_relative_file_path('ner', 'data', 'preprocessed',
'train_clean_split.json'), new_data)
data = reduce(lambda a, b: a + b, new_data)
entity_list = list(map(lambda x: x['entities'], data))
type_list = list(
map(lambda x: list(set(list(map(lambda a: a['type'], x)))), entity_list))
type_list = list(set(reduce(lambda a, b: a + b, type_list)))
type_list.sort()
print('\n---------------------------------------------------------------')
print(f'entity types: {type_list}')
print(f'len entity types: {len(type_list)}')
def generate_doc_list(dir_path, new_dir_path, id_index=0):
"""
Generate the doc for each dealer in each date
there is only one feature (the bond itself) and there is no filter
"""
print('\nstart generating doc list from %s to %s ...' %
(dir_path, new_dir_path))
file_list = os.listdir(dir_path)
len_file_list = len(file_list)
for i, file_name in enumerate(file_list):
if i % 5 == 0:
progress = float(i + 1) / len_file_list * 100.
print('\rprogress: %.2f%% ' % progress, end='')
file_name_prefix = os.path.splitext(file_name)[0]
# load bonds data
file_path = os.path.join(dir_path, file_name)
data = utils.load_json(file_path)
# to fill the date gaps (because there are dates that no bond transactions happen)
d_date = {}
# parse the date to timestamp
start_date = str(data[0][-2])[:-9] + ' 00:00:00'
end_date = str(data[-1][-2])[:-9] + ' 23:00:00'
cur_time_stamp = time.mktime(
time.strptime(start_date, '%Y-%m-%d %H:%M:%S'))
end_time_stamp = time.mktime(
time.strptime(end_date, '%Y-%m-%d %H:%M:%S'))
# traverse the dates and fill the gaps
while cur_time_stamp <= end_time_stamp:
date_time = time.localtime(cur_time_stamp)
date = time.strftime('%Y-%m-%d', date_time)
cur_time_stamp += 86400
d_date[date] = []
# fill the bonds to the d_date dict
for v in data:
_date_time = str(v[-2])
_date = _date_time[:-9]
if id_index == 0:
_id = v[id_index]
val = [v[0], v[-1], _date_time]
else:
val = [v[1], v[2], v[-1], _date_time]
d_date[_date].append(val)
# convert the d_date (dict) to list
l_date = list(d_date.items())
l_date.sort(key=lambda x: x[0])
while l_date[-1][0][:4] != l_date[0][0][:4]:
del l_date[-1]
dealer_dir = os.path.join(new_dir_path, file_name_prefix)
if not os.path.isdir(dealer_dir):
os.mkdir(dealer_dir)
for _date, _data_in_a_day in l_date:
if not _data_in_a_day:
continue
_data_in_a_day.sort(key=lambda x: x[-1])
_data_in_a_day = list(map(lambda x: x[:-1], _data_in_a_day))
utils.write_json(os.path.join(dealer_dir, f'doc_{_date}.json'),
_data_in_a_day)
print('\nfinish generating ')
elif 'tar_' in k:
k = k.replace('tar_', 'src_')
new_info[k] = v
for ro in translations:
if ro not in __ro_en_dict:
__ro_en_dict[ro] = new_info
else:
ro_info = __ro_en_dict[ro]
if 'translation' not in ro_info or not ro_info['translation']:
__ro_en_dict[ro]['translation'] = [en]
elif en not in ro_info['translation'] and len(
ro_info['translation']) < 5:
en = en.replace(',', '')
__ro_en_dict[ro]['translation'].append(en)
print('filtering duplicate ...')
__ro_en_dict = filter_duplicate(__ro_en_dict)
__en_ro_dict = filter_duplicate(__en_ro_dict)
print('writing data to files ... ')
write_json(filtered_union_ro_en_dict_path, __ro_en_dict)
write_json(filtered_union_en_ro_dict_path, __en_ro_dict)
print('\ndone')
def boot_report(config):
connection, jobs, duration = parse_json(config.get("boot"))
# TODO: Fix this when multi-lab sync is working
#download_log2html(log2html)
results_directory = os.getcwd() + '/results'
results = {}
dt_tests = False
utils.mkdir(results_directory)
for job_id in jobs:
print 'Job ID: %s' % job_id
# Init
boot_meta = {}
api_url = None
arch = None
board_instance = None
boot_retries = 0
kernel_defconfig_full = None
kernel_defconfig = None
kernel_defconfig_base = None
kernel_version = None
device_tree = None
kernel_endian = None
kernel_tree = None
kernel_addr = None
initrd_addr = None
dtb_addr = None
dtb_append = None
fastboot = None
fastboot_cmd = None
test_plan = None
job_file = ''
dt_test = None
dt_test_result = None
dt_tests_passed = None
dt_tests_failed = None
board_offline = False
kernel_boot_time = None
boot_failure_reason = None
efi_rtc = False
# Retrieve job details
device_type = ''
job_details = connection.scheduler.job_details(job_id)
if job_details['requested_device_type_id']:
device_type = job_details['requested_device_type_id']
if job_details['description']:
job_name = job_details['description']
device_name = job_details['_actual_device_cache']['hostname']
result = jobs[job_id]['result']
bundle = jobs[job_id]['bundle']
if not device_type:
device_type = job_details['_actual_device_cache']['device_type_id']
if bundle is None and device_type == 'dynamic-vm':
host_job_id = job_id.replace('.1', '.0')
bundle = jobs[host_job_id]['bundle']
if bundle is None:
print '%s bundle is empty, skipping...' % device_type
continue
# Retrieve the log file
try:
binary_job_file = connection.scheduler.job_output(job_id)
except xmlrpclib.Fault:
print 'Job output not found for %s' % device_type
continue
# Parse LAVA messages out of log
raw_job_file = str(binary_job_file)
for line in raw_job_file.splitlines():
if 'Infrastructure Error:' in line:
print 'Infrastructure Error detected!'
index = line.find('Infrastructure Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Bootloader Error:' in line:
print 'Bootloader Error detected!'
index = line.find('Bootloader Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Kernel Error:' in line:
print 'Kernel Error detected!'
index = line.find('Kernel Error:')
boot_failure_reason = line[index:]
if 'Userspace Error:' in line:
print 'Userspace Error detected!'
index = line.find('Userspace Error:')
boot_failure_reason = line[index:]
if '<LAVA_DISPATCHER>' not in line:
if len(line) != 0:
job_file += line + '\n'
if '### dt-test ### end of selftest' in line:
dt_tests = True
regex = re.compile("(?P<test>\d+\*?)")
dt_test_results = regex.findall(line)
if len(dt_test_results) > 2:
dt_tests_passed = dt_test_results[2]
dt_tests_failed = dt_test_results[3]
else:
dt_tests_passed = dt_test_results[0]
dt_tests_failed = dt_test_results[1]
if int(dt_tests_failed) > 0:
dt_test_result = 'FAIL'
else:
dt_test_result = 'PASS'
if 'rtc-efi rtc-efi: setting system clock to' in line:
if device_type == 'dynamic-vm':
efi_rtc = True
# Retrieve bundle
if bundle is not None:
json_bundle = connection.dashboard.get(bundle)
bundle_data = json.loads(json_bundle['content'])
# Get the boot data from LAVA
for test_results in bundle_data['test_runs']:
# Check for the LAVA self boot test
if test_results['test_id'] == 'lava':
for test in test_results['test_results']:
# TODO for compat :(
if test['test_case_id'] == 'kernel_boot_time':
kernel_boot_time = test['measurement']
if test['test_case_id'] == 'test_kernel_boot_time':
kernel_boot_time = test['measurement']
bundle_attributes = bundle_data['test_runs'][-1][
'attributes']
if utils.in_bundle_attributes(bundle_attributes,
'kernel.defconfig'):
print bundle_attributes['kernel.defconfig']
if utils.in_bundle_attributes(bundle_attributes, 'target'):
board_instance = bundle_attributes['target']
if utils.in_bundle_attributes(bundle_attributes,
'kernel.defconfig'):
kernel_defconfig = bundle_attributes['kernel.defconfig']
defconfig_list = kernel_defconfig.split('-')
#arch = defconfig_list[0]
arch = defconfig_list[-1]
# Remove arch
defconfig_list.pop(0)
kernel_defconfig_full = '-'.join(defconfig_list)
kernel_defconfig_base = ''.join(
kernel_defconfig_full.split('+')[:1])
if kernel_defconfig_full == kernel_defconfig_base:
kernel_defconfig_full = None
if utils.in_bundle_attributes(bundle_attributes, 'kernel.version'):
kernel_version = bundle_attributes['kernel.version']
if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
device_tree = bundle_attributes['device.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
kernel_endian = bundle_attributes['kernel.endian']
if utils.in_bundle_attributes(bundle_attributes,
'platform.fastboot'):
fastboot = bundle_attributes['platform.fastboot']
if kernel_boot_time is None:
if utils.in_bundle_attributes(bundle_attributes,
'kernel-boot-time'):
kernel_boot_time = bundle_attributes['kernel-boot-time']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
kernel_tree = bundle_attributes['kernel.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
kernel_addr = bundle_attributes['kernel-addr']
if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
initrd_addr = bundle_attributes['initrd-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
dtb_addr = bundle_attributes['dtb-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
dtb_append = bundle_attributes['dtb-append']
if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
boot_retries = int(bundle_attributes['boot_retries'])
if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
test_plan = bundle_attributes['test.plan']
# Check if we found efi-rtc
if test_plan == 'boot-kvm-uefi' and not efi_rtc:
if device_type == 'dynamic-vm':
boot_failure_reason = 'Unable to read EFI rtc'
result = 'FAIL'
# Record the boot log and result
# TODO: Will need to map device_types to dashboard device types
if kernel_defconfig and device_type and result:
if ('arm' == arch or 'arm64' == arch) and device_tree is None:
platform_name = device_map[device_type][0] + ',legacy'
else:
if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
platform_name = device_map[device_type][0] + '_rootfs:nfs'
else:
platform_name = device_map[device_type][0]
print 'Creating boot log for %s' % platform_name
log = 'boot-%s.txt' % (platform_name + job_name)
html = 'boot-%s.html' % (platform_name + job_name)
if config.get("lab"):
directory = os.path.join(
results_directory,
kernel_defconfig + '/' + config.get("lab"))
else:
directory = os.path.join(results_directory, kernel_defconfig)
utils.ensure_dir(directory)
utils.write_file(job_file, log, directory)
if kernel_boot_time is None:
kernel_boot_time = '0.0'
if results.has_key(kernel_defconfig):
results[kernel_defconfig].append({
'device_type': platform_name,
'dt_test_result': dt_test_result,
'dt_tests_passed': dt_tests_passed,
'dt_tests_failed': dt_tests_failed,
'kernel_boot_time': kernel_boot_time,
'result': result,
'device_name': device_name
})
else:
results[kernel_defconfig] = [{
'device_type': platform_name,
'dt_test_result': dt_test_result,
'dt_tests_passed': dt_tests_passed,
'dt_tests_failed': dt_tests_failed,
'kernel_boot_time': kernel_boot_time,
'result': result,
'device_name': device_name
}]
# Create JSON format boot metadata
print 'Creating JSON format boot metadata'
if config.get("lab"):
boot_meta['lab_name'] = config.get("lab")
else:
boot_meta['lab_name'] = None
if board_instance:
boot_meta['board_instance'] = board_instance
boot_meta['retries'] = boot_retries
boot_meta['boot_log'] = log
boot_meta['boot_log_html'] = html
# TODO: Fix this
boot_meta['version'] = '1.0'
boot_meta['arch'] = arch
boot_meta['defconfig'] = kernel_defconfig_base
if kernel_defconfig_full is not None:
boot_meta['defconfig_full'] = kernel_defconfig_full
if device_map[device_type][1]:
boot_meta['mach'] = device_map[device_type][1]
boot_meta['kernel'] = kernel_version
boot_meta['job'] = kernel_tree
boot_meta['board'] = platform_name
if board_offline and result == 'FAIL':
boot_meta['boot_result'] = 'OFFLINE'
#results[kernel_defconfig]['result'] = 'OFFLINE'
else:
boot_meta['boot_result'] = result
if result == 'FAIL' or result == 'OFFLINE':
if boot_failure_reason:
boot_meta['boot_result_description'] = boot_failure_reason
else:
boot_meta[
'boot_result_description'] = 'Unknown Error: platform failed to boot'
boot_meta['boot_time'] = kernel_boot_time
# TODO: Fix this
boot_meta['boot_warnings'] = None
if device_tree:
if arch == 'arm64':
boot_meta['dtb'] = 'dtbs/' + device_map[device_type][
1] + '/' + device_tree
else:
boot_meta['dtb'] = 'dtbs/' + device_tree
else:
boot_meta['dtb'] = device_tree
boot_meta['dtb_addr'] = dtb_addr
boot_meta['dtb_append'] = dtb_append
boot_meta['dt_test'] = dt_test
boot_meta['endian'] = kernel_endian
boot_meta['fastboot'] = fastboot
# TODO: Fix this
boot_meta['initrd'] = None
boot_meta['initrd_addr'] = initrd_addr
if arch == 'arm':
boot_meta['kernel_image'] = 'zImage'
elif arch == 'arm64':
boot_meta['kernel_image'] = 'Image'
else:
boot_meta['kernel_image'] = 'bzImage'
boot_meta['loadaddr'] = kernel_addr
json_file = 'boot-%s.json' % (platform_name + job_name)
utils.write_json(json_file, directory, boot_meta)
#print 'Creating html version of boot log for %s' % platform_name
#cmd = 'python log2html.py %s' % os.path.join(directory, log)
#subprocess.check_output(cmd, shell=True)
#if config.get("lab") and config.get("api") and config.get("token"):
# print 'Sending boot result to %s for %s' % (config.get("api"), platform_name)
# headers = {
# 'Authorization': config.get("token"),
# 'Content-Type': 'application/json'
# }
# api_url = urlparse.urljoin(config.get("api"), '/boot')
# push('POST', api_url, data=json.dumps(boot_meta), headers=headers)
# headers = {
# 'Authorization': config.get("token"),
# }
# print 'Uploading text version of boot log'
# with open(os.path.join(directory, log)) as lh:
# data = lh.read()
# api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s' % (kernel_tree,
# kernel_version,
# kernel_defconfig,
# config.get("lab"),
# log))
# push('PUT', api_url, data=data, headers=headers)
# print 'Uploading html version of boot log'
# with open(os.path.join(directory, html)) as lh:
# data = lh.read()
# api_url = urlparse.urljoin(config.get("api"), '/upload/%s/%s/%s/%s/%s' % (kernel_tree,
# kernel_version,
# kernel_defconfig,
# config.get("lab"),
# html))
# push('PUT', api_url, data=data, headers=headers)
if config.get("lab"):
report_directory = os.path.join(results_directory, config.get("lab"))
utils.mkdir(report_directory)
else:
report_directory = results_directory
if results and kernel_tree and kernel_version and 'boot' in test_plan or 'BOOT' in test_plan:
print 'Creating summary for %s' % (kernel_version)
boot = '%s-boot-report.txt' % (kernel_version)
boot = boot.replace('boot', test_plan)
passed = 0
failed = 0
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'PASS':
passed += 1
else:
failed += 1
total = passed + failed
with open(os.path.join(report_directory, boot), 'a') as f:
#f.write('To: %s\n' % config.get("email"))
#f.write('From: [email protected]\n')
f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' %
(kernel_tree, str(total), str(passed), str(failed),
kernel_version))
f.write('\n')
#f.write('Full Build Report: http://192.168.1.108:5000/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
#f.write('Full Boot Report: http://192.168.1.108:5000/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
#f.write('\n')
f.write('Total Duration: %.2f minutes\n' % (duration / 60))
f.write('Tree/Branch: %s\n' % kernel_tree)
f.write('Git Describe: %s\n' % kernel_version)
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'OFFLINE':
if first:
f.write('\n')
f.write('Boards Offline:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'OFFLINE':
f.write(' %s %s %ss boot-test: %s\n' %
(result['device_type'], result['device_name'],
result['kernel_boot_time'], result['result']))
f.write('\n')
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'FAIL':
if first:
f.write('\n')
f.write('Failed Boot Tests:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'FAIL':
f.write(' %s %s %ss boot-test: %s\n' %
(result['device_type'], result['device_name'],
result['kernel_boot_time'], result['result']))
if config.get("lab"):
f.write(
' http://192.168.1.108:8083/kernel-ci/%s/%s/%s/%s/boot-%s.html'
% (kernel_tree, kernel_version, defconfig,
config.get("lab"), result['device_type']))
else:
f.write(
' http://192.168.1.108:8083/kernel-ci/%s/%s/%s/boot-%s.html'
% (kernel_tree, kernel_version, defconfig,
result['device_type']))
f.write('\n')
f.write('\n')
f.write('Full Boot Report:\n')
for defconfig, results_list in results.items():
f.write('\n')
f.write(defconfig)
f.write('\n')
for result in results_list:
f.write(' %s %s %ss boot-test: %s\n' %
(result['device_type'], result['device_name'],
result['kernel_boot_time'], result['result']))
# add by wuyanjun
if results and directory:
parser_and_get_result(results, directory, report_directory)
#get_ip_board_mapping(results, directory, report_directory)
# dt-self-test
if results and kernel_tree and kernel_version and dt_tests:
print 'Creating device tree runtime self test summary for %s' % kernel_version
dt_self_test = '%s-dt-runtime-self-test-report.txt' % kernel_version
passed = 0
failed = 0
for defconfig, results_list in results.items():
for result in results_list:
if result['dt_test_result'] == 'PASS':
passed += 1
elif result['dt_test_result'] == 'FAIL':
failed += 1
total = passed + failed
with open(os.path.join(report_directory, dt_self_test), 'a') as f:
#f.write('To: %s\n' % config.get("email"))
#f.write('From: [email protected]\n')
f.write(
'Subject: %s dt-runtime-unit-tests: %s boards tested: %s passed, %s failed (%s)\n'
% (kernel_tree, str(total), str(passed), str(failed),
kernel_version))
f.write('\n')
#f.write('Full Build Report: http://192.168.1.108:5000/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
#f.write('Full Boot Report: http://192.168.1.108:5000/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
#f.write('Full Test Report: http://192.168.1.108:5000/test/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
#f.write('\n')
f.write('Tree/Branch: %s\n' % kernel_tree)
f.write('Git Describe: %s\n' % kernel_version)
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['dt_test_result'] == 'FAIL':
if first:
f.write('\n')
f.write('Failed Device Tree Unit Tests:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['dt_test_result'] == "FAIL":
f.write(
' %s %s passed: %s / failed: %s dt-runtime-unit-tests: %s\n'
% (result['device_type'], result['device_name'],
result['dt_tests_passed'],
result['dt_tests_failed'],
result['dt_test_result']))
if config.get("lab"):
f.write(
' http://192.168.1.108:8083/kernel-ci/%s/%s/%s/%s/boot-%s.html'
% (kernel_tree, kernel_version, defconfig,
config.get("lab"), result['device_type']))
else:
f.write(
' http://192.168.1.108:8083/kernel-ci/%s/%s/%s/boot-%s.html'
% (kernel_tree, kernel_version, defconfig,
result['device_type']))
f.write('\n')
f.write('\n')
f.write('Full Unit Test Report:\n')
for defconfig, results_list in results.items():
first = True
for result in results_list:
if result['dt_test_result']:
if first:
f.write('\n')
f.write(defconfig)
f.write('\n')
first = False
f.write(
' %s %s passed: %s / failed: %s dt-runtime-unit-tests: %s\n'
% (result['device_type'], result['device_name'],
result['dt_tests_passed'],
result['dt_tests_failed'],
result['dt_test_result']))
# sendmail
if config.get("email"):
print 'Sending e-mail summary to %s' % config.get("email")
if os.path.exists(report_directory):
cmd = 'cat %s | sendmail -t' % os.path.join(report_directory, boot)
subprocess.check_output(cmd, shell=True)
if dt_tests:
if os.path.exists(report_directory):
cmd = 'cat %s | sendmail -t' % os.path.join(
report_directory, dt_self_test)
subprocess.check_output(cmd, shell=True)
def generate_current_test_report():
print "generate_current_test_report"
suite_list = [] #all test suite list
# test suite data
test_suite_dict = {}
test_scope_dict = {}
# Statistics of each test suite
for job_id in job_result_dict.keys():
for item in job_result_dict[job_id]:
if item['suite'] not in test_suite_dict:
test_suite_dict[item['suite']] = {}
if item['result'] not in test_suite_dict[item['suite']]:
test_suite_dict[item['suite']][item['result']] = 1
else:
value = test_suite_dict[item['suite']][item['result']]
value = value + 1
test_suite_dict[item['suite']][item['result']] = value
else:
if item['result'] not in test_suite_dict[item['suite']]:
test_suite_dict[item['suite']][item['result']] = 1
else:
value = test_suite_dict[item['suite']][item['result']]
value = value + 1
test_suite_dict[item['suite']][item['result']] = value
print_base_info_pie_chart(test_suite_dict,
"Base Pass Rate Situation Chart")
# scope data
test_suite_scope_dict = {}
for job_id in job_result_dict.keys():
for item in job_result_dict[job_id]:
if 'metadata' in item:
metadata = item['metadata']
if 'path' in metadata and 'repository' in metadata:
count_scope_pass_number(test_suite_scope_dict,
metadata['path'], item['result'])
elif 'extra' in metadata:
path = ""
repository = ""
for extra in metadata['extra']:
if 'path' in extra:
path = extra['path']
continue
if 'repository' in extra:
repository = extra['repository']
continue
if path != "" and repository != "":
count_scope_pass_number(test_suite_scope_dict, path,
item['result'])
# print test_suite_scope_dict
print_scope_info_bar_chart(test_suite_scope_dict, "Pass Number Bar Chart")
create_test_report_pdf(job_result_dict)
current_test_result_dir = os.getcwd()
test_result_file = os.path.join(current_test_result_dir,
TEST_RESULT_FILE_NAME)
if os.path.exists(test_result_file):
os.remove(test_result_file)
utils.write_json(TEST_RESULT_FILE_NAME, current_test_result_dir,
job_result_dict)
del ro_en_dict[ro]['translation']
if not __check_has_val(ro_en_dict[ro]):
delete_ro_keys.append(ro)
else:
ro_en_dict[ro]['translation'] = translations
print('filtering deleted keys for zh_en_dict ...')
for k in delete_ro_keys:
del ro_en_dict[k]
print('writing data to file for zh_en_dict ... ')
write_json(filtered_ro_en_dict_path, ro_en_dict)
del ro_en_dict
print('\nloading en_zh_dict ...')
en_ro_dict = load_json(merged_en_ro_dict_path)
en_ro_dict = filter_duplicate(en_ro_dict)
print('filtering en_zh_dict ...')
for en, val in en_ro_dict.items():
if 'translation' not in val:
continue
translations = val['translation']
stem_dictionary = {}
len_stems = len(stem_words)
for i, (stem, word) in enumerate(stem_words):
if i % 100 == 0:
progress = float(i + 1) / len_stems * 100.
print('\rprogress: %.2f%% ' % progress, end='')
if stem not in stem_dictionary:
stem_dictionary[stem] = []
stem_dictionary[stem].append(word)
print('removing duplicate values ...')
for stem, words in stem_dictionary.items():
words = list(set(words))
stem_dictionary[stem] = words
print(f'\nwriting stem dictionary to file ...')
write_json(stem_dictionary_path, stem_dictionary)
print(f'\nlen_stem_dictionary: {len(stem_dictionary)}')
"""
The format of the stem dictionary is :
{
'like': ['likely', 'liking', 'alike'],
'cry': ['cries', 'crying'],
...
}
"""
None,
list(
map(
lambda x: x.name().split(".")[1] if
(word == x.name().split(".")[0]) else None, syns)))))
pos = list(filter(lambda x: x, pos))
src_meanings = list(filter(lambda x: x, src_meanings))
src_synonyms = list(filter(lambda x: x, src_synonyms))
if pos:
word_details["pos"] = pos
if src_meanings:
word_details["src_meanings"] = src_meanings
if src_synonyms:
word_details["src_synonyms"] = list(src_synonyms)
if not word_details:
continue
# convert coding and full 2 half
word = utils.process(word, utils.weak_pl)
dictionary[word] = word_details
write_json(write_dict_path, dictionary)
print('done')
lines = list(map(lambda x: x.strip().split('\t'), lines))
lines = list(filter(lambda x: x and len(x) == 2, lines))
if file_name[:2].lower() == 'ro':
for ro, en in lines:
ro = process(ro, pipeline)
en = process(en, pipeline)
__add_dict(ro_en_dict, ro, en)
__add_dict(en_ro_dict, en, ro)
else:
for en, ro in lines:
ro = process(ro, pipeline)
en = process(en, pipeline)
__add_dict(ro_en_dict, ro, en)
__add_dict(en_ro_dict, en, ro)
print('filtering duplicate ...')
ro_en_dict = filter_duplicate(ro_en_dict)
en_ro_dict = filter_duplicate(en_ro_dict)
print('writing dictionary to files ... ')
write_json(ro_en_dict_path, ro_en_dict)
write_json(en_ro_dict_path, en_ro_dict)
print('\ndone')
def update(self):
utils.write_json(self._data, self.file_path)
lines = list(map(lambda x: x.strip().split(' '), lines))
lines = list(filter(lambda x: x and len(x) == 2, lines))
if file_name[:2].lower() == 'zh':
for zh, en in lines:
zh = process(zh, pipeline)
en = process(en, pipeline)
__add_dict(zh_en_dict, zh, en)
__add_dict(en_zh_dict, en, zh)
else:
for en, zh in lines:
zh = process(zh, pipeline)
en = process(en, pipeline)
__add_dict(zh_en_dict, zh, en)
__add_dict(en_zh_dict, en, zh)
print('filtering duplicate ...')
zh_en_dict = filter_duplicate(zh_en_dict)
en_zh_dict = filter_duplicate(en_zh_dict)
print('writing dictionary to files ... ')
write_json(zh_en_dict_path, zh_en_dict)
write_json(en_zh_dict_path, en_zh_dict)
print('\ndone')
def boot_report(config):
connection, jobs, duration = parse_json(config.get("boot"))
# TODO: Fix this when multi-lab sync is working
#download_log2html(log2html)
results_directory = os.getcwd() + '/results'
results = {}
utils.mkdir(results_directory)
test_plan = None
for job_id in jobs:
print 'Job ID: %s' % job_id
# Init
boot_meta = {}
api_url = None
arch = None
board_instance = None
boot_retries = 0
kernel_defconfig_full = None
kernel_defconfig = None
kernel_defconfig_base = None
kernel_version = None
device_tree = None
kernel_endian = None
kernel_tree = None
kernel_addr = None
initrd_addr = None
dtb_addr = None
dtb_append = None
fastboot = None
fastboot_cmd = None
job_file = ''
board_offline = False
kernel_boot_time = None
boot_failure_reason = None
efi_rtc = False
# Retrieve job details
device_type = ''
job_details = connection.scheduler.job_details(job_id)
if job_details['requested_device_type_id']:
device_type = job_details['requested_device_type_id']
if job_details['description']:
job_name = job_details['description']
try:
job_short_name = re.search(".*?([A-Z]+.*)", job_name).group(1)
except Exception:
job_short_name = 'boot-test'
device_name = job_details['_actual_device_cache']['hostname']
result = jobs[job_id]['result']
bundle = jobs[job_id]['bundle']
if not device_type:
device_type = job_details['_actual_device_cache']['device_type_id']
if bundle is None and device_type == 'dynamic-vm':
host_job_id = job_id.replace('.1', '.0')
bundle = jobs[host_job_id]['bundle']
if bundle is None:
print '%s bundle is empty, skipping...' % device_type
continue
# Retrieve the log file
try:
binary_job_file = connection.scheduler.job_output(job_id)
except xmlrpclib.Fault:
print 'Job output not found for %s' % device_type
continue
# Parse LAVA messages out of log
raw_job_file = str(binary_job_file)
for line in raw_job_file.splitlines():
if 'Infrastructure Error:' in line:
print 'Infrastructure Error detected!'
index = line.find('Infrastructure Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Bootloader Error:' in line:
print 'Bootloader Error detected!'
index = line.find('Bootloader Error:')
boot_failure_reason = line[index:]
board_offline = True
if 'Kernel Error:' in line:
print 'Kernel Error detected!'
index = line.find('Kernel Error:')
boot_failure_reason = line[index:]
if 'Userspace Error:' in line:
print 'Userspace Error detected!'
index = line.find('Userspace Error:')
boot_failure_reason = line[index:]
if '<LAVA_DISPATCHER>' not in line:
if len(line) != 0:
job_file += line + '\n'
if 'rtc-efi rtc-efi: setting system clock to' in line:
if device_type == 'dynamic-vm':
efi_rtc = True
# Retrieve bundle
if bundle is not None:
json_bundle = connection.dashboard.get(bundle)
bundle_data = json.loads(json_bundle['content'])
# Get the boot data from LAVA
for test_results in bundle_data['test_runs']:
# Check for the LAVA self boot test
if test_results['test_id'] == 'lava':
for test in test_results['test_results']:
# TODO for compat :(
if test['test_case_id'] == 'kernel_boot_time':
kernel_boot_time = test['measurement']
if test['test_case_id'] == 'test_kernel_boot_time':
kernel_boot_time = test['measurement']
bundle_attributes = bundle_data['test_runs'][-1][
'attributes']
if utils.in_bundle_attributes(bundle_attributes,
'kernel.defconfig'):
print bundle_attributes['kernel.defconfig']
if utils.in_bundle_attributes(bundle_attributes, 'target'):
board_instance = bundle_attributes['target']
if utils.in_bundle_attributes(bundle_attributes,
'kernel.defconfig'):
kernel_defconfig = bundle_attributes['kernel.defconfig']
defconfig_list = kernel_defconfig.split('-')
#arch = defconfig_list[0]
arch = defconfig_list[-1]
# Remove arch
defconfig_list.pop(0)
kernel_defconfig_full = '-'.join(defconfig_list)
kernel_defconfig_base = ''.join(
kernel_defconfig_full.split('+')[:1])
if kernel_defconfig_full == kernel_defconfig_base:
kernel_defconfig_full = None
if utils.in_bundle_attributes(bundle_attributes, 'kernel.version'):
kernel_version = bundle_attributes['kernel.version']
if utils.in_bundle_attributes(bundle_attributes, 'device.tree'):
device_tree = bundle_attributes['device.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.endian'):
kernel_endian = bundle_attributes['kernel.endian']
if utils.in_bundle_attributes(bundle_attributes,
'platform.fastboot'):
fastboot = bundle_attributes['platform.fastboot']
if kernel_boot_time is None:
if utils.in_bundle_attributes(bundle_attributes,
'kernel-boot-time'):
kernel_boot_time = bundle_attributes['kernel-boot-time']
if utils.in_bundle_attributes(bundle_attributes, 'kernel.tree'):
kernel_tree = bundle_attributes['kernel.tree']
if utils.in_bundle_attributes(bundle_attributes, 'kernel-addr'):
kernel_addr = bundle_attributes['kernel-addr']
if utils.in_bundle_attributes(bundle_attributes, 'initrd-addr'):
initrd_addr = bundle_attributes['initrd-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-addr'):
dtb_addr = bundle_attributes['dtb-addr']
if utils.in_bundle_attributes(bundle_attributes, 'dtb-append'):
dtb_append = bundle_attributes['dtb-append']
if utils.in_bundle_attributes(bundle_attributes, 'boot_retries'):
boot_retries = int(bundle_attributes['boot_retries'])
if utils.in_bundle_attributes(bundle_attributes, 'test.plan'):
test_tmp = bundle_attributes['test.plan']
if test_tmp:
test_plan = test_tmp
else:
if not kernel_defconfig or not kernel_version or not kernel_tree:
job_defnition = {}
if 'original_definition' in job_details.keys():
job_definition = job_details['original_definition']
try:
job_dictionary = eval(job_definition)
except Exception:
pass
if job_dictionary:
if 'actions' in job_dictionary.keys():
actions = job_dictionary['actions']
for i in range(0, len(actions)):
try:
kernel_defconfig = actions[i]['metadata'][
'kernel.defconfig']
kernel_version = actions[i]['metadata'][
'kernel.version']
kernel_tree = actions[i]['metadata'][
'kernel.tree']
kernel_endian = actions[i]['metadata'][
'kernel.endian']
platform_fastboot = actions[i]['metadata'][
'platform.fastboot']
device_tree = actions[i]['metadata'][
'kernel.tree']
break
except KeyError:
continue
if 'target' in job_details.keys():
print job_details.keys()
# Check if we found efi-rtc
if test_plan == 'boot-kvm-uefi' and not efi_rtc:
if device_type == 'dynamic-vm':
boot_failure_reason = 'Unable to read EFI rtc'
result = 'FAIL'
# Record the boot log and result
# TODO: Will need to map device_types to dashboard device types
if kernel_defconfig and device_type and result:
if ('arm' == arch or 'arm64' == arch) and device_tree is None:
platform_name = device_map[device_type][0] + ',legacy'
else:
if test_plan == 'boot-nfs' or test_plan == 'boot-nfs-mp':
platform_name = device_map[device_type][0] + '_rootfs:nfs'
else:
platform_name = device_map[device_type][0]
print 'Creating boot log for %s' % (platform_name + job_name +
'_' + job_id)
log = 'boot-%s.txt' % (platform_name + job_name + '_' + job_id)
html = 'boot-%s.html' % (platform_name + job_name + '_' + job_id)
if config.get("lab"):
directory = os.path.join(
results_directory,
kernel_defconfig + '/' + config.get("lab"))
else:
directory = os.path.join(results_directory, kernel_defconfig)
utils.ensure_dir(directory)
utils.write_file(job_file, log, directory)
if kernel_boot_time is None:
kernel_boot_time = '0.0'
if results.has_key(kernel_defconfig):
results[kernel_defconfig].append({
'device_type': platform_name,
'job_id': job_id,
'job_name': job_short_name,
'kernel_boot_time': kernel_boot_time,
'result': result,
'device_name': device_name
})
else:
results[kernel_defconfig] = [{
'device_type': platform_name,
'job_id': job_id,
'job_name': job_short_name,
'kernel_boot_time': kernel_boot_time,
'result': result,
'device_name': device_name
}]
# Create JSON format boot metadata
print 'Creating JSON format boot metadata'
if config.get("lab"):
boot_meta['lab_name'] = config.get("lab")
else:
boot_meta['lab_name'] = None
if board_instance:
boot_meta['board_instance'] = board_instance
boot_meta['retries'] = boot_retries
boot_meta['boot_log'] = log
boot_meta['boot_log_html'] = html
# TODO: Fix this
boot_meta['version'] = '1.0'
boot_meta['arch'] = arch
boot_meta['defconfig'] = kernel_defconfig_base
if kernel_defconfig_full is not None:
boot_meta['defconfig_full'] = kernel_defconfig_full
if device_map[device_type][1]:
boot_meta['mach'] = device_map[device_type][1]
boot_meta['kernel'] = kernel_version
boot_meta['job'] = kernel_tree
boot_meta['board'] = platform_name
if board_offline and result == 'FAIL':
boot_meta['boot_result'] = 'OFFLINE'
#results[kernel_defconfig]['result'] = 'OFFLINE'
else:
boot_meta['boot_result'] = result
if result == 'FAIL' or result == 'OFFLINE':
if boot_failure_reason:
boot_meta['boot_result_description'] = boot_failure_reason
else:
boot_meta[
'boot_result_description'] = 'Unknown Error: platform failed to boot'
boot_meta['boot_time'] = kernel_boot_time
# TODO: Fix this
boot_meta['boot_warnings'] = None
if device_tree:
if arch == 'arm64':
boot_meta['dtb'] = 'dtbs/' + device_map[device_type][
1] + '/' + device_tree
else:
boot_meta['dtb'] = 'dtbs/' + device_tree
else:
boot_meta['dtb'] = device_tree
boot_meta['dtb_addr'] = dtb_addr
boot_meta['dtb_append'] = dtb_append
boot_meta['endian'] = kernel_endian
boot_meta['fastboot'] = fastboot
# TODO: Fix this
boot_meta['initrd'] = None
boot_meta['initrd_addr'] = initrd_addr
if arch == 'arm':
boot_meta['kernel_image'] = 'zImage'
elif arch == 'arm64':
boot_meta['kernel_image'] = 'Image'
else:
boot_meta['kernel_image'] = 'bzImage'
boot_meta['loadaddr'] = kernel_addr
json_file = 'boot-%s.json' % (platform_name + job_name + '_' +
job_id)
utils.write_json(json_file, directory, boot_meta)
if config.get("lab"):
report_directory = os.path.join(results_directory, config.get("lab"))
utils.mkdir(report_directory)
else:
report_directory = results_directory
if results and kernel_tree and kernel_version:
print 'Creating summary for %s' % (kernel_version)
boot = '%s-boot-report.txt' % (kernel_version)
if test_plan and ('boot' in test_plan or 'BOOT' in test_plan):
boot = boot.replace('boot', test_plan)
passed = 0
failed = 0
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'PASS':
passed += 1
else:
failed += 1
total = passed + failed
with open(os.path.join(report_directory, boot), 'a') as f:
f.write('Subject: %s boot: %s boots: %s passed, %s failed (%s)\n' %
(kernel_tree, str(total), str(passed), str(failed),
kernel_version))
f.write('\n')
f.write('Total Duration: %.2f minutes\n' % (duration / 60))
f.write('Tree/Branch: %s\n' % kernel_tree)
f.write('Git Describe: %s\n' % kernel_version)
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'OFFLINE':
if first:
f.write('\n')
f.write('Boards Offline:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'OFFLINE':
f.write(
' %s %s %s %ss %s: %s\n' %
(result['job_id'], result['device_type'],
result['device_name'], result['kernel_boot_time'],
result['job_name'], result['result']))
f.write('\n')
first = True
for defconfig, results_list in results.items():
for result in results_list:
if result['result'] == 'FAIL':
if first:
f.write('\n')
f.write('Failed Boot Tests:\n')
first = False
f.write('\n')
f.write(defconfig)
f.write('\n')
break
for result in results_list:
if result['result'] == 'FAIL':
f.write(
' %s %s %s %ss %s: %s\n' %
(result['job_id'], result['device_type'],
result['device_name'], result['kernel_boot_time'],
result['job_name'], result['result']))
f.write('\n')
f.write('Full Boot Report:\n')
for defconfig, results_list in results.items():
f.write('\n')
f.write(defconfig)
f.write('\n')
for result in results_list:
f.write(' %s %s %s %ss %s: %s\n' %
(result['job_id'], result['device_type'],
result['device_name'], result['kernel_boot_time'],
result['job_name'], result['result']))
# add by wuyanjun
if results and directory:
parser_and_get_result(results, directory, report_directory)
get_ip_board_mapping(results, directory, report_directory)
def process(file_path, new_screen_name):
if not os.path.isfile(file_path):
sys.exit('File not found')
data = read_json(file_path)
new_data = dict()
new_data['layout'] = data['layout']
new_data['info'] = {
'versionCode': data['info']['versionCode'],
'versionName': data['info']['versionName'],
}
new_data['preferences'] = {
'Display.Theme': 'WhiteHCTheme',
'EarthModel': 'WGS84',
}
if '-por2lan' in sys.argv:
new_data['layout']['landscape'] = new_data['layout']['portrait']
if '-lan2por' in sys.argv:
new_data['layout']['landscape'] = new_data['layout']['portrait']
# nocomp and onlycomp split
for orient in ['portrait', 'landscape']:
if orient not in new_data['layout']:
continue
if '-nocomp' in sys.argv:
new_data['layout'][orient] = [
i for i in new_data['layout'][orient]
if 'org.xcontest.XCTrack.navig.TaskCompetition' not in
i['navigations']
]
if '-onlycomp' in sys.argv:
new_data['layout'][orient] = [
i for i in new_data['layout'][orient]
if 'org.xcontest.XCTrack.navig.TaskCompetition' in
i['navigations']
]
# portrait and landscape modes
if '-por' in sys.argv:
if 'landscape' in new_data['layout']:
del new_data['layout']['landscape']
new_data['preferences']['Display.Orientation'] = 'PORTRAIT'
elif '-lan' in sys.argv:
if 'portrait' in new_data['layout']:
del new_data['layout']['portrait']
new_data['preferences']['Display.Orientation'] = 'LANDSCAPE'
else:
sys.exit('Please specify either -por (portrait) or -lan (landscape)')
sort_widgets_by_name(new_data['layout'])
ensure_dir('screens')
target_file = os.path.join('screens', new_screen_name + '.xcfg')
write_json(target_file, new_data)