def main(args):
global directory
config = configuration.get_config(args)
if config.get("jobs"):
directory = setup_job_dir(config.get("jobs"))
else:
directory = setup_job_dir(os.getcwd() + '/jobs')
print 'Scanning %s for kernel information...' % config.get("url")
walk_url(config.get("url"), config.get("plans"), config.get("arch"), config.get("targets"), config.get("priority"))
print 'Done scanning for kernel information'
print 'Done creating JSON jobs'
exit(0)
def main(args):
global directory
config = configuration.get_config(args)
if config.get("jobs"):
directory = setup_job_dir(config.get("jobs"))
else:
directory = setup_job_dir(os.getcwd() + '/jobs')
print 'Scanning %s for kernel information...' % config.get("url")
walk_url(config.get("url"), config.get("plans"), config.get("arch"),
config.get("targets"), config.get("priority"))
print 'Done scanning for kernel information'
print 'Done creating JSON jobs'
exit(0)
def write_jobs(config, jobs):
job_dir = setup_job_dir(config.get('jobs') or config.get('lab'))
for job in jobs:
job_file = os.path.join(job_dir, '.'.join([job['name'], 'yaml']))
with open(job_file, 'w') as f:
env = Environment(loader=FileSystemLoader('templates'))
template = env.get_template(job['short_template_file'])
data = template.render(job)
f.write(data)
print("Job written: {}".format(job_file))
def write_jobs(config, jobs):
job_dir = setup_job_dir(config.get('jobs') or config.get('lab'))
for job in jobs:
job_file = os.path.join(job_dir, '.'.join([job['name'], 'yaml']))
with open(job_file, 'w') as f:
env = Environment(loader=FileSystemLoader('templates'))
template = env.get_template(job['short_template_file'])
data = template.render(job)
f.write(data)
print("Job written: {}".format(job_file))
def main(args):
config = configuration.get_config(args)
plans = config.get("plans")
targets = config.get("targets")
lab_name = config.get('lab')
job_dir = setup_job_dir(config.get('jobs') or lab_name)
token = config.get('token')
api = config.get('api')
storage = config.get('storage')
if not token:
raise Exception("No token provided")
if not api:
raise Exception("No KernelCI API URL provided")
if not storage:
raise Exception("No KernelCI storage URL provided")
arch = args.get('arch')
plans = args.get('plans')
branch = args.get('branch')
git_describe = args.get('describe')
tree = args.get('tree')
expected = int(args.get('defconfigs'))
headers = {
"Authorization": token,
}
print "Working on kernel %s/%s" % (tree, branch)
url_params = urllib.urlencode({
'job': tree,
'kernel': git_describe,
'git_branch': branch,
'arch': arch,
})
url = urlparse.urljoin(api, 'build?{}'.format(url_params))
print "Calling KernelCI API: %s" % url
builds = []
loops = 10
retry_time = 30
for loop in range(loops):
response = requests.get(url, headers=headers)
response.raise_for_status()
data = json.loads(response.content)
builds = data['result']
if len(builds) >= expected:
break
print "Got less builds (%s) than expected (%s), retry in %s seconds" % (
len(builds), expected, retry_time)
time.sleep(retry_time)
print("Number of builds: {}".format(len(builds)))
jobs = []
cwd = os.getcwd()
for build in builds:
defconfig = build['defconfig_full']
arch_defconfig = ("%s-%s" % (arch, defconfig))
print "Working on build %s %s %s %s %s" % (tree, branch, git_describe,
arch, defconfig)
test_suite = None
test_set = None
test_desc = None
test_type = None
plan_defconfigs = []
modules = build['modules']
if build['kernel_image']:
if build['kernel_image'] == 'bzImage' and arch == 'x86':
build['dtb_dir_data'].extend(LEGACY_X86_PLATFORMS)
else:
continue
if 'PASS' not in build.get('status', ''):
continue
if arch in ['arm', 'arm64', 'x86'] and 'defconfig' in defconfig:
build['dtb_dir_data'].append('qemu')
for plan in plans:
if plan != 'boot':
config = ConfigParser.ConfigParser()
try:
config.read(cwd + '/templates/' + plan + '/' + plan +
'.ini')
test_suite = config.get(plan, 'suite')
test_set = config.get(plan, 'set')
test_desc = config.get(plan, 'description')
test_type = config.get(plan, 'type')
plan_defconfigs = config.get(plan, 'defconfigs').split(',')
except:
print "Unable to load test configuration"
exit(1)
if build['kernel_image']:
for dtb in build['dtb_dir_data']:
# hack for arm64 dtbs in subfolders
dtb_full = dtb
if arch == 'arm64':
dtb = str(dtb).split('/')[-1]
if dtb in device_map:
# print "device %s was in the device_map" % dtb
for device in device_map[dtb]:
# print "working on device %s" % dtb
lpae = device['lpae']
device_type = device['device_type']
mach = device['mach']
fastboot = str(device['fastboot']).lower()
blacklist = False
nfs_blacklist = False
if defconfig in device['defconfig_blacklist']:
print "defconfig %s is blacklisted for device %s" % (
defconfig, device['device_type'])
continue
elif device.has_key('defconfig_whitelist'
) and defconfig not in device[
'defconfig_whitelist']:
print "defconfig %s is not in whitelist for device %s" % (
defconfig, device['device_type'])
continue
elif device.has_key(
'arch_blacklist'
) and arch in device['arch_blacklist']:
print "arch %s is blacklisted for device %s" % (
arch, device['device_type'])
continue
elif device.has_key(
'lab_blacklist'
) and lab_name in device['lab_blacklist']:
print "device %s is blacklisted for lab %s" % (
device['device_type'], lab_name)
continue
elif "BIG_ENDIAN" in defconfig and plan != 'boot-be':
print "BIG_ENDIAN is not supported on %s" % device_type
continue
elif "LPAE" in defconfig and not lpae:
print "LPAE is not support on %s" % device_type
continue
elif any([
x for x in device['kernel_blacklist']
if x in git_describe
]):
print "git_describe %s is blacklisted for device %s" % (
git_describe, device_type)
continue
elif any([x for x in device['nfs_blacklist'] if x in git_describe]) \
and plan in ['boot-nfs', 'boot-nfs-mp']:
print "git_describe %s is blacklisted for NFS on device %s" % (
git_describe, device_type)
continue
elif 'be_blacklist' in device \
and any([x for x in device['be_blacklist'] if x in git_describe]) \
and plan in ['boot-be']:
print "git_describe %s is blacklisted for BE on device %s" % (
git_describe, device_type)
continue
elif (arch_defconfig
not in plan_defconfigs) and (plan != "boot"):
print "defconfig %s not in test plan %s" % (
arch_defconfig, plan)
continue
elif targets is not None and device_type not in targets:
print "device_type %s is not in targets %s" % (
device_type, targets)
elif arch == 'x86' and dtb == 'x86-32' and 'i386' not in arch_defconfig:
print "%s is not a 32-bit x86 build, skipping for 32-bit device %s" % (
defconfig, device_type)
elif 'kselftest' in defconfig and plan != 'kselftest':
print "Skipping kselftest defconfig because plan was not kselftest"
else:
for template in device['templates']:
short_template_file = plan + '/' + str(
template)
template_file = cwd + '/templates/' + short_template_file
if os.path.exists(
template_file
) and template_file.endswith('.jinja2'):
job_name = tree + '-' + branch + '-' + git_describe + '-' + arch + '-' + defconfig[:
100] + '-' + dtb + '-' + device_type + '-' + plan
base_url = "%s/%s/%s/%s/%s/%s/" % (
storage, build['job'],
build['git_branch'],
build['kernel'], arch, defconfig)
nfsrootfs_url = None
initrd_url = None
callback_name = 'lava/boot' if plan == 'boot' else 'lava/test'
context = device[
'context'] if 'context' in device else None
if dtb_full.endswith('.dtb'):
dtb_url = base_url + "dtbs/" + dtb_full
platform = dtb[:-4]
else:
dtb_url = None
platform = device_type
kernel_url = urlparse.urljoin(
base_url, build['kernel_image'])
defconfig_base = ''.join(
defconfig.split('+')[:1])
endian = 'little'
if 'BIG_ENDIAN' in defconfig and plan == 'boot-be':
endian = 'big'
initrd_arch = arch
if arch == 'arm64' and endian == 'big':
initrd_arch = 'arm64be'
if arch == 'arm':
if endian == 'big':
initrd_arch = 'armeb'
else:
initrd_arch = 'armel'
if 'kselftest' in plan:
initrd_url = KSELFTEST_INITRD_URL.format(
initrd_arch)
else:
initrd_url = INITRD_URL.format(
initrd_arch)
if 'nfs' in plan:
nfsrootfs_url = NFSROOTFS_URL.format(
initrd_arch)
initrd_url = None
platform = platform + "_rootfs:nfs"
if build['modules']:
modules_url = urlparse.urljoin(
base_url, build['modules'])
else:
modules_url = None
device_type = device['device_type']
if device_type.startswith(
'qemu'
) or device_type == 'kvm':
device_type = 'qemu'
job = {
'name': job_name,
'dtb_url': dtb_url,
'dtb_full': dtb_full,
'platform': platform,
'mach': mach,
'kernel_url': kernel_url,
'image_type': 'kernel-ci',
'image_url': base_url,
'modules_url': modules_url,
'plan': plan,
'kernel': git_describe,
'tree': tree,
'defconfig': defconfig,
'fastboot': fastboot,
'priority': args.get('priority'),
'device_type': device_type,
'template_file': template_file,
'base_url': base_url,
'endian': endian,
'test_suite': test_suite,
'test_set': test_set,
'test_desc': test_desc,
'test_type': test_type,
'short_template_file':
short_template_file,
'arch': arch,
'arch_defconfig': arch_defconfig,
'git_branch': branch,
'git_commit': build['git_commit'],
'git_describe': git_describe,
'git_url': build['git_url'],
'defconfig_base': defconfig_base,
'initrd_url': initrd_url,
'kernel_image':
build['kernel_image'],
'dtb_short': dtb,
'nfsrootfs_url': nfsrootfs_url,
'callback': args.get('callback'),
'api': api,
'lab_name': lab_name,
'callback_name': callback_name,
'context': context,
}
jobs.append(job)
else:
print "no kernel_image for %s" % build['defconfig_full']
for job in jobs:
job_file = job_dir + '/' + job['name'] + '.yaml'
with open(job_file, 'w') as f:
f.write(jinja_render(job))
print "Job written: %s" % job_file
def main(args):
config = configuration.get_config(args)
plans = config.get("plans")
targets = config.get("targets")
if config.get("jobs"):
job_dir = setup_job_dir(config.get("jobs"))
else:
job_dir = setup_job_dir(os.getcwd() + '/jobs')
arch = args.get('arch')
plans = args.get('plans')
branch = args.get('branch')
tree = args.get('tree')
kernel = tree
api = args.get('api')
headers = {"Authorization": config.get('token')}
print "Working on kernel %s/%s" % (tree, branch)
url = urlparse.urljoin(api,
("/build?job=%s&kernel=%s&status=PASS&arch=%s" %
(tree, branch, arch)))
print "Calling KernelCI API: %s" % url
response = requests.get(url, headers=headers)
if response.status_code != 200:
print "Error calling KernelCI API"
print response
sys.exit(1)
data = json.loads(response.content)
builds = data['result']
print len(builds)
jobs = []
cwd = os.getcwd()
for build in builds:
defconfig = build['defconfig_full']
arch_defconfig = ("%s-%s" % (arch, defconfig))
print "Working on build %s" % arch_defconfig
test_suite = None
test_set = None
test_desc = None
test_type = None
plan_defconfigs = []
modules = build['modules']
for plan in plans:
if plan != 'boot':
config = ConfigParser.ConfigParser()
try:
config.read(cwd + '/templates/' + plan + '/' + plan +
'.ini')
test_suite = config.get(plan, 'suite')
test_set = config.get(plan, 'set')
test_desc = config.get(plan, 'description')
test_type = config.get(plan, 'type')
plan_defconfigs = config.get(plan, 'defconfigs').split(',')
except:
print "Unable to load test configuration"
exit(1)
if build['kernel_image']:
# handle devices without a DTB, hacky :/
if build['kernel_image'] == 'bzImage' and arch == 'x86':
build['dtb_dir_data'].extend(LEGACY_X86_PLATFORMS)
if arch == 'arm64' and 'arm64-defconfig' in arch_defconfig:
build['dtb_dir_data'].extend(LEGACY_ARM64_PLATFORMS)
for dtb in build['dtb_dir_data']:
# hack for arm64 dtbs in subfolders
dtb_full = dtb
if arch == 'arm64':
dtb = str(dtb).split('/')[-1]
if dtb in device_map:
# print "device %s was in the device_map" % dtb
for device in device_map[dtb]:
# print "working on device %s" % dtb
lpae = device['lpae']
device_type = device['device_type']
fastboot = str(device['fastboot']).lower()
blacklist = False
nfs_blacklist = False
if arch_defconfig in device['defconfig_blacklist']:
print "defconfig %s is blacklisted for device %s" % (
defconfig, device['device_type'])
continue
elif "BIG_ENDIAN" in defconfig and plan != 'boot-be':
print "BIG_ENDIAN is not supported on %s" % device_type
continue
elif "LPAE" in defconfig and not lpae:
print "LPAE is not support on %s" % device_type
continue
elif any([
x for x in device['kernel_blacklist']
if x in kernel
]):
print "kernel %s is blacklisted for device %s" % (
kernel, device_type)
continue
elif any([x for x in device['nfs_blacklist'] if x in kernel]) \
and plan in ['boot-nfs', 'boot-nfs-mp']:
print "kernel %s is blacklisted for NFS on device %s" % (
kernel, device_type)
continue
elif 'be_blacklist' in device \
and any([x for x in device['be_blacklist'] if x in kernel]) \
and plan in ['boot-be']:
print "kernel %s is blacklisted for BE on device %s" % (
kernel, device_type)
continue
elif (arch_defconfig
not in plan_defconfigs) and (plan != "boot"):
print "defconfig %s not in test plan %s" % (
arch_defconfig, plan)
continue
elif targets is not None and device_type not in targets:
print "device_type %s is not in targets %s" % (
device_type, targets)
else:
for template in device['templates']:
short_template_file = plan + '/' + str(
template)
template_file = cwd + '/templates/' + short_template_file
if os.path.exists(
template_file
) and template_file.endswith('.jinja2'):
job_name = tree + '-' + branch + '-' + arch + "-" + defconfig[:
100] + '-' + dtb + '-' + device_type + '-' + plan
base_url = "https://storage.kernelci.org/%s/%s/%s/" % (
build['job'], build['kernel'],
arch_defconfig)
dtb_url = base_url + "dtbs/" + dtb_full
kernel_url = urlparse.urljoin(
base_url, build['kernel_image'])
defconfig_base = ''.join(
defconfig.split('+')[:1])
endian = 'little'
if 'BIG_ENDIAN' in defconfig and plan == 'boot-be':
endian = 'big'
initrd_arch = arch
if arch not in INITRD_URLS.keys():
if arch == 'arm64' and endian == 'big':
initrd_arch = 'arm64be'
if arch == 'arm':
if endian == 'big':
initrd_arch = 'armeb'
else:
initrd_arch = 'armel'
initrd_url = INITRD_URLS[initrd_arch]
if build['modules']:
modules_url = urlparse.urljoin(
base_url, build['modules'])
else:
modules_url = None
if device['device_type'].startswith(
'qemu'
) or device['device_type'] == 'kvm':
device['device_type'] = 'qemu'
job = {
'name': job_name,
'dtb_url': dtb_url,
'platform': dtb_full,
'kernel_url': kernel_url,
'image_type': 'kernel-ci',
'image_url': base_url,
'modules_url': modules_url,
'plan': plan,
'kernel': branch,
'tree': tree,
'defconfig': defconfig,
'fastboot': fastboot,
'priority': args.get('priority'),
'device': device,
'template_file': template_file,
'base_url': base_url,
'endian': endian,
'test_suite': test_suite,
'test_set': test_set,
'test_desc': test_desc,
'test_type': test_type,
'short_template_file':
short_template_file,
'arch': arch,
'arch_defconfig': arch_defconfig,
'git_branch': build['git_branch'],
'git_commit': build['git_commit'],
'git_describe':
build['git_describe'],
'defconfig_base': defconfig_base,
'initrd_url': initrd_url,
'kernel_image':
build['kernel_image'],
'dtb_short': dtb
}
jobs.append(job)
else:
print "no kernel_image for %s" % build['defconfig_full']
for job in jobs:
job_file = job_dir + '/' + job['name'] + '.yaml'
with open(job_file, 'w') as f:
f.write(jinja_render(job))
print "Job written: %s" % job_file
def main(args):
config = configuration.get_config(args)
plans = config.get("plans")
targets = config.get("targets")
if config.get("jobs"):
job_dir = setup_job_dir(config.get("jobs"))
else:
job_dir = setup_job_dir(os.getcwd() + '/jobs')
arch = args.get('arch')
plans = args.get('plans')
branch = args.get('branch')
tree = args.get('tree')
kernel = tree
api = args.get('api')
headers = {
"Authorization": config.get('token')
}
print "Working on kernel %s/%s" % (tree, branch)
url = urlparse.urljoin(api, ("/build?job=%s&kernel=%s&status=PASS&arch=%s" % (tree, branch, arch)))
print "Calling KernelCI API: %s" % url
response = requests.get(url, headers=headers)
print response
data = json.loads(response.content)
builds = data['result']
print len(builds)
jobs = []
cwd = os.getcwd()
for build in builds:
defconfig = build['defconfig_full']
arch_defconfig = ("%s-%s" % (arch, defconfig))
print "Working on build %s" % arch_defconfig
test_suite = None
test_set = None
test_desc = None
test_type = None
plan_defconfigs = []
modules = build['modules']
for plan in plans:
if plan != 'boot':
config = ConfigParser.ConfigParser()
try:
config.read(cwd + '/templates/' + plan + '/' + plan + '.ini')
test_suite = config.get(plan, 'suite')
test_set = config.get(plan, 'set')
test_desc = config.get(plan, 'description')
test_type = config.get(plan, 'type')
plan_defconfigs = config.get(plan, 'defconfigs').split(',')
except:
print "Unable to load test configuration"
exit(1)
if build['kernel_image']:
# handle devices without a DTB, hacky :/
if build['kernel_image'] == 'bzImage' and arch == 'x86':
build['dtb_dir_data'].extend(LEGACY_X86_PLATFORMS)
if arch == 'arm64' and 'arm64-defconfig' in arch_defconfig:
build['dtb_dir_data'].extend(LEGACY_ARM64_PLATFORMS)
for dtb in build['dtb_dir_data']:
# hack for arm64 dtbs in subfolders
dtb_full = dtb
if arch == 'arm64':
dtb = str(dtb).split('/')[-1]
if dtb in device_map:
# print "device %s was in the device_map" % dtb
for device in device_map[dtb]:
# print "working on device %s" % dtb
lpae = device['lpae']
device_type = device['device_type']
fastboot = str(device['fastboot']).lower()
blacklist = False
nfs_blacklist = False
if arch_defconfig in device['defconfig_blacklist']:
print "defconfig %s is blacklisted for device %s" % (defconfig, device['device_type'])
continue
elif "BIG_ENDIAN" in defconfig and plan != 'boot-be':
print "BIG_ENDIAN is not supported on %s" % device_type
continue
elif "LPAE" in defconfig and not lpae:
print "LPAE is not support on %s" % device_type
continue
elif any([x for x in device['kernel_blacklist'] if x in kernel]):
print "kernel %s is blacklisted for device %s" % (kernel, device_type)
continue
elif any([x for x in device['nfs_blacklist'] if x in kernel]) \
and plan in ['boot-nfs', 'boot-nfs-mp']:
print "kernel %s is blacklisted for NFS on device %s" % (kernel, device_type)
continue
elif 'be_blacklist' in device \
and any([x for x in device['be_blacklist'] if x in kernel]) \
and plan in ['boot-be']:
print "kernel %s is blacklisted for BE on device %s" % (kernel, device_type)
continue
elif (arch_defconfig not in plan_defconfigs) and (plan != "boot"):
print "defconfig %s not in test plan %s" % (arch_defconfig, plan)
continue
elif targets is not None and device_type not in targets:
print "device_type %s is not in targets %s" % (device_type, targets)
else:
for template in device['templates']:
short_template_file = plan + '/' + str(template)
template_file = cwd + '/templates/' + short_template_file
if os.path.exists(template_file) and template_file.endswith('.jinja2'):
job_name = tree + '-' + branch + '-' + arch + "-" + defconfig[:100] + '-' + dtb + '-' + device_type + '-' + plan
base_url = "https://storage.kernelci.org/%s/%s/%s/" % (build['job'], build['kernel'], arch_defconfig)
dtb_url = base_url + "dtbs/" + dtb_full
kernel_url = urlparse.urljoin(base_url, build['kernel_image'])
endian = 'little'
if 'BIG_ENDIAN' in defconfig and plan == 'boot-be':
endian = 'big'
if build['modules']:
modules_url = urlparse.urljoin(base_url, build['modules'])
else:
modules_url = None
if device['device_type'].startswith('qemu'):
device['device_type'] = 'qemu'
job = {'name': job_name, 'dtb_url': dtb_url, 'platform': dtb, 'kernel_url': kernel_url, 'image_type': 'kernel-ci', 'image_url': base_url,
'modules_url': modules_url, 'plan': plan, 'kernel': branch, 'tree': tree, 'defconfig': defconfig, 'fastboot': fastboot,
'priority': args.get('priority'), 'device': device, 'template_file': template_file, 'base_url': base_url, 'endian': endian,
'test_suite': test_suite, 'test_set': test_set, 'test_desc': test_desc, 'test_type': test_type, 'short_template_file': short_template_file,
'arch': arch, 'arch_defconfig': arch_defconfig}
jobs.append(job)
# print job
else:
print "no kernel_image for %s" % build['defconfig_full']
for job in jobs:
job_file = job_dir + '/' + job['name'] + '.yaml'
with open(job_file, 'w') as f:
f.write(jinja_render(job))
print "Job written: %s" % job_file