def main(args): global directory config = configuration.get_config(args) if config.get("jobs"): directory = setup_job_dir(config.get("jobs")) else: directory = setup_job_dir(os.getcwd() + '/jobs') print 'Scanning %s for kernel information...' % config.get("url") walk_url(config.get("url"), config.get("plans"), config.get("arch"), config.get("targets"), config.get("priority")) print 'Done scanning for kernel information' print 'Done creating JSON jobs' exit(0)
def write_jobs(config, jobs): job_dir = setup_job_dir(config.get('jobs') or config.get('lab')) for job in jobs: job_file = os.path.join(job_dir, '.'.join([job['name'], 'yaml'])) with open(job_file, 'w') as f: env = Environment(loader=FileSystemLoader('templates')) template = env.get_template(job['short_template_file']) data = template.render(job) f.write(data) print("Job written: {}".format(job_file))
def main(args): config = configuration.get_config(args) plans = config.get("plans") targets = config.get("targets") lab_name = config.get('lab') job_dir = setup_job_dir(config.get('jobs') or lab_name) token = config.get('token') api = config.get('api') storage = config.get('storage') if not token: raise Exception("No token provided") if not api: raise Exception("No KernelCI API URL provided") if not storage: raise Exception("No KernelCI storage URL provided") arch = args.get('arch') plans = args.get('plans') branch = args.get('branch') git_describe = args.get('describe') tree = args.get('tree') expected = int(args.get('defconfigs')) headers = { "Authorization": token, } print "Working on kernel %s/%s" % (tree, branch) url_params = urllib.urlencode({ 'job': tree, 'kernel': git_describe, 'git_branch': branch, 'arch': arch, }) url = urlparse.urljoin(api, 'build?{}'.format(url_params)) print "Calling KernelCI API: %s" % url builds = [] loops = 10 retry_time = 30 for loop in range(loops): response = requests.get(url, headers=headers) response.raise_for_status() data = json.loads(response.content) builds = data['result'] if len(builds) >= expected: break print "Got less builds (%s) than expected (%s), retry in %s seconds" % ( len(builds), expected, retry_time) time.sleep(retry_time) print("Number of builds: {}".format(len(builds))) jobs = [] cwd = os.getcwd() for build in builds: defconfig = build['defconfig_full'] arch_defconfig = ("%s-%s" % (arch, defconfig)) print "Working on build %s %s %s %s %s" % (tree, branch, git_describe, arch, defconfig) test_suite = None test_set = None test_desc = None test_type = None plan_defconfigs = [] modules = build['modules'] if build['kernel_image']: if build['kernel_image'] == 'bzImage' and arch == 'x86': build['dtb_dir_data'].extend(LEGACY_X86_PLATFORMS) else: continue if 'PASS' not in build.get('status', ''): continue if arch in ['arm', 'arm64', 'x86'] and 'defconfig' in defconfig: build['dtb_dir_data'].append('qemu') for plan in plans: if plan != 'boot': config = ConfigParser.ConfigParser() try: config.read(cwd + '/templates/' + plan + '/' + plan + '.ini') test_suite = config.get(plan, 'suite') test_set = config.get(plan, 'set') test_desc = config.get(plan, 'description') test_type = config.get(plan, 'type') plan_defconfigs = config.get(plan, 'defconfigs').split(',') except: print "Unable to load test configuration" exit(1) if build['kernel_image']: for dtb in build['dtb_dir_data']: # hack for arm64 dtbs in subfolders dtb_full = dtb if arch == 'arm64': dtb = str(dtb).split('/')[-1] if dtb in device_map: # print "device %s was in the device_map" % dtb for device in device_map[dtb]: # print "working on device %s" % dtb lpae = device['lpae'] device_type = device['device_type'] mach = device['mach'] fastboot = str(device['fastboot']).lower() blacklist = False nfs_blacklist = False if defconfig in device['defconfig_blacklist']: print "defconfig %s is blacklisted for device %s" % ( defconfig, device['device_type']) continue elif device.has_key('defconfig_whitelist' ) and defconfig not in device[ 'defconfig_whitelist']: print "defconfig %s is not in whitelist for device %s" % ( defconfig, device['device_type']) continue elif device.has_key( 'arch_blacklist' ) and arch in device['arch_blacklist']: print "arch %s is blacklisted for device %s" % ( arch, device['device_type']) continue elif device.has_key( 'lab_blacklist' ) and lab_name in device['lab_blacklist']: print "device %s is blacklisted for lab %s" % ( device['device_type'], lab_name) continue elif "BIG_ENDIAN" in defconfig and plan != 'boot-be': print "BIG_ENDIAN is not supported on %s" % device_type continue elif "LPAE" in defconfig and not lpae: print "LPAE is not support on %s" % device_type continue elif any([ x for x in device['kernel_blacklist'] if x in git_describe ]): print "git_describe %s is blacklisted for device %s" % ( git_describe, device_type) continue elif any([x for x in device['nfs_blacklist'] if x in git_describe]) \ and plan in ['boot-nfs', 'boot-nfs-mp']: print "git_describe %s is blacklisted for NFS on device %s" % ( git_describe, device_type) continue elif 'be_blacklist' in device \ and any([x for x in device['be_blacklist'] if x in git_describe]) \ and plan in ['boot-be']: print "git_describe %s is blacklisted for BE on device %s" % ( git_describe, device_type) continue elif (arch_defconfig not in plan_defconfigs) and (plan != "boot"): print "defconfig %s not in test plan %s" % ( arch_defconfig, plan) continue elif targets is not None and device_type not in targets: print "device_type %s is not in targets %s" % ( device_type, targets) elif arch == 'x86' and dtb == 'x86-32' and 'i386' not in arch_defconfig: print "%s is not a 32-bit x86 build, skipping for 32-bit device %s" % ( defconfig, device_type) elif 'kselftest' in defconfig and plan != 'kselftest': print "Skipping kselftest defconfig because plan was not kselftest" else: for template in device['templates']: short_template_file = plan + '/' + str( template) template_file = cwd + '/templates/' + short_template_file if os.path.exists( template_file ) and template_file.endswith('.jinja2'): job_name = tree + '-' + branch + '-' + git_describe + '-' + arch + '-' + defconfig[: 100] + '-' + dtb + '-' + device_type + '-' + plan base_url = "%s/%s/%s/%s/%s/%s/" % ( storage, build['job'], build['git_branch'], build['kernel'], arch, defconfig) nfsrootfs_url = None initrd_url = None callback_name = 'lava/boot' if plan == 'boot' else 'lava/test' context = device[ 'context'] if 'context' in device else None if dtb_full.endswith('.dtb'): dtb_url = base_url + "dtbs/" + dtb_full platform = dtb[:-4] else: dtb_url = None platform = device_type kernel_url = urlparse.urljoin( base_url, build['kernel_image']) defconfig_base = ''.join( defconfig.split('+')[:1]) endian = 'little' if 'BIG_ENDIAN' in defconfig and plan == 'boot-be': endian = 'big' initrd_arch = arch if arch == 'arm64' and endian == 'big': initrd_arch = 'arm64be' if arch == 'arm': if endian == 'big': initrd_arch = 'armeb' else: initrd_arch = 'armel' if 'kselftest' in plan: initrd_url = KSELFTEST_INITRD_URL.format( initrd_arch) else: initrd_url = INITRD_URL.format( initrd_arch) if 'nfs' in plan: nfsrootfs_url = NFSROOTFS_URL.format( initrd_arch) initrd_url = None platform = platform + "_rootfs:nfs" if build['modules']: modules_url = urlparse.urljoin( base_url, build['modules']) else: modules_url = None device_type = device['device_type'] if device_type.startswith( 'qemu' ) or device_type == 'kvm': device_type = 'qemu' job = { 'name': job_name, 'dtb_url': dtb_url, 'dtb_full': dtb_full, 'platform': platform, 'mach': mach, 'kernel_url': kernel_url, 'image_type': 'kernel-ci', 'image_url': base_url, 'modules_url': modules_url, 'plan': plan, 'kernel': git_describe, 'tree': tree, 'defconfig': defconfig, 'fastboot': fastboot, 'priority': args.get('priority'), 'device_type': device_type, 'template_file': template_file, 'base_url': base_url, 'endian': endian, 'test_suite': test_suite, 'test_set': test_set, 'test_desc': test_desc, 'test_type': test_type, 'short_template_file': short_template_file, 'arch': arch, 'arch_defconfig': arch_defconfig, 'git_branch': branch, 'git_commit': build['git_commit'], 'git_describe': git_describe, 'git_url': build['git_url'], 'defconfig_base': defconfig_base, 'initrd_url': initrd_url, 'kernel_image': build['kernel_image'], 'dtb_short': dtb, 'nfsrootfs_url': nfsrootfs_url, 'callback': args.get('callback'), 'api': api, 'lab_name': lab_name, 'callback_name': callback_name, 'context': context, } jobs.append(job) else: print "no kernel_image for %s" % build['defconfig_full'] for job in jobs: job_file = job_dir + '/' + job['name'] + '.yaml' with open(job_file, 'w') as f: f.write(jinja_render(job)) print "Job written: %s" % job_file
def main(args): config = configuration.get_config(args) plans = config.get("plans") targets = config.get("targets") if config.get("jobs"): job_dir = setup_job_dir(config.get("jobs")) else: job_dir = setup_job_dir(os.getcwd() + '/jobs') arch = args.get('arch') plans = args.get('plans') branch = args.get('branch') tree = args.get('tree') kernel = tree api = args.get('api') headers = {"Authorization": config.get('token')} print "Working on kernel %s/%s" % (tree, branch) url = urlparse.urljoin(api, ("/build?job=%s&kernel=%s&status=PASS&arch=%s" % (tree, branch, arch))) print "Calling KernelCI API: %s" % url response = requests.get(url, headers=headers) if response.status_code != 200: print "Error calling KernelCI API" print response sys.exit(1) data = json.loads(response.content) builds = data['result'] print len(builds) jobs = [] cwd = os.getcwd() for build in builds: defconfig = build['defconfig_full'] arch_defconfig = ("%s-%s" % (arch, defconfig)) print "Working on build %s" % arch_defconfig test_suite = None test_set = None test_desc = None test_type = None plan_defconfigs = [] modules = build['modules'] for plan in plans: if plan != 'boot': config = ConfigParser.ConfigParser() try: config.read(cwd + '/templates/' + plan + '/' + plan + '.ini') test_suite = config.get(plan, 'suite') test_set = config.get(plan, 'set') test_desc = config.get(plan, 'description') test_type = config.get(plan, 'type') plan_defconfigs = config.get(plan, 'defconfigs').split(',') except: print "Unable to load test configuration" exit(1) if build['kernel_image']: # handle devices without a DTB, hacky :/ if build['kernel_image'] == 'bzImage' and arch == 'x86': build['dtb_dir_data'].extend(LEGACY_X86_PLATFORMS) if arch == 'arm64' and 'arm64-defconfig' in arch_defconfig: build['dtb_dir_data'].extend(LEGACY_ARM64_PLATFORMS) for dtb in build['dtb_dir_data']: # hack for arm64 dtbs in subfolders dtb_full = dtb if arch == 'arm64': dtb = str(dtb).split('/')[-1] if dtb in device_map: # print "device %s was in the device_map" % dtb for device in device_map[dtb]: # print "working on device %s" % dtb lpae = device['lpae'] device_type = device['device_type'] fastboot = str(device['fastboot']).lower() blacklist = False nfs_blacklist = False if arch_defconfig in device['defconfig_blacklist']: print "defconfig %s is blacklisted for device %s" % ( defconfig, device['device_type']) continue elif "BIG_ENDIAN" in defconfig and plan != 'boot-be': print "BIG_ENDIAN is not supported on %s" % device_type continue elif "LPAE" in defconfig and not lpae: print "LPAE is not support on %s" % device_type continue elif any([ x for x in device['kernel_blacklist'] if x in kernel ]): print "kernel %s is blacklisted for device %s" % ( kernel, device_type) continue elif any([x for x in device['nfs_blacklist'] if x in kernel]) \ and plan in ['boot-nfs', 'boot-nfs-mp']: print "kernel %s is blacklisted for NFS on device %s" % ( kernel, device_type) continue elif 'be_blacklist' in device \ and any([x for x in device['be_blacklist'] if x in kernel]) \ and plan in ['boot-be']: print "kernel %s is blacklisted for BE on device %s" % ( kernel, device_type) continue elif (arch_defconfig not in plan_defconfigs) and (plan != "boot"): print "defconfig %s not in test plan %s" % ( arch_defconfig, plan) continue elif targets is not None and device_type not in targets: print "device_type %s is not in targets %s" % ( device_type, targets) else: for template in device['templates']: short_template_file = plan + '/' + str( template) template_file = cwd + '/templates/' + short_template_file if os.path.exists( template_file ) and template_file.endswith('.jinja2'): job_name = tree + '-' + branch + '-' + arch + "-" + defconfig[: 100] + '-' + dtb + '-' + device_type + '-' + plan base_url = "https://storage.kernelci.org/%s/%s/%s/" % ( build['job'], build['kernel'], arch_defconfig) dtb_url = base_url + "dtbs/" + dtb_full kernel_url = urlparse.urljoin( base_url, build['kernel_image']) defconfig_base = ''.join( defconfig.split('+')[:1]) endian = 'little' if 'BIG_ENDIAN' in defconfig and plan == 'boot-be': endian = 'big' initrd_arch = arch if arch not in INITRD_URLS.keys(): if arch == 'arm64' and endian == 'big': initrd_arch = 'arm64be' if arch == 'arm': if endian == 'big': initrd_arch = 'armeb' else: initrd_arch = 'armel' initrd_url = INITRD_URLS[initrd_arch] if build['modules']: modules_url = urlparse.urljoin( base_url, build['modules']) else: modules_url = None if device['device_type'].startswith( 'qemu' ) or device['device_type'] == 'kvm': device['device_type'] = 'qemu' job = { 'name': job_name, 'dtb_url': dtb_url, 'platform': dtb_full, 'kernel_url': kernel_url, 'image_type': 'kernel-ci', 'image_url': base_url, 'modules_url': modules_url, 'plan': plan, 'kernel': branch, 'tree': tree, 'defconfig': defconfig, 'fastboot': fastboot, 'priority': args.get('priority'), 'device': device, 'template_file': template_file, 'base_url': base_url, 'endian': endian, 'test_suite': test_suite, 'test_set': test_set, 'test_desc': test_desc, 'test_type': test_type, 'short_template_file': short_template_file, 'arch': arch, 'arch_defconfig': arch_defconfig, 'git_branch': build['git_branch'], 'git_commit': build['git_commit'], 'git_describe': build['git_describe'], 'defconfig_base': defconfig_base, 'initrd_url': initrd_url, 'kernel_image': build['kernel_image'], 'dtb_short': dtb } jobs.append(job) else: print "no kernel_image for %s" % build['defconfig_full'] for job in jobs: job_file = job_dir + '/' + job['name'] + '.yaml' with open(job_file, 'w') as f: f.write(jinja_render(job)) print "Job written: %s" % job_file
def main(args): config = configuration.get_config(args) plans = config.get("plans") targets = config.get("targets") if config.get("jobs"): job_dir = setup_job_dir(config.get("jobs")) else: job_dir = setup_job_dir(os.getcwd() + '/jobs') arch = args.get('arch') plans = args.get('plans') branch = args.get('branch') tree = args.get('tree') kernel = tree api = args.get('api') headers = { "Authorization": config.get('token') } print "Working on kernel %s/%s" % (tree, branch) url = urlparse.urljoin(api, ("/build?job=%s&kernel=%s&status=PASS&arch=%s" % (tree, branch, arch))) print "Calling KernelCI API: %s" % url response = requests.get(url, headers=headers) print response data = json.loads(response.content) builds = data['result'] print len(builds) jobs = [] cwd = os.getcwd() for build in builds: defconfig = build['defconfig_full'] arch_defconfig = ("%s-%s" % (arch, defconfig)) print "Working on build %s" % arch_defconfig test_suite = None test_set = None test_desc = None test_type = None plan_defconfigs = [] modules = build['modules'] for plan in plans: if plan != 'boot': config = ConfigParser.ConfigParser() try: config.read(cwd + '/templates/' + plan + '/' + plan + '.ini') test_suite = config.get(plan, 'suite') test_set = config.get(plan, 'set') test_desc = config.get(plan, 'description') test_type = config.get(plan, 'type') plan_defconfigs = config.get(plan, 'defconfigs').split(',') except: print "Unable to load test configuration" exit(1) if build['kernel_image']: # handle devices without a DTB, hacky :/ if build['kernel_image'] == 'bzImage' and arch == 'x86': build['dtb_dir_data'].extend(LEGACY_X86_PLATFORMS) if arch == 'arm64' and 'arm64-defconfig' in arch_defconfig: build['dtb_dir_data'].extend(LEGACY_ARM64_PLATFORMS) for dtb in build['dtb_dir_data']: # hack for arm64 dtbs in subfolders dtb_full = dtb if arch == 'arm64': dtb = str(dtb).split('/')[-1] if dtb in device_map: # print "device %s was in the device_map" % dtb for device in device_map[dtb]: # print "working on device %s" % dtb lpae = device['lpae'] device_type = device['device_type'] fastboot = str(device['fastboot']).lower() blacklist = False nfs_blacklist = False if arch_defconfig in device['defconfig_blacklist']: print "defconfig %s is blacklisted for device %s" % (defconfig, device['device_type']) continue elif "BIG_ENDIAN" in defconfig and plan != 'boot-be': print "BIG_ENDIAN is not supported on %s" % device_type continue elif "LPAE" in defconfig and not lpae: print "LPAE is not support on %s" % device_type continue elif any([x for x in device['kernel_blacklist'] if x in kernel]): print "kernel %s is blacklisted for device %s" % (kernel, device_type) continue elif any([x for x in device['nfs_blacklist'] if x in kernel]) \ and plan in ['boot-nfs', 'boot-nfs-mp']: print "kernel %s is blacklisted for NFS on device %s" % (kernel, device_type) continue elif 'be_blacklist' in device \ and any([x for x in device['be_blacklist'] if x in kernel]) \ and plan in ['boot-be']: print "kernel %s is blacklisted for BE on device %s" % (kernel, device_type) continue elif (arch_defconfig not in plan_defconfigs) and (plan != "boot"): print "defconfig %s not in test plan %s" % (arch_defconfig, plan) continue elif targets is not None and device_type not in targets: print "device_type %s is not in targets %s" % (device_type, targets) else: for template in device['templates']: short_template_file = plan + '/' + str(template) template_file = cwd + '/templates/' + short_template_file if os.path.exists(template_file) and template_file.endswith('.jinja2'): job_name = tree + '-' + branch + '-' + arch + "-" + defconfig[:100] + '-' + dtb + '-' + device_type + '-' + plan base_url = "https://storage.kernelci.org/%s/%s/%s/" % (build['job'], build['kernel'], arch_defconfig) dtb_url = base_url + "dtbs/" + dtb_full kernel_url = urlparse.urljoin(base_url, build['kernel_image']) endian = 'little' if 'BIG_ENDIAN' in defconfig and plan == 'boot-be': endian = 'big' if build['modules']: modules_url = urlparse.urljoin(base_url, build['modules']) else: modules_url = None if device['device_type'].startswith('qemu'): device['device_type'] = 'qemu' job = {'name': job_name, 'dtb_url': dtb_url, 'platform': dtb, 'kernel_url': kernel_url, 'image_type': 'kernel-ci', 'image_url': base_url, 'modules_url': modules_url, 'plan': plan, 'kernel': branch, 'tree': tree, 'defconfig': defconfig, 'fastboot': fastboot, 'priority': args.get('priority'), 'device': device, 'template_file': template_file, 'base_url': base_url, 'endian': endian, 'test_suite': test_suite, 'test_set': test_set, 'test_desc': test_desc, 'test_type': test_type, 'short_template_file': short_template_file, 'arch': arch, 'arch_defconfig': arch_defconfig} jobs.append(job) # print job else: print "no kernel_image for %s" % build['defconfig_full'] for job in jobs: job_file = job_dir + '/' + job['name'] + '.yaml' with open(job_file, 'w') as f: f.write(jinja_render(job)) print "Job written: %s" % job_file