def main(args): global CONFIG CONFIG = configuration.get_config(args) test_case_definition_dir = CONFIG.get("testDir") + "/" + TEST_DIR_BASE_NAME test_plan_definition_dir = CONFIG.get("testDir") + "/" + PLAN_DIR_BASE_NAME test_case_definition_file_list = common.find_all_test_case( CONFIG.get("plan"), test_case_definition_dir, test_plan_definition_dir) test_case_definition_url = "https://github.com/qinshulei/ci-test-cases" if CONFIG.get("testUrl") is not None and CONFIG.get("testUrl") != "": test_case_definition_url = CONFIG.get("testUrl") setup_job_dir(os.getcwd() + '/jobs') print 'Scanning %s for kernel information...' % CONFIG.get("url") distro = CONFIG.get("distro") if distro is None: distro = "Ubuntu" if CONFIG.get("tree") == "open-estuary": walk_url(CONFIG.get("url"), CONFIG.get("url"), CONFIG.get("plans"), CONFIG.get("arch"), CONFIG.get("targets"), CONFIG.get("priority"), distro, CONFIG.get("scope"), CONFIG.get("level"), test_case_definition_dir, test_case_definition_file_list, test_case_definition_url) elif CONFIG.get("tree") == "linaro": create_jobs2(CONFIG.get("plans"), "D05", CONFIG.get("targets"), CONFIG.get("priority"), distro, CONFIG.get("scope"), CONFIG.get("level"), test_case_definition_dir, test_case_definition_file_list, test_case_definition_url) print 'Done scanning for kernel information' print 'Done creating YAML jobs' exit(0)
def main(args): config = configuration.get_config(args) token = config.get('token') api = config.get('api') storage = config.get('storage') builds_json = config.get('builds') print("Working on kernel {}/{}".format( config.get('tree'), config.get('branch'))) if not storage: raise Exception("No KernelCI storage URL provided") if builds_json: print("Getting builds from {}".format(builds_json)) with open(builds_json) as json_file: builds = json.load(json_file) else: print("Getting builds from KernelCI API") if not token: raise Exception("No KernelCI API token provided") if not api: raise Exception("No KernelCI API URL provided") builds = get_builds(api, token, config) print("Number of builds: {}".format(len(builds))) jobs = get_jobs_from_builds(config, builds) print("Number of jobs: {}".format(len(jobs))) write_jobs(config, jobs)
def main(args): config = configuration.get_config(args) url = utils.validate_input(config.get("username"), config.get("token"), config.get("server")) connection = utils.connect(url) if config.get("repo"): retrieve_jobs(config.get("repo")) load_jobs() start_time = time.time() bundle_stream = None if config.get("stream"): bundle_stream = config.get("stream") submit_jobs(connection, config.get("server"), bundle_stream=bundle_stream) if config.get("poll"): jobs = poll_jobs(connection, config.get("timeout")) end_time = time.time() if config.get("bisect"): for job_id in jobs: if 'result' in jobs[job_id]: if jobs[job_id]['result'] == 'FAIL': exit(1) jobs['duration'] = end_time - start_time jobs['username'] = config.get("username") jobs['token'] = config.get("token") jobs['server'] = config.get("server") results_directory = os.getcwd() + '/results' utils.mkdir(results_directory) utils.write_json(config.get("poll"), results_directory, jobs) exit(0)
def main(args): config = configuration.get_config(args) token = config.get('token') api = config.get('api') storage = config.get('storage') builds_json = config.get('builds') print("Working on kernel {}/{}".format(config.get('tree'), config.get('branch'))) if not storage: raise Exception("No KernelCI storage URL provided") if builds_json: print("Getting builds from {}".format(builds_json)) with open(builds_json) as json_file: builds = json.load(json_file) else: print("Getting builds from KernelCI API") if not token: raise Exception("No KernelCI API token provided") if not api: raise Exception("No KernelCI API URL provided") builds = get_builds(api, token, config) print("Number of builds: {}".format(len(builds))) jobs = get_jobs_from_builds(config, builds) print("Number of jobs: {}".format(len(jobs))) write_jobs(config, jobs)
def main(args): config = configuration.get_config(args) url = utils.validate_input(config.get("username"), config.get("token"), config.get("server")) connection = utils.connect(url) if config.get("repo"): retrieve_jobs(config.get("repo")) load_jobs() start_time = time.time() bundle_stream = None if config.get("stream"): bundle_stream = config.get("stream") submit_jobs(connection, config.get("server"), bundle_stream=bundle_stream) if config.get("poll"): jobs = poll_jobs(connection, config.get("timeout")) end_time = time.time() jobs['duration'] = end_time - start_time jobs['username'] = config.get("username") jobs['token'] = config.get("token") jobs['server'] = config.get("server") results_directory = os.getcwd() + '/results' utils.mkdir(results_directory) utils.write_yaml(config.get("poll"), results_directory, jobs) exit(0)
def airflow_task_success(context): section = get_section(context['task'].task_id) dag_id = context['dag'].dag_id from lib.configuration import get_config if dag_id in [ '01_update_granted_patent', '02_gi_post_manual', '03_disambiguation_support', '04_disambiguation_postprocessing', '99_daily_checks' ]: type = 'granted_patent' else: type = 'application' config = get_config(type) message = 'AIRFLOW TASK Success:\n' \ 'DAG: {dag_id}\n' \ 'TASKS: {task_id}\n' \ 'Duration: {duration}\n' \ .format(dag_id=context['task_instance'].dag_id, task_id=context['task_instance'].task_id, duration=context['task_instance'].duration) report_message = get_report_message(context['task'].task_id, config) send_slack_notification(report_message, config, section=section, level='success') send_slack_notification(message, config, section=section, level='success')
def clean_data(): yield config = get_config() config["DATABASE"]["NEW_DB"] = 'sarvo_test_db' from sqlalchemy import create_engine engine = create_engine(get_connection_string(config, "NEW_DB")) engine.execute("TRUNCATE TABLE inventor;") engine.execute("TRUNCATE TABLE disambiguated_inventor_ids;")
def main(args): config = configuration.get_config(args) setup_job_dir(os.getcwd() + "/jobs") print "Scanning %s for kernel information..." % config.get("url") walk_url(config.get("url"), config.get("plans"), config.get("arch"), config.get("targets"), config.get("priority")) print "Done scanning for kernel information" print "Done creating JSON jobs" exit(0)
def main(args): config = configuration.get_config(args) if config.get("boot"): boot_report(config) generate_email_test_report() generate_current_test_report() generate_history_test_report() exit(0)
def main(args): config = configuration.get_config(args) setup_job_dir(os.getcwd() + '/jobs') print 'Scanning %s for kernel information...' % config.get("url") walk_url(config.get("url"), config.get("url"), config.get("plans"), config.get("arch"), config.get("targets"), config.get("priority")) print 'Done scanning for kernel information' print 'Done creating JSON jobs' exit(0)
def main(args): global directory config = configuration.get_config(args) if config.get("jobs"): directory = setup_job_dir(config.get("jobs")) else: directory = setup_job_dir(os.getcwd() + '/jobs') print 'Scanning %s for kernel information...' % config.get("url") walk_url(config.get("url"), config.get("plans"), config.get("arch"), config.get("targets"), config.get("priority")) print 'Done scanning for kernel information' print 'Done creating JSON jobs' exit(0)
def main(args): config = configuration.get_config(args) global TEST_CASE_DEFINITION_DIR TEST_CASE_DEFINITION_DIR = config.get("testDir") if config.get("boot"): boot_report(config) generate_current_test_report() generate_email_test_report() generate_history_test_report() exit(0)
def airflow_daily_check_failure(context): section = get_section(context['task'].task_id) from lib.configuration import get_config config = get_config(type) message = 'AIRFLOW TASK FAILURE:\n' \ 'DAG: {dag_id}\n' \ 'TASKS: {task_id}\n' \ 'Duration: {duration}\n' \ 'Reason: {exception}\n' \ .format(dag_id=context['task_instance'].dag_id, task_id=context['task_instance'].task_id, duration=context['task_instance'].duration, exception=context['exception']) config["SLACK"]["CHANNEL"] = "pv_server_status" send_slack_notification(message, config, section=section, level='error')
def main(args): config = configuration.get_config(args) url = utils.validate_input(config.get("username"), config.get("token"), config.get("server")) connection = utils.connect(url) load_jobs() start_time = time.time() bundle_stream = None if config.get("stream"): bundle_stream = config.get("stream") create_jobs(connection, config.get("server"), bundle_stream=bundle_stream) exit(0)
def main(args): global directory config = configuration.get_config(args) if config.get("jobs"): directory = setup_job_dir(config.get("jobs")) else: directory = setup_job_dir(os.getcwd() + '/jobs') url = config.get('url') arch = config.get('arch') if arch: url += arch + "/" print 'Scanning %s for kernel information...' % config.get("url") walk_url(url, config.get("plans"), arch, config.get("targets"), config.get("priority")) print 'Done scanning for kernel information' print 'Done creating JSON jobs' exit(0)
def airflow_daily_check_success(context): section = get_section(context['task'].task_id) from lib.configuration import get_config config = get_config() message = 'AIRFLOW TASK Success:\n' \ 'DAG: {dag_id}\n' \ 'TASKS: {task_id}\n' \ 'Duration: {duration}\n' \ .format(dag_id=context['task_instance'].dag_id, task_id=context['task_instance'].task_id, duration=context['task_instance'].duration) report_message = get_report_message(context['task'].task_id, config) config["SLACK"]["CHANNEL"] = "pv_server_status" send_slack_notification(report_message, config, section=section, level='success') send_slack_notification(message, config, section=section, level='success')
def main(args): config = configuration.get_config(args) lava_connection = LavaConnection(config) lava_job = LavaRunJob(lava_connection, config.get('job'), 2) lava_job.connect() if config.get("curses"): output_handler = CursesOutput(lava_job, config.get("interval")) else: output_handler = FileOutputHandler(sys.stdout, lava_job, config.get("interval")) output_handler.run() exit(0)
def main(args): config = configuration.get_config(args) if config.get("repo"): retrieve_jobs(config.get("repo")) if config.get("jobs"): load_jobs(config.get("jobs")) print "Loading jobs from top folder " + str(config.get("jobs")) else: load_jobs(os.getcwd()) if job_map: url = utils.validate_input(config.get("username"), config.get("token"), config.get("server")) connection = utils.connect(url) submit_jobs(connection) exit(0)
def main(args): config = configuration.get_config(args) if config.get("repo"): retrieve_jobs(config.get("repo")) jobs = config.get("jobs") print("Loading jobs from {}".format(jobs)) load_jobs(jobs) if JOBS: start_time = time.time() url = utils.validate_input(config.get("username"), config.get("token"), config.get("server")) connection = utils.connect(url) submit_jobs(connection) exit(0)
def main(args): config = configuration.get_config(args) jenkins_build_url = os.getenv('BUILD_URL', '') global TEST_CASE_DEFINITION_DIR TEST_CASE_DEFINITION_DIR = config.get("testDir") distro = config.get("distro") if config.get("boot"): boot_report(config) module_dict = generate_module_dict(job_result_dict, TEST_CASE_DEFINITION_DIR) generate_scope_test_report(TEST_CASE_DEFINITION_DIR, module_dict, jenkins_build_url, distro) generate_current_test_report() generate_email_test_report(distro, module_dict, jenkins_build_url) generate_history_test_report() exit(0)
def main(args): global test_kind config = configuration.get_config(args) setup_job_dir(os.getcwd() + '/jobs') print 'Scanning %s for kernel information...' % config.get("url") distro = config.get("distro") if distro is None: distro = "Ubuntu" test_kind = config.get("testClassify") if test_kind is None: test_kind = "BOTH" walk_url(config.get("url"), config.get("url"), config.get("plans"), config.get("arch"), config.get("targets"), config.get("priority"), distro, config.get("SasFlag")) print 'Done scanning for kernel information' print 'Done creating JSON jobs' exit(0)
def main(args): config = configuration.get_config(args) url = utils.validate_input(config.get("username"), config.get("token"), config.get("server")) connection = utils.connect(url) if config.get("repo"): retrieve_jobs(config.get("repo")) if config.get("jobs"): load_jobs(config.get("jobs")) print "Loading jobs from top folder " + str(config.get("jobs")) else: load_jobs(os.getcwd()) submit_jobs(connection) exit(0)
def main(args): conf = configuration.get_config(args) api, token = (conf.get(x) for x in ['api', 'token']) if not token: raise Exception("No token provided") if not api: raise Exception("No KernelCI API URL provided") with open(args['submitted']) as jobs_file: jobs = json.load(jobs_file) lab = jobs['lab'] all_passed = True boots = {} for job_def, job_id in jobs['jobs'].iteritems(): with open(job_def) as job_yaml: job = yaml.safe_load(job_yaml) meta = job['metadata'] boot = poll_boot(token, api, meta, lab, args) if not boot: print("Boot not found, skipping") all_passed = False continue keys = [ 'job', 'git_branch', 'kernel', 'arch', 'defconfig_full', 'lab_name', 'board' ] name = '-'.join(boot[k] for k in keys) print(" id: {}, result: {}".format(boot['_id']['$oid'], boot['status'])) print(" {}".format(name)) if boot['status'] != 'PASS': all_passed = False boots[boot['_id']['$oid']] = boot output_path = args['output'] print("Saving boot results in {}".format(output_path)) with open(output_path, 'w') as output: json.dump(boots, output) sys.exit(0 if all_passed is True else 2)
def main(args): config = configuration.get_config(args) jobs_submitted = config.get('submitted') lab = config.get('lab') if jobs_submitted: if not lab: raise Exception("Lab name required when saving submitted jobs") if os.path.exists(jobs_submitted): os.unlink(jobs_submitted) if config.get("repo"): retrieve_jobs(config.get("repo")) jobs = config.get("jobs") print("Loading jobs from {}".format(jobs)) load_jobs(jobs) if not JOBS: print("No jobs to submit") result = False else: start_time = time.time() labs_config = configparser.ConfigParser() labs_config.read('labs.ini') lava_api = labs_config[config.get("lab")]['api'] print("LAVA API: {}".format(lava_api)) url = utils.validate_input(config.get("username"), config.get("token"), lava_api) connection = utils.connect(url) result = submit_jobs(connection) if jobs_submitted and SUBMITTED: print("Saving submitted jobs data in {}".format(jobs_submitted)) data = { 'start_time': start_time, 'lab': config.get('lab'), 'jobs': {k: v for k, v in SUBMITTED.iteritems() if v}, } with open(jobs_submitted, 'w') as json_file: json.dump(data, json_file) exit(0 if result is True else 1)
def airflow_task_failure(context): section = get_section(context['task'].task_id) from lib.configuration import get_config dag_id = context['dag'].dag_id if dag_id in [ '01_update_granted_patent', '02_gi_post_manual', '03_disambiguation_support', '04_disambiguation_postprocessing' ]: type = 'granted_patent' else: type = 'application' config = get_config(type) message = 'AIRFLOW TASK FAILURE:\n' \ 'DAG: {dag_id}\n' \ 'TASKS: {task_id}\n' \ 'Duration: {duration}\n' \ 'Reason: {exception}\n' \ .format(dag_id=context['task_instance'].dag_id, task_id=context['task_instance'].task_id, duration=context['task_instance'].duration, exception=context['exception']) send_slack_notification(message, config, section=section, level='error')
def main(args): config = configuration.get_config(args) token = config.get('token') api = config.get('api') storage = config.get('storage') builds_json = config.get('builds') print("Working on kernel {}/{}".format(config.get('tree'), config.get('branch'))) if not storage: raise Exception("No KernelCI storage URL provided") if builds_json: print("Getting builds from {}".format(builds_json)) with open(builds_json) as json_file: builds = json.load(json_file) else: print("Getting builds from KernelCI API") if not token: raise Exception("No KernelCI API token provided") if not api: raise Exception("No KernelCI API URL provided") builds = get_builds(api, token, config) print("Number of builds: {}".format(len(builds))) config_data = kernelci.config.test.from_yaml(config.get('test_configs')) plan_configs = config_data['test_plans'] base_plans = config.get('plans') plans = list(plan_config for plan_config in plan_configs.values() if plan_config.base_name in base_plans) tests = config_data['test_configs'] print("Number of test configs: {}".format(len(tests))) jobs = get_jobs_from_builds(config, builds, tests, plans) print("Number of jobs: {}".format(len(jobs))) write_jobs(config, jobs)
def main(args): global test_kind global TEST_CASE_DEFINITION_DIR global TEST_CASE_DEFINITION_FILE_LIST global TEST_CASE_DEFINITION_URL global CONFIG CONFIG = configuration.get_config(args) TEST_CASE_DEFINITION_DIR = CONFIG.get("testDir") TEST_CASE_DEFINITION_FILE_LIST = find_all_test_case( TEST_CASE_DEFINITION_DIR) TEST_CASE_DEFINITION_URL = "https://github.com/qinshulei/ci-test-cases" if CONFIG.get("testUrl") != None and CONFIG.get("testUrl") != "": TEST_CASE_DEFINITION_URL = CONFIG.get("testUrl") setup_job_dir(os.getcwd() + '/jobs') print 'Scanning %s for kernel information...' % CONFIG.get("url") distro = CONFIG.get("distro") if distro is None: distro = "Ubuntu" test_kind = CONFIG.get("testClassify") if test_kind is None: test_kind = "BOTH" if CONFIG.get("tree") == "open-estuary": walk_url(CONFIG.get("url"), CONFIG.get("url"), CONFIG.get("plans"), CONFIG.get("arch"), CONFIG.get("targets"), CONFIG.get("priority"), distro, CONFIG.get("scope"), CONFIG.get("level")) elif CONFIG.get("tree") == "linaro": create_new_jobs(CONFIG.get("plans"), "D05", CONFIG.get("targets"), CONFIG.get("priority"), distro, CONFIG.get("scope"), CONFIG.get("level")) print 'Done scanning for kernel information' print 'Done creating YAML jobs' exit(0)
def main(args): config = configuration.get_config(args) token = config.get('token') api = config.get('api') if not token: raise Exception("No KernelCI API token provided") if not api: raise Exception("No KernelCI API URL provided") upload_path = '/'.join(args[k] for k in [ 'tree', 'branch', 'kernel', 'arch', 'defconfig', 'lab']) log_file_name = 'bisect-{}.json'.format(args['target']) print("Uploading bisect log: {}".format(upload_path)) upload_log(args, upload_path, log_file_name, token, api) print("Sending bisection results") send_result(args, log_file_name, token, api) print("Sending bisection report email") send_report(args, log_file_name, token, api)
def main(args): config = configuration.get_config(args) url = utils.validate_input(config.get("username"), config.get("token"), config.get("server")) connection = utils.connect(url) if config.get("repo"): retrieve_jobs(config.get("repo")) if config.get("jobs"): load_jobs(config.get("jobs")) print "Loading jobs from top folder " + str(config.get("jobs")) else: load_jobs(os.getcwd()) start_time = time.time() bundle_stream = None if config.get("stream"): bundle_stream = config.get("stream") submit_jobs(connection, config.get("server"), bundle_stream=bundle_stream) if config.get("poll"): jobs = poll_jobs(connection, config.get("timeout")) end_time = time.time() if config.get("bisect"): for job_id in jobs: if 'result' in jobs[job_id]: if jobs[job_id]['result'] == 'FAIL': exit(1) jobs['duration'] = end_time - start_time jobs['username'] = config.get("username") jobs['token'] = config.get("token") jobs['server'] = config.get("server") results_directory = os.getcwd() + '/results' utils.mkdir(results_directory) utils.write_json(config.get("poll"), results_directory, jobs) exit(0)
def main(args): config = configuration.get_config(args) url = utils.validate_input(config.get("username"), config.get("token"), config.get("server")) connection = utils.connect(url) if config.get("repo"): retrieve_jobs(config.get("repo")) if config.get("jobs"): load_jobs(config.get("jobs")) print "Loading jobs from top folder " + str(config.get("jobs")) else: load_jobs(os.getcwd()) start_time = time.time() bundle_stream = None if config.get("stream"): bundle_stream = config.get("stream") submit_jobs(connection, config.get("server"), bundle_stream=bundle_stream) if config.get("poll"): jobs = poll_jobs(connection, config.get("timeout")) end_time = time.time() if config.get("bisect"): for job_id in jobs: if "result" in jobs[job_id]: if jobs[job_id]["result"] == "FAIL": exit(1) jobs["duration"] = end_time - start_time jobs["username"] = config.get("username") jobs["token"] = config.get("token") jobs["server"] = config.get("server") results_directory = os.getcwd() + "/results" utils.mkdir(results_directory) utils.write_json(config.get("poll"), results_directory, jobs) exit(0)
def main(args): config = configuration.get_config(args) if config.get("boot"): boot_report(config) exit(0)
def main(args): config = configuration.get_config(args) plans = config.get("plans") targets = config.get("targets") if config.get("jobs"): job_dir = setup_job_dir(config.get("jobs")) else: job_dir = setup_job_dir(os.getcwd() + '/jobs') arch = args.get('arch') plans = args.get('plans') branch = args.get('branch') tree = args.get('tree') kernel = tree api = args.get('api') headers = { "Authorization": config.get('token') } print "Working on kernel %s/%s" % (tree, branch) url = urlparse.urljoin(api, ("/build?job=%s&kernel=%s&status=PASS&arch=%s" % (tree, branch, arch))) print "Calling KernelCI API: %s" % url response = requests.get(url, headers=headers) print response data = json.loads(response.content) builds = data['result'] print len(builds) jobs = [] cwd = os.getcwd() for build in builds: defconfig = build['defconfig_full'] arch_defconfig = ("%s-%s" % (arch, defconfig)) print "Working on build %s" % arch_defconfig test_suite = None test_set = None test_desc = None test_type = None plan_defconfigs = [] modules = build['modules'] for plan in plans: if plan != 'boot': config = ConfigParser.ConfigParser() try: config.read(cwd + '/templates/' + plan + '/' + plan + '.ini') test_suite = config.get(plan, 'suite') test_set = config.get(plan, 'set') test_desc = config.get(plan, 'description') test_type = config.get(plan, 'type') plan_defconfigs = config.get(plan, 'defconfigs').split(',') except: print "Unable to load test configuration" exit(1) if build['kernel_image']: # handle devices without a DTB, hacky :/ if build['kernel_image'] == 'bzImage' and arch == 'x86': build['dtb_dir_data'].extend(LEGACY_X86_PLATFORMS) if arch == 'arm64' and 'arm64-defconfig' in arch_defconfig: build['dtb_dir_data'].extend(LEGACY_ARM64_PLATFORMS) for dtb in build['dtb_dir_data']: # hack for arm64 dtbs in subfolders dtb_full = dtb if arch == 'arm64': dtb = str(dtb).split('/')[-1] if dtb in device_map: # print "device %s was in the device_map" % dtb for device in device_map[dtb]: # print "working on device %s" % dtb lpae = device['lpae'] device_type = device['device_type'] fastboot = str(device['fastboot']).lower() blacklist = False nfs_blacklist = False if arch_defconfig in device['defconfig_blacklist']: print "defconfig %s is blacklisted for device %s" % (defconfig, device['device_type']) continue elif "BIG_ENDIAN" in defconfig and plan != 'boot-be': print "BIG_ENDIAN is not supported on %s" % device_type continue elif "LPAE" in defconfig and not lpae: print "LPAE is not support on %s" % device_type continue elif any([x for x in device['kernel_blacklist'] if x in kernel]): print "kernel %s is blacklisted for device %s" % (kernel, device_type) continue elif any([x for x in device['nfs_blacklist'] if x in kernel]) \ and plan in ['boot-nfs', 'boot-nfs-mp']: print "kernel %s is blacklisted for NFS on device %s" % (kernel, device_type) continue elif 'be_blacklist' in device \ and any([x for x in device['be_blacklist'] if x in kernel]) \ and plan in ['boot-be']: print "kernel %s is blacklisted for BE on device %s" % (kernel, device_type) continue elif (arch_defconfig not in plan_defconfigs) and (plan != "boot"): print "defconfig %s not in test plan %s" % (arch_defconfig, plan) continue elif targets is not None and device_type not in targets: print "device_type %s is not in targets %s" % (device_type, targets) else: for template in device['templates']: short_template_file = plan + '/' + str(template) template_file = cwd + '/templates/' + short_template_file if os.path.exists(template_file) and template_file.endswith('.jinja2'): job_name = tree + '-' + branch + '-' + arch + "-" + defconfig[:100] + '-' + dtb + '-' + device_type + '-' + plan base_url = "https://storage.kernelci.org/%s/%s/%s/" % (build['job'], build['kernel'], arch_defconfig) dtb_url = base_url + "dtbs/" + dtb_full kernel_url = urlparse.urljoin(base_url, build['kernel_image']) endian = 'little' if 'BIG_ENDIAN' in defconfig and plan == 'boot-be': endian = 'big' if build['modules']: modules_url = urlparse.urljoin(base_url, build['modules']) else: modules_url = None if device['device_type'].startswith('qemu'): device['device_type'] = 'qemu' job = {'name': job_name, 'dtb_url': dtb_url, 'platform': dtb, 'kernel_url': kernel_url, 'image_type': 'kernel-ci', 'image_url': base_url, 'modules_url': modules_url, 'plan': plan, 'kernel': branch, 'tree': tree, 'defconfig': defconfig, 'fastboot': fastboot, 'priority': args.get('priority'), 'device': device, 'template_file': template_file, 'base_url': base_url, 'endian': endian, 'test_suite': test_suite, 'test_set': test_set, 'test_desc': test_desc, 'test_type': test_type, 'short_template_file': short_template_file, 'arch': arch, 'arch_defconfig': arch_defconfig} jobs.append(job) # print job else: print "no kernel_image for %s" % build['defconfig_full'] for job in jobs: job_file = job_dir + '/' + job['name'] + '.yaml' with open(job_file, 'w') as f: f.write(jinja_render(job)) print "Job written: %s" % job_file
con=engine) assignee_org_data = assignee_organization_with_count.sort_values( "org_count", ascending=False).groupby("assignee_id").head(1).reset_index(drop=True) assignee_org_data = assignee_org_data.drop("org_count", axis=1).assign(name_first=None, name_last=None) assignee_org_data.rename({ "assignee_id": "id" }, axis=1).to_sql(name='assignee', con=engine, if_exists='append', index=False) def post_process_assignee(config): upload_disambig_results(config) update_rawassignee(config) create_assignee(config) def post_process_qc(config): qc = AssigneePostProcessingQC(config) qc.runTests() if __name__ == '__main__': config = get_config() # post_process_assignee(config) post_process_qc(config)
xml_helpers.process_patent_numbers(line.strip('\n'))) withdrawn_patents_frame = pd.DataFrame(withdrawn_patents) withdrawn_patents_frame.columns = ['patent_id'] withdrawn_patents_frame.to_sql(con=engine, name="temp_withdrawn_patent", index=False) def update_withdrawn(update_config): update_query = "UPDATE patent p join temp_withdrawn_patent twp on twp.patent_id = p.id set p.withdrawn = 1; " engine = create_engine(get_connection_string(update_config, "NEW_DB")) engine.execute(update_query) def process_withdrawn(config): withdrawn_folder = '{}/withdrawn'.format( config['FOLDERS']['WORKING_FOLDER']) download_withdrawn_patent_numbers(withdrawn_folder) load_withdrawn(update_config=config) update_withdrawn(update_config=config) def post_withdrawn(config): qc = WithdrawnTest(config) qc.runTests() if __name__ == '__main__': config = get_config(type="granted_patent") #process_withdrawn(config) post_withdrawn(config)
def test_yearly_count(self, table, strict=True): start_date = datetime.datetime.strptime( self.config['DATES']['START_DATE'], '%Y%m%d') end_date = datetime.datetime.strptime(self.config['DATES']['END_DATE'], '%Y%m%d') start_date_string = start_date.strftime('%Y-%m-%d') end_date_string = end_date.strftime('%Y-%m-%d') if table in ['cpc_current', 'wipo']: if not self.connection.open: self.connection.connect() with self.connection.cursor() as count_cursor: in_between_query = "SELECT count(1) as new_count from {table} t join patent p on p.id =t.patent_id and p.date between '{start_dt}' and '{end_dt}'".format( table=table, start_dt=start_date_string, end_dt=end_date_string) count_cursor.execute(in_between_query) count_value = count_cursor.fetchall()[0][0] if count_value < 1: raise AssertionError( "Table doesn't not have new data : {table}, date range '{start_dt}' to '{end_dt}' " .format(table=table, start_dt=start_date_string, end_dt=end_date_string)) super().test_yearly_count(table) if __name__ == '__main__': qc = CPCTest(get_config()) qc.runTests()
def main(args): config = configuration.get_config(args) plans = config.get("plans") targets = config.get("targets") lab_name = config.get('lab') job_dir = setup_job_dir(config.get('jobs') or lab_name) token = config.get('token') api = config.get('api') storage = config.get('storage') if not token: raise Exception("No token provided") if not api: raise Exception("No KernelCI API URL provided") if not storage: raise Exception("No KernelCI storage URL provided") arch = args.get('arch') plans = args.get('plans') branch = args.get('branch') git_describe = args.get('describe') tree = args.get('tree') expected = int(args.get('defconfigs')) headers = { "Authorization": token, } print "Working on kernel %s/%s" % (tree, branch) url_params = urllib.urlencode({ 'job': tree, 'kernel': git_describe, 'git_branch': branch, 'arch': arch, }) url = urlparse.urljoin(api, 'build?{}'.format(url_params)) print "Calling KernelCI API: %s" % url builds = [] loops = 10 retry_time = 30 for loop in range(loops): response = requests.get(url, headers=headers) response.raise_for_status() data = json.loads(response.content) builds = data['result'] if len(builds) >= expected: break print "Got less builds (%s) than expected (%s), retry in %s seconds" % ( len(builds), expected, retry_time) time.sleep(retry_time) print("Number of builds: {}".format(len(builds))) jobs = [] cwd = os.getcwd() for build in builds: defconfig = build['defconfig_full'] arch_defconfig = ("%s-%s" % (arch, defconfig)) print "Working on build %s %s %s %s %s" % (tree, branch, git_describe, arch, defconfig) test_suite = None test_set = None test_desc = None test_type = None plan_defconfigs = [] modules = build['modules'] if build['kernel_image']: if build['kernel_image'] == 'bzImage' and arch == 'x86': build['dtb_dir_data'].extend(LEGACY_X86_PLATFORMS) else: continue if 'PASS' not in build.get('status', ''): continue if arch in ['arm', 'arm64', 'x86'] and 'defconfig' in defconfig: build['dtb_dir_data'].append('qemu') for plan in plans: if plan != 'boot': config = ConfigParser.ConfigParser() try: config.read(cwd + '/templates/' + plan + '/' + plan + '.ini') test_suite = config.get(plan, 'suite') test_set = config.get(plan, 'set') test_desc = config.get(plan, 'description') test_type = config.get(plan, 'type') plan_defconfigs = config.get(plan, 'defconfigs').split(',') except: print "Unable to load test configuration" exit(1) if build['kernel_image']: for dtb in build['dtb_dir_data']: # hack for arm64 dtbs in subfolders dtb_full = dtb if arch == 'arm64': dtb = str(dtb).split('/')[-1] if dtb in device_map: # print "device %s was in the device_map" % dtb for device in device_map[dtb]: # print "working on device %s" % dtb lpae = device['lpae'] device_type = device['device_type'] mach = device['mach'] fastboot = str(device['fastboot']).lower() blacklist = False nfs_blacklist = False if defconfig in device['defconfig_blacklist']: print "defconfig %s is blacklisted for device %s" % ( defconfig, device['device_type']) continue elif device.has_key('defconfig_whitelist' ) and defconfig not in device[ 'defconfig_whitelist']: print "defconfig %s is not in whitelist for device %s" % ( defconfig, device['device_type']) continue elif device.has_key( 'arch_blacklist' ) and arch in device['arch_blacklist']: print "arch %s is blacklisted for device %s" % ( arch, device['device_type']) continue elif device.has_key( 'lab_blacklist' ) and lab_name in device['lab_blacklist']: print "device %s is blacklisted for lab %s" % ( device['device_type'], lab_name) continue elif "BIG_ENDIAN" in defconfig and plan != 'boot-be': print "BIG_ENDIAN is not supported on %s" % device_type continue elif "LPAE" in defconfig and not lpae: print "LPAE is not support on %s" % device_type continue elif any([ x for x in device['kernel_blacklist'] if x in git_describe ]): print "git_describe %s is blacklisted for device %s" % ( git_describe, device_type) continue elif any([x for x in device['nfs_blacklist'] if x in git_describe]) \ and plan in ['boot-nfs', 'boot-nfs-mp']: print "git_describe %s is blacklisted for NFS on device %s" % ( git_describe, device_type) continue elif 'be_blacklist' in device \ and any([x for x in device['be_blacklist'] if x in git_describe]) \ and plan in ['boot-be']: print "git_describe %s is blacklisted for BE on device %s" % ( git_describe, device_type) continue elif (arch_defconfig not in plan_defconfigs) and (plan != "boot"): print "defconfig %s not in test plan %s" % ( arch_defconfig, plan) continue elif targets is not None and device_type not in targets: print "device_type %s is not in targets %s" % ( device_type, targets) elif arch == 'x86' and dtb == 'x86-32' and 'i386' not in arch_defconfig: print "%s is not a 32-bit x86 build, skipping for 32-bit device %s" % ( defconfig, device_type) elif 'kselftest' in defconfig and plan != 'kselftest': print "Skipping kselftest defconfig because plan was not kselftest" else: for template in device['templates']: short_template_file = plan + '/' + str( template) template_file = cwd + '/templates/' + short_template_file if os.path.exists( template_file ) and template_file.endswith('.jinja2'): job_name = tree + '-' + branch + '-' + git_describe + '-' + arch + '-' + defconfig[: 100] + '-' + dtb + '-' + device_type + '-' + plan base_url = "%s/%s/%s/%s/%s/%s/" % ( storage, build['job'], build['git_branch'], build['kernel'], arch, defconfig) nfsrootfs_url = None initrd_url = None callback_name = 'lava/boot' if plan == 'boot' else 'lava/test' context = device[ 'context'] if 'context' in device else None if dtb_full.endswith('.dtb'): dtb_url = base_url + "dtbs/" + dtb_full platform = dtb[:-4] else: dtb_url = None platform = device_type kernel_url = urlparse.urljoin( base_url, build['kernel_image']) defconfig_base = ''.join( defconfig.split('+')[:1]) endian = 'little' if 'BIG_ENDIAN' in defconfig and plan == 'boot-be': endian = 'big' initrd_arch = arch if arch == 'arm64' and endian == 'big': initrd_arch = 'arm64be' if arch == 'arm': if endian == 'big': initrd_arch = 'armeb' else: initrd_arch = 'armel' if 'kselftest' in plan: initrd_url = KSELFTEST_INITRD_URL.format( initrd_arch) else: initrd_url = INITRD_URL.format( initrd_arch) if 'nfs' in plan: nfsrootfs_url = NFSROOTFS_URL.format( initrd_arch) initrd_url = None platform = platform + "_rootfs:nfs" if build['modules']: modules_url = urlparse.urljoin( base_url, build['modules']) else: modules_url = None device_type = device['device_type'] if device_type.startswith( 'qemu' ) or device_type == 'kvm': device_type = 'qemu' job = { 'name': job_name, 'dtb_url': dtb_url, 'dtb_full': dtb_full, 'platform': platform, 'mach': mach, 'kernel_url': kernel_url, 'image_type': 'kernel-ci', 'image_url': base_url, 'modules_url': modules_url, 'plan': plan, 'kernel': git_describe, 'tree': tree, 'defconfig': defconfig, 'fastboot': fastboot, 'priority': args.get('priority'), 'device_type': device_type, 'template_file': template_file, 'base_url': base_url, 'endian': endian, 'test_suite': test_suite, 'test_set': test_set, 'test_desc': test_desc, 'test_type': test_type, 'short_template_file': short_template_file, 'arch': arch, 'arch_defconfig': arch_defconfig, 'git_branch': branch, 'git_commit': build['git_commit'], 'git_describe': git_describe, 'git_url': build['git_url'], 'defconfig_base': defconfig_base, 'initrd_url': initrd_url, 'kernel_image': build['kernel_image'], 'dtb_short': dtb, 'nfsrootfs_url': nfsrootfs_url, 'callback': args.get('callback'), 'api': api, 'lab_name': lab_name, 'callback_name': callback_name, 'context': context, } jobs.append(job) else: print "no kernel_image for %s" % build['defconfig_full'] for job in jobs: job_file = job_dir + '/' + job['name'] + '.yaml' with open(job_file, 'w') as f: f.write(jinja_render(job)) print "Job written: %s" % job_file
detail_desc_length(config) consolidate_uspc(config) pct_data_doc_type(config) consolidate_claim(config) consolidate_usreldoc(config) yearly_claim(config) yearly_brf_sum_text(config) yearly_draw_desc_text(config) yearly_detail_desc_text(config) def post_upload_database(**kwargs): config = update_config_date(**kwargs) qc = AppUploadTest(config) qc.runTests() if __name__ == "__main__": config = get_config('application') consolidate_rawlocation(config) consolidate_cpc(config) detail_desc_length(config) consolidate_uspc(config) pct_data_doc_type(config) consolidate_claim(config) consolidate_usreldoc(config) yearly_claim(config) yearly_brf_sum_text(config) yearly_draw_desc_text(config) yearly_detail_desc_text(config)
def main(args): config = configuration.get_config(args) plans = config.get("plans") targets = config.get("targets") if config.get("jobs"): job_dir = setup_job_dir(config.get("jobs")) else: job_dir = setup_job_dir(os.getcwd() + '/jobs') arch = args.get('arch') plans = args.get('plans') branch = args.get('branch') tree = args.get('tree') kernel = tree api = args.get('api') headers = {"Authorization": config.get('token')} print "Working on kernel %s/%s" % (tree, branch) url = urlparse.urljoin(api, ("/build?job=%s&kernel=%s&status=PASS&arch=%s" % (tree, branch, arch))) print "Calling KernelCI API: %s" % url response = requests.get(url, headers=headers) if response.status_code != 200: print "Error calling KernelCI API" print response sys.exit(1) data = json.loads(response.content) builds = data['result'] print len(builds) jobs = [] cwd = os.getcwd() for build in builds: defconfig = build['defconfig_full'] arch_defconfig = ("%s-%s" % (arch, defconfig)) print "Working on build %s" % arch_defconfig test_suite = None test_set = None test_desc = None test_type = None plan_defconfigs = [] modules = build['modules'] for plan in plans: if plan != 'boot': config = ConfigParser.ConfigParser() try: config.read(cwd + '/templates/' + plan + '/' + plan + '.ini') test_suite = config.get(plan, 'suite') test_set = config.get(plan, 'set') test_desc = config.get(plan, 'description') test_type = config.get(plan, 'type') plan_defconfigs = config.get(plan, 'defconfigs').split(',') except: print "Unable to load test configuration" exit(1) if build['kernel_image']: # handle devices without a DTB, hacky :/ if build['kernel_image'] == 'bzImage' and arch == 'x86': build['dtb_dir_data'].extend(LEGACY_X86_PLATFORMS) if arch == 'arm64' and 'arm64-defconfig' in arch_defconfig: build['dtb_dir_data'].extend(LEGACY_ARM64_PLATFORMS) for dtb in build['dtb_dir_data']: # hack for arm64 dtbs in subfolders dtb_full = dtb if arch == 'arm64': dtb = str(dtb).split('/')[-1] if dtb in device_map: # print "device %s was in the device_map" % dtb for device in device_map[dtb]: # print "working on device %s" % dtb lpae = device['lpae'] device_type = device['device_type'] fastboot = str(device['fastboot']).lower() blacklist = False nfs_blacklist = False if arch_defconfig in device['defconfig_blacklist']: print "defconfig %s is blacklisted for device %s" % ( defconfig, device['device_type']) continue elif "BIG_ENDIAN" in defconfig and plan != 'boot-be': print "BIG_ENDIAN is not supported on %s" % device_type continue elif "LPAE" in defconfig and not lpae: print "LPAE is not support on %s" % device_type continue elif any([ x for x in device['kernel_blacklist'] if x in kernel ]): print "kernel %s is blacklisted for device %s" % ( kernel, device_type) continue elif any([x for x in device['nfs_blacklist'] if x in kernel]) \ and plan in ['boot-nfs', 'boot-nfs-mp']: print "kernel %s is blacklisted for NFS on device %s" % ( kernel, device_type) continue elif 'be_blacklist' in device \ and any([x for x in device['be_blacklist'] if x in kernel]) \ and plan in ['boot-be']: print "kernel %s is blacklisted for BE on device %s" % ( kernel, device_type) continue elif (arch_defconfig not in plan_defconfigs) and (plan != "boot"): print "defconfig %s not in test plan %s" % ( arch_defconfig, plan) continue elif targets is not None and device_type not in targets: print "device_type %s is not in targets %s" % ( device_type, targets) else: for template in device['templates']: short_template_file = plan + '/' + str( template) template_file = cwd + '/templates/' + short_template_file if os.path.exists( template_file ) and template_file.endswith('.jinja2'): job_name = tree + '-' + branch + '-' + arch + "-" + defconfig[: 100] + '-' + dtb + '-' + device_type + '-' + plan base_url = "https://storage.kernelci.org/%s/%s/%s/" % ( build['job'], build['kernel'], arch_defconfig) dtb_url = base_url + "dtbs/" + dtb_full kernel_url = urlparse.urljoin( base_url, build['kernel_image']) defconfig_base = ''.join( defconfig.split('+')[:1]) endian = 'little' if 'BIG_ENDIAN' in defconfig and plan == 'boot-be': endian = 'big' initrd_arch = arch if arch not in INITRD_URLS.keys(): if arch == 'arm64' and endian == 'big': initrd_arch = 'arm64be' if arch == 'arm': if endian == 'big': initrd_arch = 'armeb' else: initrd_arch = 'armel' initrd_url = INITRD_URLS[initrd_arch] if build['modules']: modules_url = urlparse.urljoin( base_url, build['modules']) else: modules_url = None if device['device_type'].startswith( 'qemu' ) or device['device_type'] == 'kvm': device['device_type'] = 'qemu' job = { 'name': job_name, 'dtb_url': dtb_url, 'platform': dtb_full, 'kernel_url': kernel_url, 'image_type': 'kernel-ci', 'image_url': base_url, 'modules_url': modules_url, 'plan': plan, 'kernel': branch, 'tree': tree, 'defconfig': defconfig, 'fastboot': fastboot, 'priority': args.get('priority'), 'device': device, 'template_file': template_file, 'base_url': base_url, 'endian': endian, 'test_suite': test_suite, 'test_set': test_set, 'test_desc': test_desc, 'test_type': test_type, 'short_template_file': short_template_file, 'arch': arch, 'arch_defconfig': arch_defconfig, 'git_branch': build['git_branch'], 'git_commit': build['git_commit'], 'git_describe': build['git_describe'], 'defconfig_base': defconfig_base, 'initrd_url': initrd_url, 'kernel_image': build['kernel_image'], 'dtb_short': dtb } jobs.append(job) else: print "no kernel_image for %s" % build['defconfig_full'] for job in jobs: job_file = job_dir + '/' + job['name'] + '.yaml' with open(job_file, 'w') as f: f.write(jinja_render(job)) print "Job written: %s" % job_file