def get_build_data(self, depth=1): build_url = "/".join([JENKINS["url"], 'job', self.name, str(self.number), 'api/json?depth={depth}'.format(depth=depth)]) logger.debug("Request build data from {}".format(build_url)) return requests.get(build_url).json()
def get_test_build(build_name, build_number, check_rebuild=False, force_rebuild_search=False): """Get test data from Jenkins job build :param build_name: string :param build_number: string :param check_rebuild: bool, if True then look for newer job rebuild(s) :param force_rebuild_search: bool, if True then force rebuild(s) search :return: dict """ test_build = Build(build_name, build_number) first_case = test_build.test_data()['suites'][0]['cases'].pop()['name'] if (force_rebuild_search or first_case == 'jenkins') and check_rebuild: iso_magnet = get_job_parameter(test_build.build_data, 'MAGNET_LINK') if not iso_magnet: return test_build latest_build_number = Build(build_name, 'latest').number builds_to_check = [i for i in range(build_number + 1, latest_build_number + 1)] if force_rebuild_search: builds_to_check.reverse() for n in builds_to_check: test_rebuild = Build(build_name, n) if get_job_parameter(test_rebuild.build_data, 'MAGNET_LINK') \ == iso_magnet: logger.debug("Found test job rebuild: " "{0}".format(test_rebuild.url)) return test_rebuild return test_build
def _is_excluded(case_name, exclude): if exclude and exclude in case_name: logger.debug("Skipping '{0}' test because it contains" " '{1}' in method name".format(case_name, exclude)) return True else: return False
def _is_not_included(case_name, include): if include and include not in case_name: logger.debug("Skipping '{0}' test because it doesn't " "contain '{1}' in method name".format(case_name, include)) return True else: return False
def get_tests_groups_from_jenkins(runner_name, build_number, distros): runner_build = Build(runner_name, build_number) res = {} for b in runner_build.build_data['subBuilds']: if b['result'] is None: logger.debug("Skipping '{0}' job (build #{1}) because it's still " "running...".format(b['jobName'], b['buildNumber'],)) continue # Get the test group from the console of the job z = Build(b['jobName'], b['buildNumber']) console = z.get_job_console() groups = [keyword.split('=')[1] for line in console for keyword in line.split() if 'run_system_test.py' in line and '--group=' in keyword] if not groups: logger.error("No test group found in console of the job {0}/{1}" .format(b['jobName'], b['buildNumber'])) continue # Use the last group (there can be several groups in upgrade jobs) test_group = groups[-1] # Get the job suffix job_name = b['jobName'] for distro in distros: if distro in job_name: sep = '.' + distro + '.' job_suffix = job_name.split(sep)[-1] break else: job_suffix = job_name.split('.')[-1] res[job_suffix] = test_group return res
def get_jobs_for_view(view): """Return list of jobs from specified view """ view_url = "/".join([JENKINS["url"], 'view', view, 'api/json']) logger.debug("Request view data from {}".format(view_url)) view_data = requests.get(view_url).json() jobs = [job["name"] for job in view_data["jobs"]] return jobs
def get_tests_groups_from_jenkins(runner_name, build_number, distros): runner_build = Build(runner_name, build_number) res = {} sub_builds = \ runner_build.build_data.get('subBuilds', [runner_build.build_data]) for b in sub_builds: if b['result'] is None: logger.debug("Skipping '{0}' job (build #{1}) because it's still " "running...".format( b['jobName'], b['buildNumber'], )) continue # Get the test group from the console of the job # Get the job suffix if b.get('jobName'): z = Build(b['jobName'], b['buildNumber']) console = z.get_job_console() job_name = b['jobName'] job_info = z.job_info env_vars = z.injected_vars else: console = runner_build.get_job_console() job_name = runner_build.name job_info = runner_build.job_info env_vars = runner_build.injected_vars groups = re.findall(TEST_GROUP_PATTERN, console) if not groups: # maybe it's failed baremetal job? # because of a design baremetal tests run pre-setup job # and when it fails there are no test groups in common meaning: # groups which could be parsed by TEST_GROUP_PATTERN baremetal_pattern = re.compile(r'Jenkins Build.*jenkins-(.*)-\d+') baremetal_groups = re.findall(baremetal_pattern, console) if not baremetal_groups: logger.error( "No test group found in console of the job {0}/{1}".format( b['jobName'], b['buildNumber'])) continue # we should get the group via jobName because the test group name # inside the log could be cut and some symbols will be changed to * groups = b['jobName'].split('.') # Use the last group (there can be several groups in upgrade jobs) test_group = groups[-1] for distro in distros: if distro in job_name: sep = '.' + distro + '.' job_suffix = job_name.split(sep)[-1] break else: job_suffix = job_name.split('.')[-1] res[job_suffix] = \ {'group': test_group, 'job_info': job_info, 'env_vars': env_vars} return res
def bugs_statistics(self): if self._bugs_statistics != {}: return self._bugs_statistics logger.info( 'Collecting stats for TestRun "{0}" on "{1}"...'.format( self.run["name"], self.run["config"] or "default config" ) ) for test in self.tests: logger.debug('Checking "{0}" test...'.format(test["title"])) test_results = sorted( self.project.get_results_for_test(test["id"], self.results), key=lambda x: x["id"], reverse=True ) linked_bugs = [] is_blocked = False for result in test_results: if result["status_id"] in self.blocked_statuses: if self.check_blocked: new_bug_link = self.handle_blocked(test, result) if new_bug_link: linked_bugs.append(new_bug_link) is_blocked = True break if result["custom_launchpad_bug"]: linked_bugs.append(result["custom_launchpad_bug"]) is_blocked = True break if result["status_id"] in self.failed_statuses and result["custom_launchpad_bug"]: linked_bugs.append(result["custom_launchpad_bug"]) bug_ids = set( [ re.search(r".*bugs?/(\d+)/?", link).group(1) for link in linked_bugs if re.search(r".*bugs?/(\d+)/?", link) ] ) for bug_id in bug_ids: if bug_id in self._bugs_statistics: self._bugs_statistics[bug_id][test["id"]] = { "group": test["custom_test_group"] or "manual", "config": self.run["config"] or "default", "blocked": is_blocked, } else: self._bugs_statistics[bug_id] = { test["id"]: { "group": test["custom_test_group"] or "manual", "config": self.run["config"] or "default", "blocked": is_blocked, } } return self._bugs_statistics
def publish_results(project, milestone_id, test_plan, suite_id, config_id, results): test_run_ids = [ run['id'] for entry in test_plan['entries'] for run in entry['runs'] if suite_id == run['suite_id'] and config_id in run['config_ids'] ] logger.debug('Looking for previous tests runs on "{0}" using tests suite ' '"{1}"...'.format( project.get_config(config_id)['name'], project.get_suite(suite_id)['name'])) previous_tests_runs = project.get_previous_runs( milestone_id=milestone_id, suite_id=suite_id, config_id=config_id, limit=TestRailSettings.previous_results_depth) cases = project.get_cases(suite_id=suite_id) tests = project.get_tests(run_id=test_run_ids[0]) results_to_publish = [] for result in results: test = project.get_test_by_group(run_id=test_run_ids[0], group=result.group, tests=tests) if not test: logger.error("Test for '{0}' group not found: {1}".format( result.group, result.url)) continue existing_results_versions = [ r['version'] for r in project.get_results_for_test(test['id']) ] if result.version in existing_results_versions: continue if result.status not in ('passed', 'blocked'): case_id = project.get_case_by_group(suite_id=suite_id, group=result.group, cases=cases)['id'] run_ids = [ run['id'] for run in previous_tests_runs[0:int(TestRailSettings. previous_results_depth)] ] previous_results = project.get_all_results_for_case( run_ids=run_ids, case_id=case_id) lp_bug = get_existing_bug_link(previous_results) if lp_bug: result.launchpad_bug = lp_bug['bug_link'] results_to_publish.append(result) try: if len(results_to_publish) > 0: project.add_results_for_cases(run_id=test_run_ids[0], suite_id=suite_id, tests_results=results_to_publish) except: logger.error('Failed to add new results for tests: {0}'.format( [r.group for r in results_to_publish])) raise return results_to_publish
def get_test_data(url, result_path=None): if result_path: test_url = "/".join( [url.rstrip("/"), 'testReport'] + result_path + ['api/json']) else: test_url = "/".join([url.rstrip("/"), 'testReport', 'api/json']) logger.debug("Request test data from {}".format(test_url)) return requests.get(test_url).json()
def add_case(testrail_project, suite, test_case, section, do_check=False): try_msg = "try to add test: {} to section: {}" miss_msg = "test: {} is already on section: {}" if do_check and testrail_project.get_case_by_name(suite['id'], test_case['title']): logger.debug(miss_msg.format(test_case["title"], test_case["section"])) return logger.debug(try_msg.format(test_case["title"], test_case["section"])) testrail_project.add_case(section_id=section["id"], case=test_case)
def get_test_data(url, result_path=None): if result_path: test_url = "/".join([url.rstrip("/"), 'testReport'] + result_path + ['api/json']) else: test_url = "/".join([url.rstrip("/"), 'testReport', 'api/json']) logger.debug("Request test data from {}".format(test_url)) return requests.get(test_url).json()
def publish_results(project, milestone_id, test_plan, suite_id, config_id, results): test_run_ids = [run['id'] for entry in test_plan['entries'] for run in entry['runs'] if suite_id == run['suite_id'] and config_id in run['config_ids']] logger.debug('Looking for previous tests runs on "{0}" using tests suite ' '"{1}"...'.format(project.get_config(config_id)['name'], project.get_suite(suite_id)['name'])) previous_tests_runs = project.get_previous_runs( milestone_id=milestone_id, suite_id=suite_id, config_id=config_id, limit=TestRailSettings.previous_results_depth) logger.debug('Found next test runs: {0}'.format( [test_run['description'] for test_run in previous_tests_runs])) cases = project.get_cases(suite_id=suite_id) tests = project.get_tests(run_id=test_run_ids[0]) results_to_publish = [] for result in results: test = project.get_test_by_group(run_id=test_run_ids[0], group=result.group, tests=tests) if not test: logger.error("Test for '{0}' group not found: {1}".format( result.group, result.url)) continue existing_results_versions = [r['version'] for r in project.get_results_for_test(test['id'])] if result.version in existing_results_versions: continue if result.status not in ('passed', 'blocked'): case_id = project.get_case_by_group(suite_id=suite_id, group=result.group, cases=cases)['id'] run_ids = [run['id'] for run in previous_tests_runs[0: int(TestRailSettings.previous_results_depth)]] previous_results = project.get_all_results_for_case( run_ids=run_ids, case_id=case_id) lp_bug = get_existing_bug_link(previous_results) if lp_bug: result.launchpad_bug = lp_bug['bug_link'] results_to_publish.append(result) try: if len(results_to_publish) > 0: project.add_results_for_cases(run_id=test_run_ids[0], suite_id=suite_id, tests_results=results_to_publish) except: logger.error('Failed to add new results for tests: {0}'.format( [r.group for r in results_to_publish] )) raise return results_to_publish
def get_tests_descriptions(milestone_id, testrail_default_test_priority, testrail_project): # To get the Tempest tests list, need to execute the following commands: # git clone https://github.com/openstack/tempest & cd tempest & tox -evenv # .tox/venv/bin/pip install nose get_tempest_tests = ("cd tempest && .tox/venv/bin/nosetests " "--collect-only tempest/{0} -v 2>&1 | grep 'id-.*'") get_commit = "cd tempest && git rev-parse HEAD" commit = subprocess.Popen(get_commit, shell=True, stdout=subprocess.PIPE) logger.info("Generate test suite for tempest" " commit:{}".format(commit.stdout.readline())) custom_cases_fields = _get_custom_cases_fields( case_fields=testrail_project.get_case_fields(), project_id=testrail_project.project['id']) tests = {} for group in TEST_GROUPS: p = subprocess.Popen(get_tempest_tests.format(group.lower()), shell=True, stdout=subprocess.PIPE) for line in iter(p.stdout.readline, b''): section = generate_groups(line) if group == "API" else group test_class = [] for r in line.split("."): if "id-" in r: title = r.strip() break else: test_class.append(r) steps = [{"run this tempest test": "passed"}, ] test_case = { "title": title, "type_id": 1, "milestone_id": milestone_id, "priority_id": testrail_default_test_priority, "estimate": "1m", "refs": "", "custom_report_label": title.split('id-')[1][:-1], "custom_test_group": ".".join(test_class), "custom_test_case_description": title, "custom_test_case_steps": steps, "section": section } for case_field, default_value in custom_cases_fields.items(): if case_field not in test_case: test_case[case_field] = default_value if section not in tests: tests[section] = [] tests[section].append(test_case) logger.debug(tests) logger.info("total test cases: " "{}".format(sum(map(lambda x: len(x), tests.values())))) return tests
def get_build_artifact(url, artifact): """Return content of job build artifact """ url = "/".join([url, 'artifact', artifact]) logger.debug("Request artifact content from {}".format(url)) req = request.Request(url) opener = request.build_opener(request.HTTPHandler) s = opener.open(req).read() opener.close() return s
def get_test_data(url, result_path=None): if result_path: test_url = "/".join([url.rstrip("/"), 'testReport'] + result_path + ['api/json']) else: test_url = "/".join([url.rstrip("/"), 'testReport', 'api/json']) logger.debug("Request test data from {}".format(test_url)) response = request.urlopen(test_url) return json.load(response)
def bugs_statistics(self): if self._bugs_statistics != {}: return self._bugs_statistics logger.info('Collecting stats for TestRun "{0}" on "{1}"...'.format( self.run['name'], self.run['config'] or 'default config')) for test in self.tests: logger.debug('Checking "{0}" test...'.format(test['title'])) test_results = sorted(self.project.get_results_for_test( test['id'], self.results), key=lambda x: x['id'], reverse=True) linked_bugs = [] is_blocked = False for result in test_results: if result['status_id'] in self.blocked_statuses: if self.check_blocked: new_bug_link = self.handle_blocked(test, result) if new_bug_link: linked_bugs.append(new_bug_link) is_blocked = True break if result['custom_launchpad_bug']: linked_bugs.append(result['custom_launchpad_bug']) is_blocked = True break if result['status_id'] in self.failed_statuses \ and result['custom_launchpad_bug']: linked_bugs.append(result['custom_launchpad_bug']) bug_ids = set([ re.search(r'.*bug/(\d+)/?', link).group(1) for link in linked_bugs if re.search(r'.*bug/(\d+)/?', link) ]) for bug_id in bug_ids: if bug_id in self._bugs_statistics: self._bugs_statistics[bug_id][test['id']] = { 'group': test['custom_test_group'] or 'manual', 'config': self.run['config'] or 'default', 'blocked': is_blocked } else: self._bugs_statistics[bug_id] = { test['id']: { 'group': test['custom_test_group'] or 'manual', 'config': self.run['config'] or 'default', 'blocked': is_blocked } } return self._bugs_statistics
def get_jobs_for_view(view): """Return list of jobs from specified view """ view_url = "/".join([JENKINS["url"], 'view', view, 'api/json']) logger.debug("Request view data from {}".format(view_url)) req = request.Request(view_url) opener = request.build_opener(request.HTTPHandler) s = opener.open(req).read() opener.close() view_data = json.loads(s) jobs = [job["name"] for job in view_data["jobs"]] return jobs
def bugs_statistics(self): if self._bugs_statistics != {}: return self._bugs_statistics logger.info('Collecting stats for TestRun "{0}" on "{1}"...'.format( self.run['name'], self.run['config'] or 'default config')) for test in self.tests: logger.debug('Checking "{0}" test...'.format(test['title'])) test_results = sorted( self.project.get_results_for_test(test['id'], self.results), key=lambda x: x['id'], reverse=True) linked_bugs = [] is_blocked = False for result in test_results: if result['status_id'] in self.blocked_statuses: if self.check_blocked: new_bug_link = self.handle_blocked(test, result) if new_bug_link: linked_bugs.append(new_bug_link) is_blocked = True break if result['custom_launchpad_bug']: linked_bugs.append(result['custom_launchpad_bug']) is_blocked = True break if result['status_id'] in self.failed_statuses \ and result['custom_launchpad_bug']: linked_bugs.append(result['custom_launchpad_bug']) bug_ids = set([re.search(r'.*bug/(\d+)/?', link).group(1) for link in linked_bugs if re.search(r'.*bug/(\d+)/?', link)]) for bug_id in bug_ids: if bug_id in self._bugs_statistics: self._bugs_statistics[bug_id][test['id']] = { 'group': test['custom_test_group'] or 'manual', 'config': self.run['config'] or 'default', 'blocked': is_blocked } else: self._bugs_statistics[bug_id] = { test['id']: { 'group': test['custom_test_group'] or 'manual', 'config': self.run['config'] or 'default', 'blocked': is_blocked } } return self._bugs_statistics
def handle_blocked(self, test, result): if result['custom_launchpad_bug']: return False m = re.search(r'Blocked by "(\S+)" test.', result['comment']) if m: blocked_test_group = m.group(1) else: logger.debug('Blocked result #{0} for test {1} does ' 'not have upstream test name in its ' 'comments!'.format(result['id'], test['custom_test_group'])) return False if not result['version']: logger.debug('Blocked result #{0} for test {1} does ' 'not have version, can\'t find upstream ' 'test case!'.format(result['id'], test['custom_test_group'])) return False bug_link = None blocked_test = self.get_test_by_group(blocked_test_group, result['version']) if not blocked_test: return False logger.debug('Test {0} was blocked by failed test {1}'.format( test['custom_test_group'], blocked_test_group)) blocked_results = self.project.get_results_for_test( blocked_test['id']) # Since we manually add results to failed tests with statuses # ProdFailed, TestFailed, etc. and attach bugs links to them, # we could skip original version copying. So look for test # results with target version, but allow to copy links to bugs # from other results of the same test (newer are checked first) if not any(br['version'] == result['version'] and br['status_id'] in self.failed_statuses for br in blocked_results): logger.debug('Did not find result for test {0} with version ' '{1}!'.format(blocked_test_group, result['version'])) return False for blocked_result in sorted(blocked_results, key=lambda x: x['id'], reverse=True): if blocked_result['status_id'] not in self.failed_statuses: continue if blocked_result['custom_launchpad_bug']: bug_link = blocked_result['custom_launchpad_bug'] break if bug_link is not None: result['custom_launchpad_bug'] = bug_link self.project.add_raw_results_for_test(test['id'], result) logger.info('Added bug {0} to blocked result of {1} test.'.format( bug_link, test['custom_test_group'])) return bug_link return False
def handle_blocked(self, test, result): if result['custom_launchpad_bug']: return False m = re.search(r'Blocked by "(\S+)" test.', result['comment']) if m: blocked_test_group = m.group(1) else: logger.debug('Blocked result #{0} for test {1} does ' 'not have upstream test name in its ' 'comments!'.format(result['id'], test['custom_test_group'])) return False if not result['version']: logger.debug('Blocked result #{0} for test {1} does ' 'not have version, can\'t find upstream ' 'test case!'.format(result['id'], test['custom_test_group'])) return False bug_link = None blocked_test = self.get_test_by_group(blocked_test_group, result['version']) if not blocked_test: return False logger.debug('Test {0} was blocked by failed test {1}'.format( test['custom_test_group'], blocked_test_group)) blocked_results = self.project.get_results_for_test(blocked_test['id']) # Since we manually add results to failed tests with statuses # ProdFailed, TestFailed, etc. and attach bugs links to them, # we could skip original version copying. So look for test # results with target version, but allow to copy links to bugs # from other results of the same test (newer are checked first) if not any(br['version'] == result['version'] and br['status_id'] in self.failed_statuses for br in blocked_results): logger.debug('Did not find result for test {0} with version ' '{1}!'.format(blocked_test_group, result['version'])) return False for blocked_result in sorted(blocked_results, key=lambda x: x['id'], reverse=True): if blocked_result['status_id'] not in self.failed_statuses: continue if blocked_result['custom_launchpad_bug']: bug_link = blocked_result['custom_launchpad_bug'] break if bug_link is not None: result['custom_launchpad_bug'] = bug_link self.project.add_raw_results_for_test(test['id'], result) logger.info('Added bug {0} to blocked result of {1} test.'.format( bug_link, test['custom_test_group'])) return bug_link return False
def get_injected_vars(self, depth=1, build_number=None): if not build_number: return [] job_url = "/".join([JENKINS["url"], 'job', self.name, str(build_number), 'injectedEnvVars', 'api/json?depth={depth}'.format(depth=depth)]) logger.debug("Request injected variables from job {}".format(job_url)) try: result = requests.get(job_url).json() except JSONDecodeError: logger.debug( "Failed to decode injected variables from job {}".format( job_url)) result = [] return result
def upload_tests_descriptions(testrail_project, section_id, tests, check_all_sections): tests_suite = testrail_project.get_suite_by_name( TestRailSettings.tests_suite) check_section = None if check_all_sections else section_id existing_cases = [ case['custom_test_group'] for case in testrail_project.get_cases(suite_id=tests_suite['id'], section_id=check_section) ] custom_cases_fields = {} for field in testrail_project.get_case_fields(): for config in field['configs']: if ((testrail_project.project['id'] in config['context']['project_ids'] or not config['context']['project_ids']) and config['options']['is_required']): try: custom_cases_fields[field['system_name']] = \ int(config['options']['items'].split(',')[0]) except: logger.error("Couldn't find default value for required " "field '{0}', setting '1' (index)!".format( field['system_name'])) custom_cases_fields[field['system_name']] = 1 for test_case in tests: if test_case['custom_test_group'] in existing_cases: logger.debug('Skipping uploading "{0}" test case because it ' 'already exists in "{1}" tests section.'.format( test_case['custom_test_group'], TestRailSettings.tests_suite)) continue for case_field, default_value in custom_cases_fields.items(): if case_field not in test_case: test_case[case_field] = default_value logger.debug('Uploading test "{0}" to TestRail project "{1}", ' 'suite "{2}", section "{3}"'.format( test_case["custom_test_group"], TestRailSettings.project, TestRailSettings.tests_suite, TestRailSettings.tests_section)) testrail_project.add_case(section_id=section_id, case=test_case)
def wrapper(*args, **kwargs): iter_number = 0 while True: try: response = func(*args, **kwargs) except HTTPError as e: if e.code in codes: if iter_number < codes[e.code]: wait = 5 if 'Retry-After' in e.hdrs: wait = int(e.hdrs['Retry-after']) logger.debug(log_msg.format(e.code, wait)) time.sleep(wait) iter_number += 1 continue raise else: return response
def _is_case_processable(case, tests): if not case.entry.info.enabled or not hasattr(case.entry, 'parent'): return False parent_home = case.entry.parent.home if issubclass(parent_home, ActionTest) and \ any([test[GROUP_FIELD] == parent_home.__name__ for test in tests]): return False # Skip @before_class methods without doc strings: # they are just pre-checks, not separate tests cases if case.entry.info.before_class: if case.entry.home.func_doc is None: logger.debug('Skipping method "{0}", because it is not a ' 'test case'.format(case.entry.home.func_name)) return False return True
def get_tests_groups_from_jenkins(runner_name, build_number, distros): runner_build = Build(runner_name, build_number) res = {} sub_builds = \ runner_build.build_data.get('subBuilds', [runner_build.build_data]) for b in sub_builds: if b['result'] is None: logger.debug("Skipping '{0}' job (build #{1}) because it's still " "running...".format(b['jobName'], b['buildNumber'],)) continue # Get the test group from the console of the job # Get the job suffix if b.get('jobName'): z = Build(b['jobName'], b['buildNumber']) console = z.get_job_console() job_name = b['jobName'] job_info = z.job_info env_vars = z.injected_vars else: console = runner_build.get_job_console() job_name = runner_build.name job_info = runner_build.job_info env_vars = runner_build.injected_vars groups = re.findall(TEST_GROUP_PATTERN, console) if not groups: logger.error("No test group found in console of the job {0}/{1}" .format(b['jobName'], b['buildNumber'])) continue # Use the last group (there can be several groups in upgrade jobs) test_group = groups[-1] for distro in distros: if distro in job_name: sep = '.' + distro + '.' job_suffix = job_name.split(sep)[-1] break else: job_suffix = job_name.split('.')[-1] res[job_suffix] = \ {'group': test_group, 'job_info': job_info, 'env_vars': env_vars} return res
def wrapper(*args, **kwargs): iter_number = 0 while True: try: response = func(*args, **kwargs) response.raise_for_status() except HTTPError as e: error_code = e.response.status_code if error_code in codes: if iter_number < codes[error_code]: wait = 5 if 'Retry-After' in e.response.headers: wait = int(e.response.headers['Retry-after']) logger.debug(log_msg.format(error_code, wait)) time.sleep(wait) iter_number += 1 continue raise else: return response.json()
def get_downstream_builds_from_html(url): """Return list of downstream jobs builds from specified job """ url = "/".join([url, 'downstreambuildview/']) logger.debug("Request downstream builds data from {}".format(url)) response = requests.get(url).text jobs = [] raw_downstream_builds = re.findall( '.*downstream-buildview.*href="(/job/\S+/[0-9]+/).*', response) for raw_build in raw_downstream_builds: sub_job_name = raw_build.split('/')[2] sub_job_build = raw_build.split('/')[3] build = Build(name=sub_job_name, number=sub_job_build) jobs.append({ 'name': build.name, 'number': build.number, 'result': build.build_data['result'] }) return jobs
def get_tests_groups_from_jenkins(runner_name, build_number, distros): runner_build = Build(runner_name, build_number) res = {} for b in runner_build.build_data['subBuilds']: if b['result'] is None: logger.debug("Skipping '{0}' job (build #{1}) because it's still " "running...".format( b['jobName'], b['buildNumber'], )) continue # Get the test group from the console of the job z = Build(b['jobName'], b['buildNumber']) console = z.get_job_console() groups = [ keyword.split('=')[1] for line in console for keyword in line.split() if 'run_system_test.py' in line and '--group=' in keyword ] if not groups: logger.error( "No test group found in console of the job {0}/{1}".format( b['jobName'], b['buildNumber'])) continue # Use the last group (there can be several groups in upgrade jobs) test_group = groups[-1] # Get the job suffix job_name = b['jobName'] for distro in distros: if distro in job_name: sep = '.' + distro + '.' job_suffix = job_name.split(sep)[-1] break else: job_suffix = job_name.split('.')[-1] res[job_suffix] = test_group return res
def get_downstream_builds_from_html(url): """Return list of downstream jobs builds from specified job """ url = "/".join([url, 'downstreambuildview/']) logger.debug("Request downstream builds data from {}".format(url)) response = requests.get(url).text jobs = [] raw_downstream_builds = re.findall( '.*downstream-buildview.*href="(/job/\S+/[0-9]+/).*', response) for raw_build in raw_downstream_builds: sub_job_name = raw_build.split('/')[2] sub_job_build = raw_build.split('/')[3] build = Build(name=sub_job_name, number=sub_job_build) jobs.append( { 'name': build.name, 'number': build.number, 'result': build.build_data['result'] } ) return jobs
def get_test_build(build_name, build_number, check_rebuild=False): """Get test data from Jenkins job build :param build_name: string :param build_number: string :param check_rebuild: bool, if True then look for newer job rebuild(s) :return: dict """ test_build = Build(build_name, build_number) if test_build.test_data()['suites'][0]['cases'].pop()['name'] == 'jenkins': if not check_rebuild: return test_build iso_magnet = get_job_parameter(test_build.build_data, 'MAGNET_LINK') if not iso_magnet: return test_build latest_build_number = Build(build_name, 'latest').number for n in range(build_number, latest_build_number): test_rebuild = Build(build_name, n + 1) if get_job_parameter(test_rebuild.build_data, 'MAGNET_LINK') \ == iso_magnet: logger.debug("Found test job rebuild: " "{0}".format(test_rebuild.url)) return test_rebuild return test_build
def upload_tests_descriptions(testrail_project, section_id, tests, check_all_sections): tests_suite = testrail_project.get_suite_by_name( TestRailSettings.tests_suite) check_section = None if check_all_sections else section_id cases = testrail_project.get_cases(suite_id=tests_suite['id'], section_id=check_section) existing_cases = [case[GROUP_FIELD] for case in cases] custom_cases_fields = _get_custom_cases_fields( case_fields=testrail_project.get_case_fields(), project_id=testrail_project.project['id']) for test_case in tests: if test_case[GROUP_FIELD] in existing_cases: testrail_case = _get_testrail_case(testrail_cases=cases, test_case=test_case, group_field=GROUP_FIELD) fields_to_update = _get_fields_to_update(test_case, testrail_case) if fields_to_update: logger.debug('Updating test "{0}" in TestRail project "{1}", ' 'suite "{2}", section "{3}". Updated fields: {4}' .format( test_case[GROUP_FIELD], TestRailSettings.project, TestRailSettings.tests_suite, TestRailSettings.tests_section, ', '.join(fields_to_update.keys()))) testrail_project.update_case(case_id=testrail_case['id'], fields=fields_to_update) else: logger.debug('Skipping "{0}" test case uploading because ' 'it is up-to-date in "{1}" suite' .format(test_case[GROUP_FIELD], TestRailSettings.tests_suite)) else: for case_field, default_value in custom_cases_fields.items(): if case_field not in test_case: test_case[case_field] = default_value logger.debug('Uploading test "{0}" to TestRail project "{1}", ' 'suite "{2}", section "{3}"'.format( test_case[GROUP_FIELD], TestRailSettings.project, TestRailSettings.tests_suite, TestRailSettings.tests_section)) testrail_project.add_case(section_id=section_id, case=test_case)
def upload_tests_descriptions(testrail_project, section_id, tests, check_all_sections): tests_suite = testrail_project.get_suite_by_name( TestRailSettings.tests_suite) check_section = None if check_all_sections else section_id cases = testrail_project.get_cases(suite_id=tests_suite['id'], section_id=check_section) existing_cases = [case[GROUP_FIELD] for case in cases] custom_cases_fields = _get_custom_cases_fields( case_fields=testrail_project.get_case_fields(), project_id=testrail_project.project['id']) for test_case in tests: if test_case[GROUP_FIELD] in existing_cases: testrail_case = _get_testrail_case(testrail_cases=cases, test_case=test_case, group_field=GROUP_FIELD) fields_to_update = _get_fields_to_update(test_case, testrail_case) if fields_to_update: logger.debug( 'Updating test "{0}" in TestRail project "{1}", ' 'suite "{2}", section "{3}". Updated fields: {4}'.format( test_case[GROUP_FIELD], TestRailSettings.project, TestRailSettings.tests_suite, TestRailSettings.tests_section, ', '.join(fields_to_update.keys()))) testrail_project.update_case(case_id=testrail_case['id'], fields=fields_to_update) else: logger.debug('Skipping "{0}" test case uploading because ' 'it is up-to-date in "{1}" suite'.format( test_case[GROUP_FIELD], TestRailSettings.tests_suite)) else: for case_field, default_value in custom_cases_fields.items(): if case_field not in test_case: test_case[case_field] = default_value logger.debug('Uploading test "{0}" to TestRail project "{1}", ' 'suite "{2}", section "{3}"'.format( test_case[GROUP_FIELD], TestRailSettings.project, TestRailSettings.tests_suite, TestRailSettings.tests_section)) testrail_project.add_case(section_id=section_id, case=test_case)
def get_build_artifact(url, artifact): """Return content of job build artifact """ url = "/".join([url, 'artifact', artifact]) logger.debug("Request artifact content from {}".format(url)) return requests.get(url).text
def main(): parser = OptionParser( description="Publish results of system tests from Jenkins build to " "TestRail. See settings.py for configuration.") parser.add_option('-j', '--job-name', dest='job_name', default=None, help='Jenkins swarm runner job name') parser.add_option('-N', '--build-number', dest='build_number', default='latest', help='Jenkins swarm runner build number') parser.add_option("-l", "--live", dest="live_report", action="store_true", help="Get tests results from running swarm") parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="Enable debug output") (options, _) = parser.parse_args() if options.verbose: logger.setLevel(DEBUG) if options.live_report and options.build_number == 'latest': build_number = 'latest_started' else: build_number = options.build_number # STEP #1 # Initialize TestRail Project and define configuration logger.info('Initializing TestRail Project configuration...') project = TestRailProject(url=TestRailSettings.url, user=TestRailSettings.user, password=TestRailSettings.password, project=TestRailSettings.project) logger.info('Initializing TestRail Project configuration... done') operation_systems = [{ 'name': config['name'], 'id': config['id'], 'distro': config['name'].split()[0].lower() } for config in project.get_config_by_name('Operation System')['configs']] os_mile = { '6.1': ['Centos 6.5', 'Ubuntu 14.04'], '6.0.1': ['Centos 6.5', 'Ubuntu 12.04'] } tests_results = {} # STEP #2 # Get tests results from Jenkins runner_build = Build(options.job_name, build_number) runs = runner_build.build_data['runs'] # Analyze each test individually for run_one in runs: if '5.1' in run_one['url']: continue # Release 5.1 to skip tests_result = get_job_info(run_one['url']) if not tests_result['description']: continue # Not completed results to skip if 'skipping' in tests_result['description']: continue # Not performed tests to skip tests_job = { 'result': tests_result['result'], 'name': (options.job_name + '/' + tests_result['url'].split('/')[-3]), 'number': int(tests_result['url'].split('/')[-2]), 'mile': (tests_result['description'].split()[0].split('-')[0]), 'iso': (int(tests_result['description'].split()[0].split('-')[1])) } if tests_job['mile'] not in tests_results: tests_results[tests_job['mile']] = {} test_mile = tests_results[tests_job['mile']] if tests_job['iso'] not in test_mile: test_mile[tests_job['iso']] = {} test_iso = test_mile[tests_job['iso']] for os in operation_systems: if os['distro'] in tests_job['name'].lower() and\ os['name'] in os_mile[tests_job['mile']]: if os['id'] not in test_iso: test_iso[os['id']] = [] test_os_id = test_iso[os['id']] test_os_id.extend(get_tests_results(tests_job, os['distro'])) # STEP #3 # Create new TestPlan in TestRail (or get existing) and add TestRuns for mile in tests_results: mile_tests_suite = '{0}{1}'.format(TestRailSettings.tests_suite, mile) logger.info(mile_tests_suite) tests_suite = project.get_suite_by_name(mile_tests_suite) milestone = project.get_milestone_by_name(name=mile) for iso_number in tests_results.get(mile, {}): # Create new TestPlan name check the same name in testrail test_plan_name = '{milestone} iso #{iso_number}'.format( milestone=milestone['name'], iso_number=iso_number) test_plan = project.get_plan_by_name(test_plan_name) if not test_plan: test_plan = project.add_plan(test_plan_name, description='/'.join([ JENKINS['url'], 'job', '{0}.all'.format( milestone['name']), str(iso_number) ]), milestone_id=milestone['id'], entries=[]) logger.info( 'Created new TestPlan "{0}".'.format(test_plan_name)) else: logger.info( 'Found existing TestPlan "{0}".'.format(test_plan_name)) plan_entries = [] # Create a test plan entry config_ids = [] for os in operation_systems: if os['name'] in os_mile[mile]: config_ids.append(os['id']) cases_ids = [] plan_entries.append( project.test_run_struct( name=tests_suite['name'], suite_id=tests_suite['id'], milestone_id=milestone['id'], description=('Results of system tests ({t_suite})' ' on iso #"{iso_number}"'.format( t_suite=tests_suite['name'], iso_number=iso_number)), config_ids=[os['id']], include_all=True, case_ids=cases_ids)) # Create a test plan entry with the test run run = find_run_by_name(test_plan, tests_suite['name']) if not run: logger.info('Adding a test plan entry with test run %s ...', tests_suite['name']) entry = project.add_plan_entry(plan_id=test_plan['id'], suite_id=tests_suite['id'], config_ids=config_ids, runs=plan_entries) logger.info('The test plan entry has been added.') run = entry['runs'][0] test_plan = project.get_plan(test_plan['id']) # STEP #4 # Upload tests results to TestRail logger.info('Uploading tests results to TestRail...') for os_id in tests_results.get(mile, {})\ .get(iso_number, {}): logger.info('Checking tests results for %s...', project.get_config(os_id)['name']) tests_added = publish_results( project=project, milestone_id=milestone['id'], test_plan=test_plan, suite_id=tests_suite['id'], config_id=os_id, results=tests_results[mile][iso_number][os_id]) logger.debug('Added new results for tests (%s): %s', project.get_config(os_id)['name'], [r.group for r in tests_added]) logger.info('Report URL: %s', test_plan['url'])
def get_tests_descriptions(milestone_id, tests_include, tests_exclude, groups, default_test_priority): discover_import_tests(get_basepath(), tests_directory) define_custom_groups() for one in groups: register_system_test_cases(one) plan = TestPlan.create_from_registry(DEFAULT_REGISTRY) all_plan_tests = plan.tests[:] tests = [] for jenkins_suffix in groups: group = groups[jenkins_suffix] plan.filter(group_names=[group]) for case in plan.tests: if not case.entry.info.enabled: continue home = case.entry.home if not hasattr(case.entry, 'parent'): # Not a real case, some stuff needed by template based tests continue parent_home = case.entry.parent.home case_state = case.state if issubclass(parent_home, ActionTest): case_name = parent_home.__name__ test_group = parent_home.__name__ if any([x['custom_test_group'] == test_group for x in tests]): continue else: case_name = home.func_name test_group = case.entry.home.func_name if tests_include: if tests_include not in case_name: logger.debug("Skipping '{0}' test because it doesn't " "contain '{1}' in method name".format( case_name, tests_include)) continue if tests_exclude: if tests_exclude in case_name: logger.debug("Skipping '{0}' test because it contains" " '{1}' in method name".format( case_name, tests_exclude)) continue if issubclass(parent_home, ActionTest): docstring = parent_home.__doc__.split('\n') case_state.instance._load_config() configuration = case_state.instance.config_name docstring[0] = "{0} on {1}".format(docstring[0], configuration) docstring = '\n'.join(docstring) else: docstring = home.func_doc or '' configuration = None docstring = '\n'.join([s.strip() for s in docstring.split('\n')]) steps = [{ "content": s, "expected": "pass" } for s in docstring.split('\n') if s and s[0].isdigit()] test_duration = re.search(r'Duration\s+(\d+[s,m])\b', docstring) title = docstring.split('\n')[0] or case.entry.home.func_name if case.entry.home.func_name in GROUPS_TO_EXPAND: """Expand specified test names with the group names that are used in jenkins jobs where this test is started. """ title = ' - '.join([title, jenkins_suffix]) test_group = '_'.join( [case.entry.home.func_name, jenkins_suffix])
def get_job_info(url): job_url = "/".join([url, 'api/json']) logger.debug("Request job info from %s", job_url) return requests.get(job_url).json()
def main(): parser = argparse.ArgumentParser( description="Generate statistics for bugs linked to TestRun. Publish " "statistics to testrail if necessary.") parser.add_argument('plan_id', type=int, nargs='?', default=None, help='Test plan ID in TestRail') parser.add_argument('-j', '--job-name', dest='job_name', type=str, default=None, help='Name of Jenkins job which runs tests (runner). ' 'It will be used for TestPlan search instead ID') parser.add_argument('-n', '--build-number', dest='build_number', default='latest', help='Jenkins job build number') parser.add_argument('-r', '--run-id', dest='run_ids', type=str, default=None, help='(optional) IDs of TestRun to check (skip other)') parser.add_argument('-b', '--handle-blocked', action="store_true", dest='handle_blocked', default=False, help='Copy bugs links to downstream blocked results') parser.add_argument('-s', '--separate-runs', action="store_true", dest='separate_runs', default=False, help='Create separate statistics for each test run') parser.add_argument('-p', '--publish', action="store_true", help='Publish statistics to TestPlan description') parser.add_argument('-o', '--out-file', dest='output_file', default=None, type=str, help='Path to file to save statistics as JSON and/or ' 'HTML. Filename extension is added automatically') parser.add_argument('-H', '--html', action="store_true", help='Save statistics in HTML format to file ' '(used with --out-file option)') parser.add_argument('-q', '--quiet', action="store_true", help='Be quiet (disable logging except critical) ' 'Overrides "--verbose" option.') parser.add_argument("-v", "--verbose", action="store_true", help="Enable debug logging.") args = parser.parse_args() if args.verbose: logger.setLevel(DEBUG) if args.quiet: logger.setLevel(CRITICAL) testrail_project = get_testrail() if args.job_name: logger.info('Inspecting {0} build of {1} Jenkins job for TestPlan ' 'details...'.format(args.build_number, args.job_name)) test_plan_name = generate_test_plan_name(args.job_name, args.build_number) test_plan = testrail_project.get_plan_by_name(test_plan_name) if test_plan: args.plan_id = test_plan['id'] else: logger.warning('TestPlan "{0}" not found!'.format(test_plan_name)) if not args.plan_id: logger.error('There is no TestPlan to process, exiting...') return 1 run_ids = () if not args.run_ids else tuple( int(arg) for arg in args.run_ids.split(',')) generator = StatisticsGenerator(testrail_project, args.plan_id, run_ids, args.handle_blocked) generator.generate() stats = generator.dump() if args.publish: logger.debug('Publishing bugs statistics to TestRail..') generator.publish(stats) if args.output_file: html = generator.dump_html(stats) if args.html else args.html save_stats_to_file(stats, args.output_file, html) if args.separate_runs: for run in generator.test_runs_stats: file_name = '{0}_{1}'.format(args.output_file, run['id']) stats = generator.dump(run_id=run['id']) html = (generator.dump_html(stats, run['id']) if args.html else args.html) save_stats_to_file(stats, file_name, html) logger.info('Statistics generation complete!')
def get_job_console(self): job_url = "/".join([JENKINS["url"], 'job', self.name, str(self.number), 'consoleText']) logger.debug("Request job console from {}".format(job_url)) return requests.get(job_url).text.split('\n')
def get_tests_descriptions(milestone_id, testrail_default_test_priority, testrail_project): # To get the Tempest tests list, need to execute the following commands: # git clone https://github.com/openstack/tempest & cd tempest & tox -evenv # .tox/venv/bin/pip install nose get_tempest_tests = ("cd tempest && .tox/venv/bin/nosetests " "--collect-only tempest/{0} -v 2>&1 | grep 'id-.*'") get_commit = "cd tempest && git rev-parse HEAD" commit = subprocess.Popen(get_commit, shell=True, stdout=subprocess.PIPE) logger.info("Generate test suite for tempest" " commit:{}".format(commit.stdout.readline())) custom_cases_fields = _get_custom_cases_fields( case_fields=testrail_project.get_case_fields(), project_id=testrail_project.project['id']) tests = {} for group in TEST_GROUPS: p = subprocess.Popen(get_tempest_tests.format(group.lower()), shell=True, stdout=subprocess.PIPE) for line in iter(p.stdout.readline, b''): section = generate_groups(line) if group == "API" else group test_class = [] for r in line.split("."): if "id-" in r: title = r.strip() break else: test_class.append(r) steps = [ { "run this tempest test": "passed" }, ] test_case = { "title": title, "type_id": 1, "milestone_id": milestone_id, "priority_id": testrail_default_test_priority, "estimate": "1m", "refs": "", "custom_report_label": title.split('id-')[1][:-1], "custom_test_group": ".".join(test_class), "custom_test_case_description": title, "custom_test_case_steps": steps, "section": section } for case_field, default_value in custom_cases_fields.items(): if case_field not in test_case: test_case[case_field] = default_value if section not in tests: tests[section] = [] tests[section].append(test_case) logger.debug(tests) logger.info("total test cases: " "{}".format(sum(map(lambda x: len(x), tests.values())))) return tests
def main(): parser = OptionParser( description="Publish results of system tests from Jenkins build to " "TestRail. See settings.py for configuration.") parser.add_option('-j', '--job-name', dest='job_name', default=None, help='Jenkins swarm runner job name') parser.add_option('-N', '--build-number', dest='build_number', default='latest', help='Jenkins swarm runner build number') parser.add_option('-o', '--one-job', dest='one_job_name', default=None, help=('Process only one job name from the specified ' 'parent job or view')) parser.add_option("-w", "--view", dest="jenkins_view", default=False, help="Get system tests jobs from Jenkins view") parser.add_option("-l", "--live", dest="live_report", action="store_true", help="Get tests results from running swarm") parser.add_option("-m", "--manual", dest="manual_run", action="store_true", help="Manually add tests cases to TestRun (tested only)") parser.add_option('-c', '--create-plan-only', action="store_true", dest="create_plan_only", default=False, help='Jenkins swarm runner job name') parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="Enable debug output") (options, _) = parser.parse_args() if options.verbose: logger.setLevel(DEBUG) if options.live_report and options.build_number == 'latest': options.build_number = 'latest_started' # STEP #1 # Initialize TestRail Project and define configuration logger.info('Initializing TestRail Project configuration...') project = TestRailProject(url=TestRailSettings.url, user=TestRailSettings.user, password=TestRailSettings.password, project=TestRailSettings.project) tests_suite = project.get_suite_by_name(TestRailSettings.tests_suite) operation_systems = [ { 'name': config['name'], 'id': config['id'], 'distro': config['name'].split()[0].lower() } for config in project.get_config_by_name('Operation System')['configs'] if config['name'] in TestRailSettings.operation_systems ] tests_results = {os['distro']: [] for os in operation_systems} # STEP #2 # Get tests results from Jenkins logger.info('Getting tests results from Jenkins...') if options.jenkins_view: jobs = get_jobs_for_view(options.jenkins_view) tests_jobs = [{'name': j, 'number': 'latest'} for j in jobs if 'system_test' in j] if \ not options.create_plan_only else [] runner_job = [j for j in jobs if 'runner' in j][0] runner_build = Build(runner_job, 'latest') elif options.job_name: runner_build = Build(options.job_name, options.build_number) tests_jobs = get_downstream_builds(runner_build.build_data) if \ not options.create_plan_only else [] else: logger.error("Please specify either Jenkins swarm runner job name (-j)" " or Jenkins view with system tests jobs (-w). Exiting..") return for systest_build in tests_jobs: if (options.one_job_name and options.one_job_name != systest_build['name']): logger.debug( "Skipping '{0}' because --one-job is specified".format( systest_build['name'])) continue if options.job_name: if 'result' not in systest_build.keys(): logger.debug("Skipping '{0}' job because it does't run tests " "(build #{1} contains no results)".format( systest_build['name'], systest_build['number'])) continue if systest_build['result'] is None: logger.debug("Skipping '{0}' job (build #{1}) because it's sti" "ll running...".format( systest_build['name'], systest_build['number'], )) continue for os in tests_results.keys(): if os in systest_build['name'].lower(): tests_results[os].extend(get_tests_results(systest_build, os)) # STEP #3 # Create new TestPlan in TestRail (or get existing) and add TestRuns milestone, iso_number, prefix = get_version(runner_build.build_data) milestone = project.get_milestone_by_name(name=milestone) test_plan_name = ' '.join( filter(lambda x: bool(x), (milestone['name'], prefix, 'iso', '#' + str(iso_number)))) test_plan = project.get_plan_by_name(test_plan_name) iso_link = '/'.join([ JENKINS['url'], 'job', '{0}.all'.format(milestone['name']), str(iso_number) ]) if not test_plan: test_plan = project.add_plan(test_plan_name, description=iso_link, milestone_id=milestone['id'], entries=[]) logger.info('Created new TestPlan "{0}".'.format(test_plan_name)) else: logger.info('Found existing TestPlan "{0}".'.format(test_plan_name)) if options.create_plan_only: return plan_entries = [] all_cases = project.get_cases(suite_id=tests_suite['id']) for os in operation_systems: cases_ids = [] if options.manual_run: all_results_groups = [r.group for r in tests_results[os['distro']]] for case in all_cases: if case['custom_test_group'] in all_results_groups: cases_ids.append(case['id']) plan_entries.append( project.test_run_struct( name='{suite_name}'.format(suite_name=tests_suite['name']), suite_id=tests_suite['id'], milestone_id=milestone['id'], description='Results of system tests ({tests_suite}) on is' 'o #"{iso_number}"'.format(tests_suite=tests_suite['name'], iso_number=iso_number), config_ids=[os['id']], include_all=True, case_ids=cases_ids)) if not any(entry['suite_id'] == tests_suite['id'] for entry in test_plan['entries']): if project.add_plan_entry( plan_id=test_plan['id'], suite_id=tests_suite['id'], config_ids=[os['id'] for os in operation_systems], runs=plan_entries): test_plan = project.get_plan(test_plan['id']) # STEP #4 # Upload tests results to TestRail logger.info('Uploading tests results to TestRail...') for os in operation_systems: logger.info('Checking tests results for "{0}"...'.format(os['name'])) results_to_publish = publish_results( project=project, milestone_id=milestone['id'], test_plan=test_plan, suite_id=tests_suite['id'], config_id=os['id'], results=tests_results[os['distro']]) logger.debug('Added new results for tests ({os}): {tests}'.format( os=os['name'], tests=[r.group for r in results_to_publish])) logger.info('Report URL: {0}'.format(test_plan['url']))
def main(): parser = OptionParser( description="Publish results of system tests from Jenkins build to " "TestRail. See settings.py for configuration." ) parser.add_option('-j', '--job-name', dest='job_name', default=None, help='Jenkins swarm runner job name') parser.add_option('-N', '--build-number', dest='build_number', default='latest', help='Jenkins swarm runner build number') parser.add_option("-l", "--live", dest="live_report", action="store_true", help="Get tests results from running swarm") parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="Enable debug output") (options, _) = parser.parse_args() if options.verbose: logger.setLevel(DEBUG) if options.live_report and options.build_number == 'latest': build_number = 'latest_started' else: build_number = options.build_number # STEP #1 # Initialize TestRail Project and define configuration logger.info('Initializing TestRail Project configuration...') project = TestRailProject(url=TestRailSettings.url, user=TestRailSettings.user, password=TestRailSettings.password, project=TestRailSettings.project) logger.info('Initializing TestRail Project configuration... done') operation_systems = [{'name': config['name'], 'id': config['id'], 'distro': config['name'].split()[0].lower()} for config in project.get_config_by_name( 'Operation System')['configs']] os_mile = {'6.1': ['Centos 6.5', 'Ubuntu 14.04'], '6.0.1': ['Centos 6.5', 'Ubuntu 12.04']} tests_results = {} # STEP #2 # Get tests results from Jenkins runner_build = Build(options.job_name, build_number) runs = runner_build.build_data['runs'] # Analyze each test individually for run_one in runs: if '5.1' in run_one['url']: continue # Release 5.1 to skip tests_result = get_job_info(run_one['url']) if not tests_result['description']: continue # Not completed results to skip if 'skipping' in tests_result['description']: continue # Not performed tests to skip tests_job = {'result': tests_result['result'], 'name': (options.job_name + '/' + tests_result['url'].split('/')[-3]), 'number': int(tests_result['url'].split('/')[-2]), 'mile': (tests_result['description']. split()[0].split('-')[0]), 'iso': (int(tests_result['description']. split()[0].split('-')[1]))} if tests_job['mile'] not in tests_results: tests_results[tests_job['mile']] = {} test_mile = tests_results[tests_job['mile']] if tests_job['iso'] not in test_mile: test_mile[tests_job['iso']] = {} test_iso = test_mile[tests_job['iso']] for os in operation_systems: if os['distro'] in tests_job['name'].lower() and\ os['name'] in os_mile[tests_job['mile']]: if os['id'] not in test_iso: test_iso[os['id']] = [] test_os_id = test_iso[os['id']] test_os_id.extend(get_tests_results(tests_job, os['distro'])) # STEP #3 # Create new TestPlan in TestRail (or get existing) and add TestRuns for mile in tests_results: mile_tests_suite = '{0}{1}'.format(TestRailSettings.tests_suite, mile) logger.info(mile_tests_suite) tests_suite = project.get_suite_by_name(mile_tests_suite) milestone = project.get_milestone_by_name(name=mile) for iso_number in tests_results.get(mile, {}): # Create new TestPlan name check the same name in testrail test_plan_name = '{milestone} iso #{iso_number}'.format( milestone=milestone['name'], iso_number=iso_number) test_plan = project.get_plan_by_name(test_plan_name) if not test_plan: test_plan = project.add_plan( test_plan_name, description='/'.join([JENKINS['url'], 'job', '{0}.all'.format(milestone['name']), str(iso_number)]), milestone_id=milestone['id'], entries=[]) logger.info('Created new TestPlan "{0}".' .format(test_plan_name)) else: logger.info('Found existing TestPlan "{0}".' .format(test_plan_name)) plan_entries = [] # Create a test plan entry config_ids = [] for os in operation_systems: if os['name'] in os_mile[mile]: config_ids.append(os['id']) cases_ids = [] plan_entries.append( project.test_run_struct( name=tests_suite['name'], suite_id=tests_suite['id'], milestone_id=milestone['id'], description=('Results of system tests ({t_suite})' ' on iso #"{iso_number}"' .format(t_suite=tests_suite['name'], iso_number=iso_number)), config_ids=[os['id']], include_all=True, case_ids=cases_ids)) # Create a test plan entry with the test run run = find_run_by_name(test_plan, tests_suite['name']) if not run: logger.info('Adding a test plan entry with test run %s ...', tests_suite['name']) entry = project.add_plan_entry(plan_id=test_plan['id'], suite_id=tests_suite['id'], config_ids=config_ids, runs=plan_entries) logger.info('The test plan entry has been added.') run = entry['runs'][0] test_plan = project.get_plan(test_plan['id']) # STEP #4 # Upload tests results to TestRail logger.info('Uploading tests results to TestRail...') for os_id in tests_results.get(mile, {})\ .get(iso_number, {}): logger.info('Checking tests results for %s...', project.get_config(os_id)['name']) tests_added = publish_results( project=project, milestone_id=milestone['id'], test_plan=test_plan, suite_id=tests_suite['id'], config_id=os_id, results=tests_results[mile][iso_number][os_id]) logger.debug('Added new results for tests (%s): %s', project.get_config(os_id)['name'], [r.group for r in tests_added]) logger.info('Report URL: %s', test_plan['url'])
def get_job_info(url): job_url = "/".join([url, 'api/json']) logger.debug("Request job info from %s", job_url) return json.load(urlopen(job_url))
def get_job_info(self, depth=1): job_url = "/".join([JENKINS["url"], 'job', self.name, 'api/json?depth={depth}'.format(depth=depth)]) logger.debug("Request job info from {}".format(job_url)) return requests.get(job_url).json()
def main(): parser = OptionParser( description="Publish results of system tests from Jenkins build to " "TestRail. See settings.py for configuration." ) parser.add_option('-j', '--job-name', dest='job_name', default=None, help='Jenkins swarm runner job name') parser.add_option('-N', '--build-number', dest='build_number', default='latest', help='Jenkins swarm runner build number') parser.add_option('-o', '--one-job', dest='one_job_name', default=None, help=('Process only one job name from the specified ' 'parent job or view')) parser.add_option("-w", "--view", dest="jenkins_view", default=False, help="Get system tests jobs from Jenkins view") parser.add_option("-l", "--live", dest="live_report", action="store_true", help="Get tests results from running swarm") parser.add_option("-m", "--manual", dest="manual_run", action="store_true", help="Manually add tests cases to TestRun (tested only)") parser.add_option('-c', '--create-plan-only', action="store_true", dest="create_plan_only", default=False, help='Jenkins swarm runner job name') parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="Enable debug output") (options, _) = parser.parse_args() if options.verbose: logger.setLevel(DEBUG) if options.live_report and options.build_number == 'latest': options.build_number = 'latest_started' # STEP #1 # Initialize TestRail Project and define configuration logger.info('Initializing TestRail Project configuration...') project = TestRailProject(url=TestRailSettings.url, user=TestRailSettings.user, password=TestRailSettings.password, project=TestRailSettings.project) tests_suite = project.get_suite_by_name(TestRailSettings.tests_suite) operation_systems = [{'name': config['name'], 'id': config['id'], 'distro': config['name'].split()[0].lower()} for config in project.get_config_by_name( 'Operation System')['configs'] if config['name'] in TestRailSettings.operation_systems] tests_results = {os['distro']: [] for os in operation_systems} # STEP #2 # Get tests results from Jenkins logger.info('Getting tests results from Jenkins...') if options.jenkins_view: jobs = get_jobs_for_view(options.jenkins_view) tests_jobs = [{'name': j, 'number': 'latest'} for j in jobs if 'system_test' in j] if \ not options.create_plan_only else [] runner_job = [j for j in jobs if 'runner' in j][0] runner_build = Build(runner_job, 'latest') elif options.job_name: runner_build = Build(options.job_name, options.build_number) tests_jobs = get_downstream_builds(runner_build.build_data) if \ not options.create_plan_only else [] else: logger.error("Please specify either Jenkins swarm runner job name (-j)" " or Jenkins view with system tests jobs (-w). Exiting..") return for systest_build in tests_jobs: if (options.one_job_name and options.one_job_name != systest_build['name']): logger.debug("Skipping '{0}' because --one-job is specified" .format(systest_build['name'])) continue if options.job_name: if 'result' not in systest_build.keys(): logger.debug("Skipping '{0}' job because it does't run tests " "(build #{1} contains no results)".format( systest_build['name'], systest_build['number'])) continue if systest_build['result'] is None: logger.debug("Skipping '{0}' job (build #{1}) because it's sti" "ll running...".format(systest_build['name'], systest_build['number'],)) continue for os in tests_results.keys(): if os in systest_build['name'].lower(): tests_results[os].extend(get_tests_results(systest_build, os)) # STEP #3 # Create new TestPlan in TestRail (or get existing) and add TestRuns milestone, iso_number, prefix = get_version(runner_build.build_data) milestone = project.get_milestone_by_name(name=milestone) test_plan_name = ' '.join( filter(lambda x: bool(x), (milestone['name'], prefix, 'iso', '#' + str(iso_number)))) test_plan = project.get_plan_by_name(test_plan_name) iso_link = '/'.join([JENKINS['url'], 'job', '{0}.all'.format(milestone['name']), str(iso_number)]) if not test_plan: test_plan = project.add_plan(test_plan_name, description=iso_link, milestone_id=milestone['id'], entries=[] ) logger.info('Created new TestPlan "{0}".'.format(test_plan_name)) else: logger.info('Found existing TestPlan "{0}".'.format(test_plan_name)) if options.create_plan_only: return plan_entries = [] all_cases = project.get_cases(suite_id=tests_suite['id']) for os in operation_systems: cases_ids = [] if options.manual_run: all_results_groups = [r.group for r in tests_results[os['distro']]] for case in all_cases: if case['custom_test_group'] in all_results_groups: cases_ids.append(case['id']) plan_entries.append( project.test_run_struct( name='{suite_name}'.format(suite_name=tests_suite['name']), suite_id=tests_suite['id'], milestone_id=milestone['id'], description='Results of system tests ({tests_suite}) on is' 'o #"{iso_number}"'.format(tests_suite=tests_suite['name'], iso_number=iso_number), config_ids=[os['id']], include_all=True, case_ids=cases_ids ) ) if not any(entry['suite_id'] == tests_suite['id'] for entry in test_plan['entries']): if project.add_plan_entry(plan_id=test_plan['id'], suite_id=tests_suite['id'], config_ids=[os['id'] for os in operation_systems], runs=plan_entries): test_plan = project.get_plan(test_plan['id']) # STEP #4 # Upload tests results to TestRail logger.info('Uploading tests results to TestRail...') for os in operation_systems: logger.info('Checking tests results for "{0}"...'.format(os['name'])) results_to_publish = publish_results( project=project, milestone_id=milestone['id'], test_plan=test_plan, suite_id=tests_suite['id'], config_id=os['id'], results=tests_results[os['distro']] ) logger.debug('Added new results for tests ({os}): {tests}'.format( os=os['name'], tests=[r.group for r in results_to_publish] )) logger.info('Report URL: {0}'.format(test_plan['url']))
def main(): parser = argparse.ArgumentParser( description="Generate statistics for bugs linked to TestRun. Publish " "statistics to testrail if necessary." ) parser.add_argument("plan_id", type=int, nargs="?", default=None, help="Test plan ID in TestRail") parser.add_argument( "-j", "--job-name", dest="job_name", type=str, default=None, help="Name of Jenkins job which runs tests (runner). " "It will be used for TestPlan search instead ID", ) parser.add_argument("-n", "--build-number", dest="build_number", default="latest", help="Jenkins job build number") parser.add_argument( "-r", "--run-id", dest="run_ids", type=str, default=None, help="(optional) IDs of TestRun to check (skip other)" ) parser.add_argument( "-b", "--handle-blocked", action="store_true", dest="handle_blocked", default=False, help="Copy bugs links to downstream blocked results", ) parser.add_argument( "-s", "--separate-runs", action="store_true", dest="separate_runs", default=False, help="Create separate statistics for each test run", ) parser.add_argument("-p", "--publish", action="store_true", help="Publish statistics to TestPlan description") parser.add_argument( "-o", "--out-file", dest="output_file", default=None, type=str, help="Path to file to save statistics as JSON and/or " "HTML. Filename extension is added automatically", ) parser.add_argument( "-H", "--html", action="store_true", help="Save statistics in HTML format to file " "(used with --out-file option)", ) parser.add_argument( "-q", "--quiet", action="store_true", help="Be quiet (disable logging except critical) " 'Overrides "--verbose" option.', ) parser.add_argument("-v", "--verbose", action="store_true", help="Enable debug logging.") args = parser.parse_args() if args.verbose: logger.setLevel(DEBUG) if args.quiet: logger.setLevel(CRITICAL) testrail_project = get_testrail() if args.job_name: logger.info( "Inspecting {0} build of {1} Jenkins job for TestPlan " "details...".format(args.build_number, args.job_name) ) test_plan_name = generate_test_plan_name(args.job_name, args.build_number) test_plan = testrail_project.get_plan_by_name(test_plan_name) if test_plan: args.plan_id = test_plan["id"] else: logger.warning('TestPlan "{0}" not found!'.format(test_plan_name)) if not args.plan_id: logger.error("There is no TestPlan to process, exiting...") return 1 run_ids = () if not args.run_ids else tuple(int(arg) for arg in args.run_ids.split(",")) generator = StatisticsGenerator(testrail_project, args.plan_id, run_ids, args.handle_blocked) generator.generate() stats = generator.dump() if args.publish: logger.debug("Publishing bugs statistics to TestRail..") generator.publish(stats) if args.output_file: html = generator.dump_html(stats) if args.html else args.html save_stats_to_file(stats, args.output_file, html) if args.separate_runs: for run in generator.test_runs_stats: file_name = "{0}_{1}".format(args.output_file, run["id"]) stats = generator.dump(run_id=run["id"]) html = generator.dump_html(stats, run["id"]) if args.html else args.html save_stats_to_file(stats, file_name, html) logger.info("Statistics generation complete!")
def main(): parser = argparse.ArgumentParser( description="Generate statistics for bugs linked to TestRun. Publish " "statistics to testrail if necessary." ) parser.add_argument('plan_id', type=int, nargs='?', default=None, help='Test plan ID in TestRail') parser.add_argument('-j', '--job-name', dest='job_name', type=str, default=None, help='Name of Jenkins job which runs tests (runner). ' 'It will be used for TestPlan search instead ID') parser.add_argument('-n', '--build-number', dest='build_number', default='latest', help='Jenkins job build number') parser.add_argument('-r', '--run-id', dest='run_ids', type=str, default=None, help='(optional) IDs of TestRun to check (skip other)') parser.add_argument('-b', '--handle-blocked', action="store_true", dest='handle_blocked', default=False, help='Copy bugs links to downstream blocked results') parser.add_argument('-s', '--separate-runs', action="store_true", dest='separate_runs', default=False, help='Create separate statistics for each test run') parser.add_argument('-p', '--publish', action="store_true", help='Publish statistics to TestPlan description') parser.add_argument('-o', '--out-file', dest='output_file', default=None, type=str, help='Path to file to save statistics as JSON and/or ' 'HTML. Filename extension is added automatically') parser.add_argument('-H', '--html', action="store_true", help='Save statistics in HTML format to file ' '(used with --out-file option)') parser.add_argument('-q', '--quiet', action="store_true", help='Be quiet (disable logging except critical) ' 'Overrides "--verbose" option.') parser.add_argument("-v", "--verbose", action="store_true", help="Enable debug logging.") args = parser.parse_args() if args.verbose: logger.setLevel(DEBUG) if args.quiet: logger.setLevel(CRITICAL) testrail_project = get_testrail() if args.job_name: logger.info('Inspecting {0} build of {1} Jenkins job for TestPlan ' 'details...'.format(args.build_number, args.job_name)) test_plan_name = generate_test_plan_name(args.job_name, args.build_number) test_plan = testrail_project.get_plan_by_name(test_plan_name) if test_plan: args.plan_id = test_plan['id'] else: logger.warning('TestPlan "{0}" not found!'.format(test_plan_name)) if not args.plan_id: logger.error('There is no TestPlan to process, exiting...') return 1 run_ids = () if not args.run_ids else tuple( int(arg) for arg in args.run_ids.split(',')) generator = StatisticsGenerator(testrail_project, args.plan_id, run_ids, args.handle_blocked) generator.generate() stats = generator.dump() if args.publish: logger.debug('Publishing bugs statistics to TestRail..') generator.publish(stats) if args.output_file: html = generator.dump_html(stats) if args.html else args.html save_stats_to_file(stats, args.output_file, html) if args.separate_runs: for run in generator.test_runs_stats: file_name = '{0}_{1}'.format(args.output_file, run['id']) stats = generator.dump(run_id=run['id']) html = (generator.dump_html(stats, run['id']) if args.html else args.html) save_stats_to_file(stats, file_name, html) logger.info('Statistics generation complete!')