def test_inactive_by_project(self):
build = self.project.builds.create(
datetime=self.date,
version=self.date.strftime("%Y%m%d"),
)
test_run = build.test_runs.create(environment=self.env1)
# create failed test
foo_metadata, _ = SuiteMetadata.objects.get_or_create(
suite=self.suite1.slug, name='test_foo', kind='test')
test = test_run.tests.create(build=test_run.build,
environment=test_run.environment,
suite=self.suite1,
metadata=foo_metadata,
result=False)
known_issue = KnownIssue.objects.create(title="foo",
test_name=test.full_name)
known_issue.save()
known_issue.environments.add(test_run.environment)
known_issue.active = False
known_issue.save()
self.assertEqual(
0,
len(
KnownIssue.active_by_project_and_test(self.project,
test.full_name)))
def __init__(self, project, full_test_name, top=None, page=1, per_page=20):
suite_slug, test_name = parse_name(full_test_name)
self.test = full_test_name
self.paginator = Paginator(project.builds.prefetch_related(Prefetch('test_runs', queryset=TestRun.objects.prefetch_related('environment').all())).reverse(), per_page)
if top:
self.number = 0
builds = project.builds.filter(datetime__lte=top.datetime).reverse()[0:per_page - 1]
else:
self.number = page
builds = self.paginator.page(page)
self.top = builds[0]
environments = OrderedDict()
results = OrderedDict()
for build in builds:
results[build] = {}
issues_by_env = {}
for issue in KnownIssue.active_by_project_and_test(project, full_test_name).all():
for env in issue.environments.all():
if env.id not in issues_by_env:
issues_by_env[env.id] = []
issues_by_env[env.id].append(issue)
suite = project.suites.prefetch_related('metadata').get(slug=suite_slug)
metadata = SuiteMetadata.objects.get(kind='test', suite=suite_slug, name=test_name)
tests = Test.objects.filter(build__in=builds, metadata_id=metadata.id).prefetch_related('build', 'environment', 'test_run', 'metadata').order_by()
all_envs = set(project.environments.all())
for test in tests:
build = test.build
environment = test.environment
environments[environment] = True
known_issues = issues_by_env.get(environment.id)
is_duplicate = False
if environment in results[build]:
is_duplicate = True
results[build][environment] = TestResult(test, suite, metadata, known_issues, is_duplicate)
for build in results.keys():
recorded_envs = set(results[build].keys())
remaining_envs = all_envs - recorded_envs
for env in remaining_envs:
results[build][env] = None
environments[env] = True
# Make sure all builds that don't have the test have None at least
for b in builds:
if not results[b]:
for env in all_envs:
results[build][env] = None
environments[env] = True
self.environments = sorted(environments.keys(), key=lambda env: env.slug)
self.results = results
def __call__(test_run):
if test_run.data_processed:
return
issues = {}
for issue in KnownIssue.active_by_environment(test_run.environment):
issues.setdefault(issue.test_name, [])
issues[issue.test_name].append(issue)
for test in test_parser()(test_run.tests_file):
# TODO: remove check below when test_name size changes in the schema
if len(test['test_name']) > 256:
continue
suite = get_suite(
test_run,
test['group_name']
)
metadata, _ = SuiteMetadata.objects.get_or_create(suite=suite.slug, name=test['test_name'], kind='test')
full_name = join_name(suite.slug, test['test_name'])
test_issues = issues.get(full_name, [])
test_obj = Test.objects.create(
test_run=test_run,
suite=suite,
metadata=metadata,
result=test['pass'],
log=test['log'],
has_known_issues=bool(test_issues),
build=test_run.build,
environment=test_run.environment,
)
for issue in test_issues:
test_obj.known_issues.add(issue)
for metric in metric_parser()(test_run.metrics_file):
# TODO: remove check below when test_name size changes in the schema
if len(metric['name']) > 256:
continue
suite = get_suite(
test_run,
metric['group_name']
)
metadata, _ = SuiteMetadata.objects.get_or_create(suite=suite.slug, name=metric['name'], kind='metric')
Metric.objects.create(
test_run=test_run,
suite=suite,
metadata=metadata,
name=metric['name'],
result=metric['result'],
measurements=','.join([str(m) for m in metric['measurements']]),
unit=metric['unit'],
)
test_run.data_processed = True
test_run.save()
def _extract_cts_results(self, buf, testrun, suite_prefix):
tradefed_tree = self.__parse_xml_results(buf)
if tradefed_tree is None:
return
issues = {}
for issue in KnownIssue.active_by_environment(testrun.environment):
issues.setdefault(issue.test_name, [])
issues[issue.test_name].append(issue)
test_elems = tradefed_tree.findall(".//Test")
logger.debug("Tests: {}".format(len(test_elems)))
elems = tradefed_tree.findall('Module')
logger.debug("Modules: {}".format(len(elems)))
task_list = []
for elem in elems:
# Naming: Module Name + Test Case Name + Test Name
if 'abi' in elem.attrib.keys():
module_name = '.'.join(
[elem.attrib['abi'], elem.attrib['name']])
else:
module_name = elem.attrib['name']
logger.debug("Extracting tests for module: {}".format(module_name))
test_cases = elem.findall('.//TestCase')
suite_metadata_list = []
suite_list = []
logger.debug("Extracting suite names")
atomic_test_suite_name = "{suite_prefix}/{module_name}".format(
suite_prefix=suite_prefix, module_name=module_name)
logger.debug(
"creating suite metadata: {}".format(atomic_test_suite_name))
suite_metadata, _ = SuiteMetadata.objects.get_or_create(
suite=atomic_test_suite_name, kind='suite')
suite, _ = Suite.objects.get_or_create(
slug=atomic_test_suite_name,
project=testrun.build.project,
defaults={"metadata": suite_metadata})
logger.debug(
"Adding status with suite: {suite_prefix}/{module_name}".
format(suite_prefix=suite_prefix, module_name=module_name))
logger.debug("Creating subtasks for extracting results")
for test_case in test_cases:
plugin_scratch = PluginScratch.objects.create(
build=testrun.build,
storage=ET.tostring(test_case).decode('utf-8'))
logger.debug("Created plugin scratch with ID: %s" %
plugin_scratch.pk)
task = create_testcase_tests.s(plugin_scratch.pk,
atomic_test_suite_name,
testrun.pk, suite.pk)
task_list.append(task)
celery_chord(task_list)(update_build_status.s(testrun.pk))
def test_active_by_project(self):
build = self.project.builds.create(
datetime=self.date,
version=self.date.strftime("%Y%m%d"),
)
test_run = build.test_runs.create(environment=self.env1)
# create failed test
test = test_run.tests.create(suite=self.suite1, name="test_foo", result=False)
known_issue = KnownIssue.objects.create(
title="foo",
test_name=test.full_name
)
known_issue.save()
known_issue.environments.add(test_run.environment)
self.assertEqual(1, len(KnownIssue.active_by_project_and_test(self.project, test.full_name)))
def test_active_by_project(self):
build = self.project.builds.create(
datetime=self.date,
version=self.date.strftime("%Y%m%d"),
)
test_run = build.test_runs.create(environment=self.env1)
# create failed test
test = test_run.tests.create(suite=self.suite1, name="test_foo", result=False)
known_issue = KnownIssue.objects.create(
title="foo",
test_name=test.full_name
)
known_issue.save()
known_issue.environments.add(test_run.environment)
self.assertEqual(1, len(KnownIssue.active_by_project_and_test(self.project, test.full_name)))
def __call__(test_run):
if test_run.data_processed:
return
issues = {}
for issue in KnownIssue.active_by_environment(test_run.environment):
issues.setdefault(issue.test_name, [])
issues[issue.test_name].append(issue)
for test in test_parser()(test_run.tests_file):
suite = get_suite(
test_run,
test['group_name']
)
metadata, _ = SuiteMetadata.objects.get_or_create(suite=suite.slug, name=test['test_name'], kind='test')
full_name = join_name(suite.slug, test['test_name'])
test_issues = issues.get(full_name, [])
test = Test.objects.create(
test_run=test_run,
suite=suite,
metadata=metadata,
name=test['test_name'],
result=test['pass'],
has_known_issues=bool(test_issues),
)
for issue in test_issues:
test.known_issues.add(issue)
for metric in metric_parser()(test_run.metrics_file):
suite = get_suite(
test_run,
metric['group_name']
)
metadata, _ = SuiteMetadata.objects.get_or_create(suite=suite.slug, name=metric['name'], kind='metric')
Metric.objects.create(
test_run=test_run,
suite=suite,
metadata=metadata,
name=metric['name'],
result=metric['result'],
measurements=','.join([str(m) for m in metric['measurements']]),
)
test_run.data_processed = True
test_run.save()
def __call__(test_run):
if test_run.data_processed:
return
issues = {}
for issue in KnownIssue.active_by_environment(test_run.environment):
issues.setdefault(issue.test_name, [])
issues[issue.test_name].append(issue)
for test in test_parser()(test_run.tests_file):
suite = get_suite(test_run, test['group_name'])
metadata, _ = SuiteMetadata.objects.get_or_create(
suite=suite.slug, name=test['test_name'], kind='test')
full_name = join_name(suite.slug, test['test_name'])
test_issues = issues.get(full_name, [])
test = Test.objects.create(
test_run=test_run,
suite=suite,
metadata=metadata,
name=test['test_name'],
result=test['pass'],
has_known_issues=bool(test_issues),
)
for issue in test_issues:
test.known_issues.add(issue)
for metric in metric_parser()(test_run.metrics_file):
suite = get_suite(test_run, metric['group_name'])
metadata, _ = SuiteMetadata.objects.get_or_create(
suite=suite.slug, name=metric['name'], kind='metric')
Metric.objects.create(
test_run=test_run,
suite=suite,
metadata=metadata,
name=metric['name'],
result=metric['result'],
measurements=','.join([str(m)
for m in metric['measurements']]),
)
test_run.data_processed = True
test_run.save()
def __call__(test_run):
if test_run.data_processed:
return
issues = {}
for issue in KnownIssue.active_by_environment(test_run.environment):
issues.setdefault(issue.test_name, [])
issues[issue.test_name].append(issue)
# Issues' test_name should be interpreted as regexes
# so compile them prior to matching against test names
# The * character should be replaced by .*?, which is regex for "everything"
issues_regex = {}
for test_name_regex in issues.keys():
pattern = re.escape(test_name_regex).replace('\\*', '.*?')
regex = re.compile(pattern)
issues_regex[regex] = issues[test_name_regex]
for test in test_parser()(test_run.tests_file):
# TODO: remove check below when test_name size changes in the schema
if len(test['test_name']) > 256:
continue
suite = get_suite(test_run, test['group_name'])
metadata, _ = SuiteMetadata.objects.get_or_create(
suite=suite.slug, name=test['test_name'], kind='test')
full_name = join_name(suite.slug, test['test_name'])
test_issues = list(
itertools.chain(*[
issue for regex, issue in issues_regex.items()
if regex.match(full_name)
]))
test_obj = Test.objects.create(
test_run=test_run,
suite=suite,
metadata=metadata,
result=test['pass'],
log=test['log'],
has_known_issues=bool(test_issues),
build=test_run.build,
environment=test_run.environment,
)
for issue in test_issues:
test_obj.known_issues.add(issue)
for metric in metric_parser()(test_run.metrics_file):
# TODO: remove check below when test_name size changes in the schema
if len(metric['name']) > 256:
continue
suite = get_suite(test_run, metric['group_name'])
metadata, _ = SuiteMetadata.objects.get_or_create(
suite=suite.slug, name=metric['name'], kind='metric')
Metric.objects.create(
test_run=test_run,
suite=suite,
metadata=metadata,
result=metric['result'],
measurements=','.join([str(m)
for m in metric['measurements']]),
unit=metric['unit'],
build=test_run.build,
environment=test_run.environment,
)
test_run.data_processed = True
test_run.save()
def known_issues(self):
return KnownIssue.active_by_project_and_test(self.project)
def create_testcase_tests(test_case_string_storage_id, atomic_test_suite_name,
testrun_id, suite_id):
test_case_string = None
scratch_object = None
try:
scratch_object = PluginScratch.objects.get(
pk=test_case_string_storage_id)
test_case_string = scratch_object.storage
except PluginScratch.DoesNotExist:
logger.warning("PluginScratch with ID: %s doesn't exist" %
test_case_string_storage_id)
return
test_case = ET.fromstring(test_case_string)
testrun = TestRun.objects.get(pk=testrun_id)
suite = Suite.objects.get(pk=suite_id)
local_status = {
'tests_pass': 0,
'tests_xfail': 0,
'tests_fail': 0,
'tests_skip': 0
}
issues = {}
for issue in KnownIssue.active_by_environment(testrun.environment):
issues.setdefault(issue.test_name, [])
issues[issue.test_name].append(issue)
test_case_name = test_case.get("name")
tests = test_case.findall('.//Test')
logger.debug("Extracting TestCase: {test_case_name}".format(
test_case_name=test_case_name))
logger.debug("Adding {} testcases".format(len(tests)))
test_list = []
for atomic_test in tests:
atomic_test_result = atomic_test.get("result")
decoded_test_result = atomic_test_result == 'pass'
if atomic_test_result == 'skip' or atomic_test.get(
"skipped") == "true":
decoded_test_result = None
atomic_test_name = "{test_case_name}.{test_name}".format(
test_case_name=test_case_name, test_name=atomic_test.get("name"))
atomic_test_log = ""
trace_node = atomic_test.find('.//StackTrace')
if trace_node is not None:
atomic_test_log = trace_node.text
metadata, _ = SuiteMetadata.objects.get_or_create(
suite=atomic_test_suite_name, name=atomic_test_name, kind='test')
full_name = join_name(suite.slug, atomic_test_name)
test_issues = issues.get(full_name, [])
test_list.append(
Test(
test_run=testrun,
suite=suite,
metadata=metadata,
result=decoded_test_result,
log=atomic_test_log,
has_known_issues=bool(test_issues),
))
if decoded_test_result is True:
local_status['tests_pass'] += 1
elif decoded_test_result is False:
if test_issues:
local_status['tests_xfail'] += 1
else:
local_status['tests_fail'] += 1
else:
local_status['tests_skip'] += 1
created_tests = Test.objects.bulk_create(test_list)
for test in created_tests:
if test.name in issues.keys():
test.known_issues.add(issues[test.name])
with transaction.atomic():
tr_status = testrun.status.select_for_update().get(suite=None)
tr_status.tests_pass += local_status['tests_pass']
tr_status.tests_xfail += local_status['tests_xfail']
tr_status.tests_fail += local_status['tests_fail']
tr_status.tests_skip += local_status['tests_skip']
tr_status.save()
suite_status, _ = Status.objects.get_or_create(test_run=testrun,
suite=suite)
with transaction.atomic():
suite_status_for_update = Status.objects.select_for_update().get(
pk=suite_status.pk)
suite_status_for_update.tests_pass += local_status['tests_pass']
suite_status_for_update.tests_xfail += local_status['tests_xfail']
suite_status_for_update.tests_fail += local_status['tests_fail']
suite_status_for_update.tests_skip += local_status['tests_skip']
suite_status_for_update.save()
logger.info("Deleting PluginScratch with ID: %s" % scratch_object.pk)
scratch_object.delete()
return 0
def known_issues(self):
return KnownIssue.active_by_project_and_test(self.project)