def __init__(self, project, full_test_name):
suite, test_name = parse_name(full_test_name)
self.test = full_test_name
tests = Test.objects.filter(
suite__slug=suite,
name=test_name,
test_run__build__project=project,
)
Test.prefetch_related(tests)
environments = OrderedDict()
results = OrderedDict()
for build in project.builds.reverse():
results[build] = {}
for test in tests:
build = test.test_run.build
environment = test.test_run.environment
environments[environment] = True
results[build][environment] = TestResult(test)
self.environments = list(environments.keys())
self.results = results
def __init__(self, project, full_test_name, top=None, page=1, per_page=20):
suite_slug, test_name = parse_name(full_test_name)
self.test = full_test_name
self.paginator = Paginator(project.builds.prefetch_related(Prefetch('test_runs', queryset=TestRun.objects.prefetch_related('environment').all())).reverse(), per_page)
if top:
self.number = 0
builds = project.builds.filter(datetime__lte=top.datetime).reverse()[0:per_page - 1]
else:
self.number = page
builds = self.paginator.page(page)
self.top = builds[0]
environments = OrderedDict()
results = OrderedDict()
for build in builds:
results[build] = {}
issues_by_env = {}
for issue in KnownIssue.active_by_project_and_test(project, full_test_name).all():
for env in issue.environments.all():
if env.id not in issues_by_env:
issues_by_env[env.id] = []
issues_by_env[env.id].append(issue)
suite = project.suites.prefetch_related('metadata').get(slug=suite_slug)
metadata = SuiteMetadata.objects.get(kind='test', suite=suite_slug, name=test_name)
tests = Test.objects.filter(build__in=builds, metadata_id=metadata.id).prefetch_related('build', 'environment', 'test_run', 'metadata').order_by()
all_envs = set(project.environments.all())
for test in tests:
build = test.build
environment = test.environment
environments[environment] = True
known_issues = issues_by_env.get(environment.id)
is_duplicate = False
if environment in results[build]:
is_duplicate = True
results[build][environment] = TestResult(test, suite, metadata, known_issues, is_duplicate)
for build in results.keys():
recorded_envs = set(results[build].keys())
remaining_envs = all_envs - recorded_envs
for env in remaining_envs:
results[build][env] = None
environments[env] = True
# Make sure all builds that don't have the test have None at least
for b in builds:
if not results[b]:
for env in all_envs:
results[build][env] = None
environments[env] = True
self.environments = sorted(environments.keys(), key=lambda env: env.slug)
self.results = results
def regressions_grouped_by_suite(self):
regressions = self.regressions
result = OrderedDict()
for env, tests in regressions.items():
this_env = OrderedDict()
for test in tests:
suite, testname = parse_name(test)
if suite not in this_env:
this_env[suite] = []
this_env[suite].append(testname)
result[env] = this_env
return result
def __call__(test_data):
if test_data is None or test_data == '':
return []
input_data = json.loads(test_data)
data = []
for key, value in input_data.items():
group_name, test_name = parse_name(key)
data.append({
"group_name": group_name,
"test_name": test_name,
"pass": parse_test_result(value),
})
return data
def __status_changes_by_suite__(self, regression=True):
comparisons = self.regressions
if not regression:
comparisons = self.fixes
result = OrderedDict()
for env, tests in comparisons.items():
this_env = OrderedDict()
for test in tests:
suite, testname = parse_name(test)
if suite not in this_env:
this_env[suite] = []
this_env[suite].append(testname)
result[env] = this_env
return result
def __call__(test_data):
if test_data is None or test_data == '':
return []
input_data = json.loads(test_data)
data = []
for key, value in input_data.items():
group_name, test_name = parse_name(key)
data.append({
"group_name": group_name,
"test_name": test_name,
"pass": parse_test_result(value),
})
return data
def __status_changes_by_suite__(self, regression=True):
comparisons = self.regressions
if not regression:
comparisons = self.fixes
result = OrderedDict()
for env, tests in comparisons.items():
this_env = OrderedDict()
for test in tests:
suite, testname = parse_name(test)
if suite not in this_env:
this_env[suite] = []
this_env[suite].append(testname)
result[env] = this_env
return result
def __status_changes_by_suite__(self, regression=True):
comparisons = self.__regressions__
if not regression:
comparisons = self.__fixes__
result = OrderedDict()
for env, items in comparisons.items():
this_env = OrderedDict()
for item in items:
suite, itemname = parse_name(item)
if suite not in this_env:
this_env[suite] = []
this_env[suite].append(itemname)
result[env] = this_env
return result
def get_dynamic_summary(project, environments, metrics, date_start, date_end):
entry = {}
filters = []
if not metrics:
for env in environments:
entry[env] = []
return entry
for m in metrics:
suite, metric = parse_name(m)
filters.append(Q(suite__slug=suite) & Q(name=metric))
metric_filter = reduce(lambda x, y: x | y, filters)
data = models.Metric.objects.filter(
test_run__build__project=project,
test_run__environment__slug__in=environments,
test_run__created_at__range=(date_start, date_end),
).filter(metric_filter).prefetch_related(
'test_run',
'test_run__environment',
'test_run__build',
'test_run__build__annotation',
).order_by('test_run__environment__id', 'test_run__build__id')
for environment, metrics_by_environment in groupby(
data, lambda m: m.test_run.environment):
envdata = []
metrics_by_build = groupby(metrics_by_environment,
lambda m: m.test_run.build)
for build, metric_list in metrics_by_build:
values = []
for metric in metric_list:
if not metric.is_outlier:
values = values + metric.measurement_list
try:
description = build.annotation.description
except ObjectDoesNotExist:
description = ""
envdata.append([
build.datetime.timestamp(),
geomean(values),
build.version,
description,
])
entry[environment.slug] = sorted(envdata, key=(lambda e: e[0]))
return entry
def __call__(json_text):
if json_text is None or json_text == '':
return []
input_data = json.loads(json_text)
data = []
for key, value in input_data.items():
group_name, name = parse_name(key)
result, measurements = parse_metric(value)
if result is not None and not (math.isnan(result) or math.isinf(result)):
data.append({
"name": name,
"group_name": group_name,
"result": result,
"measurements": measurements,
})
return data
def __call__(json_text):
if json_text is None or json_text == '':
return []
input_data = json.loads(json_text)
data = []
for key, value in input_data.items():
group_name, name = parse_name(key)
result, measurements = parse_metric(value)
if result is not None and not (math.isnan(result) or math.isinf(result)):
data.append({
"name": name,
"group_name": group_name,
"result": result,
"measurements": measurements,
})
return data
def __call__(test_data):
if test_data is None or test_data == '':
return []
input_data = json.loads(test_data)
data = []
for key, value in input_data.items():
group_name, test_name = parse_name(key)
result = value
log = None
if isinstance(value, dict):
result = value.get('result', None)
log = value.get('log', None)
data.append({
"group_name": group_name,
"test_name": test_name,
"pass": parse_test_result(result),
"log": log
})
return data
def __init__(self, project, full_test_name, top=None, page=1, per_page=20):
suite, test_name = parse_name(full_test_name)
self.test = full_test_name
self.paginator = Paginator(project.builds.reverse(), per_page)
if top:
self.number = 0
builds = project.builds.filter(
datetime__lte=top.datetime).reverse()[0:per_page - 1]
else:
self.number = page
builds = self.paginator.page(page)
suite = project.suites.get(slug=suite)
tests = Test.objects.filter(
suite=suite,
name=test_name,
test_run__build__in=builds,
)
Test.prefetch_related(tests)
environments = OrderedDict()
results = OrderedDict()
for build in builds:
results[build] = {}
self.top = builds[0]
for test in tests:
build = test.test_run.build
environment = test.test_run.environment
environments[environment] = True
results[build][environment] = TestResult(test)
self.environments = sorted(environments.keys(),
key=lambda env: env.slug)
self.results = results
def __call__(json_text):
if json_text is None or json_text == '':
return []
input_data = json.loads(json_text)
data = []
for metric, value_dict in input_data.items():
unit = None
if type(value_dict) is dict:
unit = value_dict.get('unit', None)
value = value_dict.get('value', None)
else:
value = value_dict
group_name, name = parse_name(metric)
result, measurements = parse_metric(value)
if result is not None and not (math.isnan(result) or math.isinf(result)):
data.append({
"name": name,
"group_name": group_name,
"result": result,
"measurements": measurements,
"unit": unit,
})
return data
def __init__(self, project, full_test_name, top=None, page=1, per_page=20):
suite, test_name = parse_name(full_test_name)
self.test = full_test_name
self.paginator = Paginator(project.builds.reverse(), per_page)
if top:
self.number = 0
builds = project.builds.filter(datetime__lte=top.datetime).reverse()[0:per_page - 1]
else:
self.number = page
builds = self.paginator.page(page)
suite = project.suites.get(slug=suite)
tests = Test.objects.filter(
suite=suite,
name=test_name,
test_run__build__in=builds,
)
Test.prefetch_related(tests)
environments = OrderedDict()
results = OrderedDict()
for build in builds:
results[build] = {}
self.top = builds[0]
for test in tests:
build = test.test_run.build
environment = test.test_run.environment
environments[environment] = True
results[build][environment] = TestResult(test)
self.environments = sorted(environments.keys(), key=lambda env: env.slug)
self.results = results
def test_nested(self):
self.assertEqual(('foo/bar', 'baz'), parse_name('foo/bar/baz'))
def test_simple(self):
self.assertEqual(('foo', 'bar'), parse_name('foo/bar'))
def test_variants_missing_opening_bracket(self):
self.assertEqual(('long/special/case.forresult', 'variants]'),
parse_name("long/special/case.forresult/variants]"))
def test_nested(self):
self.assertEqual(('foo/bar', 'baz'), parse_name('foo/bar/baz'))
def test_variants_simple(self):
self.assertEqual(('special', 'case.for[result/variants]'),
parse_name("special/case.for[result/variants]"))
def test_variants_simple(self):
self.assertEqual(('special', 'case.for[result/variants]'),
parse_name("special/case.for[result/variants]"))
def test_multiple_leading_slashes(self):
self.assertEqual(('/', 'foo'), parse_name('//foo'))
def test_multiple_leading_slashes(self):
self.assertEqual(('/', 'foo'), parse_name('//foo'))
def test_variants_nested(self):
self.assertEqual(('long/special', 'case.for[result/variants]'),
parse_name("long/special/case.for[result/variants]"))
def test_variants_missing_opening_bracket(self):
self.assertEqual(('long/special/case.forresult', 'variants]'),
parse_name("long/special/case.forresult/variants]"))
def test_simple(self):
self.assertEqual(('foo', 'bar'), parse_name('foo/bar'))
def test_variants_multiple_leading_slashes(self):
self.assertEqual(('/', 'case.for[result/variants]'),
parse_name("//case.for[result/variants]"))
def test_variants_ungrouped(self):
self.assertEqual(('/', 'case.for[result/variants]'),
parse_name("case.for[result/variants]"))
def test_variants_multiple_leading_slashes(self):
self.assertEqual(('/', 'case.for[result/variants]'),
parse_name("//case.for[result/variants]"))
def test_variants_ungrouped(self):
self.assertEqual(('/', 'case.for[result/variants]'),
parse_name("case.for[result/variants]"))
def test_variants_nested(self):
self.assertEqual(('long/special', 'case.for[result/variants]'),
parse_name("long/special/case.for[result/variants]"))
def by_full_name(self, name):
(suite, metric) = parse_name(name)
return self.filter(suite__slug=suite, name=metric)
def by_full_name(self, name):
(suite, metric) = parse_name(name)
return self.filter(suite__slug=suite, name=metric)
def test_ungrouped(self):
self.assertEqual(('/', 'foo'), parse_name('foo'))
def test_ungrouped(self):
self.assertEqual(('/', 'foo'), parse_name('foo'))
