def test_baseline_1(self):
env = G(models.Environment)
baseline = G(models.Result,
manifest=MANIFEST(),
branch_name="master",
gerrit_change_number=None)
current = G(models.Result,
manifest=MANIFEST(),
branch_name="master",
gerrit_change_number=123)
G(models.ResultData,
result=current,
benchmark__name="load",
name="load-avg",
measurement=10)
G(models.TestJob, result=current, environment=env)
G(models.ResultData,
result=baseline,
benchmark__name="load",
name="load-avg",
measurement=10)
G(models.TestJob, result=baseline, environment=env)
response = self.client.get('/api/result/%s/baseline/' % current.pk)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.data['id'], baseline.id)
def setUp(self):
self.now = timezone.now()
self.past = self.now - relativedelta(days=1)
self.baseline = G(Result,
manifest=MANIFEST(),
branch_name='master',
gerrit_change_number=None,
created_at=self.past)
self.current = G(Result,
manifest=MANIFEST(),
branch_name='master',
gerrit_change_number=123,
gerrit_change_id='I8adbccfe4b39a1e849b5d7a976fd4cdc',
created_at=self.now)
benchmark1 = G(Benchmark)
G(ResultData,
result=self.baseline,
benchmark=benchmark1,
name="benchmark1",
measurement=5)
G(ResultData,
result=self.current,
benchmark=benchmark1,
name="benchmark1",
measurement=6)
fake_gerrit.clear()
def __progress__(self, past, **kwargs):
build1 = G(
Result,
manifest=MANIFEST(),
branch_name='master',
name='myproject',
gerrit_change_number=None,
created_at=past,
)
build2 = G(
Result,
manifest=MANIFEST(),
branch_name="master",
name='myproject',
gerrit_change_number=None,
created_at=timezone.now(),
)
env = G(Environment, identifier="myenv1")
job1 = G(TestJob, result=build1, environment=env, completed=True)
job1.data = get_file("then.json")
job1.save()
job2 = G(TestJob, result=build2, environment=env, completed=True)
job2.data = get_file("now.json")
job2.save()
progress = Progress("myproject", "master", env, job1, job2)
return [progress]
def test_to_compare_for_baseline_builds(self):
now = timezone.now()
then = now - relativedelta(days=7)
current_master = G(Result,
manifest=MANIFEST(),
branch_name="master",
gerrit_change_number=None,
created_at=now)
previous_master = G(Result,
manifest=MANIFEST(),
branch_name="master",
gerrit_change_number=None,
created_at=then)
G(ResultData,
result=current_master,
benchmark__name="load",
name="load-avg",
measurement=10)
G(ResultData,
result=previous_master,
benchmark__name="load",
name="load-avg",
measurement=10)
self.assertEqual(previous_master, current_master.to_compare())
def test_returns_only_mainline_results(self):
mainline_result = G(models.Result,
manifest=MANIFEST(),
branch_name='master',
gerrit_change_number=None)
patch_result = G(models.Result,
manifest=MANIFEST(),
branch_name='master',
gerrit_change_number=999)
group = G(models.BenchmarkGroup)
env = G(models.Environment)
G(models.BenchmarkGroupSummary,
group=group,
environment=env,
result=mainline_result)
G(models.BenchmarkGroupSummary,
group=group,
environment=env,
result=patch_result)
response = self.client.get(
'/api/benchmark_group_summary/', {
'benchmark_group': group.name,
'environment': env.identifier,
'branch': 'master',
})
self.assertEqual(1, len(response.data))
def test_only_baseline_changes_ignore_gerrit(self):
now = timezone.now()
yesterday = now - relativedelta(days=1)
baseline = G(models.Result,
manifest=MANIFEST(),
branch_name='master',
name="TheProject",
created_at=yesterday,
gerrit_change_number=None)
patched = G(models.Result,
manifest=MANIFEST(),
branch_name='master',
name="TheProject",
created_at=now,
gerrit_change_number=123)
benchmark = G(models.Benchmark,
group__name='foo/bar/',
name="TheBenchmark")
environment = G(models.Environment, identifier='myenv')
testjob_baseline = G(models.TestJob,
id=u'888888',
environment=environment,
result=baseline,
completed=True)
testjob_patched = G(models.TestJob,
id=u'999999',
environment=environment,
result=patched,
completed=True)
G(models.ResultData,
result=baseline,
benchmark=benchmark,
test_job_id=testjob_baseline.id,
name="TheBenchmark",
measurement=5)
G(models.ResultData,
result=patched,
benchmark=benchmark,
test_job_id=testjob_patched.id,
name="TheBenchmark",
measurement=10)
response = self.client.get(
'/api/stats/', {
'branch': 'master',
'benchmark': benchmark.full_name,
'project': 'TheProject',
'environment': 'myenv',
})
self.assertEqual(len(response.data), 2)
self.assertEqual(response.data[0]['measurement'], 10)
self.assertEqual(response.data[1]['measurement'], 5)
def test_baseline_3(self):
now = timezone.now()
yesterday = now - relativedelta(days=1)
env = G(models.Environment)
env2 = G(models.Environment)
wrong_baseline = G(models.Result,
manifest=MANIFEST(),
branch_name="master",
created_at=now,
gerrit_change_number=None)
baseline = G(models.Result,
manifest=MANIFEST(),
branch_name="master",
created_at=yesterday,
gerrit_change_number=None)
current = G(models.Result,
manifest=MANIFEST(),
branch_name="master",
gerrit_change_number=123)
G(models.ResultData,
result=current,
benchmark__name="load",
name="load-avg",
measurement=10)
G(models.TestJob,
result=current,
environment=env)
G(models.ResultData,
result=wrong_baseline,
benchmark__name="load",
name="load-avg",
measurement=10)
G(models.TestJob,
result=wrong_baseline,
environment=env2)
G(models.ResultData,
result=baseline,
benchmark__name="load",
name="load-avg",
measurement=10)
G(models.TestJob,
result=baseline,
environment=env)
response = self.client.get('/api/result/%s/baseline/' % current.pk)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.data['id'], baseline.id)
def test_missing_parameters(self):
now = timezone.now()
yesterday = now - relativedelta(days=1)
baseline = G(models.Result,
manifest=MANIFEST(),
branch_name='master',
name="TheProject",
created_at=yesterday,
gerrit_change_number=None)
patched = G(models.Result,
manifest=MANIFEST(),
branch_name='master',
name="TheProject",
created_at=now,
gerrit_change_number=123)
group = G(models.BenchmarkGroup, name='foo/')
benchmark = G(models.Benchmark, group=group, name="TheBenchmark")
G(models.ResultData,
result=baseline,
benchmark=benchmark,
name="TheBenchmark",
measurement=5)
G(models.ResultData,
result=patched,
benchmark=benchmark,
name="TheBenchmark",
measurement=10)
response = self.client.get('/api/stats/', {
'branch': 'master',
'benchmark': benchmark.full_name,
})
self.assertEqual(len(response.data), 0)
response = self.client.get('/api/stats/', {
'benchmark': benchmark.full_name,
'project': 'TheProject',
})
self.assertEqual(len(response.data), 0)
response = self.client.get('/api/stats/', {
'branch': 'master',
'project': 'TheProject',
})
self.assertEqual(len(response.data), 0)
def test_set_testjob_result_saves_testjob(self):
result = G(Result, manifest=MANIFEST())
testjob = G(TestJob, result=result, status='Submitted')
set_testjob_results.apply(args=[testjob.id])
testjob_from_db = TestJob.objects.get(pk=testjob.id)
self.assertEqual("Complete", testjob_from_db.status)
def test_result_data_with_benchmark_group(self):
result = G(Result, manifest=MANIFEST())
testjob = N(TestJob, result=result, status='Complete')
test_results = [
{
'benchmark_group':
'foo',
'benchmark_name':
'bar',
'subscore': [
{
'name': 'test1',
'measurement': 1
},
{
'name': 'test1',
'measurement': 2
},
]
},
]
store_testjob_data(testjob, test_results)
self.assertEqual(BenchmarkGroup.objects.count(), 2) # foo + /
benchmark_group = BenchmarkGroup.objects.order_by('id').last()
benchmark = Benchmark.objects.order_by('id').last()
self.assertEqual(benchmark.group, benchmark_group)
def test_result_data(self):
result = G(Result, manifest=MANIFEST())
testjob = N(TestJob, result=result, status='Complete')
test_results = [
{
'benchmark_name':
'bar',
'subscore': [
{
'name': 'test1',
'measurement': 1
},
{
'name': 'test1',
'measurement': 2
},
]
},
]
store_testjob_data(testjob, test_results)
self.assertEqual(TestJob.objects.count(), 1)
self.assertEqual(Benchmark.objects.count(), 1)
self.assertEqual(ResultData.objects.count(), 1)
result_data = ResultData.objects.order_by('id').last()
self.assertEqual(result_data.values, [1, 2])
self.assertEqual(result_data.benchmark.name, 'bar')
def setUp(self):
self.testjob = G(models.TestJob,
id="0001",
status="Complete",
completed=True,
result__manifest=MANIFEST())
user = User.objects.create_superuser('test', '*****@*****.**', 'test')
self.client.force_authenticate(user=user)
def test_complete_2(self):
result = G(Result, manifest=MANIFEST())
testjob = G(TestJob, result=result, completed=False)
self.assertEqual(result.completed, False)
testjob.completed = True
self.assertEqual(result.completed, False)
def test_dont_duplicate_test_results(self):
result = G(Result, manifest=MANIFEST())
testjob = G(TestJob, result=result, status='Submitted')
set_testjob_results.apply(args=[testjob.id])
set_testjob_results.apply(args=[testjob.id])
self.assertEqual(2, result.data.count())
def test_resubmit_with_status_running(self):
testjob = G(models.TestJob, status='Running', result__manifest=MANIFEST())
response = self.client.get('/api/testjob/%s/resubmit/' % testjob.id)
self.assertEqual(response.status_code, 400)
self.assertEqual(models.TestJob.objects.count(), 1)
self.assertEqual(models.TestJob.objects.filter(id="111").exists(), False)
def test_complete_1(self):
result = G(Result, manifest=MANIFEST())
self.assertEqual(result.completed, False)
G(TestJob, result=result, completed=True)
G(TestJob, result=result, completed=True)
self.assertEqual(result.completed, True)
def setUp(self):
self.now = timezone.now()
past = self.now - relativedelta(days=1)
self.current = G(
Result,
manifest=MANIFEST(),
branch_name='master',
gerrit_change_number=123,
created_at=self.now,
)
self.baseline = G(
Result,
manifest=MANIFEST(),
branch_name='master',
gerrit_change_number=None,
created_at=past,
)
self.environment = G(Environment)
def test_resubmit_canceled(self):
testjob = G(models.TestJob, id="000", status="Canceled", result__manifest=MANIFEST())
response = self.client.get('/api/testjob/%s/resubmit/' % testjob.id)
self.assertEqual(response.data[0]['id'], "111")
self.assertEqual(models.TestJob.objects.count(), 2)
self.assertEqual(models.TestJob.objects.filter(id="000").exists(), True)
self.assertEqual(models.TestJob.objects.filter(id="111").exists(), True)
def test_basic(self):
result = G(models.Result, manifest=MANIFEST(), gerrit_change_number=None)
group = G(models.BenchmarkGroup)
env = G(models.Environment)
G(models.BenchmarkGroupSummary, group=group, environment=env, result=result)
response = self.client.get('/api/benchmark_group_summary/', {
'benchmark_group': group.name,
'environment': env.identifier,
'branch': result.branch_name,
})
self.assertEqual(1, len(response.data))
def test_render_comparison(self):
now = timezone.now()
then = now - relativedelta(days=7)
result_now = G(Result,
name="name1",
manifest=MANIFEST(),
branch_name="master",
gerrit_change_number=None,
created_at=now)
result_then = G(Result,
name="name2",
manifest=MANIFEST(),
branch_name="master",
gerrit_change_number=None,
created_at=now)
testjob_now = G(
TestJob,
result=result_now,
completed=True,
)
testjob_now.data = get_file("now.json")
testjob_now.save()
testjob_then = G(
TestJob,
result=result_now,
completed=True,
)
testjob_then.data = get_file("then.json")
testjob_then.save()
output = render_comparison(testjob_then, testjob_now)
self.assertTrue("benchmark1" in output)
def test_to_compare_missing_results_previous(self):
now = timezone.now()
then = now - relativedelta(days=7)
result_1 = G(Result,
manifest=MANIFEST(),
branch_name="master",
gerrit_change_number=123,
created_at=now)
result_2 = G(Result,
manifest=MANIFEST(),
branch_name="master",
gerrit_change_number=None,
created_at=then)
G(ResultData,
result=result_1,
benchmark__name="load",
name="load-avg",
measurement=10)
self.assertEqual(result_1.to_compare(), None)
def test_baseline_2(self):
result_1 = G(models.Result,
manifest=MANIFEST(),
branch_name="master",
gerrit_change_number=123)
G(models.ResultData,
result=result_1,
benchmark__name="load",
name="load-avg",
measurement=10)
response = self.client.get('/api/result/%s/baseline/' % result_1.pk)
self.assertEqual(response.status_code, 204)
def test_testjobs_updated(self):
present = timezone.now()
past = present - relativedelta(days=1)
result = G(Result, manifest=MANIFEST())
testjob = G(TestJob,
result=result,
completed=False,
updated_at=present)
result.updated_at = past
self.assertEqual(True, result.testjobs_updated)
result.updated_at = present
self.assertEqual(False, result.testjobs_updated)
result.updated_at = None
result.testjobs_updated # we are happy it this just doesn't crash
def test_can_resubmit_test_single_node(self):
result = G(Result, manifest=MANIFEST())
job = TestJob(result=result)
job.save()
self.assertFalse(job.can_resubmit())
job.status = "Complete"
job.save()
self.assertFalse(job.can_resubmit())
job.status = "Running"
job.save()
self.assertFalse(job.can_resubmit())
job.status = "Incomplete"
job.save()
self.assertTrue(job.can_resubmit())
job.status = "Canceled"
job.save()
self.assertTrue(job.can_resubmit())
job.status = "Results Missing"
job.save()
self.assertTrue(job.can_resubmit())
def test_can_resubmit_test_multi_node_target(self):
result = G(Result, manifest=MANIFEST())
job = TestJob(result=result, id="12345.1")
job.save()
self.assertFalse(job.can_resubmit())
job.status = "Complete"
job.save()
self.assertFalse(job.can_resubmit())
job.status = "Running"
job.save()
self.assertFalse(job.can_resubmit())
job.status = "Incomplete"
job.save()
self.assertFalse(job.can_resubmit())
job.status = "Canceled"
job.save()
self.assertFalse(job.can_resubmit())
job.status = "Results Missing"
job.save()
self.assertFalse(job.can_resubmit())
def test_geomean_with_only_zeros(self):
result = G(Result, manifest=MANIFEST())
progress = G(BenchmarkGroupSummary, values=[0,0,0], result=result)
self.assertAlmostEqual(0, progress.measurement, delta=0.0001)
def test_geomean(self):
result = G(Result, manifest=MANIFEST())
progress = G(BenchmarkGroupSummary, values=[1,2], result=result)
self.assertAlmostEqual(1.4142, progress.measurement, delta=0.0001)
def test_data_filetype(self):
result = G(Result, manifest=MANIFEST())
job = G(TestJob, result=result, id="12345.1")
job.data = get_file('now.json')
self.assertEqual('json', job.data_filetype)
def test_updated_at_on_creation(self):
result = G(Result, manifest=MANIFEST())
self.assertTrue(result.updated_at is not None)