def setUp(self):
self.data1 = {
'commitid': '2',
'branch': 'default', # Always use default for trunk/master/tip
'project': 'MyProject',
'executable': 'myexe O3 64bits',
'benchmark': 'float',
'environment': "Bulldozer",
'result_value': 4000,
}
self.data_optional = {
'std_dev': 0.2,
'val_min': 2.23,
'val_max': 3.42,
'date': datetime.now().strftime(self.DATETIME_FORMAT),
}
project_data = dict(
name="PyPy",
repo_type="M",
repo_path="ssh://[email protected]/pypy/pypy",
repo_user="******",
repo_pass="******",
)
self.project = Project(**project_data)
self.project.save()
self.env1 = Environment(name='Bulldozer')
self.env1.save()
def setUp(self):
self.env1_data = dict(name="env1",
cpu="cpu1",
memory="48kB",
os="ZX Spectrum OS",
kernel="2.6.32")
self.env1 = Environment(**self.env1_data)
self.env1.save()
self.env2_data = dict(name="env2",
cpu="z80",
memory="64kB",
os="ZX Spectrum OS",
kernel="2.6.32")
self.client = Client()
super(EnvironmentTest, self).setUp()
def setUp(self):
self.path = reverse('codespeed.views.add_result')
self.e = Environment(name='Dual Core', cpu='Core 2 Duo 8200')
self.e.save()
temp = datetime.today()
self.cdate = datetime(
temp.year, temp.month, temp.day, temp.hour, temp.minute, temp.second)
self.data = {
'commitid': '23',
'branch': 'default',
'project': 'MyProject',
'executable': 'myexe O3 64bits',
'benchmark': 'float',
'environment': 'Dual Core',
'result_value': 456,
}
def setUp(self):
self.data1 = {
'commitid': '2',
'branch': 'default', # Always use default for trunk/master/tip
'project': 'MyProject',
'executable': 'myexe O3 64bits',
'benchmark': 'float',
'environment': "Bulldozer",
'result_value': 4000,
}
self.data_optional = {
'std_dev': 0.2,
'val_min': 2.23,
'val_max': 3.42,
'date': datetime.now().strftime(self.DATETIME_FORMAT),
}
project_data = dict(
name="PyPy",
repo_type="M",
repo_path="ssh://[email protected]/pypy/pypy",
repo_user="******",
repo_pass="******",
)
self.project = Project(**project_data)
self.project.save()
self.env1 = Environment(name='Bulldozer')
self.env1.save()
def setUp(self):
self.path = reverse('codespeed.views.add_json_results')
self.client = Client()
self.e = Environment(name='bigdog', cpu='Core 2 Duo 8200')
self.e.save()
temp = datetime.today()
self.cdate = datetime(temp.year, temp.month, temp.day, temp.hour,
temp.minute, temp.second)
self.data = [
{
'commitid': '123',
'project': 'pypy',
'branch': 'default',
'executable': 'pypy-c',
'benchmark': 'Richards',
'environment': 'bigdog',
'result_value': 456,
},
{
'commitid': '456',
'project': 'pypy',
'branch': 'default',
'executable': 'pypy-c',
'benchmark': 'Richards',
'environment': 'bigdog',
'result_value': 457,
},
{
'commitid': '456',
'project': 'pypy',
'branch': 'default',
'executable': 'pypy-c',
'benchmark': 'Richards2',
'environment': 'bigdog',
'result_value': 34,
},
{
'commitid': '789',
'project': 'pypy',
'branch': 'default',
'executable': 'pypy-c',
'benchmark': 'Richards',
'environment': 'bigdog',
'result_value': 458,
},
]
def setUp(self):
self.env1_data = dict(name="env1",
cpu="cpu1",
memory="48kB",
os="ZX Spectrum OS",
kernel="2.6.32")
self.env1 = Environment(**self.env1_data)
self.env1.save()
self.env2_data = dict(name="env2",
cpu="z80",
memory="64kB",
os="ZX Spectrum OS",
kernel="2.6.32")
env_db1 = Environment.objects.get(id=1)
self.env_db1_data = dict([(k, getattr(env_db1, k))
for k in self.env1_data.keys()])
self.client = Client()
super(EnvironmentTest, self).setUp()
def setUp(self):
self.path = reverse('codespeed.views.addresult')
self.client = Client()
self.e = Environment(name='bigdog', cpu='Core 2 Duo 8200')
self.e.save()
temp = datetime.today()
self.cdate = datetime(temp.year, temp.month, temp.day, temp.hour, temp.minute, temp.second)
self.data = {
'commitid': '23232',
'project': 'pypy',
'executable_name': 'pypy-c',
'benchmark': 'Richards',
'environment': 'bigdog',
'result_value': 456,
}
def setUp(self):
self.path = reverse("codespeed.views.add_result")
self.client = Client()
self.e = Environment(name="Dual Core", cpu="Core 2 Duo 8200")
self.e.save()
temp = datetime.today()
self.cdate = datetime(temp.year, temp.month, temp.day, temp.hour, temp.minute, temp.second)
self.data = {
"commitid": "23",
"branch": "default",
"project": "MyProject",
"executable": "myexe O3 64bits",
"benchmark": "float",
"environment": "Dual Core",
"result_value": 456,
}
def setUp(self):
self.path = reverse("codespeed.views.add_json_results")
self.client = Client()
self.e = Environment(name="bigdog", cpu="Core 2 Duo 8200")
self.e.save()
temp = datetime.today()
self.cdate = datetime(temp.year, temp.month, temp.day, temp.hour, temp.minute, temp.second)
self.data = [
{
"commitid": "123",
"project": "pypy",
"branch": "default",
"executable": "pypy-c",
"benchmark": "Richards",
"environment": "bigdog",
"result_value": 456,
},
{
"commitid": "456",
"project": "pypy",
"branch": "default",
"executable": "pypy-c",
"benchmark": "Richards",
"environment": "bigdog",
"result_value": 457,
},
{
"commitid": "456",
"project": "pypy",
"branch": "default",
"executable": "pypy-c",
"benchmark": "Richards2",
"environment": "bigdog",
"result_value": 34,
},
{
"commitid": "789",
"project": "pypy",
"branch": "default",
"executable": "pypy-c",
"benchmark": "Richards",
"environment": "bigdog",
"result_value": 458,
},
]
def setUp(self):
self.days = 0
self.starttime = datetime.now() + timedelta(days=-100)
Project(repo_type='G', name='pro',
repo_path='/home/foo/codespeed').save()
self.pro = Project.objects.get(name='pro')
Branch(project=self.pro, name='branch').save()
self.b = Branch.objects.get(name='branch')
Environment(name='Walden Pond').save()
Executable(name='walden', project=self.pro).save()
Benchmark(name='TestBench').save()
self.env = Environment.objects.get(name='Walden Pond')
self.exe = Executable.objects.get(name='walden')
self.bench = Benchmark.objects.get(name='TestBench')
def setUp(self):
self.env1_data = dict(
name="env1",
cpu="cpu1",
memory="48kB",
os="ZX Spectrum OS",
kernel="2.6.32"
)
self.env1 = Environment(**self.env1_data)
self.env1.save()
self.env2_data = dict(
name="env2",
cpu="z80",
memory="64kB",
os="ZX Spectrum OS",
kernel="2.6.32"
)
self.client = Client()
super(EnvironmentTest, self).setUp()
def setUp(self):
self.path = reverse('add-json-results')
self.e = Environment(name='bigdog', cpu='Core 2 Duo 8200')
self.e.save()
temp = datetime.today()
self.cdate = datetime(
temp.year, temp.month, temp.day, temp.hour, temp.minute,
temp.second)
self.data = [
{'commitid': '123',
'project': 'pypy',
'branch': 'default',
'executable': 'pypy-c',
'benchmark': 'Richards',
'environment': 'bigdog',
'result_value': 456},
{'commitid': '456',
'project': 'pypy',
'branch': 'default',
'executable': 'pypy-c',
'benchmark': 'Richards',
'environment': 'bigdog',
'result_value': 457},
{'commitid': '456',
'project': 'pypy',
'branch': 'default',
'executable': 'pypy-c',
'benchmark': 'Richards2',
'environment': 'bigdog',
'result_value': 34},
{'commitid': '789',
'project': 'pypy',
'branch': 'default',
'executable': 'pypy-c',
'benchmark': 'Richards',
'environment': 'bigdog',
'result_value': 458},
]
def setUp(self):
self.env1_data = dict(
name="env1",
cpu="cpu1",
memory="48kB",
os="ZX Spectrum OS",
kernel="2.6.32"
)
self.env1 = Environment(**self.env1_data)
self.env1.save()
self.env2_data = dict(
name="env2",
cpu="z80",
memory="64kB",
os="ZX Spectrum OS",
kernel="2.6.32"
)
env_db1 = Environment.objects.get(id=1)
self.env_db1_data = dict(
[(k, getattr(env_db1, k)) for k in self.env1_data.keys()]
)
self.client = Client()
super(EnvironmentTest, self).setUp()
class AddResultTest(TestCase):
def setUp(self):
self.path = reverse('codespeed.views.addresult')
self.client = Client()
self.e = Environment(name='bigdog', cpu='Core 2 Duo 8200')
self.e.save()
temp = datetime.today()
self.cdate = datetime(temp.year, temp.month, temp.day, temp.hour, temp.minute, temp.second)
self.data = {
'commitid': '23232',
'project': 'pypy',
'executable_name': 'pypy-c',
'benchmark': 'Richards',
'environment': 'bigdog',
'result_value': 456,
}
def test_add_default_result(self):
"""
Add result data using default options
"""
response = self.client.post(self.path, self.data)
self.assertEquals(response.status_code, 200)
self.assertEquals(response.content, "Result data saved succesfully")
e = Environment.objects.get(name='bigdog')
b = Benchmark.objects.get(name='Richards')
self.assertEquals(b.benchmark_type, "C")
self.assertEquals(b.units, "seconds")
self.assertEquals(b.lessisbetter, True)
p = Project.objects.get(name='pypy')
r = Revision.objects.get(commitid='23232', project=p)
i = Executable.objects.get(name='pypy-c', coptions='')
res = Result.objects.get(
revision=r,
executable=i,
benchmark=b,
environment=e
)
self.assertTrue(res.value, 456)
resdate = res.date.strftime("%Y%m%dT%H%M%S")
selfdate = self.cdate.strftime("%Y%m%dT%H%M%S")
self.assertTrue(resdate, selfdate)
def test_add_non_default_result(self):
"""
Add result data with non-default options
"""
modified_data = copy.deepcopy(self.data)
modified_data['executable_coptions'] = 'gc=Böhm'
modified_data['benchmark_type'] = "O"
modified_data['units'] = "fps"
modified_data['lessisbetter'] = False
modified_data['result_date'] = self.cdate
modified_data['std_dev'] = 1.11111
modified_data['max'] = 2
modified_data['min'] = 1.0
response = self.client.post(self.path, modified_data)
self.assertEquals(response.status_code, 200)
self.assertEquals(response.content, "Result data saved succesfully")
e = Environment.objects.get(name='bigdog')
p = Project.objects.get(name='pypy')
r = Revision.objects.get(commitid='23232', project=p)
self.assertEquals(r.date, self.cdate)
i = Executable.objects.get(name='pypy-c', coptions='gc=Böhm')
b = Benchmark.objects.get(name='Richards')
res = Result.objects.get(
revision=r,
executable=i,
benchmark=b,
environment=e
)
self.assertEquals(res.std_dev, 1.11111)
self.assertEquals(res.val_max, 2)
self.assertEquals(res.val_min, 1)
def test_bad_environment(self):
"""
Add result associated with non-existing environment
"""
bad_name = 'bigdog1'
self.data['environment'] = bad_name
response = self.client.post(self.path, self.data)
self.assertEquals(response.status_code, 404)
self.assertEquals(response.content, "Environment " + bad_name + " not found")
self.data['environment'] = 'bigdog'
def test_empty_argument(self):
"""
Make POST request with an empty argument.
"""
for key in self.data:
backup = self.data[key]
self.data[key] = ""
response = self.client.post(self.path, self.data)
self.assertEquals(response.status_code, 400)
self.assertEquals(response.content, 'Key "' + key + '" empty in request')
self.data[key] = backup
def test_missing_argument(self):
"""
Make POST request with a missing argument.
"""
for key in self.data:
backup = self.data[key]
del(self.data[key])
response = self.client.post(self.path, self.data)
self.assertEquals(response.status_code, 400)
self.assertEquals(response.content, 'Key "' + key + '" missing from request')
self.data[key] = backup
class TestAddJSONResults(TestCase):
def setUp(self):
self.path = reverse('codespeed.views.add_json_results')
self.e = Environment(name='bigdog', cpu='Core 2 Duo 8200')
self.e.save()
temp = datetime.today()
self.cdate = datetime(
temp.year, temp.month, temp.day, temp.hour, temp.minute, temp.second)
self.data = [
{'commitid': '123',
'project': 'pypy',
'branch': 'default',
'executable': 'pypy-c',
'benchmark': 'Richards',
'environment': 'bigdog',
'result_value': 456},
{'commitid': '456',
'project': 'pypy',
'branch': 'default',
'executable': 'pypy-c',
'benchmark': 'Richards',
'environment': 'bigdog',
'result_value': 457},
{'commitid': '456',
'project': 'pypy',
'branch': 'default',
'executable': 'pypy-c',
'benchmark': 'Richards2',
'environment': 'bigdog',
'result_value': 34},
{'commitid': '789',
'project': 'pypy',
'branch': 'default',
'executable': 'pypy-c',
'benchmark': 'Richards',
'environment': 'bigdog',
'result_value': 458},
]
def test_get_returns_405(self):
response = self.client.get(self.path,
{'json': json.dumps(self.data)})
self.assertEquals(response.status_code, 405)
def test_add_correct_results(self):
"""Should add all results when the request data is valid"""
response = self.client.post(self.path,
{'json': json.dumps(self.data)})
# Check that we get a success response
self.assertEquals(response.status_code, 202)
self.assertEquals(response.content.decode(),
"All result data saved successfully")
# Check that the data was correctly saved
e = Environment.objects.get(name='bigdog')
b = Benchmark.objects.get(name='Richards')
self.assertEquals(b.benchmark_type, "C")
self.assertEquals(b.units, "seconds")
self.assertEquals(b.lessisbetter, True)
p = Project.objects.get(name='pypy')
branch = Branch.objects.get(name='default', project=p)
r = Revision.objects.get(commitid='123', branch=branch)
i = Executable.objects.get(name='pypy-c')
res = Result.objects.get(
revision=r,
executable=i,
benchmark=b,
environment=e
)
self.assertTrue(res.value, 456)
resdate = res.date.strftime("%Y%m%dT%H%M%S")
selfdate = self.cdate.strftime("%Y%m%dT%H%M%S")
self.assertTrue(resdate, selfdate)
r = Revision.objects.get(commitid='456', branch=branch)
res = Result.objects.get(
revision=r,
executable=i,
benchmark=b,
environment=e
)
self.assertTrue(res.value, 457)
r = Revision.objects.get(commitid='789', branch=branch)
res = Result.objects.get(
revision=r,
executable=i,
benchmark=b,
environment=e
)
self.assertTrue(res.value, 458)
def test_bad_environment(self):
"""Add result associated with non-existing environment.
Only change one item in the list.
"""
data = self.data[0]
bad_name = 'bigdog1'
data['environment'] = bad_name
response = self.client.post(self.path,
{'json': json.dumps(self.data)})
self.assertEquals(response.status_code, 400)
self.assertEquals(response.content.decode(), "Environment " + bad_name + " not found")
data['environment'] = 'bigdog'
def test_empty_argument(self):
'''Should return 400 when making a request with an empty argument'''
data = self.data[1]
for key in data:
backup = data[key]
data[key] = ""
response = self.client.post(self.path,
{'json': json.dumps(self.data)})
self.assertEquals(response.status_code, 400)
self.assertEquals(response.content.decode(), 'Value for key "' + key + '" empty in request')
data[key] = backup
def test_missing_argument(self):
'''Should return 400 when making a request with a missing argument'''
data = self.data[2]
for key in data:
backup = data[key]
del(data[key])
response = self.client.post(self.path,
{'json': json.dumps(self.data)})
self.assertEquals(response.status_code, 400)
self.assertEquals(response.content.decode(), 'Key "' + key + '" missing from request')
data[key] = backup
def test_report_is_created(self):
'''Should create a report when adding json results for two revisions
plus a third revision with one result less than the last one'''
response = self.client.post(self.path,
{'json': json.dumps(self.data)})
# Check that we get a success response
self.assertEquals(response.status_code, 202)
number_of_reports = len(Report.objects.all())
# After adding 4 result for 3 revisions, only 2 reports should be created
# The third revision will need an extra result for Richards2 in order
# to trigger report creation
self.assertEquals(number_of_reports, 1)
class AddJSONResults(TestCase):
def setUp(self):
self.path = reverse('codespeed.views.add_json_results')
self.client = Client()
self.e = Environment(name='bigdog', cpu='Core 2 Duo 8200')
self.e.save()
temp = datetime.today()
self.cdate = datetime(
temp.year, temp.month, temp.day, temp.hour, temp.minute, temp.second)
self.data = [
{'commitid': '123',
'project': 'pypy',
'branch': 'default',
'executable': 'pypy-c',
'benchmark': 'Richards',
'environment': 'bigdog',
'result_value': 456,},
{'commitid': '456',
'project': 'pypy',
'branch': 'default',
'executable': 'pypy-c',
'benchmark': 'Richards',
'environment': 'bigdog',
'result_value': 457,},
{'commitid': '456',
'project': 'pypy',
'branch': 'default',
'executable': 'pypy-c',
'benchmark': 'Richards2',
'environment': 'bigdog',
'result_value': 34,},
{'commitid': '789',
'project': 'pypy',
'branch': 'default',
'executable': 'pypy-c',
'benchmark': 'Richards',
'environment': 'bigdog',
'result_value': 458,},
]
def test_add_correct_results(self):
"""Should add all results when the request data is valid"""
response = self.client.post(self.path, {'json' : json.dumps(self.data)})
# Check that we get a success response
self.assertEquals(response.status_code, 202)
self.assertEquals(response.content, "All result data saved successfully")
# Check that the data was correctly saved
e = Environment.objects.get(name='bigdog')
b = Benchmark.objects.get(name='Richards')
self.assertEquals(b.benchmark_type, "C")
self.assertEquals(b.units, "seconds")
self.assertEquals(b.lessisbetter, True)
p = Project.objects.get(name='pypy')
branch = Branch.objects.get(name='default', project=p)
r = Revision.objects.get(commitid='123', branch=branch)
i = Executable.objects.get(name='pypy-c')
res = Result.objects.get(
revision=r,
executable=i,
benchmark=b,
environment=e
)
self.assertTrue(res.value, 456)
resdate = res.date.strftime("%Y%m%dT%H%M%S")
selfdate = self.cdate.strftime("%Y%m%dT%H%M%S")
self.assertTrue(resdate, selfdate)
r = Revision.objects.get(commitid='456', branch=branch)
res = Result.objects.get(
revision=r,
executable=i,
benchmark=b,
environment=e
)
self.assertTrue(res.value, 457)
r = Revision.objects.get(commitid='789', branch=branch)
res = Result.objects.get(
revision=r,
executable=i,
benchmark=b,
environment=e
)
self.assertTrue(res.value, 458)
def test_bad_environment(self):
"""Add result associated with non-existing environment.
Only change one item in the list.
"""
data = self.data[0]
bad_name = 'bigdog1'
data['environment'] = bad_name
response = self.client.post(self.path, {'json' : json.dumps(self.data)})
self.assertEquals(response.status_code, 400)
self.assertEquals(response.content, "Environment " + bad_name + " not found")
data['environment'] = 'bigdog'
def test_empty_argument(self):
'''Should return 400 when making a request with an empty argument'''
data = self.data[1]
for key in data:
backup = data[key]
data[key] = ""
response = self.client.post(self.path,
{'json' : json.dumps(self.data)})
self.assertEquals(response.status_code, 400)
self.assertEquals(response.content, 'Value for key "' + key + '" empty in request')
data[key] = backup
def test_missing_argument(self):
'''Should return 400 when making a request with a missing argument'''
data = self.data[2]
for key in data:
backup = data[key]
del(data[key])
response = self.client.post(self.path,
{'json' : json.dumps(self.data)})
self.assertEquals(response.status_code, 400)
self.assertEquals(response.content, 'Key "' + key + '" missing from request')
data[key] = backup
def test_report_is_created(self):
'''Should create a report when adding json results for two revisions
plus a third revision with one result less than the last one'''
response = self.client.post(self.path, {'json' : json.dumps(self.data)})
# Check that we get a success response
self.assertEquals(response.status_code, 202)
number_of_reports = len(Report.objects.all())
# After adding 4 result for 3 revisions, only 2 reports should be created
# The third revision will need an extra result for Richards2 in order
# to trigger report creation
self.assertEquals(number_of_reports, 1)
class AddResult(TestCase):
def setUp(self):
self.path = reverse('codespeed.views.add_result')
self.client = Client()
self.e = Environment(name='Dual Core', cpu='Core 2 Duo 8200')
self.e.save()
temp = datetime.today()
self.cdate = datetime(
temp.year, temp.month, temp.day, temp.hour, temp.minute, temp.second)
self.data = {
'commitid': '23',
'branch': 'default',
'project': 'MyProject',
'executable': 'myexe O3 64bits',
'benchmark': 'float',
'environment': 'Dual Core',
'result_value': 456,
}
def test_add_correct_result(self):
"""Add correct result data"""
response = self.client.post(self.path, self.data)
# Check that we get a success response
self.assertEquals(response.status_code, 202)
self.assertEquals(response.content, "Result data saved successfully")
# Check that the data was correctly saved
e = Environment.objects.get(name='Dual Core')
b = Benchmark.objects.get(name='float')
self.assertEquals(b.benchmark_type, "C")
self.assertEquals(b.units, "seconds")
self.assertEquals(b.lessisbetter, True)
p = Project.objects.get(name='MyProject')
branch = Branch.objects.get(name='default', project=p)
r = Revision.objects.get(commitid='23', branch=branch)
i = Executable.objects.get(name='myexe O3 64bits')
res = Result.objects.get(
revision=r,
executable=i,
benchmark=b,
environment=e
)
self.assertTrue(res.value, 456)
def test_add_non_default_result(self):
"""Add result data with non-mandatory options"""
modified_data = copy.deepcopy(self.data)
revision_date = self.cdate - timedelta(minutes=2)
modified_data['revision_date'] = revision_date
result_date = self.cdate + timedelta(minutes=2)
modified_data['result_date'] = result_date
modified_data['std_dev'] = 1.11111
modified_data['max'] = 2
modified_data['min'] = 1.0
response = self.client.post(self.path, modified_data)
self.assertEquals(response.status_code, 202)
self.assertEquals(response.content, "Result data saved successfully")
e = Environment.objects.get(name='Dual Core')
p = Project.objects.get(name='MyProject')
branch = Branch.objects.get(name='default', project=p)
r = Revision.objects.get(commitid='23', branch=branch)
# Tweak the resolution down to avoid failing over very slight differences:
self.assertEquals(r.date, revision_date)
i = Executable.objects.get(name='myexe O3 64bits')
b = Benchmark.objects.get(name='float')
res = Result.objects.get(
revision=r,
executable=i,
benchmark=b,
environment=e
)
self.assertEquals(res.date, result_date)
self.assertEquals(res.std_dev, 1.11111)
self.assertEquals(res.val_max, 2)
self.assertEquals(res.val_min, 1)
def test_bad_environment(self):
"""Should return 400 when environment does not exist"""
bad_name = '10 Core'
self.data['environment'] = bad_name
response = self.client.post(self.path, self.data)
self.assertEquals(response.status_code, 400)
self.assertEquals(response.content, "Environment " + bad_name + " not found")
self.data['environment'] = 'Dual Core'
def test_empty_argument(self):
"""Should respond 400 when a POST request has an empty argument"""
for key in self.data:
backup = self.data[key]
self.data[key] = ""
response = self.client.post(self.path, self.data)
self.assertEquals(response.status_code, 400)
self.assertEquals(
response.content, 'Value for key "' + key + '" empty in request')
self.data[key] = backup
def test_missing_argument(self):
"""Should respond 400 when a POST request is missing an argument"""
for key in self.data:
backup = self.data[key]
del(self.data[key])
response = self.client.post(self.path, self.data)
self.assertEquals(response.status_code, 400)
self.assertEquals(
response.content, 'Key "' + key + '" missing from request')
self.data[key] = backup
def test_report_is_not_created(self):
'''Should not create a report when adding a single result'''
response = self.client.post(self.path, self.data)
number_of_reports = len(Report.objects.all())
# After adding one result for one revision, there should be no reports
self.assertEquals(number_of_reports, 0)
def test_report_is_created(self):
"""Should create a report when adding a result for two revisions"""
response = self.client.post(self.path, self.data)
modified_data = copy.deepcopy(self.data)
modified_data['commitid'] = "23233"
response = self.client.post(self.path, modified_data)
number_of_reports = len(Report.objects.all())
# After adding a result for a second revision, a report should be created
self.assertEquals(number_of_reports, 1)
def test_submit_data_with_none_timestamp(self):
"""Should add a default revision date when timestamp is None"""
modified_data = copy.deepcopy(self.data)
# The value None will get urlencoded and converted to a "None" string
modified_data['revision_date'] = None
response = self.client.post(self.path, modified_data)
self.assertEquals(response.status_code, 202)
def test_add_result_with_no_project(self):
"""Should add a revision with the project"""
modified_data = copy.deepcopy(self.data)
modified_data['project'] = "My new project"
modified_data['executable'] = "My new executable"
response = self.client.post(self.path, modified_data)
self.assertEquals(response.status_code, 202)
self.assertEquals(response.content, "Result data saved successfully")
class ResultBundleResourceTestCase(FixtureTestCase):
"""Submitting new benchmark results"""
DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S'
def setUp(self):
self.data1 = {
'commitid': '2',
'branch': 'default', # Always use default for trunk/master/tip
'project': 'MyProject',
'executable': 'myexe O3 64bits',
'benchmark': 'float',
'environment': "Bulldozer",
'result_value': 4000,
}
self.data_optional = {
'std_dev': 0.2,
'val_min': 2.23,
'val_max': 3.42,
'date': datetime.now().strftime(self.DATETIME_FORMAT),
}
project_data = dict(
name="PyPy",
repo_type="M",
repo_path="ssh://[email protected]/pypy/pypy",
repo_user="******",
repo_pass="******",
)
self.project = Project(**project_data)
self.project.save()
self.env1 = Environment(name='Bulldozer')
self.env1.save()
def test_post_mandatory(self):
"""Should save a new result with only mandatory data"""
response = self.client.post('/api/v1/benchmark-result/',
data=json.dumps(self.data1),
content_type='application/json')
self.assertEquals(response.status_code, 201)
id = response['Location'].rsplit('/', 2)[-2]
result = Result.objects.get(pk=int(id))
# just to make the point
self.assertIsInstance(result, Result)
self.assertEqual(result.value, self.data1['result_value'])
def test_post_all_data(self):
"""Should save a new result with mandatory and optional data"""
data = dict(self.data1, **self.data_optional)
response = self.client.post('/api/v1/benchmark-result/',
data=json.dumps(data),
content_type='application/json')
self.assertEquals(response.status_code, 201)
def test_get_one(self):
"""Should get a result bundle"""
response = self.client.get('/api/v1/benchmark-result/1/',
content_type='application/json')
self.assertEquals(response.status_code, 200)
response_data = json.loads(response.content)
for k in ('project', 'result', 'branch', 'benchmark', 'environment',
'executable', 'revision'):
self.assertEqual(
response_data[k],
'/api/v1/{0}/1/'.format(k,))
class ResultBundleResourceTestCase(FixtureTestCase):
"""Submitting new benchmark results"""
DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S'
def setUp(self):
self.data1 = {
'commitid': '2',
'branch': 'default', # Always use default for trunk/master/tip
'project': 'MyProject',
'executable': 'myexe O3 64bits',
'benchmark': 'float',
'environment': "Bulldozer",
'result_value': 4000,
}
self.data_optional = {
'std_dev': 0.2,
'val_min': 2.23,
'val_max': 3.42,
'date': datetime.now().strftime(self.DATETIME_FORMAT),
}
project_data = dict(
name="PyPy",
repo_type="M",
repo_path="ssh://[email protected]/pypy/pypy",
repo_user="******",
repo_pass="******",
)
self.project = Project(**project_data)
self.project.save()
self.env1 = Environment(name='Bulldozer')
self.env1.save()
def test_post_mandatory(self):
"""Should save a new result with only mandatory data"""
response = self.client.post('/api/v1/benchmark-result/',
data=json.dumps(self.data1),
content_type='application/json')
self.assertEquals(response.status_code, 201)
id = response['Location'].rsplit('/', 2)[-2]
result = Result.objects.get(pk=int(id))
# just to make the point
self.assertIsInstance(result, Result)
self.assertEqual(result.value, self.data1['result_value'])
def test_post_all_data(self):
"""Should save a new result with mandatory and optional data"""
data = dict(self.data1, **self.data_optional)
response = self.client.post('/api/v1/benchmark-result/',
data=json.dumps(data),
content_type='application/json')
self.assertEquals(response.status_code, 201)
def test_get_one(self):
"""Should get a result bundle"""
response = self.client.get('/api/v1/benchmark-result/1/',
content_type='application/json')
self.assertEquals(response.status_code, 200)
response_data = json.loads(response.content)
for k in ('project', 'result', 'branch', 'benchmark', 'environment',
'executable', 'revision'):
self.assertEqual(response_data[k], '/api/v1/{0}/1/'.format(k, ))
class ResultBundleTestCase(FixtureTestCase):
def setUp(self):
self.data1 = {
'commitid': '2',
'branch': 'default', # Always use default for trunk/master/tip
'project': 'MyProject',
'executable': 'myexe O3 64bits',
'benchmark': 'float',
'environment': "Bulldozer",
'result_value': 4000,
}
DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S'
self.data_optional = {
'std_dev': 0.2,
'val_min': 2.23,
'val_max': 3.42,
'date': datetime.now().strftime(DATETIME_FORMAT),
}
project_data = dict(
name="PyPy",
repo_type="M",
repo_path="ssh://[email protected]/pypy/pypy",
repo_user="******",
repo_pass="******",
)
self.project = Project(**project_data)
self.project.save()
self.env1 = Environment(name='Bulldozer')
self.env1.save()
def test_populate_and_save(self):
bundle = ResultBundle(**self.data1)
bundle._populate_obj_by_data()
# should raise exception if not OK
bundle.save()
self.assert_(True)
def test_save_same_result_again(self):
"""Save a previously saved result. Expected is an IntegrityError"""
modified_data = copy.deepcopy(self.data1)
modified_data['environment'] = "Dual Core"
bundle = ResultBundle(**modified_data)
bundle._populate_obj_by_data()
self.assertRaises(IntegrityError, bundle.save)
def test_for_nonexistent_environment(self):
"""Save data using non existing environment. Expected is an
ImmediateHttpResponse
"""
modified_data = copy.deepcopy(self.data1)
modified_data['environment'] = "Foo the Bar"
self.assertRaises(ImmediateHttpResponse, ResultBundle, **modified_data)
def test_insufficient_data(self):
"""See if Result() is saved w/ insufficient data"""
modified_data = copy.deepcopy(self.data1)
modified_data.pop('environment')
self.assertRaises(ImmediateHttpResponse, ResultBundle, **modified_data)
def test_date_attr_set(self):
"""Check if date attr of Result() is set if not given"""
# date is set automatically
modified_data = copy.deepcopy(self.data1)
bundle = ResultBundle(**modified_data)
bundle.save()
self.assertIsInstance(bundle.obj.date, datetime)
# date set by value
modified_data['date'] = '2011-05-05 03:01:45'
ResultBundle(**modified_data)
# wrong date string
modified_data['date'] = '2011-05-05T03:01:45'
self.assertRaises(ImmediateHttpResponse, ResultBundle, **modified_data)
def test_optional_data(self):
"""Check handling of optional data"""
data = dict(self.data1.items() + self.data_optional.items())
bundle = ResultBundle(**data)
bundle.save()
self.assertIsInstance(bundle.obj.date, datetime)
self.assertEqual(bundle.obj.std_dev,
float(self.data_optional['std_dev']))
self.assertEqual(bundle.obj.val_max,
float(self.data_optional['val_max']))
self.assertEqual(bundle.obj.val_min,
float(self.data_optional['val_min']))
def test_non_exiting_items(self):
"""Check handling of optional data"""
modified_data = copy.deepcopy(self.data1)
modified_data['commitid'] = '0b31bf33a469ac2cb1949666eea54d69a36c3724'
modified_data['project'] = 'Cython'
modified_data['benchmark'] = 'Django Template'
modified_data['executable'] = 'pypy-jit'
bundle = ResultBundle(**modified_data)
bundle.save()
self.assertEqual(bundle.obj.revision.commitid,
modified_data['commitid'])
self.assertEqual(bundle.obj.benchmark.name, modified_data['benchmark'])
self.assertEqual(bundle.obj.project.name, modified_data['project'])
class EnvironmentTest(FixtureTestCase):
"""Test Environment() API
"""
def setUp(self):
self.env1_data = dict(
name="env1",
cpu="cpu1",
memory="48kB",
os="ZX Spectrum OS",
kernel="2.6.32"
)
self.env1 = Environment(**self.env1_data)
self.env1.save()
self.env2_data = dict(
name="env2",
cpu="z80",
memory="64kB",
os="ZX Spectrum OS",
kernel="2.6.32"
)
self.client = Client()
super(EnvironmentTest, self).setUp()
def test_get_environment(self):
"""Should get an existing environment"""
response = self.client.get('/api/v1/environment/1/')
self.assertEquals(response.status_code, 200)
self.assertEqual(json.loads(response.content)['name'], "Dual Core")
def test_get_environment_all_fields(self):
"""Should get all fields for an environment"""
response = self.client.get('/api/v1/environment/%s/' % (self.env1.id,))
self.assertEquals(response.status_code, 200)
for k in self.env1_data.keys():
self.assertEqual(
json.loads(response.content)[k], getattr(self.env1, k))
def test_post(self):
"""Should save a new environment"""
response = self.client.post('/api/v1/environment/',
data=json.dumps(self.env2_data),
content_type='application/json')
self.assertEquals(response.status_code, 201)
response = self.client.get('/api/v1/environment/3/')
for k, v in self.env2_data.items():
self.assertEqual(
json.loads(response.content)[k], v)
def test_put(self):
"""Should modify an existing environment"""
modified_data = copy.deepcopy(self.env2_data)
modified_data['name'] = "env2.2"
modified_data['memory'] = "128kB"
response = self.client.put('/api/v1/environment/3/',
data=json.dumps(modified_data),
content_type='application/json')
self.assertEquals(response.status_code, 201)
response = self.client.get('/api/v1/environment/3/')
for k, v in modified_data.items():
self.assertEqual(
json.loads(response.content)[k], v)
def test_delete(self):
"""Should delete an environment"""
response = self.client.delete('/api/v1/environment/1/',
content_type='application/json')
self.assertEquals(response.status_code, 204)
response = self.client.get('/api/v1/environment/1/')
self.assertEquals(response.status_code, 404)
class EnvironmentTest(FixtureTestCase):
"""Test Environment() API"""
def setUp(self):
self.env1_data = dict(name="env1",
cpu="cpu1",
memory="48kB",
os="ZX Spectrum OS",
kernel="2.6.32")
self.env1 = Environment(**self.env1_data)
self.env1.save()
self.env2_data = dict(name="env2",
cpu="z80",
memory="64kB",
os="ZX Spectrum OS",
kernel="2.6.32")
env_db1 = Environment.objects.get(id=1)
self.env_db1_data = dict([(k, getattr(env_db1, k))
for k in self.env1_data.keys()])
self.client = Client()
super(EnvironmentTest, self).setUp()
def test_get_environment(self):
"""Should get an existing environment"""
response = self.client.get('/api/v1/environment/1/')
self.assertEquals(response.status_code, 200)
self.assertEqual(json.loads(response.content)['name'], "Dual Core")
def test_get_environment_all_fields(self):
"""Should get all fields for an environment"""
response = self.client.get('/api/v1/environment/%s/' %
(self.env1.id, ))
self.assertEquals(response.status_code, 200)
for k in self.env1_data.keys():
self.assertEqual(
json.loads(response.content)[k], getattr(self.env1, k))
def test_post(self):
"""Should save a new environment"""
response = self.client.post('/api/v1/environment/',
data=json.dumps(self.env2_data),
content_type='application/json')
self.assertEquals(response.status_code, 201)
id = response['Location'].rsplit('/', 2)[-2]
response = self.client.get('/api/v1/environment/{0}/'.format(id))
for k, v in self.env2_data.items():
self.assertEqual(json.loads(response.content)[k], v)
response = self.client.delete('/api/v1/environment/{0}/'.format(id),
content_type='application/json')
self.assertEquals(response.status_code, 204)
def test_put(self):
"""Should modify an existing environment"""
modified_data = copy.deepcopy(self.env_db1_data)
modified_data['name'] = "env2.2"
modified_data['memory'] = "128kB"
response = self.client.put('/api/v1/environment/1/',
data=json.dumps(modified_data),
content_type='application/json')
self.assertEquals(response.status_code, 204)
response = self.client.get('/api/v1/environment/1/')
for k, v in modified_data.items():
self.assertEqual(json.loads(response.content)[k], v)
def test_delete(self):
"""Should delete an environment"""
response = self.client.get('/api/v1/environment/1/')
self.assertEquals(response.status_code, 200)
# from fixture
response = self.client.delete('/api/v1/environment/1/',
content_type='application/json')
self.assertEquals(response.status_code, 204)
response = self.client.get('/api/v1/environment/1/')
self.assertEquals(response.status_code, 404)
# from just created data
response = self.client.get('/api/v1/environment/{0}/'.format(
self.env1.id))
self.assertEquals(response.status_code, 200)
response = self.client.delete('/api/v1/environment/{0}/'.format(
self.env1.id),
content_type='application/json')
self.assertEquals(response.status_code, 204)
response = self.client.get('/api/v1/environment/{0}/'.format(
self.env1.id))
self.assertEquals(response.status_code, 404)
class ResultBundleTestCase(FixtureTestCase):
def setUp(self):
self.data1 = {
'commitid': '2',
'branch': 'default', # Always use default for trunk/master/tip
'project': 'MyProject',
'executable': 'myexe O3 64bits',
'benchmark': 'float',
'environment': "Bulldozer",
'result_value': 4000,
}
DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S'
self.data_optional = {
'std_dev': 0.2,
'val_min': 2.23,
'val_max': 3.42,
'date': datetime.now().strftime(DATETIME_FORMAT),
}
project_data = dict(
name="PyPy",
repo_type="M",
repo_path="ssh://[email protected]/pypy/pypy",
repo_user="******",
repo_pass="******",
)
self.project = Project(**project_data)
self.project.save()
self.env1 = Environment(name='Bulldozer')
self.env1.save()
def test_populate_and_save(self):
bundle = ResultBundle(**self.data1)
bundle._populate_obj_by_data()
# should raise exception if not OK
bundle.save()
self.assert_(True)
def test_save_same_result_again(self):
"""Save a previously saved result. Expected is an IntegrityError"""
modified_data = copy.deepcopy(self.data1)
modified_data['environment'] = "Dual Core"
bundle = ResultBundle(**modified_data)
bundle._populate_obj_by_data()
self.assertRaises(IntegrityError, bundle.save)
def test_for_nonexistent_environment(self):
"""Save data using non existing environment. Expected is an
ImmediateHttpResponse
"""
modified_data = copy.deepcopy(self.data1)
modified_data['environment'] = "Foo the Bar"
self.assertRaises(ImmediateHttpResponse, ResultBundle, **modified_data)
def test_insufficient_data(self):
"""See if Result() is saved w/ insufficient data"""
modified_data = copy.deepcopy(self.data1)
modified_data.pop('environment')
self.assertRaises(ImmediateHttpResponse, ResultBundle, **modified_data)
def test_date_attr_set(self):
"""Check if date attr of Result() is set if not given"""
# date is set automatically
modified_data = copy.deepcopy(self.data1)
bundle = ResultBundle(**modified_data)
bundle.save()
self.assertIsInstance(bundle.obj.date, datetime)
# date set by value
modified_data['date'] = '2011-05-05 03:01:45'
ResultBundle(**modified_data)
# wrong date string
modified_data['date'] = '2011-05-05T03:01:45'
self.assertRaises(ImmediateHttpResponse, ResultBundle, **modified_data)
def test_optional_data(self):
"""Check handling of optional data"""
data = dict(self.data1.items() + self.data_optional.items())
bundle = ResultBundle(**data)
bundle.save()
self.assertIsInstance(bundle.obj.date, datetime)
self.assertEqual(bundle.obj.std_dev,
float(self.data_optional['std_dev']))
self.assertEqual(bundle.obj.val_max,
float(self.data_optional['val_max']))
self.assertEqual(bundle.obj.val_min,
float(self.data_optional['val_min']))
def test_non_exiting_items(self):
"""Check handling of optional data"""
modified_data = copy.deepcopy(self.data1)
modified_data['commitid'] = '0b31bf33a469ac2cb1949666eea54d69a36c3724'
modified_data['project'] = 'Cython'
modified_data['benchmark'] = 'Django Template'
modified_data['executable'] = 'pypy-jit'
bundle = ResultBundle(**modified_data)
bundle.save()
self.assertEqual(bundle.obj.revision.commitid,
modified_data['commitid'])
self.assertEqual(bundle.obj.benchmark.name,
modified_data['benchmark'])
self.assertEqual(bundle.obj.project.name,
modified_data['project'])
class AddJSONResults(TestCase):
def setUp(self):
self.path = reverse("codespeed.views.add_json_results")
self.client = Client()
self.e = Environment(name="bigdog", cpu="Core 2 Duo 8200")
self.e.save()
temp = datetime.today()
self.cdate = datetime(temp.year, temp.month, temp.day, temp.hour, temp.minute, temp.second)
self.data = [
{
"commitid": "123",
"project": "pypy",
"branch": "default",
"executable": "pypy-c",
"benchmark": "Richards",
"environment": "bigdog",
"result_value": 456,
},
{
"commitid": "456",
"project": "pypy",
"branch": "default",
"executable": "pypy-c",
"benchmark": "Richards",
"environment": "bigdog",
"result_value": 457,
},
{
"commitid": "456",
"project": "pypy",
"branch": "default",
"executable": "pypy-c",
"benchmark": "Richards2",
"environment": "bigdog",
"result_value": 34,
},
{
"commitid": "789",
"project": "pypy",
"branch": "default",
"executable": "pypy-c",
"benchmark": "Richards",
"environment": "bigdog",
"result_value": 458,
},
]
def test_add_correct_results(self):
"""Should add all results when the request data is valid"""
response = self.client.post(self.path, {"json": json.dumps(self.data)})
# Check that we get a success response
self.assertEquals(response.status_code, 202)
self.assertEquals(response.content, "All result data saved successfully")
# Check that the data was correctly saved
e = Environment.objects.get(name="bigdog")
b = Benchmark.objects.get(name="Richards")
self.assertEquals(b.benchmark_type, "C")
self.assertEquals(b.units, "seconds")
self.assertEquals(b.lessisbetter, True)
p = Project.objects.get(name="pypy")
branch = Branch.objects.get(name="default", project=p)
r = Revision.objects.get(commitid="123", branch=branch)
i = Executable.objects.get(name="pypy-c")
res = Result.objects.get(revision=r, executable=i, benchmark=b, environment=e)
self.assertTrue(res.value, 456)
resdate = res.date.strftime("%Y%m%dT%H%M%S")
selfdate = self.cdate.strftime("%Y%m%dT%H%M%S")
self.assertTrue(resdate, selfdate)
r = Revision.objects.get(commitid="456", branch=branch)
res = Result.objects.get(revision=r, executable=i, benchmark=b, environment=e)
self.assertTrue(res.value, 457)
r = Revision.objects.get(commitid="789", branch=branch)
res = Result.objects.get(revision=r, executable=i, benchmark=b, environment=e)
self.assertTrue(res.value, 458)
def test_bad_environment(self):
"""Add result associated with non-existing environment.
Only change one item in the list.
"""
data = self.data[0]
bad_name = "bigdog1"
data["environment"] = bad_name
response = self.client.post(self.path, {"json": json.dumps(self.data)})
self.assertEquals(response.status_code, 400)
self.assertEquals(response.content, "Environment " + bad_name + " not found")
data["environment"] = "bigdog"
def test_empty_argument(self):
"""Should return 400 when making a request with an empty argument"""
data = self.data[1]
for key in data:
backup = data[key]
data[key] = ""
response = self.client.post(self.path, {"json": json.dumps(self.data)})
self.assertEquals(response.status_code, 400)
self.assertEquals(response.content, 'Value for key "' + key + '" empty in request')
data[key] = backup
def test_missing_argument(self):
"""Should return 400 when making a request with a missing argument"""
data = self.data[2]
for key in data:
backup = data[key]
del (data[key])
response = self.client.post(self.path, {"json": json.dumps(self.data)})
self.assertEquals(response.status_code, 400)
self.assertEquals(response.content, 'Key "' + key + '" missing from request')
data[key] = backup
def test_report_is_created(self):
"""Should create a report when adding json results for two revisions
plus a third revision with one result less than the last one"""
response = self.client.post(self.path, {"json": json.dumps(self.data)})
# Check that we get a success response
self.assertEquals(response.status_code, 202)
number_of_reports = len(Report.objects.all())
# After adding 4 result for 3 revisions, only 2 reports should be created
# The third revision will need an extra result for Richards2 in order
# to trigger report creation
self.assertEquals(number_of_reports, 1)
