def test_00_update_machine(self):
"""Check PUT request to /machines/n"""
client = self.client
# We are going to set the 'os' field to none, remove the 'uname'
# parameter and add the 'new_parameter' parameter.
# Make sure none of those things happened yet:
machine_before = check_json(client, 'api/db_default/v4/nts/machines/1')
machine_before = machine_before['machine']
self.assertIsNotNone(machine_before.get('os', None))
self.assertIsNone(machine_before.get('new_parameter', None))
self.assertIsNotNone(machine_before.get('uname', None))
data = {
'machine': {
'hardware': 'hal 9000',
'os': None,
'hostname': 'localhost',
'new_parameter': True,
},
}
json_data = json.dumps(data)
resp = client.put('api/db_default/v4/nts/machines/1', data=json_data,
headers={'AuthToken': 'test_token'})
self.assertEqual(resp.status_code, 200)
machine_after = check_json(client, 'api/db_default/v4/nts/machines/1')
machine_after = machine_after['machine']
for key in ('hardware', 'os', 'hostname', 'new_parameter', 'uname'):
self.assertEquals(machine_after.get(key, None),
data['machine'].get(key, None))
def test_04_merge_into(self):
"""Check POST/merge into request for /machines."""
client = self.client
# Download existing machines.
machine_1 = check_json(client, 'api/db_default/v4/nts/machines/1')
machine_3 = check_json(client, 'api/db_default/v4/nts/machines/3')
# The test is boring if we don't have at least 1 run in each machine.
self.assertTrue(len(machine_1['runs']) > 0)
self.assertTrue(len(machine_3['runs']) > 0)
data = {
'action': 'merge',
'into': '3',
}
resp = client.post('api/db_default/v4/nts/machines/1', data=data,
headers={'AuthToken': 'test_token'})
self.assertEqual(resp.status_code, 200)
# Old machine should have disappeared.
resp_2 = client.get('api/db_default/v4/nts/machines/1')
self.assertEqual(resp_2.status_code, 404)
# The other machine should have the union of all runs.
machine_1['runs'] = [_hashabledict(run) for run in machine_1['runs']]
machine_3['runs'] = [_hashabledict(run) for run in machine_3['runs']]
allruns = set(machine_1['runs']).union(machine_3['runs'])
resp_3 = check_json(client, 'api/db_default/v4/nts/machines/3')
resp_3['runs'] = [_hashabledict(run) for run in resp_3['runs']]
self.assertEqual(set(resp_3['runs']), allruns)
def test_roundtrip(self):
"""Check /runs GET, POST roundtrip"""
client = self.client
# Download originl
original = check_json(client, 'api/db_default/v4/nts/runs/2')
# Remove the run
resp = client.delete('api/db_default/v4/nts/runs/2',
headers={'AuthToken': 'test_token'})
self.assertEqual(resp.status_code, 200)
# Post it back
resp = client.post('api/db_default/v4/nts/runs',
data=json.dumps(original),
headers={'AuthToken': 'test_token'})
self.assertEqual(resp.status_code, 301)
new_location = resp.headers['Location']
# Download new data
reimported = check_json(client, new_location)
# The 'id' field may be the different, the rest must be the same.
reimported['run']['id'] = original['run']['id']
self.assertEqual(original, reimported)
def test_00_rename_machine(self):
"""Check rename POST request to /machines/n"""
client = self.client
# Make sure the environment is as expected.
j = check_json(client, 'api/db_default/v4/nts/machines/1')
self.assertEqual(j['machine']['name'], 'localhost__clang_DEV__x86_64')
data = {
'action': 'rename',
'name': 'new_machine_name',
}
resp = client.post('api/db_default/v4/nts/machines/1', data=data)
self.assertEqual(resp.status_code, 401)
resp = client.post('api/db_default/v4/nts/machines/1', data=data,
headers={'AuthToken': 'wrong token'})
self.assertEqual(resp.status_code, 401)
resp = client.post('api/db_default/v4/nts/machines/1', data=data,
headers={'AuthToken': 'test_token'})
self.assertEqual(resp.status_code, 200)
# Machine should be renamed now.
j = check_json(client, 'api/db_default/v4/nts/machines/1')
self.assertEqual(j['machine']['name'], 'new_machine_name')
def test_00_update_machine(self):
"""Check PUT request to /machines/n"""
client = self.client
# We are going to set the 'os' field to none, remove the 'uname'
# parameter and add the 'new_parameter' parameter.
# Make sure none of those things happened yet:
machine_before = check_json(client, 'api/db_default/v4/nts/machines/1')
machine_before = machine_before['machine']
self.assertIsNotNone(machine_before.get('os', None))
self.assertIsNone(machine_before.get('new_parameter', None))
self.assertIsNotNone(machine_before.get('uname', None))
data = {
'machine': {
'hardware': 'hal 9000',
'os': None,
'hostname': 'localhost',
'new_parameter': True,
},
}
json_data = json.dumps(data)
resp = client.put('api/db_default/v4/nts/machines/1', data=json_data,
headers={'AuthToken': 'test_token'})
self.assertEqual(resp.status_code, 200)
machine_after = check_json(client, 'api/db_default/v4/nts/machines/1')
machine_after = machine_after['machine']
for key in ('hardware', 'os', 'hostname', 'new_parameter', 'uname'):
self.assertEquals(machine_after.get(key, None),
data['machine'].get(key, None))
def test_00_rename_machine(self):
"""Check rename POST request to /machines/n"""
client = self.client
# Make sure the environment is as expected.
j = check_json(client, 'api/db_default/v4/nts/machines/1')
self.assertEqual(j['machine']['name'], 'localhost__clang_DEV__x86_64')
data = {
'action': 'rename',
'name': 'new_machine_name',
}
resp = client.post('api/db_default/v4/nts/machines/1', data=data)
self.assertEqual(resp.status_code, 401)
resp = client.post('api/db_default/v4/nts/machines/1', data=data,
headers={'AuthToken': 'wrong token'})
self.assertEqual(resp.status_code, 401)
resp = client.post('api/db_default/v4/nts/machines/1', data=data,
headers={'AuthToken': 'test_token'})
self.assertEqual(resp.status_code, 200)
# Machine should be renamed now.
j = check_json(client, 'api/db_default/v4/nts/machines/1')
self.assertEqual(j['machine']['name'], 'new_machine_name')
def test_04_merge_into(self):
"""Check POST/merge into request for /machines."""
client = self.client
# Download existing machines.
machine_1 = check_json(client, 'api/db_default/v4/nts/machines/1')
machine_3 = check_json(client, 'api/db_default/v4/nts/machines/3')
# The test is boring if we don't have at least 1 run in each machine.
self.assertTrue(len(machine_1['runs']) > 0)
self.assertTrue(len(machine_3['runs']) > 0)
data = {
'action': 'merge',
'into': '3',
}
resp = client.post('api/db_default/v4/nts/machines/1', data=data,
headers={'AuthToken': 'test_token'})
self.assertEqual(resp.status_code, 200)
# Old machine should have disappeared.
resp_2 = client.get('api/db_default/v4/nts/machines/1')
self.assertEqual(resp_2.status_code, 404)
# The other machine should have the union of all runs.
machine_1['runs'] = [_hashabledict(run) for run in machine_1['runs']]
machine_3['runs'] = [_hashabledict(run) for run in machine_3['runs']]
allruns = set(machine_1['runs']).union(machine_3['runs'])
resp_3 = check_json(client, 'api/db_default/v4/nts/machines/3')
resp_3['runs'] = [_hashabledict(run) for run in resp_3['runs']]
self.assertEqual(set(resp_3['runs']), allruns)
def test_run_api(self):
"""Check /run/n returns expected run information."""
client = self.client
j = check_json(client, 'api/db_default/v4/nts/run/1')
self.assertEquals(j, run_expected_response[0])
for i in xrange(0, len(run_expected_response)):
j = check_json(client, 'api/db_default/v4/nts/run/' + str(i + 1))
self.assertEquals(j, run_expected_response[i])
def test_order_api(self):
""" Check /orders/n returns the expected order information."""
client = self.client
j = check_json(client, 'api/db_default/v4/nts/orders/1')
self.assertEquals(j['orders'][0], order_expected_response)
self._check_response_is_well_formed(j)
check_json(client,
'api/db_default/v4/nts/orders/100',
expected_code=404)
def test_single_sample_api(self):
""" Check /samples/n returns the expected sample information."""
client = self.client
j = check_json(client, 'api/db_default/v4/nts/samples/1')
self._check_response_is_well_formed(j)
self.assertEquals(sample_expected_response, j['samples'][0])
check_json(client,
'api/db_default/v4/nts/samples/1000',
expected_code=404)
def test_run_api(self):
"""Check /run/n returns expected run information."""
client = self.client
j = check_json(client, 'api/db_default/v4/nts/run/1')
self.assertEquals(j, run_expected_response[0])
for i in xrange(0, len(run_expected_response)):
j = check_json(client, 'api/db_default/v4/nts/run/' + str(i + 1))
self.assertEquals(j, run_expected_response[i])
def test_graph_api(self):
"""Check that /graph/x/y/z returns what we expect."""
client = self.client
j = check_json(client, 'api/db_default/v4/nts/graph/2/4/3')
self.assertEqual(graph_data, j)
# Now check that limit works.
j2 = check_json(client, 'api/db_default/v4/nts/graph/2/4/3?limit=1')
self.assertEqual(graph_data2, j2)
def test_machine_api(self):
"""Check /machines and /machine/n return expected results from
testdb.
"""
client = self.client
j = check_json(client, 'api/db_default/v4/nts/machines')
self.assertEquals(j, machines_expected_response)
for i in xrange(0, len(machine_expected_response)):
j = check_json(client, 'api/db_default/v4/nts/machine/' +
str(i + 1))
self.assertEquals(j, machine_expected_response[i])
def test_machine_api(self):
"""Check /machines and /machine/n return expected results from
testdb.
"""
client = self.client
j = check_json(client, 'api/db_default/v4/nts/machines')
self.assertEquals(j, machines_expected_response)
for i in xrange(0, len(machine_expected_response)):
j = check_json(client,
'api/db_default/v4/nts/machine/' + str(i + 1))
self.assertEquals(j, machine_expected_response[i])
def test_graph_api(self):
"""Check that /graph/x/y/z returns what we expect."""
client = self.client
j = check_json(client, 'api/db_default/v4/nts/graph/2/4/2')
# TODO: Graph API needs redesign to be well formed.
# self._check_response_is_well_formed(j)
self.assertEqual(graph_data, j)
# Now check that limit works.
j2 = check_json(client, 'api/db_default/v4/nts/graph/2/4/2?limit=1')
# self._check_response_is_well_formed(j)
self.assertEqual(graph_data2, j2)
def test_run_api(self):
"""Check /runs/n returns expected run information."""
client = self.client
j = check_json(client, 'api/db_default/v4/nts/runs/1')
self._check_response_is_well_formed(j)
expected = {
"end_time": "2012-04-11T16:28:58",
"start_time": "2012-04-11T16:28:23",
"id": 1,
"llvm_project_revision": u'154331'
}
self.assertDictContainsSubset(expected, j['run'])
self.assertEqual(len(j['tests']), 2)
# This should not be a run.
check_json(client, 'api/db_default/v4/nts/runs/100', expected_code=404)
def test_run_round_trip(self):
"""Output of /runs/n can be fed back to /submitRun
"""
client = self.client
orig_api_run = check_json(client, 'api/db_default/v4/nts/runs/1')
# Do some slight modification to avoid LNT rejecting the new submission
# as a duplicate.
orig_api_run['run']['llvm_project_revision'] = u'154333'
self.assertEqual(orig_api_run['run']['id'], 1)
# Machine and run id should be ignored on submission so we should be
# able to set them to anything without anyone complaining.
self.assertEqual(orig_api_run['machine']['id'], 1)
modified_run = copy.deepcopy(orig_api_run)
modified_run['run']['llvm_project_revision'] = u'666'
new_api_run, result_url = self._resubmit(modified_run)
self.assertEqual(result_url, 'http://localhost/db_default/v4/nts/10')
self.assertEqual(new_api_run['run']['id'], 10)
self.assertEqual(new_api_run['run']['llvm_project_revision'], u'666')
new_api_run['run']['llvm_project_revision'] = orig_api_run['run']['llvm_project_revision']
# We change run id and machine id back to the original and after that
# we should have a perfect match.
self._compare_results(new_api_run, orig_api_run)
def test_01_delete_run(self):
"""Check /runs/n can be deleted."""
client = self.client
j = check_json(client, 'api/db_default/v4/nts/runs/1')
sample_ids = [s['id'] for s in j['tests']]
self.assertNotEqual(len(sample_ids), 0)
for sid in sample_ids:
resp = client.get('api/db_default/v4/nts/samples/{}'.format(sid))
self.assertEqual(resp.status_code, 200)
resp = client.delete('api/db_default/v4/nts/runs/1')
self.assertEqual(resp.status_code, 401)
resp = client.delete('api/db_default/v4/nts/runs/1',
headers={'AuthToken': 'wrong token'})
self.assertEqual(resp.status_code, 401)
resp = client.delete('api/db_default/v4/nts/runs/1',
headers={'AuthToken': 'test_token'})
self.assertEqual(resp.status_code, 200)
resp = client.get('api/db_default/v4/nts/runs/1')
self.assertEqual(resp.status_code, 404)
for sid in sample_ids:
resp = client.get('api/db_default/v4/nts/samples/{}'.format(sid))
self.assertEqual(resp.status_code, 404)
def test_run_round_trip(self):
"""Output of /runs/n can be fed back to /submitRun
"""
client = self.client
orig_api_run = check_json(client, 'api/db_default/v4/nts/runs/1')
# Do some slight modification to avoid LNT rejecting the new submission
# as a duplicate.
orig_api_run['run']['llvm_project_revision'] = u'154333'
self.assertEqual(orig_api_run['run']['id'], 1)
# Machine and run id should be ignored on submission so we should be
# able to set them to anything without anyone complaining.
self.assertEqual(orig_api_run['machine']['id'], 1)
modified_run = copy.deepcopy(orig_api_run)
modified_run['run']['llvm_project_revision'] = u'666'
new_api_run, result_url = self._resubmit(modified_run)
self.assertEqual(result_url, 'http://localhost/db_default/v4/nts/10')
self.assertEqual(new_api_run['run']['id'], 10)
self.assertEqual(new_api_run['run']['llvm_project_revision'], u'666')
new_api_run['run']['llvm_project_revision'] = orig_api_run['run']['llvm_project_revision']
# We change run id and machine id back to the original and after that
# we should have a perfect match.
self._compare_results(new_api_run, orig_api_run)
def test_01_delete_run(self):
"""Check /runs/n can be deleted."""
client = self.client
j = check_json(client, 'api/db_default/v4/nts/runs/1')
sample_ids = [s['id'] for s in j['tests']]
self.assertNotEqual(len(sample_ids), 0)
for sid in sample_ids:
resp = client.get('api/db_default/v4/nts/samples/{}'.format(sid))
self.assertEqual(resp.status_code, 200)
resp = client.delete('api/db_default/v4/nts/runs/1')
self.assertEqual(resp.status_code, 401)
resp = client.delete('api/db_default/v4/nts/runs/1',
headers={'AuthToken': 'wrong token'})
self.assertEqual(resp.status_code, 401)
resp = client.delete('api/db_default/v4/nts/runs/1',
headers={'AuthToken': 'test_token'})
self.assertEqual(resp.status_code, 200)
resp = client.get('api/db_default/v4/nts/runs/1')
self.assertEqual(resp.status_code, 404)
for sid in sample_ids:
resp = client.get('api/db_default/v4/nts/samples/{}'.format(sid))
self.assertEqual(resp.status_code, 404)
def test_02_delete_machine(self):
"""Check /machines/n can be deleted."""
client = self.client
j = check_json(client, 'api/db_default/v4/nts/machines/2')
run_ids = [s['id'] for s in j['runs']]
self.assertNotEqual(len(run_ids), 0)
sample_ids = []
for run_id in run_ids:
resp = check_json(client,
'api/db_default/v4/nts/runs/{}'.format(run_id))
sample_ids.append([s['id'] for s in resp['tests']])
self.assertNotEqual(len(sample_ids), 0)
resp = client.delete('api/db_default/v4/nts/machines/2')
self.assertEqual(resp.status_code, 401)
resp = client.delete('api/db_default/v4/nts/machines/2',
headers={'AuthToken': 'wrong token'})
self.assertEqual(resp.status_code, 401)
resp = client.delete('api/db_default/v4/nts/machines/2',
headers={'AuthToken': 'test_token'})
self.assertEqual(resp.status_code, 200)
self.assertEqual(resp.get_data(),
'''Deleting runs 3 5 6 7 8 9 (6/6)
Deleted machine machine2:2
''')
resp = client.get('api/db_default/v4/nts/machines/2')
self.assertEqual(resp.status_code, 404)
for run_id in run_ids:
resp = client.get('api/db_default/v4/nts/runs/{}'.format(run_id))
self.assertEqual(resp.status_code, 404)
for sid in sample_ids:
resp = client.get('api/db_default/v4/nts/samples/{}'.format(sid))
self.assertEqual(resp.status_code, 404)
def test_02_delete_machine(self):
"""Check /machines/n can be deleted."""
client = self.client
j = check_json(client, 'api/db_default/v4/nts/machines/2')
run_ids = [s['id'] for s in j['runs']]
self.assertNotEqual(len(run_ids), 0)
sample_ids = []
for run_id in run_ids:
resp = check_json(client,
'api/db_default/v4/nts/runs/{}'.format(run_id))
sample_ids.append([s['id'] for s in resp['tests']])
self.assertNotEqual(len(sample_ids), 0)
resp = client.delete('api/db_default/v4/nts/machines/2')
self.assertEqual(resp.status_code, 401)
resp = client.delete('api/db_default/v4/nts/machines/2',
headers={'AuthToken': 'wrong token'})
self.assertEqual(resp.status_code, 401)
resp = client.delete('api/db_default/v4/nts/machines/2',
headers={'AuthToken': 'test_token'})
self.assertEqual(resp.status_code, 200)
self.assertEqual(resp.get_data(),
'''Deleting runs 3 5 6 7 8 9 (6/6)
Deleted machine machine2:2
''')
resp = client.get('api/db_default/v4/nts/machines/2')
self.assertEqual(resp.status_code, 404)
for run_id in run_ids:
resp = client.get('api/db_default/v4/nts/runs/{}'.format(run_id))
self.assertEqual(resp.status_code, 404)
for sid in sample_ids:
resp = client.get('api/db_default/v4/nts/samples/{}'.format(sid))
self.assertEqual(resp.status_code, 404)
def test_fields_api(self):
"""Fields API."""
client = self.client
j = check_json(client, 'api/db_default/v4/nts/fields')
fields = j['fields']
# check number of fields
self.assertEqual(9, len(fields))
# check first field
f0 = fields[0]
self.assertEqual(0, f0['column_id'])
self.assertEqual('compile_time', f0['column_name'])
def test_samples_api(self):
"""Samples API."""
client = self.client
# Run IDs must be passed, so 400 if they are not.
check_json(client, 'api/db_default/v4/nts/samples', expected_code=400)
# Simple single run.
j = check_json(client, 'api/db_default/v4/nts/samples?runid=1')
self._check_response_is_well_formed(j)
expected = [{
u'compile_time': 0.007,
u'llvm_project_revision': u'154331',
u'name': u'SingleSource/UnitTests/2006-12-01-float_varg',
u'run_id': 1,
u'execution_time': 0.0003,
u'id': 1
}, {
u'compile_time': 0.0072,
u'llvm_project_revision': u'154331',
u'name': u'SingleSource/UnitTests/2006-12-04-DynAllocAndRestore',
u'run_id': 1,
u'execution_time': 0.0003,
u'id': 2
}]
self.assertEqual(j['samples'], expected)
# Check that other args are ignored.
extra_param = check_json(
client, 'api/db_default/v4/nts/samples?runid=1&foo=bar')
self._check_response_is_well_formed(extra_param)
self.assertEqual(j, extra_param)
# There is only one run in the DB.
two_runs = check_json(client,
'api/db_default/v4/nts/samples?runid=1&runid=2')
self._check_response_is_well_formed(two_runs)
self.assertEqual(j, two_runs)
def test_tests_api(self):
"""Tests API."""
client = self.client
j = check_json(client, 'api/db_default/v4/nts/tests')
tests = j['tests']
# check number of tests
self.assertEqual(9, len(tests))
# check first test
t0 = tests[0]
self.assertEqual(1, t0['id'])
self.assertEqual('SingleSource/UnitTests/2006-12-01-float_varg',
t0['name'])
def _resubmit(self, run_results):
"""Send the results to the server.
Convert the results to json, post them to the server's submitRun
"""
# Submit the data
data_to_send = {'commit': '1', 'input_data': json.dumps(run_results)}
response = check_code_post(self.client,
'db_default/v4/nts/submitRun',
data_to_send=data_to_send)
submit_result = json.loads(response.data)
result_url = submit_result.get('result_url')
run_id = result_url.split("/")[-1]
new_api_run = check_json(
self.client, 'api/db_default/v4/nts/runs/{}'.format(run_id))
return new_api_run, result_url
def _resubmit(self, run_results):
"""Send the results to the server.
Convert the results to json, post them to the server's submitRun
"""
# Submit the data
data_to_send = {
'commit': '1',
'input_data': json.dumps(run_results)
}
response = check_code_post(self.client, 'db_default/v4/nts/submitRun',
data_to_send=data_to_send)
submit_result = json.loads(response.data)
result_url = submit_result.get('result_url')
run_id = result_url.split("/")[-1]
new_api_run = check_json(self.client, 'api/db_default/v4/nts/runs/{}'.format(run_id))
return new_api_run, result_url
def test_machine_api(self):
"""Check /machines/ and /machines/n return expected results from testdb.
"""
client = self.client
# All machines returns the list of machines with parameters, but no runs.
j = check_json(client, 'api/db_default/v4/nts/machines/')
self._check_response_is_well_formed(j)
self.assertEquals(j['machines'], machines_expected_response)
self.assertIsNone(j.get('runs'))
j = check_json(client, 'api/db_default/v4/nts/machines')
self._check_response_is_well_formed(j)
self.assertEquals(j['machines'], machines_expected_response)
self.assertIsNone(j.get('runs'))
# Machine + properties + run information.
j = check_json(client, 'api/db_default/v4/nts/machines/1')
self._check_response_is_well_formed(j)
self.assertEqual(j['machine'], machines_expected_response[0])
self.assertEqual(len(j['runs']), 2)
for run in j['runs']:
self.assertSetEqual(set(run.keys()), possible_run_keys)
# Specify machine by name
j = check_json(
client,
'api/db_default/v4/nts/machines/localhost__clang_DEV__x86_64')
self._check_response_is_well_formed(j)
self.assertEqual(j['machine'], machines_expected_response[0])
# Invalid machine ids are 404.
check_json(client,
'api/db_default/v4/nts/machines/99',
expected_code=404)
check_json(client,
'api/db_default/v4/nts/machines/foo',
expected_code=404)
def test_schema(self):
client = self.client
rest_schema = check_json(client, 'api/db_default/v4/nts/schema')
# The reported schema should be the same as the yaml one on the top.
with open('%s/schemas/nts.yaml' % self.instance_path) as syaml:
yaml_schema = yaml.load(syaml)
# Do some massaging to make it similar to the rest API result.
for m in yaml_schema['metrics']:
if 'unit' not in m:
m['unit'] = None
if 'unit_abbrev' not in m:
m['unit_abbrev'] = None
if 'display_name' not in m:
m['display_name'] = m['name']
if 'bigger_is_better' not in m:
m['bigger_is_better'] = False
yaml_schema['metrics'].sort(key=lambda x: x['name'])
yaml_schema['run_fields'].sort(key=lambda x: x['name'])
yaml_schema['machine_fields'].sort(key=lambda x: x['name'])
self.assertEqual(rest_schema, yaml_schema)
def test_order_api(self):
""" Check /order/n returns the expected order information."""
client = self.client
j = check_json(client, 'api/db_default/v4/nts/order/1')
self.assertEquals(j, order_expected_response)
def test_order_api(self):
""" Check /order/n returns the expected order information."""
client = self.client
j = check_json(client, 'api/db_default/v4/nts/order/1')
self.assertEquals(j, order_expected_response)
def test_graph_api(self):
"""Check that /graph/x/y/z returns what we expect."""
client = self.client
j = check_json(client, 'api/db_default/v4/nts/graph/2/4/3')
self.assertEqual(graph_data, j)