def post_collection(project, th_collection, status=None, expect_errors=False, consumer_key=None, consumer_secret=None): # Set the credentials OAuthCredentials.set_credentials(SampleData.get_credentials()) credentials = OAuthCredentials.get_credentials(project) # The only time the credentials should be overridden are when # a client needs to test authentication failure confirmation if consumer_key: credentials["consumer_key"] = consumer_key if consumer_secret: credentials["consumer_secret"] = consumer_secret tr = TreeherderRequest( protocol="http", host="localhost", project=project, oauth_key=credentials["consumer_key"], oauth_secret=credentials["consumer_secret"], ) signed_uri = tr.oauth_client.get_signed_uri( th_collection.to_json(), tr.get_uri(th_collection.endpoint_base), "POST" ) response = TestApp(application).post_json( str(signed_uri), params=th_collection.get_collection_data(), status=status ) return response
def _post_json_data(url, data): th_collection = data[jm.project] OAuthCredentials.set_credentials( SampleData.get_credentials() ) credentials = OAuthCredentials.get_credentials(jm.project) tr = TreeherderRequest( protocol='http', host='localhost', project=jm.project, oauth_key=credentials['consumer_key'], oauth_secret=credentials['consumer_secret'] ) signed_uri = tr.oauth_client.get_signed_uri( th_collection.to_json(), tr.get_uri(th_collection.endpoint_base), "POST" ) response = TestApp(application).post_json( str(signed_uri), params=th_collection.get_collection_data() ) response.getcode = lambda: response.status_int return response
def post_job_data( project, uri, data, status=None, expect_errors=False): # Since the uri is passed in it's not generated by the # treeherder request or collection and is missing the protocol # and host. Add those missing elements here. uri = 'http://localhost{0}'.format(uri) # Set the credentials OAuthCredentials.set_credentials(SampleData.get_credentials()) credentials = OAuthCredentials.get_credentials(project) tr = TreeherderRequest( protocol='http', host='localhost', project=project, oauth_key=credentials['consumer_key'], oauth_secret=credentials['consumer_secret'] ) signed_uri = tr.get_signed_uri( json.dumps(data), uri ) response = TestApp(application).post_json( str(signed_uri), params=data, status=status, expect_errors=expect_errors ) return response
def check_json(self, filename, expected_timestamps): """Parse JSON produced by http://graphs.mozilla.org/api/test/runs""" # Configuration for TalosAnalyzer FORE_WINDOW = 12 BACK_WINDOW = 12 THRESHOLD = 7 MACHINE_THRESHOLD = 15 MACHINE_HISTORY_SIZE = 5 payload = SampleData.get_perf_data(os.path.join('graphs', filename)) runs = payload['test_runs'] data = [ PerfDatum(r[2], r[3], testrun_id=r[0], machine_id=r[6], testrun_timestamp=r[2], buildid=r[1][1], revision=r[1][2]) for r in runs ] a = TalosAnalyzer() a.addData(data) results = a.analyze_t(BACK_WINDOW, FORE_WINDOW, THRESHOLD, MACHINE_THRESHOLD, MACHINE_HISTORY_SIZE) regression_timestamps = [ d.testrun_timestamp for d in results if d.state == 'regression' ] self.assertEqual(regression_timestamps, expected_timestamps)
def post_collection( project, th_collection, status=None, expect_errors=False, consumer_key=None, consumer_secret=None): # Set the credentials OAuthCredentials.set_credentials(SampleData.get_credentials()) credentials = OAuthCredentials.get_credentials(project) # The only time the credentials should be overridden are when # a client needs to test authentication failure confirmation consumer_key = consumer_key or credentials['consumer_key'] consumer_secret = consumer_secret or credentials['consumer_secret'] auth = TreeherderAuth(consumer_key, consumer_secret, project) client = TreeherderClient(protocol='http', host='localhost', auth=auth) uri = client._get_project_uri(project, th_collection.endpoint_base) req = Request('POST', uri, json=th_collection.get_collection_data(), auth=auth) prepped_request = req.prepare() response = TestApp(application).post_json( prepped_request.url, params=th_collection.get_collection_data(), status=status ) return response
def post_collection( project, th_collection, status=None, expect_errors=False, consumer_key=None, consumer_secret=None): # Set the credentials OAuthCredentials.set_credentials(SampleData.get_credentials()) credentials = OAuthCredentials.get_credentials(project) # The only time the credentials should be overridden are when # a client needs to test authentication failure confirmation if consumer_key: credentials['consumer_key'] = consumer_key if consumer_secret: credentials['consumer_secret'] = consumer_secret cli = TreeherderClient( protocol='http', host='localhost', ) jsondata = th_collection.to_json() signed_uri = cli._get_uri(project, th_collection.endpoint_base, data=jsondata, oauth_key=credentials['consumer_key'], oauth_secret=credentials['consumer_secret'], method='POST') response = TestApp(application).post_json( str(signed_uri), params=th_collection.get_collection_data(), status=status ) return response
def post_job_data(project, uri, data, status=None, expect_errors=False): # Since the uri is passed in it's not generated by the # treeherder request or collection and is missing the protocol # and host. Add those missing elements here. uri = 'http://localhost{0}'.format(uri) # Set the credentials OAuthCredentials.set_credentials(SampleData.get_credentials()) credentials = OAuthCredentials.get_credentials(project) tr = TreeherderRequest(protocol='http', host='localhost', project=project, oauth_key=credentials['consumer_key'], oauth_secret=credentials['consumer_secret']) signed_uri = tr.get_signed_uri(json.dumps(data), uri) response = TestApp(application).post_json(str(signed_uri), params=data, status=status, expect_errors=expect_errors) return response
def load_exp(filename): """ Load in an expected result json and return as an obj. """ path = SampleData().get_log_path(filename) with open(path) as f: return json.load(f)
def test_post_talos_artifact(test_project, test_repository, result_set_stored, mock_post_json): test_repository.save() tjc = client.TreeherderJobCollection() job_guid = 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33' tj = client.TreeherderJob({ 'project': test_repository.name, 'revision_hash': result_set_stored[0]['revision_hash'], 'job': { 'job_guid': job_guid, 'state': 'completed', 'project': test_repository.name, 'option_collection': {'opt': True}, 'artifacts': [{ 'blob': {'talos_data': SampleData.get_minimal_talos_perf_data()}, 'type': 'json', 'name': 'talos_data', 'job_guid': job_guid }] } }) tjc.add(tj) post_collection(test_project, tjc) # we'll just validate that we got the expected number of results for # talos (we have validation elsewhere for the actual data adapters) assert PerformanceSignature.objects.count() == 1 assert PerformanceDatum.objects.count() == 1
def add_log_response(filename): """ Set up responses for a local gzipped log and return the url for it. """ log_path = SampleData().get_log_path(filename) log_url = "http://my-log.mozilla.org/{}".format(filename) with open(log_path, 'rb') as log_file: responses.add(responses.GET, log_url, body=log_file.read(), adding_headers={ "Content-Encoding": "gzip", }) return log_url
def load_exp(filename): """ Load in an expected result json and return as an obj. If the file doesn't exist, it will be created, but the test will fail, due to no content. This is to make it easier during test development. """ path = SampleData().get_log_path(filename) with open(path, "a+") as f: try: return json.loads(f.read()) except ValueError: # if it's not parse-able, return an empty dict return {}
def check_json(self, filename, expected_timestamps): """Parse JSON produced by http://graphs.mozilla.org/api/test/runs""" # Configuration for Analyzer FORE_WINDOW = 12 BACK_WINDOW = 12 THRESHOLD = 7 payload = SampleData.get_perf_data(os.path.join('graphs', filename)) runs = payload['test_runs'] a = Analyzer() for r in runs: a.add_data(r[2], r[3], testrun_id=r[0], revision_id=r[1][2]) results = a.analyze_t(BACK_WINDOW, FORE_WINDOW, THRESHOLD) regression_timestamps = [d.push_timestamp for d in results if d.state == 'regression'] self.assertEqual(regression_timestamps, expected_timestamps)
def _send(th_request, th_collection): OAuthCredentials.set_credentials(SampleData.get_credentials()) credentials = OAuthCredentials.get_credentials(jm.project) th_request.oauth_key = credentials['consumer_key'] th_request.oauth_secret = credentials['consumer_secret'] signed_uri = th_request.get_signed_uri( th_collection.to_json(), th_request.get_uri(th_collection) ) response = TestApp(application).post_json( str(signed_uri), params=th_collection.get_collection_data() ) response.getcode = lambda: response.status_int return response
def test_adapt_and_load(): talos_perf_data = SampleData.get_talos_perf_data() tda = TalosDataAdapter() result_count = 0 for datum in talos_perf_data: datum = { "job_guid": 'oqiwy0q847365qiu', "name": "test", "type": "test", "blob": datum } job_data = { "oqiwy0q847365qiu": { "id": 1, "result_set_id": 1, "push_timestamp": 1402692388 } } reference_data = { "property1": "value1", "property2": "value2", "property3": "value3" } # one extra result for the summary series result_count += len(datum['blob']["results"]) + 1 # we create one performance series per counter if 'talos_counters' in datum['blob']: result_count += len(datum['blob']["talos_counters"]) # Mimic production environment, the blobs are serialized # when the web service receives them datum['blob'] = json.dumps({'talos_data': [datum['blob']]}) tda.adapt_and_load(reference_data, job_data, datum) assert result_count == len(tda.performance_artifact_placeholders)
def add_log_response(filename): """ Set up responses for a local gzipped log and return the url for it. """ log_path = SampleData().get_log_path(filename) log_url = "http://my-log.mozilla.org/{}".format(filename) with open(log_path, 'rb') as log_file: content = log_file.read() responses.add( responses.GET, log_url, body=content, adding_headers={ 'Content-Encoding': 'gzip', 'Content-Length': str(len(content)), }, ) return log_url
def test_detect_changes_historical_data(filename, expected_timestamps): """Parse JSON produced by http://graphs.mozilla.org/api/test/runs""" # Configuration for Analyzer FORE_WINDOW = 12 MIN_BACK_WINDOW = 12 MAX_BACK_WINDOW = 24 THRESHOLD = 7 payload = SampleData.get_perf_data(os.path.join('graphs', filename)) runs = payload['test_runs'] data = [RevisionDatum(r[2], r[2], [r[3]]) for r in runs] results = detect_changes(data, min_back_window=MIN_BACK_WINDOW, max_back_window=MAX_BACK_WINDOW, fore_window=FORE_WINDOW, t_threshold=THRESHOLD) regression_timestamps = [d.push_timestamp for d in results if d.change_detected] assert regression_timestamps == expected_timestamps
def _send(th_request, endpoint, method=None, data=None): OAuthCredentials.set_credentials(SampleData.get_credentials()) credentials = OAuthCredentials.get_credentials(jm.project) th_request.oauth_key = credentials['consumer_key'] th_request.oauth_secret = credentials['consumer_secret'] if data and not isinstance(data, str): data = json.dumps(data) signed_uri = th_request.oauth_client.get_signed_uri( data, th_request.get_uri(endpoint), method) response = getattr(TestApp(application), method.lower())(str(signed_uri), params=data, content_type='application/json') response.getcode = lambda: response.status_int return response
def test_post_talos_artifact(test_project, test_repository, result_set_stored, mock_post_json): test_repository.save() tjc = client.TreeherderJobCollection() job_guid = 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33' tj = client.TreeherderJob({ 'project': test_repository.name, 'revision_hash': result_set_stored[0]['revision_hash'], 'job': { 'job_guid': job_guid, 'state': 'completed', 'project': test_repository.name, 'option_collection': { 'opt': True }, 'artifacts': [{ 'blob': { 'talos_data': SampleData.get_minimal_talos_perf_data() }, 'type': 'json', 'name': 'talos_data', 'job_guid': job_guid }] } }) tjc.add(tj) post_collection(test_project, tjc) # we'll just validate that we got the expected number of results for # talos (we have validation elsewhere for the actual data adapters) assert PerformanceSignature.objects.count() == 1 assert PerformanceDatum.objects.count() == 1
def post_collection(project, th_collection, status=None, expect_errors=False, consumer_key=None, consumer_secret=None): # Set the credentials OAuthCredentials.set_credentials(SampleData.get_credentials()) credentials = OAuthCredentials.get_credentials(project) # The only time the credentials should be overridden are when # a client needs to test authentication failure confirmation if consumer_key: credentials['consumer_key'] = consumer_key if consumer_secret: credentials['consumer_secret'] = consumer_secret cli = TreeherderClient( protocol='http', host='localhost', ) jsondata = th_collection.to_json() signed_uri = cli._get_project_uri( project, th_collection.endpoint_base, data=jsondata, oauth_key=credentials['consumer_key'], oauth_secret=credentials['consumer_secret'], method='POST') response = TestApp(application).post_json( str(signed_uri), params=th_collection.get_collection_data(), status=status) return response
def check_json(self, filename, expected_timestamps): """Parse JSON produced by http://graphs.mozilla.org/api/test/runs""" # Configuration for TalosAnalyzer FORE_WINDOW = 12 BACK_WINDOW = 12 THRESHOLD = 7 MACHINE_THRESHOLD = 15 MACHINE_HISTORY_SIZE = 5 payload = SampleData.get_perf_data(os.path.join('graphs', filename)) runs = payload['test_runs'] data = [PerfDatum(r[2], r[3], testrun_id=r[0], machine_id=r[6], testrun_timestamp=r[2], buildid=r[1][1], revision=r[1][2]) for r in runs] a = TalosAnalyzer() a.addData(data) results = a.analyze_t(BACK_WINDOW, FORE_WINDOW, THRESHOLD, MACHINE_THRESHOLD, MACHINE_HISTORY_SIZE) regression_timestamps = [d.testrun_timestamp for d in results if d.state == 'regression'] self.assertEqual(regression_timestamps, expected_timestamps)
def check_json(self, filename, expected_timestamps): """Parse JSON produced by http://graphs.mozilla.org/api/test/runs""" # Configuration for Analyzer FORE_WINDOW = 12 MIN_BACK_WINDOW = 12 MAX_BACK_WINDOW = 24 THRESHOLD = 7 payload = SampleData.get_perf_data(os.path.join('graphs', filename)) runs = payload['test_runs'] data = [] for r in runs: data.append(Datum(r[2], r[3], testrun_id=r[0], revision_id=r[1][2])) results = detect_changes(data, min_back_window=MIN_BACK_WINDOW, max_back_window=MAX_BACK_WINDOW, fore_window=FORE_WINDOW, t_threshold=THRESHOLD) regression_timestamps = [d.push_timestamp for d in results if d.state == 'regression'] self.assertEqual(regression_timestamps, expected_timestamps)
def _send(th_request, endpoint, method=None, data=None): OAuthCredentials.set_credentials(SampleData.get_credentials()) credentials = OAuthCredentials.get_credentials(jm.project) th_request.oauth_key = credentials['consumer_key'] th_request.oauth_secret = credentials['consumer_secret'] if data and not isinstance(data, str): data = json.dumps(data) signed_uri = th_request.oauth_client.get_signed_uri( data, th_request.get_uri(endpoint), method ) response = getattr(TestApp(application), method.lower())( str(signed_uri), params=data, content_type='application/json' ) response.getcode = lambda: response.status_int return response
def test_post_talos_artifact(test_project, test_repository, result_set_stored, mock_post_json): test_repository.save() # delete any previously-created perf objects until bug 1133273 is fixed # https://bugzilla.mozilla.org/show_bug.cgi?id=1133273 (this can be really # slow if the local database has a lot of objects in it) PerformanceSignature.objects.all().delete() PerformanceDatum.objects.all().delete() tjc = client.TreeherderJobCollection() job_guid = 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33' tj = client.TreeherderJob({ 'project': test_repository.name, 'revision_hash': result_set_stored[0]['revision_hash'], 'job': { 'job_guid': job_guid, 'state': 'completed', 'project': test_repository.name, 'option_collection': {'opt': True}, 'artifacts': [{ 'blob': {'talos_data': SampleData.get_minimal_talos_perf_data()}, 'type': 'json', 'name': 'talos_data', 'job_guid': job_guid }] } }) tjc.add(tj) do_post_collection(test_project, tjc) # we'll just validate that we got the expected number of results for # talos (we have validation elsewhere for the actual data adapters) assert PerformanceSignature.objects.count() == 2 assert PerformanceDatum.objects.count() == 2
def check_json(self, filename, expected_timestamps): """Parse JSON produced by http://graphs.mozilla.org/api/test/runs""" # Configuration for Analyzer FORE_WINDOW = 12 MIN_BACK_WINDOW = 12 MAX_BACK_WINDOW = 24 THRESHOLD = 7 payload = SampleData.get_perf_data(os.path.join('graphs', filename)) runs = payload['test_runs'] data = [] for r in runs: data.append(Datum(r[2], r[3], testrun_id=r[0], revision_id=r[1][2])) results = detect_changes(data, min_back_window=MIN_BACK_WINDOW, max_back_window=MAX_BACK_WINDOW, fore_window=FORE_WINDOW, t_threshold=THRESHOLD) regression_timestamps = [ d.push_timestamp for d in results if d.state == 'regression' ] self.assertEqual(regression_timestamps, expected_timestamps)
def test_load_talos_data(test_project, test_repository, perf_option_collection, perf_platform, perf_job_data, perf_reference_data): PerformanceFramework.objects.create(name='talos') talos_perf_data = SampleData.get_talos_perf_data() for talos_datum in talos_perf_data: datum = { "job_guid": "fake_job_guid", "name": "test", "type": "test", "blob": talos_datum } # Mimic production environment, the blobs are serialized # when the web service receives them datum['blob'] = json.dumps({'talos_data': [datum['blob']]}) load_talos_artifacts(test_repository.name, perf_reference_data, perf_job_data, datum) # base: subtests + one extra result for the summary series expected_result_count = len(talos_datum["results"]) + 1 # we create one performance series per counter if 'talos_counters' in talos_datum: expected_result_count += len(talos_datum["talos_counters"]) # result count == number of signatures assert expected_result_count == PerformanceSignature.objects.all().count() expected_push_timestamp = datetime.datetime.fromtimestamp( perf_job_data['fake_job_guid']['push_timestamp']) # verify that we have signatures for the subtests for (testname, results) in talos_datum["results"].iteritems(): signature = PerformanceSignature.objects.get(test=testname) datum = PerformanceDatum.objects.get(signature=signature) if talos_datum.get('summary'): # if we have a summary, ensure the subtest summary values made # it in and that we ingested lowerIsBetter ok (if it was there) subtest = talos_datum['summary']['subtests'][testname] assert round(subtest['filtered'], 2) == datum.value assert signature.lower_is_better == subtest.get('lowerIsBetter', True) else: # this is an old style talos blob without a summary. these are # going away, so I'm not going to bother testing the # correctness. however let's at least verify that some values # are being generated here assert datum.value assert datum.push_timestamp == expected_push_timestamp # if we have counters, verify that the series for them is as expected for (counter, results) in talos_datum.get('talos_counters', {}).iteritems(): signature = PerformanceSignature.objects.get(test=counter) datum = PerformanceDatum.objects.get(signature=signature) assert round(float(results['mean']), 2) == datum.value assert datum.push_timestamp == expected_push_timestamp # we should be left with just the summary series signature = PerformanceSignature.objects.get( test='', suite=talos_datum['testrun']['suite']) datum = PerformanceDatum.objects.get(signature=signature) if talos_datum.get('summary'): assert round(talos_datum['summary']['suite'], 2) == datum.value else: # old style talos blob without summary. again, going away, # but let's at least test that we have the value assert datum.value assert datum.push_timestamp == expected_push_timestamp # delete perf objects for next iteration PerformanceSignature.objects.all().delete() PerformanceDatum.objects.all().delete()
def set_oauth_credentials(): OAuthCredentials.set_credentials(SampleData.get_credentials())
def sample_data(): """Returns a SampleData() object""" from sampledata import SampleData return SampleData()
def test_adapt_and_load(self): talos_perf_data = SampleData.get_talos_perf_data() for talos_datum in talos_perf_data: datum = { "job_guid": 'oqiwy0q847365qiu', "name": "test", "type": "test", "blob": talos_datum } job_data = { "oqiwy0q847365qiu": { "id": 1, "result_set_id": 1, "push_timestamp": 1402692388 } } reference_data = { "property1": "value1", "property2": "value2", "property3": "value3" } # Mimic production environment, the blobs are serialized # when the web service receives them datum['blob'] = json.dumps({'talos_data': [datum['blob']]}) tda = TalosDataAdapter() tda.adapt_and_load(reference_data, job_data, datum) # base: subtests + one extra result for the summary series expected_result_count = len(talos_datum["results"]) + 1 # we create one performance series per counter if 'talos_counters' in talos_datum: expected_result_count += len(talos_datum["talos_counters"]) # result count == number of signatures self.assertEqual(expected_result_count, len(tda.signatures.keys())) # verify that we have signatures for the subtests signature_placeholders = copy.copy( tda.signature_property_placeholders) for (testname, results) in talos_datum["results"].iteritems(): signature_placeholder = filter( lambda p: p[2] == testname, signature_placeholders) self.assertEqual(len(signature_placeholder), 1) signature_hash = signature_placeholder[0][0] perfdata = tda.signatures[signature_hash][0] if talos_datum.get('summary'): # if we have a summary, ensure the subtest summary values made # it in for measure in ['min', 'max', 'std', 'mean', 'median']: self.assertEqual( round(talos_datum['summary']['subtests'][testname][measure], 2), perfdata[measure]) else: # this is an old style talos blob without a summary. these are going # away, so I'm not going to bother testing the correctness. however # let's at least verify that some values are being generated here for measure in ['min', 'max', 'std', 'mean', 'median']: self.assertTrue(perfdata[measure]) # filter out this signature from data to process signature_placeholders = filter( lambda p: p[0] != signature_hash, signature_placeholders) # if we have counters, verify that the series for them is as expected for (counter, results) in talos_datum.get('talos_counters', {}).iteritems(): signature_placeholder = filter( lambda p: p[2] == counter, signature_placeholders) self.assertEqual(len(signature_placeholder), 1) signature_hash = signature_placeholder[0][0] perfdata = tda.signatures[signature_hash][0] for measure in ['max', 'mean']: self.assertEqual(round(float(results[measure]), 2), perfdata[measure]) # filter out this signature from data to process signature_placeholders = filter( lambda p: p[0] != signature_hash, signature_placeholders) # we should be left with just summary signature placeholders self.assertEqual(len(signature_placeholders), 2) perfdata = tda.signatures[signature_placeholders[0][0]][0] if talos_datum.get('summary'): self.assertEqual(round(talos_datum['summary']['suite'], 2), perfdata['geomean']) else: # old style talos blob without summary. again, going away, # but let's at least test that we have the 'geomean' value # generated self.assertTrue(perfdata['geomean'])
def test_adapt_and_load(self): talos_perf_data = SampleData.get_talos_perf_data() for talos_datum in talos_perf_data: # delete any previously-created perf objects # FIXME: because of https://bugzilla.mozilla.org/show_bug.cgi?id=1133273 # this can be really slow if we have a dev database with lots of # performance data in it (if the test succeeds, the transaction # will be rolled back so at least it won't pollute the production # database) PerformanceSignature.objects.all().delete() PerformanceDatum.objects.all().delete() datum = { "job_guid": 'oqiwy0q847365qiu', "name": "test", "type": "test", "blob": talos_datum } job_data = { "oqiwy0q847365qiu": { "id": 1, "result_set_id": 1, "push_timestamp": 1402692388 } } reference_data = { "option_collection_hash": self.OPTION_HASH, "machine_platform": self.MACHINE_PLATFORM, "property1": "value1", "property2": "value2", "property3": "value3" } # Mimic production environment, the blobs are serialized # when the web service receives them datum['blob'] = json.dumps({'talos_data': [datum['blob']]}) tda = TalosDataAdapter() tda.adapt_and_load(self.REPO_NAME, reference_data, job_data, datum) # base: subtests + one extra result for the summary series expected_result_count = len(talos_datum["results"]) + 1 # we create one performance series per counter if 'talos_counters' in talos_datum: expected_result_count += len(talos_datum["talos_counters"]) # result count == number of signatures self.assertEqual(expected_result_count, PerformanceSignature.objects.all().count()) # verify that we have signatures for the subtests for (testname, results) in talos_datum["results"].iteritems(): signature = PerformanceSignature.objects.get(test=testname) datum = PerformanceDatum.objects.get(signature=signature) if talos_datum.get('summary'): # if we have a summary, ensure the subtest summary values made # it in self.assertEqual( round(talos_datum['summary']['subtests'][testname]['filtered'], 2), datum.value) else: # this is an old style talos blob without a summary. these are going # away, so I'm not going to bother testing the correctness. however # let's at least verify that some values are being generated here self.assertTrue(datum.value) # if we have counters, verify that the series for them is as expected for (counter, results) in talos_datum.get('talos_counters', {}).iteritems(): signature = PerformanceSignature.objects.get(test=counter) datum = PerformanceDatum.objects.get(signature=signature) self.assertEqual(round(float(results['mean']), 2), datum.value) # we should be left with just the summary series signature = PerformanceSignature.objects.get( test='', suite=talos_datum['testrun']['suite']) datum = PerformanceDatum.objects.get(signature=signature) if talos_datum.get('summary'): self.assertEqual(round(talos_datum['summary']['suite'], 2), datum.value) else: # old style talos blob without summary. again, going away, # but let's at least test that we have the value self.assertTrue(datum.value)
def test_adapt_and_load(): talos_perf_data = SampleData.get_talos_perf_data() tda = TalosDataAdapter() result_count = 0 for datum in talos_perf_data: datum = { "job_guid": 'oqiwy0q847365qiu', "name": "test", "type": "test", "blob": datum } job_data = { "oqiwy0q847365qiu": { "id": 1, "result_set_id": 1, "push_timestamp": 1402692388 } } reference_data = { "property1": "value1", "property2": "value2", "property3": "value3" } # one extra result for the summary series result_count += len(datum['blob']["results"]) + 1 # we create one performance series per counter if 'talos_counters' in datum['blob']: result_count += len(datum['blob']["talos_counters"]) # Mimic production environment, the blobs are serialized # when the web service receives them datum['blob'] = json.dumps({'talos_data': [datum['blob']]}) tda.adapt_and_load(reference_data, job_data, datum) # we upload a summary with a suite and subtest values, +1 for suite if 'summary' in datum['blob']: results = json.loads(zlib.decompress(tda.performance_artifact_placeholders[-1][4])) data = json.loads(datum['blob'])['talos_data'][0] assert results["blob"]["performance_series"]["geomean"] == data['summary']['suite'] # deal with the subtests now for i in range(0, len(data['summary']['subtests'])): subresults = json.loads(zlib.decompress(tda.performance_artifact_placeholders[-1 - i][4])) if 'subtest_signatures' in subresults["blob"]['signature_properties']: # ignore summary signatures continue subdata = data['summary']['subtests'][subresults["blob"]['signature_properties']['test']] for datatype in ['min', 'max', 'mean', 'median', 'std']: assert subdata[datatype] == subresults["blob"]["performance_series"][datatype] if 'value' in subdata.keys(): assert subdata['value'] == subresults["blob"]["performance_series"]['value'] else: # FIXME: the talos data blob we're currently using contains datums with summaries and those without # we should probably test non-summarized data as well pass assert result_count == len(tda.performance_artifact_placeholders)
def test_adapt_and_load(self): talos_perf_data = SampleData.get_talos_perf_data() for talos_datum in talos_perf_data: datum = { "job_guid": 'oqiwy0q847365qiu', "name": "test", "type": "test", "blob": talos_datum } job_data = { "oqiwy0q847365qiu": { "id": 1, "result_set_id": 1, "push_timestamp": 1402692388 } } reference_data = { "property1": "value1", "property2": "value2", "property3": "value3" } # Mimic production environment, the blobs are serialized # when the web service receives them datum['blob'] = json.dumps({'talos_data': [datum['blob']]}) tda = TalosDataAdapter() tda.adapt_and_load(reference_data, job_data, datum) # base: subtests + one extra result for the summary series expected_result_count = len(talos_datum["results"]) + 1 # we create one performance series per counter if 'talos_counters' in talos_datum: expected_result_count += len(talos_datum["talos_counters"]) # result count == number of signatures self.assertEqual(expected_result_count, len(tda.signatures.keys())) # verify that we have signatures for the subtests signature_placeholders = copy.copy( tda.signature_property_placeholders) for (testname, results) in talos_datum["results"].iteritems(): signature_placeholder = filter(lambda p: p[2] == testname, signature_placeholders) self.assertEqual(len(signature_placeholder), 1) signature_hash = signature_placeholder[0][0] perfdata = tda.signatures[signature_hash][0] if talos_datum.get('summary'): # if we have a summary, ensure the subtest summary values made # it in for measure in ['min', 'max', 'std', 'mean', 'median']: self.assertEqual( round( talos_datum['summary']['subtests'][testname] [measure], 2), perfdata[measure]) else: # this is an old style talos blob without a summary. these are going # away, so I'm not going to bother testing the correctness. however # let's at least verify that some values are being generated here for measure in ['min', 'max', 'std', 'mean', 'median']: self.assertTrue(perfdata[measure]) # filter out this signature from data to process signature_placeholders = filter( lambda p: p[0] != signature_hash, signature_placeholders) # if we have counters, verify that the series for them is as expected for (counter, results) in talos_datum.get('talos_counters', {}).iteritems(): signature_placeholder = filter(lambda p: p[2] == counter, signature_placeholders) self.assertEqual(len(signature_placeholder), 1) signature_hash = signature_placeholder[0][0] perfdata = tda.signatures[signature_hash][0] for measure in ['max', 'mean']: self.assertEqual(round(float(results[measure]), 2), perfdata[measure]) # filter out this signature from data to process signature_placeholders = filter( lambda p: p[0] != signature_hash, signature_placeholders) # we should be left with just summary signature placeholders self.assertEqual(len(signature_placeholders), 2) perfdata = tda.signatures[signature_placeholders[0][0]][0] if talos_datum.get('summary'): self.assertEqual(round(talos_datum['summary']['suite'], 2), perfdata['geomean']) else: # old style talos blob without summary. again, going away, # but let's at least test that we have the 'geomean' value # generated self.assertTrue(perfdata['geomean'])
def test_load_talos_data(test_project, test_repository, perf_option_collection, perf_platform, perf_job_data, perf_reference_data): PerformanceFramework.objects.create(name='talos') talos_perf_data = SampleData.get_talos_perf_data() for talos_datum in talos_perf_data: datum = { "job_guid": "fake_job_guid", "name": "test", "type": "test", "blob": talos_datum } # Mimic production environment, the blobs are serialized # when the web service receives them datum['blob'] = json.dumps({'talos_data': [datum['blob']]}) load_talos_artifacts(test_repository.name, perf_reference_data, perf_job_data, datum) # base: subtests + one extra result for the summary series expected_result_count = len(talos_datum["results"]) + 1 # we create one performance series per counter if 'talos_counters' in talos_datum: expected_result_count += len(talos_datum["talos_counters"]) # result count == number of signatures assert expected_result_count == PerformanceSignature.objects.all( ).count() expected_push_timestamp = datetime.datetime.fromtimestamp( perf_job_data['fake_job_guid']['push_timestamp']) # verify that we have signatures for the subtests for (testname, results) in talos_datum["results"].iteritems(): signature = PerformanceSignature.objects.get(test=testname) datum = PerformanceDatum.objects.get(signature=signature) if talos_datum.get('summary'): # if we have a summary, ensure the subtest summary values made # it in and that we ingested lowerIsBetter ok (if it was there) subtest = talos_datum['summary']['subtests'][testname] assert round(subtest['filtered'], 2) == datum.value assert signature.lower_is_better == subtest.get( 'lowerIsBetter', True) else: # this is an old style talos blob without a summary. these are # going away, so I'm not going to bother testing the # correctness. however let's at least verify that some values # are being generated here assert datum.value assert datum.push_timestamp == expected_push_timestamp # if we have counters, verify that the series for them is as expected for (counter, results) in talos_datum.get('talos_counters', {}).iteritems(): signature = PerformanceSignature.objects.get(test=counter) datum = PerformanceDatum.objects.get(signature=signature) assert round(float(results['mean']), 2) == datum.value assert datum.push_timestamp == expected_push_timestamp # we should be left with just the summary series signature = PerformanceSignature.objects.get( test='', suite=talos_datum['testrun']['suite']) datum = PerformanceDatum.objects.get(signature=signature) if talos_datum.get('summary'): assert round(talos_datum['summary']['suite'], 2) == datum.value else: # old style talos blob without summary. again, going away, # but let's at least test that we have the value assert datum.value assert datum.push_timestamp == expected_push_timestamp # delete perf objects for next iteration PerformanceSignature.objects.all().delete() PerformanceDatum.objects.all().delete()
def test_load_talos_data(self): PerformanceFramework.objects.get_or_create(name='talos') talos_perf_data = SampleData.get_talos_perf_data() for talos_datum in talos_perf_data: (job_data, reference_data) = self._get_job_and_reference_data() datum = { "job_guid": self.JOB_GUID, "name": "test", "type": "test", "blob": talos_datum } # Mimic production environment, the blobs are serialized # when the web service receives them datum['blob'] = json.dumps({'talos_data': [datum['blob']]}) load_talos_artifacts(self.REPO_NAME, reference_data, job_data, datum) # base: subtests + one extra result for the summary series expected_result_count = len(talos_datum["results"]) + 1 # we create one performance series per counter if 'talos_counters' in talos_datum: expected_result_count += len(talos_datum["talos_counters"]) # result count == number of signatures self.assertEqual(expected_result_count, PerformanceSignature.objects.all().count()) # verify that we have signatures for the subtests for (testname, results) in talos_datum["results"].iteritems(): signature = PerformanceSignature.objects.get(test=testname) datum = PerformanceDatum.objects.get(signature=signature) if talos_datum.get('summary'): # if we have a summary, ensure the subtest summary values made # it in and that we ingested lowerIsBetter ok (if it was there) subtest = talos_datum['summary']['subtests'][testname] self.assertEqual( round(subtest['filtered'], 2), datum.value) self.assertEqual(signature.lower_is_better, subtest.get('lowerIsBetter', True)) else: # this is an old style talos blob without a summary. these are going # away, so I'm not going to bother testing the correctness. however # let's at least verify that some values are being generated here self.assertTrue(datum.value) self.assertEqual(datum.push_timestamp, datetime.datetime.fromtimestamp( self.PUSH_TIMESTAMP)) # if we have counters, verify that the series for them is as expected for (counter, results) in talos_datum.get('talos_counters', {}).iteritems(): signature = PerformanceSignature.objects.get(test=counter) datum = PerformanceDatum.objects.get(signature=signature) self.assertEqual(round(float(results['mean']), 2), datum.value) self.assertEqual(datum.push_timestamp, datetime.datetime.fromtimestamp( self.PUSH_TIMESTAMP)) # we should be left with just the summary series signature = PerformanceSignature.objects.get( test='', suite=talos_datum['testrun']['suite']) datum = PerformanceDatum.objects.get(signature=signature) if talos_datum.get('summary'): self.assertEqual(round(talos_datum['summary']['suite'], 2), datum.value) else: # old style talos blob without summary. again, going away, # but let's at least test that we have the value self.assertTrue(datum.value) self.assertEqual(datum.push_timestamp, datetime.datetime.fromtimestamp( self.PUSH_TIMESTAMP)) # delete perf objects for next iteration PerformanceSignature.objects.all().delete() PerformanceDatum.objects.all().delete()