def get_new_failures(dates):
bq = big_query_utils.create_big_query()
this_script_path = os.path.join(os.path.dirname(__file__))
sql_script = os.path.join(this_script_path, 'sql/new_failures_24h.sql')
with open(sql_script) as query_file:
query = query_file.read().format(
calibration_begin=dates['calibration']['begin'],
calibration_end=dates['calibration']['end'],
reporting_begin=dates['reporting']['begin'],
reporting_end=dates['reporting']['end'])
logging.debug("Query:\n%s", query)
query_job = big_query_utils.sync_query_job(bq, 'grpc-testing', query)
page = bq.jobs().getQueryResults(
pageToken=None, **query_job['jobReference']).execute(num_retries=3)
rows = page.get('rows')
if rows:
return {
row['f'][0]['v']: Row(poll_strategy=row['f'][1]['v'],
job_name=row['f'][2]['v'],
build_id=row['f'][3]['v'],
timestamp=row['f'][4]['v'])
for row in rows
}
else:
return {}
def print_summary_records(self, num_query_retries=3):
line = '-' * 120
print line
print 'Summary records'
print 'Run Id: ', self.run_id
print 'Dataset Id: ', self.dataset_id
print line
query = ('SELECT pod_name, image_type, event_type, event_date, details'
' FROM %s.%s WHERE run_id = \'%s\' ORDER by event_date;') % (
self.dataset_id, self.summary_table_id, self.run_id)
query_job = bq_utils.sync_query_job(self.bq, self.project_id, query)
print '{:<25} {:<12} {:<12} {:<30} {}'.format('Pod name', 'Image type',
'Event type', 'Date',
'Details')
print line
page_token = None
while True:
page = self.bq.jobs().getQueryResults(
pageToken=page_token,
**query_job['jobReference']).execute(num_retries=num_query_retries)
rows = page.get('rows', [])
for row in rows:
print '{:<25} {:<12} {:<12} {:<30} {}'.format(row['f'][0]['v'],
row['f'][1]['v'],
row['f'][2]['v'],
row['f'][3]['v'],
row['f'][4]['v'])
page_token = page.get('pageToken')
if not page_token:
break
def print_qps_records(self, num_query_retries=3):
line = '-' * 80
print line
print 'QPS Summary'
print 'Run Id: ', self.run_id
print 'Dataset Id: ', self.dataset_id
print line
query = (
'SELECT pod_name, recorded_at, qps FROM %s.%s WHERE run_id = \'%s\' '
'ORDER by recorded_at;') % (self.dataset_id, self.qps_table_id,
self.run_id)
query_job = bq_utils.sync_query_job(self.bq, self.project_id, query)
print '{:<25} {:30} {}'.format('Pod name', 'Recorded at', 'Qps')
print line
page_token = None
while True:
page = self.bq.jobs().getQueryResults(
pageToken=page_token,
**query_job['jobReference']).execute(num_retries=num_query_retries)
rows = page.get('rows', [])
for row in rows:
print '{:<25} {:30} {}'.format(row['f'][0]['v'], row['f'][1]['v'],
row['f'][2]['v'])
page_token = page.get('pageToken')
if not page_token:
break
def print_summary_records(self, num_query_retries=3):
line = '-' * 120
print line
print 'Summary records'
print 'Run Id: ', self.run_id
print 'Dataset Id: ', self.dataset_id
print line
query = ('SELECT pod_name, image_type, event_type, event_date, details'
' FROM %s.%s WHERE run_id = \'%s\' ORDER by event_date;') % (
self.dataset_id, self.summary_table_id, self.run_id)
query_job = bq_utils.sync_query_job(self.bq, self.project_id, query)
print '{:<25} {:<12} {:<12} {:<30} {}'.format('Pod name', 'Image type',
'Event type', 'Date',
'Details')
print line
page_token = None
while True:
page = self.bq.jobs().getQueryResults(
pageToken=page_token, **query_job['jobReference']).execute(
num_retries=num_query_retries)
rows = page.get('rows', [])
for row in rows:
print '{:<25} {:<12} {:<12} {:<30} {}'.format(
row['f'][0]['v'], row['f'][1]['v'], row['f'][2]['v'],
row['f'][3]['v'], row['f'][4]['v'])
page_token = page.get('pageToken')
if not page_token:
break
def print_qps_records(self, num_query_retries=3):
line = '-' * 80
print line
print 'QPS Summary'
print 'Run Id: ', self.run_id
print 'Dataset Id: ', self.dataset_id
print line
query = (
'SELECT pod_name, recorded_at, qps FROM %s.%s WHERE run_id = \'%s\' '
'ORDER by recorded_at;') % (self.dataset_id, self.qps_table_id,
self.run_id)
query_job = bq_utils.sync_query_job(self.bq, self.project_id, query)
print '{:<25} {:30} {}'.format('Pod name', 'Recorded at', 'Qps')
print line
page_token = None
while True:
page = self.bq.jobs().getQueryResults(
pageToken=page_token, **query_job['jobReference']).execute(
num_retries=num_query_retries)
rows = page.get('rows', [])
for row in rows:
print '{:<25} {:30} {}'.format(row['f'][0]['v'],
row['f'][1]['v'],
row['f'][2]['v'])
page_token = page.get('pageToken')
if not page_token:
break
def get_flaky_tests(days_lower_bound, days_upper_bound, limit=None):
""" period is one of "WEEK", "DAY", etc.
(see https://cloud.google.com/bigquery/docs/reference/standard-sql/functions-and-operators#date_add). """
bq = big_query_utils.create_big_query()
query = """
SELECT
REGEXP_REPLACE(test_name, r'/\d+', '') AS filtered_test_name,
job_name,
build_id,
timestamp
FROM
[grpc-testing:jenkins_test_results.aggregate_results]
WHERE
timestamp > DATE_ADD(CURRENT_DATE(), {days_lower_bound}, "DAY")
AND timestamp <= DATE_ADD(CURRENT_DATE(), {days_upper_bound}, "DAY")
AND NOT REGEXP_MATCH(job_name, '.*portability.*')
AND result != 'PASSED' AND result != 'SKIPPED'
ORDER BY timestamp desc
""".format(days_lower_bound=days_lower_bound,
days_upper_bound=days_upper_bound)
if limit:
query += '\n LIMIT {}'.format(limit)
query_job = big_query_utils.sync_query_job(bq, 'grpc-testing', query)
page = bq.jobs().getQueryResults(
pageToken=None, **query_job['jobReference']).execute(num_retries=3)
rows = page.get('rows')
if rows:
return {
row['f'][0]['v']:
(row['f'][1]['v'], row['f'][2]['v'], row['f'][3]['v'])
for row in rows
}
else:
return {}
def check_if_any_tests_failed(self, num_query_retries=3, timeout_msec=30000):
query = ('SELECT event_type FROM %s.%s WHERE run_id = \'%s\' AND '
'event_type="%s"') % (self.dataset_id, self.summary_table_id,
self.run_id, EventType.FAILURE)
page = None
try:
query_job = bq_utils.sync_query_job(self.bq, self.project_id, query)
job_id = query_job['jobReference']['jobId']
project_id = query_job['jobReference']['projectId']
page = self.bq.jobs().getQueryResults(
projectId=project_id,
jobId=job_id,
timeoutMs=timeout_msec).execute(num_retries=num_query_retries)
if not page['jobComplete']:
print('TIMEOUT ERROR: The query %s timed out. Current timeout value is'
' %d msec. Returning False (i.e assuming there are no failures)'
) % (query, timeoout_msec)
return False
num_failures = int(page['totalRows'])
print 'num rows: ', num_failures
return num_failures > 0
except:
print 'Exception in check_if_any_tests_failed(). Info: ', sys.exc_info()
print 'Query: ', query
def get_flaky_tests(days_lower_bound, days_upper_bound, limit=None):
""" period is one of "WEEK", "DAY", etc.
(see https://cloud.google.com/bigquery/docs/reference/standard-sql/functions-and-operators#date_add). """
bq = big_query_utils.create_big_query()
query = """
SELECT
REGEXP_REPLACE(test_name, r'/\d+', '') AS filtered_test_name,
job_name,
build_id,
timestamp
FROM
[grpc-testing:jenkins_test_results.aggregate_results]
WHERE
timestamp > DATE_ADD(CURRENT_DATE(), {days_lower_bound}, "DAY")
AND timestamp <= DATE_ADD(CURRENT_DATE(), {days_upper_bound}, "DAY")
AND NOT REGEXP_MATCH(job_name, '.*portability.*')
AND result != 'PASSED' AND result != 'SKIPPED'
ORDER BY timestamp desc
""".format(days_lower_bound=days_lower_bound, days_upper_bound=days_upper_bound)
if limit:
query += '\n LIMIT {}'.format(limit)
query_job = big_query_utils.sync_query_job(bq, 'grpc-testing', query)
page = bq.jobs().getQueryResults(
pageToken=None, **query_job['jobReference']).execute(num_retries=3)
testname_to_cols = {row['f'][0]['v']:
(row['f'][1]['v'], row['f'][2]['v'], row['f'][3]['v'])
for row in page['rows']}
return testname_to_cols
def _get_last_processed_buildnumber(build_name):
query = 'SELECT max(build_number) FROM [%s:%s.%s];' % (
_PROJECT_ID, _DATASET_ID, build_name)
query_job = big_query_utils.sync_query_job(bq, _PROJECT_ID, query)
page = bq.jobs().getQueryResults(
pageToken=None, **query_job['jobReference']).execute(num_retries=3)
if page['rows'][0]['f'][0]['v']:
return int(page['rows'][0]['f'][0]['v'])
return 0
def _get_last_processed_buildnumber(build_name):
query = 'SELECT max(build_number) FROM [%s:%s.%s];' % (
_PROJECT_ID, _DATASET_ID, build_name)
query_job = big_query_utils.sync_query_job(bq, _PROJECT_ID, query)
page = bq.jobs().getQueryResults(
pageToken=None, **query_job['jobReference']).execute(num_retries=3)
if page['rows'][0]['f'][0]['v']:
return int(page['rows'][0]['f'][0]['v'])
return 0
def check_if_any_tests_failed(self, num_query_retries=3):
query = ('SELECT event_type FROM %s.%s WHERE run_id = \'%s\' AND '
'event_type="%s"') % (self.dataset_id, self.summary_table_id,
self.run_id, EventType.FAILURE)
query_job = bq_utils.sync_query_job(self.bq, self.project_id, query)
page = self.bq.jobs().getQueryResults(
**query_job['jobReference']).execute(num_retries=num_query_retries)
num_failures = int(page['totalRows'])
print 'num rows: ', num_failures
return num_failures > 0
def check_if_any_tests_failed(self, num_query_retries=3):
query = ('SELECT event_type FROM %s.%s WHERE run_id = \'%s\' AND '
'event_type="%s"') % (self.dataset_id, self.summary_table_id,
self.run_id, EventType.FAILURE)
query_job = bq_utils.sync_query_job(self.bq, self.project_id, query)
page = self.bq.jobs().getQueryResults(**query_job['jobReference']).execute(
num_retries=num_query_retries)
num_failures = int(page['totalRows'])
print 'num rows: ', num_failures
return num_failures > 0
def check_if_any_tests_failed(self, num_query_retries=3):
query = ('SELECT event_type FROM %s.%s WHERE run_id = \'%s\' AND '
'event_type="%s"') % (self.dataset_id, self.summary_table_id,
self.run_id, EventType.FAILURE)
try:
query_job = bq_utils.sync_query_job(self.bq, self.project_id, query)
page = self.bq.jobs().getQueryResults(
**query_job['jobReference']).execute(num_retries=num_query_retries)
num_failures = int(page['totalRows'])
print 'num rows: ', num_failures
return num_failures > 0
# TODO (sreek): Cleanup the following lines once we have a better idea of
# why we sometimes get KeyError exceptions in long running test cases
except KeyError:
print 'KeyError in check_if_any_tests_failed()'
print 'Query:', query
print 'Query result page:', page
except:
print 'Exception in check_if_any_tests_failed(). Info: ', sys.exc_info()
print 'Query: ', query
def check_if_any_tests_failed(self, num_query_retries=3):
query = ('SELECT event_type FROM %s.%s WHERE run_id = \'%s\' AND '
'event_type="%s"') % (self.dataset_id, self.summary_table_id,
self.run_id, EventType.FAILURE)
try:
query_job = bq_utils.sync_query_job(self.bq, self.project_id,
query)
page = self.bq.jobs().getQueryResults(
**query_job['jobReference']).execute(
num_retries=num_query_retries)
num_failures = int(page['totalRows'])
print 'num rows: ', num_failures
return num_failures > 0
# TODO (sreek): Cleanup the following lines once we have a better idea of
# why we sometimes get KeyError exceptions in long running test cases
except KeyError:
print 'KeyError in check_if_any_tests_failed()'
print 'Query:', query
print 'Query result page:', page
except:
print 'Exception in check_if_any_tests_failed(). Info: ', sys.exc_info(
)
print 'Query: ', query
