def get_last_successful_hour_or_start_hour():
"""Get the last hour that ran successfully or the start hour."""
last_hour = crash_stats.get_last_successful_hour()
if last_hour:
return last_hour
return get_start_hour()
def _get_crash_occurrence_platforms_from_crash_parameters(
crash_type, crash_state, security_flag, project_name, lookbehind_days):
"""Get platforms from crash stats based on crash parameters."""
last_hour = crash_stats.get_last_successful_hour()
if not last_hour:
# No crash stats available, skip.
return []
where_clause = ('crash_type = {crash_type} AND '
'crash_state = {crash_state} AND '
'security_flag = {security_flag} AND '
'project = {project}').format(
crash_type=json.dumps(crash_type),
crash_state=json.dumps(crash_state),
security_flag=json.dumps(security_flag),
project=json.dumps(project_name),
)
_, rows = crash_stats.get(end=last_hour,
block='day',
days=lookbehind_days,
group_by='platform',
where_clause=where_clause,
group_having_clause='',
sort_by='total_count',
offset=0,
limit=1)
platforms = set()
for row in rows:
for group in row['groups']:
platform = group['name'].split(':')[0].capitalize()
platforms.add(platform)
return list(platforms)
def test_last_hour(self):
"""Get the last hour."""
hour2 = data_types.BuildCrashStatsJobHistory()
hour2.end_time_in_hours = 15
hour2.put()
hour1 = data_types.BuildCrashStatsJobHistory()
hour1.end_time_in_hours = 10
hour1.put()
self.assertEqual(15, crash_stats.get_last_successful_hour())
def get_top_crashes_for_all_projects_and_platforms():
"""Return top crashes for all projects and platforms."""
last_hour = crash_stats.get_last_successful_hour()
if not last_hour:
# No crash stats available, skip.
return {}
projects_to_jobs_and_platforms = (get_jobs_and_platforms_for_project())
top_crashes_by_project_and_platform_map = {}
for project_name in projects_to_jobs_and_platforms:
top_crashes_by_project_and_platform_map[project_name] = {}
project_map = projects_to_jobs_and_platforms[project_name]
for platform in project_map.platforms:
where_clause = (
'crash_type NOT IN UNNEST(%s) AND '
'crash_state NOT IN UNNEST(%s) AND '
'job_type IN UNNEST(%s) AND '
'platform LIKE %s AND '
'project = %s' %
(json.dumps(TOP_CRASHES_IGNORE_CRASH_TYPES),
json.dumps(TOP_CRASHES_IGNORE_CRASH_STATES),
json.dumps(list(project_map.jobs)),
json.dumps(platform.lower() + '%'), json.dumps(project_name)))
_, rows = crash_stats.get(end=last_hour,
block='day',
days=TOP_CRASHES_DAYS_LOOKBEHIND,
group_by='platform',
where_clause=where_clause,
group_having_clause='',
sort_by='total_count',
offset=0,
limit=TOP_CRASHES_LIMIT)
if not rows:
continue
top_crashes_by_project_and_platform_map[project_name][platform] = [
{
'crashState': row['crashState'],
'crashType': row['crashType'],
'isSecurity': row['isSecurity'],
'totalCount': row['totalCount'],
} for row in rows
if row['totalCount'] >= TOP_CRASHES_MIN_THRESHOLD
]
return top_crashes_by_project_and_platform_map
def get_top_crashes_for_all_projects_and_platforms():
"""Return top crashes for all projects and platforms."""
last_hour = crash_stats.get_last_successful_hour()
if not last_hour:
# No crash stats available, skip.
return {}
jobs, platforms = get_jobs_and_platforms_for_top_crashes()
project_names = data_handler.get_all_project_names()
top_crashes_by_project_and_platform_map = {}
for project_name in project_names:
top_crashes_by_project_and_platform_map[project_name] = {}
for platform in platforms:
where_clause = (
'crash_type NOT IN UNNEST(%s) AND '
'crash_state NOT IN UNNEST(%s) AND '
'job_type IN UNNEST(%s) AND '
'platform LIKE %s AND '
'project = %s' %
(json.dumps(TOP_CRASHES_IGNORE_CRASH_TYPES),
json.dumps(TOP_CRASHES_IGNORE_CRASH_STATES),
json.dumps(list(jobs)), json.dumps(platform.lower() + '%'),
json.dumps(project_name)))
for keyword in TOP_CRASHES_IGNORE_CRASH_STATE_KEYWORDS:
where_clause += ' AND crash_state NOT LIKE "%%%s%%"' % keyword
_, rows = crash_stats.get(end=last_hour,
block='day',
days=TOP_CRASHES_DAYS_LOOKBEHIND,
group_by='platform',
where_clause=where_clause,
group_having_clause='',
sort_by='total_count',
offset=0,
limit=TOP_CRASHES_LIMIT)
if rows:
rows = [
s for s in rows
if s['totalCount'] >= TOP_CRASHES_MIN_THRESHOLD
]
top_crashes_by_project_and_platform_map[project_name][platform] = (
rows or [])
return top_crashes_by_project_and_platform_map
def get_top_crashes_for_all_projects_and_platforms():
"""Return top crashes for all projects and platforms."""
last_hour = crash_stats.get_last_successful_hour()
if not last_hour:
# No crash stats available, skip.
return {}
jobs, platforms = get_jobs_and_platforms_for_top_crashes()
project_names = data_handler.get_all_project_names()
top_crashes_by_project_and_platform_map = {}
for project_name in project_names:
top_crashes_by_project_and_platform_map[project_name] = {}
for platform in platforms:
where_clause = ("crash_type NOT IN UNNEST(%s) AND "
"crash_state NOT IN UNNEST(%s) AND "
"job_type IN UNNEST(%s) AND "
"platform LIKE %s AND "
"project = %s" % (
json.dumps(TOP_CRASHES_IGNORE_CRASH_TYPES),
json.dumps(TOP_CRASHES_IGNORE_CRASH_STATES),
json.dumps(list(jobs)),
json.dumps(platform.lower() + "%"),
json.dumps(project_name),
))
_, rows = crash_stats.get(
end=last_hour,
block="day",
days=TOP_CRASHES_DAYS_LOOKBEHIND,
group_by="platform",
where_clause=where_clause,
group_having_clause="",
sort_by="total_count",
offset=0,
limit=TOP_CRASHES_LIMIT,
)
if rows:
rows = [s for s in rows if s["totalCount"] >= TOP_CRASHES_MIN_THRESHOLD]
top_crashes_by_project_and_platform_map[project_name][
platform] = rows or []
return top_crashes_by_project_and_platform_map
def _get_crash_occurrence_platforms_from_crash_parameters(
crash_type, crash_state, security_flag, project_name, lookbehind_days):
"""Get platforms from crash stats based on crash parameters."""
last_hour = crash_stats.get_last_successful_hour()
if not last_hour:
# No crash stats available, skip.
return []
where_clause = ("crash_type = {crash_type} AND "
"crash_state = {crash_state} AND "
"security_flag = {security_flag} AND "
"project = {project}").format(
crash_type=json.dumps(crash_type),
crash_state=json.dumps(crash_state),
security_flag=json.dumps(security_flag),
project=json.dumps(project_name),
)
_, rows = crash_stats.get(
end=last_hour,
block="day",
days=lookbehind_days,
group_by="platform",
where_clause=where_clause,
group_having_clause="",
sort_by="total_count",
offset=0,
limit=1,
)
platforms = set()
for row in rows:
for group in row["groups"]:
platform = group["name"].split(":")[0].capitalize()
platforms.add(platform)
return list(platforms)
def _is_crash_important(testcase):
"""Indicate if the crash is important to file."""
if not testcase.one_time_crasher_flag:
# A reproducible crash is an important crash.
return True
if testcase.status != 'Processed':
# A duplicate or unreproducible crash is not an important crash.
return False
# Testcase is unreproducible. Only those crashes that are crashing frequently
# are important.
if testcase.crash_type in UNREPRODUCIBLE_CRASH_IGNORE_CRASH_TYPES:
return False
# Ensure that there is no reproducible testcase in our group.
if testcase.group_id:
other_reproducible_testcase = data_types.Testcase.query(
data_types.Testcase.group_id == testcase.group_id,
ndb_utils.is_false(
data_types.Testcase.one_time_crasher_flag)).get()
if other_reproducible_testcase:
# There is another reproducible testcase in our group. So, this crash is
# not important.
return False
# Get crash statistics data on this unreproducible crash for last X days.
last_hour = crash_stats.get_last_successful_hour()
if not last_hour:
# No crash stats available, skip.
return False
_, rows = crash_stats.get(
end=last_hour,
block='day',
days=data_types.FILE_CONSISTENT_UNREPRODUCIBLE_TESTCASE_DEADLINE,
group_by='reproducible_flag',
where_clause=(
'crash_type = %s AND crash_state = %s AND security_flag = %s' %
(json.dumps(testcase.crash_type), json.dumps(
testcase.crash_state), json.dumps(testcase.security_flag))),
group_having_clause='',
sort_by='total_count',
offset=0,
limit=1)
# Calculate total crash count and crash days count.
crash_days_indices = set([])
total_crash_count = 0
for row in rows:
if 'groups' not in row:
continue
total_crash_count += row['totalCount']
for group in row['groups']:
for index in group['indices']:
crash_days_indices.add(index['hour'])
crash_days_count = len(crash_days_indices)
# Only those unreproducible testcases are important that happened atleast once
# everyday for the last X days and total crash count exceeded our threshold
# limit.
return (crash_days_count
== data_types.FILE_CONSISTENT_UNREPRODUCIBLE_TESTCASE_DEADLINE
and total_crash_count >=
data_types.FILE_UNREPRODUCIBLE_TESTCASE_MIN_CRASH_THRESHOLD)
def test_none(self): """Get none because there's no last hour.""" self.assertIsNone(crash_stats.get_last_successful_hour())