def __iter__(self):
records = []
with transaction_context(self.database_class) as conn:
records.extend([
record[0] for record in
execute_query_fetchall(conn, 'SELECT app_name FROM cron_job')
])
return iter(records)
def __iter__(self):
records = []
with transaction_context(self.database_class) as conn:
records.extend([
record[0] for record in execute_query_fetchall(
conn, 'SELECT app_name FROM cron_job')
])
return iter(records)
def update_bug_data(self, bug_id, signature_set):
with transaction_context(self.database) as connection:
self.logger.debug('bug %s: %s', bug_id, signature_set)
# If there's no associated signatures, delete everything for this bug id
if not signature_set:
sql = """
DELETE FROM crashstats_bugassociation WHERE bug_id = %s
"""
execute_no_results(connection, sql, (bug_id, ))
return
try:
sql = """
SELECT signature FROM crashstats_bugassociation WHERE bug_id = %s
"""
signature_rows = execute_query_fetchall(
connection, sql, (bug_id, ))
signatures_db = [x[0] for x in signature_rows]
for signature in signatures_db:
if signature not in signature_set:
sql = """
DELETE FROM crashstats_bugassociation
WHERE signature = %s and bug_id = %s
"""
execute_no_results(connection, sql,
(signature, bug_id))
self.logger.info('association removed: %s - "%s"',
bug_id, signature)
except SQLDidNotReturnSingleRow:
signatures_db = []
for signature in signature_set:
if signature not in signatures_db:
sql = """
INSERT INTO crashstats_bugassociation (signature, bug_id)
VALUES (%s, %s)
"""
execute_no_results(connection, sql, (signature, bug_id))
self.logger.info('association added: %s - "%s"', bug_id,
signature)
def items(self):
"""return all the app_names and their values as tuples"""
sql = """
SELECT
app_name,
next_run,
first_run,
last_run,
last_success,
depends_on,
error_count,
last_error
FROM cron_job"""
columns = ('app_name', 'next_run', 'first_run', 'last_run',
'last_success', 'depends_on', 'error_count', 'last_error')
items = []
with transaction_context(self.database_class) as conn:
for record in execute_query_fetchall(conn, sql):
row = dict(zip(columns, record))
items.append((row.pop('app_name'), row))
return items
def check_past_missing(self):
"""Check the table for missing crashes and check to see if they exist."""
connection_source = self.crashstorage.connection_source
bucket_name = connection_source.config.bucket_name
boto_conn = connection_source._connect()
crash_ids = []
with transaction_context(self.database) as conn:
sql = """
SELECT crash_id
FROM crashstats_missingprocessedcrash
WHERE is_processed=False
"""
params = ()
crash_ids = [item[0] for item in execute_query_fetchall(conn, sql, params)]
no_longer_missing = []
for crash_id in crash_ids:
bucket = boto_conn.get_bucket(bucket_name)
processed_crash_key = bucket.get_key(PROCESSED_CRASH_TEMPLATE % crash_id)
if processed_crash_key is not None:
no_longer_missing.append(crash_id)
if no_longer_missing:
with transaction_context(self.database) as conn:
sql = """
UPDATE crashstats_missingprocessedcrash
SET is_processed=True
WHERE crash_id IN %s
"""
params = (tuple(no_longer_missing),)
execute_no_results(conn, sql, params)
self.logger.info(
'Updated %s missing crashes which have since been processed',
len(no_longer_missing)
)
def update_bug_data(self, bug_id, signature_set):
with transaction_context(self.database) as connection:
self.logger.debug('bug %s: %s', bug_id, signature_set)
# If there's no associated signatures, delete everything for this bug id
if not signature_set:
sql = """
DELETE FROM crashstats_bugassociation WHERE bug_id = %s
"""
execute_no_results(connection, sql, (bug_id,))
return
try:
sql = """
SELECT signature FROM crashstats_bugassociation WHERE bug_id = %s
"""
signature_rows = execute_query_fetchall(connection, sql, (bug_id,))
signatures_db = [x[0] for x in signature_rows]
for signature in signatures_db:
if signature not in signature_set:
sql = """
DELETE FROM crashstats_bugassociation
WHERE signature = %s and bug_id = %s
"""
execute_no_results(connection, sql, (signature, bug_id))
self.logger.info('association removed: %s - "%s"', bug_id, signature)
except SQLDidNotReturnSingleRow:
signatures_db = []
for signature in signature_set:
if signature not in signatures_db:
sql = """
INSERT INTO crashstats_bugassociation (signature, bug_id)
VALUES (%s, %s)
"""
execute_no_results(connection, sql, (signature, bug_id))
self.logger.info('association added: %s - "%s"', bug_id, signature)
def items(self):
"""return all the app_names and their values as tuples"""
sql = """
SELECT
app_name,
next_run,
first_run,
last_run,
last_success,
depends_on,
error_count,
last_error
FROM cron_job"""
columns = (
'app_name',
'next_run', 'first_run', 'last_run', 'last_success',
'depends_on', 'error_count', 'last_error'
)
items = []
with transaction_context(self.database_class) as conn:
for record in execute_query_fetchall(conn, sql):
row = dict(zip(columns, record))
items.append((row.pop('app_name'), row))
return items
def _get_real_version(self, product, channel, build_id):
"""Return real version number from crashstats_productversion table
:arg str product: the product
:arg str channel: the release channel
:arg int build_id: the build id as a string
:returns: ``None`` or the version string that should be used
"""
# Fix the product so it matches the data in the table
if (product, channel) == ('firefox', 'aurora') and build_id > '20170601':
product = 'DevEdition'
elif product == 'firefox':
product = 'Firefox'
elif product in ('fennec', 'fennecandroid'):
product = 'Fennec'
key = '%s:%s:%s' % (product, channel, build_id)
if key in self.cache:
self.metrics.incr('cache', tags=['result:hit'])
return self.cache[key]
sql = """
SELECT version_string
FROM crashstats_productversion
WHERE product_name = %(product)s
AND release_channel = %(channel)s
AND build_id = %(build_id)s
"""
params = {
'product': product,
'channel': channel,
'build_id': build_id
}
with self.conn_context() as conn:
versions = execute_query_fetchall(conn, sql, params)
if versions:
# Flatten version results from a list of tuples to a list of versions
versions = [version[0] for version in versions]
if versions:
if 'b' in versions[0]:
# If we're looking at betas which have a "b" in the versions,
# then ignore "rc" versions because they didn't get released
versions = [version for version in versions if 'rc' not in version]
else:
# If we're looking at non-betas, then only return "rc"
# versions because this crash report is in the beta channel
# and not the release channel
versions = [version for version in versions if 'rc' in version]
if not versions:
# We didn't get an answer which could mean that this is a weird
# build and there is no answer or it could mean that Buildhub
# doesn't know, yet. Maybe in the future we get a better answer so
# we use the short ttl.
self.metrics.incr('lookup', tags=['result:fail'])
self.cache.set(key, value=None, ttl=self.SHORT_CACHE_TTL)
return None
# If we got an answer we should keep it around for a while because it's
# a real answer and it's not going to change so use the long ttl plus
# a fudge factor.
real_version = versions[0]
self.metrics.incr('lookup', tags=['result:success'])
self.cache.set(key, value=real_version, ttl=self.LONG_CACHE_TTL)
return real_version