def _remember_failure(self, class_, duration, exc_type, exc_value, exc_tb):
with transaction_context(self.database_class) as connection:
exc_traceback = ''.join(traceback.format_tb(exc_tb))
app_name = class_.app_name
execute_no_results(
connection,
"""INSERT INTO cron_log (
app_name,
duration,
exc_type,
exc_value,
exc_traceback,
log_time
) VALUES (
%s,
%s,
%s,
%s,
%s,
%s
)""",
(app_name, '%.5f' % duration, repr(exc_type), repr(exc_value),
exc_traceback, utc_now()),
)
metrics.gauge('job_failure_runtime',
value=duration,
tags=['job:%s' % app_name])
def update_crashstats_signature(self, signature, report_date,
report_build):
with transaction_context(self.database) as connection:
# Pull the data from the db. If it's there, then do an update. If it's
# not there, then do an insert.
try:
sql = """
SELECT signature, first_build, first_date
FROM crashstats_signature
WHERE signature=%s
"""
sig = single_row_sql(connection, sql, (signature, ))
sql = """
UPDATE crashstats_signature
SET first_build=%s, first_date=%s
WHERE signature=%s
"""
params = (min(sig[1], int(report_build)),
min(sig[2], string_to_datetime(report_date)), sig[0])
except SQLDidNotReturnSingleRow:
sql = """
INSERT INTO crashstats_signature (signature, first_build, first_date)
VALUES (%s, %s, %s)
"""
params = (signature, report_build, report_date)
execute_no_results(connection, sql, params)
def _remember_failure(self, class_, duration, exc_type, exc_value, exc_tb):
with transaction_context(self.database_class) as connection:
exc_traceback = ''.join(traceback.format_tb(exc_tb))
app_name = class_.app_name
execute_no_results(
connection,
"""INSERT INTO cron_log (
app_name,
duration,
exc_type,
exc_value,
exc_traceback,
log_time
) VALUES (
%s,
%s,
%s,
%s,
%s,
%s
)""",
(
app_name,
'%.5f' % duration,
repr(exc_type),
repr(exc_value),
exc_traceback,
utc_now()
),
)
metrics.gauge('job_failure_runtime', value=duration, tags=['job:%s' % app_name])
def update_crashstats_signature(self, signature, report_date, report_build):
with transaction_context(self.database) as connection:
# Pull the data from the db. If it's there, then do an update. If it's
# not there, then do an insert.
try:
sql = """
SELECT signature, first_build, first_date
FROM crashstats_signature
WHERE signature=%s
"""
sig = single_row_sql(connection, sql, (signature,))
sql = """
UPDATE crashstats_signature
SET first_build=%s, first_date=%s
WHERE signature=%s
"""
params = (
min(sig[1], int(report_build)),
min(sig[2], string_to_datetime(report_date)),
sig[0]
)
except SQLDidNotReturnSingleRow:
sql = """
INSERT INTO crashstats_signature (signature, first_build, first_date)
VALUES (%s, %s, %s)
"""
params = (signature, report_build, report_date)
execute_no_results(connection, sql, params)
def test_execute_no_results(self):
m_execute = Mock()
m_cursor = MagicMock()
m_cursor.execute = m_execute
conn = MagicMock()
conn.cursor.return_value.__enter__.return_value = m_cursor
dbutil.execute_no_results(
conn, "insert into table (a, b, c) values (%s, %s, %s)", (1, 2, 3))
assert conn.cursor.call_count == 1
assert m_cursor.execute.call_count == 1
m_cursor.execute.assert_called_once_with(
"insert into table (a, b, c) values (%s, %s, %s)", (1, 2, 3))
def __delitem__(self, key):
"""remove the item by key or raise KeyError"""
with transaction_context(self.database_class) as connection:
try:
# result intentionally ignored
single_value_sql(
connection, """SELECT app_name
FROM cron_job
WHERE
app_name = %s""", (key, ))
except SQLDidNotReturnSingleValue:
raise KeyError(key)
# item exists
execute_no_results(
connection, """DELETE FROM cron_job
WHERE app_name = %s""", (key, ))
def _remember_success(self, connection, class_, success_date, duration):
app_name = class_.app_name
execute_no_results(
connection,
"""INSERT INTO cron_log (
app_name,
success,
duration,
log_time
) VALUES (
%s,
%s,
%s,
%s
)""",
(app_name, success_date, '%.5f' % duration, utc_now()),
)
metrics.gauge('job_success_runtime', value=duration, tags=['job:%s' % app_name])
def test_execute_no_results(self):
m_execute = Mock()
m_cursor = MagicMock()
m_cursor.execute = m_execute
conn = MagicMock()
conn.cursor.return_value.__enter__.return_value = m_cursor
dbutil.execute_no_results(
conn,
"insert into table (a, b, c) values (%s, %s, %s)",
(1, 2, 3)
)
assert conn.cursor.call_count == 1
assert m_cursor.execute.call_count == 1
m_cursor.execute.assert_called_once_with(
"insert into table (a, b, c) values (%s, %s, %s)",
(1, 2, 3)
)
def _remember_success(self, class_, success_date, duration):
with transaction_context(self.database_class) as connection:
app_name = class_.app_name
execute_no_results(
connection,
"""INSERT INTO cron_log (
app_name,
success,
duration,
log_time
) VALUES (
%s,
%s,
%s,
%s
)""",
(app_name, success_date, '%.5f' % duration, utc_now()),
)
metrics.gauge('job_success_runtime', value=duration, tags=['job:%s' % app_name])
def handle_missing(self, date, missing):
"""Report crash ids for missing processed crashes."""
metrics.gauge('missing_processed', len(missing))
if missing:
for crash_id in missing:
self.logger.info('Missing: %s', crash_id)
with transaction_context(self.database) as conn:
sql = """
INSERT INTO crashstats_missingprocessedcrash (crash_id, is_processed, created)
VALUES (%s, False, current_timestamp)
"""
params = (crash_id,)
try:
execute_no_results(conn, sql, params)
except IntegrityError:
# If there's already one, that's fine--just move on
pass
else:
self.logger.info('All crashes for %s were processed.', date)
def check_past_missing(self):
"""Check the table for missing crashes and check to see if they exist."""
connection_source = self.crashstorage.connection_source
bucket_name = connection_source.config.bucket_name
boto_conn = connection_source._connect()
crash_ids = []
with transaction_context(self.database) as conn:
sql = """
SELECT crash_id
FROM crashstats_missingprocessedcrash
WHERE is_processed=False
"""
params = ()
crash_ids = [item[0] for item in execute_query_fetchall(conn, sql, params)]
no_longer_missing = []
for crash_id in crash_ids:
bucket = boto_conn.get_bucket(bucket_name)
processed_crash_key = bucket.get_key(PROCESSED_CRASH_TEMPLATE % crash_id)
if processed_crash_key is not None:
no_longer_missing.append(crash_id)
if no_longer_missing:
with transaction_context(self.database) as conn:
sql = """
UPDATE crashstats_missingprocessedcrash
SET is_processed=True
WHERE crash_id IN %s
"""
params = (tuple(no_longer_missing),)
execute_no_results(conn, sql, params)
self.logger.info(
'Updated %s missing crashes which have since been processed',
len(no_longer_missing)
)
def __delitem__(self, key):
"""remove the item by key or raise KeyError"""
with transaction_context(self.database_class) as connection:
try:
# result intentionally ignored
single_value_sql(
connection,
"""SELECT app_name
FROM cron_job
WHERE
app_name = %s""",
(key,)
)
except SQLDidNotReturnSingleValue:
raise KeyError(key)
# item exists
execute_no_results(
connection,
"""DELETE FROM cron_job
WHERE app_name = %s""",
(key,)
)
def update_bug_data(self, bug_id, signature_set):
with transaction_context(self.database) as connection:
self.logger.debug('bug %s: %s', bug_id, signature_set)
# If there's no associated signatures, delete everything for this bug id
if not signature_set:
sql = """
DELETE FROM crashstats_bugassociation WHERE bug_id = %s
"""
execute_no_results(connection, sql, (bug_id, ))
return
try:
sql = """
SELECT signature FROM crashstats_bugassociation WHERE bug_id = %s
"""
signature_rows = execute_query_fetchall(
connection, sql, (bug_id, ))
signatures_db = [x[0] for x in signature_rows]
for signature in signatures_db:
if signature not in signature_set:
sql = """
DELETE FROM crashstats_bugassociation
WHERE signature = %s and bug_id = %s
"""
execute_no_results(connection, sql,
(signature, bug_id))
self.logger.info('association removed: %s - "%s"',
bug_id, signature)
except SQLDidNotReturnSingleRow:
signatures_db = []
for signature in signature_set:
if signature not in signatures_db:
sql = """
INSERT INTO crashstats_bugassociation (signature, bug_id)
VALUES (%s, %s)
"""
execute_no_results(connection, sql, (signature, bug_id))
self.logger.info('association added: %s - "%s"', bug_id,
signature)
def update_bug_data(self, bug_id, signature_set):
with transaction_context(self.database) as connection:
self.logger.debug('bug %s: %s', bug_id, signature_set)
# If there's no associated signatures, delete everything for this bug id
if not signature_set:
sql = """
DELETE FROM crashstats_bugassociation WHERE bug_id = %s
"""
execute_no_results(connection, sql, (bug_id,))
return
try:
sql = """
SELECT signature FROM crashstats_bugassociation WHERE bug_id = %s
"""
signature_rows = execute_query_fetchall(connection, sql, (bug_id,))
signatures_db = [x[0] for x in signature_rows]
for signature in signatures_db:
if signature not in signature_set:
sql = """
DELETE FROM crashstats_bugassociation
WHERE signature = %s and bug_id = %s
"""
execute_no_results(connection, sql, (signature, bug_id))
self.logger.info('association removed: %s - "%s"', bug_id, signature)
except SQLDidNotReturnSingleRow:
signatures_db = []
for signature in signature_set:
if signature not in signatures_db:
sql = """
INSERT INTO crashstats_bugassociation (signature, bug_id)
VALUES (%s, %s)
"""
execute_no_results(connection, sql, (signature, bug_id))
self.logger.info('association added: %s - "%s"', bug_id, signature)
def __setitem__(self, key, value):
class LastErrorEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, type):
return repr(obj)
return json.JSONEncoder.default(self, obj)
with transaction_context(self.database_class) as connection:
try:
single_value_sql(
connection, """
SELECT ongoing
FROM cron_job
WHERE
app_name = %s
FOR UPDATE NOWAIT
""", (key, ))
# If the above single_value_sql() didn't raise a
# SQLDidNotReturnSingleValue exception, it means
# there is a row by this app_name.
# Therefore, the next SQL is an update.
next_sql = """
UPDATE cron_job
SET
next_run = %(next_run)s,
first_run = %(first_run)s,
last_run = %(last_run)s,
last_success = %(last_success)s,
depends_on = %(depends_on)s,
error_count = %(error_count)s,
last_error = %(last_error)s,
ongoing = %(ongoing)s
WHERE
app_name = %(app_name)s
"""
except OperationalError as exception:
if 'could not obtain lock' in exception.args[0]:
raise RowLevelLockError(exception.args[0])
else:
raise
except SQLDidNotReturnSingleValue:
# the key does not exist, do an insert
next_sql = """
INSERT INTO cron_job (
app_name,
next_run,
first_run,
last_run,
last_success,
depends_on,
error_count,
last_error,
ongoing
) VALUES (
%(app_name)s,
%(next_run)s,
%(first_run)s,
%(last_run)s,
%(last_success)s,
%(depends_on)s,
%(error_count)s,
%(last_error)s,
%(ongoing)s
)
"""
# serialize last_error if it's a {}
last_error = value['last_error']
if isinstance(last_error, dict):
last_error = json.dumps(value['last_error'],
cls=LastErrorEncoder)
parameters = {
'app_name': key,
'next_run': value['next_run'],
'first_run': value['first_run'],
'last_run': value['last_run'],
'last_success': value.get('last_success'),
'depends_on': value['depends_on'],
'error_count': value['error_count'],
'last_error': last_error,
'ongoing': value.get('ongoing'),
}
try:
execute_no_results(connection, next_sql, parameters)
except IntegrityError as exception:
# See CREATE_CRONTABBER_APP_NAME_UNIQUE_INDEX for why
# we know to look for this mentioned in the error message.
if 'crontabber_unique_app_name_idx' in exception.args[0]:
raise RowLevelLockError(exception.args[0])
raise
def __setitem__(self, key, value):
class LastErrorEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, type):
return repr(obj)
return json.JSONEncoder.default(self, obj)
with transaction_context(self.database_class) as connection:
try:
single_value_sql(
connection,
"""
SELECT ongoing
FROM cron_job
WHERE
app_name = %s
FOR UPDATE NOWAIT
""",
(key,)
)
# If the above single_value_sql() didn't raise a
# SQLDidNotReturnSingleValue exception, it means
# there is a row by this app_name.
# Therefore, the next SQL is an update.
next_sql = """
UPDATE cron_job
SET
next_run = %(next_run)s,
first_run = %(first_run)s,
last_run = %(last_run)s,
last_success = %(last_success)s,
depends_on = %(depends_on)s,
error_count = %(error_count)s,
last_error = %(last_error)s,
ongoing = %(ongoing)s
WHERE
app_name = %(app_name)s
"""
except OperationalError as exception:
if 'could not obtain lock' in exception.args[0]:
raise RowLevelLockError(exception.args[0])
else:
raise
except SQLDidNotReturnSingleValue:
# the key does not exist, do an insert
next_sql = """
INSERT INTO cron_job (
app_name,
next_run,
first_run,
last_run,
last_success,
depends_on,
error_count,
last_error,
ongoing
) VALUES (
%(app_name)s,
%(next_run)s,
%(first_run)s,
%(last_run)s,
%(last_success)s,
%(depends_on)s,
%(error_count)s,
%(last_error)s,
%(ongoing)s
)
"""
# serialize last_error if it's a {}
last_error = value['last_error']
if isinstance(last_error, dict):
last_error = json.dumps(value['last_error'], cls=LastErrorEncoder)
parameters = {
'app_name': key,
'next_run': value['next_run'],
'first_run': value['first_run'],
'last_run': value['last_run'],
'last_success': value.get('last_success'),
'depends_on': value['depends_on'],
'error_count': value['error_count'],
'last_error': last_error,
'ongoing': value.get('ongoing'),
}
try:
execute_no_results(connection, next_sql, parameters)
except IntegrityError as exception:
# See CREATE_CRONTABBER_APP_NAME_UNIQUE_INDEX for why
# we know to look for this mentioned in the error message.
if 'crontabber_unique_app_name_idx' in exception.args[0]:
raise RowLevelLockError(exception.args[0])
raise
