def _has_signature_report(self, signature, connection):
try:
single_row_sql(
connection, """
SELECT 1 FROM reports
WHERE signature = %s LIMIT 1""", (signature, ))
return True
except SQLDidNotReturnSingleRow:
return False
def _has_signature_report(self, signature, connection):
try:
single_row_sql(
connection,
"""
SELECT 1 FROM reports
WHERE signature = %s LIMIT 1""",
(signature,)
)
return True
except SQLDidNotReturnSingleRow:
return False
def test_single_value_sql5(self):
m_execute = Mock()
m_fetchall = Mock(return_value=((17, 22), ))
m_cursor = Mock()
m_cursor.execute = m_execute
m_cursor.fetchall = m_fetchall
conn = Mock()
conn.cursor.return_value = m_cursor
dbapi2_util.single_row_sql(conn, "select 17, 22", (1, 2, 3))
eq_(conn.cursor.call_count, 1)
eq_(m_cursor.execute.call_count, 1)
m_cursor.execute.assert_called_once_with("select 17, 22", (1, 2, 3))
def test_single_value_sql5(self):
m_execute = Mock()
m_fetchall = Mock(return_value=((17, 22),))
m_cursor = Mock()
m_cursor.execute = m_execute
m_cursor.fetchall = m_fetchall
conn = Mock()
conn.cursor.return_value = m_cursor
dbapi2_util.single_row_sql(conn, "select 17, 22", (1, 2, 3))
eq_(conn.cursor.call_count, 1)
eq_(m_cursor.execute.call_count, 1)
m_cursor.execute.assert_called_once_with("select 17, 22", (1, 2, 3))
def test_single_value_sql4(self):
m_execute = Mock()
m_fetchall = Mock(return_value=None)
m_cursor = MagicMock()
m_cursor.execute = m_execute
m_cursor.fetchall = m_fetchall
conn = MagicMock()
conn.cursor.return_value.__enter__.return_value = m_cursor
with pytest.raises(dbapi2_util.SQLDidNotReturnSingleRow):
dbapi2_util.single_row_sql(conn, 'select 17, 22', (1, 2, 3))
assert conn.cursor.call_count == 1
assert m_cursor.execute.call_count == 1
m_cursor.execute.assert_called_once_with("select 17, 22", (1, 2, 3))
def test_single_value_sql4(self):
m_execute = Mock()
m_fetchall = Mock(return_value=None)
m_cursor = MagicMock()
m_cursor.execute = m_execute
m_cursor.fetchall = m_fetchall
conn = MagicMock()
conn.cursor.return_value.__enter__.return_value = m_cursor
with pytest.raises(dbapi2_util.SQLDidNotReturnSingleRow):
dbapi2_util.single_row_sql(conn, 'select 17, 22', (1, 2, 3))
assert conn.cursor.call_count == 1
assert m_cursor.execute.call_count == 1
m_cursor.execute.assert_called_once_with("select 17, 22", (1, 2, 3))
def update_crashstats_signature(self, connection, signature, report_date,
report_build):
# Pull the data in the db. If it's there, then do an update. If it's
# not there, then do an insert.
try:
sig = single_row_sql(
connection, """
SELECT signature, first_build, first_date
FROM crashstats_signature
WHERE signature=%s
""", (signature, ))
sql = """
UPDATE crashstats_signature
SET first_build=%s, first_date=%s
WHERE signature=%s
"""
params = (min(sig[1], int(report_build)),
min(sig[2], string_to_datetime(report_date)), sig[0])
except SQLDidNotReturnSingleRow:
sql = """
INSERT INTO crashstats_signature (signature, first_build, first_date)
VALUES (%s, %s, %s)
"""
params = (signature, report_build, report_date)
execute_no_results(connection, sql, params)
def test_single_row_sql1(self):
m_execute = Mock()
m_fetchall = Mock(return_value=((17, 22), ))
m_cursor = Mock()
m_cursor.execute = m_execute
m_cursor.fetchall = m_fetchall
conn = Mock()
conn.cursor.return_value = m_cursor
r = dbapi2_util.single_row_sql(conn, "select 17, 22")
self.assertEqual(r, (17, 22))
self.assertEqual(conn.cursor.call_count, 1)
self.assertEqual(m_cursor.execute.call_count, 1)
m_cursor.execute.assert_called_once_with('select 17, 22', None)
def test_single_row_sql1(self):
m_execute = Mock()
m_fetchall = Mock(return_value=((17, 22), ))
m_cursor = MagicMock()
m_cursor.execute = m_execute
m_cursor.fetchall = m_fetchall
conn = MagicMock()
conn.cursor.return_value.__enter__.return_value = m_cursor
r = dbapi2_util.single_row_sql(conn, "select 17, 22")
assert r == (17, 22)
assert conn.cursor.call_count == 1
assert m_cursor.execute.call_count == 1
m_cursor.execute.assert_called_once_with('select 17, 22', None)
def test_single_row_sql1(self):
m_execute = Mock()
m_fetchall = Mock(return_value=((17, 22),))
m_cursor = MagicMock()
m_cursor.execute = m_execute
m_cursor.fetchall = m_fetchall
conn = MagicMock()
conn.cursor.return_value.__enter__.return_value = m_cursor
r = dbapi2_util.single_row_sql(conn, "select 17, 22")
eq_(r, (17, 22))
eq_(conn.cursor.call_count, 1)
eq_(m_cursor.execute.call_count, 1)
m_cursor.execute.assert_called_once_with('select 17, 22', None)
def inner_transaction(self, connection, bug_id, status, resolution,
short_desc, signature_set):
self.config.logger.debug("bug %s (%s, %s) %s: %s", bug_id, status,
resolution, short_desc, signature_set)
if not signature_set:
execute_no_results(connection, "DELETE FROM bugs WHERE id = %s",
(bug_id, ))
return
useful = False
insert_made = False
try:
status_db, resolution_db, short_desc_db = single_row_sql(
connection, """SELECT status, resolution, short_desc
FROM bugs
WHERE id = %s""", (bug_id, ))
if (status_db != status or resolution_db != resolution
or short_desc_db != short_desc):
execute_no_results(
connection, """
UPDATE bugs SET
status = %s, resolution = %s, short_desc = %s
WHERE id = %s""", (status, resolution, short_desc, bug_id))
self.config.logger.info("bug status updated: %s - %s, %s",
bug_id, status, resolution)
useful = True
signature_rows = execute_query_fetchall(
connection,
"SELECT signature FROM bug_associations WHERE bug_id = %s",
(bug_id, ))
signatures_db = [x[0] for x in signature_rows]
for signature in signatures_db:
if signature not in signature_set:
execute_no_results(
connection, """
DELETE FROM bug_associations
WHERE signature = %s and bug_id = %s""",
(signature, bug_id))
self.config.logger.info('association removed: %s - "%s"',
bug_id, signature)
useful = True
except SQLDidNotReturnSingleRow:
execute_no_results(
connection, """
INSERT INTO bugs
(id, status, resolution, short_desc)
VALUES (%s, %s, %s, %s)""",
(bug_id, status, resolution, short_desc))
insert_made = True
signatures_db = []
for signature in signature_set:
if signature not in signatures_db:
if self._has_signature_report(signature, connection):
execute_no_results(
connection, """
INSERT INTO bug_associations (signature, bug_id)
VALUES (%s, %s)""", (signature, bug_id))
self.config.logger.info('new association: %s - "%s"',
bug_id, signature)
useful = True
else:
self.config.logger.info(
'rejecting association (no reports with this '
'signature): %s - "%s"', bug_id, signature)
if useful:
if insert_made:
self.config.logger.info('new bug: %s - %s, %s, "%s"', bug_id,
status, resolution, short_desc)
else:
if insert_made:
self.config.logger.info(
'rejecting bug (no useful information): '
'%s - %s, %s, "%s"', bug_id, status, resolution,
short_desc)
else:
self.config.logger.info(
'skipping bug (no new information): '
'%s - %s, %s, "%s"', bug_id, status, resolution,
short_desc)
raise NothingUsefulHappened('nothing useful done')
def inner_transaction(
self,
connection,
bug_id,
status,
resolution,
short_desc,
signature_set
):
self.config.logger.debug(
"bug %s (%s, %s) %s: %s",
bug_id, status, resolution, short_desc, signature_set)
if not signature_set:
execute_no_results(
connection,
"DELETE FROM bugs WHERE id = %s",
(bug_id,)
)
return
useful = False
insert_made = False
try:
status_db, resolution_db, short_desc_db = single_row_sql(
connection,
"""SELECT status, resolution, short_desc
FROM bugs
WHERE id = %s""",
(bug_id,)
)
if (status_db != status
or resolution_db != resolution
or short_desc_db != short_desc):
execute_no_results(
connection,
"""
UPDATE bugs SET
status = %s, resolution = %s, short_desc = %s
WHERE id = %s""",
(status, resolution, short_desc, bug_id)
)
self.config.logger.info(
"bug status updated: %s - %s, %s",
bug_id,
status,
resolution
)
useful = True
signature_rows = execute_query_fetchall(
connection,
"SELECT signature FROM bug_associations WHERE bug_id = %s",
(bug_id,)
)
signatures_db = [x[0] for x in signature_rows]
for signature in signatures_db:
if signature not in signature_set:
execute_no_results(
connection,
"""
DELETE FROM bug_associations
WHERE signature = %s and bug_id = %s""",
(signature, bug_id)
)
self.config.logger.info(
'association removed: %s - "%s"',
bug_id, signature)
useful = True
except SQLDidNotReturnSingleRow:
execute_no_results(
connection,
"""
INSERT INTO bugs
(id, status, resolution, short_desc)
VALUES (%s, %s, %s, %s)""",
(bug_id, status, resolution, short_desc)
)
insert_made = True
signatures_db = []
for signature in signature_set:
if signature not in signatures_db:
if self._has_signature_report(signature, connection):
execute_no_results(
connection,
"""
INSERT INTO bug_associations (signature, bug_id)
VALUES (%s, %s)""",
(signature, bug_id)
)
self.config.logger.info(
'new association: %s - "%s"',
bug_id,
signature
)
useful = True
else:
self.config.logger.info(
'rejecting association (no reports with this '
'signature): %s - "%s"',
bug_id,
signature
)
if useful:
if insert_made:
self.config.logger.info(
'new bug: %s - %s, %s, "%s"',
bug_id,
status,
resolution,
short_desc
)
else:
if insert_made:
self.config.logger.info(
'rejecting bug (no useful information): '
'%s - %s, %s, "%s"',
bug_id, status, resolution, short_desc)
else:
self.config.logger.info(
'skipping bug (no new information): '
'%s - %s, %s, "%s"',
bug_id,
status,
resolution,
short_desc
)
raise NothingUsefulHappened('nothing useful done')
def _sweep_dead_processors_transaction(self, connection):
"""this function is a single database transaction: look for dead
processors - find all the jobs of dead processors and assign them to
live processors then delete the dead processor registrations"""
self.config.logger.info("looking for dead processors")
try:
self.config.logger.info("threshold %s",
self.config.registrar.check_in_frequency)
threshold = single_value_sql(
connection, "select now() - %s - %s",
(self.config.registrar.processor_grace_period,
self.config.registrar.check_in_frequency))
dead_processors = execute_query_fetchall(
connection,
"select id from processors where lastSeenDateTime < %s",
(threshold, ))
if dead_processors:
self.config.logger.info("found dead processor(s):")
for a_dead_processor in dead_processors:
self.config.logger.info("%d is dead", a_dead_processor[0])
self.config.logger.debug("getting list of live processor(s):")
live_processors = execute_query_fetchall(
connection,
"select id from processors where lastSeenDateTime >= %s",
(threshold, ))
if not live_processors:
if self.config.registrar.quit_if_no_processors:
raise NoProcessorsRegisteredError(
"There are no processors registered")
else:
self.config.logger.critical(
'There are no live processors, nothing to do. '
'Waiting for processors to come on line.')
return
number_of_live_processors = len(live_processors)
self.config.logger.debug(
"getting range of queued date for jobs associated with "
"dead processor(s):")
dead_processor_ids_str = ", ".join(
[str(x[0]) for x in dead_processors])
earliest_dead_job, latest_dead_job = single_row_sql(
connection,
"select min(queueddatetime), max(queueddatetime) from jobs "
"where owner in (%s)" % dead_processor_ids_str)
# take dead processor jobs and reallocate them to live
# processors in equal sized chunks
if (earliest_dead_job is not None
and latest_dead_job is not None):
time_increment = ((latest_dead_job - earliest_dead_job) /
number_of_live_processors)
for x, live_processor_id in enumerate(live_processors):
low_queued_time = (x * time_increment +
earliest_dead_job)
high_queued_time = ((x + 1) * time_increment +
earliest_dead_job)
self.config.logger.info(
"assigning jobs from %s to %s to processor %s:",
low_queued_time, high_queued_time,
live_processor_id)
# why is the range >= at both ends? the range must be
# inclusive, the risk of moving a job twice is low and
# consequences low, too.
# 1st step: take any jobs of a dead processor that were
# in progress and reset them to unprocessed
execute_no_results(
connection, "update jobs set"
" starteddatetime = NULL "
"where"
" %%s >= queueddatetime"
" and queueddatetime >= %%s"
" and owner in (%s)"
" and success is NULL" % dead_processor_ids_str,
(high_queued_time, low_queued_time))
# 2nd step: take all jobs of a dead processor and give
# them to a new owner
execute_no_results(
connection, "update jobs set"
" set owner = %%s "
"where"
" %%s >= queueddatetime"
" and queueddatetime >= %%s"
" and owner in (%s)" % dead_processor_ids_str,
(live_processor_id, high_queued_time,
low_queued_time))
# transfer stalled priority jobs to new processors
for dead_processor_tuple in dead_processors:
self.config.logger.info(
"re-assigning priority jobs from processor %d:",
dead_processor_tuple[0])
execute_no_results(
connection,
"insert into priorityjobs (uuid) select uuid "
"from priority_jobs_%d" % dead_processor_tuple)
self.config.logger.info("removing all dead processors")
execute_no_results(
connection,
"delete from processors where lastSeenDateTime < %s",
(threshold, ))
# remove dead processors' priority tables
for a_dead_processor in dead_processors:
execute_no_results(
connection, "drop table if exists priority_jobs_%d" %
a_dead_processor[0])
except NoProcessorsRegisteredError:
self.quit = True
self.config.logger.critical('there are no live processors')
def _sweep_dead_processors_transaction(self, connection):
"""this function is a single database transaction: look for dead
processors - find all the jobs of dead processors and assign them to
live processors then delete the dead processor registrations"""
self.config.logger.info("looking for dead processors")
try:
self.config.logger.info(
"threshold %s",
self.config.registrar.check_in_frequency
)
threshold = single_value_sql(
connection,
"select now() - %s - %s",
(self.config.registrar.processor_grace_period,
self.config.registrar.check_in_frequency)
)
dead_processors = execute_query_fetchall(
connection,
"select id from processors where lastSeenDateTime < %s",
(threshold,)
)
if dead_processors:
self.config.logger.info("found dead processor(s):")
for a_dead_processor in dead_processors:
self.config.logger.info("%d is dead", a_dead_processor[0])
self.config.logger.debug("getting list of live processor(s):")
live_processors = execute_query_fetchall(
connection,
"select id from processors where lastSeenDateTime >= %s",
(threshold,)
)
if not live_processors:
if self.config.registrar.quit_if_no_processors:
raise NoProcessorsRegisteredError(
"There are no processors registered"
)
else:
self.config.logger.critical(
'There are no live processors, nothing to do. '
'Waiting for processors to come on line.'
)
return
number_of_live_processors = len(live_processors)
self.config.logger.debug(
"getting range of queued date for jobs associated with "
"dead processor(s):"
)
dead_processor_ids_str = ", ".join(
[str(x[0]) for x in dead_processors]
)
earliest_dead_job, latest_dead_job = single_row_sql(
connection,
"select min(queueddatetime), max(queueddatetime) from jobs "
"where owner in (%s)" % dead_processor_ids_str
)
# take dead processor jobs and reallocate them to live
# processors in equal sized chunks
if (earliest_dead_job is not None and
latest_dead_job is not None):
time_increment = (
(latest_dead_job - earliest_dead_job) /
number_of_live_processors
)
for x, live_processor_id in enumerate(live_processors):
low_queued_time = (
x * time_increment + earliest_dead_job
)
high_queued_time = (
(x + 1) * time_increment + earliest_dead_job
)
self.config.logger.info(
"assigning jobs from %s to %s to processor %s:",
low_queued_time,
high_queued_time,
live_processor_id
)
# why is the range >= at both ends? the range must be
# inclusive, the risk of moving a job twice is low and
# consequences low, too.
# 1st step: take any jobs of a dead processor that were
# in progress and reset them to unprocessed
execute_no_results(
connection,
"update jobs set"
" starteddatetime = NULL "
"where"
" %%s >= queueddatetime"
" and queueddatetime >= %%s"
" and owner in (%s)"
" and success is NULL" % dead_processor_ids_str,
(high_queued_time, low_queued_time)
)
# 2nd step: take all jobs of a dead processor and give
# them to a new owner
execute_no_results(
connection,
"update jobs set"
" set owner = %%s "
"where"
" %%s >= queueddatetime"
" and queueddatetime >= %%s"
" and owner in (%s)" % dead_processor_ids_str,
(live_processor_id, high_queued_time,
low_queued_time)
)
# transfer stalled priority jobs to new processors
for dead_processor_tuple in dead_processors:
self.config.logger.info(
"re-assigning priority jobs from processor %d:",
dead_processor_tuple[0]
)
execute_no_results(
connection,
"insert into priorityjobs (uuid) select uuid "
"from priority_jobs_%d" % dead_processor_tuple
)
self.config.logger.info("removing all dead processors")
execute_no_results(
connection,
"delete from processors where lastSeenDateTime < %s",
(threshold,)
)
# remove dead processors' priority tables
for a_dead_processor in dead_processors:
execute_no_results(
connection,
"drop table if exists priority_jobs_%d" %
a_dead_processor[0]
)
except NoProcessorsRegisteredError:
self.quit = True
self.config.logger.critical('there are no live processors')