def _read_signature_rules_from_database(self, connection):
for category, category_re in (
('prefix', 'prefix_signature_re'),
('irrelevant', 'irrelevant_signature_re'),
('line_number', 'signatures_with_line_numbers_re')
):
rule_element_list = [
a_rule
for (a_rule,) in execute_query_fetchall(
connection,
"select rule from skiplist "
"where category = %s",
(category, )
)
]
setattr(
self,
category_re,
re.compile('|'.join(rule_element_list))
)
# get sentinel rules
self.signature_sentinels = [
eval(sentinel_rule) # eval quoted strings and tuples
if sentinel_rule[0] in "'\"(" else
sentinel_rule # already a string, don't need to eval
for (sentinel_rule,) in execute_query_fetchall(
connection,
"select rule from csignature_rules where category = 'sentinel'"
)
]
def _read_signature_rules_from_database(self, connection):
for category, category_re in (
('prefix', 'prefix_signature_re'),
('irrelevant', 'irrelevant_signature_re'),
('line_number', 'signatures_with_line_numbers_re')
):
rule_element_list = [
a_rule
for (a_rule,) in execute_query_fetchall(
connection,
"select rule from skiplist "
"where category = %s",
(category, )
)
]
setattr(
self,
category_re,
re.compile('|'.join(rule_element_list))
)
# get sentinel rules
self.signature_sentinels = [
eval(sentinel_rule) # eval quoted strings and tuples
if sentinel_rule[0] in "'\"(" else
sentinel_rule # already a string, don't need to eval
for (sentinel_rule,) in execute_query_fetchall(
connection,
"select rule from csignature_rules where category = 'sentinel'"
)
]
def test_basic_run_job(self, rget):
config_manager = self._setup_config_manager()
def mocked_get(url):
return Response({
'hits': [
{
'product': 'Firefox',
'is_featured': True,
'version': '24.5.0'
},
],
'total':
1
})
rget.side_effect = mocked_get
rows = execute_query_fetchall(
self.conn, 'select product_name, version_string, featured_version '
'from product_versions')
eq_(sorted(rows), [('Firefox', '15.0a1', True),
('Firefox', '24.5.0', False)])
# and the view `product_info`...
rows = execute_query_fetchall(
self.conn, 'select product_name, version_string, is_featured '
'from product_info')
eq_(sorted(rows), [('Firefox', '15.0a1', True),
('Firefox', '24.5.0', False)])
# This is necessary so we get a new cursor when we do other
# selects after the crontabber app has run.
self.conn.commit()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['featured-versions-sync']
assert not information['featured-versions-sync']['last_error']
assert information['featured-versions-sync']['last_success']
config.logger.info.assert_called_with(
'Set featured versions for Firefox %r' % ([u'24.5.0'], ))
rows = execute_query_fetchall(
self.conn, 'select product_name, version_string, featured_version '
'from product_versions')
eq_(sorted(rows), [('Firefox', '15.0a1', False),
('Firefox', '24.5.0', True)])
# and the view `product_info`...
rows = execute_query_fetchall(
self.conn, 'select product_name, version_string, is_featured '
'from product_info')
eq_(sorted(rows), [('Firefox', '15.0a1', False),
('Firefox', '24.5.0', True)])
def inner_transaction(self, connection, bug_id, signature_set):
self.config.logger.debug("bug %s: %s", bug_id, signature_set)
if not signature_set:
execute_no_results(
connection, "DELETE FROM bug_associations WHERE bug_id = %s",
(bug_id, ))
return
try:
signature_rows = execute_query_fetchall(
connection,
"SELECT signature FROM bug_associations WHERE bug_id = %s",
(bug_id, ))
signatures_db = [x[0] for x in signature_rows]
for signature in signatures_db:
if signature not in signature_set:
execute_no_results(
connection, """
DELETE FROM bug_associations
WHERE signature = %s and bug_id = %s""",
(signature, bug_id))
self.config.logger.info('association removed: %s - "%s"',
bug_id, signature)
except SQLDidNotReturnSingleRow:
signatures_db = []
for signature in signature_set:
if signature not in signatures_db:
execute_no_results(
connection, """
INSERT INTO bug_associations (signature, bug_id)
VALUES (%s, %s)""", (signature, bug_id))
self.config.logger.info('new association: %s - "%s"', bug_id,
signature)
def _get_priority_jobs_transaction(self, connection):
"""this method implements a single transaction that just returns a
set of priority jobs."""
priority_jobs_list = execute_query_fetchall(
connection,
"select * from priorityjobs"
)
return set(x[0] for x in priority_jobs_list)
def _get_live_processors_transaction(self, connection):
"""this transaction just fetches a list of live processors"""
processor_ids = execute_query_fetchall(
connection, "select id, name from processors "
"where lastSeenDateTime > now() - interval %s",
(self.config.registrar.check_in_frequency, ))
# remove the row tuples, just give out a pure list of ids
return [(a_row[0], a_row[1]) for a_row in processor_ids]
def _get_live_processors_transaction(self, connection):
"""this transaction just fetches a list of live processors"""
processor_ids = execute_query_fetchall(
connection,
"select id, name from processors "
"where lastSeenDateTime > now() - interval %s",
(self.config.registrar.check_in_frequency,)
)
# remove the row tuples, just give out a pure list of ids
return [(a_row[0], a_row[1]) for a_row in processor_ids]
def test_basic_run_job(self, rget):
config_manager = self._setup_config_manager()
def mocked_get(url):
return Response({"hits": [{"product": "Firefox", "is_featured": True, "version": "24.5.0"}], "total": 1})
rget.side_effect = mocked_get
rows = execute_query_fetchall(
self.conn, "select product_name, version_string, featured_version " "from product_versions"
)
eq_(sorted(rows), [("Firefox", "15.0a1", True), ("Firefox", "24.5.0", False)])
# and the view `product_info`...
rows = execute_query_fetchall(
self.conn, "select product_name, version_string, is_featured " "from product_info"
)
eq_(sorted(rows), [("Firefox", "15.0a1", True), ("Firefox", "24.5.0", False)])
# This is necessary so we get a new cursor when we do other
# selects after the crontabber app has run.
self.conn.commit()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information["featured-versions-sync"]
assert not information["featured-versions-sync"]["last_error"]
assert information["featured-versions-sync"]["last_success"]
config.logger.info.assert_called_with("Set featured versions for Firefox %r" % ([u"24.5.0"],))
rows = execute_query_fetchall(
self.conn, "select product_name, version_string, featured_version " "from product_versions"
)
eq_(sorted(rows), [("Firefox", "15.0a1", False), ("Firefox", "24.5.0", True)])
# and the view `product_info`...
rows = execute_query_fetchall(
self.conn, "select product_name, version_string, is_featured " "from product_info"
)
eq_(sorted(rows), [("Firefox", "15.0a1", False), ("Firefox", "24.5.0", True)])
def _get_processors_and_loads_transaction(self, connection):
"""this transaction fetches a list of live processors and how many
jobs each curretly has assigned to it"""
sql = ("with live_processors as "
" (select * from processors where "
" lastSeenDateTime > now() - %s)"
"select"
" p.id,"
" count(j.owner),"
" p.name "
"from"
" live_processors p left join jobs j "
" on p.id = j.owner"
" and j.success is null "
"group by p.id, p.name")
processors_and_load = execute_query_fetchall(
connection, sql, (self.config.registrar.check_in_frequency, ))
# convert row tuples to muteable lists
return [[a_row[0], a_row[1], a_row[2]]
for a_row in processors_and_load]
def inner_transaction(self, connection, bug_id, signature_set):
self.config.logger.debug("bug %s: %s", bug_id, signature_set)
if not signature_set:
execute_no_results(
connection,
"DELETE FROM bug_associations WHERE bug_id = %s",
(bug_id,)
)
return
try:
signature_rows = execute_query_fetchall(
connection,
"SELECT signature FROM bug_associations WHERE bug_id = %s",
(bug_id,)
)
signatures_db = [x[0] for x in signature_rows]
for signature in signatures_db:
if signature not in signature_set:
execute_no_results(
connection,
"""
DELETE FROM bug_associations
WHERE signature = %s and bug_id = %s""",
(signature, bug_id)
)
self.config.logger.info('association removed: %s - "%s"', bug_id, signature)
except SQLDidNotReturnSingleRow:
signatures_db = []
for signature in signature_set:
if signature not in signatures_db:
execute_no_results(
connection,
"""
INSERT INTO bug_associations (signature, bug_id)
VALUES (%s, %s)""",
(signature, bug_id)
)
self.config.logger.info('association added: %s - "%s"', bug_id, signature)
def update_bug_data(self, connection, bug_id, signature_set):
self.config.logger.debug('bug %s: %s', bug_id, signature_set)
# If there's no associated signatures, delete everything for this bug id
if not signature_set:
execute_no_results(
connection, """
DELETE FROM crashstats_bugassociation WHERE bug_id = %s
""", (bug_id, ))
return
try:
signature_rows = execute_query_fetchall(
connection, """
SELECT signature FROM crashstats_bugassociation WHERE bug_id = %s
""", (bug_id, ))
signatures_db = [x[0] for x in signature_rows]
for signature in signatures_db:
if signature not in signature_set:
execute_no_results(
connection, """
DELETE FROM crashstats_bugassociation
WHERE signature = %s and bug_id = %s
""", (signature, bug_id))
self.config.logger.info('association removed: %s - "%s"',
bug_id, signature)
except SQLDidNotReturnSingleRow:
signatures_db = []
for signature in signature_set:
if signature not in signatures_db:
execute_no_results(
connection, """
INSERT INTO crashstats_bugassociation (signature, bug_id)
VALUES (%s, %s)
""", (signature, bug_id))
self.config.logger.info('association added: %s - "%s"', bug_id,
signature)
def _get_processors_and_loads_transaction(self, connection):
"""this transaction fetches a list of live processors and how many
jobs each curretly has assigned to it"""
sql = ("with live_processors as "
" (select * from processors where "
" lastSeenDateTime > now() - %s)"
"select"
" p.id,"
" count(j.owner),"
" p.name "
"from"
" live_processors p left join jobs j "
" on p.id = j.owner"
" and j.success is null "
"group by p.id, p.name")
processors_and_load = execute_query_fetchall(
connection,
sql,
(self.config.registrar.check_in_frequency,)
)
# convert row tuples to muteable lists
return [[a_row[0], a_row[1], a_row[2]]
for a_row in processors_and_load]
def test_basic_run_job(self, rget):
config_manager = self._setup_config_manager()
def mocked_get(url):
return Response({
'hits': {
'Firefox': [
{'featured': True, 'version': '24.5.0'},
{'featured': False, 'version': '15.0'},
],
'Camino': [
{'featured': False, 'version': '2.0.2'},
]
},
'total': 1
})
rget.side_effect = mocked_get
rows = execute_query_fetchall(
self.conn,
'select product_name, version_string, featured_version '
'from product_versions'
)
eq_(
sorted(rows),
[('Firefox', '15.0a1', True), ('Firefox', '24.5.0', False)]
)
# and the view `product_info`...
rows = execute_query_fetchall(
self.conn,
'select product_name, version_string, is_featured '
'from product_info'
)
eq_(
sorted(rows),
[('Firefox', '15.0a1', True), ('Firefox', '24.5.0', False)]
)
# This is necessary so we get a new cursor when we do other
# selects after the crontabber app has run.
self.conn.commit()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['featured-versions-sync']
assert not information['featured-versions-sync']['last_error']
assert information['featured-versions-sync']['last_success']
config.logger.info.assert_called_with(
'Set featured versions for Firefox %r' % (
[u'24.5.0'],
)
)
rows = execute_query_fetchall(
self.conn,
'select product_name, version_string, featured_version '
'from product_versions'
)
eq_(
sorted(rows),
[('Firefox', '15.0a1', False), ('Firefox', '24.5.0', True)]
)
# and the view `product_info`...
rows = execute_query_fetchall(
self.conn,
'select product_name, version_string, is_featured '
'from product_info'
)
eq_(
sorted(rows),
[('Firefox', '15.0a1', False), ('Firefox', '24.5.0', True)]
)
def test_basic_run_job(self, rget):
config_manager = self._setup_config_manager()
def mocked_get(url):
if 'firefox_versions.json' in url:
return Response({
'FIREFOX_NIGHTLY': '52.0a1',
'FIREFOX_AURORA': '51.0a2',
'FIREFOX_ESR': '45.4.0esr',
'FIREFOX_ESR_NEXT': '',
'LATEST_FIREFOX_DEVEL_VERSION': '50.0b7',
'LATEST_FIREFOX_OLDER_VERSION': '3.6.28',
'LATEST_FIREFOX_RELEASED_DEVEL_VERSION': '50.0b7',
'LATEST_FIREFOX_VERSION': '49.0.1'
})
elif 'mobile_versions.json' in url:
return Response({
'nightly_version': '52.0a1',
'alpha_version': '51.0a2',
'beta_version': '50.0b6',
'version': '49.0',
'ios_beta_version': '6.0',
'ios_version': '5.0'
})
elif 'thunderbird_versions.json' in url:
return Response({
'LATEST_THUNDERBIRD_VERSION': '45.4.0',
'LATEST_THUNDERBIRD_DEVEL_VERSION': '50.0b1',
'LATEST_THUNDERBIRD_ALPHA_VERSION': '51.0a2',
'LATEST_THUNDERBIRD_NIGHTLY_VERSION': '52.0a1',
})
else:
raise NotImplementedError(url)
rget.side_effect = mocked_get
# Check what's set up in the fixture
rows = execute_query_fetchall(
self.conn, 'select product_name, version_string, featured_version '
'from product_versions order by version_string')
assert sorted(rows) == [
('Firefox', '15.0a1', True),
('Firefox', '24.5.0', True),
('Firefox', '49.0.1', False),
('Firefox', '50.0b', False),
('Firefox', '51.0a2', False),
('Firefox', '52.0a1', False),
]
# This is necessary so we get a new cursor when we do other
# selects after the crontabber app has run.
self.conn.commit()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['featured-versions-automatic']
assert not information['featured-versions-automatic']['last_error']
assert information['featured-versions-automatic']['last_success']
config.logger.info.assert_called_with(
'Set featured versions for Thunderbird to: '
'45.4.0, 50.0b1, 51.0a2, 52.0a1')
rows = execute_query_fetchall(
self.conn, 'select product_name, version_string, featured_version '
'from product_versions')
eq_(sorted(rows), [
('Firefox', '15.0a1', False),
('Firefox', '24.5.0', False),
('Firefox', '49.0.1', True),
('Firefox', '50.0b', True),
('Firefox', '51.0a2', True),
('Firefox', '52.0a1', True),
])
def test_basic_run_job(self, req_mock):
config_manager = self._setup_config_manager()
req_mock.get(
'http://example.com/firefox_versions.json',
json={
'FIREFOX_NIGHTLY': '52.0a1',
# Kept for legacy and smooth transition. We USED to consider
# the latest AURORA version a featured version but we no
# longer build aurora so Socorro shouldn't pick this up any
# more even if product-details.mozilla.org supplies it.
'FIREFOX_AURORA': '51.0a2',
'FIREFOX_ESR': '45.4.0esr',
'FIREFOX_ESR_NEXT': '',
'LATEST_FIREFOX_DEVEL_VERSION': '50.0b7',
'LATEST_FIREFOX_OLDER_VERSION': '3.6.28',
'LATEST_FIREFOX_RELEASED_DEVEL_VERSION': '50.0b7',
'LATEST_FIREFOX_VERSION': '49.0.1',
})
req_mock.get('http://example.com/mobile_versions.json',
json={
'nightly_version': '52.0a1',
'alpha_version': '51.0a2',
'beta_version': '50.0b6',
'version': '49.0',
'ios_beta_version': '6.0',
'ios_version': '5.0',
})
req_mock.get('http://example.com/thunderbird_versions.json',
json={
'LATEST_THUNDERBIRD_VERSION': '45.4.0',
'LATEST_THUNDERBIRD_DEVEL_VERSION': '50.0b1',
'LATEST_THUNDERBIRD_ALPHA_VERSION': '51.0a2',
'LATEST_THUNDERBIRD_NIGHTLY_VERSION': '52.0a1',
})
# Check what's set up in the fixture
rows = execute_query_fetchall(
self.conn, 'select product_name, version_string, featured_version '
'from product_versions order by version_string')
assert sorted(rows) == [
('Firefox', '15.0a1', True),
('Firefox', '24.5.0', True),
('Firefox', '49.0.1', False),
('Firefox', '50.0b', False),
('Firefox', '51.0a2', False),
('Firefox', '52.0a1', False),
]
# This is necessary so we get a new cursor when we do other selects
# after the crontabber app has run.
self.conn.commit()
with config_manager.context() as config:
tab = CronTabberApp(config)
tab.run_all()
information = self._load_structure()
assert information['featured-versions-automatic']
assert not information['featured-versions-automatic']['last_error']
assert information['featured-versions-automatic']['last_success']
config.logger.info.assert_called_with(
'Set featured versions for Thunderbird to: '
'45.4.0, 50.0b1, 52.0a1')
rows = execute_query_fetchall(
self.conn, 'select product_name, version_string, featured_version '
'from product_versions')
expected = [
('Firefox', '15.0a1', False),
('Firefox', '24.5.0', False),
('Firefox', '49.0.1', True),
('Firefox', '50.0b', True),
# Note that the 'Aurora' branch is still mentioned but note that it's NOT featured
# (hence 'False').
('Firefox', '51.0a2', False),
('Firefox', '52.0a1', True),
]
assert sorted(rows) == expected
def inner_transaction(self, connection, bug_id, status, resolution,
short_desc, signature_set):
self.config.logger.debug("bug %s (%s, %s) %s: %s", bug_id, status,
resolution, short_desc, signature_set)
if not signature_set:
execute_no_results(connection, "DELETE FROM bugs WHERE id = %s",
(bug_id, ))
return
useful = False
insert_made = False
try:
status_db, resolution_db, short_desc_db = single_row_sql(
connection, """SELECT status, resolution, short_desc
FROM bugs
WHERE id = %s""", (bug_id, ))
if (status_db != status or resolution_db != resolution
or short_desc_db != short_desc):
execute_no_results(
connection, """
UPDATE bugs SET
status = %s, resolution = %s, short_desc = %s
WHERE id = %s""", (status, resolution, short_desc, bug_id))
self.config.logger.info("bug status updated: %s - %s, %s",
bug_id, status, resolution)
useful = True
signature_rows = execute_query_fetchall(
connection,
"SELECT signature FROM bug_associations WHERE bug_id = %s",
(bug_id, ))
signatures_db = [x[0] for x in signature_rows]
for signature in signatures_db:
if signature not in signature_set:
execute_no_results(
connection, """
DELETE FROM bug_associations
WHERE signature = %s and bug_id = %s""",
(signature, bug_id))
self.config.logger.info('association removed: %s - "%s"',
bug_id, signature)
useful = True
except SQLDidNotReturnSingleRow:
execute_no_results(
connection, """
INSERT INTO bugs
(id, status, resolution, short_desc)
VALUES (%s, %s, %s, %s)""",
(bug_id, status, resolution, short_desc))
insert_made = True
signatures_db = []
for signature in signature_set:
if signature not in signatures_db:
if self._has_signature_report(signature, connection):
execute_no_results(
connection, """
INSERT INTO bug_associations (signature, bug_id)
VALUES (%s, %s)""", (signature, bug_id))
self.config.logger.info('new association: %s - "%s"',
bug_id, signature)
useful = True
else:
self.config.logger.info(
'rejecting association (no reports with this '
'signature): %s - "%s"', bug_id, signature)
if useful:
if insert_made:
self.config.logger.info('new bug: %s - %s, %s, "%s"', bug_id,
status, resolution, short_desc)
else:
if insert_made:
self.config.logger.info(
'rejecting bug (no useful information): '
'%s - %s, %s, "%s"', bug_id, status, resolution,
short_desc)
else:
self.config.logger.info(
'skipping bug (no new information): '
'%s - %s, %s, "%s"', bug_id, status, resolution,
short_desc)
raise NothingUsefulHappened('nothing useful done')
def _sweep_dead_processors_transaction(self, connection):
"""this function is a single database transaction: look for dead
processors - find all the jobs of dead processors and assign them to
live processors then delete the dead processor registrations"""
self.config.logger.info("looking for dead processors")
try:
self.config.logger.info(
"threshold %s",
self.config.registrar.check_in_frequency
)
threshold = single_value_sql(
connection,
"select now() - %s - %s",
(self.config.registrar.processor_grace_period,
self.config.registrar.check_in_frequency)
)
dead_processors = execute_query_fetchall(
connection,
"select id from processors where lastSeenDateTime < %s",
(threshold,)
)
if dead_processors:
self.config.logger.info("found dead processor(s):")
for a_dead_processor in dead_processors:
self.config.logger.info("%d is dead", a_dead_processor[0])
self.config.logger.debug("getting list of live processor(s):")
live_processors = execute_query_fetchall(
connection,
"select id from processors where lastSeenDateTime >= %s",
(threshold,)
)
if not live_processors:
if self.config.registrar.quit_if_no_processors:
raise NoProcessorsRegisteredError(
"There are no processors registered"
)
else:
self.config.logger.critical(
'There are no live processors, nothing to do. '
'Waiting for processors to come on line.'
)
return
number_of_live_processors = len(live_processors)
self.config.logger.debug(
"getting range of queued date for jobs associated with "
"dead processor(s):"
)
dead_processor_ids_str = ", ".join(
[str(x[0]) for x in dead_processors]
)
earliest_dead_job, latest_dead_job = single_row_sql(
connection,
"select min(queueddatetime), max(queueddatetime) from jobs "
"where owner in (%s)" % dead_processor_ids_str
)
# take dead processor jobs and reallocate them to live
# processors in equal sized chunks
if (earliest_dead_job is not None and
latest_dead_job is not None):
time_increment = (
(latest_dead_job - earliest_dead_job) /
number_of_live_processors
)
for x, live_processor_id in enumerate(live_processors):
low_queued_time = (
x * time_increment + earliest_dead_job
)
high_queued_time = (
(x + 1) * time_increment + earliest_dead_job
)
self.config.logger.info(
"assigning jobs from %s to %s to processor %s:",
low_queued_time,
high_queued_time,
live_processor_id
)
# why is the range >= at both ends? the range must be
# inclusive, the risk of moving a job twice is low and
# consequences low, too.
# 1st step: take any jobs of a dead processor that were
# in progress and reset them to unprocessed
execute_no_results(
connection,
"update jobs set"
" starteddatetime = NULL "
"where"
" %%s >= queueddatetime"
" and queueddatetime >= %%s"
" and owner in (%s)"
" and success is NULL" % dead_processor_ids_str,
(high_queued_time, low_queued_time)
)
# 2nd step: take all jobs of a dead processor and give
# them to a new owner
execute_no_results(
connection,
"update jobs set"
" set owner = %%s "
"where"
" %%s >= queueddatetime"
" and queueddatetime >= %%s"
" and owner in (%s)" % dead_processor_ids_str,
(live_processor_id, high_queued_time,
low_queued_time)
)
# transfer stalled priority jobs to new processors
for dead_processor_tuple in dead_processors:
self.config.logger.info(
"re-assigning priority jobs from processor %d:",
dead_processor_tuple[0]
)
execute_no_results(
connection,
"insert into priorityjobs (uuid) select uuid "
"from priority_jobs_%d" % dead_processor_tuple
)
self.config.logger.info("removing all dead processors")
execute_no_results(
connection,
"delete from processors where lastSeenDateTime < %s",
(threshold,)
)
# remove dead processors' priority tables
for a_dead_processor in dead_processors:
execute_no_results(
connection,
"drop table if exists priority_jobs_%d" %
a_dead_processor[0]
)
except NoProcessorsRegisteredError:
self.quit = True
self.config.logger.critical('there are no live processors')
def inner_transaction(
self,
connection,
bug_id,
status,
resolution,
short_desc,
signature_set
):
self.config.logger.debug(
"bug %s (%s, %s) %s: %s",
bug_id, status, resolution, short_desc, signature_set)
if not signature_set:
execute_no_results(
connection,
"DELETE FROM bugs WHERE id = %s",
(bug_id,)
)
return
useful = False
insert_made = False
try:
status_db, resolution_db, short_desc_db = single_row_sql(
connection,
"""SELECT status, resolution, short_desc
FROM bugs
WHERE id = %s""",
(bug_id,)
)
if (status_db != status
or resolution_db != resolution
or short_desc_db != short_desc):
execute_no_results(
connection,
"""
UPDATE bugs SET
status = %s, resolution = %s, short_desc = %s
WHERE id = %s""",
(status, resolution, short_desc, bug_id)
)
self.config.logger.info(
"bug status updated: %s - %s, %s",
bug_id,
status,
resolution
)
useful = True
signature_rows = execute_query_fetchall(
connection,
"SELECT signature FROM bug_associations WHERE bug_id = %s",
(bug_id,)
)
signatures_db = [x[0] for x in signature_rows]
for signature in signatures_db:
if signature not in signature_set:
execute_no_results(
connection,
"""
DELETE FROM bug_associations
WHERE signature = %s and bug_id = %s""",
(signature, bug_id)
)
self.config.logger.info(
'association removed: %s - "%s"',
bug_id, signature)
useful = True
except SQLDidNotReturnSingleRow:
execute_no_results(
connection,
"""
INSERT INTO bugs
(id, status, resolution, short_desc)
VALUES (%s, %s, %s, %s)""",
(bug_id, status, resolution, short_desc)
)
insert_made = True
signatures_db = []
for signature in signature_set:
if signature not in signatures_db:
if self._has_signature_report(signature, connection):
execute_no_results(
connection,
"""
INSERT INTO bug_associations (signature, bug_id)
VALUES (%s, %s)""",
(signature, bug_id)
)
self.config.logger.info(
'new association: %s - "%s"',
bug_id,
signature
)
useful = True
else:
self.config.logger.info(
'rejecting association (no reports with this '
'signature): %s - "%s"',
bug_id,
signature
)
if useful:
if insert_made:
self.config.logger.info(
'new bug: %s - %s, %s, "%s"',
bug_id,
status,
resolution,
short_desc
)
else:
if insert_made:
self.config.logger.info(
'rejecting bug (no useful information): '
'%s - %s, %s, "%s"',
bug_id, status, resolution, short_desc)
else:
self.config.logger.info(
'skipping bug (no new information): '
'%s - %s, %s, "%s"',
bug_id,
status,
resolution,
short_desc
)
raise NothingUsefulHappened('nothing useful done')
def test_basic_run_job(self, rget):
config_manager = self._setup_config_manager()
def mocked_get(url):
if 'firefox_versions.json' in url:
return Response({
'FIREFOX_NIGHTLY': '52.0a1',
# Kept for legacy and smooth transition.
# We USED to consider the latest AURORA version a
# featured version but we no longer build aurora
# so Socorro shouldn't pick this up any more
# even if product-details.mozilla.org supplies it.
'FIREFOX_AURORA': '51.0a2',
'FIREFOX_ESR': '45.4.0esr',
'FIREFOX_ESR_NEXT': '',
'LATEST_FIREFOX_DEVEL_VERSION': '50.0b7',
'LATEST_FIREFOX_OLDER_VERSION': '3.6.28',
'LATEST_FIREFOX_RELEASED_DEVEL_VERSION': '50.0b7',
'LATEST_FIREFOX_VERSION': '49.0.1'
})
elif 'mobile_versions.json' in url:
return Response({
'nightly_version': '52.0a1',
'alpha_version': '51.0a2',
'beta_version': '50.0b6',
'version': '49.0',
'ios_beta_version': '6.0',
'ios_version': '5.0'
})
elif 'thunderbird_versions.json' in url:
return Response({
'LATEST_THUNDERBIRD_VERSION': '45.4.0',
'LATEST_THUNDERBIRD_DEVEL_VERSION': '50.0b1',
'LATEST_THUNDERBIRD_ALPHA_VERSION': '51.0a2',
'LATEST_THUNDERBIRD_NIGHTLY_VERSION': '52.0a1',
})
else:
raise NotImplementedError(url)
rget.side_effect = mocked_get
# Check what's set up in the fixture
rows = execute_query_fetchall(
self.conn,
'select product_name, version_string, featured_version '
'from product_versions order by version_string'
)
assert sorted(rows) == [
('Firefox', '15.0a1', True),
('Firefox', '24.5.0', True),
('Firefox', '49.0.1', False),
('Firefox', '50.0b', False),
('Firefox', '51.0a2', False),
('Firefox', '52.0a1', False),
]
# This is necessary so we get a new cursor when we do other
# selects after the crontabber app has run.
self.conn.commit()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['featured-versions-automatic']
assert not information['featured-versions-automatic']['last_error']
assert information['featured-versions-automatic']['last_success']
config.logger.info.assert_called_with(
'Set featured versions for Thunderbird to: '
'45.4.0, 50.0b1, 52.0a1'
)
rows = execute_query_fetchall(
self.conn,
'select product_name, version_string, featured_version '
'from product_versions'
)
eq_(
sorted(rows),
[
('Firefox', '15.0a1', False),
('Firefox', '24.5.0', False),
('Firefox', '49.0.1', True),
('Firefox', '50.0b', True),
# Note that the 'Aurora' branch is still mentioned but
# note that it's NOT featured (hence 'False').
('Firefox', '51.0a2', False),
('Firefox', '52.0a1', True),
]
)
def _get_priority_jobs_transaction(self, connection):
"""this method implements a single transaction that just returns a
set of priority jobs."""
priority_jobs_list = execute_query_fetchall(
connection, "select * from priorityjobs")
return set(x[0] for x in priority_jobs_list)
def _sweep_dead_processors_transaction(self, connection):
"""this function is a single database transaction: look for dead
processors - find all the jobs of dead processors and assign them to
live processors then delete the dead processor registrations"""
self.config.logger.info("looking for dead processors")
try:
self.config.logger.info("threshold %s",
self.config.registrar.check_in_frequency)
threshold = single_value_sql(
connection, "select now() - %s - %s",
(self.config.registrar.processor_grace_period,
self.config.registrar.check_in_frequency))
dead_processors = execute_query_fetchall(
connection,
"select id from processors where lastSeenDateTime < %s",
(threshold, ))
if dead_processors:
self.config.logger.info("found dead processor(s):")
for a_dead_processor in dead_processors:
self.config.logger.info("%d is dead", a_dead_processor[0])
self.config.logger.debug("getting list of live processor(s):")
live_processors = execute_query_fetchall(
connection,
"select id from processors where lastSeenDateTime >= %s",
(threshold, ))
if not live_processors:
if self.config.registrar.quit_if_no_processors:
raise NoProcessorsRegisteredError(
"There are no processors registered")
else:
self.config.logger.critical(
'There are no live processors, nothing to do. '
'Waiting for processors to come on line.')
return
number_of_live_processors = len(live_processors)
self.config.logger.debug(
"getting range of queued date for jobs associated with "
"dead processor(s):")
dead_processor_ids_str = ", ".join(
[str(x[0]) for x in dead_processors])
earliest_dead_job, latest_dead_job = single_row_sql(
connection,
"select min(queueddatetime), max(queueddatetime) from jobs "
"where owner in (%s)" % dead_processor_ids_str)
# take dead processor jobs and reallocate them to live
# processors in equal sized chunks
if (earliest_dead_job is not None
and latest_dead_job is not None):
time_increment = ((latest_dead_job - earliest_dead_job) /
number_of_live_processors)
for x, live_processor_id in enumerate(live_processors):
low_queued_time = (x * time_increment +
earliest_dead_job)
high_queued_time = ((x + 1) * time_increment +
earliest_dead_job)
self.config.logger.info(
"assigning jobs from %s to %s to processor %s:",
low_queued_time, high_queued_time,
live_processor_id)
# why is the range >= at both ends? the range must be
# inclusive, the risk of moving a job twice is low and
# consequences low, too.
# 1st step: take any jobs of a dead processor that were
# in progress and reset them to unprocessed
execute_no_results(
connection, "update jobs set"
" starteddatetime = NULL "
"where"
" %%s >= queueddatetime"
" and queueddatetime >= %%s"
" and owner in (%s)"
" and success is NULL" % dead_processor_ids_str,
(high_queued_time, low_queued_time))
# 2nd step: take all jobs of a dead processor and give
# them to a new owner
execute_no_results(
connection, "update jobs set"
" set owner = %%s "
"where"
" %%s >= queueddatetime"
" and queueddatetime >= %%s"
" and owner in (%s)" % dead_processor_ids_str,
(live_processor_id, high_queued_time,
low_queued_time))
# transfer stalled priority jobs to new processors
for dead_processor_tuple in dead_processors:
self.config.logger.info(
"re-assigning priority jobs from processor %d:",
dead_processor_tuple[0])
execute_no_results(
connection,
"insert into priorityjobs (uuid) select uuid "
"from priority_jobs_%d" % dead_processor_tuple)
self.config.logger.info("removing all dead processors")
execute_no_results(
connection,
"delete from processors where lastSeenDateTime < %s",
(threshold, ))
# remove dead processors' priority tables
for a_dead_processor in dead_processors:
execute_no_results(
connection, "drop table if exists priority_jobs_%d" %
a_dead_processor[0])
except NoProcessorsRegisteredError:
self.quit = True
self.config.logger.critical('there are no live processors')
def test_basic_run_job(self, rget):
config_manager = self._setup_config_manager()
def mocked_get(url):
if 'firefox_versions.json' in url:
return Response({
'FIREFOX_NIGHTLY': '52.0a1',
'FIREFOX_AURORA': '51.0a2',
'FIREFOX_ESR': '45.4.0esr',
'FIREFOX_ESR_NEXT': '',
'LATEST_FIREFOX_DEVEL_VERSION': '50.0b7',
'LATEST_FIREFOX_OLDER_VERSION': '3.6.28',
'LATEST_FIREFOX_RELEASED_DEVEL_VERSION': '50.0b7',
'LATEST_FIREFOX_VERSION': '49.0.1'
})
elif 'mobile_versions.json' in url:
return Response({
'nightly_version': '52.0a1',
'alpha_version': '51.0a2',
'beta_version': '50.0b6',
'version': '49.0',
'ios_beta_version': '6.0',
'ios_version': '5.0'
})
elif 'thunderbird_versions.json' in url:
return Response({
'LATEST_THUNDERBIRD_VERSION': '45.4.0',
'LATEST_THUNDERBIRD_DEVEL_VERSION': '50.0b1',
'LATEST_THUNDERBIRD_ALPHA_VERSION': '51.0a2',
'LATEST_THUNDERBIRD_NIGHTLY_VERSION': '52.0a1',
})
else:
raise NotImplementedError(url)
rget.side_effect = mocked_get
# Check what's set up in the fixture
rows = execute_query_fetchall(
self.conn,
'select product_name, version_string, featured_version '
'from product_versions order by version_string'
)
assert sorted(rows) == [
('Firefox', '15.0a1', True),
('Firefox', '24.5.0', True),
('Firefox', '49.0.1', False),
('Firefox', '50.0b', False),
('Firefox', '51.0a2', False),
('Firefox', '52.0a1', False),
]
# This is necessary so we get a new cursor when we do other
# selects after the crontabber app has run.
self.conn.commit()
with config_manager.context() as config:
tab = CronTabber(config)
tab.run_all()
information = self._load_structure()
assert information['featured-versions-automatic']
assert not information['featured-versions-automatic']['last_error']
assert information['featured-versions-automatic']['last_success']
config.logger.info.assert_called_with(
'Set featured versions for Thunderbird to: '
'45.4.0, 50.0b1, 51.0a2, 52.0a1'
)
rows = execute_query_fetchall(
self.conn,
'select product_name, version_string, featured_version '
'from product_versions'
)
eq_(
sorted(rows),
[
('Firefox', '15.0a1', False),
('Firefox', '24.5.0', False),
('Firefox', '49.0.1', True),
('Firefox', '50.0b', True),
('Firefox', '51.0a2', True),
('Firefox', '52.0a1', True),
]
)