def _save_plugins(self, connection, processed_crash, report_id):
""" Electrolysis Support - Optional - processed_crash may contain a
ProcessType of plugin. In the future this value would be default,
content, maybe even Jetpack... This indicates which process was the
crashing process.
plugin - When set to plugin, the jsonDocument MUST calso contain
PluginFilename and PluginName
"""
process_type = processed_crash['process_type']
if not process_type:
return
if process_type == "plugin":
# Bug#543776 We actually will are relaxing the non-null policy...
# a null filename, name, and version is OK. We'll use empty strings
try:
plugin_filename = processed_crash['PluginFilename']
plugin_name = processed_crash['PluginName']
except KeyError as x:
self.config.logger.error(
'the crash is missing a required field: %s', str(x))
return
find_plugin_sql = ('select id from plugins '
'where filename = %s '
'and name = %s')
try:
single_value_sql(connection, find_plugin_sql,
(plugin_filename, plugin_name))
except SQLDidNotReturnSingleValue:
insert_plugsins_sql = ("insert into plugins (filename, name) "
"values (%s, %s) returning id")
execute_no_results(connection, insert_plugsins_sql,
(plugin_filename, plugin_name))
def _assume_identity_by_host(self, connection, threshold, hostname,
req_id):
"""This function implements the case where a newly registering
processor wants to take over for a dead processor with the same host
name as the registering processor.
Parameters:
connection - a connection object
threshold - a datetime instance that represents an absolute date
made from the current datetime minus the timedelta
that defines how often a processor must update its
registration. If the threshold is greater than the
'lastseendatetime' of a registered processor, that
processor is considered dead.
hostname - the name of the host of the registering processor.
req_id - not used by this method, but present to meet the required
api for a registration method.
Returns:
an integer representing the new id of the newly registered
processor."""
self.config.logger.debug(
"looking for a dead processor for host %s",
hostname
)
try:
sql = ("select id from processors"
" where lastseendatetime < %s"
" and name like %s limit 1")
hostname_phrase = hostname + '%'
processor_id = single_value_sql(
connection,
sql,
(threshold, hostname_phrase)
)
self.config.logger.info(
"will step in for processor %s",
processor_id
)
# a dead processor for this host was found
self._take_over_dead_processor(connection, processor_id)
return processor_id
except SQLDidNotReturnSingleValue:
# no dead processor was found for this host, is there already
# a live processor for it?
self.config.logger.debug("no dead processor found for host, %s",
hostname)
try:
sql = ("select id from processors"
" where name like '%s%%'" % hostname)
self.processor_id = single_value_sql(connection, sql)
message = ('a live processor already exists for host %s' %
hostname)
# there was a live processor found for this host, we cannot
# complete registration
raise RegistrationError(message)
except SQLDidNotReturnSingleValue:
# there was no processor for this host was found, make new one
return self._assume_new_identity(connection, threshold,
hostname, req_id)
def _assume_identity_by_host(self, connection, threshold, hostname,
req_id):
"""This function implements the case where a newly registering
processor wants to take over for a dead processor with the same host
name as the registering processor.
Parameters:
connection - a connection object
threshold - a datetime instance that represents an absolute date
made from the current datetime minus the timedelta
that defines how often a processor must update its
registration. If the threshold is greater than the
'lastseendatetime' of a registered processor, that
processor is considered dead.
hostname - the name of the host of the registering processor.
req_id - not used by this method, but present to meet the required
api for a registration method.
Returns:
an integer representing the new id of the newly registered
processor."""
self.config.logger.debug(
"looking for a dead processor for host %s",
hostname
)
try:
sql = ("select id from processors"
" where lastseendatetime < %s"
" and name like %s limit 1")
hostname_phrase = hostname + '%'
processor_id = single_value_sql(
connection,
sql,
(threshold, hostname_phrase)
)
self.config.logger.info(
"will step in for processor %s",
processor_id
)
# a dead processor for this host was found
self._take_over_dead_processor(connection, processor_id)
return processor_id
except SQLDidNotReturnSingleValue:
# no dead processor was found for this host, is there already
# a live processor for it?
self.config.logger.debug("no dead processor found for host, %s",
hostname)
try:
sql = ("select id from processors"
" where name like '%s%%'" % hostname)
self.processor_id = single_value_sql(connection, sql)
message = ('a live processor already exists for host %s' %
hostname)
# there was a live processor found for this host, we cannot
# complete registration
raise RegistrationError(message)
except SQLDidNotReturnSingleValue:
# there was no processor for this host was found, make new one
return self._assume_new_identity(connection, threshold,
hostname, req_id)
def _prioritize_previously_enqueued_jobs_transaction(self, connection,
crash_id):
"""priorty jobs come into the system at random times. A given crash_id
may already be queued for processing when a priority request comes in
for it. To avoid repeating processing, a priority crash_id is checked
to see if it is already queued. If it is, the processor already
assigned to it is told to expedite processing. This done just by
entering the crash_id into the processors private 'priority_jobs_XXX'
table."""
try:
job_owner = single_value_sql(
connection,
"select owner from jobs where uuid = %s",
(crash_id,)
)
except SQLDidNotReturnSingleValue:
return False
priority_job_table_name = 'priority_jobs_%d' % job_owner
self.config.logger.debug(
"priority job %s was already in the queue, assigned to %d",
crash_id,
job_owner
)
try:
# detect if the found job was assigned to a processor that was
# dead by checking to see if the priority jobs table exists or
# not. If id doesn't exist, wait for the job to get reassigned
# to a live processor. It in the future, it may be better to
# just reassign the job immediately.
single_value_sql( # return value intentionally ignored
connection,
"select 1 from pg_stat_user_tables where relname = %s",
(priority_job_table_name,)
)
except SQLDidNotReturnSingleValue:
self.config.logger.debug(
"%s assigned to dead processor %d - "
"wait for reassignment",
crash_id,
job_owner
)
# likely that the job is assigned to a dead processor
# skip processing it this time around - by next time
# hopefully it will have been
# re assigned to a live processor
return False
execute_no_results(
connection,
"insert into %s (uuid) values (%%s)" %
priority_job_table_name,
(crash_id,)
)
execute_no_results(
connection,
"delete from priorityjobs where uuid = %s",
(crash_id,)
)
return True
def _save_plugins(self, connection, processed_crash, report_id):
""" Electrolysis Support - Optional - processed_crash may contain a
ProcessType of plugin. In the future this value would be default,
content, maybe even Jetpack... This indicates which process was the
crashing process.
plugin - When set to plugin, the jsonDocument MUST calso contain
PluginFilename, PluginName, and PluginVersion
"""
process_type = processed_crash['process_type']
if not process_type:
return
if process_type == "plugin":
# Bug#543776 We actually will are relaxing the non-null policy...
# a null filename, name, and version is OK. We'll use empty strings
try:
plugin_filename = processed_crash['PluginFilename']
plugin_name = processed_crash['PluginName']
plugin_version = processed_crash['PluginVersion']
except KeyError, x:
self.config.logger.error(
'the crash is missing a required field: %s', str(x)
)
return
find_plugin_sql = ('select id from plugins '
'where filename = %s '
'and name = %s')
try:
plugin_id = single_value_sql(connection,
find_plugin_sql,
(plugin_filename,
plugin_name))
except SQLDidNotReturnSingleValue:
insert_plugsins_sql = ("insert into plugins (filename, name) "
"values (%s, %s) returning id")
plugin_id = single_value_sql(connection,
insert_plugsins_sql,
(plugin_filename,
plugin_name))
crash_id = processed_crash['uuid']
table_suffix = self._table_suffix_for_crash_id(crash_id)
plugin_reports_table_name = 'plugins_reports_%s' % table_suffix
plugins_reports_insert_sql = (
'insert into %s '
' (report_id, plugin_id, date_processed, version) '
'values '
' (%%s, %%s, %%s, %%s)' % plugin_reports_table_name
)
values_tuple = (report_id,
plugin_id,
processed_crash['date_processed'],
plugin_version)
execute_no_results(connection,
plugins_reports_insert_sql,
values_tuple)
def test_single_value_sql2(self):
m_execute = Mock()
m_fetchall = Mock(return_value=((17,),))
m_cursor = Mock()
m_cursor.execute = m_execute
m_cursor.fetchall = m_fetchall
conn = Mock()
conn.cursor.return_value = m_cursor
dbapi2_util.single_value_sql(conn, "select 17", (1, 2, 3))
eq_(conn.cursor.call_count, 1)
eq_(m_cursor.execute.call_count, 1)
m_cursor.execute.assert_called_once_with('select 17', (1, 2, 3))
def test_single_value_sql2(self):
m_execute = Mock()
m_fetchall = Mock(return_value=((17, ), ))
m_cursor = Mock()
m_cursor.execute = m_execute
m_cursor.fetchall = m_fetchall
conn = Mock()
conn.cursor.return_value = m_cursor
dbapi2_util.single_value_sql(conn, "select 17", (1, 2, 3))
eq_(conn.cursor.call_count, 1)
eq_(m_cursor.execute.call_count, 1)
m_cursor.execute.assert_called_once_with('select 17', (1, 2, 3))
def _save_processed_report(self, connection, processed_crash):
column_list = []
placeholder_list = []
value_list = []
for pro_crash_name, report_name in self._reports_table_mappings:
column_list.append(report_name)
placeholder_list.append('%s')
value_list.append(processed_crash[pro_crash_name])
crash_id = processed_crash['uuid']
reports_table_name = (
'reports_%s' % self._table_suffix_for_crash_id(crash_id)
)
insert_sql = "insert into %s (%s) values (%s) returning id" % (
reports_table_name,
', '.join(column_list),
', '.join(placeholder_list)
)
# we want to insert directly into the report table. There is a
# chance however that the record already exists. If it does, then
# the insert would fail and the connection fall into a "broken" state.
# To avoid this, we set a savepoint to which we can roll back if the
# record already exists - essentially a nested transaction.
# We use the name of the executing thread as the savepoint name.
# alternatively we could get a uuid.
savepoint_name = threading.currentThread().getName().replace('-', '')
execute_no_results(connection, "savepoint %s" % savepoint_name)
try:
report_id = single_value_sql(connection, insert_sql, value_list)
execute_no_results(
connection,
"release savepoint %s" % savepoint_name
)
except self.config.database_class.IntegrityError:
# report already exists
execute_no_results(
connection,
"rollback to savepoint %s" % savepoint_name
)
execute_no_results(
connection,
"release savepoint %s" % savepoint_name
)
execute_no_results(
connection,
"delete from %s where uuid = %%s" % reports_table_name,
(processed_crash.uuid,)
)
report_id = single_value_sql(connection, insert_sql, value_list)
return report_id
def test_single_value_sql3(self):
m_execute = Mock()
m_fetchall = Mock(return_value=None)
m_cursor = MagicMock()
m_cursor.execute = m_execute
m_cursor.fetchall = m_fetchall
conn = MagicMock()
conn.cursor.return_value.__enter__.return_value = m_cursor
with pytest.raises(dbapi2_util.SQLDidNotReturnSingleValue):
dbapi2_util.single_value_sql(conn, 'select 17', (1, 2, 3))
assert conn.cursor.call_count == 1
assert m_cursor.execute.call_count == 1
m_cursor.execute.assert_called_once_with('select 17', (1, 2, 3))
def __delitem__(self, connection, key):
"""remove the item by key or raise KeyError"""
try:
# result intentionally ignored
single_value_sql(
connection, """SELECT app_name
FROM crontabber
WHERE
app_name = %s""", (key, ))
except SQLDidNotReturnSingleValue:
raise KeyError(key)
# item exists
execute_no_results(
connection, """DELETE FROM crontabber
WHERE app_name = %s""", (key, ))
def test_single_value_sql3(self):
m_execute = Mock()
m_fetchall = Mock(return_value=None)
m_cursor = MagicMock()
m_cursor.execute = m_execute
m_cursor.fetchall = m_fetchall
conn = MagicMock()
conn.cursor.return_value.__enter__.return_value = m_cursor
with pytest.raises(dbapi2_util.SQLDidNotReturnSingleValue):
dbapi2_util.single_value_sql(conn, 'select 17', (1, 2, 3))
assert conn.cursor.call_count == 1
assert m_cursor.execute.call_count == 1
m_cursor.execute.assert_called_once_with('select 17', (1, 2, 3))
def _assume_new_identity(self, connection, threshold, hostname, req_id):
"""This function implements the method of registering a brand new
processor. It will cause a new row to be entered into the 'processors'
table within the database.
Parameters:
cursor - a cursor object
threshold - not used by this method, but present to meet the
required api for a registration method.
hostname - the name of the host of the registering processor.
not used by the method, but present to meet the
required api for a registration method.
req_id - not used by this method, but present to meet the required
api for a registration method.
Returns:
an integer representing the new id of the newly registered
processor."""
self.config.logger.debug("becoming a new processor")
return single_value_sql(
connection,
"insert into processors"
" (id,"
" name,"
" startdatetime,"
" lastseendatetime) "
"values"
" (default,"
" %s,"
" now(),"
" now()) "
"returning id",
(self.processor_name,),
)
def _assume_specific_identity(self, connection, threshold, hostname, req_id):
"""This function implements the case where we want the processor to
take over for a specific existing but dead processor without regard
to what host the dead processor was running on. If the dead processor
was not found, or the processor was not really dead, the function will
raise a RegistrationError and decline to register the new processor.
Parameters:
connection - a connection object
threshold - a datetime instance that represents an absolute date
made from the current datetime minus the timedelta
that defines how often a processor must update its
registration. If the threshold is greater than the
'lastseendatetime' of a registered processor, that
processor is considered dead.
hostname - the name of the host of the registering processor.
not used by the method, but present to meet the
required api for a registration method.
req_id - an integer representing the 'id' (from the 'id' column of
'processors' database table) of the allegedly dead
processor.
Returns:
an integer representing the new id of the newly registered
processor."""
self.config.logger.debug("looking for a specific dead processor")
try:
check_sql = "select id from processors " "where lastSeenDateTime < %s " "and id = %s"
processor_id = single_value_sql(connection, check_sql, (threshold, req_id))
self.config.logger.info("stepping in for processor %s", processor_id)
self._take_over_dead_processor(connection, processor_id)
return processor_id
except SQLDidNotReturnSingleValue:
raise RegistrationError("%s doesn't exist or is not dead" % req_id)
def _assume_any_identity(self, connection, threshold, hostname, req_id):
"""This function implements the case where we're interested in taking
over for any dead processor regardless of what host it was running on.
Parameters:
connection - a connection object
threshold - a datetime instance that represents an absolute date
made from the current datetime minus the timedelta
that defines how often a processor must update its
registration. If the threshold is greater than the
'lastseendatetime' of a registered processor, that
processor is considered dead.
hostname - the name of the host of the registering processor.
not used by the method, but present to meet the
required api for a registration method.
req_id - not used by this method, but present to meet the required
api for a registration method.
Returns:
an integer representing the new id of the newly registered
processor."""
self.config.logger.debug("looking for any dead processor")
try:
sql = "select id from processors" " where lastseendatetime < %s limit 1"
processor_id = single_value_sql(connection, sql, (threshold,))
self.config.logger.info("will step in for processor %s", processor_id)
self._take_over_dead_processor(connection, processor_id)
return processor_id
except SQLDidNotReturnSingleValue:
self.config.logger.debug("no dead processor found, registering as new")
return self._assume_new_identity(connection, threshold, hostname, req_id)
def _force_assume_identity_by_host(self, connection, threshold, hostname, req_id):
"""This function implements the case where a newly registering
processor wants to take over for a processor with the same host
name as the registering processor. This is the case where the
existing processor is likely dead but didn't manage to halt cleanly.
Parameters:
connection - a connection object
threshold - a datetime instance that represents an absolute date
made from the current datetime minus the timedelta
that defines how often a processor must update its
registration. If the threshold is greater than the
'lastseendatetime' of a registered processor, that
processor is considered dead.
hostname - the name of the host of the registering processor.
req_id - not used by this method, but present to meet the required
api for a registration method.
Returns:
an integer representing the new id of the newly registered
processor."""
self.config.logger.debug("looking for a processor for host %s", hostname)
try:
sql = "select id from processors" " where name like %s limit 1"
hostname_phrase = hostname + "%"
processor_id = single_value_sql(connection, sql, (hostname_phrase,))
self.config.logger.info("will take over processor %s", processor_id)
# a processor for this host was found
self._take_over_dead_processor(connection, processor_id)
return processor_id
except SQLDidNotReturnSingleValue:
return self._assume_new_identity(connection, threshold, hostname, req_id)
def _get_processed_crash_transaction(self, connection, crash_id):
fetch_sql = (
'select processed_crash from processed_crashes where uuid = %s')
try:
return single_value_sql(connection, fetch_sql, (crash_id, ))
except SQLDidNotReturnSingleValue:
raise CrashIDNotFound(crash_id)
def _assume_new_identity(self, connection, threshold, hostname, req_id):
"""This function implements the method of registering a brand new
processor. It will cause a new row to be entered into the 'processors'
table within the database.
Parameters:
cursor - a cursor object
threshold - not used by this method, but present to meet the
required api for a registration method.
hostname - the name of the host of the registering processor.
not used by the method, but present to meet the
required api for a registration method.
req_id - not used by this method, but present to meet the required
api for a registration method.
Returns:
an integer representing the new id of the newly registered
processor."""
self.config.logger.debug("becoming a new processor")
return single_value_sql(
connection, "insert into processors"
" (id,"
" name,"
" startdatetime,"
" lastseendatetime) "
"values"
" (default,"
" %s,"
" now(),"
" now()) "
"returning id", (self.processor_name, ))
def _assume_any_identity(self, connection, threshold, hostname, req_id):
"""This function implements the case where we're interested in taking
over for any dead processor regardless of what host it was running on.
Parameters:
connection - a connection object
threshold - a datetime instance that represents an absolute date
made from the current datetime minus the timedelta
that defines how often a processor must update its
registration. If the threshold is greater than the
'lastseendatetime' of a registered processor, that
processor is considered dead.
hostname - the name of the host of the registering processor.
not used by the method, but present to meet the
required api for a registration method.
req_id - not used by this method, but present to meet the required
api for a registration method.
Returns:
an integer representing the new id of the newly registered
processor."""
self.config.logger.debug("looking for any dead processor")
try:
sql = ("select id from processors"
" where lastseendatetime < %s limit 1")
processor_id = single_value_sql(connection, sql, (threshold, ))
self.config.logger.info("will step in for processor %s",
processor_id)
self._take_over_dead_processor(connection, processor_id)
return processor_id
except SQLDidNotReturnSingleValue:
self.config.logger.debug(
"no dead processor found, registering as new")
return self._assume_new_identity(connection, threshold, hostname,
req_id)
def _prioritize_previously_enqueued_jobs_transaction(
self, connection, crash_id):
"""priorty jobs come into the system at random times. A given crash_id
may already be queued for processing when a priority request comes in
for it. To avoid repeating processing, a priority crash_id is checked
to see if it is already queued. If it is, the processor already
assigned to it is told to expedite processing. This done just by
entering the crash_id into the processors private 'priority_jobs_XXX'
table."""
try:
job_owner = single_value_sql(
connection, "select owner from jobs where uuid = %s",
(crash_id, ))
except SQLDidNotReturnSingleValue:
return False
priority_job_table_name = 'priority_jobs_%d' % job_owner
self.config.logger.debug(
"priority job %s was already in the queue, assigned to %d",
crash_id, job_owner)
try:
# detect if the found job was assigned to a processor that was
# dead by checking to see if the priority jobs table exists or
# not. If id doesn't exist, wait for the job to get reassigned
# to a live processor. It in the future, it may be better to
# just reassign the job immediately.
single_value_sql( # return value intentionally ignored
connection,
"select 1 from pg_stat_user_tables where relname = %s",
(priority_job_table_name, ))
except SQLDidNotReturnSingleValue:
self.config.logger.debug(
"%s assigned to dead processor %d - "
"wait for reassignment", crash_id, job_owner)
# likely that the job is assigned to a dead processor
# skip processing it this time around - by next time
# hopefully it will have been
# re assigned to a live processor
return False
execute_no_results(
connection,
"insert into %s (uuid) values (%%s)" % priority_job_table_name,
(crash_id, ))
execute_no_results(connection,
"delete from priorityjobs where uuid = %s",
(crash_id, ))
return True
def _get_processed_crash_transaction(self, connection, crash_id):
fetch_sql = (
'select processed_crash from processed_crashes where uuid = %s'
)
try:
return single_value_sql(connection, fetch_sql, (crash_id,))
except SQLDidNotReturnSingleValue:
raise CrashIDNotFound(crash_id)
def _get_raw_crash_transaction(self, connection, crash_id):
raw_crash_table_name = ('raw_crash_%s' %
self._table_suffix_for_crash_id(crash_id))
fetch_sql = 'select raw_crash from %s where uuid = %ss' % \
raw_crash_table_name
try:
return single_value_sql(connection, fetch_sql, (crash_id, ))
except SQLDidNotReturnSingleValue:
raise CrashIDNotFound(crash_id)
def _get_raw_crash_transaction(self, connection, crash_id):
raw_crash_table_name = (
'raw_crash_%s' % self._table_suffix_for_crash_id(crash_id)
)
fetch_sql = 'select raw_crash from %s where uuid = %ss' % \
raw_crash_table_name
try:
return single_value_sql(connection, fetch_sql, (crash_id,))
except SQLDidNotReturnSingleValue:
raise CrashIDNotFound(crash_id)
def _create_priority_jobs(self, connection):
self.processor_id = single_value_sql(
connection, "select id from processors where name = %s", (self.processor_name,)
)
priority_jobs_table_name = "priority_jobs_%d" % self.processor_id
execute_no_results(connection, "drop table if exists %s" % priority_jobs_table_name)
execute_no_results(
connection, "create table %s (uuid varchar(50) not null primary key)" % priority_jobs_table_name
)
self.config.logger.info("created priority jobs table: %s", priority_jobs_table_name)
return priority_jobs_table_name
def _load_from_postgres(self, connection, file_path):
try:
json_dump = single_value_sql(
connection,
'SELECT state FROM crontabber_state'
)
if json_dump != '{}':
with open(file_path, 'w') as f:
f.write(json_dump)
except SQLDidNotReturnSingleValue:
pass
def _get_processed_crash_transaction(self, connection, crash_id):
processed_crash_table_name = (
'processed_crashes_%s' % self._table_suffix_for_crash_id(crash_id))
fetch_sql = 'select processed_crash from %s where uuid = %%s' % \
processed_crash_table_name
try:
return single_value_sql(connection, fetch_sql, (crash_id, ))
except ProgrammingError, e:
err = 'relation "%s" does not exist' % processed_crash_table_name
if err in str(e):
raise CrashIDNotFound(crash_id)
raise
def __delitem__(self, connection, key):
"""remove the item by key or raise KeyError"""
try:
# result intentionally ignored
single_value_sql(
connection,
"""SELECT app_name
FROM crontabber
WHERE
app_name = %s""",
(key,)
)
except SQLDidNotReturnSingleValue:
raise KeyError(key)
# item exists
execute_no_results(
connection,
"""DELETE FROM crontabber
WHERE app_name = %s""",
(key,)
)
def edit_featured_versions(self, connection, product, versions):
sql = """
SELECT
edit_featured_versions(%s, {})
""".format(','.join('%s' for _ in versions))
worked = single_value_sql(connection, sql, [product] + versions)
if worked:
self.config.logger.info('Set featured versions for %s %r' %
(product, versions))
else:
self.config.logger.warning(
'Unable to set featured versions for %s %r' %
(product, versions))
def _get_processed_crash_transaction(self, connection, crash_id):
processed_crash_table_name = (
'processed_crashes_%s' % self._table_suffix_for_crash_id(crash_id)
)
fetch_sql = 'select processed_crash from %s where uuid = %%s' % \
processed_crash_table_name
try:
return single_value_sql(connection, fetch_sql, (crash_id,))
except ProgrammingError, e:
err = 'relation "%s" does not exist' % processed_crash_table_name
if err in str(e):
raise CrashIDNotFound(crash_id)
raise
def _save_processed_report(self, connection, processed_crash):
column_list = []
placeholder_list = []
value_list = []
for pro_crash_name, report_name in self._reports_table_mappings:
column_list.append(report_name)
placeholder_list.append('%s')
value_list.append(processed_crash[pro_crash_name])
crash_id = processed_crash['uuid']
reports_table_name = ('reports_%s' %
self._table_suffix_for_crash_id(crash_id))
insert_sql = "insert into %s (%s) values (%s) returning id" % (
reports_table_name, ', '.join(column_list),
', '.join(placeholder_list))
# we want to insert directly into the report table. There is a
# chance however that the record already exists. If it does, then
# the insert would fail and the connection fall into a "broken" state.
# To avoid this, we set a savepoint to which we can roll back if the
# record already exists - essentially a nested transaction.
# We use the name of the executing thread as the savepoint name.
# alternatively we could get a uuid.
savepoint_name = threading.currentThread().getName().replace('-', '')
execute_no_results(connection, "savepoint %s" % savepoint_name)
try:
report_id = single_value_sql(connection, insert_sql, value_list)
execute_no_results(connection,
"release savepoint %s" % savepoint_name)
except self.config.database_class.IntegrityError:
# report already exists
execute_no_results(connection,
"rollback to savepoint %s" % savepoint_name)
execute_no_results(connection,
"release savepoint %s" % savepoint_name)
execute_no_results(
connection,
"delete from %s where uuid = %%s" % reports_table_name,
(processed_crash.uuid, ))
report_id = single_value_sql(connection, insert_sql, value_list)
return report_id
def test_single_value_sql1(self):
m_execute = Mock()
m_fetchall = Mock(return_value=((17, ), ))
m_cursor = Mock()
m_cursor.execute = m_execute
m_cursor.fetchall = m_fetchall
conn = Mock()
conn.cursor.return_value = m_cursor
r = dbapi2_util.single_value_sql(conn, "select 17")
self.assertEqual(r, 17)
self.assertEqual(conn.cursor.call_count, 1)
self.assertEqual(m_cursor.execute.call_count, 1)
m_cursor.execute.assert_called_once_with('select 17', None)
def _get_raw_crash_transaction(self, connection, crash_id):
raw_crash_table_name = ('raw_crashes_%s' %
self._table_suffix_for_crash_id(crash_id))
fetch_sql = 'select raw_crash from %s where uuid = %%s' % \
raw_crash_table_name
try:
return single_value_sql(connection, fetch_sql, (crash_id, ))
except ProgrammingError as e:
err = 'relation "%s" does not exist' % raw_crash_table_name
if err in str(e):
raise CrashIDNotFound(crash_id)
raise
except SQLDidNotReturnSingleValue:
raise CrashIDNotFound(crash_id)
def test_single_value_sql1(self):
m_execute = Mock()
m_fetchall = Mock(return_value=((17, ), ))
m_cursor = MagicMock()
m_cursor.execute = m_execute
m_cursor.fetchall = m_fetchall
conn = MagicMock()
conn.cursor.return_value.__enter__.return_value = m_cursor
r = dbapi2_util.single_value_sql(conn, "select 17")
assert r == 17
assert conn.cursor.call_count == 1
assert m_cursor.execute.call_count == 1
m_cursor.execute.assert_called_once_with('select 17', None)
def test_single_value_sql1(self):
m_execute = Mock()
m_fetchall = Mock(return_value=((17,),))
m_cursor = MagicMock()
m_cursor.execute = m_execute
m_cursor.fetchall = m_fetchall
conn = MagicMock()
conn.cursor.return_value.__enter__.return_value = m_cursor
r = dbapi2_util.single_value_sql(conn, "select 17")
eq_(r, 17)
eq_(conn.cursor.call_count, 1)
eq_(m_cursor.execute.call_count, 1)
m_cursor.execute.assert_called_once_with('select 17', None)
def _create_priority_jobs(self, connection):
self.processor_id = single_value_sql(
connection, "select id from processors where name = %s",
(self.processor_name, ))
priority_jobs_table_name = "priority_jobs_%d" % self.processor_id
execute_no_results(
connection, "drop table if exists %s" % priority_jobs_table_name)
execute_no_results(
connection,
"create table %s (uuid varchar(50) not null primary key)" %
priority_jobs_table_name)
self.config.logger.info('created priority jobs table: %s',
priority_jobs_table_name)
return priority_jobs_table_name
def _get_raw_crash_transaction(self, connection, crash_id):
raw_crash_table_name = (
'raw_crashes_%s' % self._table_suffix_for_crash_id(crash_id)
)
fetch_sql = 'select raw_crash from %s where uuid = %%s' % \
raw_crash_table_name
try:
return single_value_sql(connection, fetch_sql, (crash_id,))
except ProgrammingError as e:
err = 'relation "%s" does not exist' % raw_crash_table_name
if err in str(e):
raise CrashIDNotFound(crash_id)
raise
except SQLDidNotReturnSingleValue:
raise CrashIDNotFound(crash_id)
def _assume_specific_identity(self, connection, threshold, hostname,
req_id):
"""This function implements the case where we want the processor to
take over for a specific existing but dead processor without regard
to what host the dead processor was running on. If the dead processor
was not found, or the processor was not really dead, the function will
raise a RegistrationError and decline to register the new processor.
Parameters:
connection - a connection object
threshold - a datetime instance that represents an absolute date
made from the current datetime minus the timedelta
that defines how often a processor must update its
registration. If the threshold is greater than the
'lastseendatetime' of a registered processor, that
processor is considered dead.
hostname - the name of the host of the registering processor.
not used by the method, but present to meet the
required api for a registration method.
req_id - an integer representing the 'id' (from the 'id' column of
'processors' database table) of the allegedly dead
processor.
Returns:
an integer representing the new id of the newly registered
processor."""
self.config.logger.debug("looking for a specific dead processor")
try:
check_sql = ("select id from processors "
"where lastSeenDateTime < %s "
"and id = %s")
processor_id = single_value_sql(connection,
check_sql,
(threshold, req_id))
self.config.logger.info(
"stepping in for processor %s",
processor_id
)
self._take_over_dead_processor(connection, processor_id)
return processor_id
except SQLDidNotReturnSingleValue:
raise RegistrationError("%s doesn't exist or is not dead" %
req_id)
def edit_featured_versions(self, connection, product, versions):
sql = """
SELECT
edit_featured_versions(%s, {})
""".format(','.join('%s' for _ in versions))
worked = single_value_sql(
connection,
sql,
[product] + versions
)
if worked:
self.config.logger.info(
'Set featured versions for %s %r' % (
product,
versions
)
)
else:
self.config.logger.warning(
'Unable to set featured versions for %s %r' % (
product,
versions
)
)
def _force_assume_identity_by_host(self, connection, threshold, hostname,
req_id):
"""This function implements the case where a newly registering
processor wants to take over for a processor with the same host
name as the registering processor. This is the case where the
existing processor is likely dead but didn't manage to halt cleanly.
Parameters:
connection - a connection object
threshold - a datetime instance that represents an absolute date
made from the current datetime minus the timedelta
that defines how often a processor must update its
registration. If the threshold is greater than the
'lastseendatetime' of a registered processor, that
processor is considered dead.
hostname - the name of the host of the registering processor.
req_id - not used by this method, but present to meet the required
api for a registration method.
Returns:
an integer representing the new id of the newly registered
processor."""
self.config.logger.debug("looking for a processor for host %s",
hostname)
try:
sql = ("select id from processors" " where name like %s limit 1")
hostname_phrase = hostname + '%'
processor_id = single_value_sql(connection, sql,
(hostname_phrase, ))
self.config.logger.info("will take over processor %s",
processor_id)
# a processor for this host was found
self._take_over_dead_processor(connection, processor_id)
return processor_id
except SQLDidNotReturnSingleValue:
return self._assume_new_identity(connection, threshold, hostname,
req_id)
def _sweep_dead_processors_transaction(self, connection):
"""this function is a single database transaction: look for dead
processors - find all the jobs of dead processors and assign them to
live processors then delete the dead processor registrations"""
self.config.logger.info("looking for dead processors")
try:
self.config.logger.info("threshold %s",
self.config.registrar.check_in_frequency)
threshold = single_value_sql(
connection, "select now() - %s - %s",
(self.config.registrar.processor_grace_period,
self.config.registrar.check_in_frequency))
dead_processors = execute_query_fetchall(
connection,
"select id from processors where lastSeenDateTime < %s",
(threshold, ))
if dead_processors:
self.config.logger.info("found dead processor(s):")
for a_dead_processor in dead_processors:
self.config.logger.info("%d is dead", a_dead_processor[0])
self.config.logger.debug("getting list of live processor(s):")
live_processors = execute_query_fetchall(
connection,
"select id from processors where lastSeenDateTime >= %s",
(threshold, ))
if not live_processors:
if self.config.registrar.quit_if_no_processors:
raise NoProcessorsRegisteredError(
"There are no processors registered")
else:
self.config.logger.critical(
'There are no live processors, nothing to do. '
'Waiting for processors to come on line.')
return
number_of_live_processors = len(live_processors)
self.config.logger.debug(
"getting range of queued date for jobs associated with "
"dead processor(s):")
dead_processor_ids_str = ", ".join(
[str(x[0]) for x in dead_processors])
earliest_dead_job, latest_dead_job = single_row_sql(
connection,
"select min(queueddatetime), max(queueddatetime) from jobs "
"where owner in (%s)" % dead_processor_ids_str)
# take dead processor jobs and reallocate them to live
# processors in equal sized chunks
if (earliest_dead_job is not None
and latest_dead_job is not None):
time_increment = ((latest_dead_job - earliest_dead_job) /
number_of_live_processors)
for x, live_processor_id in enumerate(live_processors):
low_queued_time = (x * time_increment +
earliest_dead_job)
high_queued_time = ((x + 1) * time_increment +
earliest_dead_job)
self.config.logger.info(
"assigning jobs from %s to %s to processor %s:",
low_queued_time, high_queued_time,
live_processor_id)
# why is the range >= at both ends? the range must be
# inclusive, the risk of moving a job twice is low and
# consequences low, too.
# 1st step: take any jobs of a dead processor that were
# in progress and reset them to unprocessed
execute_no_results(
connection, "update jobs set"
" starteddatetime = NULL "
"where"
" %%s >= queueddatetime"
" and queueddatetime >= %%s"
" and owner in (%s)"
" and success is NULL" % dead_processor_ids_str,
(high_queued_time, low_queued_time))
# 2nd step: take all jobs of a dead processor and give
# them to a new owner
execute_no_results(
connection, "update jobs set"
" set owner = %%s "
"where"
" %%s >= queueddatetime"
" and queueddatetime >= %%s"
" and owner in (%s)" % dead_processor_ids_str,
(live_processor_id, high_queued_time,
low_queued_time))
# transfer stalled priority jobs to new processors
for dead_processor_tuple in dead_processors:
self.config.logger.info(
"re-assigning priority jobs from processor %d:",
dead_processor_tuple[0])
execute_no_results(
connection,
"insert into priorityjobs (uuid) select uuid "
"from priority_jobs_%d" % dead_processor_tuple)
self.config.logger.info("removing all dead processors")
execute_no_results(
connection,
"delete from processors where lastSeenDateTime < %s",
(threshold, ))
# remove dead processors' priority tables
for a_dead_processor in dead_processors:
execute_no_results(
connection, "drop table if exists priority_jobs_%d" %
a_dead_processor[0])
except NoProcessorsRegisteredError:
self.quit = True
self.config.logger.critical('there are no live processors')
def _sweep_dead_processors_transaction(self, connection):
"""this function is a single database transaction: look for dead
processors - find all the jobs of dead processors and assign them to
live processors then delete the dead processor registrations"""
self.config.logger.info("looking for dead processors")
try:
self.config.logger.info(
"threshold %s",
self.config.registrar.check_in_frequency
)
threshold = single_value_sql(
connection,
"select now() - %s - %s",
(self.config.registrar.processor_grace_period,
self.config.registrar.check_in_frequency)
)
dead_processors = execute_query_fetchall(
connection,
"select id from processors where lastSeenDateTime < %s",
(threshold,)
)
if dead_processors:
self.config.logger.info("found dead processor(s):")
for a_dead_processor in dead_processors:
self.config.logger.info("%d is dead", a_dead_processor[0])
self.config.logger.debug("getting list of live processor(s):")
live_processors = execute_query_fetchall(
connection,
"select id from processors where lastSeenDateTime >= %s",
(threshold,)
)
if not live_processors:
if self.config.registrar.quit_if_no_processors:
raise NoProcessorsRegisteredError(
"There are no processors registered"
)
else:
self.config.logger.critical(
'There are no live processors, nothing to do. '
'Waiting for processors to come on line.'
)
return
number_of_live_processors = len(live_processors)
self.config.logger.debug(
"getting range of queued date for jobs associated with "
"dead processor(s):"
)
dead_processor_ids_str = ", ".join(
[str(x[0]) for x in dead_processors]
)
earliest_dead_job, latest_dead_job = single_row_sql(
connection,
"select min(queueddatetime), max(queueddatetime) from jobs "
"where owner in (%s)" % dead_processor_ids_str
)
# take dead processor jobs and reallocate them to live
# processors in equal sized chunks
if (earliest_dead_job is not None and
latest_dead_job is not None):
time_increment = (
(latest_dead_job - earliest_dead_job) /
number_of_live_processors
)
for x, live_processor_id in enumerate(live_processors):
low_queued_time = (
x * time_increment + earliest_dead_job
)
high_queued_time = (
(x + 1) * time_increment + earliest_dead_job
)
self.config.logger.info(
"assigning jobs from %s to %s to processor %s:",
low_queued_time,
high_queued_time,
live_processor_id
)
# why is the range >= at both ends? the range must be
# inclusive, the risk of moving a job twice is low and
# consequences low, too.
# 1st step: take any jobs of a dead processor that were
# in progress and reset them to unprocessed
execute_no_results(
connection,
"update jobs set"
" starteddatetime = NULL "
"where"
" %%s >= queueddatetime"
" and queueddatetime >= %%s"
" and owner in (%s)"
" and success is NULL" % dead_processor_ids_str,
(high_queued_time, low_queued_time)
)
# 2nd step: take all jobs of a dead processor and give
# them to a new owner
execute_no_results(
connection,
"update jobs set"
" set owner = %%s "
"where"
" %%s >= queueddatetime"
" and queueddatetime >= %%s"
" and owner in (%s)" % dead_processor_ids_str,
(live_processor_id, high_queued_time,
low_queued_time)
)
# transfer stalled priority jobs to new processors
for dead_processor_tuple in dead_processors:
self.config.logger.info(
"re-assigning priority jobs from processor %d:",
dead_processor_tuple[0]
)
execute_no_results(
connection,
"insert into priorityjobs (uuid) select uuid "
"from priority_jobs_%d" % dead_processor_tuple
)
self.config.logger.info("removing all dead processors")
execute_no_results(
connection,
"delete from processors where lastSeenDateTime < %s",
(threshold,)
)
# remove dead processors' priority tables
for a_dead_processor in dead_processors:
execute_no_results(
connection,
"drop table if exists priority_jobs_%d" %
a_dead_processor[0]
)
except NoProcessorsRegisteredError:
self.quit = True
self.config.logger.critical('there are no live processors')
def _save_processed_report(self, connection, processed_crash):
""" Here we INSERT or UPDATE a row in the reports table.
This is the first stop before imported data gets into our normalized
batch reporting (next table: reports_clean).
At some point in the future, we will switch to using the raw_crash
table and JSON transforms instead. This work will require an overhaul
and optimization of the update_reports_clean() and
update_reports_duplicates() stored procedures.
We perform an UPSERT using a PostgreSQL CTE (aka WITH clause) that
first tests whether a row exists and performs an UPDATE if it can, or
it performs an INSERT. Because we're using raw SQL in this function,
we've got a substantial parameterized query that requires two sets of
parameters to be passed in via value_list. The value_list ends up
having an extra crash_id added to the list, and being doubled before
being passed to single_value_sql().
The SQL produced isn't beautiful, but a side effect of the CTE style of
UPSERT-ing. We look forward to SQL UPSERT being adopted as a
first-class citizen in PostgreSQL.
Similar code is present for _save_raw_crash() and
_save_processed_crash(), but is much simpler seeming because there are
far fewer columns being passed into the parameterized query.
"""
column_list = []
placeholder_list = []
# create a list of values to go into the reports table
value_list = []
for pro_crash_name, report_name, length in (
self._reports_table_mappings
):
column_list.append(report_name)
placeholder_list.append('%s')
value = processed_crash[pro_crash_name]
if isinstance(value, basestring) and length:
value_list.append(value[:length])
else:
value_list.append(value)
def print_eq(a, b):
# Helper for UPDATE SQL clause
return a + ' = ' + b
def print_as(a, b):
# Helper for INSERT SQL clause
return b + ' as ' + a
crash_id = processed_crash['uuid']
reports_table_name = (
'reports_%s' % self._table_suffix_for_crash_id(crash_id)
)
upsert_sql = """
WITH
update_report AS (
UPDATE %(table)s SET
%(joined_update_clause)s
WHERE uuid = %%s
RETURNING id
),
insert_report AS (
INSERT INTO %(table)s (%(column_list)s)
( SELECT
%(joined_select_clause)s
WHERE NOT EXISTS (
SELECT uuid from %(table)s
WHERE
uuid = %%s
LIMIT 1
)
)
RETURNING id
)
SELECT * from update_report
UNION ALL
SELECT * from insert_report
""" % {
'joined_update_clause':
", ".join(map(print_eq, column_list, placeholder_list)),
'table': reports_table_name,
'column_list': ', '. join(column_list),
'joined_select_clause':
", ".join(map(print_as, column_list, placeholder_list)),
}
value_list.append(crash_id)
value_list.extend(value_list)
report_id = single_value_sql(connection, upsert_sql, value_list)
return report_id
def _save_plugins(self, connection, processed_crash, report_id):
""" Electrolysis Support - Optional - processed_crash may contain a
ProcessType of plugin. In the future this value would be default,
content, maybe even Jetpack... This indicates which process was the
crashing process.
plugin - When set to plugin, the jsonDocument MUST calso contain
PluginFilename, PluginName, and PluginVersion
"""
process_type = processed_crash['process_type']
if not process_type:
return
if process_type == "plugin":
# Bug#543776 We actually will are relaxing the non-null policy...
# a null filename, name, and version is OK. We'll use empty strings
try:
plugin_filename = processed_crash['PluginFilename']
plugin_name = processed_crash['PluginName']
plugin_version = processed_crash['PluginVersion']
except KeyError, x:
self.config.logger.error(
'the crash is missing a required field: %s', str(x))
return
find_plugin_sql = ('select id from plugins '
'where filename = %s '
'and name = %s')
try:
plugin_id = single_value_sql(connection, find_plugin_sql,
(plugin_filename, plugin_name))
except SQLDidNotReturnSingleValue:
insert_plugsins_sql = ("insert into plugins (filename, name) "
"values (%s, %s) returning id")
plugin_id = single_value_sql(connection, insert_plugsins_sql,
(plugin_filename, plugin_name))
crash_id = processed_crash['uuid']
table_suffix = self._table_suffix_for_crash_id(crash_id)
plugin_reports_table_name = 'plugins_reports_%s' % table_suffix
# why are we deleting first? This might be a reprocessing job and
# the plugins_reports data might already be in the table: a
# straight insert might fail. Why not check to see if there is
# data already there and then just not insert if data is there?
# We may be reprocessing to deal with missing plugin_reports data,
# so just because there is already data there doesn't mean that we
# can skip this. What about using "upsert" sql - that would be fine
# and result in one fewer round trip between client and database,
# but "upsert" sql is opaque and not easy to understand at a
# glance. This was faster to implement. What about using
# "transaction check points"? Too many round trips between the
# client and the server.
plugins_reports_delete_sql = (
'delete from %s where report_id = %%s' %
plugin_reports_table_name)
execute_no_results(connection, plugins_reports_delete_sql,
(report_id, ))
plugins_reports_insert_sql = (
'insert into %s '
' (report_id, plugin_id, date_processed, version) '
'values '
' (%%s, %%s, %%s, %%s)' % plugin_reports_table_name)
values_tuple = (report_id, plugin_id,
processed_crash['date_processed'], plugin_version)
execute_no_results(connection, plugins_reports_insert_sql,
values_tuple)
def _save_processed_report(self, connection, processed_crash):
""" Here we INSERT or UPDATE a row in the reports table.
This is the first stop before imported data gets into our normalized
batch reporting (next table: reports_clean).
At some point in the future, we will switch to using the raw_crash
table and JSON transforms instead. This work will require an overhaul
and optimization of the update_reports_clean() and
update_reports_duplicates() stored procedures.
We perform an UPSERT using a PostgreSQL CTE (aka WITH clause) that
first tests whether a row exists and performs an UPDATE if it can, or
it performs an INSERT. Because we're using raw SQL in this function,
we've got a substantial parameterized query that requires two sets of
parameters to be passed in via value_list. The value_list ends up
having an extra crash_id added to the list, and being doubled before
being passed to single_value_sql().
The SQL produced isn't beautiful, but a side effect of the CTE style of
UPSERT-ing. We look forward to SQL UPSERT being adopted as a
first-class citizen in PostgreSQL.
Similar code is present for _save_raw_crash() and
_save_processed_crash(), but is much simpler seeming because there are
far fewer columns being passed into the parameterized query.
"""
column_list = []
placeholder_list = []
# create a list of values to go into the reports table
value_list = []
for pro_crash_name, report_name, length in \
self._reports_table_mappings:
column_list.append(report_name)
placeholder_list.append('%s')
value = processed_crash[pro_crash_name]
if isinstance(value, basestring) and length:
value_list.append(value[:length])
else:
value_list.append(value)
def print_eq(a, b):
# Helper for UPDATE SQL clause
return a + ' = ' + b
def print_as(a, b):
# Helper for INSERT SQL clause
return b + ' as ' + a
crash_id = processed_crash['uuid']
reports_table_name = ('reports_%s' %
self._table_suffix_for_crash_id(crash_id))
upsert_sql = """
WITH
update_report AS (
UPDATE %(table)s SET
%(joined_update_clause)s
WHERE uuid = %%s
RETURNING id
),
insert_report AS (
INSERT INTO %(table)s (%(column_list)s)
( SELECT
%(joined_select_clause)s
WHERE NOT EXISTS (
SELECT uuid from %(table)s
WHERE
uuid = %%s
LIMIT 1
)
)
RETURNING id
)
SELECT * from update_report
UNION ALL
SELECT * from insert_report
""" % {
'joined_update_clause':
", ".join(map(print_eq, column_list, placeholder_list)),
'table':
reports_table_name,
'column_list':
', '.join(column_list),
'joined_select_clause':
", ".join(map(print_as, column_list, placeholder_list)),
}
value_list.append(crash_id)
value_list.extend(value_list)
report_id = single_value_sql(connection, upsert_sql, value_list)
return report_id
def __setitem__(self, connection, key, value):
class LastErrorEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, type):
return repr(obj)
return json.JSONEncoder.default(self, obj)
try:
single_value_sql(
connection, """SELECT ongoing
FROM crontabber
WHERE
app_name = %s
FOR UPDATE NOWAIT
""", (key, ))
# If the above single_value_sql() didn't raise a
# SQLDidNotReturnSingleValue exception, it means
# there is a row by this app_name.
# Therefore, the next SQL is an update.
next_sql = """
UPDATE crontabber
SET
next_run = %(next_run)s,
first_run = %(first_run)s,
last_run = %(last_run)s,
last_success = %(last_success)s,
depends_on = %(depends_on)s,
error_count = %(error_count)s,
last_error = %(last_error)s,
ongoing = %(ongoing)s
WHERE
app_name = %(app_name)s
"""
except OperationalError as exception:
if 'could not obtain lock' in exception.args[0]:
raise RowLevelLockError(exception.args[0])
else:
raise
except SQLDidNotReturnSingleValue:
# the key does not exist, do an insert
next_sql = """
INSERT INTO crontabber (
app_name,
next_run,
first_run,
last_run,
last_success,
depends_on,
error_count,
last_error,
ongoing
) VALUES (
%(app_name)s,
%(next_run)s,
%(first_run)s,
%(last_run)s,
%(last_success)s,
%(depends_on)s,
%(error_count)s,
%(last_error)s,
%(ongoing)s
)
"""
parameters = {
'app_name': key,
'next_run': value['next_run'],
'first_run': value['first_run'],
'last_run': value['last_run'],
'last_success': value.get('last_success'),
'depends_on': value['depends_on'],
'error_count': value['error_count'],
'last_error': json.dumps(value['last_error'],
cls=LastErrorEncoder),
'ongoing': value.get('ongoing'),
}
try:
execute_no_results(connection, next_sql, parameters)
except IntegrityError as exception:
# See CREATE_CRONTABBER_APP_NAME_UNIQUE_INDEX for why
# we know to look for this mentioned in the error message.
if 'crontabber_unique_app_name_idx' in exception.args[0]:
raise RowLevelLockError(exception.args[0])
raise
def __setitem__(self, connection, key, value):
class LastErrorEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, type):
return repr(obj)
return json.JSONEncoder.default(self, obj)
try:
single_value_sql(
connection,
"""SELECT app_name
FROM crontabber
WHERE
app_name = %s""",
(key,)
)
# the key exists, do an update
next_sql = """UPDATE crontabber
SET
next_run = %(next_run)s,
first_run = %(first_run)s,
last_run = %(last_run)s,
last_success = %(last_success)s,
depends_on = %(depends_on)s,
error_count = %(error_count)s,
last_error = %(last_error)s
WHERE
app_name = %(app_name)s"""
except SQLDidNotReturnSingleValue:
# the key does not exist, do an insert
next_sql = """
INSERT INTO crontabber (
app_name,
next_run,
first_run,
last_run,
last_success,
depends_on,
error_count,
last_error
) VALUES (
%(app_name)s,
%(next_run)s,
%(first_run)s,
%(last_run)s,
%(last_success)s,
%(depends_on)s,
%(error_count)s,
%(last_error)s
)"""
parameters = {
'app_name': key,
'next_run': value['next_run'],
'first_run': value['first_run'],
'last_run': value['last_run'],
'last_success': value.get('last_success'),
'depends_on': value['depends_on'],
'error_count': value['error_count'],
'last_error': json.dumps(value['last_error'], cls=LastErrorEncoder)
}
execute_no_results(
connection,
next_sql,
parameters
)
def _save_plugins(self, connection, processed_crash, report_id):
""" Electrolysis Support - Optional - processed_crash may contain a
ProcessType of plugin. In the future this value would be default,
content, maybe even Jetpack... This indicates which process was the
crashing process.
plugin - When set to plugin, the jsonDocument MUST calso contain
PluginFilename, PluginName, and PluginVersion
"""
process_type = processed_crash['process_type']
if not process_type:
return
if process_type == "plugin":
# Bug#543776 We actually will are relaxing the non-null policy...
# a null filename, name, and version is OK. We'll use empty strings
try:
plugin_filename = processed_crash['PluginFilename']
plugin_name = processed_crash['PluginName']
plugin_version = processed_crash['PluginVersion']
except KeyError, x:
self.config.logger.error(
'the crash is missing a required field: %s', str(x)
)
return
find_plugin_sql = ('select id from plugins '
'where filename = %s '
'and name = %s')
try:
plugin_id = single_value_sql(connection,
find_plugin_sql,
(plugin_filename,
plugin_name))
except SQLDidNotReturnSingleValue:
insert_plugsins_sql = ("insert into plugins (filename, name) "
"values (%s, %s) returning id")
plugin_id = single_value_sql(connection,
insert_plugsins_sql,
(plugin_filename,
plugin_name))
crash_id = processed_crash['uuid']
table_suffix = self._table_suffix_for_crash_id(crash_id)
plugin_reports_table_name = 'plugins_reports_%s' % table_suffix
# why are we deleting first? This might be a reprocessing job and
# the plugins_reports data might already be in the table: a
# straight insert might fail. Why not check to see if there is
# data already there and then just not insert if data is there?
# We may be reprocessing to deal with missing plugin_reports data,
# so just because there is already data there doesn't mean that we
# can skip this. What about using "upsert" sql - that would be fine
# and result in one fewer round trip between client and database,
# but "upsert" sql is opaque and not easy to understand at a
# glance. This was faster to implement. What about using
# "transaction check points"? Too many round trips between the
# client and the server.
plugins_reports_delete_sql = (
'delete from %s where report_id = %%s'
% plugin_reports_table_name
)
execute_no_results(connection,
plugins_reports_delete_sql,
(report_id,))
plugins_reports_insert_sql = (
'insert into %s '
' (report_id, plugin_id, date_processed, version) '
'values '
' (%%s, %%s, %%s, %%s)' % plugin_reports_table_name
)
values_tuple = (report_id,
plugin_id,
processed_crash['date_processed'],
plugin_version)
execute_no_results(connection,
plugins_reports_insert_sql,
values_tuple)
