def _log_error_rollback_and_raise(self, e: Exception, query_log_line: str):
self._log_database_notifications()
error_string = single_line_with_single_spaces(e)
logger.error("Rolling back query=\'%s\' due to error: \'%s\'" % (query_log_line, error_string))
self.rollback()
if isinstance(e, PostgresDBError):
# just re-raise our PostgresDBError
raise
else:
# wrap original error in PostgresDBQueryExecutionError
raise PostgresDBQueryExecutionError("Could not execute query '%s' error=%s" % (query_log_line, error_string))
def connect(self):
if self.is_connected:
logger.debug("already connected to database: %s", self)
return
for retry_cntr in range(self.__num_connect_retries+1):
try:
logger.debug("connecting to database: %s", self)
self._connection = psycopg2.connect(host=self._dbcreds.host,
user=self._dbcreds.user,
password=self._dbcreds.password,
database=self._dbcreds.database,
port=self._dbcreds.port,
connect_timeout=5)
if self._connection:
self._cursor = self._connection.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
logger.info("connected to database: %s", self)
# see http://initd.org/psycopg/docs/connection.html#connection.notices
# try to set the notices attribute with a non-list collection,
# so we can log more than 50 messages. Is only available since 2.7, so encapsulate in try/except.
try:
self._connection.notices = collections.deque()
except TypeError:
logger.warning("Cannot overwrite self._connection.notices with a deque... only max 50 notifications available per query. (That's ok, no worries.)")
# we have a proper connection, so return
return
except psycopg2.DatabaseError as dbe:
error_string = single_line_with_single_spaces(dbe)
logger.error(error_string)
if self._is_recoverable_connection_error(dbe):
# try to reconnect on connection-like-errors
if retry_cntr == self.__num_connect_retries:
raise PostgresDBConnectionError("Error while connecting to %s. error=%s" % (self, error_string))
logger.info('retrying to connect to %s in %s seconds', self.database, self.__connect_retry_interval)
time.sleep(self.__connect_retry_interval)
else:
# non-connection-error, raise generic PostgresDBError
raise PostgresDBError(error_string)
def _do_execute_query(self, query, qargs=None, fetch=FETCH_NONE):
'''execute the query and reconnect upon OperationalError'''
query_log_line = self._queryAsSingleLine(query, qargs)
try:
self.connect_if_needed()
# log
logger.debug('executing query: %s', query_log_line)
# execute (and time it)
start = datetime.utcnow()
self._cursor.execute(query, qargs)
elapsed = datetime.utcnow() - start
elapsed_ms = 1000.0 * totalSeconds(elapsed)
# log execution result
logger.info('executed query in %.1fms%s yielding %s rows: %s', elapsed_ms,
' (SLOW!)' if elapsed_ms > 250 else '', # for easy log grep'ing
self._cursor.rowcount,
query_log_line)
# log any notifications from within the database itself
self._log_database_notifications()
self._commit_selects_if_needed(query)
# fetch and return results
if fetch == FETCH_ONE:
row = self._cursor.fetchone()
return dict(row) if row is not None else None
if fetch == FETCH_ALL:
return [dict(row) for row in self._cursor.fetchall() if row is not None]
return []
except psycopg2.OperationalError as oe:
if self._is_recoverable_connection_error(oe):
raise PostgresDBConnectionError("Could not execute query due to connection errors. '%s' error=%s" %
(query_log_line,
single_line_with_single_spaces(oe)))
else:
self._log_error_rollback_and_raise(oe, query_log_line)
except Exception as e:
self._log_error_rollback_and_raise(e, query_log_line)
def _send_task_status_notification(self, spec, new_status):
"""
Sends a message about the task's status on the RA notification bus
:param spec: the task concerned
:param new_status: the task's status
:raises Exception if sending the notification fails
"""
#TODO can maybe move to Specification in the future? Logically it should be resource_assigner that sends the notification
content = {
'radb_id': spec.radb_id,
'otdb_id': spec.otdb_id,
'mom_id': spec.mom_id
}
subject = 'Task' + new_status[0].upper() + new_status[
1:] #TODO this is MAGIC, needs explanation!
event_message = EventMessage(subject="%s.%s" %
(DEFAULT_RA_NOTIFICATION_PREFIX, subject),
content=content)
logger.info('Sending notification %s: %s' %
(subject, single_line_with_single_spaces(content)))
self.ra_notification_bus.send(event_message)
def onDataWritersFinished(self, msg_content):
logger.info("received DataWritersFinished event: %s",
single_line_with_single_spaces(str(msg_content)))
# signal that we're done waiting
self._waiter.wait_event.set()
def onDataWritersStopped(self, msg_content):
logger.info("received DataWritersStopped event: %s",
single_line_with_single_spaces(str(msg_content)))
def _queryAsSingleLine(query, qargs=None):
line = ' '.join(single_line_with_single_spaces(query).split())
if qargs:
line = line % tuple(['\'%s\'' % a if isinstance(a, str) else a for a in qargs])
return line
def handle_message(self, msg: LofarMessage):
# try to handle an incoming message, and call the associated on<SomeMessage> method
if not isinstance(msg, EventMessage):
raise ValueError("%s: Ignoring non-EventMessage: %s" % (self.__class__.__name__, msg))
stripped_subject = msg.subject.replace("%s." % DEFAULT_TBB_NOTIFICATION_PREFIX, '')
logger.debug("TBBEventMessageHandler.handle_message: on%s content=%s", stripped_subject, single_line_with_single_spaces(str(msg.content)))
if stripped_subject == 'DataWritersStarting':
self.onDataWritersStarting(msg.content)
elif stripped_subject == 'DataWritersStarted':
self.onDataWritersStarted(msg.content)
elif stripped_subject == 'DataWritersFinished':
self.onDataWritersFinished(msg.content)
elif stripped_subject == 'DataWritersStopping':
self.onDataWritersStopping(msg.content)
elif stripped_subject == 'DataWritersStopped':
self.onDataWritersStopped(msg.content)
else:
raise ValueError("TBBEventMessageHandler.handleMessage: unknown subject: %s" % msg.subject)
def onError(self, msg_content):
logger.info("%s.onError(%s)", self.__class__.__name__, single_line_with_single_spaces(msg_content))
def onCreatedInspectionPlots(self, msg_content):
logger.info("%s.onCreatedInspectionPlots(%s)", self.__class__.__name__, single_line_with_single_spaces(msg_content))
def onConvertedBF2Hdf5(self, msg_content):
logger.info("%s.onConvertedBF2Hdf5(%s)", self.__class__.__name__, single_line_with_single_spaces(msg_content))
