def __log_receival_one_message(self, message):
if self.__first_message_receival:
logdebug(LOGGER, 'Handing over first message to rabbit thread...')
self.__first_message_receival = False
logtrace(LOGGER, 'Handing over one message over to the rabbit thread (%s)', message)
log_every_x_times(LOGGER, self.__logcounter_received, self.__LOGFREQUENCY, 'Handing over one message over to the rabbit thread (no. %i).', self.__logcounter_received)
self.__logcounter_received += 1
def __put_all_messages_into_queue_of_unsent_messages(self, messages):
counter = 1
for message in messages:
logtrace(LOGGER, 'Adding message %i/%i to stack to be sent.',
counter, len(messages))
counter += 1
self.__put_one_message_into_queue_of_unsent_messages(message)
def __add_file(self, **args):
logdebug(LOGGER, 'Adding file "%s" with handle "%s".', args['file_name'], args['file_handle'])
self.__add_file_to_datasets_children(args['file_handle'])
self.__adapt_file_args(args)
self.__create_and_store_file_publication_message(args)
self.__set_machine_state_to_files_added()
logtrace(LOGGER, 'Adding file done.')
def first_connection(self):
logdebug(LOGGER, 'Trigger connection to rabbit...')
self.__trigger_connection_to_rabbit_etc()
logdebug(LOGGER, 'Trigger connection to rabbit... done.')
logdebug(LOGGER, 'Start waiting for events...')
self.__start_waiting_for_events()
logtrace(LOGGER, 'Had started waiting for events, but stopped.')
def __get_message_from_stack(self, seconds=0):
message = self.thread.get_message_from_unpublished_stack(seconds)
logtrace(
LOGGER,
'Found message to be published. Now left in queue to be published: %i messages.',
self.thread.get_num_unpublished())
return message
def __publish_message_to_channel(self):
# Find a message to publish.
# If no messages left, well, nothing to publish!
try:
message = self.__get_message_from_stack()
except queue.Empty as e:
logtrace(LOGGER, 'Queue empty. No more messages to be published.')
return
# Now try to publish it.
# If anything goes wrong, you need to put it back to
# the stack of unpublished messages!
try:
success = self.__try_publishing_otherwise_put_back_to_stack(message)
if success:
self.__postparations_after_successful_feeding(message)
# Treat various errors that may occur during publishing:
except pika.exceptions.ChannelClosed as e:
logwarn(LOGGER, 'Cannot publish message %i to RabbitMQ because the Channel is closed (%s)', self.__delivery_number+1, repr(e))
except AttributeError as e:
if self.thread._channel is None:
logwarn(LOGGER, 'Cannot publish message %i to RabbitMQ because there is no channel.', self.__delivery_number+1)
else:
logwarn(LOGGER, 'Cannot publish message %i to RabbitMQ (unexpected error %s:%s)', self.__delivery_number+1, e.__class__.__name__, repr(e))
except AssertionError as e:
logwarn(LOGGER, 'Cannot publish message to RabbitMQ %i because of AssertionError: "%s"', self.__delivery_number+1,e)
if 'A non-string value was supplied for self.exchange' in repr(e):
exch = self.thread.get_exchange_name()
logwarn(LOGGER, 'Exchange was "%s" (type %s)', exch, type(exch))
def get_routing_key_and_string_message_from_message_if_possible(msg):
# Try to convert message to json:
json_ok = False
msg_json = None
msg_string = None
if msg is None:
raise ValueError('The message that was passed is None.')
# Get JSON from message, if possible!
if isinstance(msg, basestring):
try:
# Valid string message --> JSON
msg_string = msg
msg_json = json.loads(msg)
json_ok = True
logdebug(LOGGER, 'Message was transformed to json.')
except ValueError as e:
# Invalid string message
loginfo(LOGGER, 'Message seems to be invalid json: %s', msg)
msg_string = str(msg)
json_ok = False
else:
try:
# Message is json already.
msg_string = json.dumps(msg)
msg_json = msg
json_ok = True
logtrace(LOGGER, 'Message was already json.')
except TypeError as e:
if 'not JSON serializable' in e.message:
# Message was whatever.
msg_string = str(msg)
json_ok = False
msg = (
'Message was neither JSON nor string and not understandable: %s'
% msg_string)
loginfo(LOGGER, msg)
raise ValueError(msg)
# If we succeeded, try to get routing key:
routing_key = None
if json_ok:
try:
routing_key = msg_json['ROUTING_KEY']
logtrace(LOGGER, 'Routing key extracted from message.')
except (KeyError, TypeError) as e:
logdebug(LOGGER, 'No routing key in message.')
routing_key = esgfpid.defaults.RABBIT_DEFAULT_ROUTING_KEY
pass # There is no routing key in the message
else:
routing_key = esgfpid.defaults.RABBIT_DEFAULT_ROUTING_KEY
return routing_key, msg_string
def __iterate_over_all_hosts(self):
success = False
print_conn = True
print_chan = True
self.channel_ok = False
while True:
try:
if print_conn:
self.__loginfo(
' .. checking authentication and connection ...')
print_conn = False
print_chan = True
self.connection = self.__check_making_rabbit_connection()
if print_chan:
self.__loginfo(
' .. checking authentication and connection ... ok.')
self.__loginfo(' .. checking channel ...')
print_chan = False
print_conn = True
self.channel_ok = False
self.channel = self.__check_opening_channel(self.connection)
self.channel_ok = True
self.__check_exchange_existence(self.channel)
if self.__send_message:
if self.__prefix is None:
# Cannot happen, as we get it from the connector object.
raise esgfpid.exceptions.ArgumentError(
'Missing handle prefix!')
self.__check_send_print_message(self.channel)
success = True
break # success, leave loop
except ValueError as e:
self.__loginfo(' .. giving this node a lower priority..')
self.__nodemanager.set_priority_low_for_current()
if self.__nodemanager.has_more_urls(
): # stay in loop, try next host
utils.logtrace(LOGGER, 'Left URLs: %s',
self.__nodemanager.get_num_left_urls())
self.__nodemanager.set_next_host()
self.__current_rabbit_host = self.__nodemanager.get_connection_parameters(
).host
utils.logtrace(LOGGER, 'Now trying: %s',
self.__current_rabbit_host)
else: # definitive fail, leave loop
break
return success
def __remove_delivery_tag_and_message_single(self, deliv_tag):
try:
self.__unconfirmed_delivery_tags.remove(deliv_tag)
ms = self.__unconfirmed_messages_dict.pop(str(deliv_tag))
logtrace(LOGGER, 'Received ack for message %s.', ms)
except ValueError as e:
logdebug(LOGGER, 'Could not remove %i from unconfirmed.',
deliv_tag)
def __react_on_single_delivery_ack(self, deliv_tag):
self.__remove_delivery_tag_and_message_single(deliv_tag)
logdebug(LOGGER,
'Received ack for delivery tag %i. Waiting for %i confirms.',
deliv_tag, len(self.__unconfirmed_delivery_tags))
logtrace(LOGGER, 'Received ack for delivery tag %i.', deliv_tag)
logtrace(LOGGER, 'Now left in queue to be confirmed: %i messages.',
len(self.__unconfirmed_delivery_tags))
def __make_ready_for_publishing(self):
logdebug(
LOGGER,
'(Re)connection established, making ready for publication...')
# Check for unexpected errors:
if self.thread._channel is None:
logerror(
LOGGER,
'Channel is None after connecting to server. This should not happen.'
)
self.statemachine.set_to_permanently_unavailable()
if self.thread._connection is None:
logerror(
LOGGER,
'Connection is None after connecting to server. This should not happen.'
)
self.statemachine.set_to_permanently_unavailable()
# Normally, it should already be waiting to be available:
if self.statemachine.is_WAITING_TO_BE_AVAILABLE():
logdebug(LOGGER, 'Setup is finished. Publishing may start.')
logtrace(LOGGER, 'Publishing will use channel no. %s!',
self.thread._channel.channel_number)
self.statemachine.set_to_available()
self.__check_for_already_arrived_messages_and_publish_them()
# It was asked to close in the meantime (but might be able to publish the last messages):
elif self.statemachine.is_AVAILABLE_BUT_WANTS_TO_STOP():
logdebug(
LOGGER,
'Setup is finished, but the module was already asked to be closed in the meantime.'
)
self.__check_for_already_arrived_messages_and_publish_them()
# It was force-closed in the meantime:
elif self.statemachine.is_PERMANENTLY_UNAVAILABLE(
): # state was set in shutter module's __close_down()
if self.statemachine.get_detail_closed_by_publisher():
logdebug(
LOGGER,
'Setup is finished now, but the module was already force-closed in the meantime.'
)
self.shutter.safety_finish(
'closed before connection was ready. reclosing.')
elif self.statemachine.detail_could_not_connect:
logerror(
LOGGER,
'This is not supposed to happen. If the connection failed, this part of the code should not be reached.'
)
else:
logerror(
LOGGER,
'This is not supposed to happen. An unknown event set this module to be unavailable. When was this set to unavailable?'
)
else:
logdebug(LOGGER, 'Unexpected state.')
def __log_publication_trigger(self):
if self.__first_publication_trigger:
logdebug(
LOGGER,
'Received first trigger for publishing message to RabbitMQ.')
self.__first_publication_trigger = False
logtrace(
LOGGER,
'Received trigger for publishing message to RabbitMQ, and module is ready to accept it.'
)
def on_channel_open(self, channel):
time_passed = datetime.datetime.now() - self.__start_connect_time
logdebug(
LOGGER, 'Opening channel... done. Took %s seconds.' %
time_passed.total_seconds())
logtrace(LOGGER, 'Channel has number: %s.', channel.channel_number)
self.thread._channel = channel
self.__reconnect_counter = 0
self.__add_on_channel_close_callback()
self.__add_on_return_callback()
self.__make_channel_confirm_delivery()
self.__make_ready_for_publishing()
def __wait_and_trigger_reconnection(self, connection, wait_seconds):
if self.statemachine.is_FORCE_FINISHED():
errormsg = 'Permanently failed to connect to RabbitMQ. Tried all hosts until received a force-finish. Giving up. No PID requests will be sent.'
self.__give_up_reconnecting_and_raise_exception(errormsg)
else:
self.statemachine.set_to_waiting_to_be_available()
loginfo(LOGGER, 'Trying to reconnect to RabbitMQ in %s seconds.',
wait_seconds)
connection.add_timeout(wait_seconds, self.reconnect)
logtrace(LOGGER,
'Reconnect event added to connection %s (not to %s)',
connection, self.thread._connection)
def set_priority_low_for_current(self):
# We do not change the priority stored ass attribute in the
# dicts, BUT we change the priority under which it is stored in
# the list of nodes to be used.
# Deal with open or trusted node:
if self.__current_node['is_open']:
where_to_look = self.__open_nodes_archive
else:
where_to_look = self.__trusted_nodes_archive
# Go over all nodes of the current prio to find the
# current one, then move it to a different prio:
moved = False
try:
current_prio = self.__current_node['priority']
moved = self.__move_to_last_prio(current_prio, where_to_look)
if moved: return # changed successfully!
except KeyError as e:
logsafe = make_logsafe(where_to_look)
errmsg = 'No node of prio %s found. Nodes: %s.' % (current_prio,
logsafe)
logwarn(LOGGER, errmsg)
# The node had already been added to the last-prio nodes ?!
last_already = self.__is_this_node_in_last_prio_already(
where_to_look)
if last_already:
logdebug(LOGGER, 'Node already had lowest priority.')
return # nothing to change!
# This is extremely unlikely - in fact I don't see how it could occur:
if (not moved) and (not last_already):
errmsg = 'Could not find this node\'s priority (%s), nor the last-priority (%s). Somehow this node\'s priority was changed weirdly.' % (
current_prio, LAST_PRIO)
logwarn(LOGGER, errmsg)
logsafe = make_logsafe(where_to_look)
logwarn(LOGGER, 'All nodes: %s' % logsafe)
# No matter where the node is stored, move it to "last" prio:
for prio, nodes in where_to_look.items():
logtrace(LOGGER, 'Looking in prio "%s"...' % prio)
moved = self.__move_to_last_prio(prio, where_to_look)
if moved: return # changed successfully!
errmsg = 'Node definitely not found, cannot change prio.'
logwarn(LOGGER, errmsg)
raise ValueError(errmsg)
def __iterate_over_all_hosts(self):
success = False
print_conn = True
print_chan = True
self.channel_ok = False
while True:
try:
if print_conn:
self.__loginfo(
' .. checking authentication and connection ...')
print_conn = False
print_chan = True
self.connection = self.__check_making_rabbit_connection()
if print_chan:
self.__loginfo(
' .. checking authentication and connection ... ok.')
self.__loginfo(' .. checking channel ...')
print_chan = False
print_conn = True
self.channel_ok = False
self.channel = self.__check_opening_channel(self.connection)
self.channel_ok = True
self.__check_exchange_existence(self.channel)
success = True
break # success, leave loop
except ValueError as e:
if self.__nodemanager.has_more_urls(
): # stay in loop, try next host
utils.logtrace(LOGGER, 'Left URLs: %s',
self.__nodemanager.get_num_left_urls())
self.__nodemanager.set_next_host()
self.__current_rabbit_host = self.__nodemanager.get_connection_parameters(
).host
utils.logtrace(LOGGER, 'Now trying: %s',
self.__current_rabbit_host)
else: # definitive fail, leave loop
break
return success
def __react_on_ack(self, deliv_tag, multiple):
if self.__first_confirm_receival:
self.__first_confirm_receival = False
loginfo(LOGGER,
'Received first message confirmation from RabbitMQ.')
if multiple:
logtrace(
LOGGER,
'Received "ACK" for multiple messages from messaging service.')
self.__react_on_multiple_delivery_ack(deliv_tag)
else:
logtrace(
LOGGER,
'Received "ACK" for single message from messaging service.')
self.__react_on_single_delivery_ack(deliv_tag)
def __postparations_after_successful_feeding(self, msg):
# Pass the successfully published message and its delivery_number
# to the confirmer module, to wait for its confirmation.
# Increase the delivery number for the next message.
self.thread.put_to_unconfirmed_delivery_tags(self.__delivery_number)
self.thread.put_to_unconfirmed_messages_dict(self.__delivery_number, msg)
self.__delivery_number += 1
# Logging
self.__logcounter_success += 1
log_every_x_times(LOGGER, self.__logcounter_success, self.__LOGFREQUENCY, 'Actual publish to channel done (trigger no. %i, publish no. %i).', self.__logcounter_trigger, self.__logcounter_success)
logtrace(LOGGER, 'Publishing messages %i to RabbitMQ... done.', self.__delivery_number-1)
if (self.__delivery_number-1 == 1):
loginfo(LOGGER, 'First message published to RabbitMQ.')
logdebug(LOGGER, 'Message published (no. %i)', self.__delivery_number-1)
def on_delivery_confirmation(self, method_frame):
deliv_tag, confirmation_type, multiple = self.__get_confirm_info(
method_frame)
log_every_x_times(LOGGER, self.__logcounter, self.__LOGFREQUENCY,
'Received a confirm (%s)', confirmation_type)
self.__logcounter += 1
if confirmation_type == 'ack':
logtrace(LOGGER, 'Received "ACK" from messaging service.')
self.__react_on_ack(deliv_tag, multiple)
elif confirmation_type == 'nack':
logtrace(LOGGER, 'Received "NACK" from messaging service.')
self.__react_on_nack(deliv_tag, multiple)
else:
msg = 'Received asynchronous response of unknown type from messaging service.'
logwarn(LOGGER, msg)
raise UnknownServerResponse(msg + ':' + str(method_frame))
def make_permanently_closed_by_user(self):
# This changes the state of the state machine!
# This needs to be called from the shutter module
# in case there is a force_finish while the connection
# is already closed (as the callback on_connection_closed
# is not called then).
self.statemachine.set_to_permanently_unavailable()
logtrace(LOGGER, 'Stop waiting for events due to user interrupt!')
logtrace(LOGGER,
'Permanent close: Stopping ioloop of connection %s...',
self.thread._connection)
self.thread._connection.ioloop.stop()
loginfo(LOGGER, 'Stopped listening for RabbitMQ events (%s).',
get_now_utc_as_formatted_string())
logdebug(
LOGGER,
'Connection to messaging service closed by user. Will not reopen.')
def __log_why_cannot_feed_the_rabbit_now(self):
log_every_x_times(
LOGGER, self.__logcounter_trigger, self.__LOGFREQUENCY,
'Cannot publish message to RabbitMQ (trigger no. %i).',
self.__logcounter_trigger)
if self.statemachine.is_WAITING_TO_BE_AVAILABLE():
logdebug(
LOGGER,
'Cannot publish message to RabbitMQ yet, as the connection is not ready.'
)
elif self.statemachine.is_NOT_STARTED_YET():
logerror(
LOGGER,
'Cannot publish message to RabbitMQ, as the thread is not running yet.'
)
elif self.statemachine.is_PERMANENTLY_UNAVAILABLE(
) or self.statemachine.is_FORCE_FINISHED():
if self.statemachine.detail_could_not_connect:
logtrace(
LOGGER,
'Could not publish message to RabbitMQ, as the connection failed.'
)
if self.__have_not_warned_about_connection_fail_yet:
logwarn(
LOGGER,
'Could not publish message(s) to RabbitMQ. The connection failed definitively.'
)
self.__have_not_warned_about_connection_fail_yet = False
elif self.statemachine.get_detail_closed_by_publisher():
logtrace(
LOGGER,
'Cannot publish message to RabbitMQ, as the connection was closed by the user.'
)
if self.__have_not_warned_about_force_close_yet:
logwarn(
LOGGER,
'Could not publish message(s) to RabbitMQ. The sender was closed by the user.'
)
self.__have_not_warned_about_force_close_yet = False
else:
if self.thread._channel is None:
logerror(
LOGGER,
'Very unexpected. Could not publish message(s) to RabbitMQ. There is no channel.'
)
def __is_this_node_in_last_prio_already(self, where_to_look):
try:
list_candidates = where_to_look[LAST_PRIO]
except KeyError as e:
errmsg = 'No node of last prio (%s) exists.' % LAST_PRIO
logwarn(LOGGER, errmsg)
return False
for i in range(len(list_candidates)):
candidate = list_candidates[i]
if self.__compare_nodes(candidate, self.__current_node):
logtrace(
LOGGER,
'Found current node in archive (in list of last-prio nodes).'
)
return True
return False
def __wait_and_trigger_reconnection(self, connection, wait_seconds):
if self.statemachine.is_FORCE_FINISHED():
# TODO This is the same code as above. Make a give_up function from it?
#self.statemachine.set_to_permanently_unavailable()
#self.statemachine.detail_could_not_connect = True
#max_tries = defaults.RABBIT_RECONNECTION_MAX_TRIES
errormsg = (
'Permanently failed to connect to RabbitMQ. Tried all hosts %s until received a force-finish. Giving up. No PID requests will be sent.'
% list(self.__all_hosts_that_were_tried))
logerror(LOGGER, errormsg)
raise PIDServerException(errormsg)
else:
self.statemachine.set_to_waiting_to_be_available()
loginfo(LOGGER, 'Trying to reconnect to RabbitMQ in %s seconds.',
wait_seconds)
connection.add_timeout(wait_seconds, self.reconnect)
logtrace(LOGGER,
'Reconnect event added to connection %s (not to %s)',
connection, self.thread._connection)
def on_message_not_accepted(self, channel, returned_frame, props, body):
# Messages that are returned are confirmed anyways.
# If we sent 20 messages that are returned, all 20 are acked,
# so we do not need to retrieve them from the unconfirmed
# messages after resending.
# In the end, we'll have published 40 messages and received 40 acks.
# Logging...
logtrace(
LOGGER, 'Return frame: %s', returned_frame
) # <Basic.Return(['exchange=rabbitsender_integration_tests', 'reply_code=312', 'reply_text=NO_ROUTE', 'routing_key=cmip6.publisher.HASH.cart.datasets'])>
logtrace(
LOGGER, 'Return props: %s', props
) # <BasicProperties(['content_type=application/json', 'delivery_mode=2'])>
logtrace(LOGGER, 'Return body: %s', body)
# Was it the first or second time it comes back?
if returned_frame.reply_text == 'NO_ROUTE':
loginfo(
LOGGER,
'The message was returned because it could not be assigned to any queue. No binding for routing key "%s".',
returned_frame.routing_key)
if returned_frame.routing_key.startswith(
esgfpid.utils.RABBIT_EMERGENCY_ROUTING_KEY):
self.__log_about_double_return(returned_frame, body)
else:
self.__resend_message(returned_frame, props, body)
else:
logerror(
LOGGER,
'The message was returned. Routing key: %s. Unknown reason: %s',
returned_frame.routing_key, returned_frame.reply_text)
self.__resend_message(returned_frame, props, body)
def __try_publishing_otherwise_put_back_to_stack(self, message):
try:
# Getting message info:
properties = self.nodemanager.get_properties_for_message_publications()
routing_key, msg_string = rabbitutils.get_routing_key_and_string_message_from_message_if_possible(message)
routing_key = self.nodemanager.adapt_routing_key_for_untrusted(routing_key)
# Logging
logtrace(LOGGER, 'Publishing message %i (key %s) (body %s)...', self.__delivery_number+1, routing_key, msg_string) # +1 because it will be incremented after the publish.
log_every_x_times(LOGGER, self.__logcounter_trigger, self.__LOGFREQUENCY, 'Trying actual publish... (trigger no. %i).', self.__logcounter_trigger)
logtrace(LOGGER, '(Publish to channel no. %i).', self.thread._channel.channel_number)
# Actual publish to exchange
self.thread._channel.basic_publish(
exchange=self.thread.get_exchange_name(),
routing_key=routing_key,
body=msg_string,
properties=properties,
mandatory=defaults.RABBIT_MANDATORY_DELIVERY
)
return True
# If anything went wrong, put it back into the stack of
# unpublished messages before re-raising the exception
# for further handling:
except Exception as e:
success = False
logwarn(LOGGER, 'Message was not published. Putting back to queue. Reason: %s: "%s"',e.__class__.__name__, repr(e))
self.thread.put_one_message_into_queue_of_unsent_messages(message)
logtrace(LOGGER, 'Now (after putting back) left in queue to be published: %i messages.', self.thread.get_num_unpublished())
raise e
def __move_to_last_prio(self, current_prio, all_nodes):
list_candidates = all_nodes[current_prio]
logsafe = make_logsafe(list_candidates)
loginfo(LOGGER, 'Nodes of prio "%s": %s', current_prio, logsafe)
for i in range(len(list_candidates)):
candidate = list_candidates[i]
if self.__compare_nodes(candidate, self.__current_node):
logtrace(LOGGER, 'Found current node in archive.')
# Add to lowest prio:
try:
all_nodes[LAST_PRIO].append(candidate)
logdebug(
LOGGER,
'Added this host to list of lowest prio hosts...')
except KeyError:
all_nodes[LAST_PRIO] = [candidate]
logdebug(
LOGGER,
'Added this host to (newly-created) list of lowest prio hosts...'
)
# Remove from current prio:
list_candidates.pop(i)
loginfo(LOGGER,
'Removed this host from list of hosts with prio %s!',
current_prio)
if len(list_candidates) == 0:
all_nodes.pop(current_prio)
loginfo(LOGGER, 'Removed the current priority %s!',
current_prio)
return True
return False
def make_permanently_closed_by_error(self, connection, reply_text):
# This changes the state of the state machine!
# This needs to be called if there is a permanent
# error and we don't want the library to reonnect,
# and we also don't want to pretend it was closed
# by the user.
# This is really rarely needed.
self.statemachine.set_to_permanently_unavailable()
logtrace(LOGGER, 'Stop waiting for events due to permanent error!')
# In case the main thread was waiting for any synchronization event.
self.thread.unblock_events()
# Close ioloop, which blocks the thread.
logdebug(LOGGER,
'Permanent close: Stopping ioloop of connection %s...',
self.thread._connection)
self.thread._connection.ioloop.stop()
loginfo(LOGGER, 'Stopped listening for RabbitMQ events (%s).',
get_now_utc_as_formatted_string())
logdebug(
LOGGER,
'Connection to messaging service closed because of error. Will not reopen. Reason: %s',
reply_text)
def __create_and_send_dataset_publication_message_to_queue(self):
self.__remove_duplicates_from_list_of_file_handles()
message = self.__create_dataset_publication_message()
self.__send_message_to_queue(message)
logdebug(LOGGER, 'Dataset publication message handed to rabbit thread.')
logtrace(LOGGER, 'Dataset publication message: %s (%s, version %s).', self.__dataset_handle, self.__drs_id, self.__version_number)
def __make_channel_confirm_delivery(self):
logtrace(LOGGER,
'Set confirm delivery... (Issue Confirm.Select RPC command)')
self.thread._channel.confirm_delivery(
callback=self.confirmer.on_delivery_confirmation)
logdebug(LOGGER, 'Set confirm delivery... done.')
def __start_waiting_for_events(self):
'''
This waits until the whole chain of callback methods triggered by
"trigger_connection_to_rabbit_etc()" has finished, and then starts
waiting for publications.
This is done by starting the ioloop.
Note: In the pika usage example, these things are both called inside the run()
method, so I wonder if this check-and-wait here is necessary. Maybe not.
But the usage example does not implement a Thread, so it probably blocks during
the opening of the connection. Here, as it is a different thread, the run()
might get called before the __init__ has finished? I'd rather stay on the
safe side, as my experience of threading in Python is limited.
'''
# Start ioloop if connection object ready:
if self.thread._connection is not None:
try:
logdebug(LOGGER, 'Starting ioloop...')
logtrace(LOGGER, 'ioloop is owned by connection %s...',
self.thread._connection)
# Tell the main thread that we're now open for events.
# As soon as the thread._connection object is not None anymore, it
# can receive events.
self.thread.tell_publisher_to_stop_waiting_for_thread_to_accept_events(
)
self.thread.continue_gently_closing_if_applicable()
self.thread._connection.ioloop.start()
except pika.exceptions.ProbableAuthenticationError as e:
time_passed = datetime.datetime.now(
) - self.__start_connect_time
logerror(
LOGGER,
'Caught Authentication Exception after %s seconds during connection ("%s").',
time_passed.total_seconds(), e.__class__.__name__)
self.statemachine.set_to_waiting_to_be_available()
self.statemachine.detail_authentication_exception = True # TODO WHAT FOR?
# It seems that ProbableAuthenticationErrors do not cause
# RabbitMQ to call any callback, either on_connection_closed
# or on_connection_error - it just silently swallows the
# problem.
# So we need to manually trigger reconnection to the next
# host here, which we do by manually calling the callback.
errorname = 'ProbableAuthenticationError issued by pika'
self.on_connection_error(self.thread._connection, errorname)
# We start the ioloop, so it can handle the reconnection events,
# or also receive events from the publisher in the meantime.
self.thread._connection.ioloop.start()
except Exception as e:
# This catches any error during connection startup and during the entire
# time the ioloop runs, blocks and waits for events.
logerror(
LOGGER,
'Unexpected error during event listener\'s lifetime: %s: %s',
e.__class__.__name__, e.message)
# As we will try to reconnect, set state to waiting to connect.
# If reconnection fails, it will be set to permanently unavailable.
self.statemachine.set_to_waiting_to_be_available()
# In case this error is reached, it seems that no callback
# was called that handles the problem. Let's try to reconnect
# somewhere else.
errorname = 'Unexpected error (' + str(
e.__class__.__name__) + ': ' + str(e.message) + ')'
self.on_connection_error(self.thread._connection, errorname)
# We start the ioloop, so it can handle the reconnection events,
# or also receive events from the publisher in the meantime.
self.thread._connection.ioloop.start()
else:
# I'm quite sure that this cannot happen, as the connection object
# is created in "trigger_connection_...()" and thus exists, no matter
# if the actual connection to RabbitMQ succeeded (yet) or not.
logdebug(LOGGER,
'This cannot happen: Connection object is not ready.')
logerror(
LOGGER,
'Cannot happen. Cannot properly start the thread. Connection object is not ready.'
)