class MasterControlProgram:
""" Master Control Program keeps track of threads and threading needs """
def __init__(self, config, options):
logging.debug( 'MCP: Master Control Program Created' )
# If we have monitoring enabled for elasic resizing
if config['Monitor']['enabled']:
#TODO: Make this more generic. Just import whatever the user puts here dynamically.
#TODO: Replace 'alice' with 'monitor' or something throughout.
if config['Monitor']['module'] == 'Rabbit':
from rejected.monitors import Rabbit
self.alice = Rabbit()
else:
from rejected.monitors import Alice
self.alice = Alice()
else:
self.alice = None
self.bindings = []
self.config = config
self.last_poll = None
self.shutdown_pending = False
self.thread_stats = {}
def get_information(self):
""" Return the stats data collected from Poll """
pass
def poll(self):
""" Check the Alice daemon for queue depths for each binding """
global mcp_poll_delay
logging.debug( 'MCP: Master Control Program Polling' )
# Cache the monitor queue depth checks
cache_lookup = {}
# default total counts
total_processed = 0
total_throttled = 0
# Get our delay since last poll
if self.last_poll:
duration_since_last_poll = time.time() - self.last_poll
else:
duration_since_last_poll = mcp_poll_delay
# If we're shutting down, no need to do this, can make it take longer
if self.shutdown_pending:
return
# Loop through each binding to ensure all our threads are running
offset = 0
for binding in self.bindings:
# Go through the threads to check the queue depths for each server
dead_threads = []
for x in xrange(0, len(binding['threads'])):
# Make sure the thread is still alive, otherwise remove it and move on
if not binding['threads'][x].isAlive():
logging.error( 'MCP: Encountered a dead thread, removing.' )
dead_threads.append(x)
# Remove dead threads
for list_offset in dead_threads:
logging.error( 'MCP: Removing the dead thread from the stack' )
binding['threads'].pop(list_offset)
# If we don't have any consumer threads, remove the binding
if not len(binding['threads']):
logging.error( 'MCP: We have no working consumers, removing down this binding.' )
del self.bindings[offset]
# Increment our list offset
offset += 1
# If we have removed all of our bindings because they had no working threads, shutdown
if not len(self.bindings):
logging.error( 'MCP: We have no working bindings, shutting down.' )
shutdown()
return
# If we're monitoring, then run through here
if self.alice:
# Loop through each binding
offset = 0
for binding in self.bindings:
# Go through the threads to check the queue depths for each server
for thread in binding['threads']:
# Get our thread data such as the connection and queue it's using
info = thread.get_information()
# Stats are keyed on thread name
thread_name = thread.getName()
# Check our stats info
if thread_name in self.thread_stats:
# Calculate our processed & throttled amount
processed = info['processed'] - self.thread_stats[thread_name]['processed']
throttled = info['throttle_count'] - self.thread_stats[thread_name]['throttle_count']
# Totals for MCP Stats
total_processed += processed
total_throttled += throttled
logging.debug( '%s processed %i messages and throttled %i messages in %.2f seconds at a rate of %.2f mps.' %
( thread_name,
processed,
throttled,
duration_since_last_poll,
( float(processed) / duration_since_last_poll ) ) )
else:
# Initialize our thread stats dictionary
self.thread_stats[thread_name] = {}
# Totals for MCP Stats
total_processed += info['processed']
total_throttled += info['throttle_count']
# Set our thread processed # count for next time
self.thread_stats[thread_name]['processed'] = info['processed']
self.thread_stats[thread_name]['throttle_count'] = info['throttle_count']
# Check the queue depth for the connection and queue
cache_name = '%s-%s' % ( info['connection'], info['queue'] )
if cache_name in cache_lookup:
data = cache_lookup[cache_name]
else:
# Get the value from Alice
data = self.alice.get_queue_depth(info['connection'], info['monitor_port'], info['queue'])
cache_lookup[cache_name] = data
# Easier to work with variables
queue_depth = int(data['depth'])
min_threads = self.config['Bindings'][info['binding']]['consumers']['min']
max_threads = self.config['Bindings'][info['binding']]['consumers']['max']
threshold = self.config['Bindings'][info['binding']]['consumers']['threshold']
# If our queue depth exceeds the threshold and we haven't maxed out make a new worker
if queue_depth > threshold and len(binding['threads']) < max_threads:
logging.info( 'MCP: Spawning worker thread for connection "%s" binding "%s": %i messages pending, %i threshold, %i min, %i max, %i consumers active.' %
( info['connection'],
info['binding'],
queue_depth,
threshold,
min_threads,
max_threads,
len(binding['threads']) ) )
# Create the new thread making it use self.consume
new_thread = ConsumerThread( self.config,
info['binding'],
info['connection'] )
# Add to our dictionary of active threads
binding['threads'].append(new_thread)
# Start the thread
new_thread.start()
# We only want 1 new thread per poll as to not overwhelm the consumer system
break
# Check if our queue depth is below our threshold and we have more than the min amount
if queue_depth < threshold and len(binding['threads']) > min_threads:
logging.info( 'MCP: Removing worker thread for connection "%s" binding "%s": %i messages pending, %i threshold, %i min, %i max, %i threads active.' %
( info['connection'],
info['binding'],
queue_depth,
threshold,
min_threads,
max_threads,
len(binding['threads']) ) )
# Remove a thread
thread = binding['threads'].pop()
while thread.is_locked():
logging.debug( 'MCP: Waiting on %s to unlock so we can shut it down' % thread.getName() )
time.sleep(1)
# Shutdown the thread gracefully
thread.shutdown()
# We only want to remove one thread per poll
break;
logging.info('MCP: Binding #%i processed %i total messages in %.2f seconds at a rate of %.2f mps.' %
( offset,
total_processed,
duration_since_last_poll,
( float(total_processed) / duration_since_last_poll ) ) )
if len(binding['threads']) > 1:
logging.info('MCP: Binding #%i has %i threads which throttled themselves %i times.' %
( offset,
len(binding['threads']),
total_throttled ) )
else:
logging.info('MCP: Binding #%i has 1 thread which throttled itself %i times.' %
( offset, total_throttled ) )
offset += 1
# Get our last poll time
self.last_poll = time.time()
def shutdown(self):
""" Graceful shutdown of the MCP means shutting down threads too """
logging.debug( 'MCP: Master Control Program Shutting Down' )
# Get the thread count
threads = self.threadCount()
# Keep track of the fact we're shutting down
self.shutdown_pending = True
# Loop as long as we have running threads
while threads:
# Loop through all of the bindings and try and shutdown their threads
for binding in self.bindings:
# Loop through all the threads in this binding
for x in xrange(0, len(binding['threads'])):
# Let the thread know we want to shutdown
thread = binding['threads'].pop()
while not thread.shutdown():
logging.debug('MCP: Waiting on %s to shutdown properly' % thread.getName())
time.sleep(1)
# Get our updated thread count and only sleep then loop if it's > 0,
threads = self.threadCount()
# If we have any threads left, sleep for a second before trying again
if threads:
logging.debug( 'MCP: Waiting on %i threads to cleanly shutdown.' % threads )
time.sleep(1)
def start(self):
""" Initialize all of the consumer threads when the MCP comes to life """
logging.debug( 'MCP: Master Control Program Starting Up' )
# Loop through all of the bindings
for binding_name in self.config['Bindings']:
# Create the dictionary values for this binding
binding = { 'name': binding_name }
binding['queue'] = self.config['Bindings'][binding_name]['queue']
binding['threads'] = []
# For each connection, kick off the min consumers and start consuming
for connect_name in self.config['Bindings'][binding_name]['connections']:
for i in xrange( 0, self.config['Bindings'][binding_name]['consumers']['min'] ):
logging.debug( 'MCP: Creating worker thread #%i for connection "%s" binding "%s"' % ( i, connect_name, binding_name ) )
# Create the new thread making it use self.consume
thread = ConsumerThread( self.config,
binding_name,
connect_name );
# Start the thread
thread.start()
# Check to see if the thread is alive before adding it to our stack
if thread.isAlive():
# Add to our dictionary of active threads
binding['threads'].append(thread)
# Append this binding to our binding stack
self.bindings.append(binding)
def threadCount(self):
""" Return the total number of working threads managed by the MCP """
count = 0
for binding in self.bindings:
count += len(binding['threads'])
return count