def _init(self):
log.info("Initializing")
# Set confirm buffer to 1 for heartbeat messages
self.client = AMQPClient(logger=log_amqp, confirm_buffer=1)
# Connect to AMQP host
self.client.connect()
# Declare queues and exchanges
exchange = settings.EXCHANGE_GANETI
exchange_dl = queues.convert_exchange_to_dead(exchange)
self.client.exchange_declare(exchange=exchange,
type="topic")
self.client.exchange_declare(exchange=exchange_dl,
type="topic")
for queue in queues.QUEUES:
# Queues are mirrored to all RabbitMQ brokers
self.client.queue_declare(queue=queue, mirrored=True,
dead_letter_exchange=exchange_dl)
# Declare the corresponding dead-letter queue
queue_dl = queues.convert_queue_to_dead(queue)
self.client.queue_declare(queue=queue_dl, mirrored=True)
# Bind queues to handler methods
for binding in queues.BINDINGS:
try:
callback = getattr(callbacks, binding[3])
except AttributeError:
log.error("Cannot find callback %s", binding[3])
raise SystemExit(1)
queue = binding[0]
exchange = binding[1]
routing_key = binding[2]
self.client.queue_bind(queue=queue, exchange=exchange,
routing_key=routing_key)
self.client.basic_consume(queue=binding[0],
callback=callback,
prefetch_count=5)
queue_dl = queues.convert_queue_to_dead(queue)
exchange_dl = queues.convert_exchange_to_dead(exchange)
# Bind the corresponding dead-letter queue
self.client.queue_bind(queue=queue_dl,
exchange=exchange_dl,
routing_key=routing_key)
log.debug("Binding %s(%s) to queue %s with handler %s",
exchange, routing_key, queue, binding[3])
# Declare the queue that will be used for receiving requests, e.g. a
# status check request
hostname, pid = get_hostname(), os.getpid()
queue = queues.get_dispatcher_request_queue(hostname, pid)
self.client.queue_declare(queue=queue, mirrored=True,
ttl=REQUEST_QUEUE_TTL)
self.client.basic_consume(queue=queue, callback=handle_request)
log.debug("Binding %s(%s) to queue %s with handler 'hadle_request'",
exchange, routing_key, queue)
def main():
usage = "Usage: %s <instance_name>\n" % PROGNAME
if len(sys.argv) != 2:
sys.stderr.write(usage)
return 1
instance_name = sys.argv[1]
# WARNING: This assumes that instance names
# are of the form prefix-id, and uses prefix to
# determine the routekey for AMPQ
prefix = instance_name.split('-')[0]
routekey = "ganeti.%s.event.progress" % prefix
amqp_client = AMQPClient(confirm_buffer=10)
amqp_client.connect()
amqp_client.exchange_declare(settings.EXCHANGE_GANETI, "topic")
for msg in jsonstream(sys.stdin):
msg['event_time'] = split_time(time.time())
msg['instance'] = instance_name
# log to stderr
sys.stderr.write("[MONITOR] %s\n" % json.dumps(msg))
# then send it over AMQP
amqp_client.basic_publish(exchange=settings.EXCHANGE_GANETI,
routing_key=routekey,
body=json.dumps(msg))
amqp_client.close()
return 0
def __init__(self, logger, cluster_name):
pyinotify.ProcessEvent.__init__(self)
self.logger = logger
self.cluster_name = cluster_name
# Set max_retries to 0 for unlimited retries.
self.client = AMQPClient(hosts=settings.AMQP_HOSTS, confirm_buffer=25,
max_retries=0, logger=logger)
logger.info("Attempting to connect to RabbitMQ hosts")
self.client.connect()
logger.info("Connected successfully")
self.ganeti_master = get_ganeti_master()
logger.debug("Ganeti Master Node: %s", self.ganeti_master)
self.ganeti_node = get_ganeti_node()
logger.debug("Current Ganeti Node: %s", self.ganeti_node)
# Check if this is the master node
logger.info("Checking if this is Ganeti Master of %s cluster: %s",
self.cluster_name,
"YES" if self.ganeti_master == self.ganeti_node else "NO")
self.client.exchange_declare(settings.EXCHANGE_GANETI, type='topic')
self.op_handlers = {"INSTANCE": self.process_instance_op,
"NETWORK": self.process_network_op,
"CLUSTER": self.process_cluster_op,
# "GROUP": self.process_group_op}
"TAGS": self.process_tag_op}
def __init__(self, logger, cluster_name):
pyinotify.ProcessEvent.__init__(self)
self.logger = logger
self.cluster_name = cluster_name
# Set max_retries to 0 for unlimited retries.
self.client = AMQPClient(hosts=settings.AMQP_HOSTS,
confirm_buffer=25,
max_retries=0,
logger=logger)
handler_logger.info("Attempting to connect to RabbitMQ hosts")
self.client.connect()
handler_logger.info("Connected succesfully")
self.client.exchange_declare(settings.EXCHANGE_GANETI, type='topic')
self.op_handlers = {
"INSTANCE": self.process_instance_op,
"NETWORK": self.process_network_op,
"CLUSTER": self.process_cluster_op,
# "GROUP": self.process_group_op}
"TAGS": self.process_tag_op
}
def drain_queue(queue):
"""Strip a (declared) queue from all outstanding messages"""
if not queue:
return
if not queue in queues.QUEUES:
print "Queue %s not configured" % queue
return
print "Queue to be drained: %s" % queue
if not get_user_confirmation():
return
client = AMQPClient()
client.connect()
tag = client.basic_consume(queue=queue, callback=callbacks.dummy_proc)
print "Queue draining about to start, hit Ctrl+c when done"
time.sleep(2)
print "Queue draining starting"
num_processed = 0
while True:
client.basic_wait()
num_processed += 1
sys.stderr.write("Ignored %d messages\r" % num_processed)
client.basic_cancel(tag)
client.close()
def __init__(self, **params):
hosts = params['hosts']
self.exchange = params['exchange']
self.client_id = params['client_id']
self.client = AMQPClient(hosts=hosts)
self.client.connect()
self.client.exchange_declare(exchange=self.exchange, type='topic')
def main():
usage = "Usage: %s <instance_name>\n" % PROGNAME
if len(sys.argv) != 2:
sys.stderr.write(usage)
return 1
instance_name = sys.argv[1]
# WARNING: This assumes that instance names
# are of the form prefix-id, and uses prefix to
# determine the routekey for AMPQ
prefix = instance_name.split('-')[0]
routekey = "ganeti.%s.event.progress" % prefix
amqp_client = AMQPClient(confirm_buffer=10)
amqp_client.connect()
amqp_client.exchange_declare(settings.EXCHANGE_GANETI, "topic")
for msg in jsonstream(sys.stdin):
msg['event_time'] = split_time(time.time())
msg['instance'] = instance_name
# log to stderr
sys.stderr.write("[MONITOR] %s\n" % json.dumps(msg))
# then send it over AMQP
amqp_client.basic_publish(exchange=settings.EXCHANGE_GANETI,
routing_key=routekey,
body=json.dumps(msg))
amqp_client.close()
return 0
class Queue(object):
"""Queue.
Required constructor parameters: hosts, exchange, client_id.
"""
def __init__(self, **params):
hosts = params['hosts']
self.exchange = params['exchange']
self.client_id = params['client_id']
self.client = AMQPClient(hosts=hosts)
self.client.connect()
self.client.exchange_declare(exchange=self.exchange,
type='topic')
def send(self, message_key, user, instance, resource, value, details):
body = Message(
self.client_id, user, instance, resource, value, details)
self.client.basic_publish(exchange=self.exchange,
routing_key=message_key,
body=json.dumps(body.__dict__))
def close(self):
self.client.close()
class Queue(object):
"""Queue.
Required constructor parameters: hosts, exchange, client_id.
"""
def __init__(self, **params):
hosts = params['hosts']
self.exchange = params['exchange']
self.client_id = params['client_id']
self.client = AMQPClient(hosts=hosts)
self.client.connect()
self.client.exchange_declare(exchange=self.exchange,
type='topic')
def send(self, message_key, user, instance, resource, value, details):
body = Message(
self.client_id, user, instance, resource, value, details)
self.client.basic_publish(exchange=self.exchange,
routing_key=message_key,
body=json.dumps(body.__dict__))
def close(self):
self.client.close()
def _init(self):
log.info("Initializing")
self.client = AMQPClient(logger=log_amqp)
# Connect to AMQP host
self.client.connect()
# Declare queues and exchanges
exchange = settings.EXCHANGE_GANETI
exchange_dl = queues.convert_exchange_to_dead(exchange)
self.client.exchange_declare(exchange=exchange, type="topic")
self.client.exchange_declare(exchange=exchange_dl, type="topic")
for queue in queues.QUEUES:
# Queues are mirrored to all RabbitMQ brokers
self.client.queue_declare(queue=queue,
mirrored=True,
dead_letter_exchange=exchange_dl)
# Declare the corresponding dead-letter queue
queue_dl = queues.convert_queue_to_dead(queue)
self.client.queue_declare(queue=queue_dl, mirrored=True)
# Bind queues to handler methods
for binding in queues.BINDINGS:
try:
callback = getattr(callbacks, binding[3])
except AttributeError:
log.error("Cannot find callback %s", binding[3])
raise SystemExit(1)
queue = binding[0]
exchange = binding[1]
routing_key = binding[2]
self.client.queue_bind(queue=queue,
exchange=exchange,
routing_key=routing_key)
self.client.basic_consume(queue=binding[0],
callback=callback,
prefetch_count=5)
queue_dl = queues.convert_queue_to_dead(queue)
exchange_dl = queues.convert_exchange_to_dead(exchange)
# Bind the corresponding dead-letter queue
self.client.queue_bind(queue=queue_dl,
exchange=exchange_dl,
routing_key=routing_key)
log.debug("Binding %s(%s) to queue %s with handler %s", exchange,
routing_key, queue, binding[3])
def _init(self):
log.info("Initializing")
# Set confirm buffer to 1 for heartbeat messages
self.client = AMQPClient(logger=log_amqp, confirm_buffer=1)
# Connect to AMQP host
self.client.connect()
# Declare queues and exchanges
exchange = settings.EXCHANGE_GANETI
exchange_dl = queues.convert_exchange_to_dead(exchange)
self.client.exchange_declare(exchange=exchange,
type="topic")
self.client.exchange_declare(exchange=exchange_dl,
type="topic")
for queue in queues.QUEUES:
# Queues are mirrored to all RabbitMQ brokers
self.client.queue_declare(queue=queue, mirrored=True,
dead_letter_exchange=exchange_dl)
# Declare the corresponding dead-letter queue
queue_dl = queues.convert_queue_to_dead(queue)
self.client.queue_declare(queue=queue_dl, mirrored=True)
# Bind queues to handler methods
for binding in queues.BINDINGS:
try:
callback = getattr(callbacks, binding[3])
except AttributeError:
log.error("Cannot find callback %s", binding[3])
raise SystemExit(1)
queue = binding[0]
exchange = binding[1]
routing_key = binding[2]
self.client.queue_bind(queue=queue, exchange=exchange,
routing_key=routing_key)
self.client.basic_consume(queue=binding[0],
callback=callback,
prefetch_count=5)
queue_dl = queues.convert_queue_to_dead(queue)
exchange_dl = queues.convert_exchange_to_dead(exchange)
# Bind the corresponding dead-letter queue
self.client.queue_bind(queue=queue_dl,
exchange=exchange_dl,
routing_key=routing_key)
log.debug("Binding %s(%s) to queue %s with handler %s",
exchange, routing_key, queue, binding[3])
# Declare the queue that will be used for receiving requests, e.g. a
# status check request
hostname, pid = get_hostname(), os.getpid()
queue = queues.get_dispatcher_request_queue(hostname, pid)
self.client.queue_declare(queue=queue, mirrored=True,
ttl=REQUEST_QUEUE_TTL)
self.client.basic_consume(queue=queue, callback=handle_request)
log.debug("Binding %s(%s) to queue %s with handler 'hadle_request'",
exchange, routing_key, queue)
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError("Please provide a queue")
queue = args[0]
interactive = options['interactive']
requeue = options['requeue']
client = AMQPClient()
client.connect()
pp = pprint.PrettyPrinter(indent=4, width=4, stream=self.stdout)
more_msgs = True
counter = 0
sep = '-' * 80
while more_msgs:
msg = client.basic_get(queue=queue)
if msg:
counter += 1
self.stderr.write(sep + "\n")
self.stderr.write('Message %d:\n' % counter)
self.stderr.write(sep + "\n")
pp.pprint(msg)
if not requeue or interactive:
if interactive and not get_user_confirmation():
continue
# Acknowledging the message will remove it from the queue
client.basic_ack(msg)
else:
more_msgs = False
def __init__(self, **params):
hosts = params["hosts"]
self.exchange = params["exchange"]
self.client_id = params["client_id"]
self.client = AMQPClient(hosts=hosts)
self.client.connect()
self.client.exchange_declare(exchange=self.exchange, type="topic")
def __init__(self, **params):
hosts = params['hosts']
self.exchange = params['exchange']
self.client_id = params['client_id']
self.client = AMQPClient(hosts=hosts)
self.client.connect()
self.client.exchange_declare(exchange=self.exchange,
type='topic')
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError("Please provide a queue")
queue = args[0]
interactive = options['interactive']
requeue = options['requeue']
client = AMQPClient()
client.connect()
pp = pprint.PrettyPrinter(indent=4, width=4)
more_msgs = True
counter = 0
sep = '-' * 80
while more_msgs:
msg = client.basic_get(queue=queue)
if msg:
counter += 1
print sep
print 'Message %d:' % counter
print sep
pp.pprint(msg)
if not requeue or interactive:
if interactive and not get_user_confirmation():
continue
# Acknowledging the message will remove it from the queue
client.basic_ack(msg)
else:
more_msgs = False
def _init(self):
log.info("Initializing")
self.client = AMQPClient(logger=log_amqp)
# Connect to AMQP host
self.client.connect()
# Declare queues and exchanges
exchange = settings.EXCHANGE_GANETI
exchange_dl = queues.convert_exchange_to_dead(exchange)
self.client.exchange_declare(exchange=exchange,
type="topic")
self.client.exchange_declare(exchange=exchange_dl,
type="topic")
for queue in queues.QUEUES:
# Queues are mirrored to all RabbitMQ brokers
self.client.queue_declare(queue=queue, mirrored=True,
dead_letter_exchange=exchange_dl)
# Declare the corresponding dead-letter queue
queue_dl = queues.convert_queue_to_dead(queue)
self.client.queue_declare(queue=queue_dl, mirrored=True)
# Bind queues to handler methods
for binding in queues.BINDINGS:
try:
callback = getattr(callbacks, binding[3])
except AttributeError:
log.error("Cannot find callback %s", binding[3])
raise SystemExit(1)
queue = binding[0]
exchange = binding[1]
routing_key = binding[2]
self.client.queue_bind(queue=queue, exchange=exchange,
routing_key=routing_key)
self.client.basic_consume(queue=binding[0],
callback=callback,
prefetch_count=5)
queue_dl = queues.convert_queue_to_dead(queue)
exchange_dl = queues.convert_exchange_to_dead(exchange)
# Bind the corresponding dead-letter queue
self.client.queue_bind(queue=queue_dl,
exchange=exchange_dl,
routing_key=routing_key)
log.debug("Binding %s(%s) to queue %s with handler %s",
exchange, routing_key, queue, binding[3])
def __init__(self, logger, cluster_name):
pyinotify.ProcessEvent.__init__(self)
self.logger = logger
self.cluster_name = cluster_name
# Set max_retries to 0 for unlimited retries.
self.client = AMQPClient(hosts=settings.AMQP_HOSTS, confirm_buffer=25,
max_retries=0, logger=logger)
handler_logger.info("Attempting to connect to RabbitMQ hosts")
self.client.connect()
handler_logger.info("Connected succesfully")
self.client.exchange_declare(settings.EXCHANGE_GANETI, type='topic')
self.op_handlers = {"INSTANCE": self.process_instance_op,
"NETWORK": self.process_network_op}
def purge_queues():
"""
Delete declared queues from RabbitMQ. Use with care!
"""
client = AMQPClient(max_retries=120)
client.connect()
print "Queues to be deleted: ", queues.QUEUES
if not get_user_confirmation():
return
for queue in queues.QUEUES:
result = client.queue_delete(queue=queue)
print "Deleting queue %s. Result: %s" % (queue, result)
client.close()
def purge_exchanges():
"""Delete declared exchanges from RabbitMQ, after removing all queues"""
purge_queues()
client = AMQPClient()
client.connect()
exchanges = queues.EXCHANGES
print "Exchanges to be deleted: ", exchanges
if not get_user_confirmation():
return
for exch in exchanges:
result = client.exchange_delete(exchange=exch)
print "Deleting exchange %s. Result: %s" % (exch, result)
client.close()
def purge_exchanges():
"""Delete declared exchanges from RabbitMQ, after removing all queues"""
purge_queues()
client = AMQPClient()
client.connect()
exchanges = queues.EXCHANGES
print "Exchanges to be deleted: ", exchanges
if not get_user_confirmation():
return
for exch in exchanges:
result = client.exchange_delete(exchange=exch)
print "Deleting exchange %s. Result: %s" % (exch, result)
client.close()
def purge_queues():
"""
Delete declared queues from RabbitMQ. Use with care!
"""
client = AMQPClient(max_retries=120)
client.connect()
print "Queues to be deleted: ", queues.QUEUES
if not get_user_confirmation():
return
for queue in queues.QUEUES:
result = client.queue_delete(queue=queue)
print "Deleting queue %s. Result: %s" % (queue, result)
client.close()
def handle(self, *args, **options):
verbose = (options["verbosity"] == "2")
self.keep_zombies = options["keep_zombies"]
log_level = logging.DEBUG if verbose else logging.WARNING
log.setLevel(log_level)
client = AMQPClient(confirms=False)
client.connect()
self.client = client
for queue in queues.QUEUES:
dead_queue = queues.convert_queue_to_dead(queue)
while 1:
message = client.basic_get(dead_queue)
if not message:
break
log.debug("Received message %s", message)
self.handle_message(message)
client.close()
return 0
def drain_queue(queue):
"""Strip a (declared) queue from all outstanding messages"""
if not queue:
return
if not queue in queues.QUEUES:
print "Queue %s not configured" % queue
return
print "Queue to be drained: %s" % queue
if not get_user_confirmation():
return
client = AMQPClient()
client.connect()
tag = client.basic_consume(queue=queue, callback=callbacks.dummy_proc)
print "Queue draining about to start, hit Ctrl+c when done"
time.sleep(2)
print "Queue draining starting"
num_processed = 0
while True:
client.basic_wait()
num_processed += 1
sys.stderr.write("Ignored %d messages\r" % num_processed)
client.basic_cancel(tag)
client.close()
class Dispatcher:
debug = False
def __init__(self, debug=False):
self.debug = debug
self._init()
def wait(self):
log.info("Waiting for messages..")
timeout = 600
while True:
try:
# Close the Django DB connection before processing
# every incoming message. This plays nicely with
# DB connection pooling, if enabled and allows
# the dispatcher to recover from broken connections
# gracefully.
close_connection()
msg = self.client.basic_wait(timeout=timeout)
if not msg:
log.warning(
"Idle connection for %d seconds. Will connect"
" to a different host. Verify that"
" snf-ganeti-eventd is running!!",
timeout,
)
self.client.reconnect()
except SystemExit:
break
except Exception as e:
log.exception("Caught unexpected exception: %s", e)
self.client.basic_cancel()
self.client.close()
def _init(self):
log.info("Initializing")
self.client = AMQPClient(logger=log_amqp)
# Connect to AMQP host
self.client.connect()
# Declare queues and exchanges
exchange = settings.EXCHANGE_GANETI
exchange_dl = queues.convert_exchange_to_dead(exchange)
self.client.exchange_declare(exchange=exchange, type="topic")
self.client.exchange_declare(exchange=exchange_dl, type="topic")
for queue in queues.QUEUES:
# Queues are mirrored to all RabbitMQ brokers
self.client.queue_declare(queue=queue, mirrored=True, dead_letter_exchange=exchange_dl)
# Declare the corresponding dead-letter queue
queue_dl = queues.convert_queue_to_dead(queue)
self.client.queue_declare(queue=queue_dl, mirrored=True)
# Bind queues to handler methods
for binding in queues.BINDINGS:
try:
callback = getattr(callbacks, binding[3])
except AttributeError:
log.error("Cannot find callback %s", binding[3])
raise SystemExit(1)
queue = binding[0]
exchange = binding[1]
routing_key = binding[2]
self.client.queue_bind(queue=queue, exchange=exchange, routing_key=routing_key)
self.client.basic_consume(queue=binding[0], callback=callback, prefetch_count=5)
queue_dl = queues.convert_queue_to_dead(queue)
exchange_dl = queues.convert_exchange_to_dead(exchange)
# Bind the corresponding dead-letter queue
self.client.queue_bind(queue=queue_dl, exchange=exchange_dl, routing_key=routing_key)
log.debug("Binding %s(%s) to queue %s with handler %s", exchange, routing_key, queue, binding[3])
def main():
parser = OptionParser()
parser.add_option('-v',
'--verbose',
action='store_true',
default=False,
dest='verbose',
help='Enable verbose logging')
parser.add_option('--host',
default=BROKER_HOST,
dest='host',
help='RabbitMQ host (default: %s)' % BROKER_HOST)
parser.add_option('--port',
default=BROKER_PORT,
dest='port',
help='RabbitMQ port (default: %s)' % BROKER_PORT,
type='int')
parser.add_option('--user',
default=BROKER_USER,
dest='user',
help='RabbitMQ user (default: %s)' % BROKER_USER)
parser.add_option('--password',
default=BROKER_PASSWORD,
dest='password',
help='RabbitMQ password (default: %s)' % BROKER_PASSWORD)
parser.add_option('--vhost',
default=BROKER_VHOST,
dest='vhost',
help='RabbitMQ vhost (default: %s)' % BROKER_VHOST)
parser.add_option('--queue',
default=CONSUMER_QUEUE,
dest='queue',
help='RabbitMQ queue (default: %s)' % CONSUMER_QUEUE)
parser.add_option('--exchange',
default=CONSUMER_EXCHANGE,
dest='exchange',
help='RabbitMQ exchange (default: %s)' %
CONSUMER_EXCHANGE)
parser.add_option('--key',
default=CONSUMER_KEY,
dest='key',
help='RabbitMQ key (default: %s)' % CONSUMER_KEY)
parser.add_option('--callback',
default=None,
dest='callback',
help='Callback function to consume messages')
parser.add_option('--test',
action='store_true',
default=False,
dest='test',
help='Produce a dummy message for testing')
opts, args = parser.parse_args()
DEBUG = False
if opts.verbose:
DEBUG = True
logging.basicConfig(
format='%(asctime)s [%(levelname)s] %(name)s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.DEBUG if DEBUG else logging.INFO)
logger = logging.getLogger('dispatcher')
host = 'amqp://%s:%s@%s:%s' % (opts.user, opts.password, opts.host,
opts.port)
queue = opts.queue
key = opts.key
exchange = opts.exchange
client = AMQPClient(hosts=[host])
client.connect()
if opts.test:
client.exchange_declare(exchange=exchange, type='topic')
client.basic_publish(exchange=exchange,
routing_key=key,
body=json.dumps({"test": "0123456789"}))
client.close()
sys.exit()
callback = None
if opts.callback:
cb = opts.callback.rsplit('.', 1)
if len(cb) == 2:
__import__(cb[0])
cb_module = sys.modules[cb[0]]
callback = getattr(cb_module, cb[1])
def handle_message(client, msg):
logger.debug('%s', msg)
if callback:
callback(msg)
client.basic_ack(msg)
client.queue_declare(queue=queue)
client.queue_bind(queue=queue, exchange=exchange, routing_key=key)
client.basic_consume(queue=queue, callback=handle_message)
try:
while True:
client.basic_wait()
except KeyboardInterrupt:
pass
finally:
client.close()
class Dispatcher:
debug = False
def __init__(self, debug=False):
self.debug = debug
self._init()
def wait(self):
log.info("Waiting for messages..")
timeout = DISPATCHER_RECONNECT_TIMEOUT
while True:
try:
# Close the Django DB connection before processing
# every incoming message. This plays nicely with
# DB connection pooling, if enabled and allows
# the dispatcher to recover from broken connections
# gracefully.
close_connection()
msg = self.client.basic_wait(timeout=timeout)
if not msg:
log.warning(
"Idle connection for %d seconds. Will connect"
" to a different host. Verify that"
" snf-ganeti-eventd is running!!", timeout)
self.client.reconnect(timeout=1)
except AMQPConnectionError as e:
log.error("AMQP connection failed: %s" % e)
log.warning("Sleeping for %d seconds before retrying to "
"connect to an AMQP broker" %
DISPATCHER_FAILED_CONNECTION_WAIT)
time.sleep(DISPATCHER_FAILED_CONNECTION_WAIT)
except select.error as e:
if e[0] != errno.EINTR:
log.exception("Caught unexpected exception: %s", e)
log.warning("Sleeping for %d seconds before retrying to "
"connect to an AMQP broker" %
DISPATCHER_FAILED_CONNECTION_WAIT)
time.sleep(DISPATCHER_FAILED_CONNECTION_WAIT)
else:
break
except (SystemExit, KeyboardInterrupt):
break
except Exception as e:
log.exception("Caught unexpected exception: %s", e)
log.warning("Sleeping for %d seconds before retrying to "
"connect to an AMQP broker" %
DISPATCHER_FAILED_CONNECTION_WAIT)
time.sleep(DISPATCHER_FAILED_CONNECTION_WAIT)
log.info("Clean up AMQP connection before exit")
self.client.basic_cancel(timeout=1)
self.client.close(timeout=1)
def _init(self):
log.info("Initializing")
# Set confirm buffer to 1 for heartbeat messages
self.client = AMQPClient(logger=log_amqp, confirm_buffer=1)
# Connect to AMQP host
self.client.connect()
# Declare queues and exchanges
exchange = settings.EXCHANGE_GANETI
exchange_dl = queues.convert_exchange_to_dead(exchange)
self.client.exchange_declare(exchange=exchange, type="topic")
self.client.exchange_declare(exchange=exchange_dl, type="topic")
for queue in queues.QUEUES:
# Queues are mirrored to all RabbitMQ brokers
self.client.queue_declare(queue=queue,
mirrored=True,
dead_letter_exchange=exchange_dl)
# Declare the corresponding dead-letter queue
queue_dl = queues.convert_queue_to_dead(queue)
self.client.queue_declare(queue=queue_dl, mirrored=True)
# Bind queues to handler methods
for binding in queues.BINDINGS:
try:
callback = getattr(callbacks, binding[3])
except AttributeError:
log.error("Cannot find callback %s", binding[3])
raise SystemExit(1)
queue = binding[0]
exchange = binding[1]
routing_key = binding[2]
self.client.queue_bind(queue=queue,
exchange=exchange,
routing_key=routing_key)
self.client.basic_consume(queue=binding[0],
callback=callback,
prefetch_count=5)
queue_dl = queues.convert_queue_to_dead(queue)
exchange_dl = queues.convert_exchange_to_dead(exchange)
# Bind the corresponding dead-letter queue
self.client.queue_bind(queue=queue_dl,
exchange=exchange_dl,
routing_key=routing_key)
log.debug("Binding %s(%s) to queue %s with handler %s", exchange,
routing_key, queue, binding[3])
# Declare the queue that will be used for receiving requests, e.g. a
# status check request
hostname, pid = get_hostname(), os.getpid()
queue = queues.get_dispatcher_request_queue(hostname, pid)
self.client.queue_declare(queue=queue,
mirrored=True,
ttl=REQUEST_QUEUE_TTL)
self.client.basic_consume(queue=queue, callback=handle_request)
log.debug("Binding %s(%s) to queue %s with handler 'hadle_request'",
exchange, routing_key, queue)
def check_dispatcher_status(pid_file):
"""Check the status of a running snf-dispatcher process.
Check the status of a running snf-dispatcher process, the PID of which is
contained in the 'pid_file'. This function will send a 'status-check'
message to the running snf-dispatcher, wait for dispatcher's response and
pretty-print the results.
"""
dispatcher_pid = pidlockfile.read_pid_from_pidfile(pid_file)
if dispatcher_pid is None:
sys.stdout.write("snf-dispatcher with PID file '%s' is not running."
" PID file does not exist\n" % pid_file)
sys.exit(1)
sys.stdout.write("snf-dispatcher (PID: %s): running\n" % dispatcher_pid)
hostname = get_hostname()
local_queue = "snf:check_tool:%s:%s" % (hostname, os.getpid())
dispatcher_queue = queues.get_dispatcher_request_queue(
hostname, dispatcher_pid)
log_amqp.setLevel(logging.WARNING)
try:
client = AMQPClient(logger=log_amqp)
client.connect()
client.queue_declare(queue=local_queue, mirrored=False, exclusive=True)
client.basic_consume(queue=local_queue,
callback=lambda x, y: 0,
no_ack=True)
msg = json.dumps({"action": "status-check", "reply_to": local_queue})
client.basic_publish("", dispatcher_queue, msg)
except:
sys.stdout.write("Error while connecting with AMQP\nError:\n")
traceback.print_exc()
sys.exit(1)
sys.stdout.write("AMQP -> snf-dispatcher: ")
msg = client.basic_wait(timeout=CHECK_TOOL_ACK_TIMEOUT)
if msg is None:
sys.stdout.write("fail\n")
sys.stdout.write("ERROR: No reply from snf-dipatcher after '%s'"
" seconds.\n" % CHECK_TOOL_ACK_TIMEOUT)
sys.exit(1)
else:
try:
body = json.loads(msg["body"])
assert (body["action"] == "status-check"), "Invalid action"
assert (body["status"] == "started"), "Invalid status"
sys.stdout.write("ok\n")
except Exception as e:
sys.stdout.write("Received invalid msg from snf-dispatcher:"
" msg: %s error: %s\n" % (msg, e))
sys.exit(1)
msg = client.basic_wait(timeout=CHECK_TOOL_REPORT_TIMEOUT)
if msg is None:
sys.stdout.write("fail\n")
sys.stdout.write("ERROR: No status repot after '%s' seconds.\n" %
CHECK_TOOL_REPORT_TIMEOUT)
sys.exit(1)
sys.stdout.write("Backends:\n")
status = json.loads(msg["body"])["status"]
for backend, bstatus in sorted(status.items()):
sys.stdout.write(" * %s: \n" % backend)
sys.stdout.write(" snf-dispatcher -> ganeti: %s\n" % bstatus["RAPI"])
sys.stdout.write(" snf-ganeti-eventd -> AMQP: %s\n" %
bstatus["eventd"])
sys.exit(0)
def check_dispatcher_status(pid_file):
"""Check the status of a running snf-dispatcher process.
Check the status of a running snf-dispatcher process, the PID of which is
contained in the 'pid_file'. This function will send a 'status-check'
message to the running snf-dispatcher, wait for dispatcher's response and
pretty-print the results.
"""
dispatcher_pid = pidlockfile.read_pid_from_pidfile(pid_file)
if dispatcher_pid is None:
sys.stdout.write("snf-dispatcher with PID file '%s' is not running."
" PID file does not exist\n" % pid_file)
sys.exit(1)
sys.stdout.write("snf-dispatcher (PID: %s): running\n" % dispatcher_pid)
hostname = get_hostname()
local_queue = "snf:check_tool:%s:%s" % (hostname, os.getpid())
dispatcher_queue = queues.get_dispatcher_request_queue(hostname,
dispatcher_pid)
log_amqp.setLevel(logging.WARNING)
try:
client = AMQPClient(logger=log_amqp)
client.connect()
client.queue_declare(queue=local_queue, mirrored=False, exclusive=True)
client.basic_consume(queue=local_queue, callback=lambda x, y: 0,
no_ack=True)
msg = json.dumps({"action": "status-check", "reply_to": local_queue})
client.basic_publish("", dispatcher_queue, msg)
except:
sys.stdout.write("Error while connecting with AMQP\nError:\n")
traceback.print_exc()
sys.exit(1)
sys.stdout.write("AMQP -> snf-dispatcher: ")
msg = client.basic_wait(timeout=CHECK_TOOL_ACK_TIMEOUT)
if msg is None:
sys.stdout.write("fail\n")
sys.stdout.write("ERROR: No reply from snf-dipatcher after '%s'"
" seconds.\n" % CHECK_TOOL_ACK_TIMEOUT)
sys.exit(1)
else:
try:
body = json.loads(msg["body"])
assert(body["action"] == "status-check"), "Invalid action"
assert(body["status"] == "started"), "Invalid status"
sys.stdout.write("ok\n")
except Exception as e:
sys.stdout.write("Received invalid msg from snf-dispatcher:"
" msg: %s error: %s\n" % (msg, e))
sys.exit(1)
msg = client.basic_wait(timeout=CHECK_TOOL_REPORT_TIMEOUT)
if msg is None:
sys.stdout.write("fail\n")
sys.stdout.write("ERROR: No status repot after '%s' seconds.\n"
% CHECK_TOOL_REPORT_TIMEOUT)
sys.exit(1)
sys.stdout.write("Backends:\n")
status = json.loads(msg["body"])["status"]
for backend, bstatus in sorted(status.items()):
sys.stdout.write(" * %s: \n" % backend)
sys.stdout.write(" snf-dispatcher -> ganeti: %s\n" %
bstatus["RAPI"])
sys.stdout.write(" snf-ganeti-eventd -> AMQP: %s\n" %
bstatus["eventd"])
sys.exit(0)
class JobFileHandler(pyinotify.ProcessEvent):
def __init__(self, logger, cluster_name):
pyinotify.ProcessEvent.__init__(self)
self.logger = logger
self.cluster_name = cluster_name
# Set max_retries to 0 for unlimited retries.
self.client = AMQPClient(hosts=settings.AMQP_HOSTS, confirm_buffer=25,
max_retries=0, logger=logger)
logger.info("Attempting to connect to RabbitMQ hosts")
self.client.connect()
logger.info("Connected successfully")
self.ganeti_master = get_ganeti_master()
logger.debug("Ganeti Master Node: %s", self.ganeti_master)
self.ganeti_node = get_ganeti_node()
logger.debug("Current Ganeti Node: %s", self.ganeti_node)
# Check if this is the master node
logger.info("Checking if this is Ganeti Master of %s cluster: %s",
self.cluster_name,
"YES" if self.ganeti_master == self.ganeti_node else "NO")
self.client.exchange_declare(settings.EXCHANGE_GANETI, type='topic')
self.op_handlers = {"INSTANCE": self.process_instance_op,
"NETWORK": self.process_network_op,
"CLUSTER": self.process_cluster_op,
# "GROUP": self.process_group_op}
"TAGS": self.process_tag_op}
def process_IN_CLOSE_WRITE(self, event):
self.process_IN_MOVED_TO(event)
def process_IN_MOVED_TO(self, event):
jobfile = os.path.join(event.path, event.name)
if not event.name.startswith("job-"):
self.logger.debug("Not a job file: %s" % event.path)
return
try:
data = utils.ReadFile(jobfile)
except IOError:
return
data = serializer.LoadJson(data)
job = jqueue._QueuedJob.Restore(None, data, False, False)
job_id = int(job.id)
for op in job.ops:
op_id = op.input.OP_ID
msg = None
try:
handler_fn = self.op_handlers[op_id.split('_')[1]]
msg, routekey = handler_fn(op, job_id)
except KeyError:
pass
if not msg:
self.logger.debug("Ignoring job: %s: %s", job_id, op_id)
continue
# Generate a unique message identifier
event_time = get_time_from_status(op, job)
# Get the last line of the op log as message
try:
logmsg = op.log[-1][-1]
except IndexError:
logmsg = None
# Add shared attributes for all operations
msg.update({"event_time": event_time,
"operation": op_id,
"status": op.status,
"cluster": self.cluster_name,
"logmsg": logmsg,
"result": op.result,
"jobId": job_id})
if op.status == "success":
msg["result"] = op.result
if op_id == "OP_INSTANCE_CREATE" and op.status == "error":
# In case an instance creation fails send the job input
# so that the job can be retried if needed.
msg["job_fields"] = op.Serialize()["input"]
# Check if this is the master node. Only the master node should
# deliver messages to RabbitMQ.
current_master = get_ganeti_master()
if self.ganeti_master != current_master:
self.logger.warning("Ganeti Master changed! New Master: %s",
current_master)
if self.ganeti_node == current_master:
self.logger.info("This node became Ganeti Master.")
else:
self.logger.info("This node is not Ganeti Master.")
self.ganeti_master = current_master
msg = json.dumps(msg)
if self.ganeti_node != self.ganeti_master:
self.logger.debug(
"Ignoring msg for job: %s: %s. Reason: Not Master",
job_id, op_id)
continue
self.logger.debug("Delivering msg: %s (key=%s)", msg, routekey)
# Send the message to RabbitMQ. Since the master node test and the
# message delivery isn't atomic, race conditions may occur. We can
# live with that.
self.client.basic_publish(settings.EXCHANGE_GANETI,
routekey,
msg)
def process_instance_op(self, op, job_id):
""" Process OP_INSTANCE_* opcodes.
"""
input = op.input
op_id = input.OP_ID
instances = None
instances = get_field(input, 'instance_name')
if not instances:
instances = get_field(input, 'instances')
if not instances or len(instances) > 1:
# Do not publish messages for jobs with no or multiple
# instances. Currently snf-dispatcher can not normally handle
# these messages
return None, None
else:
instances = instances[0]
self.logger.debug("Job: %d: %s(%s) %s", job_id, op_id,
instances, op.status)
job_fields = {}
if op_id in ["OP_INSTANCE_SET_PARAMS", "OP_INSTANCE_CREATE"]:
job_fields = {"nics": get_field(input, "nics"),
"disks": get_field(input, "disks"),
"hvparams": get_field(input, "hvparams"),
"beparams": get_field(input, "beparams")}
elif op_id == "OP_INSTANCE_SNAPSHOT":
# Cyclades store the UUID of the snapshot as the 'reason' attribute
# of the Ganeti job in order to be able to update the status of
# the snapshot based on the result of the Ganeti job. Parse this
# attribute and include it in the msg.
# NOTE: This will fill the 'snapshot_info' attribute only for the
# first disk, but this is ok because Cyclades do not issue jobs to
# create snapshots of many disks.
disks = get_field(input, "disks")
if disks:
reason = get_field(input, "reason")
snapshot_info = None
try:
reason = reason[0]
assert (reason[0] == "gnt:user")
snapshot_info = reason[1]
disks[0][1]["snapshot_info"] = snapshot_info
except:
self.logger.warning("Malformed snapshot job '%s'", job_id)
job_fields = {"disks": disks}
msg = {"type": "ganeti-op-status",
"instance": instances,
"operation": op_id,
"job_fields": job_fields}
if ((op_id in ["OP_INSTANCE_CREATE", "OP_INSTANCE_STARTUP"] and
op.status == "success") or
(op_id in ["OP_INSTANCE_SET_PARAMS", "OP_INSTANCE_GROW_DISK"] and
op.status in ["success", "error", "cancelled"])):
instance_info = get_instance_info(msg["instance"], self.logger)
msg["instance_nics"] = instance_info["nics"]
msg["instance_disks"] = instance_info["disks"]
msg["instance_hvparams"] = instance_info["hvparams"]
routekey = "ganeti.%s.event.op" % prefix_from_name(instances)
return msg, routekey
def process_network_op(self, op, job_id):
""" Process OP_NETWORK_* opcodes.
"""
input = op.input
op_id = input.OP_ID
network_name = get_field(input, 'network_name')
if not network_name:
return None, None
self.logger.debug("Job: %d: %s(%s) %s", job_id, op_id,
network_name, op.status)
job_fields = {
'subnet': get_field(input, 'network'),
'gateway': get_field(input, 'gateway'),
"add_reserved_ips": get_field(input, "add_reserved_ips"),
"remove_reserved_ips": get_field(input, "remove_reserved_ips"),
# 'network_mode': get_field(input, 'network_mode'),
# 'network_link': get_field(input, 'network_link'),
'group_name': get_field(input, 'group_name')}
msg = {'operation': op_id,
'type': "ganeti-network-status",
'network': network_name,
'job_fields': job_fields}
routekey = "ganeti.%s.event.network" % prefix_from_name(network_name)
return msg, routekey
def process_cluster_op(self, op, job_id):
""" Process OP_CLUSTER_* opcodes.
"""
input = op.input
op_id = input.OP_ID
self.logger.debug("Job: %d: %s %s", job_id, op_id, op.status)
if op_id != "OP_CLUSTER_SET_PARAMS":
# Send only modifications of cluster
return None, None
msg = {'operation': op_id,
'type': "ganeti-cluster-status"}
routekey = "ganeti.event.cluster"
return msg, routekey
def process_tag_op(self, op, job_id):
""" Process OP_TAGS_* opcodes.
"""
input = op.input
op_id = input.OP_ID
if op_id == "OP_TAGS_SET":
# NOTE: Check 'dry_run' after 'cluster' because networks and groups
# do not support the 'dry_run' option.
if (op.status == "waiting" and input.tags and
input.kind == "cluster" and input.dry_run):
# Special where a prefixed cluster tag operation in dry-run
# mode is used in order to trigger eventd to send a
# heartbeat message.
tag = input.tags[0]
if tag.startswith("snf:eventd:heartbeat"):
self.logger.debug("Received heartbeat tag '%s'."
" Sending response.", tag)
msg = {"type": "eventd-heartbeat",
"cluster": self.cluster_name}
return msg, "eventd.heartbeat"
return None, None
class JobFileHandler(pyinotify.ProcessEvent):
def __init__(self, logger, cluster_name):
pyinotify.ProcessEvent.__init__(self)
self.logger = logger
self.cluster_name = cluster_name
# Set max_retries to 0 for unlimited retries.
self.client = AMQPClient(hosts=settings.AMQP_HOSTS,
confirm_buffer=25,
max_retries=0,
logger=logger)
handler_logger.info("Attempting to connect to RabbitMQ hosts")
self.client.connect()
handler_logger.info("Connected succesfully")
self.client.exchange_declare(settings.EXCHANGE_GANETI, type='topic')
self.op_handlers = {
"INSTANCE": self.process_instance_op,
"NETWORK": self.process_network_op,
"CLUSTER": self.process_cluster_op
}
# "GROUP": self.process_group_op}
def process_IN_CLOSE_WRITE(self, event):
self.process_IN_MOVED_TO(event)
def process_IN_MOVED_TO(self, event):
jobfile = os.path.join(event.path, event.name)
if not event.name.startswith("job-"):
self.logger.debug("Not a job file: %s" % event.path)
return
try:
data = utils.ReadFile(jobfile)
except IOError:
return
data = serializer.LoadJson(data)
job = jqueue._QueuedJob.Restore(None, data, False, False)
job_id = int(job.id)
for op in job.ops:
op_id = op.input.OP_ID
msg = None
try:
handler_fn = self.op_handlers[op_id.split('_')[1]]
msg, routekey = handler_fn(op, job_id)
except KeyError:
pass
if not msg:
self.logger.debug("Ignoring job: %s: %s", job_id, op_id)
continue
# Generate a unique message identifier
event_time = get_time_from_status(op, job)
# Get the last line of the op log as message
try:
logmsg = op.log[-1][-1]
except IndexError:
logmsg = None
# Add shared attributes for all operations
msg.update({
"event_time": event_time,
"operation": op_id,
"status": op.status,
"cluster": self.cluster_name,
"logmsg": logmsg,
"result": op.result,
"jobId": job_id
})
if op.status == "success":
msg["result"] = op.result
if op_id == "OP_INSTANCE_CREATE" and op.status == "error":
# In case an instance creation fails send the job input
# so that the job can be retried if needed.
msg["job_fields"] = op.Serialize()["input"]
msg = json.dumps(msg)
self.logger.debug("Delivering msg: %s (key=%s)", msg, routekey)
# Send the message to RabbitMQ
self.client.basic_publish(settings.EXCHANGE_GANETI, routekey, msg)
def process_instance_op(self, op, job_id):
""" Process OP_INSTANCE_* opcodes.
"""
input = op.input
op_id = input.OP_ID
instances = None
instances = get_field(input, 'instance_name')
if not instances:
instances = get_field(input, 'instances')
if not instances or len(instances) > 1:
# Do not publish messages for jobs with no or multiple
# instances. Currently snf-dispatcher can not normally handle
# these messages
return None, None
else:
instances = instances[0]
self.logger.debug("Job: %d: %s(%s) %s", job_id, op_id, instances,
op.status)
job_fields = {}
if op_id in ["OP_INSTANCE_SET_PARAMS", "OP_INSTANCE_CREATE"]:
job_fields = {
"nics": get_field(input, "nics"),
"disks": get_field(input, "disks"),
"beparams": get_field(input, "beparams")
}
msg = {
"type": "ganeti-op-status",
"instance": instances,
"operation": op_id,
"job_fields": job_fields
}
if ((op_id in ["OP_INSTANCE_CREATE", "OP_INSTANCE_STARTUP"]
and op.status == "success")
or (op_id == "OP_INSTANCE_SET_PARAMS"
and op.status in ["success", "error", "cancelled"])):
nics = get_instance_nics(msg["instance"], self.logger)
msg["instance_nics"] = nics
routekey = "ganeti.%s.event.op" % prefix_from_name(instances)
return msg, routekey
def process_network_op(self, op, job_id):
""" Process OP_NETWORK_* opcodes.
"""
input = op.input
op_id = input.OP_ID
network_name = get_field(input, 'network_name')
if not network_name:
return None, None
self.logger.debug("Job: %d: %s(%s) %s", job_id, op_id, network_name,
op.status)
job_fields = {
'subnet': get_field(input, 'network'),
'gateway': get_field(input, 'gateway'),
"add_reserved_ips": get_field(input, "add_reserved_ips"),
"remove_reserved_ips": get_field(input, "remove_reserved_ips"),
# 'network_mode': get_field(input, 'network_mode'),
# 'network_link': get_field(input, 'network_link'),
'group_name': get_field(input, 'group_name')
}
msg = {
'operation': op_id,
'type': "ganeti-network-status",
'network': network_name,
'job_fields': job_fields
}
routekey = "ganeti.%s.event.network" % prefix_from_name(network_name)
return msg, routekey
def process_cluster_op(self, op, job_id):
""" Process OP_CLUSTER_* opcodes.
"""
input = op.input
op_id = input.OP_ID
self.logger.debug("Job: %d: %s %s", job_id, op_id, op.status)
if op_id != "OP_CLUSTER_SET_PARAMS":
# Send only modifications of cluster
return None, None
msg = {'operation': op_id, 'type': "ganeti-cluster-status"}
routekey = "ganeti.event.cluster"
return msg, routekey
def main():
parser = OptionParser()
parser.add_option('-v', '--verbose', action='store_true', default=False,
dest='verbose', help='Enable verbose logging')
parser.add_option('--host', default=BROKER_HOST, dest='host',
help='RabbitMQ host (default: %s)' % BROKER_HOST)
parser.add_option('--port', default=BROKER_PORT, dest='port',
help='RabbitMQ port (default: %s)' % BROKER_PORT, type='int')
parser.add_option('--user', default=BROKER_USER, dest='user',
help='RabbitMQ user (default: %s)' % BROKER_USER)
parser.add_option('--password', default=BROKER_PASSWORD, dest='password',
help='RabbitMQ password (default: %s)' % BROKER_PASSWORD)
parser.add_option('--vhost', default=BROKER_VHOST, dest='vhost',
help='RabbitMQ vhost (default: %s)' % BROKER_VHOST)
parser.add_option('--queue', default=CONSUMER_QUEUE, dest='queue',
help='RabbitMQ queue (default: %s)' % CONSUMER_QUEUE)
parser.add_option('--exchange', default=CONSUMER_EXCHANGE, dest='exchange',
help='RabbitMQ exchange (default: %s)' % CONSUMER_EXCHANGE)
parser.add_option('--key', default=CONSUMER_KEY, dest='key',
help='RabbitMQ key (default: %s)' % CONSUMER_KEY)
parser.add_option('--callback', default=None, dest='callback',
help='Callback function to consume messages')
parser.add_option('--test', action='store_true', default=False,
dest='test', help='Produce a dummy message for testing')
opts, args = parser.parse_args()
DEBUG = False
if opts.verbose:
DEBUG = True
logging.basicConfig(
format='%(asctime)s [%(levelname)s] %(name)s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.DEBUG if DEBUG else logging.INFO)
logger = logging.getLogger('dispatcher')
host = 'amqp://%s:%s@%s:%s' % (opts.user, opts.password, opts.host, opts.port)
queue = opts.queue
key = opts.key
exchange = opts.exchange
client = AMQPClient(hosts=[host])
client.connect()
if opts.test:
client.exchange_declare(exchange=exchange,
type='topic')
client.basic_publish(exchange=exchange,
routing_key=key,
body= json.dumps({"test": "0123456789"}))
client.close()
sys.exit()
callback = None
if opts.callback:
cb = opts.callback.rsplit('.', 1)
if len(cb) == 2:
__import__(cb[0])
cb_module = sys.modules[cb[0]]
callback = getattr(cb_module, cb[1])
def handle_message(client, msg):
logger.debug('%s', msg)
if callback:
callback(msg)
client.basic_ack(msg)
client.queue_declare(queue=queue)
client.queue_bind(queue=queue,
exchange=exchange,
routing_key=key)
client.basic_consume(queue=queue, callback=handle_message)
try:
while True:
client.basic_wait()
except KeyboardInterrupt:
pass
finally:
client.close()
class Dispatcher:
debug = False
def __init__(self, debug=False):
self.debug = debug
self._init()
def wait(self):
log.info("Waiting for messages..")
timeout = 600
while True:
try:
# Close the Django DB connection before processing
# every incoming message. This plays nicely with
# DB connection pooling, if enabled and allows
# the dispatcher to recover from broken connections
# gracefully.
close_connection()
msg = self.client.basic_wait(timeout=timeout)
if not msg:
log.warning(
"Idle connection for %d seconds. Will connect"
" to a different host. Verify that"
" snf-ganeti-eventd is running!!", timeout)
self.client.reconnect()
except SystemExit:
break
except Exception as e:
log.exception("Caught unexpected exception: %s", e)
self.client.basic_cancel()
self.client.close()
def _init(self):
log.info("Initializing")
self.client = AMQPClient(logger=log_amqp)
# Connect to AMQP host
self.client.connect()
# Declare queues and exchanges
exchange = settings.EXCHANGE_GANETI
exchange_dl = queues.convert_exchange_to_dead(exchange)
self.client.exchange_declare(exchange=exchange, type="topic")
self.client.exchange_declare(exchange=exchange_dl, type="topic")
for queue in queues.QUEUES:
# Queues are mirrored to all RabbitMQ brokers
self.client.queue_declare(queue=queue,
mirrored=True,
dead_letter_exchange=exchange_dl)
# Declare the corresponding dead-letter queue
queue_dl = queues.convert_queue_to_dead(queue)
self.client.queue_declare(queue=queue_dl, mirrored=True)
# Bind queues to handler methods
for binding in queues.BINDINGS:
try:
callback = getattr(callbacks, binding[3])
except AttributeError:
log.error("Cannot find callback %s", binding[3])
raise SystemExit(1)
queue = binding[0]
exchange = binding[1]
routing_key = binding[2]
self.client.queue_bind(queue=queue,
exchange=exchange,
routing_key=routing_key)
self.client.basic_consume(queue=binding[0],
callback=callback,
prefetch_count=5)
queue_dl = queues.convert_queue_to_dead(queue)
exchange_dl = queues.convert_exchange_to_dead(exchange)
# Bind the corresponding dead-letter queue
self.client.queue_bind(queue=queue_dl,
exchange=exchange_dl,
routing_key=routing_key)
log.debug("Binding %s(%s) to queue %s with handler %s", exchange,
routing_key, queue, binding[3])
class JobFileHandler(pyinotify.ProcessEvent):
def __init__(self, logger, cluster_name):
pyinotify.ProcessEvent.__init__(self)
self.logger = logger
self.cluster_name = cluster_name
# Set max_retries to 0 for unlimited retries.
self.client = AMQPClient(hosts=settings.AMQP_HOSTS, confirm_buffer=25,
max_retries=0, logger=logger)
handler_logger.info("Attempting to connect to RabbitMQ hosts")
self.client.connect()
handler_logger.info("Connected succesfully")
self.client.exchange_declare(settings.EXCHANGE_GANETI, type='topic')
self.op_handlers = {"INSTANCE": self.process_instance_op,
"NETWORK": self.process_network_op,
"CLUSTER": self.process_cluster_op}
# "GROUP": self.process_group_op}
def process_IN_CLOSE_WRITE(self, event):
self.process_IN_MOVED_TO(event)
def process_IN_MOVED_TO(self, event):
jobfile = os.path.join(event.path, event.name)
if not event.name.startswith("job-"):
self.logger.debug("Not a job file: %s" % event.path)
return
try:
data = utils.ReadFile(jobfile)
except IOError:
return
data = serializer.LoadJson(data)
job = jqueue._QueuedJob.Restore(None, data, False, False)
job_id = int(job.id)
for op in job.ops:
op_id = op.input.OP_ID
msg = None
try:
handler_fn = self.op_handlers[op_id.split('_')[1]]
msg, routekey = handler_fn(op, job_id)
except KeyError:
pass
if not msg:
self.logger.debug("Ignoring job: %s: %s", job_id, op_id)
continue
# Generate a unique message identifier
event_time = get_time_from_status(op, job)
# Get the last line of the op log as message
try:
logmsg = op.log[-1][-1]
except IndexError:
logmsg = None
# Add shared attributes for all operations
msg.update({"event_time": event_time,
"operation": op_id,
"status": op.status,
"cluster": self.cluster_name,
"logmsg": logmsg,
"result": op.result,
"jobId": job_id})
if op.status == "success":
msg["result"] = op.result
if op_id == "OP_INSTANCE_CREATE" and op.status == "error":
# In case an instance creation fails send the job input
# so that the job can be retried if needed.
msg["job_fields"] = op.Serialize()["input"]
msg = json.dumps(msg)
self.logger.debug("Delivering msg: %s (key=%s)", msg, routekey)
# Send the message to RabbitMQ
self.client.basic_publish(settings.EXCHANGE_GANETI,
routekey,
msg)
def process_instance_op(self, op, job_id):
""" Process OP_INSTANCE_* opcodes.
"""
input = op.input
op_id = input.OP_ID
instances = None
instances = get_field(input, 'instance_name')
if not instances:
instances = get_field(input, 'instances')
if not instances or len(instances) > 1:
# Do not publish messages for jobs with no or multiple
# instances. Currently snf-dispatcher can not normally handle
# these messages
return None, None
else:
instances = instances[0]
self.logger.debug("Job: %d: %s(%s) %s", job_id, op_id,
instances, op.status)
job_fields = {}
if op_id in ["OP_INSTANCE_SET_PARAMS", "OP_INSTANCE_CREATE"]:
job_fields = {"nics": get_field(input, "nics"),
"disks": get_field(input, "disks"),
"beparams": get_field(input, "beparams")}
msg = {"type": "ganeti-op-status",
"instance": instances,
"operation": op_id,
"job_fields": job_fields}
if ((op_id in ["OP_INSTANCE_CREATE", "OP_INSTANCE_STARTUP"] and
op.status == "success") or
(op_id == "OP_INSTANCE_SET_PARAMS" and
op.status in ["success", "error", "cancelled"])):
nics = get_instance_nics(msg["instance"], self.logger)
msg["instance_nics"] = nics
routekey = "ganeti.%s.event.op" % prefix_from_name(instances)
return msg, routekey
def process_network_op(self, op, job_id):
""" Process OP_NETWORK_* opcodes.
"""
input = op.input
op_id = input.OP_ID
network_name = get_field(input, 'network_name')
if not network_name:
return None, None
self.logger.debug("Job: %d: %s(%s) %s", job_id, op_id,
network_name, op.status)
job_fields = {
'subnet': get_field(input, 'network'),
'gateway': get_field(input, 'gateway'),
"add_reserved_ips": get_field(input, "add_reserved_ips"),
"remove_reserved_ips": get_field(input, "remove_reserved_ips"),
# 'network_mode': get_field(input, 'network_mode'),
# 'network_link': get_field(input, 'network_link'),
'group_name': get_field(input, 'group_name')}
msg = {'operation': op_id,
'type': "ganeti-network-status",
'network': network_name,
'job_fields': job_fields}
routekey = "ganeti.%s.event.network" % prefix_from_name(network_name)
return msg, routekey
def process_cluster_op(self, op, job_id):
""" Process OP_CLUSTER_* opcodes.
"""
input = op.input
op_id = input.OP_ID
self.logger.debug("Job: %d: %s %s", job_id, op_id, op.status)
if op_id != "OP_CLUSTER_SET_PARAMS":
# Send only modifications of cluster
return None, None
msg = {'operation': op_id,
'type': "ganeti-cluster-status"}
routekey = "ganeti.event.cluster"
return msg, routekey
class Dispatcher:
debug = False
def __init__(self, debug=False):
self.debug = debug
self._init()
def wait(self):
log.info("Waiting for messages..")
timeout = DISPATCHER_RECONNECT_TIMEOUT
while True:
try:
# Close the Django DB connection before processing
# every incoming message. This plays nicely with
# DB connection pooling, if enabled and allows
# the dispatcher to recover from broken connections
# gracefully.
close_connection()
msg = self.client.basic_wait(timeout=timeout)
if not msg:
log.warning("Idle connection for %d seconds. Will connect"
" to a different host. Verify that"
" snf-ganeti-eventd is running!!", timeout)
self.client.reconnect(timeout=1)
except AMQPConnectionError as e:
log.error("AMQP connection failed: %s" % e)
log.warning("Sleeping for %d seconds before retrying to "
"connect to an AMQP broker" %
DISPATCHER_FAILED_CONNECTION_WAIT)
time.sleep(DISPATCHER_FAILED_CONNECTION_WAIT)
except select.error as e:
if e[0] != errno.EINTR:
log.exception("Caught unexpected exception: %s", e)
log.warning("Sleeping for %d seconds before retrying to "
"connect to an AMQP broker" %
DISPATCHER_FAILED_CONNECTION_WAIT)
time.sleep(DISPATCHER_FAILED_CONNECTION_WAIT)
else:
break
except (SystemExit, KeyboardInterrupt):
break
except Exception as e:
log.exception("Caught unexpected exception: %s", e)
log.warning("Sleeping for %d seconds before retrying to "
"connect to an AMQP broker" %
DISPATCHER_FAILED_CONNECTION_WAIT)
time.sleep(DISPATCHER_FAILED_CONNECTION_WAIT)
log.info("Clean up AMQP connection before exit")
self.client.basic_cancel(timeout=1)
self.client.close(timeout=1)
def _init(self):
log.info("Initializing")
# Set confirm buffer to 1 for heartbeat messages
self.client = AMQPClient(logger=log_amqp, confirm_buffer=1)
# Connect to AMQP host
self.client.connect()
# Declare queues and exchanges
exchange = settings.EXCHANGE_GANETI
exchange_dl = queues.convert_exchange_to_dead(exchange)
self.client.exchange_declare(exchange=exchange,
type="topic")
self.client.exchange_declare(exchange=exchange_dl,
type="topic")
for queue in queues.QUEUES:
# Queues are mirrored to all RabbitMQ brokers
self.client.queue_declare(queue=queue, mirrored=True,
dead_letter_exchange=exchange_dl)
# Declare the corresponding dead-letter queue
queue_dl = queues.convert_queue_to_dead(queue)
self.client.queue_declare(queue=queue_dl, mirrored=True)
# Bind queues to handler methods
for binding in queues.BINDINGS:
try:
callback = getattr(callbacks, binding[3])
except AttributeError:
log.error("Cannot find callback %s", binding[3])
raise SystemExit(1)
queue = binding[0]
exchange = binding[1]
routing_key = binding[2]
self.client.queue_bind(queue=queue, exchange=exchange,
routing_key=routing_key)
self.client.basic_consume(queue=binding[0],
callback=callback,
prefetch_count=5)
queue_dl = queues.convert_queue_to_dead(queue)
exchange_dl = queues.convert_exchange_to_dead(exchange)
# Bind the corresponding dead-letter queue
self.client.queue_bind(queue=queue_dl,
exchange=exchange_dl,
routing_key=routing_key)
log.debug("Binding %s(%s) to queue %s with handler %s",
exchange, routing_key, queue, binding[3])
# Declare the queue that will be used for receiving requests, e.g. a
# status check request
hostname, pid = get_hostname(), os.getpid()
queue = queues.get_dispatcher_request_queue(hostname, pid)
self.client.queue_declare(queue=queue, mirrored=True,
ttl=REQUEST_QUEUE_TTL)
self.client.basic_consume(queue=queue, callback=handle_request)
log.debug("Binding %s(%s) to queue %s with handler 'hadle_request'",
exchange, routing_key, queue)
