def test_run(self):
config = {
'name': 'east_coast.prod.global',
'durable_queue': False,
'rabbit_host': '10.0.0.1',
'rabbit_port': 5672,
'rabbit_userid': 'rabbit',
'rabbit_password': '******',
'rabbit_virtual_host': '/'
}
self.mox.StubOutWithMock(db, 'get_or_create_deployment')
deployment = self.mox.CreateMockAnything()
db.get_or_create_deployment(config['name'])\
.AndReturn((deployment, True))
self.mox.StubOutWithMock(kombu.connection, 'BrokerConnection')
params = dict(hostname=config['rabbit_host'],
port=config['rabbit_port'],
userid=config['rabbit_userid'],
password=config['rabbit_password'],
transport="librabbitmq",
virtual_host=config['rabbit_virtual_host'])
self.mox.StubOutWithMock(worker, "continue_running")
worker.continue_running().AndReturn(True)
conn = self.mox.CreateMockAnything()
kombu.connection.BrokerConnection(**params).AndReturn(conn)
conn.__enter__().AndReturn(conn)
conn.__exit__(None, None, None).AndReturn(None)
self.mox.StubOutClassWithMocks(worker, 'NovaConsumer')
consumer = worker.NovaConsumer(config['name'], conn, deployment,
config['durable_queue'], {})
consumer.run()
worker.continue_running().AndReturn(False)
self.mox.ReplayAll()
worker.run(config)
self.mox.VerifyAll()
def test_run(self):
config = {
'name': 'east_coast.prod.global',
'durable_queue': False,
'rabbit_host': '10.0.0.1',
'rabbit_port': 5672,
'rabbit_userid': 'rabbit',
'rabbit_password': '******',
'rabbit_virtual_host': '/'
}
self.mox.StubOutWithMock(db, 'get_or_create_deployment')
deployment = self.mox.CreateMockAnything()
db.get_or_create_deployment(config['name'])\
.AndReturn((deployment, True))
self.mox.StubOutWithMock(kombu.connection, 'BrokerConnection')
params = dict(hostname=config['rabbit_host'],
port=config['rabbit_port'],
userid=config['rabbit_userid'],
password=config['rabbit_password'],
transport="librabbitmq",
virtual_host=config['rabbit_virtual_host'])
self.mox.StubOutWithMock(worker, "continue_running")
worker.continue_running().AndReturn(True)
conn = self.mox.CreateMockAnything()
kombu.connection.BrokerConnection(**params).AndReturn(conn)
conn.__enter__().AndReturn(conn)
conn.__exit__(None, None, None).AndReturn(None)
self.mox.StubOutClassWithMocks(worker, 'NovaConsumer')
consumer = worker.NovaConsumer(config['name'], conn, deployment,
config['durable_queue'], {})
consumer.run()
worker.continue_running().AndReturn(False)
self.mox.ReplayAll()
worker.run(config)
self.mox.VerifyAll()
def test_get_or_create_deployment(self):
deployment = self.mox.CreateMockAnything()
models.Deployment.objects.get_or_create(name='test').AndReturn(deployment)
self.mox.ReplayAll()
returned = db.get_or_create_deployment('test')
self.assertEqual(returned, deployment)
self.mox.VerifyAll()
def run(deployment_config):
name = deployment_config['name']
host = deployment_config.get('rabbit_host', 'localhost')
port = deployment_config.get('rabbit_port', 5672)
user_id = deployment_config.get('rabbit_userid', 'rabbit')
password = deployment_config.get('rabbit_password', 'rabbit')
virtual_host = deployment_config.get('rabbit_virtual_host', '/')
durable = deployment_config.get('durable_queue', True)
queue_arguments = deployment_config.get('queue_arguments', {})
deployment, new = db.get_or_create_deployment(name)
print "Starting worker for '%s'" % name
LOG.info("%s: %s %s %s %s" % (name, host, port, user_id, virtual_host))
params = dict(hostname=host,
port=port,
userid=user_id,
password=password,
transport="librabbitmq",
virtual_host=virtual_host)
while continue_running():
LOG.debug("Processing on '%s'" % name)
with kombu.connection.BrokerConnection(**params) as conn:
try:
consumer = NovaConsumer(name, conn, deployment, durable,
queue_arguments)
consumer.run()
except Exception as e:
LOG.exception("name=%s, exception=%s. Reconnecting in 5s" %
(name, e))
time.sleep(5)
LOG.debug("Completed processing on '%s'" % name)
def test_get_or_create_deployment(self):
deployment = self.mox.CreateMockAnything()
models.Deployment.objects.get_or_create(
name='test').AndReturn(deployment)
self.mox.ReplayAll()
returned = db.get_or_create_deployment('test')
self.assertEqual(returned, deployment)
self.mox.VerifyAll()
def test_run_queue_args(self):
config = {
'name': 'east_coast.prod.global',
'durable_queue': False,
'rabbit_host': '10.0.0.1',
'rabbit_port': 5672,
'rabbit_userid': 'rabbit',
'rabbit_password': '******',
'rabbit_virtual_host': '/',
'queue_arguments': {'x-ha-policy': 'all'},
'queue_name_prefix': "test_name_",
"services": ["nova"],
"topics": {"nova": ["monitor.info", "monitor.error"]}
}
self.mox.StubOutWithMock(db, 'get_or_create_deployment')
deployment = self.mox.CreateMockAnything()
db.get_or_create_deployment(config['name'])\
.AndReturn((deployment, True))
self.mox.StubOutWithMock(kombu.connection, 'BrokerConnection')
params = dict(hostname=config['rabbit_host'],
port=config['rabbit_port'],
userid=config['rabbit_userid'],
password=config['rabbit_password'],
transport="librabbitmq",
virtual_host=config['rabbit_virtual_host'])
self.mox.StubOutWithMock(worker, "continue_running")
worker.continue_running().AndReturn(True)
conn = self.mox.CreateMockAnything()
kombu.connection.BrokerConnection(**params).AndReturn(conn)
conn.__enter__().AndReturn(conn)
conn.__exit__(None, None, None).AndReturn(None)
self.mox.StubOutClassWithMocks(worker, 'Consumer')
exchange = 'nova'
consumer = worker.Consumer(config['name'], conn, deployment,
config['durable_queue'],
config['queue_arguments'], exchange,
["monitor.info", "monitor.error"],
"test_name_")
consumer.run()
worker.continue_running().AndReturn(False)
self.mox.ReplayAll()
worker.run(config, exchange)
self.mox.VerifyAll()
def run(deployment_config, exchange):
name = deployment_config['name']
host = deployment_config.get('rabbit_host', 'localhost')
port = deployment_config.get('rabbit_port', 5672)
user_id = deployment_config.get('rabbit_userid', 'rabbit')
password = deployment_config.get('rabbit_password', 'rabbit')
virtual_host = deployment_config.get('rabbit_virtual_host', '/')
durable = deployment_config.get('durable_queue', True)
queue_arguments = deployment_config.get('queue_arguments', {})
exit_on_exception = deployment_config.get('exit_on_exception', False)
topics = deployment_config.get('topics', {})
queue_name_prefix = deployment_config.get('queue_name_prefix', 'stacktach_')
deployment, new = db.get_or_create_deployment(name)
print "Starting worker for '%s %s'" % (name, exchange)
LOG.info("%s: %s %s %s %s %s" % (name, exchange, host, port, user_id,
virtual_host))
params = dict(hostname=host,
port=port,
userid=user_id,
password=password,
transport="librabbitmq",
virtual_host=virtual_host)
# continue_running() is used for testing
while continue_running():
try:
LOG.debug("Processing on '%s %s'" % (name, exchange))
with kombu.connection.BrokerConnection(**params) as conn:
try:
consumer = Consumer(name, conn, deployment, durable,
queue_arguments, exchange,
topics[exchange],
queue_name_prefix)
consumer.run()
except Exception as e:
LOG.error("!!!!Exception!!!!")
LOG.exception("name=%s, exchange=%s, exception=%s. "
"Reconnecting in 5s" %
(name, exchange, e))
exit_or_sleep(exit_on_exception)
LOG.debug("Completed processing on '%s %s'" % (name, exchange))
except:
LOG.error("!!!!Exception!!!!")
e = sys.exc_info()[0]
msg = "Uncaught exception: deployment=%s, exchange=%s, " \
"exception=%s. Retrying in 5s"
LOG.exception(msg % (name, exchange, e))
exit_or_sleep(exit_on_exception)
def create_proc_table(manager):
for deployment in config.deployments():
if deployment.get('enabled', True):
name = deployment['name']
db_deployment, new = db.get_or_create_deployment(name)
for exchange in deployment.get('topics').keys():
stats = manager.dict()
proc_info = dict(process=None,
pid=0,
deployment=deployment,
deploy_id=db_deployment.id,
exchange=exchange,
stats=stats)
processes[(name, exchange)] = proc_info
def create_proc_table(manager):
for deployment in config.deployments():
if deployment.get('enabled', True):
name = deployment['name']
db_deployment, new = db.get_or_create_deployment(name)
for exchange in deployment.get('topics').keys():
stats = manager.dict()
proc_info = dict(process=None,
pid=0,
deployment=deployment,
deploy_id=db_deployment.id,
exchange=exchange,
stats=stats)
processes[(name, exchange)] = proc_info
def run(deployment_config):
name = deployment_config['name']
host = deployment_config.get('rabbit_host', 'localhost')
port = deployment_config.get('rabbit_port', 5672)
user_id = deployment_config.get('rabbit_userid', 'rabbit')
password = deployment_config.get('rabbit_password', 'rabbit')
virtual_host = deployment_config.get('rabbit_virtual_host', '/')
durable = deployment_config.get('durable_queue', True)
queue_arguments = deployment_config.get('queue_arguments', {})
exit_on_exception = deployment_config.get('exit_on_exception', False)
deployment, new = db.get_or_create_deployment(name)
print "Starting worker for '%s'" % name
LOG.info("%s: %s %s %s %s" % (name, host, port, user_id, virtual_host))
params = dict(hostname=host,
port=port,
userid=user_id,
password=password,
transport="librabbitmq",
virtual_host=virtual_host)
# continue_running() is used for testing
while continue_running():
try:
LOG.debug("Processing on '%s'" % name)
with kombu.connection.BrokerConnection(**params) as conn:
try:
consumer = NovaConsumer(name, conn, deployment, durable,
queue_arguments)
consumer.run()
except Exception as e:
LOG.error("!!!!Exception!!!!")
LOG.exception("name=%s, exception=%s. Reconnecting in 5s" %
(name, e))
exit_or_sleep(exit_on_exception)
LOG.debug("Completed processing on '%s'" % name)
except:
LOG.error("!!!!Exception!!!!")
e = sys.exc_info()[0]
msg = "Uncaught exception: deployment=%s, exception=%s. Retrying in 5s"
LOG.exception(msg % (name, e))
exit_or_sleep(exit_on_exception)
def run(deployment_config):
name = deployment_config["name"]
host = deployment_config.get("rabbit_host", "localhost")
port = deployment_config.get("rabbit_port", 5672)
user_id = deployment_config.get("rabbit_userid", "rabbit")
password = deployment_config.get("rabbit_password", "rabbit")
virtual_host = deployment_config.get("rabbit_virtual_host", "/")
durable = deployment_config.get("durable_queue", True)
queue_arguments = deployment_config.get("queue_arguments", {})
deployment, new = db.get_or_create_deployment(name)
print "Starting worker for '%s'" % name
LOG.info("%s: %s %s %s %s" % (name, host, port, user_id, virtual_host))
params = dict(
hostname=host, port=port, userid=user_id, password=password, transport="librabbitmq", virtual_host=virtual_host
)
# continue_running() is used for testing
while continue_running():
try:
LOG.debug("Processing on '%s'" % name)
with kombu.connection.BrokerConnection(**params) as conn:
try:
consumer = NovaConsumer(name, conn, deployment, durable, queue_arguments)
consumer.run()
except Exception as e:
LOG.exception("name=%s, exception=%s. Reconnecting in 5s" % (name, e))
time.sleep(5)
LOG.debug("Completed processing on '%s'" % name)
except:
e = sys.exc_info()[0]
msg = "Uncaught exception: deployment=%s, exception=%s. Retrying in 5s"
LOG.exception(msg % (name, e))
time.sleep(5)
log_listener.end()
print "dying ..."
for process in processes:
process.terminate()
print "rose"
for process in processes:
process.join()
print "bud"
sys.exit(0)
if __name__ == '__main__':
log_listener = stacklog.LogListener(_get_parent_logger()).start()
for deployment in config.deployments():
if deployment.get('enabled', True):
db_deployment, new = db.get_or_create_deployment(deployment['name'])
# NOTE (apmelton)
# Close the connection before spinning up the child process,
# otherwise the child process will attempt to use the connection
# the parent process opened up to get/create the deployment.
close_connection()
for exchange in deployment.get('topics').keys():
process = Process(target=worker.run, args=(deployment,
db_deployment.id,
exchange,))
process.daemon = True
process.start()
processes.append(process)
signal.signal(signal.SIGINT, kill_time)
signal.signal(signal.SIGTERM, kill_time)
signal.pause()
for process in processes:
process.terminate()
print "rose"
for process in processes:
process.join()
log_listener.end()
print "bud"
sys.exit(0)
if __name__ == '__main__':
log_listener = stacklog.LogListener(_get_parent_logger())
log_listener.start()
for deployment in config.deployments():
if deployment.get('enabled', True):
db_deployment, new = db.get_or_create_deployment(
deployment['name'])
# NOTE (apmelton)
# Close the connection before spinning up the child process,
# otherwise the child process will attempt to use the connection
# the parent process opened up to get/create the deployment.
close_connection()
for exchange in deployment.get('topics').keys():
process = Process(target=worker.run,
args=(
deployment,
db_deployment.id,
exchange,
))
process.daemon = True
process.start()
processes.append(process)
