def amqp_subscribe(exchange, callback, queue='',
ex_type='fanout', routing_keys=None):
def json_parse_dec(func):
@functools.wraps(func)
def wrapped(msg):
try:
msg.body = json.loads(msg.body)
except:
pass
return func(msg)
return wrapped
connection = Connection(config.AMQP_URI)
channel = connection.channel()
channel.exchange_declare(exchange=exchange, type=ex_type, auto_delete=True)
resp = channel.queue_declare(queue, exclusive=True)
if not routing_keys:
channel.queue_bind(resp.queue, exchange)
else:
for routing_key in routing_keys:
channel.queue_bind(resp.queue, exchange, routing_key=routing_key)
channel.basic_consume(queue=queue,
callback=json_parse_dec(callback),
no_ack=True)
try:
while True:
channel.wait()
except BaseException as exc:
# catch BaseException so that it catches KeyboardInterrupt
channel.close()
connection.close()
amqp_log("SUBSCRIPTION ENDED: %s %s %r" % (exchange, queue, exc))
def amqp_subscribe(exchange, queue, callback):
def json_parse_dec(func):
@functools.wraps(func)
def wrapped(msg):
try:
msg.body = json.loads(msg.body)
except:
pass
return func(msg)
return wrapped
if not queue:
queue = "mist-tmp_%d" % random.randrange(2 ** 20)
connection = Connection()
channel = connection.channel()
channel.exchange_declare(exchange=exchange, type='fanout')
channel.queue_declare(queue, exclusive=True)
channel.queue_bind(queue, exchange)
channel.basic_consume(queue=queue,
callback=json_parse_dec(callback),
no_ack=True)
try:
while True:
channel.wait()
except BaseException as exc:
# catch BaseException so that it catches KeyboardInterrupt
channel.close()
connection.close()
amqp_log("SUBSCRIPTION ENDED: %s %s %r" % (exchange, queue, exc))
def amqp_publish(exchange, routing_key, data):
connection = Connection()
channel = connection.channel()
msg = Message(json.dumps(data))
channel.basic_publish(msg, exchange=exchange, routing_key=routing_key)
channel.close()
connection.close()
def amqp_subscribe(exchange, callback, queue='',
ex_type='fanout', routing_keys=None):
def json_parse_dec(func):
@functools.wraps(func)
def wrapped(msg):
try:
msg.body = json.loads(msg.body)
except:
pass
return func(msg)
return wrapped
connection = Connection()
channel = connection.channel()
channel.exchange_declare(exchange=exchange, type=ex_type, auto_delete=true)
resp = channel.queue_declare(queue, exclusive=True)
if not routing_keys:
channel.queue_bind(resp.queue, exchange)
else:
for routing_key in routing_keys:
channel.queue_bind(resp.queue, exchange, routing_key=routing_key)
channel.basic_consume(queue=queue,
callback=json_parse_dec(callback),
no_ack=True)
try:
while True:
channel.wait()
except BaseException as exc:
# catch BaseException so that it catches KeyboardInterrupt
channel.close()
connection.close()
amqp_log("SUBSCRIPTION ENDED: %s %s %r" % (exchange, queue, exc))
def amqp_subscribe(exchange, queue, callback):
def json_parse_dec(func):
@functools.wraps(func)
def wrapped(msg):
try:
msg.body = json.loads(msg.body)
except:
pass
return func(msg)
return wrapped
if not queue:
queue = "mist-tmp_%d" % random.randrange(2**20)
connection = Connection()
channel = connection.channel()
channel.exchange_declare(exchange=exchange, type='fanout')
channel.queue_declare(queue, exclusive=True)
channel.queue_bind(queue, exchange)
channel.basic_consume(queue=queue,
callback=json_parse_dec(callback),
no_ack=True)
try:
while True:
channel.wait()
except BaseException as exc:
# catch BaseException so that it catches KeyboardInterrupt
channel.close()
connection.close()
amqp_log("SUBSCRIPTION ENDED: %s %s %r" % (exchange, queue, exc))
def amqp_publish(exchange, routing_key, data):
connection = Connection()
channel = connection.channel()
msg = Message(json.dumps(data))
channel.basic_publish(msg, exchange=exchange, routing_key=routing_key)
channel.close()
connection.close()
def setup_conn(self):
self.connection = Connection(**self.connection_meta)
self.channel = Channel(self.connection)
# Since amqp v2.0, you should explicitly call Connection.connect() and Channel.open()
if VERSION[0] >= 2:
self.connection.connect()
self.channel.open()
def amqp_publish(exchange, routing_key, data, ex_type="fanout", ex_declare=False):
connection = Connection(config.AMQP_URI)
channel = connection.channel()
if ex_declare:
channel.exchange_declare(exchange=exchange, type=ex_type)
msg = Message(json.dumps(data))
channel.basic_publish(msg, exchange=exchange, routing_key=routing_key)
channel.close()
connection.close()
def amqp_user_listening(user):
connection = Connection(config.AMQP_URI)
channel = connection.channel()
try:
channel.exchange_declare(exchange=_amqp_user_exchange(user), type="fanout", passive=True)
except AmqpNotFound:
return False
else:
return True
finally:
channel.close()
connection.close()
def amqp_owner_listening(owner):
connection = Connection(config.AMQP_URI)
channel = connection.channel()
try:
channel.exchange_declare(exchange=_amqp_owner_exchange(owner),
type='fanout', passive=True)
except AmqpNotFound:
return False
else:
return True
finally:
channel.close()
connection.close()
def amqp_publish(exchange,
routing_key,
data,
ex_type='fanout',
ex_declare=False):
connection = Connection()
channel = connection.channel()
if ex_declare:
channel.exchange_declare(exchange=exchange, type=ex_type)
msg = Message(json.dumps(data))
channel.basic_publish(msg, exchange=exchange, routing_key=routing_key)
channel.close()
connection.close()
def amqp_publish(exchange, routing_key, data,
ex_type='fanout', ex_declare=False, auto_delete=True,
connection=None):
close = False
if connection is None:
connection = Connection(config.AMQP_URI)
close = True
channel = connection.channel()
if ex_declare:
channel.exchange_declare(exchange=exchange, type=ex_type,
auto_delete=auto_delete)
msg = Message(json.dumps(data))
channel.basic_publish(msg, exchange=exchange, routing_key=routing_key)
channel.close()
if close:
connection.close()
def setupConnectivity(self):
if self.kwargs.heartbeat > 0:
self.logging.info("Sending heartbeat every %s seconds." % (self.kwargs.heartbeat))
self.sendToBackground(self.heartbeat)
while self.loop():
self.connect.wait()
self.logging.debug("Connecting to %s" % (self.kwargs.host))
try:
self.connection = Connection(
host=self.kwargs.host,
heartbeat=self.kwargs.heartbeat,
virtual_host=self.kwargs.vhost,
userid=self.kwargs.user,
password=self.kwargs.password,
ssl=self.kwargs.ssl,
connect_timeout=5
)
self.connection.connect()
self.channel = self.connection.channel()
if self.kwargs.exchange != "":
self.channel.exchange_declare(
self.kwargs.exchange,
self.kwargs.exchange_type,
durable=self.kwargs.exchange_durable,
auto_delete=self.kwargs.exchange_auto_delete,
passive=self.kwargs.exchange_passive,
arguments=self._exchange_arguments
)
self.logging.debug("Declared exchange %s." % (self.kwargs.exchange))
if self.kwargs.queue_declare:
self.channel.queue_declare(
self.kwargs.queue,
durable=self.kwargs.queue_durable,
exclusive=self.kwargs.queue_exclusive,
auto_delete=self.kwargs.queue_auto_delete,
arguments=self._queue_arguments
)
self.logging.debug("Declared queue %s." % (self.kwargs.queue))
if self.kwargs.exchange != "":
self.channel.queue_bind(
self.kwargs.queue,
self.kwargs.exchange,
routing_key=self.kwargs.routing_key
)
self.logging.debug("Bound queue %s to exchange %s." % (self.kwargs.queue, self.kwargs.exchange))
self.logging.info("Connected to broker %s." % (self.kwargs.host))
except Exception as err:
self.logging.error("Failed to connect to broker. Reason %s " % (err))
sleep(1)
else:
self.do_consume.set()
self.connect.clear()
self.connected = True
def __init__(self):
self.response = None
self.connection = Connection()
self.channel = Channel(self.connection)
(self.queue, _, _) = self.channel.queue_declare(exclusive=True)
self.channel.queue_bind(self.queue, exchange='django_amqp_example')
def _create_resource(conn, res_id, user_id, project_id, source_id, rmeta):
# TODO(gordc): implement lru_cache to improve performance
try:
res = models.Resource.__table__
m_hash = hashlib.md5(jsonutils.dumps(rmeta,
sort_keys=True)).hexdigest()
trans = conn.begin_nested()
if conn.dialect.name == 'sqlite':
trans = conn.begin()
with trans:
res_row = conn.execute(
sa.select([res.c.internal_id]).where(
sa.and_(res.c.resource_id == res_id,
res.c.user_id == user_id,
res.c.project_id == project_id,
res.c.source_id == source_id,
res.c.metadata_hash == m_hash))).first()
internal_id = res_row[0] if res_row else None
if internal_id is None:
result = conn.execute(res.insert(),
resource_id=res_id,
user_id=user_id,
project_id=project_id,
source_id=source_id,
resource_metadata=rmeta,
metadata_hash=m_hash)
internal_id = result.inserted_primary_key[0]
if rmeta and isinstance(rmeta, dict):
meta_map = {}
for key, v in utils.dict_to_keyval(rmeta):
try:
_model = sql_utils.META_TYPE_MAP[type(v)]
if meta_map.get(_model) is None:
meta_map[_model] = []
meta_map[_model].append({
'id': internal_id,
'meta_key': key,
'value': v
})
except KeyError:
LOG.warn(
_("Unknown metadata type. Key (%s) "
"will not be queryable."), key)
for _model in meta_map.keys():
conn.execute(_model.__table__.insert(),
meta_map[_model])
except dbexc.DBDuplicateEntry:
# retry function to pick up duplicate committed object
internal_id = Connection._create_resource(conn, res_id, user_id,
project_id, source_id,
rmeta)
return internal_id
class RpcClient(object):
def __init__(self):
self.response = None
self.connection = Connection()
self.channel = Channel(self.connection)
(self.queue, _, _) = self.channel.queue_declare(exclusive=True)
self.channel.queue_bind(self.queue, exchange='django_amqp_example')
def request(self, body):
message = Message(
body=json.dumps(body),
reply_to=self.queue,
content_type='application/json')
self.channel.basic_publish(
message,
exchange='django_amqp_example',
routing_key='task_queue')
print "Task submitted:", json.dumps(body)
def callback(msg):
self.response = json.loads(msg.body)
self.channel.basic_consume(
callback=callback,
queue=self.queue,
no_ack=True)
while True:
self.connection.drain_events(timeout=60)
if self.response is not None:
break
self.connection.close()
return self.response
def list_networks(self, persist=True):
"""Return list of networks for cloud
A list of networks is fetched from libcloud, data is processed, stored
on network models, and a list of network models is returned.
Subclasses SHOULD NOT override or extend this method.
This method wraps `_list_networks` which contains the core
implementation.
"""
task_key = 'cloud:list_networks:%s' % self.cloud.id
task = PeriodicTaskInfo.get_or_add(task_key)
with task.task_runner(persist=persist):
cached_networks = {
'%s' % n.id: n.as_dict()
for n in self.list_cached_networks()
}
networks = self._list_networks()
# Initialize AMQP connection to reuse for multiple messages.
amqp_conn = Connection(config.AMQP_URI)
if amqp_owner_listening(self.cloud.owner.id):
networks_dict = [n.as_dict() for n in networks]
if cached_networks and networks_dict:
# Publish patches to rabbitmq.
new_networks = {'%s' % n['id']: n for n in networks_dict}
patch = jsonpatch.JsonPatch.from_diff(cached_networks,
new_networks).patch
if patch:
amqp_publish_user(self.cloud.owner.id,
routing_key='patch_networks',
connection=amqp_conn,
data={
'cloud_id': self.cloud.id,
'patch': patch
})
else:
# TODO: remove this block, once patches
# are implemented in the UI
amqp_publish_user(self.cloud.owner.id,
routing_key='list_networks',
connection=amqp_conn,
data={
'cloud_id': self.cloud.id,
'networks': networks_dict
})
return networks
def _create_resource(conn, res_id, user_id, project_id, source_id,
rmeta):
# TODO(gordc): implement lru_cache to improve performance
try:
res = models.Resource.__table__
m_hash = hashlib.md5(jsonutils.dumps(rmeta,
sort_keys=True)).hexdigest()
trans = conn.begin_nested()
if conn.dialect.name == 'sqlite':
trans = conn.begin()
with trans:
res_row = conn.execute(
sa.select([res.c.internal_id])
.where(sa.and_(res.c.resource_id == res_id,
res.c.user_id == user_id,
res.c.project_id == project_id,
res.c.source_id == source_id,
res.c.metadata_hash == m_hash))).first()
internal_id = res_row[0] if res_row else None
if internal_id is None:
result = conn.execute(res.insert(), resource_id=res_id,
user_id=user_id,
project_id=project_id,
source_id=source_id,
resource_metadata=rmeta,
metadata_hash=m_hash)
internal_id = result.inserted_primary_key[0]
if rmeta and isinstance(rmeta, dict):
meta_map = {}
for key, v in utils.dict_to_keyval(rmeta):
try:
_model = sql_utils.META_TYPE_MAP[type(v)]
if meta_map.get(_model) is None:
meta_map[_model] = []
meta_map[_model].append(
{'id': internal_id, 'meta_key': key,
'value': v})
except KeyError:
LOG.warn(_("Unknown metadata type. Key (%s) "
"will not be queryable."), key)
for _model in meta_map.keys():
conn.execute(_model.__table__.insert(),
meta_map[_model])
except dbexc.DBDuplicateEntry:
# retry function to pick up duplicate committed object
internal_id = Connection._create_resource(
conn, res_id, user_id, project_id, source_id, rmeta)
return internal_id
def _stats_result_to_model(result, period, period_start,
period_end, groupby, aggregate):
stats_args = Connection._stats_result_aggregates(result, aggregate)
stats_args['unit'] = result.unit
duration = (timeutils.delta_seconds(result.tsmin, result.tsmax)
if result.tsmin is not None and result.tsmax is not None
else None)
stats_args['duration'] = duration
stats_args['duration_start'] = result.tsmin
stats_args['duration_end'] = result.tsmax
stats_args['period'] = period
stats_args['period_start'] = period_start
stats_args['period_end'] = period_end
stats_args['groupby'] = (dict(
(g, getattr(result, g)) for g in groupby) if groupby else None)
return api_models.Statistics(**stats_args)
def _stats_result_to_model(result, period, period_start, period_end,
groupby, aggregate):
stats_args = Connection._stats_result_aggregates(result, aggregate)
stats_args['unit'] = result.unit
duration = (timeutils.delta_seconds(result.tsmin, result.tsmax)
if result.tsmin is not None and result.tsmax is not None
else None)
stats_args['duration'] = duration
stats_args['duration_start'] = result.tsmin
stats_args['duration_end'] = result.tsmax
stats_args['period'] = period
stats_args['period_start'] = period_start
stats_args['period_end'] = period_end
stats_args['groupby'] = (dict(
(g, getattr(result, g)) for g in groupby) if groupby else None)
return api_models.Statistics(**stats_args)
def open(self, channel=None):
if channel:
saved_channel = self.channel
self.channel = channel
yield self.channel
self.channel = saved_channel
elif self.channel:
yield self.channel
else:
try:
args = {}
if self.host: args['host'] = self.host
if self.userid: args['userid'] = self.userid
self.connection = Connection(**args)
self.channel = self.connection.channel()
yield self.channel
finally:
self.close()
return
def _create_meter(conn, name, type, unit):
# TODO(gordc): implement lru_cache to improve performance
try:
meter = models.Meter.__table__
trans = conn.begin_nested()
if conn.dialect.name == 'sqlite':
trans = conn.begin()
with trans:
meter_row = conn.execute(
sa.select([meter.c.id])
.where(sa.and_(meter.c.name == name,
meter.c.type == type,
meter.c.unit == unit))).first()
meter_id = meter_row[0] if meter_row else None
if meter_id is None:
result = conn.execute(meter.insert(), name=name,
type=type, unit=unit)
meter_id = result.inserted_primary_key[0]
except dbexc.DBDuplicateEntry:
# retry function to pick up duplicate committed object
meter_id = Connection._create_meter(conn, name, type, unit)
return meter_id
def _create_meter(conn, name, type, unit):
# TODO(gordc): implement lru_cache to improve performance
try:
meter = models.Meter.__table__
trans = conn.begin_nested()
if conn.dialect.name == 'sqlite':
trans = conn.begin()
with trans:
meter_row = conn.execute(
sa.select([meter.c.id]).where(
sa.and_(meter.c.name == name, meter.c.type == type,
meter.c.unit == unit))).first()
meter_id = meter_row[0] if meter_row else None
if meter_id is None:
result = conn.execute(meter.insert(),
name=name,
type=type,
unit=unit)
meter_id = result.inserted_primary_key[0]
except dbexc.DBDuplicateEntry:
# retry function to pick up duplicate committed object
meter_id = Connection._create_meter(conn, name, type, unit)
return meter_id
def test_module_amqp_submit_message():
actor_config = ActorConfig('amqp', 100, 1, {}, "", disable_exception_handling=True)
amqp = AMQPIn(actor_config, exchange="wishbone")
amqp.pool.queue.outbox.disableFallThrough()
amqp.start()
sleep(1)
conn = Connection()
conn.connect()
channel = conn.channel()
channel.basic_publish(basic_message.Message("test"), exchange="wishbone")
channel.close()
conn.close()
sleep(1)
event = getter(amqp.pool.queue.outbox)
assert event.get() == "test"
amqp.stop()
def test_module_amqp_submit_message():
actor_config = ActorConfig('amqp', 100, 1, {}, "", disable_exception_handling=True)
amqp = AMQPOut(actor_config, exchange="wishbone_submit_message", queue="wishbone_submit_message", exchange_durable=False, queue_durable=False)
amqp.pool.queue.inbox.disableFallThrough()
amqp.start()
event = Event("test")
amqp.submit(event, "inbox")
sleep(1)
conn = Connection()
conn.connect()
channel = conn.channel()
message = channel.basic_get("wishbone_submit_message")
channel.close()
conn.close()
sleep(1)
amqp.stop()
assert message.body == "test"
def test_module_amqp_submit_message_encode():
c = ComponentManager()
protocol = c.getComponentByName("wishbone.protocol.encode.json")()
actor_config = ActorConfig('amqp', 100, 1, {}, "", protocol=protocol, disable_exception_handling=True)
amqp = AMQPOut(actor_config, exchange="wishbone_submit_encode", queue="wishbone_submit_encode", exchange_durable=False, queue_durable=False)
amqp.pool.queue.inbox.disableFallThrough()
amqp.start()
event = Event({"one": 1})
amqp.submit(event, "inbox")
sleep(1)
conn = Connection()
conn.connect()
channel = conn.channel()
message = channel.basic_get("wishbone_submit_encode")
channel.close()
conn.close()
sleep(1)
amqp.stop()
assert message.body == '{"one": 1}'
class AMQPOut(OutputModule):
'''
Submits messages to AMQP.
Submits messages to an AMQP message broker.
If <exchange> is not provided, no exchange will be created during initialisation.
If <queue> is not provided, queue will be created during initialisation
If <exchange> and <queue> are provided, they will both be created and
bound during initialisation.
Parameters::
- delivery_mode(int)(1)
| Sets the delivery mode of the messages.
- exchange(str)("")
| The exchange to declare.
- exchange_type(str)("direct")
| The exchange type to create. (direct, topic, fanout)
- exchange_durable(bool)(false)
| Declare a durable exchange.
- exchange_auto_delete(bool)(true)
| If set, the exchange is deleted when all queues have finished using it.
- exchange_passive(bool)(false)
| If set, the server will not create the exchange. The client can use
| this to check whether an exchange exists without modifying the server state.
- exchange_arguments(dict)({})
| Additional arguments for exchange declaration.
- heartbeat(int)(0)
| Enable AMQP heartbeat. The value is the interval in seconds.
| 0 disables heartbeat support.
- host(str)("localhost:5672")
| The host broker to connect to.
- native_events(bool)(False)
| Outgoing events should be native Wishbone events
- parallel_streams(int)(1)
| The number of outgoing parallel data streams.
- password(str)("guest")
| The password to authenticate.
- payload(str)(None)
| The string to submit.
| If defined takes precedence over `selection`.
- queue(str)("wishbone")
| The queue to declare and bind to <exchange>. This will also the
| the destination queue of the submitted messages unless
| <routing_key> is set to another value and <exchange_type> is
| "topic".
- queue_arguments(dict)({})
| Additional arguments for queue declaration.
- queue_auto_delete(bool)(true)
| Whether to autodelete the queue.
- queue_declare(bool)(true)
| Whether to actually declare the queue.
- queue_durable(bool)(false)
| Declare a durable queue.
- queue_exclusive(bool)(false)
| Declare an exclusive queue.
- routing_key(str)("")
| The routing key to use when submitting messages.
- selection(str)("data")
| The part of the event to submit externally.
- ssl(bool)(False)
| If True expects SSL
- user(str)("guest")
| The username to authenticate.
- vhost(str)("/")
| The virtual host to connect to.
Queues::
- inbox
| Messages going to the defined broker.
'''
def __init__(self, actor_config, native_events=False, selection="data", payload=None, parallel_streams=1,
host="localhost:5672", vhost="/", user="******", password="******", ssl=False, heartbeat=0,
exchange="wishbone", exchange_type="direct", exchange_durable=False, exchange_auto_delete=True, exchange_passive=False,
exchange_arguments={},
queue="wishbone", queue_durable=False, queue_exclusive=False, queue_auto_delete=True, queue_declare=True,
queue_arguments={},
routing_key="", delivery_mode=1):
OutputModule.__init__(self, actor_config)
self.pool.createQueue("inbox")
self.registerConsumer(self.consume, "inbox")
self.connect = Event()
self.connect.set()
self.do_consume = Event()
self.do_consume.clear()
self.channel = None
self.connected = False
def heartbeat(self):
while self.loop():
sleep(self.kwargs.heartbeat)
try:
if self.connected:
self.connection.send_heartbeat()
except Exception as err:
self.logging.error("Failed to send heartbeat. Reason: %s" % (err))
def preHook(self):
self._queue_arguments = dict(self.kwargs.queue_arguments)
self._exchange_arguments = dict(self.kwargs.exchange_arguments)
self.sendToBackground(self.setupConnectivity)
def consume(self, event):
self.do_consume.wait()
if self.channel is None:
self.logging.error("Failed to submit message. Initial connection not established yet.")
else:
data = self.getDataToSubmit(event)
data = self.encode(data)
message = basic_message.Message(
body=data,
delivery_mode=self.kwargs.delivery_mode
)
try:
self.channel.basic_publish(
message,
exchange=self.kwargs.exchange,
routing_key=self.kwargs.routing_key
)
except Exception as err:
self.logging.error("Failed to submit event to broker. Reason: %s" % (err))
self.connected = False
self.connect.set()
def setupConnectivity(self):
if self.kwargs.heartbeat > 0:
self.logging.info("Sending heartbeat every %s seconds." % (self.kwargs.heartbeat))
self.sendToBackground(self.heartbeat)
while self.loop():
self.connect.wait()
self.logging.debug("Connecting to %s" % (self.kwargs.host))
try:
self.connection = Connection(
host=self.kwargs.host,
heartbeat=self.kwargs.heartbeat,
virtual_host=self.kwargs.vhost,
userid=self.kwargs.user,
password=self.kwargs.password,
ssl=self.kwargs.ssl,
connect_timeout=5
)
self.connection.connect()
self.channel = self.connection.channel()
if self.kwargs.exchange != "":
self.channel.exchange_declare(
self.kwargs.exchange,
self.kwargs.exchange_type,
durable=self.kwargs.exchange_durable,
auto_delete=self.kwargs.exchange_auto_delete,
passive=self.kwargs.exchange_passive,
arguments=self._exchange_arguments
)
self.logging.debug("Declared exchange %s." % (self.kwargs.exchange))
if self.kwargs.queue_declare:
self.channel.queue_declare(
self.kwargs.queue,
durable=self.kwargs.queue_durable,
exclusive=self.kwargs.queue_exclusive,
auto_delete=self.kwargs.queue_auto_delete,
arguments=self._queue_arguments
)
self.logging.debug("Declared queue %s." % (self.kwargs.queue))
if self.kwargs.exchange != "":
self.channel.queue_bind(
self.kwargs.queue,
self.kwargs.exchange,
routing_key=self.kwargs.routing_key
)
self.logging.debug("Bound queue %s to exchange %s." % (self.kwargs.queue, self.kwargs.exchange))
self.logging.info("Connected to broker %s." % (self.kwargs.host))
except Exception as err:
self.logging.error("Failed to connect to broker. Reason %s " % (err))
sleep(1)
else:
self.do_consume.set()
self.connect.clear()
self.connected = True
def postHook(self):
try:
self.channel.close()
except Exception as err:
del(err)
try:
self.connection.close()
except Exception as err:
del(err)
def send(self, users=None, dismiss=False):
# FIXME Imported here due to circular dependency issues.
from mist.api.notifications.models import InAppNotification
from mist.api.notifications.models import UserNotificationPolicy
# Get the list of `InAppNotifications`s in the current context before
# any update takes place.
owner_old_ntfs = list(InAppNotification.objects(owner=self.ntf.owner))
if not users:
users = self.ntf.owner.members
elif not isinstance(users, list):
users = [users]
# Save/update/dismiss notifications.
if dismiss:
dismissed_by = set(self.ntf.dismissed_by)
old_dismissed_by = list(dismissed_by)
dismissed_by |= set(user.id for user in users)
self.ntf.dismissed_by = list(dismissed_by)
# Is anyone listening?
if not amqp_owner_listening(self.ntf.owner.id):
return
# Initialize AMQP connection to reuse for multiple messages.
amqp_conn = Connection(config.AMQP_URI)
# Re-fetch all notifications in order to calculate the diff between
# the two lists.
owner_new_ntfs = list(InAppNotification.objects(owner=self.ntf.owner))
# Apply each user's notification policy on the above lists to get rid
# of notifications users are not interested in.
for user in users:
user_old_ntfs, user_new_ntfs = [], []
try:
np = UserNotificationPolicy.objects.get(user_id=user.id)
except UserNotificationPolicy.DoesNotExist:
log.debug('No UserNotificationPolicy found for %s', user)
user_old_ntfs = [
ntf.as_dict() for ntf in owner_old_ntfs if not (
self.ntf.id == ntf.id and user.id in old_dismissed_by)
]
user_new_ntfs = [
ntf.as_dict() for ntf in owner_new_ntfs
if not (self.ntf.id == ntf.id and user.id in dismissed_by)
]
else:
user_old_ntfs = [
ntf.as_dict() for ntf in owner_old_ntfs
if not np.has_blocked(ntf) and not (
self.ntf.id == ntf.id and user.id in old_dismissed_by)
]
user_new_ntfs = [
ntf.as_dict() for ntf in owner_new_ntfs
if not np.has_blocked(ntf)
and not (self.ntf.id == ntf.id and user.id in dismissed_by)
]
# Now we can save the dismissed notification
self.ntf.save()
# Calculate diff.
patch = jsonpatch.JsonPatch.from_diff(user_old_ntfs,
user_new_ntfs).patch
if patch:
amqp_publish_user(self.ntf.owner.id,
routing_key='patch_notifications',
connection=amqp_conn,
data={
'user': user.id,
'patch': patch
})
# Finally, try to close the AMQP connection.
try:
amqp_conn.close()
except Exception as exc:
log.exception(repr(exc))
#!/usr/bin/env python
from amqp.connection import Connection
from amqp.channel import Channel
from amqp.basic_message import Message
import json
connection = Connection()
channel = Channel(connection)
channel.exchange_declare('django_amqp_example', 'topic', auto_delete=False)
channel.queue_declare(queue='task_queue', durable=True)
channel.queue_bind('task_queue', 'django_amqp_example', 'task_queue')
def callback(msg):
print "Received request:", msg.body
content = json.loads(msg.body)['content']
response = {
'rot13': content.encode('rot13')
}
response_msg = Message(
body=json.dumps(response),
exchange='django_amqp_example')
print "Sending response:", json.dumps(response)
channel.basic_publish(
response_msg,
routing_key=msg.reply_to)
channel.basic_consume(callback=callback, queue='task_queue')
def queue():
c = Connection()
ch = c.channel()
return QueueManager(channel=ch)
class QueueManager(object):
TTL_MAX = 86400000 # 604800000 one-week # 2147483647 max
TIME_FORMAT = '%Y%m%d%H%M%S.%f'
TAGS = {
'HEADER' : 'PQ4R',
'READY' : 'R',
'SCHEDULE' : 'S',
'EXCHANGE' : 'O',
}
SCHEDULING_ACCURACY = 1000 # second
def setup_tag(self, tag):
rtag = self.get_tag(tag, 'READY')
if not self.tags_tbl.get(rtag):
stag = self.get_tag(tag, 'SCHEDULE')
otag = self.get_tag(tag, 'EXCHANGE')
self.channel.queue_declare(queue=rtag, durable=True, auto_delete=False)
self.channel.exchange_declare(exchange=otag, type='topic', durable=True)
self.channel.queue_bind(exchange=otag, routing_key=stag+'.*', queue=rtag)
self.tags_tbl[rtag] = True
def __init__(self, host=None, userid=None, on_dead=lambda m: m, channel=None, error_times_to_ignore = 3):
self.host = host
self.userid = userid
self.connection = None
self.channel = channel
self.serializer = lambda d: json.dumps(d, separators=(',',':'))
self.deserializer = lambda d: json.loads(d)
self.on_dead = on_dead
self.listenning = False
self.consumer = None
self.error_times_to_ignore = error_times_to_ignore
self.tags_tbl = {}
@contextmanager
def open(self, channel=None):
if channel:
saved_channel = self.channel
self.channel = channel
yield self.channel
self.channel = saved_channel
elif self.channel:
yield self.channel
else:
try:
args = {}
if self.host: args['host'] = self.host
if self.userid: args['userid'] = self.userid
self.connection = Connection(**args)
self.channel = self.connection.channel()
yield self.channel
finally:
self.close()
return
def close(self):
if self.channel:
self.channel.close()
self.channel = None
if self.connection:
self.connection.close()
self.connection = None
def compose_tag(self, tags):
if isinstance(tags, str):
tags = [tags]
for tag in tags:
if not re.match(r'^[A-Za-z0-9_$\-]+$', tag):
raise Exception('validation error: tag must be matched /^[A-Za-z0-9_$\\-]+$/')
return ':'.join(tags)
def get_tag(self, tags, typ):
return ('%s.%s.%s' % (self.TAGS['HEADER'], self.TAGS[typ], self.compose_tag(tags)))
def count(self, tag=None):
# This function is really slow.
proc = subprocess.Popen("sudo rabbitmqctl list_queues", shell=True, stdout=subprocess.PIPE)
stdout_value = proc.communicate()[0]
rt = stdout_value.decode('utf-8')
tag_count_pairs = map(lambda p: p.split('\t'), rt.rstrip().split('\n')[1:-1])
rtag = self.get_tag(tag, 'READY')
stag = self.get_tag(tag, 'SCHEDULE')
rt = { 'READY': 0, 'SCHEDULED': 0 }
for tag, count in tag_count_pairs:
if tag == rtag:
rt['READY'] = int(count)
elif tag[0:len(stag)] == stag:
rt['SCHEDULED'] += int(count)
else:
pass
return rt
def gensym(self, l=5):
src = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'
h = ''
for i in range(l): h = h + random.choice(src)
return h + '_' + datetime.now().strftime('%H%M%S%f')
def enqueue(self, tag, data, schedule = None, error_times=0, channel=None):
with self.open(channel):
self.setup_tag(tag)
msg = Message()
msg.properties['delivery_mode'] = 2
body = {'i': self.gensym(), 'c': data, 'e': error_times}
id = None
if schedule:
ttl = round_by_accuracy(get_millis_to_the_date(schedule), self.SCHEDULING_ACCURACY)
ttl1 = min(ttl, self.TTL_MAX)
stag = ('%s.%s' % (self.get_tag(tag, 'SCHEDULE'), str(ttl1), ))
self.channel.queue_declare(
queue=stag, durable=True, auto_delete=False,
arguments = { 'x-message-ttl' : ttl1, "x-dead-letter-exchange" : self.get_tag(tag, 'EXCHANGE') }
)
body['s'] = schedule.strftime(self.TIME_FORMAT)
msg.body = self.serializer(body)
id = self.channel.basic_publish(msg, routing_key=stag)
log(" [%s] Sent scheduled -> ttl: %s(%s) schedule: %s" % (body['i'], ttl1, ttl, body['s'], ))
else:
rtag = self.get_tag(tag, 'READY')
msg.body = self.serializer(body)
id = self.channel.basic_publish(msg, routing_key=rtag)
log(" [%s] Sent immediate -> tag: %s" % (body['i'], rtag))
return body['i']
def notify_dead(self, msg):
self.on_dead(msg)
return
# @contextmanager
# def dequeue_item(self, tag, channel=None):
# with self.open(channel):
# ctag = self.compose_tag(tag)
# receiver = QueueReceiver.get(ctag, self)
# res = receiver.fetchone()
# if res:
# if ((0 < self.error_times_to_ignore) and
# (self.error_times_to_ignore <= res['e'])):
# try:
# self.notify_dead(res)
# except:
# pass
# yield None
# else:
# try:
# yield res
# except:
# self.enqueue(tag, res['c'], error_times = res['e'] + 1)
# raise
# else:
# yield None
# return
#
# @contextmanager
# def dequeue(self, tag, channel=None):
# with self.dequeue_item(tag, channel) as res:
# if res:
# yield res['c']
# else:
# yield res
# return
def listen_item(self, tag, on_recv, channel=None):
#
def listen_item_iter(res):
nonlocal on_recv
if res:
if ((0 < self.error_times_to_ignore) and
(self.error_times_to_ignore <= res['e'])):
try:
self.notify_dead(res)
except:
pass
else:
on_recv(res)
#
self.stop_listen()
self.listenning = True
while True:
if self.listenning:
with self.open(channel):
self.setup_tag(tag)
self.consumer = QueueConsumer(tag, self, listen_item_iter)
self.consumer.start()
try:
while self.consumer.wait1():
pass
finally:
self.consumer.stop()
else:
return None
def stop_listen(self):
if self.listenning:
if self.consumer:
self.consumer.stop()
self.listenning = False
def listen(self, tag, on_recv, channel=None):
wrap = lambda d: on_recv(d['c'] if d != None else None)
self.listen_item(tag, wrap, channel)
def dequeue_item_immediate(self, tag, channel=None):
ctag = self.compose_tag(tag)
with self.open(channel):
self.setup_tag(tag)
msg = self.channel.basic_get(tag)
if msg:
headers = msg.properties.get('application_headers')
death = headers.get('x-death') if headers else None
reason = death[0]['reason'] if death else None
expired = (reason == 'expired')
body = self.deserializer(msg.body)
id = body['i']
schedule = body.get('s')
#
self.channel.basic_ack(msg.delivery_tag)
#
if schedule and expired:
ready_time = datetime.strptime(schedule, self.TIME_FORMAT)
n = datetime.now()
if ready_time < n:
log(" [%s] Done Scheduled missed %sms" % (id, timedelta_to_millis(n - ready_time), ))
return body
else:
self.enqueue(self.tag, body['c'], ready_time)
log(" [%s] Re-Published -> %s" % (id, schedule, ))
elif not schedule and not expired:
log(" [%s] Done Immediately" % id)
return body
else:
log(" [%s] Not-Done because of (%s)" % (id, reason))
return None
def dequeue_immediate(self, tag, channel=None):
res = self.dequeue_item_immediate(tag, channel)
return res['c'] if res else None
def list_machines(self, persist=True):
"""Return list of machines for cloud
A list of nodes is fetched from libcloud, the data is processed, stored
on machine models, and a list of machine models is returned.
Subclasses SHOULD NOT override or extend this method.
This method wraps `_list_machines` which contains the core
implementation.
"""
task_key = 'cloud:list_machines:%s' % self.cloud.id
task = PeriodicTaskInfo.get_or_add(task_key)
try:
with task.task_runner(persist=persist):
old_machines = {
'%s-%s' % (m.id, m.machine_id): m.as_dict()
for m in self.list_cached_machines()
}
machines = self._list_machines()
except PeriodicTaskThresholdExceeded:
self.cloud.disable()
raise
# Initialize AMQP connection to reuse for multiple messages.
amqp_conn = Connection(config.AMQP_URI)
if amqp_owner_listening(self.cloud.owner.id):
if not config.MACHINE_PATCHES:
amqp_publish_user(
self.cloud.owner.id,
routing_key='list_machines',
connection=amqp_conn,
data={
'cloud_id': self.cloud.id,
'machines':
[machine.as_dict() for machine in machines]
})
else:
# Publish patches to rabbitmq.
new_machines = {
'%s-%s' % (m.id, m.machine_id): m.as_dict()
for m in machines
}
# Exclude last seen and probe fields from patch.
for md in old_machines, new_machines:
for m in md.values():
m.pop('last_seen')
m.pop('probe')
patch = jsonpatch.JsonPatch.from_diff(old_machines,
new_machines).patch
if patch:
amqp_publish_user(self.cloud.owner.id,
routing_key='patch_machines',
connection=amqp_conn,
data={
'cloud_id': self.cloud.id,
'patch': patch
})
# Push historic information for inventory and cost reporting.
for machine in machines:
data = {
'owner_id': self.cloud.owner.id,
'machine_id': machine.id,
'cost_per_month': machine.cost.monthly
}
amqp_publish(exchange='machines_inventory',
routing_key='',
auto_delete=False,
data=data,
connection=amqp_conn)
return machines
class App(object):
handlers = []
@classmethod # register single or multiple handlers to the application
def register_handlers(cls, *handlers):
cls.handlers = list(handlers)
def __init__(self, host, userid, password, virtual_host):
self.connection_meta = {}
if host:
self.connection_meta['host'] = host
if userid:
self.connection_meta['userid'] = userid
if password:
self.connection_meta['password'] = password
if virtual_host:
self.connection_meta['virtual_host'] = virtual_host
self.terminated = False
self.connection = None
self.channel = None
self.setup_conn()
def setup_conn(self):
self.connection = Connection(**self.connection_meta)
self.channel = Channel(self.connection)
# Since amqp v2.0, you should explicitly call Connection.connect() and Channel.open()
if VERSION[0] >= 2:
self.connection.connect()
self.channel.open()
@staticmethod
def welcome():
UserInterface.output(
banner.lstrip() + "Connected to the channel.\n"
"Type `help` to see the help document.\n"
"Type `exit` or `Ctrl+C` whenever to exit this shell.")
@staticmethod
def help():
for handler in App.handlers:
command = handler.group + "." + handler.name
for meta_argument in handler.meta_arguments:
command += ' ' + str(meta_argument)
UserInterface.output(command)
def terminate(self):
UserInterface.output("Oops! Please don't go... /(ㄒoㄒ)/~~")
self.terminated = True
def event_loop(self):
self.terminated = False
try:
# event_loop starts a event loop to "read - parse - handle - output"
while not self.terminated:
cmd = UserInterface.read()
try:
if len(cmd) == 0:
UserInterface.output(
"Nothing entered, please type any command")
continue
if cmd.split()[0] == "exit":
self.terminate()
continue
if cmd.split()[0] == "help":
App.help()
continue
self.dispatch(cmd)
except UnsupportedCommandError:
UserInterface.output("Unsupported Command: {}".format(cmd))
except InvalidArgumentValueError as e:
UserInterface.output(e.message)
except KeyboardInterrupt:
self.terminate()
def dispatch(self, cmd):
# dispatch forwards the cmd and its arguments to corresponding handler
cmd, arguments = cmd.split()[0], cmd.split()[1:]
for handler in App.handlers:
if cmd == handler.group + "." + handler.name:
return handler(channel=self.channel,
arguments=arguments).perform()
raise UnsupportedCommandError
