def close_channel(self):
"""Call to close the channel with RabbitMQ cleanly by issuing the
Channel.Close RPC command.
"""
zlogger.info('Closing the channel')
self._channel.close()
def post_push_acknowledgement(self, **kwargs):
"""
Each client sends an acknowledgement message when they receive a push message.
This acknowledgement must be saved to DeliveryInfo bucket.
Args:
**kwargs (dict): project_id, target_id, client_id, message_id:
- project_id: project_id
- target_id: alphanumeric string, represents the target of the push message
- client_id: string, represents the target's device-application pair
- message_id: alphanumeric string, represents the push message
"""
client_bucket = self.get_bucket(kwargs['project_id'], 'client')
message_bucket = self.get_bucket(kwargs['project_id'], 'push_message')
client = client_bucket.get(kwargs['client_id'])
message = message_bucket.get(kwargs['message_id'])
if client.exists and message.exists:
bucket = self.get_bucket(kwargs['project_id'], 'delivery_info')
kwargs.update(self.get_creation_info())
bucket.new(data=kwargs).store()
zlogger.info("New delivery info record is created. Project id:{}, client id:{}".format(
kwargs['project_id'], kwargs['client_id'])
)
def add_on_connection_close_callback(self):
"""This method adds an on close callback that will be invoked by pika
when RabbitMQ closes the connection to the publisher unexpectedly.
"""
zlogger.info('Adding connection close callback')
self._connection.add_on_close_callback(self.on_connection_closed)
def add_on_channel_close_callback(self):
"""This method tells pika to call the on_channel_closed method if
RabbitMQ unexpectedly closes the channel.
"""
zlogger.info('Adding channel close callback')
self._channel.add_on_close_callback(self.on_channel_closed)
def create_push_segment(self, project_id=None, **kwargs):
"""
Creates a push segment with given args.
Args:
**kwargs (dict): project_id, name, residents
- project_id: project_id
- name (str): name of segment
- residents (dict): object of expressions which represent the audience set of
segment
Returns:
"""
project_id = project_id or kwargs['project_id']
bucket = self.get_bucket(project_id, 'segment')
segment = bucket.get(kwargs['segment_id'])
if segment.exists:
# todo: let client know the new segment id
kwargs['segment_id'] = generate_uuid()
segment = bucket.get(kwargs['segment_id'])
segment.data = {'name': kwargs['name'], 'residents': kwargs['residents'], 'id': kwargs['segment_id']}
segment.add_index('segment_bin', project_id)
for val in kwargs['residents']['sets'].values():
segment.add_index('tag_seg_bin', val['key'])
segment.data.update(self.get_creation_info())
segment.store()
zlogger.info("New segment is created. Project id:{}, segment id:{}".format(
project_id, segment.key))
def create_push_tag(self, **kwargs):
"""
Creates a single tag with given id.
Args:
**kwargs (dict): project_id, name, tag_type, value_type
- project_id: project id
- name: name of tag
- tag_type: tag type
- value_type: value type
Returns:
"""
bucket = self.get_bucket(kwargs['project_id'], 'tag')
tag_key = "{}_{}".format(kwargs['project_id'], kwargs['name'])
if bucket.get(tag_key).exists:
zlogger.error(" Tag to be saved is already exists. Project id:{}, tag name:{}".format(
kwargs['project_id'], kwargs['name']))
return {"error": {"code": -32005, "message": "Invalid value, tag is already exists."}}
kwargs['possible_values'] = {}
kwargs.update(self.get_creation_info())
obj = bucket.new(data=kwargs)
obj.add_index('tag_bin', kwargs['project_id'])
obj.key = tag_key
obj.store()
zlogger.info("New push tag is created. Project id:{}, tag name:{}".format(
kwargs['project_id'], kwargs['name']))
def update_client(self, **kwargs):
"""
Updates the client info with given values.
Args:
**kwargs (dict): project_id, client_id, target_id, validated_client:
- project_id: project_id
- client_id: id of client which is desired to be updated
- target_id: id of user who wants to update the client information, possibly user
of the client
- validated_client: dict of updated client information including clientId, token,
appVersion, deviceType, language, country, osVersion.
Returns:
"""
client = self.get_obj(kwargs['project_id'], 'client', kwargs['client_id'])
if client.data['target_id'] != kwargs['target_id']:
zlogger.error("Target id is not equal to wanted client's target. Project id:{}, Target "
"id:{}, client id:{}".format(kwargs['project_id'], kwargs['target_id'],
kwargs['client_id']))
return {"error": {"code": -32004, "message": "Unauthorized Operation Error"}}
client.data.update(kwargs['validated_client'])
client.data['last_update_time'] = datetime.now().strftime(DATETIME_FORMAT)
client.store()
zlogger.info("Client data is updated. Project id:{}, "
"client id:{}".format(kwargs['project_id'], kwargs['client_id']))
def main():
t1 = Thread(target=sd_rabbit.watch_rabbit)
t2 = Thread(target=sd_riak.watch_riak)
t4 = Thread(target=sd_redis.watch_redis)
t1.start()
t2.start()
t4.start()
while not getattr(sd_rabbit, 'rabbit_nodes') or \
not getattr(sd_riak, 'riak_pb') or not getattr(sd_riak, 'log_bucket') or \
not getattr(sd_redis, 'redis_master'):
zlogger.info(
"Still waiting for rabbit_nodes, riak_pb, log_bucket, redis_master"
)
time.sleep(0.1)
rabbit_cl = sd_rabbit.get_suitable_client(
json.loads(sd_rabbit.rabbit_nodes))
rpcserver = RPCServer(sd_riak.riak_pb, rabbit_cl, sd_redis.redis_master)
t3 = Thread(target=rpcserver.run)
t3.start()
zlogger.info("Started RpcServer...")
def rpc(self):
"""
Makes an rpc call to get raw data of object.
Args:
bucket_name(str):
project_id(str):
object_id(str):
service(str):
Returns:
dict: returns riak_object.data for given object.
"""
if not (self.bucket_name or self.project or self.object_id
or self.service):
zlogger.info(
"Invalid value for bucket_name, project_id, object_id or service."
)
raise ValueError(
"Invalid value for bucket_name, project_id, object_id or service."
)
params = {
"bucket_name": self.bucket_name,
"project_id": self.project,
"object_id": self.object_id,
"service": self.service,
}
return self.rpc_client.rpc_call("get_obj_data", params)
def open_channel(self):
"""Open a new channel with RabbitMQ by issuing the Channel.Open RPC
command. When RabbitMQ responds that the channel is open, the
on_channel_open callback will be invoked by pika.
"""
zlogger.info('Creating a new channel')
self._connection.channel(on_open_callback=self.on_channel_open)
def stop_consuming(self):
"""Tell RabbitMQ that you would like to stop consuming by sending the
Basic.Cancel RPC command.
"""
if self._channel:
zlogger.info('Sending a Basic.Cancel RPC command to RabbitMQ')
self._channel.basic_cancel(self.on_cancelok, self._consumer_tag)
def add_on_cancel_callback(self):
"""Add a callback that will be invoked if RabbitMQ cancels the consumer
for some reason. If RabbitMQ does cancel the consumer,
on_consumer_cancelled will be invoked by pika.
"""
zlogger.info('Adding consumer cancellation callback')
self._channel.add_on_cancel_callback(self.on_consumer_cancelled)
def acknowledge_message(self, delivery_tag):
"""Acknowledge the message delivery from RabbitMQ by sending a
Basic.Ack RPC method for the delivery tag.
:param int delivery_tag: The delivery tag from the Basic.Deliver frame
"""
zlogger.info('Acknowledging message %s', delivery_tag)
self._channel.basic_ack(delivery_tag)
def on_exchange_declareok(self, unused_frame):
"""Invoked by pika when RabbitMQ has finished the Exchange.Declare RPC
command.
:param pika.Frame.Method unused_frame: Exchange.DeclareOk response frame
"""
zlogger.info('Exchange declared')
self.setup_queue(self.QUEUE)
def on_channel_open(self, channel):
"""
Input channel creation callback, exchange declaring.
"""
zlogger.info('PikaClient: Channel open, Declaring exchange')
self.channel = channel
self.channel.exchange_declare(exchange='messages',
type='topic',
durable=True)
def on_bindok(self, unused_frame):
"""Invoked by pika when the Queue.Bind method has completed. At this
point we will start consuming messages by calling start_consuming
which will invoke the needed RPC commands to start the process.
:param pika.frame.Method unused_frame: The Queue.BindOk response frame
"""
zlogger.info('Queue bound')
self.start_consuming()
def on_connection_open(self, unused_connection):
"""This method is called by pika once the connection to RabbitMQ has
been established. It passes the handle to the connection object in
case we need it, but in this case, we'll just mark it unused.
:type unused_connection: pika.SelectConnection
"""
zlogger.info('Connection opened')
self.add_on_connection_close_callback()
self.open_channel()
def on_cancelok(self, unused_frame):
"""This method is invoked by pika when RabbitMQ acknowledges the
cancellation of a consumer. At this point we will close the channel.
This will invoke the on_channel_closed method once the channel has been
closed, which will in-turn close the connection.
:param pika.frame.Method unused_frame: The Basic.CancelOk frame
"""
zlogger.info('RabbitMQ acknowledged the cancellation of the consumer')
self.close_channel()
def on_consumer_cancelled(self, method_frame):
"""Invoked by pika when RabbitMQ sends a Basic.Cancel for a consumer
receiving messages.
:param pika.frame.Method method_frame: The Basic.Cancel frame
"""
zlogger.info('Consumer was cancelled remotely, shutting down: %r',
method_frame)
if self._channel:
self._channel.close()
def on_connected(self, connection):
"""
AMQP connection callback.
Creates input channel.
Args:
connection: AMQP connection
"""
zlogger.info('PikaClient: connected to RabbitMQ')
self.connection.channel(self.on_channel_open)
def start_consuming(self):
"""
Overrided this method to use on_request callback
Original method in rpc_server.py
"""
self.add_on_cancel_callback()
self._channel.basic_qos(prefetch_count=1)
self._consumer_tag = self._channel.basic_consume(
consumer_callback=self.on_request, queue=self.QUEUE)
zlogger.info("Started consuming...")
def setup_queue(self, queue_name):
"""Setup the queue on RabbitMQ by invoking the Queue.Declare RPC
command. When it is complete, the on_queue_declareok method will
be invoked by pika.
:param str|unicode queue_name: The name of the queue to declare.
"""
zlogger.info('Declaring queue %s', queue_name)
self._channel.queue_declare(self.on_queue_declareok,
queue_name,
durable=True)
def send_push_message(self, project_id, service, client_ids, validated_message, message_id):
device_tokens = self.group_by_device_type(client_ids, project_id)
for device_type, tokens in device_tokens.items():
send_push_message(device_type, tokens, validated_message, project_id)
zlogger.info(
"Send Push Message. Project id:{}, message id:{}".format(
project_id, message_id),
extra={
"project_id": project_id,
"service": service,
"purpose": "counter",
})
def on_channel_open(self, channel):
"""This method is invoked by pika when the channel has been opened.
The channel object is passed in so we can make use of it.
Since the channel is now open, we'll declare the exchange to use.
:param pika.channel.Channel channel: The channel object
"""
zlogger.info('Channel opened')
self._channel = channel
self.add_on_channel_close_callback()
self.setup_exchange(self.EXCHANGE)
def setup_exchange(self, exchange_name):
"""Setup the exchange on RabbitMQ by invoking the Exchange.Declare RPC
command. When it is complete, the on_exchange_declareok method will
be invoked by pika.
:param str|unicode exchange_name: The name of the exchange to declare
"""
zlogger.info('Declaring exchange %s', exchange_name)
self._channel.exchange_declare(self.on_exchange_declareok,
exchange_name,
self.EXCHANGE_TYPE,
durable=True)
def start_consuming(self):
"""This method sets up the consumer by first calling
add_on_cancel_callback so that the object is notified if RabbitMQ
cancels the consumer. It then issues the Basic.Consume RPC command
which returns the consumer tag that is used to uniquely identify the
consumer with RabbitMQ. We keep the value to use it when we want to
cancel consuming. The on_message method is passed in as a callback pika
will invoke when a message is fully received.
"""
zlogger.info('Issuing consumer related RPC commands')
self.add_on_cancel_callback()
self._channel.basic_qos(prefetch_count=1)
self._consumer_tag = self._channel.basic_consume(
consumer_callback=self.on_message, queue=self.QUEUE)
def watch_postgres(single=False):
cur_index = None
while True:
index, data = consul_client.catalog.service('postgresql',
index=cur_index,
wait='10m')
if index != cur_index:
cur_index = index
global postgres_nodes
postgres_nodes = data[0]['ServiceAddress']
zlogger.info(
f"Postgresql Nodes are changed. New nodes are {postgres_nodes}."
)
if single:
return postgres_nodes
def on_queue_declareok(self, method_frame):
"""Method invoked by pika when the Queue.Declare RPC call made in
setup_queue has completed. In this method we will bind the queue
and exchange together with the routing key by issuing the Queue.Bind
RPC command. When this command is complete, the on_bindok method will
be invoked by pika.
:param pika.frame.Method method_frame: The Queue.DeclareOk frame
"""
zlogger.info('Binding %s to %s with %s', self.EXCHANGE, self.QUEUE,
self.ROUTING_KEYS)
for ROUTING_KEY in self.ROUTING_KEYS:
self._channel.queue_bind(self.on_bindok, self.QUEUE, self.EXCHANGE,
str(ROUTING_KEY))
def delete_push_segment(self, **kwargs):
"""
Deletes a push segment with given id.
Args:
**kwargs (dict): project_id, segment_id
- project_id: project_id
- segment_id: unique identifier of segment
Returns:
"""
segment = self.get_obj(kwargs['project_id'], 'segment', kwargs['segment_id'])
segment.delete()
zlogger.info("Segment is deleted. Project id:{}, segment id:{}".format(
kwargs['project_id'], kwargs['segment_id']))
def stop(self):
"""Cleanly shutdown the connection to RabbitMQ by stopping the consumer
with RabbitMQ. When RabbitMQ confirms the cancellation, on_cancelok
will be invoked by pika, which will then closing the channel and
connection. The IOLoop is started again because this method is invoked
when CTRL-C is pressed raising a KeyboardInterrupt exception. This
exception stops the IOLoop which needs to be running for pika to
communicate with RabbitMQ. All of the commands issued prior to starting
the IOLoop will be buffered but not processed.
"""
zlogger.info('Stopping')
self._closing = True
self.stop_consuming()
self._connection.ioloop.start()
zlogger.info('Stopped')