def close_channel(self): """Call to close the channel with RabbitMQ cleanly by issuing the Channel.Close RPC command. """ zlogger.info('Closing the channel') self._channel.close()
def post_push_acknowledgement(self, **kwargs): """ Each client sends an acknowledgement message when they receive a push message. This acknowledgement must be saved to DeliveryInfo bucket. Args: **kwargs (dict): project_id, target_id, client_id, message_id: - project_id: project_id - target_id: alphanumeric string, represents the target of the push message - client_id: string, represents the target's device-application pair - message_id: alphanumeric string, represents the push message """ client_bucket = self.get_bucket(kwargs['project_id'], 'client') message_bucket = self.get_bucket(kwargs['project_id'], 'push_message') client = client_bucket.get(kwargs['client_id']) message = message_bucket.get(kwargs['message_id']) if client.exists and message.exists: bucket = self.get_bucket(kwargs['project_id'], 'delivery_info') kwargs.update(self.get_creation_info()) bucket.new(data=kwargs).store() zlogger.info("New delivery info record is created. Project id:{}, client id:{}".format( kwargs['project_id'], kwargs['client_id']) )
def add_on_connection_close_callback(self): """This method adds an on close callback that will be invoked by pika when RabbitMQ closes the connection to the publisher unexpectedly. """ zlogger.info('Adding connection close callback') self._connection.add_on_close_callback(self.on_connection_closed)
def add_on_channel_close_callback(self): """This method tells pika to call the on_channel_closed method if RabbitMQ unexpectedly closes the channel. """ zlogger.info('Adding channel close callback') self._channel.add_on_close_callback(self.on_channel_closed)
def create_push_segment(self, project_id=None, **kwargs): """ Creates a push segment with given args. Args: **kwargs (dict): project_id, name, residents - project_id: project_id - name (str): name of segment - residents (dict): object of expressions which represent the audience set of segment Returns: """ project_id = project_id or kwargs['project_id'] bucket = self.get_bucket(project_id, 'segment') segment = bucket.get(kwargs['segment_id']) if segment.exists: # todo: let client know the new segment id kwargs['segment_id'] = generate_uuid() segment = bucket.get(kwargs['segment_id']) segment.data = {'name': kwargs['name'], 'residents': kwargs['residents'], 'id': kwargs['segment_id']} segment.add_index('segment_bin', project_id) for val in kwargs['residents']['sets'].values(): segment.add_index('tag_seg_bin', val['key']) segment.data.update(self.get_creation_info()) segment.store() zlogger.info("New segment is created. Project id:{}, segment id:{}".format( project_id, segment.key))
def create_push_tag(self, **kwargs): """ Creates a single tag with given id. Args: **kwargs (dict): project_id, name, tag_type, value_type - project_id: project id - name: name of tag - tag_type: tag type - value_type: value type Returns: """ bucket = self.get_bucket(kwargs['project_id'], 'tag') tag_key = "{}_{}".format(kwargs['project_id'], kwargs['name']) if bucket.get(tag_key).exists: zlogger.error(" Tag to be saved is already exists. Project id:{}, tag name:{}".format( kwargs['project_id'], kwargs['name'])) return {"error": {"code": -32005, "message": "Invalid value, tag is already exists."}} kwargs['possible_values'] = {} kwargs.update(self.get_creation_info()) obj = bucket.new(data=kwargs) obj.add_index('tag_bin', kwargs['project_id']) obj.key = tag_key obj.store() zlogger.info("New push tag is created. Project id:{}, tag name:{}".format( kwargs['project_id'], kwargs['name']))
def update_client(self, **kwargs): """ Updates the client info with given values. Args: **kwargs (dict): project_id, client_id, target_id, validated_client: - project_id: project_id - client_id: id of client which is desired to be updated - target_id: id of user who wants to update the client information, possibly user of the client - validated_client: dict of updated client information including clientId, token, appVersion, deviceType, language, country, osVersion. Returns: """ client = self.get_obj(kwargs['project_id'], 'client', kwargs['client_id']) if client.data['target_id'] != kwargs['target_id']: zlogger.error("Target id is not equal to wanted client's target. Project id:{}, Target " "id:{}, client id:{}".format(kwargs['project_id'], kwargs['target_id'], kwargs['client_id'])) return {"error": {"code": -32004, "message": "Unauthorized Operation Error"}} client.data.update(kwargs['validated_client']) client.data['last_update_time'] = datetime.now().strftime(DATETIME_FORMAT) client.store() zlogger.info("Client data is updated. Project id:{}, " "client id:{}".format(kwargs['project_id'], kwargs['client_id']))
def main(): t1 = Thread(target=sd_rabbit.watch_rabbit) t2 = Thread(target=sd_riak.watch_riak) t4 = Thread(target=sd_redis.watch_redis) t1.start() t2.start() t4.start() while not getattr(sd_rabbit, 'rabbit_nodes') or \ not getattr(sd_riak, 'riak_pb') or not getattr(sd_riak, 'log_bucket') or \ not getattr(sd_redis, 'redis_master'): zlogger.info( "Still waiting for rabbit_nodes, riak_pb, log_bucket, redis_master" ) time.sleep(0.1) rabbit_cl = sd_rabbit.get_suitable_client( json.loads(sd_rabbit.rabbit_nodes)) rpcserver = RPCServer(sd_riak.riak_pb, rabbit_cl, sd_redis.redis_master) t3 = Thread(target=rpcserver.run) t3.start() zlogger.info("Started RpcServer...")
def rpc(self): """ Makes an rpc call to get raw data of object. Args: bucket_name(str): project_id(str): object_id(str): service(str): Returns: dict: returns riak_object.data for given object. """ if not (self.bucket_name or self.project or self.object_id or self.service): zlogger.info( "Invalid value for bucket_name, project_id, object_id or service." ) raise ValueError( "Invalid value for bucket_name, project_id, object_id or service." ) params = { "bucket_name": self.bucket_name, "project_id": self.project, "object_id": self.object_id, "service": self.service, } return self.rpc_client.rpc_call("get_obj_data", params)
def open_channel(self): """Open a new channel with RabbitMQ by issuing the Channel.Open RPC command. When RabbitMQ responds that the channel is open, the on_channel_open callback will be invoked by pika. """ zlogger.info('Creating a new channel') self._connection.channel(on_open_callback=self.on_channel_open)
def stop_consuming(self): """Tell RabbitMQ that you would like to stop consuming by sending the Basic.Cancel RPC command. """ if self._channel: zlogger.info('Sending a Basic.Cancel RPC command to RabbitMQ') self._channel.basic_cancel(self.on_cancelok, self._consumer_tag)
def add_on_cancel_callback(self): """Add a callback that will be invoked if RabbitMQ cancels the consumer for some reason. If RabbitMQ does cancel the consumer, on_consumer_cancelled will be invoked by pika. """ zlogger.info('Adding consumer cancellation callback') self._channel.add_on_cancel_callback(self.on_consumer_cancelled)
def acknowledge_message(self, delivery_tag): """Acknowledge the message delivery from RabbitMQ by sending a Basic.Ack RPC method for the delivery tag. :param int delivery_tag: The delivery tag from the Basic.Deliver frame """ zlogger.info('Acknowledging message %s', delivery_tag) self._channel.basic_ack(delivery_tag)
def on_exchange_declareok(self, unused_frame): """Invoked by pika when RabbitMQ has finished the Exchange.Declare RPC command. :param pika.Frame.Method unused_frame: Exchange.DeclareOk response frame """ zlogger.info('Exchange declared') self.setup_queue(self.QUEUE)
def on_channel_open(self, channel): """ Input channel creation callback, exchange declaring. """ zlogger.info('PikaClient: Channel open, Declaring exchange') self.channel = channel self.channel.exchange_declare(exchange='messages', type='topic', durable=True)
def on_bindok(self, unused_frame): """Invoked by pika when the Queue.Bind method has completed. At this point we will start consuming messages by calling start_consuming which will invoke the needed RPC commands to start the process. :param pika.frame.Method unused_frame: The Queue.BindOk response frame """ zlogger.info('Queue bound') self.start_consuming()
def on_connection_open(self, unused_connection): """This method is called by pika once the connection to RabbitMQ has been established. It passes the handle to the connection object in case we need it, but in this case, we'll just mark it unused. :type unused_connection: pika.SelectConnection """ zlogger.info('Connection opened') self.add_on_connection_close_callback() self.open_channel()
def on_cancelok(self, unused_frame): """This method is invoked by pika when RabbitMQ acknowledges the cancellation of a consumer. At this point we will close the channel. This will invoke the on_channel_closed method once the channel has been closed, which will in-turn close the connection. :param pika.frame.Method unused_frame: The Basic.CancelOk frame """ zlogger.info('RabbitMQ acknowledged the cancellation of the consumer') self.close_channel()
def on_consumer_cancelled(self, method_frame): """Invoked by pika when RabbitMQ sends a Basic.Cancel for a consumer receiving messages. :param pika.frame.Method method_frame: The Basic.Cancel frame """ zlogger.info('Consumer was cancelled remotely, shutting down: %r', method_frame) if self._channel: self._channel.close()
def on_connected(self, connection): """ AMQP connection callback. Creates input channel. Args: connection: AMQP connection """ zlogger.info('PikaClient: connected to RabbitMQ') self.connection.channel(self.on_channel_open)
def start_consuming(self): """ Overrided this method to use on_request callback Original method in rpc_server.py """ self.add_on_cancel_callback() self._channel.basic_qos(prefetch_count=1) self._consumer_tag = self._channel.basic_consume( consumer_callback=self.on_request, queue=self.QUEUE) zlogger.info("Started consuming...")
def setup_queue(self, queue_name): """Setup the queue on RabbitMQ by invoking the Queue.Declare RPC command. When it is complete, the on_queue_declareok method will be invoked by pika. :param str|unicode queue_name: The name of the queue to declare. """ zlogger.info('Declaring queue %s', queue_name) self._channel.queue_declare(self.on_queue_declareok, queue_name, durable=True)
def send_push_message(self, project_id, service, client_ids, validated_message, message_id): device_tokens = self.group_by_device_type(client_ids, project_id) for device_type, tokens in device_tokens.items(): send_push_message(device_type, tokens, validated_message, project_id) zlogger.info( "Send Push Message. Project id:{}, message id:{}".format( project_id, message_id), extra={ "project_id": project_id, "service": service, "purpose": "counter", })
def on_channel_open(self, channel): """This method is invoked by pika when the channel has been opened. The channel object is passed in so we can make use of it. Since the channel is now open, we'll declare the exchange to use. :param pika.channel.Channel channel: The channel object """ zlogger.info('Channel opened') self._channel = channel self.add_on_channel_close_callback() self.setup_exchange(self.EXCHANGE)
def setup_exchange(self, exchange_name): """Setup the exchange on RabbitMQ by invoking the Exchange.Declare RPC command. When it is complete, the on_exchange_declareok method will be invoked by pika. :param str|unicode exchange_name: The name of the exchange to declare """ zlogger.info('Declaring exchange %s', exchange_name) self._channel.exchange_declare(self.on_exchange_declareok, exchange_name, self.EXCHANGE_TYPE, durable=True)
def start_consuming(self): """This method sets up the consumer by first calling add_on_cancel_callback so that the object is notified if RabbitMQ cancels the consumer. It then issues the Basic.Consume RPC command which returns the consumer tag that is used to uniquely identify the consumer with RabbitMQ. We keep the value to use it when we want to cancel consuming. The on_message method is passed in as a callback pika will invoke when a message is fully received. """ zlogger.info('Issuing consumer related RPC commands') self.add_on_cancel_callback() self._channel.basic_qos(prefetch_count=1) self._consumer_tag = self._channel.basic_consume( consumer_callback=self.on_message, queue=self.QUEUE)
def watch_postgres(single=False): cur_index = None while True: index, data = consul_client.catalog.service('postgresql', index=cur_index, wait='10m') if index != cur_index: cur_index = index global postgres_nodes postgres_nodes = data[0]['ServiceAddress'] zlogger.info( f"Postgresql Nodes are changed. New nodes are {postgres_nodes}." ) if single: return postgres_nodes
def on_queue_declareok(self, method_frame): """Method invoked by pika when the Queue.Declare RPC call made in setup_queue has completed. In this method we will bind the queue and exchange together with the routing key by issuing the Queue.Bind RPC command. When this command is complete, the on_bindok method will be invoked by pika. :param pika.frame.Method method_frame: The Queue.DeclareOk frame """ zlogger.info('Binding %s to %s with %s', self.EXCHANGE, self.QUEUE, self.ROUTING_KEYS) for ROUTING_KEY in self.ROUTING_KEYS: self._channel.queue_bind(self.on_bindok, self.QUEUE, self.EXCHANGE, str(ROUTING_KEY))
def delete_push_segment(self, **kwargs): """ Deletes a push segment with given id. Args: **kwargs (dict): project_id, segment_id - project_id: project_id - segment_id: unique identifier of segment Returns: """ segment = self.get_obj(kwargs['project_id'], 'segment', kwargs['segment_id']) segment.delete() zlogger.info("Segment is deleted. Project id:{}, segment id:{}".format( kwargs['project_id'], kwargs['segment_id']))
def stop(self): """Cleanly shutdown the connection to RabbitMQ by stopping the consumer with RabbitMQ. When RabbitMQ confirms the cancellation, on_cancelok will be invoked by pika, which will then closing the channel and connection. The IOLoop is started again because this method is invoked when CTRL-C is pressed raising a KeyboardInterrupt exception. This exception stops the IOLoop which needs to be running for pika to communicate with RabbitMQ. All of the commands issued prior to starting the IOLoop will be buffered but not processed. """ zlogger.info('Stopping') self._closing = True self.stop_consuming() self._connection.ioloop.start() zlogger.info('Stopped')