예제 #1
0
    def _init_(self, **kwargs):
        """
        Loads various variables and calls :py:meth:connect() when it's ready.

        :return:
        """
        self.user_id = "gw_" + self._Configs.get('core', 'gwid', 'local',
                                                 False)
        self.login_gwuuid = self.user_id + "_" + self._Configs.get(
            "core", "gwuuid")
        self.request_configs = False
        self.controlHandler = AmqpControlHandler(self)
        self.configHandler = AmqpConfigHandler(self)
        self.systemHandler = AmqpSystemHandler(self)

        self.amqpyombo_options = {   # Stores data from sub-modules
            'connected': [],
            'disconnected': [],
            'routing': {
                'config': [self.configHandler.amqp_incoming, ],
                'control': [self.controlHandler.amqp_incoming, ],
                'system': [self.systemHandler.amqp_incoming, ],
                'sslcerts': [self._SSLCerts.amqp_incoming, ],
            },
        }

        self.amqp = None  # holds our pointer for out amqp connection.
        self._getAllConfigsLoggerLoop = None
        self.send_local_information_loop = None  # used to periodically send yombo servers updated information

        self.connected = False
        self.init_deferred = Deferred()
        self.connect()
        return self.init_deferred
예제 #2
0
    def _init_(self, **kwargs):
        """
        Loads various variables and calls :py:meth:connect() when it's ready.

        :return:
        """
        self.gateway_id = "gw_" + self._Configs.get('core', 'gwid', 'local',
                                                    False)
        self.login_gwuuid = self.gateway_id + "_" + self._Configs.get(
            "core", "gwuuid")
        self._LocalDBLibrary = self._Libraries['localdb']
        self.request_configs = False

        self.amqp = None  # holds our pointer for out amqp connection.
        self._getAllConfigsLoggerLoop = None
        self.send_local_information_loop = None  # used to periodically send yombo servers updated information

        self.controlHandler = AmqpControlHandler(self)
        self.configHandler = AmqpConfigHandler(self)

        self._States.set('amqp.amqpyombo.state', False)
        self.init_deferred = Deferred()
        self.connect()
        return self.init_deferred
예제 #3
0
    def _init_(self):
        """
        Loads various variables and calls :py:meth:connect() when it's ready.

        :return:
        """
        self.user_id = "gw_" + self._Configs.get("core", "gwid")
        self.login_user_id = self.user_id + "_" + self._Configs.get("core", "gwuuid")
        self.__pending_updates = []  # Holds a list of configuration items we've asked for, but not response yet.
        self._LocalDBLibrary = self._Libraries['localdb']
        self.init_startup_count = 0

        self.amqp = None  # holds our pointer for out amqp connection.
        self._getAllConfigsLoggerLoop = None

        self._getAllConfigsLoggerLoop = None
        self.reconnect = True  # If we get disconnected, we should reconnect to the server.
        self.sendLocalInformationLoop = None  # used to periodically send yombo servers updated information

        self.controlHandler = AmqpControlHandler(self)
        self.configHandler = AmqpConfigHandler(self)
        return self.connect()
예제 #4
0
class AMQPYombo(YomboLibrary):
    """
    Handles interactions with Yombo servers through the AMQP library.
    """
    @property
    def connected(self):
        return self._States.get('amqp.amqpyombo.state', None)

    @connected.setter
    def connected(self, val):
        return self._States.set('amqp.amqpyombo.state', val)

    def __str__(self):
        """
        Returns the name of the library.
        :return: Name of the library
        :rtype: string
        """
        return "Yombo amqp yombo library"

    def _init_(self, **kwargs):
        """
        Loads various variables and calls :py:meth:connect() when it's ready.

        :return:
        """
        self.user_id = "gw_" + self._Configs.get('core', 'gwid', 'local',
                                                 False)
        self.login_gwuuid = self.user_id + "_" + self._Configs.get(
            "core", "gwuuid")
        self.request_configs = False
        self.controlHandler = AmqpControlHandler(self)
        self.configHandler = AmqpConfigHandler(self)
        self.systemHandler = AmqpSystemHandler(self)

        self.amqpyombo_options = {   # Stores data from sub-modules
            'connected': [],
            'disconnected': [],
            'routing': {
                'config': [self.configHandler.amqp_incoming, ],
                'control': [self.controlHandler.amqp_incoming, ],
                'system': [self.systemHandler.amqp_incoming, ],
                'sslcerts': [self._SSLCerts.amqp_incoming, ],
            },
        }

        self.amqp = None  # holds our pointer for out amqp connection.
        self._getAllConfigsLoggerLoop = None
        self.send_local_information_loop = None  # used to periodically send yombo servers updated information

        self.connected = False
        self.init_deferred = Deferred()
        self.connect()
        return self.init_deferred

    @inlineCallbacks
    def _load_(self, **kwargs):
        # print("################# about to process_amqpyombo_options")
        results = yield global_invoke_all('_amqpyombo_options_',
                                          called_by=self)
        for component_name, data in results.items():
            if 'connected' in data:
                self.amqpyombo_options['connected'].append(data['connected'])
            if 'disconnected' in data:
                self.amqpyombo_options['disconnected'].append(
                    data['disconnected'])
            if 'routing' in data:
                for key, the_callback in data['gateway_routing'].items():
                    if key not in self.amqpyombo_options['gateway_routing']:
                        self.amqpyombo_options['gateway_routing'][key] = []
                    self.amqpyombo_options['gateway_routing'][key].append(
                        the_callback)

    def _stop_(self, **kwargs):
        """
        Called by the Yombo system when it's time to shutdown. This in turn calls the disconnect.
        :return:
        """

        if self.init_deferred is not None and self.init_deferred.called is False:
            self.init_deferred.callback(
                1)  # if we don't check for this, we can't stop!

        self.configHandler._stop_()
        self.controlHandler._stop_()
        self.systemHandler._stop_()
        self.disconnect(
        )  # will be cleaned up by amqp library anyways, but it's good to be nice.

    def _unload_(self, **kwargs):
        self.amqp.disconnect()

    @inlineCallbacks
    def connect(self):
        """
        Connect to Yombo amqp server.

        :return:
        """
        if self.amqp is None:
            already_have_amqp = None
        else:
            already_have_amqp = True

        environment = self._Configs.get('server', 'environment', "production",
                                        False)
        if self._Configs.get("amqpyombo", 'hostname', "", False) != "":
            amqp_host = self._Configs.get("amqpyombo", 'hostname')
            amqp_port = self._Configs.get("amqpyombo", 'port', 5671, False)
        else:
            amqp_port = 5671
            if environment == "production":
                amqp_host = "amqp.yombo.net"
            elif environment == "staging":
                amqp_host = "amqpstg.yombo.net"
            elif environment == "development":
                amqp_host = "amqpdev.yombo.net"
            else:
                amqp_host = "amqp.yombo.net"

        # get a new amqp instance and connect.
        if self.amqp is None:
            self.amqp = yield self._AMQP.new(
                hostname=amqp_host,
                port=amqp_port,
                virtual_host='yombo',
                username=self.login_gwuuid,
                password=self._Configs.get("core", "gwhash"),
                client_id='amqpyombo',
                prefetch_count=PREFETCH_COUNT,
                connected_callback=self.amqp_connected,
                disconnected_callback=self.amqp_disconnected,
                critical=True)
        self.amqp.connect()

        # The servers will have a dedicated queue for us. All pending messages will be held there for us. If we
        # connect to a different server, they wil automagically be re-routed to our new queue.
        if already_have_amqp is None:
            self.amqp.subscribe("ygw.q." + self.user_id,
                                incoming_callback=self.amqp_incoming,
                                queue_no_ack=False,
                                persistent=True)

        self.configHandler.connect_setup(self.init_deferred)

    def disconnect(self):
        """
        Called by the yombo system when it's time to shutdown.

        :return:
        """
        if self._Loader.operating_mode != 'run':
            return
        requestmsg = self.generate_message_request(
            exchange_name='ysrv.e.gw_system',
            source='yombo.gateway.lib.amqpyombo',
            destination='yombo.server.gw_system',
            request_type="disconnect",
            headers={'request_type': 'disconnect,'},
        )

        self.publish(**requestmsg)

        logger.debug("Disconnected from Yombo message server.")

    def amqp_connected(self):
        """
        Called by AQMP when connected. This function was define above when setting up self.ampq.

        :return:
        """
        self.connected = True
        for the_callback in self.amqpyombo_options['connected']:
            the_callback()

        if self.send_local_information_loop is None:
            self.send_local_information_loop = LoopingCall(
                self.send_local_information)

        # Sends various information, helps Yombo cloud know we are alive and where to find us.
        if self.send_local_information_loop.running is False:
            self.send_local_information_loop.start(random_int(60 * 60 * 4, .2))

    def amqp_disconnected(self):
        """
        Called by AQMP when disconnected.
        :return:
        """
        # logger.info("amqpyombo yombo disconnected: {state}", state=self._States.get('amqp.amqpyombo.state'))
        # If we have at least connected once, then we don't toss errors.  We just wait for reconnection...

        if self.connected is False:
            logger.error(
                "Unable to connect. This may be due to multiple connections or bad gateway hash. See: http://g2.yombo.net/noconnect"
            )
            self._Loader.sigint = True
            self.amqp.disconnect()
            reactor.stop()
            return

        self.connected = False
        for the_callback in self.amqpyombo_options['disconnected']:
            the_callback()
        if self.send_local_information_loop is not None and self.send_local_information_loop.running:
            self.send_local_information_loop.stop()

    def send_local_information(self):
        """
        Say hello, send some information about us. What we use these IP addresses for:

        Devices in your home can connect directly to the gateway for faster access. We give this information
        to your clients so they can find the gateway easier/faster.

        Your external IP is also given to your clients so they can attempt to connect directly to the gateway when
        you are not at home.

        If either of these conenctions are not available, applications can use the Yombo servers as proxy and will
        be delivered here as either a control or status request message.

        :return:
        """
        gwid = self._Configs.get('core', 'gwid', 'local', False)
        body = {
            "is_master":
            self._Configs.get("core", "is_master", True, False),
            "master_gateway":
            self._Configs.get("core", "master_gateway", gwid, False),
            "internal_ipv4":
            self._Configs.get("core", "localipaddress_v4"),
            "external_ipv4":
            self._Configs.get("core", "externalipaddress_v4"),
            # "internal_ipv6": self._Configs.get("core", "externalipaddress_v6"),
            # "external_ipv6": self._Configs.get("core", "externalipaddress_v6"),
            "internal_port":
            self._Configs.get("webinterface", "nonsecure_port"),
            "external_port":
            self._Configs.get("webinterface", "nonsecure_port"),
            "internal_secure_port":
            self._Configs.get("webinterface", "secure_port"),
            "external_secure_port":
            self._Configs.get("webinterface", "secure_port"),
            "internal_mqtt":
            self._Configs.get("mqtt", "server_listen_port"),
            "internal_mqtt_le":
            self._Configs.get("mqtt", "server_listen_port_le_ssl"),
            "internal_mqtt_ss":
            self._Configs.get("mqtt", "server_listen_port_ss_ssl"),
            "internal_mqtt_ws":
            self._Configs.get("mqtt", "server_listen_port_websockets"),
            "internal_mqtt_ws_le":
            self._Configs.get("mqtt", "server_listen_port_websockets_le_ssl"),
            "internal_mqtt_ws_ss":
            self._Configs.get("mqtt", "server_listen_port_websockets_ss_ssl"),
            "external_mqtt":
            self._Configs.get("mqtt", "server_listen_port"),
            "external_mqtt_le":
            self._Configs.get("mqtt", "server_listen_port_le_ssl"),
            "external_mqtt_ss":
            self._Configs.get("mqtt", "server_listen_port_ss_ssl"),
            "external_mqtt_ws":
            self._Configs.get("mqtt", "server_listen_port_websockets"),
            "external_mqtt_ws_le":
            self._Configs.get("mqtt", "server_listen_port_websockets_le_ssl"),
            "external_mqtt_ws_ss":
            self._Configs.get("mqtt", "server_listen_port_websockets_ss_ssl"),
        }

        # logger.info("sending local information: {body}", body=body)

        requestmsg = self.generate_message_request(
            exchange_name='ysrv.e.gw_system',
            source='yombo.gateway.lib.amqpyombo',
            destination='yombo.server.gw_system',
            body=body,
            request_type="connected",
            callback=self.receive_local_information,
        )
        self.publish(**requestmsg)

    def receive_local_information(self,
                                  body=None,
                                  properties=None,
                                  correlation_info=None,
                                  send_message_meta=None,
                                  receied_message_meta=None,
                                  **kwargs):
        if self.request_configs is False:  # this is where we start requesting information - after we have sent out info.
            self.request_configs = True
            if 'owner_id' in body:
                self._Configs.set("core", "owner_id", body['owner_id'])

            return self.configHandler.connected()

    def amqp_incoming_parse(self, channel, deliver, properties, msg):
        """
        :param deliver:
        :param properties:
        :param msg:
        :param queue:
        :return:
        """
        # print("amqp_incoming_parse............")

        if not hasattr(properties, 'user_id') or properties.user_id is None:
            self._Statistics.increment(
                "lib.amqpyombo.received.discarded.nouserid",
                bucket_size=15,
                anon=True)
            raise YomboWarning("user_id missing.")
        if not hasattr(properties,
                       'content_type') or properties.content_type is None:
            self._Statistics.increment(
                "lib.amqpyombo.received.discarded.content_type_missing",
                bucket_size=15,
                anon=True)
            raise YomboWarning("content_type missing.")
        if not hasattr(
                properties,
                'content_encoding') or properties.content_encoding is None:
            self._Statistics.increment(
                "lib.amqpyombo.received.discarded.content_encoding_missing",
                bucket_size=15,
                anon=True)
            raise YomboWarning("content_encoding missing.")
        if properties.content_encoding != 'text' and properties.content_encoding != 'zlib':
            self._Statistics.increment(
                "lib.amqpyombo.received.discarded.content_encoding_invalid",
                bucket_size=15,
                anon=True)
            raise YomboWarning(
                "Content Encoding must be either  'text' or 'zlib'. Got: " +
                properties.content_encoding)
        if properties.content_type != 'text/plain' and properties.content_type != 'application/msgpack' and properties.content_type != 'application/json':
            self._Statistics.increment(
                "lib.amqpyombo.received.discarded.content_type_invalid",
                bucket_size=15,
                anon=True)
            logger.warn('Error with contentType!')
            raise YomboWarning(
                "Content type must be 'application/msgpack', 'application/json' or 'text/plain'. Got: "
                + properties.content_type)

        received_message_meta = {}
        received_message_meta['content_encoding'] = properties.content_encoding
        received_message_meta['content_type'] = properties.content_type
        if properties.content_encoding == 'zlib':
            compressed_size = len(msg)
            msg = zlib.decompress(msg)
            uncompressed_size = len(msg)
            # logger.info(
            #     "Message sizes: msg_size_compressed = {compressed}, non-compressed = {uncompressed}, percent: {percent}",
            #     compressed=beforeZlib, uncompressed=afterZlib, percent=abs(percentage(beforeZlib, afterZlib)-1))
            received_message_meta['payload_size'] = uncompressed_size
            received_message_meta['compressed_size'] = compressed_size
            received_message_meta['compression_percent'] = abs(
                (compressed_size / uncompressed_size) - 1) * 100
        else:
            received_message_meta['payload_size'] = len(msg)
            received_message_meta['compressed_size'] = len(msg)
            received_message_meta['compression_percent'] = None

        if properties.content_type == 'application/json':
            if self._Validate.is_json(msg):
                msg = bytes_to_unicode(json.loads(msg))
            else:
                raise YomboWarning("Receive msg reported json, but isn't: %s" %
                                   msg)
        elif properties.content_type == 'application/msgpack':
            if self._Validate.is_msgpack(msg):
                msg = bytes_to_unicode(msgpack.loads(msg))
                # print("msg: %s" % type(msg))
                # print("msg: %s" % msg['headers'])
            else:
                raise YomboWarning(
                    "Received msg reported msgpack, but isn't: %s" % msg)

        # todo: Validate signatures/encryption here!
        return {
            'headers': msg['headers'],
            'body': msg['body'],
            'received_message_meta': received_message_meta,
        }

    @inlineCallbacks
    def amqp_incoming(self,
                      body=None,
                      properties=None,
                      headers=None,
                      deliver=None,
                      correlation_info=None,
                      received_message_meta=None,
                      sent_message_meta=None,
                      subscription_callback=None,
                      **kwargs):
        # if subscription_callback is None:
        #     logger.warn("Received invalid message, no subscription_callback!")
        #     return
        # arguments = {
        #     'body': body,
        #     'properties': properties,
        #     'headers': headers,
        #     'deliver': deliver,
        #     'correlation_info': correlation_info,
        #     'received_message_meta': received_message_meta,
        #     'sent_message_meta': sent_message_meta,
        # }
        # print("amqp_incoming................")
        # print(headers)
        ## Valiate that we have the required headers
        if 'message_type' not in headers:
            raise YomboWarning(
                "Discarding request message, header 'message_type' is missing."
            )
        if 'gateway_routing' not in headers:
            raise YomboWarning(
                "Discarding request message, header 'gateway_routing' is missing."
            )
        if headers['message_type'] == 'request':
            if 'request_type' not in headers:
                raise YomboWarning(
                    "Discarding request message, header 'request_type' is missing."
                )
        if headers['message_type'] == 'response':
            if 'response_type' not in headers:
                raise YomboWarning(
                    "Discarding request message, header 'response_type' is missing."
                )

        # Now, route the message. If it's a yombo message, send it to your AQMPYombo for delivery
        if correlation_info is not None and correlation_info['callback'] is not None and \
                        isinstance(correlation_info['callback'], collections.Callable) is True:
            # print("amqp_incoming................ routing to correlation callback")
            logger.debug(
                "calling message callback, not incoming queue callback")
            sent = maybeDeferred(
                correlation_info['callback'],
                body=body,
                properties=properties,
                headers=headers,
                deliver=deliver,
                correlation_info=correlation_info,
                received_message_meta=received_message_meta,
                sent_message_meta=sent_message_meta,
                subscription_callback=subscription_callback,
            )
            yield sent

        # Lastly, send it to the callback defined by the hooks returned.
        else:
            # print("amqp_incoming..correlation info: %s" % correlation_info)
            routing = headers['gateway_routing']
            # print("amqp_incoming..routing to amqpyombo_options callback: %s" % routing)
            # print("amqp_incoming...details: %s" % self.amqpyombo_options['routing'])
            if routing in self.amqpyombo_options['routing']:
                for the_callback in self.amqpyombo_options['routing'][routing]:
                    # print("about to call callback: %s" % the_callback)
                    sent = maybeDeferred(
                        the_callback,
                        deliver=deliver,
                        properties=properties,
                        headers=headers,
                        body=body,
                        correlation_info=correlation_info,
                        received_message_meta=received_message_meta,
                        sent_message_meta=sent_message_meta,
                    )
                    # d.callback(1)
                    yield sent

    def generate_message_response(self,
                                  exchange_name=None,
                                  source=None,
                                  destination=None,
                                  headers=None,
                                  body=None,
                                  routing_key=None,
                                  callback=None,
                                  correlation_id=None,
                                  message_type=None,
                                  response_type=None,
                                  data_type=None,
                                  previous_properties=None,
                                  previous_headers=None):
        if self._Loader.operating_mode != 'run':
            return
        if previous_properties is None:
            raise YomboWarning(
                "generate_message_response() requires 'previous_properties' argument."
            )
        if previous_headers is None:
            raise YomboWarning(
                "generate_message_response() requires 'previous_headers' argument."
            )

        if message_type is None:
            message_type = "response"

        reply_to = None
        if 'correlation_id' in previous_headers and previous_headers['correlation_id'] is not None and \
            previous_headers['correlation_id'][0:2] != "xx_":
            reply_to = previous_headers['correlation_id']
            if headers is None:
                headers = {}
            headers['reply_to'] = reply_to

        response_msg = self.generate_message(
            exchange_name=exchange_name,
            source=source,
            destination=destination,
            message_type=message_type,
            data_type=data_type,
            body=body,
            routing_key=routing_key,
            callback=callback,
            correlation_id=correlation_id,
            headers=headers,
        )
        if response_type is not None:
            response_msg['body']['headers']['response_type'] = response_type

        return response_msg

    def generate_message_request(self,
                                 exchange_name=None,
                                 source=None,
                                 destination=None,
                                 headers=None,
                                 body=None,
                                 routing_key=None,
                                 callback=None,
                                 correlation_id=None,
                                 message_type=None,
                                 request_type=None,
                                 data_type=None):
        if self._Loader.operating_mode != 'run':
            return

        if message_type is None:
            message_type = "request"

        request_msg = self.generate_message(
            exchange_name=exchange_name,
            source=source,
            destination=destination,
            message_type=message_type,
            data_type=data_type,
            body=body,
            routing_key=routing_key,
            callback=callback,
            correlation_id=correlation_id,
            headers=headers,
        )
        if request_type is not None:
            request_msg['body']['headers']['request_type'] = request_type

        return request_msg

    def generate_message(self,
                         exchange_name,
                         source,
                         destination,
                         message_type,
                         data_type=None,
                         body=None,
                         routing_key=None,
                         callback=None,
                         correlation_id=None,
                         headers=None,
                         reply_to=None):
        """
        When interacting with Yombo AMQP servers, we use a standard messaging layout. The format
        below helps other functions and libraries conform to this standard.

        This only creates the message, it doesn't send it. Use the publish() function to complete that.

        **Usage**:

        .. code-block:: python

           requestData = {
               "exchange_name" : "gw_config",
               "source"        : "yombo.gateway.lib.configurationupdate",
               "destination"   : "yombo.server.configs",
               "callback"      : self.amqp_direct_incoming,
               "data_type      : "object",
               "body"          : payload_content,  # Usually a dictionary.
               },
           }
           request = self.AMQPYombo.generateRequest(**requestData)

        :param exchange_name: The exchange the request should go to.
        :type exchange_name: str
        :param source: Value for the 'source' field.
        :type source: str
        :param destination: Value of the 'destination' field.
        :type destination: str
        :param message_type: Type of header. Usually one of: request, response
        :type message_type: str
        :param data_type: One of: Object, Objects, or String
        :type data_type: string
        :param body: The part that will become the body, or payload, of the message.
        :type body: str, dict, list
        :param routing_key: Routing key to use for message delivery. Usually '*'.
        :type routing_key: str
        :param callback: A pointer to the function to return results to. This function will receive 4 arguments:
          sendInfo (Dict) - Various details of the sent packet. deliver (Dict) - Deliver fields as returned by Pika.
          props (Pika Object) - Message properties, includes headers. msg (dict) - The actual content of the message.
        :type callback: function
        :param correlation_id: A correlation_id to use.
        :type correlation_id: string
        :param headers: Extra headers. Note: these cannot be validated with GPG/PGP keys.
        :type headers: dict

        :return: A dictionary that can be directly returned to Yombo SErvers via AMQP
        :rtype: dict
        """
        if self._Loader.operating_mode != 'run':
            return

        # print("body: %s" % exchange_name)
        # print("body: %s" % body)
        if routing_key is None:
            routing_key = '*'

        if body is None:
            body = {}

        if correlation_id is None:
            correlation_id = random_string(length=24)

        if data_type is None:
            if isinstance(body, list):
                data_type = 'objects'
            elif isinstance(body, str):
                data_type = 'string'
            else:
                data_type = 'object'

        msg_created_at = float(time())
        request_msg = {
            "exchange_name": exchange_name,
            "routing_key": routing_key,
            "body": {
                "headers": {
                    "source": source + ":" + self.user_id,
                    "destination": destination,
                    "message_type": message_type.lower(),
                    "yombo_msg_protocol_verion": PROTOCOL_VERSION,
                    "correlation_id": correlation_id,
                    "msg_created_at": msg_created_at,
                    "data_type": data_type.lower(),
                },
                "body": body,
            },
            "properties": {
                "user_id":
                self.user_id,  # system id is required to be able to send it.
                "content_type": 'application/msgpack',
                "content_encoding": None,
                "headers": {
                    "yombo_msg_protocol_verion": PROTOCOL_VERSION,
                    "route": "yombo.gw.amqpyombo:" + self.user_id,
                    "body_signature": "",
                },
            },
            "meta": {
                'finalized_for_sending': False,
                'msg_created_at': msg_created_at,
            },
        }

        if isinstance(headers, dict):
            request_msg['body']['headers'].update(headers)

        if callback is not None:
            request_msg['callback'] = callback
            if correlation_id is None:
                request_msg['body']['headers'][
                    'correlation_id'] = random_string(length=24)

        if reply_to is not None:
            request_msg['body']['headers']['reply_to'] = reply_to

        return request_msg

    def finalize_message(self, message):
        if 'correlation_id' in message['body']['headers']:
            message['properties']['correlation_id'] = message['body'][
                'headers']['correlation_id']
        if 'reply_to' in message['body']['headers']:
            message['properties']['reply_to'] = message['body']['headers'][
                'reply_to']
        # print("finalize message: %s" % message)
        message['body'] = msgpack.dumps(message['body'])

        # Lets test if we can compress. Set headers as needed.
        if len(message['body']) > 800:
            beforeZlib = len(message['body'])
            message['body'] = zlib.compress(
                message['body'], 5
            )  # 5 appears to be the best speed/compression ratio - MSchwenk
            afterZlib = len(message['body'])
            message['meta']['compression_percent'] = percentage(
                afterZlib, beforeZlib)

            message['properties']['content_encoding'] = "zlib"
        else:
            message['properties']['content_encoding'] = 'text'
        # request_msg['meta']['content_encoding'] = request_msg['properties']['content_encoding']
        # request_msg['meta']['payload_size'] = len(request_msg['body'])
        message['meta']['finalized_for_sending'] = True
        return message

    def publish(self, **kwargs):
        """
        Publishes a message. Use generate_message(), generate_message_request, or generate_message_response to
        create the message.
        :return:
        """
        logger.debug("about to publish: {kwargs}", kwargs=kwargs)
        if kwargs['meta']['finalized_for_sending'] is False:
            kwargs = self.finalize_message(kwargs)
        if "callback" in kwargs:
            # callback = kwargs['callback']
            # del kwargs['callback']
            if 'correlation_id' not in kwargs['properties']:
                kwargs['properties']['correlation_id'] = random_string()

        kwargs['yomboprotocol'] = True
        self.amqp.publish(**kwargs)

    def process_system(self, msg, properties):
        pass

    def _local_log(self, level, location, msg=""):
        logit = func = getattr(logger, level)
        logit("In {location} : {msg}", location=location, msg=msg)
예제 #5
0
class AMQPYombo(YomboLibrary):
    """
    Handles interactions with Yombo servers through the AMQP library.
    """
    def _init_(self, **kwargs):
        """
        Loads various variables and calls :py:meth:connect() when it's ready.

        :return:
        """
        self.gateway_id = "gw_" + self._Configs.get('core', 'gwid', 'local',
                                                    False)
        self.login_gwuuid = self.gateway_id + "_" + self._Configs.get(
            "core", "gwuuid")
        self._LocalDBLibrary = self._Libraries['localdb']
        self.request_configs = False

        self.amqp = None  # holds our pointer for out amqp connection.
        self._getAllConfigsLoggerLoop = None
        self.send_local_information_loop = None  # used to periodically send yombo servers updated information

        self.controlHandler = AmqpControlHandler(self)
        self.configHandler = AmqpConfigHandler(self)

        self._States.set('amqp.amqpyombo.state', False)
        self.init_deferred = Deferred()
        self.connect()
        return self.init_deferred

    def _stop_(self, **kwargs):
        """
        Called by the Yombo system when it's time to shutdown. This in turn calls the disconnect.
        :return:
        """

        if self.init_deferred is not None and self.init_deferred.called is False:
            self.init_deferred.callback(
                1)  # if we don't check for this, we can't stop!

        self.configHandler._stop_()
        self.controlHandler._stop_()
        self.disconnect(
        )  # will be cleaned up by amqp library anyways, but it's good to be nice.

    def _unload_(self, **kwargs):
        self.amqp.disconnect()

    @inlineCallbacks
    def connect(self):
        """
        Connect to Yombo amqp server.

        :return:
        """
        if self.amqp is None:
            already_have_amqp = None
        else:
            already_have_amqp = True

        environment = self._Configs.get('server', 'environment', "production",
                                        False)
        if self._Configs.get("amqpyombo", 'hostname', "", False) != "":
            amqp_host = self._Configs.get("amqpyombo", 'hostname')
            amqp_port = self._Configs.get("amqpyombo", 'port', 5671, False)
        else:
            amqp_port = 5671
            if environment == "production":
                amqp_host = "amqp.yombo.net"
            elif environment == "staging":
                amqp_host = "amqpstg.yombo.net"
            elif environment == "development":
                amqp_host = "amqpdev.yombo.net"
            else:
                amqp_host = "amqp.yombo.net"

        # get a new amqp instance and connect.
        if self.amqp is None:
            self.amqp = yield self._AMQP.new(
                hostname=amqp_host,
                port=amqp_port,
                virtual_host='yombo',
                username=self.login_gwuuid,
                password=self._Configs.get("core", "gwhash"),
                client_id='amqpyombo',
                prefetch_count=PREFETCH_COUNT,
                connected_callback=self.amqp_connected,
                disconnected_callback=self.amqp_disconnected,
                critical=True)
        self.amqp.connect()

        # The servers will have a dedicated queue for us. All pending messages will be held there for us. If we
        # connect to a different server, they wil automagically be re-routed to our new queue.
        if already_have_amqp is None:
            self.amqp.subscribe("ygw.q." + self.gateway_id,
                                incoming_callback=self.amqp_incoming,
                                queue_no_ack=False,
                                persistent=True)

        self.configHandler.connect_setup(self.init_deferred)
        # self.init_deferred.callback(10)

    def disconnect(self):
        """
        Called by the yombo system when it's time to shutdown.

        :return:
        """
        body = {}

        requestmsg = self.generate_message_request(
            exchange_name='ysrv.e.gw_config',
            source='yombo.gateway.lib.amqpyombo',
            destination='yombo.server.configs',
            body=body,
            headers={
                "request_type": "disconnecting",
            })
        self.amqp.publish(**requestmsg)

        logger.debug("Disconnected from Yombo message server.")

    def amqp_connected(self):
        """
        Called by AQMP when connected. This function was define above when setting up self.ampq.

        :return:
        """
        self._States.set('amqp.amqpyombo.state', True)

        if self.send_local_information_loop is None:
            self.send_local_information_loop = LoopingCall(
                self.send_local_information)

        # Sends various information, helps Yombo cloud know we are alive and where to find us.
        if self.send_local_information_loop.running is False:
            self.send_local_information_loop.start(random_int(60 * 60 * 4, .2))

    def amqp_disconnected(self):
        """
        Called by AQMP when disconnected.
        :return:
        """
        # logger.info("amqpyombo yombo disconnected: {state}", state=self._States.get('amqp.amqpyombo.state'))
        # If we have at least connected once, then we don't toss errors.  We just wait for reconnection...

        if self._States.get('amqp.amqpyombo.state') is False:
            logger.error(
                "Unable to connect. This may be due to multiple connections or bad gateway hash. See: http://g2.yombo.net/noconnect"
            )
            self._Loader.sigint = True
            self.amqp.disconnect()
            reactor.stop()
            return

        self._States.set('amqp.amqpyombo.state', False)
        if self.send_local_information_loop is not None and self.send_local_information_loop.running:
            self.send_local_information_loop.stop()

    def send_local_information(self):
        """
        Say hello, send some information about us. What we use these IP addresses for:

        Devices in your home can connect directly to the gateway for faster access. We give this information
        to your clients so they can find the gateway easier/faster.

        Your external IP is also given to your clients so they can attempt to connect directly to the gateway when
        you are not at home.

        If either of these conenctions are not available, applications can use the Yombo servers as proxy and will
        be delivered here as either a control or status request message.

        :return:
        """
        body = {
            "is_master":
            self._Configs.get("core", "is_master", True, False),
            "master_gateway":
            self._Configs.get("core", "master_gateway", "", False),
            "internal_ipv4":
            self._Configs.get("core", "localipaddress_v4"),
            "external_ipv4":
            self._Configs.get("core", "externalipaddress_v4"),
            # "internal_ipv6": self._Configs.get("core", "externalipaddress_v6"),
            # "external_ipv6": self._Configs.get("core", "externalipaddress_v6"),
            "internal_port":
            self._Configs.get("webinterface", "nonsecure_port"),
            "external_port":
            self._Configs.get("webinterface", "nonsecure_port"),
            "internal_secure_port":
            self._Configs.get("webinterface", "secure_port"),
            "external_secure_port":
            self._Configs.get("webinterface", "secure_port"),
            "internal_mqtt":
            self._Configs.get("mqtt", "server_listen_port"),
            "internal_mqtt_le":
            self._Configs.get("mqtt", "server_listen_port_le_ssl"),
            "internal_mqtt_ss":
            self._Configs.get("mqtt", "server_listen_port_ss_ssl"),
            "internal_mqtt_ws":
            self._Configs.get("mqtt", "server_listen_port_websockets"),
            "internal_mqtt_ws_le":
            self._Configs.get("mqtt", "server_listen_port_websockets_le_ssl"),
            "internal_mqtt_ws_ss":
            self._Configs.get("mqtt", "server_listen_port_websockets_ss_ssl"),
            "external_mqtt":
            self._Configs.get("mqtt", "server_listen_port"),
            "external_mqtt_le":
            self._Configs.get("mqtt", "server_listen_port_le_ssl"),
            "external_mqtt_ss":
            self._Configs.get("mqtt", "server_listen_port_ss_ssl"),
            "external_mqtt_ws":
            self._Configs.get("mqtt", "server_listen_port_websockets"),
            "external_mqtt_ws_le":
            self._Configs.get("mqtt", "server_listen_port_websockets_le_ssl"),
            "external_mqtt_ws_ss":
            self._Configs.get("mqtt", "server_listen_port_websockets_ss_ssl"),
        }

        # logger.debug("sending local information.")

        requestmsg = self.generate_message_request(
            exchange_name='ysrv.e.gw_config',
            source='yombo.gateway.lib.amqpyombo',
            destination='yombo.server.configs',
            body=body,
            callback=self.receive_local_information,
            headers={
                "request_type": "gatewayInfo",
            })
        requestmsg['properties']['headers']['response_type'] = 'system'
        self.amqp.publish(**requestmsg)

    def receive_local_information(self,
                                  msg=None,
                                  properties=None,
                                  correlation_info=None,
                                  send_message_info=None,
                                  receied_message_info=None,
                                  **kwargs):
        if self.request_configs is False:  # this is where we start requesting information - after we have sent out info.
            self.request_configs = True
            return self.configHandler.connected()

    def generate_message_response(self, properties, exchange_name, source,
                                  destination, headers, body):
        response_msg = self.generate_message(exchange_name, source,
                                             destination, "response", headers,
                                             body)

        if hasattr('correlation_id',
                   properties) and properties.correlation_id is not None:
            response_msg['properties'][
                'correlation_id'] = properties.correlation_id
        if hasattr('message_id', properties) and properties.message_id is not None and \
                properties.message_id[0:2] != "xx_":
            response_msg['properties']['reply_to'] = properties.correlation_id

        # print "properties: %s" % properties
        if 'route' in properties.headers:
            route = str(properties.headers['route']
                        ) + ",yombo.gw.amqpyombo:" + self.gateway_id
            response_msg['properties']['headers']['route'] = route
        else:
            response_msg['properties']['headers'][
                'route'] = "yombo.gw.amqpyombo:" + self.gateway_id
        return response_msg

    def generate_message_request(self,
                                 exchange_name=None,
                                 source=None,
                                 destination=None,
                                 headers=None,
                                 body=None,
                                 callback=None,
                                 correlation_id=None,
                                 message_id=None):
        new_body = {
            "data_type": "object",
            "request": body,
        }
        if isinstance(body, list):
            new_body['data_type'] = 'objects'

        request_msg = self.generate_message(exchange_name,
                                            source,
                                            destination,
                                            "request",
                                            headers,
                                            new_body,
                                            callback=callback,
                                            correlation_id=correlation_id,
                                            message_id=message_id)
        return request_msg

    def generate_message(self,
                         exchange_name,
                         source,
                         destination,
                         header_type,
                         headers,
                         body,
                         callback=None,
                         correlation_id=None,
                         message_id=None):
        """
        When interacting with Yombo AMQP servers, we use a standard messaging layout. The below helps other functions
        and libraries conform to this standard.

        This only creates the message, it doesn't send it. Use the publish() function to complete that.

        **Usage**:

        .. code-block:: python

           requestData = {
               "exchange_name"  : "gw_config",
               "source"        : "yombo.gateway.lib.configurationupdate",
               "destination"   : "yombo.server.configs",
               "callback" : self.amqp_direct_incoming,
               "body"          : {
                 "DataType"        : "Object",
                 "Request"         : requestContent,
               },
           }
           request = self.AMQPYombo.generateRequest(**requestData)

        :param exchange_name: The exchange the request should go to.
        :type exchange_name: str
        :param source: Value for the 'source' field.
        :type source: str
        :param destination: Value of the 'destination' field.
        :type destination: str
        :param header_type: Type of header. Usually one of: request, response
        :type header_type: str
        :param headers: Extra headers
        :type headers: dict
        :param body: The part that will become the body, or payload, of the message.
        :type body: str, dict, list
        :param callback: A pointer to the function to return results to. This function will receive 4 arguments:
          sendInfo (Dict) - Various details of the sent packet. deliver (Dict) - Deliver fields as returned by Pika.
          props (Pika Object) - Message properties, includes headers. msg (dict) - The actual content of the message.
        :type callback: function
        :param body: The body contents for the mesage.
        :type body: dict

        :return: A dictionary that can be directly returned to Yombo Gateways via AMQP
        :rtype: dict
        """
        # print("body: %s" % body)
        request_msg = {
            "exchange_name": exchange_name,
            "routing_key": '*',
            "body": msgpack.dumps(body),
            "properties": {
                # "correlation_id" : correlation_id,
                "user_id": self.
                gateway_id,  # system id is required to be able to send it.
                "content_type": 'application/msgpack',
                "headers": {
                    # "requesting_user_id"        : user
                    "source": source + ":" + self.gateway_id,
                    "destination": destination,
                    "type": header_type,
                    "protocol_verion": PROTOCOL_VERSION,
                    "message_id": random_string(length=20),
                    "msg_created_at": str(time()),
                },
            },
            "meta": {
                "content_type": 'application/msgpack',
            },
            "created_at": time(),
        }

        if "callback" is not None:
            request_msg['callback'] = callback
            if correlation_id is None:
                request_msg['properties']['correlation_id'] = random_string(
                    length=24)
            else:
                request_msg['properties']['correlation_id'] = correlation_id

        if message_id is None:
            request_msg['properties']['message_id'] = random_string(length=26)
        else:
            request_msg['properties']['message_id'] = message_id

        # Lets test if we can compress. Set headers as needed.
        if len(request_msg['body']) > 800:
            beforeZlib = len(request_msg['body'])
            request_msg['body'] = zlib.compress(
                request_msg['body'], 5
            )  # 5 appears to be the best speed/compression ratio - MSchwenk
            afterZlib = len(request_msg['body'])
            request_msg['meta']['compression_percent'] = percentage(
                afterZlib, beforeZlib)

            request_msg['properties']['content_encoding'] = "zlib"
        else:
            request_msg['properties']['content_encoding'] = 'text'
        request_msg['properties']['headers'].update(headers)
        request_msg['meta']['content_encoding'] = request_msg['properties'][
            'content_encoding']
        request_msg['meta']['payload_size'] = len(request_msg['body'])
        return request_msg

    def publish(self, **kwargs):
        """
        Publishes a message. Use generate_message(), generate_message_request, or generate_message_response to
        create the message.
        :return:
        """
        if 'callback' in kwargs:
            callback = kwargs['callback']
            del kwargs['callback']
            if 'correlation_id' not in kwargs['properties']:
                kwargs['properties']['correlation_id'] = random_string()
        else:
            callback = None

        # print "publish kwargs: %s" % kwargs
        results = self.amqp.publish(**kwargs)
        if results['correlation_info'] is not None:
            results['correlation_info']['amqpyombo_callback'] = callback

    def amqp_incoming(self,
                      msg=None,
                      properties=None,
                      deliver=None,
                      correlation_info=None,
                      received_message_info=None,
                      sent_message_info=None,
                      **kwargs):
        """
        All incoming messages come here. It will be parsed and sorted as needed.  Routing:

        1) Device updates, changes, deletes -> Devices library
        1) Command updates, changes, deletes -> Command library
        1) Module updates, changes, deletes -> Module library
        1) Device updates, changes, deletes -> Devices library
        1) Device updates, changes, deletes -> Devices library

        Summary of tasks:

        1) Validate incoming headers.
        2) Setup ACK/Nack responses.
        3) Route the message to the proper library for final handling.
        """
        # self._local_log("info", "AMQPLibrary::amqp_incoming")
        # print "properties: %s" % properties
        # print "correlation: %s" % correlation

        #        log.msg('%s (%s): %s' % (deliver.exchange, deliver.routing_key, repr(msg)), system='Pika:<=')

        msg_meta = {}
        if properties.user_id is None:
            self._Statistics.increment(
                "lib.amqpyombo.received.discarded.nouserid",
                bucket_size=15,
                anon=True)
            raise YomboWarning("user_id missing.")
        if properties.content_type is None:
            self._Statistics.increment(
                "lib.amqpyombo.received.discarded.content_type_missing",
                bucket_size=15,
                anon=True)
            raise YomboWarning("content_type missing.")
        if properties.content_encoding is None:
            self._Statistics.increment(
                "lib.amqpyombo.received.discarded.content_encoding_missing",
                bucket_size=15,
                anon=True)
            raise YomboWarning("content_encoding missing.")
        if properties.content_encoding != 'text' and properties.content_encoding != 'zlib':
            self._Statistics.increment(
                "lib.amqpyombo.received.discarded.content_encoding_invalid",
                bucket_size=15,
                anon=True)
            raise YomboWarning(
                "Content Encoding must be either  'text' or 'zlib'. Got: " +
                properties.content_encoding)
        if properties.content_type != 'text/plain' and properties.content_type != 'application/msgpack' and properties.content_type != 'application/json':
            self._Statistics.increment(
                "lib.amqpyombo.received.discarded.content_type_invalid",
                bucket_size=15,
                anon=True)
            logger.warn('Error with contentType!')
            raise YomboWarning(
                "Content type must be 'application/msgpack', 'application/json' or 'text/plain'. Got: "
                + properties.content_type)

        msg_meta['payload_size'] = len(msg)
        if properties.content_encoding == 'zlib':
            compressed_size = len(msg)
            msg = zlib.decompress(msg)
            uncompressed_size = len(msg)
            # logger.info(
            #     "Message sizes: msg_size_compressed = {compressed}, non-compressed = {uncompressed}, percent: {percent}",
            #     compressed=beforeZlib, uncompressed=afterZlib, percent=abs(percentage(beforeZlib, afterZlib)-1))
            received_message_info['uncompressed_size'] = uncompressed_size
            received_message_info['compression_percent'] = abs(
                (compressed_size / uncompressed_size) - 1) * 100

        if properties.content_type == 'application/json':
            try:
                msg = bytes_to_unicode(json.loads(msg))
            except Exception:
                raise YomboWarning("Receive msg reported json, but isn't: %s" %
                                   msg)
        elif properties.content_type == 'application/msgpack':
            try:
                msg = bytes_to_unicode(msgpack.loads(msg))
            except Exception:
                raise YomboWarning(
                    "Received msg reported msgpack, but isn't: %s" % msg)

        # if a response, lets make sure it's something we asked for!
        elif properties.headers['type'] == "response":
            # print "send_correlation_ids: %s" % self.amqp.send_correlation_ids
            if properties.correlation_id not in self.amqp.send_correlation_ids:
                self._Statistics.increment(
                    "lib.amqpyombo.received.discarded.correlation_id_missing",
                    bucket_size=15,
                    anon=True)
                raise YomboWarning("correlation_id missing.")

            if sent_message_info is not None and sent_message_info[
                    'sent_at'] is not None:
                delay_date_at = received_message_info[
                    'received_at'] - sent_message_info['sent_at']
                milliseconds = (delay_date_at.days * 24 * 60 * 60 +
                                delay_date_at.seconds
                                ) * 1000 + delay_date_at.microseconds / 1000.0
                logger.debug(
                    "Time between sending and receiving a response:: {milliseconds}",
                    milliseconds=milliseconds)
                received_message_info['round_trip_timing'] = milliseconds

            if properties.correlation_id is None or not isinstance(
                    properties.correlation_id, six.string_types):
                self._Statistics.increment(
                    "lib.amqpyombo.received.discarded.correlation_id_invalid",
                    bucket_size=15,
                    anon=True)
                raise YomboWarning(
                    "Correlation_id must be present for 'Response' types, and must be a string."
                )
            if properties.correlation_id not in self.amqp.send_correlation_ids:
                logger.debug(
                    "{correlation_id} not in list of ids: {send_correlation_ids} ",
                    correlation_id=properties.correlation_id,
                    send_correlation_ids=list(
                        self.amqp.send_correlation_ids.keys()))
                self._Statistics.increment(
                    "lib.amqpyombo.received.discarded.nocorrelation",
                    bucket_size=15,
                    anon=True)
                raise YomboWarning(
                    "Received request %s, but never asked for it. Discarding" %
                    properties.correlation_id)
        else:
            self._Statistics.increment(
                "lib.amqpyombo.received.discarded.unknown_msg_type",
                bucket_size=15,
                anon=True)
            raise YomboWarning("Unknown message type received.")

        # self._local_log("debug", "PikaProtocol::receive_item4")

        # if we are here.. we have a valid message....
        if correlation_info is not None and 'amqpyombo_callback' in correlation_info and \
                correlation_info['amqpyombo_callback'] and isinstance(correlation_info['amqpyombo_callback'], collections.Callable):
            correlation_info['amqpyombo_callback'](
                msg=msg,
                properties=properties,
                deliver=deliver,
                correlation_info=correlation_info,
                **kwargs)
        received_message_info['meta'] = msg_meta

        if properties.headers['type'] == 'request':
            try:
                logger.debug("headers: {headers}", headers=properties.headers)
                if properties.headers['request_type'] == 'control':
                    self.controlHandler.process_control(msg, properties)
                elif properties.headers['request_type'] == 'system':
                    self.process_system_request(
                        msg=msg,
                        properties=properties,
                        deliver=deliver,
                        correlation_info=correlation_info,
                        sent_message_info=sent_message_info,
                        received_message_info=received_message_info,
                        **kwargs)

            except Exception as e:
                logger.error(
                    "--------==(Error: in response processing     )==--------")
                logger.error(
                    "--------------------------------------------------------")
                logger.error("{error}", error=sys.exc_info())
                logger.error(
                    "---------------==(Traceback)==--------------------------")
                logger.error("{trace}",
                             trace=traceback.print_exc(file=sys.stdout))
                logger.error(
                    "--------------------------------------------------------")
        elif properties.headers['type'] == 'response':
            try:
                logger.debug("headers: {headers}", headers=properties.headers)
                if properties.headers['response_type'] == 'config':
                    self.configHandler.process_config_response(
                        msg=msg,
                        properties=properties,
                        deliver=deliver,
                        correlation_info=correlation_info,
                        sent_message_info=sent_message_info,
                        received_message_info=received_message_info,
                        **kwargs)
                elif properties.headers['response_type'] == 'sslcert':
                    self._SSLCerts.amqp_incoming(
                        msg=msg,
                        properties=properties,
                        deliver=deliver,
                        correlation_info=correlation_info,
                        sent_message_info=sent_message_info,
                        received_message_info=received_message_info,
                        **kwargs)

            except Exception as e:
                logger.error(
                    "--------==(Error: in response processing     )==--------")
                logger.error(
                    "--------------------------------------------------------")
                logger.error("{error}", error=sys.exc_info())
                logger.error(
                    "---------------==(Traceback)==--------------------------")
                logger.error("{trace}",
                             trace=traceback.print_exc(file=sys.stdout))
                logger.error(
                    "--------------------------------------------------------")

    def process_system(self, msg, properties):
        pass

    def _local_log(self, level, location, msg=""):
        logit = func = getattr(logger, level)
        logit("In {location} : {msg}", location=location, msg=msg)
예제 #6
0
class AMQPYombo(YomboLibrary):
    """
    Handles interactions with Yombo servers through the AMQP library.
    """

    def _init_(self):
        """
        Loads various variables and calls :py:meth:connect() when it's ready.

        :return:
        """
        self.user_id = "gw_" + self._Configs.get("core", "gwid")
        self.login_user_id = self.user_id + "_" + self._Configs.get("core", "gwuuid")
        self.__pending_updates = []  # Holds a list of configuration items we've asked for, but not response yet.
        self._LocalDBLibrary = self._Libraries['localdb']
        self.init_startup_count = 0

        self.amqp = None  # holds our pointer for out amqp connection.
        self._getAllConfigsLoggerLoop = None

        self._getAllConfigsLoggerLoop = None
        self.reconnect = True  # If we get disconnected, we should reconnect to the server.
        self.sendLocalInformationLoop = None  # used to periodically send yombo servers updated information

        self.controlHandler = AmqpControlHandler(self)
        self.configHandler = AmqpConfigHandler(self)
        return self.connect()

    def _stop_(self):
        """
        Called by the Yombo system when it's time to shutdown. This in turn calls the disconnect.
        :return:
        """
        self.disconnect()  # will be cleaned up by amqp library anyways, but it's good to be nice.
        self.configHandler._stop_()

    def connect(self):
        """
        Connect to Yombo AMQP server.

        :return:
        """
        environment = self._Configs.get('server', 'environment', "production", False)
        if self._Configs.get("amqpyombo", 'hostname', "", False) != "":
            amqp_host = self._Configs.get("amqpyombo", 'hostname')
            amqp_port = self._Configs.get("amqpyombo", 'port', 5671, False)
        else:
            amqp_port = 5671
            if environment == "production":
                amqp_host = "amqp.yombo.net"
            elif environment == "staging":
                amqp_host = "amqpstg.yombo.net"
            elif environment == "development":
                amqp_host = "amqpdev.yombo.net"
            else:
                amqp_host = "amqp.yombo.net"

        # get a new AMPQ instance and connect.
        if self.amqp is None:
            self.amqp = self._AMQP.new(hostname=amqp_host, port=amqp_port, virtual_host='yombo', username=self.login_user_id,
                password=self._Configs.get("core", "gwhash"), client_id='amqpyombo',
                connected_callback=self.amqp_connected, disconnected_callback=self.amqp_disconnected)
        self.amqp.connect()

        # The servers will have a dedicated queue for us. All pending messages will be held there for us. If we
        # connect to a different server, they wil automagically be re-routed to our new queue.
        self.amqp.subscribe("ygw.q." + self.user_id, incoming_callback=self.amqp_incoming, queue_no_ack=False, persistent=True)

        return self.configHandler.connected()

    def disconnect(self):
        """
        Called by the yombo system when it's
        :return:
        """
        body = {
        }

        requestmsg = self.generate_message_request(
            exchange_name='ysrv.e.gw_config',
            source='yombo.gateway.lib.amqpyombo',
            destination='yombo.server.configs',
            body=body,
            headers={
                "request_type": "disconnecting",
            }
        )
        self.amqp.publish(**requestmsg)

        self.reconnect = False
        logger.debug("Disconnected from Yombo message server.")

    def amqp_connected(self):
        """
        Called by AQMP when connected. This function was define above when setting up self.ampq.

        :return:
        """
        self.sendLocalInformationLoop = LoopingCall(self.sendLocalInformation)
        self.sendLocalInformationLoop.start(60*60*4)  # Sends various information, helps Yombo cloud know we are alive and where to find us.

        self._States.set('amqp.amqpyombo.state', True)

    def amqp_disconnected(self):
        """
        Called by AQMP when disconnected.
        :return:
        """
        self._States.set('amqp.amqpyombo.state', False)
        if self.sendLocalInformationLoop is not None and self.sendLocalInformationLoop.running:
            self.sendLocalInformationLoop.stop()

        if self.reconnect is True:
            self.connect()

    def sendLocalInformation(self):
        """
        Say hello, send some information about us. What we use these IP addresses for:

        Devices in your home can connect directly to the gateway for faster access. We give this information
        to your clients so they can find the gateway easier/faster.

        Your external IP is also given to your clients so they can attempt to connect directly to the gateway when
        you are not at home.

        If either of these conenctions are not available, applications can use the Yombo servers as proxy and will
        be delivered here as either a control or status request message.

        :return:
        """
        body = {
            "local_ip_address": self._Configs.get("core", "localipaddress"),
            "external_ip_address": self._Configs.get("core", "externalipaddress"),
        }

        requestmsg = self.generate_message_request(
            exchange_name='ysrv.e.gw_config',
            source='yombo.gateway.lib.amqpyombo',
            destination='yombo.server.configs',
            body=body,
            headers={
                "request_type": "gatewayInfo",
            }
        )
        requestmsg['properties']['headers']['response_type']='system'
        self.amqp.publish(**requestmsg)

    def generate_message_response(self, properties, exchange_name, source, destination, headers, body ):
        response_msg = self.generate_message(exchange_name, source, destination, "response", headers, body)
        if properties.correlation_id:
           response_msg['properties']['correlation_id'] = properties.correlation_id
#        response_msg['properties']['headers']['response_type']=response_type
        correlation_id = random_string(length=12)

        print "properties: %s" % properties
        if 'route' in properties.headers:
            route = str(properties.headers['route']) + ",yombo.gw.amqpyombo:" + self.user_id
            response_msg['properties']['headers']['route'] = route
        else:
            response_msg['properties']['headers']['route'] = "yombo.gw.amqpyombo:" + self.user_id
        return response_msg

    def generate_message_request(self, exchange_name, source, destination, headers, body, callback=None):
        new_body = {
            "data_type": "object",
            "request"  : body,
        }
        if isinstance(body, list):
            new_body['data_type'] = 'objects'

        request_msg = self.generate_message(exchange_name, source, destination, "request",
                                            headers, new_body, callback=callback)
        request_msg['properties']['correlation_id'] = random_string(length=16)
        # request_msg['properties']['headers']['request_type']=request_type
        return request_msg

    def generate_message(self, exchange_name, source, destination, header_type, headers, body, callback=None):
        """
        When interacting with Yombo AMQP servers, we use a standard messaging layout. The below helps other functions
        and libraries conform to this standard.

        This only creates the message, it doesn't send it. Use the publish() function to complete that.

        **Usage**:

        .. code-block:: python

           requestData = {
               "exchange_name"  : "gw_config",
               "source"        : "yombo.gateway.lib.configurationupdate",
               "destination"   : "yombo.server.configs",
               "callback" : self.amqp_direct_incoming,
               "body"          : {
                 "DataType"        : "Object",
                 "Request"         : requestContent,
               },
           }
           request = self.AMQPYombo.generateRequest(**requestData)

        :param exchange_name: The exchange the request should go to.
        :type exchange_name: str
        :param source: Value for the 'source' field.
        :type source: str
        :param destination: Value of the 'destination' field.
        :type destination: str
        :param callback: A pointer to the function to return results to. This function will receive 4 arguments:
          sendInfo (Dict) - Various details of the sent packet. deliver (Dict) - Deliver fields as returned by Pika.
          props (Pika Object) - Message properties, includes headers. msg (dict) - The actual content of the message.
        :type callback: function
        :param body: The body contents for the mesage.
        :type body: dict

        :return: A dictionary that can be directly returned to Yombo Gateways via AMQP
        :rtype: dict
        """
        request_msg = {
            "exchange_name"    : exchange_name,
            "routing_key"      : '*',
            "body"             : msgpack.dumps(body),
            "properties" : {
                # "correlation_id" : correlation_id,
                "user_id"        : self.user_id,
                "content_type"   : 'application/msgpack',
                "headers"        : {
                    "source"        : source + ":" + self.user_id,
                    "destination"   : destination,
                    "type"          : header_type,
                    "protocol_verion": PROTOCOL_VERSION,
                    },
                },
            "callback": callback,
            }

        # Lets test if we can compress. Set headers as needed.

        self._Statistics.averages("lib.amqpyombo.sent.size", len(request_msg['body']), bucket_time=15, anon=True)
        if len(request_msg['body']) > 800:
            beforeZlib = len(request_msg['body'])
            request_msg['body'] = zlib.compress(request_msg['body'], 5)  # 5 appears to be the best speed/compression ratio - MSchwenk
            request_msg['properties']['content_encoding'] = "zlib"
            afterZlib = len(request_msg['body'])
            self._Statistics.increment("lib.amqpyombo.sent.compressed", bucket_time=15, anon=True)
            self._Statistics.averages("lib.amqpyombo.sent.compressed.percentage", percentage(afterZlib, beforeZlib), anon=True)
        else:
            request_msg['properties']['content_encoding'] = 'text'
            self._Statistics.increment("lib.amqpyombo.sent.uncompressed", bucket_time=15, anon=True)
        request_msg['properties']['headers'].update(headers)

        return request_msg


    def amqp_incoming(self, deliver, properties, msg, queue):
        """
        All incoming messages come here. It will be parsed and sorted as needed.  Routing:

        1) Device updates, changes, deletes -> Devices library
        1) Command updates, changes, deletes -> Command library
        1) Module updates, changes, deletes -> Module library
        1) Device updates, changes, deletes -> Devices library
        1) Device updates, changes, deletes -> Devices library

        Summary of tasks:

        1) Validate incoming headers.
        2) Setup ACK/Nack responses.
        3) Route the message to the proper library for final handling.
        """
        # self._local_log("info", "AMQPLibrary::amqp_incoming")
        # print " !!!!!!!!!!!!!!!!!!!!!!!!! "
        # print "properties: %s" % properties
#        log.msg('%s (%s): %s' % (deliver.exchange, deliver.routing_key, repr(msg)), system='Pika:<=')

        if properties.user_id is None:
            self._Statistics.increment("lib.amqpyombo.received.discarded.nouserid", bucket_time=15, anon=True)
            raise YomboWarning("user_id missing.")
        if properties.content_type is None:
            self._Statistics.increment("lib.amqpyombo.received.discarded.content_type_missing", bucket_time=15, anon=True)
            raise YomboWarning("content_type missing.")
        if properties.content_encoding is None:
            self._Statistics.increment("lib.amqpyombo.received.discarded.content_encoding_missing", bucket_time=15, anon=True)
            raise YomboWarning("content_encoding missing.")
        if properties.content_encoding != 'text' and properties.content_encoding != 'zlib':
            self._Statistics.increment("lib.amqpyombo.received.discarded.content_encoding_invalid", bucket_time=15, anon=True)
            raise YomboWarning("Content Encoding must be either  'text' or 'zlib'. Got: " + properties.content_encoding)
        if properties.content_type != 'text/plain' and properties.content_type != 'application/msgpack' and  properties.content_type != 'application/json':
            self._Statistics.increment("lib.amqpyombo.received.discarded.content_type_invalid", bucket_time=15, anon=True)
            logger.warn('Error with contentType!')
            raise YomboWarning("Content type must be 'application/msgpack', 'application/json' or 'text/plain'. Got: " + properties.content_type)

        if properties.content_encoding == 'zlib':
            beforeZlib = len(msg)
            msg = zlib.decompress(msg)
            afterZlib = len(msg)
            logger.debug("Message sizes: msg_size_compressed = {compressed}, non-compressed = {uncompressed}, percent: {percent}",
                         compressed=beforeZlib, uncompressed=afterZlib, percent=percentage(beforeZlib, afterZlib))
            self._Statistics.increment("lib.amqpyombo.received.compressed", bucket_time=15, anon=True)
            self._Statistics.averages("lib.amqpyombo.received.compressed.percentage", percentage(beforeZlib, afterZlib), bucket_time=15, anon=True)
        else:
            self._Statistics.increment("lib.amqpyombo.received.uncompressed", bucket_time=15, anon=True)
        self._Statistics.averages("lib.amqpyombo.received.payload.size", len(msg), bucket_time=15, anon=True)

        if properties.content_type == 'application/json':
            if self.is_json(msg):
                msg = json.loads(msg)
            else:
                raise YomboWarning("Receive msg reported json, but isn't.")
        elif properties.content_type == 'application/msgpack':
            if self.is_msgpack(msg):
                msg = msgpack.loads(msg)
            else:
                raise YomboWarning("Received msg reported msgpack, but isn't.")

        if properties.headers['type'] == 'request':
            self._Statistics.increment("lib.amqpyombo.received.request", bucket_time=15, anon=True)

        # if a response, lets make sure it's something we asked for!
        elif properties.headers['type'] == "response":
            # print "send_correlation_ids: %s" % self.amqp.send_correlation_ids
            if properties.correlation_id not in self.amqp.send_correlation_ids:
                self._Statistics.increment("lib.amqpyombo.received.discarded.correlation_id_missing", bucket_time=15,
                                           anon=True)
                raise YomboWarning("correlation_id missing.")

            time_info = self.amqp.send_correlation_ids[properties.correlation_id]
            daate_time = time_info['time_received'] - time_info['time_sent']
            milliseconds = (
                           daate_time.days * 24 * 60 * 60 + daate_time.seconds) * 1000 + daate_time.microseconds / 1000.0
            logger.debug("Time between sending and receiving a response:: {milliseconds}", milliseconds=milliseconds)
            self._Statistics.averages("lib.amqpyombo.amqp.response.time", milliseconds, bucket_time=15, anon=True)

            if properties.correlation_id is None or not isinstance(properties.correlation_id, six.string_types):
                self._Statistics.increment("lib.amqpyombo.received.discarded.correlation_id_invalid", bucket_time=15, anon=True)
                raise YomboWarning("Correlation_id must be present for 'Response' types, and must be a string.")
            if properties.correlation_id not in self.amqp.send_correlation_ids:
                logger.debug("{correlation_id} not in list of ids: {send_correlation_ids} ",
                             correlation_id=properties.correlation_id, send_correlation_ids=self.amqp.send_correlation_ids.keys())
                self._Statistics.increment("lib.amqpyombo.received.discarded.nocorrelation", bucket_time=15, anon=True)
                raise YomboWarning("Received request {correlation_id}, but never asked for it. Discarding",
                                   correlation_id=properties.correlation_id)
        else:
            self._Statistics.increment("lib.amqpyombo.received.discarded.unknown_msg_type", bucket_time=15, anon=True)
            raise YomboWarning("Unknown message type recieved.")

        # self._local_log("debug", "PikaProtocol::receive_item4")

        # if we are here.. we have a valid message....

        if properties.headers['type'] == 'request':
            try:
                logger.debug("headers: {headers}", headers=properties.headers)
                if properties.headers['request_type'] == 'control':
                    self.controlHandler.process_control(msg, properties)
                elif properties.headers['request_type'] == 'system':
                    self.process_system_request(msg, properties)

            except Exception, e:
                logger.error("--------==(Error: in response processing     )==--------")
                logger.error("--------------------------------------------------------")
                logger.error("{error}", error=sys.exc_info())
                logger.error("---------------==(Traceback)==--------------------------")
                logger.error("{trace}", trace=traceback.print_exc(file=sys.stdout))
                logger.error("--------------------------------------------------------")
        elif properties.headers['type'] == 'response':
            try:
                logger.debug("headers: {headers}", headers=properties.headers)
                if properties.headers['response_type'] == 'config':
                    self.configHandler.process_config_response(msg, properties)

            except Exception, e:
                logger.error("--------==(Error: in response processing     )==--------")
                logger.error("--------------------------------------------------------")
                logger.error("{error}", error=sys.exc_info())
                logger.error("---------------==(Traceback)==--------------------------")
                logger.error("{trace}", trace=traceback.print_exc(file=sys.stdout))
                logger.error("--------------------------------------------------------")