Beispiel #1
0
def historian(config_path, **kwargs):
    config = utils.load_config(config_path)
    services_topic_list = config.get('services_topic_list', [
        topics.DRIVER_TOPIC_BASE, topics.LOGGER_BASE, topics.ACTUATOR,
        topics.ANALYSIS_TOPIC_BASE
    ])
    custom_topic_list = config.get('custom_topic_list', [])
    topic_replace_list = config.get('topic_replace_list', [])
    destination_vip = config.get('destination-vip')
    destination_historian_identity = config.get(
        'destination-historian-identity', 'platform.historian')
    backup_storage_limit_gb = config.get('backup_storage_limit_gb', None)

    gather_timing_data = config.get('gather_timing_data', False)

    hosts = KnownHostsStore()
    destination_serverkey = hosts.serverkey(destination_vip)
    if destination_serverkey is None:
        _log.info(
            "Destination serverkey not found in known hosts file, using config"
        )
        destination_serverkey = config['destination-serverkey']

    return DataMover(services_topic_list,
                     custom_topic_list,
                     topic_replace_list,
                     destination_vip,
                     destination_serverkey,
                     destination_historian_identity,
                     gather_timing_data,
                     backup_storage_limit_gb=backup_storage_limit_gb,
                     **kwargs)
Beispiel #2
0
def historian(config_path, **kwargs):
    config = utils.load_config(config_path)
    services_topic_list = config.get('services_topic_list', [
        topics.DRIVER_TOPIC_BASE,
        topics.LOGGER_BASE,
        topics.ACTUATOR,
        topics.ANALYSIS_TOPIC_BASE
    ])
    custom_topic_list = config.get('custom_topic_list', [])
    topic_replace_list = config.get('topic_replace_list', [])
    destination_vip = config.get('destination-vip')
    destination_historian_identity = config.get('destination-historian-identity',
                                                'platform.historian')
    backup_storage_limit_gb = config.get('backup_storage_limit_gb', None)

    gather_timing_data = config.get('gather_timing_data', False)

    hosts = KnownHostsStore()
    destination_serverkey = hosts.serverkey(destination_vip)
    if destination_serverkey is None:
        _log.info("Destination serverkey not found in known hosts file, using config")
        destination_serverkey = config['destination-serverkey']

    return DataMover(services_topic_list,
                     custom_topic_list,
                     topic_replace_list,
                     destination_vip,
                     destination_serverkey,
                     destination_historian_identity,
                     gather_timing_data,
                     backup_storage_limit_gb=backup_storage_limit_gb,
                     **kwargs)
Beispiel #3
0
def historian(config_path, **kwargs):
    config = utils.load_config(config_path)
    custom_topic_list = config.pop('custom_topic_list', [])
    topic_replace_list = config.pop('topic_replace_list', [])
    destination_vip = config.pop('destination-vip', None)
    service_topic_list = config.pop('service_topic_list', None)
    destination_serverkey = None
    try:
        destination_address = config.pop('destination-address')
    except KeyError:
        destination_address = None
    if service_topic_list is not None:
        w = "Deprecated service_topic_list.  Use capture_device_data " \
            "capture_log_data, capture_analysis_data or capture_record_data " \
            "instead!"
        _log.warning(w)

        # Populate the new values for the kwargs based upon the old data.
        kwargs['capture_device_data'] = True if (
            "device" in service_topic_list
            or "all" in service_topic_list) else False
        kwargs['capture_log_data'] = True if (
            "datalogger" in service_topic_list
            or "all" in service_topic_list) else False
        kwargs['capture_record_data'] = True if (
            "record" in service_topic_list
            or "all" in service_topic_list) else False
        kwargs['capture_analysis_data'] = True if (
            "analysis" in service_topic_list
            or "all" in service_topic_list) else False

    if destination_vip:
        hosts = KnownHostsStore()
        destination_serverkey = hosts.serverkey(destination_vip)
        if destination_serverkey is None:
            _log.info(
                "Destination serverkey not found in known hosts file, using config"
            )
            destination_serverkey = config.pop('destination-serverkey')
        else:
            config.pop('destination-serverkey', None)

        destination_messagebus = 'zmq'

    required_target_agents = config.pop('required_target_agents', [])
    cache_only = config.pop('cache_only', False)

    utils.update_kwargs_with_config(kwargs, config)

    return ForwardHistorian(destination_vip,
                            destination_serverkey,
                            custom_topic_list=custom_topic_list,
                            topic_replace_list=topic_replace_list,
                            required_target_agents=required_target_agents,
                            cache_only=cache_only,
                            destination_address=destination_address,
                            **kwargs)
Beispiel #4
0
def get_keys():
    """Gets keys from keystore and known-hosts store

    :returns: Keys for connecting to the platform
    :rtype: dict
    """
    hosts = KnownHostsStore()
    serverkey = hosts.serverkey(get_address())
    key_store = KeyStore()
    publickey = key_store.public
    secretkey = key_store.secret
    return {'publickey': publickey,
            'secretkey': secretkey,
            'serverkey': serverkey}
Beispiel #5
0
def get_keys():
    """Gets keys from keystore and known-hosts store

    :returns: Keys for connecting to the platform
    :rtype: dict
    """
    hosts = KnownHostsStore()
    serverkey = hosts.serverkey(get_address())
    key_store = KeyStore()
    publickey = key_store.public
    secretkey = key_store.secret
    return {'publickey': publickey,
            'secretkey': secretkey,
            'serverkey': serverkey}
Beispiel #6
0
def historian(config_path, **kwargs):
    config = utils.load_config(config_path)
    destination_vip = config.get('destination-vip', None)
    assert destination_vip is not None

    hosts = KnownHostsStore()
    destination_serverkey = hosts.serverkey(destination_vip)
    if destination_serverkey is not None:
        config['destination-serverkey'] = destination_serverkey
    else:
        assert config.get('destination-serverkey') is not None
        _log.info("Destination serverkey not found in known hosts file, using config")

    utils.update_kwargs_with_config(kwargs, config)
    return DataMover(**kwargs)
Beispiel #7
0
def historian(config_path, **kwargs):
    config = utils.load_config(config_path)
    destination_vip = config.get('destination-vip', None)
    assert destination_vip is not None

    hosts = KnownHostsStore()
    serverkey = hosts.serverkey(destination_vip)
    if serverkey is not None:
        config['destination-serverkey'] = serverkey
    else:
        assert config.get('destination-serverkey') is not None
        _log.info("Destination serverkey not found in known hosts file, "
                  "using config")

    utils.update_kwargs_with_config(kwargs, config)
    return DataMover(**kwargs)
Beispiel #8
0
    def __init__(self, config_path, **kwargs):
        super(FailoverAgent, self).__init__(**kwargs)
        config = utils.load_config(config_path)

        # Get agent and remote ids
        agent_id = config["agent_id"]
        if agent_id == "primary":
            self.agent_id = "primary"
            self.remote_id = "secondary"
        elif agent_id == "secondary":
            self.agent_id = "secondary"
            self.remote_id = "primary"
        else:
            _log.error("agent_id must be either 'primary' or 'secondary'")

        # Modify ids if we're using the simple option
        # Defaults to true pending vc coordination
        use_simple = config.get("simple_behavior", True)
        if use_simple:
            self.agent_id = "simple_" + self.agent_id
            self.remote_id = "simple_" + self.remote_id

        self.remote_vip = config["remote_vip"]

        hosts = KnownHostsStore()
        self.remote_serverkey = hosts.serverkey(self.remote_vip)
        if self.remote_serverkey is None:
            self.remote_serverkey = config["remote_serverkey"]

        self.agent_vip_identity = config["agent_vip_identity"]
        self.heartbeat_period = config["heartbeat_period"]
        self.timeout = config["timeout"]

        self.vc_timeout = self.timeout
        self.remote_timeout = self.timeout
        self.agent_uuid = None
        self.heartbeat = None
        self.last_connected = None

        self._state = True, True
        self._state_machine = getattr(self, self.agent_id + '_state_machine')
Beispiel #9
0
    def __init__(self, config_path, **kwargs):
        super(FailoverAgent, self).__init__(**kwargs)
        config = utils.load_config(config_path)

        # Get agent and remote ids
        agent_id = config["agent_id"]
        if agent_id == "primary":
            self.agent_id = "primary"
            self.remote_id = "secondary"
        elif agent_id == "secondary":
            self.agent_id = "secondary"
            self.remote_id = "primary"
        else:
            _log.error("agent_id must be either 'primary' or 'secondary'")

        # Modify ids if we're using the simple option
        # Defaults to true pending vc coordination
        use_simple = config.get("simple_behavior", True)
        if use_simple:
            self.agent_id = "simple_" + self.agent_id
            self.remote_id = "simple_" + self.remote_id

        self.remote_vip = config["remote_vip"]

        hosts = KnownHostsStore()
        self.remote_serverkey = hosts.serverkey(self.remote_vip)
        if self.remote_serverkey is None:
            self.remote_serverkey = config["remote_serverkey"]

        self.agent_vip_identity = config["agent_vip_identity"]
        self.heartbeat_period = config["heartbeat_period"]
        self.timeout = config["timeout"]

        self.vc_timeout = self.timeout
        self.remote_timeout = self.timeout
        self.agent_uuid = None
        self.heartbeat = None
        self.last_connected = None

        self._state = True, True
        self._state_machine = getattr(self, self.agent_id + '_state_machine')
Beispiel #10
0
def historian(config_path, **kwargs):
    config = utils.load_config(config_path)
    custom_topic_list = config.pop('custom_topic_list', [])
    topic_replace_list = config.pop('topic_replace_list', [])
    destination_vip = config.pop('destination-vip', None)

    service_topic_list = config.pop('service_topic_list', None)
    if service_topic_list is not None:
        w = "Deprecated service_topic_list.  Use capture_device_data " \
            "capture_log_data, capture_analysis_data or capture_record_data " \
            "instead!"
        _log.warning(w)

        # Populate the new values for the kwargs based upon the old data.
        kwargs['capture_device_data'] = True if ("device" in service_topic_list or "all" in service_topic_list) else False
        kwargs['capture_log_data'] = True if ("datalogger" in service_topic_list or "all" in service_topic_list) else False
        kwargs['capture_record_data'] = True if ("record" in service_topic_list or "all" in service_topic_list) else False
        kwargs['capture_analysis_data'] = True if ("analysis" in service_topic_list or "all" in service_topic_list) else False

    hosts = KnownHostsStore()
    destination_serverkey = hosts.serverkey(destination_vip)
    if destination_serverkey is None:
        _log.info("Destination serverkey not found in known hosts file, using config")
        destination_serverkey = config.pop('destination-serverkey')
    else:
        config.pop('destination-serverkey', None)

    required_target_agents = config.pop('required_target_agents', [])
    cache_only = config.pop('cache_only', False)

    utils.update_kwargs_with_config(kwargs, config)

    return ForwardHistorian(destination_vip, destination_serverkey,
                            custom_topic_list=custom_topic_list,
                            topic_replace_list=topic_replace_list,
                            required_target_agents=required_target_agents,
                            cache_only=cache_only,
                            **kwargs)
Beispiel #11
0
    def connect_remote_platform(
            self,
            address,
            serverkey=None,
            agent_class=None
    ):
        """
        Agent attempts to connect to a remote platform to exchange data.

        address must start with http, https, tcp, ampq, or ampqs or a
        ValueError will be
        raised

        If this function is successful it will return an instance of the
        `agent_class`
        parameter if not then this function will return None.

        If the address parameter begins with http or https
        TODO: use the known host functionality here
        the agent will attempt to use Discovery to find the values
        associated with it.

        Discovery should return either an rmq-address or a vip-address or
        both.  In
        that situation the connection will be made using zmq.  In the event
        that
        fails then rmq will be tried.  If both fail then None is returned
        from this
        function.

        """
        from volttron.platform.vip.agent.utils import build_agent
        from volttron.platform.vip.agent import Agent

        if agent_class is None:
            agent_class = Agent

        parsed_address = urlparse(address)
        _log.debug("Begining auth.connect_remote_platform: {}".format(address))

        value = None
        if parsed_address.scheme == "tcp":
            # ZMQ connection
            hosts = KnownHostsStore()
            temp_serverkey = hosts.serverkey(address)
            if not temp_serverkey:
                _log.info(
                    "Destination serverkey not found in known hosts file, "
                    "using config"
                )
                destination_serverkey = serverkey
            elif not serverkey:
                destination_serverkey = temp_serverkey
            else:
                if temp_serverkey != serverkey:
                    raise ValueError(
                        "server_key passed and known hosts serverkey do not "
                        ""
                        "match!"
                    )
                destination_serverkey = serverkey

            publickey, secretkey = (
                self._core().publickey,
                self._core().secretkey,
            )
            _log.debug(
                "Connecting using: %s", get_fq_identity(self._core().identity)
            )

            value = build_agent(
                agent_class=agent_class,
                identity=get_fq_identity(self._core().identity),
                serverkey=destination_serverkey,
                publickey=publickey,
                secretkey=secretkey,
                message_bus="zmq",
                address=address,
            )
        elif parsed_address.scheme in ("https", "http"):
            from volttron.platform.web import DiscoveryInfo
            from volttron.platform.web import DiscoveryError

            try:
                # TODO: Use known host instead of looking up for discovery
                #  info if possible.

                # We need to discover which type of bus is at the other end.
                info = DiscoveryInfo.request_discovery_info(address)
                remote_identity = "{}.{}.{}".format(
                    info.instance_name,
                    get_platform_instance_name(),
                    self._core().identity,
                )
                # if the current message bus is zmq then we need
                # to connect a zmq on the remote, whether that be the
                # rmq router or proxy.  Also note that we are using the
                # fully qualified
                # version of the identity because there will be conflicts if
                # volttron central has more than one platform.agent connecting
                if get_messagebus() == "zmq":
                    if not info.vip_address or not info.serverkey:
                        err = (
                            "Discovery from {} did not return serverkey "
                            "and/or vip_address".format(address)
                        )
                        raise ValueError(err)

                    _log.debug(
                        "Connecting using: %s",
                        get_fq_identity(self._core().identity),
                    )

                    # use fully qualified identity
                    value = build_agent(
                        identity=get_fq_identity(self._core().identity),
                        address=info.vip_address,
                        serverkey=info.serverkey,
                        secretkey=self._core().secretkey,
                        publickey=self._core().publickey,
                        agent_class=agent_class,
                    )

                else:  # we are on rmq messagebus

                    # This is if both remote and local are rmq message buses.
                    if info.messagebus_type == "rmq":
                        _log.debug("Both remote and local are rmq messagebus.")
                        fqid_local = get_fq_identity(self._core().identity)

                        # Check if we already have the cert, if so use it
                        # instead of requesting cert again
                        remote_certs_dir = self.get_remote_certs_dir()
                        remote_cert_name = "{}.{}".format(
                            info.instance_name, fqid_local
                        )
                        certfile = os.path.join(
                            remote_certs_dir, remote_cert_name + ".crt"
                        )
                        if os.path.exists(certfile):
                            response = certfile
                        else:
                            response = self.request_cert(
                                address, fqid_local, info
                            )

                        if response is None:
                            _log.error("there was no response from the server")
                            value = None
                        elif isinstance(response, tuple):
                            if response[0] == "PENDING":
                                _log.info(
                                    "Waiting for administrator to accept a "
                                    "CSR request."
                                )
                            value = None
                        # elif isinstance(response, dict):
                        #     response
                        elif os.path.exists(response):
                            # info = DiscoveryInfo.request_discovery_info(
                            # address)
                            # From the remote platforms perspective the
                            # remote user name is
                            #   remoteinstance.localinstance.identity,
                            #   this is what we must
                            #   pass to the build_remote_connection_params
                            #   for a successful

                            remote_rmq_user = get_fq_identity(
                                fqid_local, info.instance_name
                            )
                            _log.debug(
                                "REMOTE RMQ USER IS: %s", remote_rmq_user
                            )
                            remote_rmq_address = self._core().rmq_mgmt.build_remote_connection_param(
                                remote_rmq_user,
                                info.rmq_address,
                                ssl_auth=True,
                                cert_dir=self.get_remote_certs_dir(),
                            )

                            value = build_agent(
                                identity=fqid_local,
                                address=remote_rmq_address,
                                instance_name=info.instance_name,
                                publickey=self._core().publickey,
                                secretkey=self._core().secretkey,
                                message_bus="rmq",
                                enable_store=False,
                                agent_class=agent_class,
                            )
                        else:
                            raise ValueError(
                                "Unknown path through discovery process!"
                            )

                    else:
                        # TODO: cache the connection so we don't always have
                        #  to ping the server to connect.

                        # This branch happens when the message bus is not
                        # the same note
                        # this writes to the agent-data directory of this
                        # agent if the agent
                        # is installed.
                        if get_messagebus() == "rmq":
                            if not os.path.exists("keystore.json"):
                                with open("keystore.json", "w") as file_pointer:
                                    file_pointer.write(
                                        jsonapi.dumps(
                                            KeyStore.generate_keypair_dict()
                                        )
                                    )

                            with open("keystore.json") as file_pointer:
                                keypair = jsonapi.loads(file_pointer.read())

                        value = build_agent(
                            agent_class=agent_class,
                            identity=remote_identity,
                            serverkey=info.serverkey,
                            publickey=keypair.get("publickey"),
                            secretkey=keypair.get("secretekey"),
                            message_bus="zmq",
                            address=info.vip_address,
                        )
            except DiscoveryError:
                _log.error(
                    "Couldn't connect to %s or incorrect response returned "
                    "response was %s",
                    address,
                    value,
                )

        else:
            raise ValueError(
                "Invalid configuration found the address: {} has an invalid "
                "scheme".format(address)
            )

        return value
Beispiel #12
0
 def _get_serverkey_from_known_hosts(self):
     known_hosts_file = os.path.join(self.volttron_home, 'known_hosts')
     known_hosts = KnownHostsStore(known_hosts_file)
     return known_hosts.serverkey(self.address)
Beispiel #13
0
def historian(config_path, **kwargs):
    config = utils.load_config(config_path)
    services_topic_list = config.get('services_topic_list', ['all'])
    custom_topic_list = config.get('custom_topic_list', [])
    topic_replace_list = config.get('topic_replace_list', [])
    destination_vip = config.get('destination-vip')

    gather_timing_data = config.get('gather_timing_data', False)

    hosts = KnownHostsStore()
    destination_serverkey = hosts.serverkey(destination_vip)
    if destination_serverkey is None:
        _log.info("Destination serverkey not found in known hosts file, using config")
        destination_serverkey = config['destination-serverkey']

    required_target_agents = config.get('required_target_agents', [])
    backup_storage_limit_gb = config.get('backup_storage_limit_gb', None)
    if 'all' in services_topic_list:
        services_topic_list = [topics.DRIVER_TOPIC_BASE, topics.LOGGER_BASE,
                               topics.ACTUATOR, topics.ANALYSIS_TOPIC_BASE]

    class ForwardHistorian(BaseHistorian):
        """
        This historian forwards data to another platform.
        """

        def __init__(self, **kwargs):
            # will be available in both threads.
            self._topic_replace_map = {}
            self._num_failures = 0
            self._last_timeout = 0
            self._target_platform = None
            super(ForwardHistorian, self).__init__(**kwargs)

        @Core.receiver("onstart")
        def starting_base(self, sender, **kwargs):
            """
            Subscribes to the platform message bus on the actuator, record,
            datalogger, and device topics to capture data.
            """

            def subscriber(subscription, callback_method):
                _log.debug("subscribing to {}".format(subscription))
                self.vip.pubsub.subscribe(peer='pubsub',
                                          prefix=subscription,
                                          callback=callback_method)

            _log.debug("Starting Forward historian")
            for topic_subscriptions in services_topic_list:
                subscriber(topic_subscriptions, self.capture_data)

            for custom_topic in custom_topic_list:
                subscriber(custom_topic, self.capture_data)

            self._started = True

        def timestamp(self):
            return time.mktime(datetime.datetime.now().timetuple())

        def capture_data(self, peer, sender, bus, topic, headers, message):

            # Grab the timestamp string from the message (we use this as the
            # value in our readings at the end of this method)
            _log.debug("In capture data")
            timestamp_string = headers.get(headers_mod.DATE, None)

            data = message
            try:
                # 2.0 agents compatability layer makes sender = pubsub.compat
                # so we can do the proper thing when it is here
                _log.debug("message in capture_data {}".format(message))
                if sender == 'pubsub.compat':
                    # data = jsonapi.loads(message[0])
                    data = compat.unpack_legacy_message(headers, message)
                    _log.debug("data in capture_data {}".format(data))
                if isinstance(data, dict):
                    data = data
                elif isinstance(data, int) or \
                        isinstance(data, float) or \
                        isinstance(data, long):
                    data = data
                    # else:
                    #     data = data[0]
            except ValueError as e:
                log_message = "message for {topic} bad message string:" \
                              "{message_string}"
                _log.error(log_message.format(topic=topic,
                                              message_string=message[0]))
                raise

            if topic_replace_list:
                if topic in self._topic_replace_map.keys():
                    topic = self._topic_replace_map[topic]
                else:
                    self._topic_replace_map[topic] = topic
                    temptopics = {}
                    for x in topic_replace_list:
                        if x['from'] in topic:
                            new_topic = temptopics.get(topic, topic)
                            temptopics[topic] = new_topic.replace(
                                x['from'], x['to'])

                    for k, v in temptopics.items():
                        self._topic_replace_map[k] = v
                    topic = self._topic_replace_map[topic]

            if gather_timing_data:
                add_timing_data_to_header(headers, self.core.agent_uuid or self.core.identity, "collected")

            payload = {'headers': headers, 'message': data}

            self._event_queue.put({'source': "forwarded",
                                   'topic': topic,
                                   'readings': [(timestamp_string, payload)]})

        @doc_inherit
        def publish_to_historian(self, to_publish_list):
            handled_records = []

            _log.debug("publish_to_historian number of items: {}"
                       .format(len(to_publish_list)))
            parsed = urlparse(self.core.address)
            next_dest = urlparse(destination_vip)
            current_time = self.timestamp()
            last_time = self._last_timeout
            _log.debug('Lasttime: {} currenttime: {}'.format(last_time,
                                                             current_time))
            timeout_occurred = False
            if self._last_timeout:
                # if we failed we need to wait 60 seconds before we go on.
                if self.timestamp() < self._last_timeout + 60:
                    _log.debug('Not allowing send < 60 seconds from failure')
                    return
            if not self._target_platform:
                self.historian_setup()
            if not self._target_platform:
                _log.debug('Could not connect to target')
                return

            for vip_id in required_target_agents:
                try:
                    self._target_platform.vip.ping(vip_id).get()
                except Unreachable:
                    skip = "Skipping publish: Target platform not running " \
                           "required agent {}".format(vip_id)
                    _log.warn(skip)
                    self.vip.health.set_status(
                        STATUS_BAD, skip)
                    return
                except Exception as e:
                    err = "Unhandled error publishing to target platform."
                    _log.error(err)
                    _log.error(traceback.format_exc())
                    self.vip.health.set_status(
                        STATUS_BAD, err)
                    return

            for x in to_publish_list:
                topic = x['topic']
                value = x['value']
                # payload = jsonapi.loads(value)
                payload = value
                headers = payload['headers']
                headers['X-Forwarded'] = True
                try:
                    del headers['Origin']
                except KeyError:
                    pass
                try:
                    del headers['Destination']
                except KeyError:
                    pass

                if gather_timing_data:
                    add_timing_data_to_header(headers, self.core.agent_uuid or self.core.identity,"forwarded")

                if timeout_occurred:
                    _log.error(
                        'A timeout has occurred so breaking out of publishing')
                    break
                with gevent.Timeout(30):
                    try:
                        _log.debug('debugger: {} {} {}'.format(topic,
                                                               headers,
                                                               payload))

                        self._target_platform.vip.pubsub.publish(
                            peer='pubsub',
                            topic=topic,
                            headers=headers,
                            message=payload['message']).get()
                    except gevent.Timeout:
                        _log.debug("Timeout occurred email should send!")
                        timeout_occurred = True
                        self._last_timeout = self.timestamp()
                        self._num_failures += 1
                        # Stop the current platform from attempting to
                        # connect
                        self._target_platform.core.stop()
                        self._target_platform = None
                        self.vip.health.set_status(
                            STATUS_BAD, "Timeout occured")
                    except Unreachable:
                        _log.error("Target not reachable. Wait till it's ready!")
                    except ZMQError as exc:
                        if exc.errno == ENOTSOCK:
                            # Stop the current platform from attempting to
                            # connect
                            _log.error("Target disconnected. Stopping target platform agent")
                            self._target_platform = None
                            self.vip.health.set_status(
                                STATUS_BAD, "Target platform disconnected")
                    except Exception as e:
                        err = "Unhandled error publishing to target platfom."
                        _log.error(err)
                        _log.error(traceback.format_exc())
                        self.vip.health.set_status(
                            STATUS_BAD, err)
                        # Before returning lets mark any that weren't errors
                        # as sent.
                        self.report_handled(handled_records)
                        return
                    else:
                        handled_records.append(x)

            _log.debug("handled: {} number of items".format(
                len(to_publish_list)))
            self.report_handled(handled_records)

            if timeout_occurred:
                _log.debug('Sending alert from the ForwardHistorian')
                status = Status.from_json(self.vip.health.get_status())
                self.vip.health.send_alert(FORWARD_TIMEOUT_KEY,
                                           status)
            else:
                self.vip.health.set_status(
                    STATUS_GOOD,"published {} items".format(
                        len(to_publish_list)))

        @doc_inherit
        def historian_setup(self):
            _log.debug("Setting up to forward to {}".format(destination_vip))
            try:
                agent = build_agent(address=destination_vip,
                                    serverkey=destination_serverkey,
                                    publickey=self.core.publickey,
                                    secretkey=self.core.secretkey,
                                    enable_store=False)

            except gevent.Timeout:
                self.vip.health.set_status(
                    STATUS_BAD, "Timeout in setup of agent")
                status = Status.from_json(self.vip.health.get_status_json())
                self.vip.health.send_alert(FORWARD_TIMEOUT_KEY,
                                           status)
            else:
                self._target_platform = agent


    return ForwardHistorian(backup_storage_limit_gb=backup_storage_limit_gb,
                            **kwargs)
Beispiel #14
0
 def _get_serverkey_from_known_hosts(self):
     known_hosts_file = os.path.join(self.volttron_home, 'known_hosts')
     known_hosts = KnownHostsStore(known_hosts_file)
     return known_hosts.serverkey(self.address)
Beispiel #15
0
def historian(config_path, **kwargs):
    config = utils.load_config(config_path)
    services_topic_list = config.get('services_topic_list', ['all'])
    custom_topic_list = config.get('custom_topic_list', [])
    topic_replace_list = config.get('topic_replace_list', [])
    destination_vip = config.get('destination-vip')

    gather_timing_data = config.get('gather_timing_data', False)

    hosts = KnownHostsStore()
    destination_serverkey = hosts.serverkey(destination_vip)
    if destination_serverkey is None:
        _log.info(
            "Destination serverkey not found in known hosts file, using config"
        )
        destination_serverkey = config['destination-serverkey']

    required_target_agents = config.get('required_target_agents', [])
    backup_storage_limit_gb = config.get('backup_storage_limit_gb', None)
    if 'all' in services_topic_list:
        services_topic_list = [
            topics.DRIVER_TOPIC_BASE, topics.LOGGER_BASE, topics.ACTUATOR,
            topics.ANALYSIS_TOPIC_BASE
        ]

    class ForwardHistorian(BaseHistorian):
        """
        This historian forwards data to another platform.
        """
        def __init__(self, **kwargs):
            # will be available in both threads.
            self._topic_replace_map = {}
            self._num_failures = 0
            self._last_timeout = 0
            self._target_platform = None
            super(ForwardHistorian, self).__init__(**kwargs)

        @Core.receiver("onstart")
        def starting_base(self, sender, **kwargs):
            """
            Subscribes to the platform message bus on the actuator, record,
            datalogger, and device topics to capture data.
            """
            def subscriber(subscription, callback_method):
                _log.debug("subscribing to {}".format(subscription))
                self.vip.pubsub.subscribe(peer='pubsub',
                                          prefix=subscription,
                                          callback=callback_method)

            _log.debug("Starting Forward historian")
            for topic_subscriptions in services_topic_list:
                subscriber(topic_subscriptions, self.capture_data)

            for custom_topic in custom_topic_list:
                subscriber(custom_topic, self.capture_data)

            self._started = True

        def timestamp(self):
            return time.mktime(datetime.datetime.now().timetuple())

        def capture_data(self, peer, sender, bus, topic, headers, message):

            # Grab the timestamp string from the message (we use this as the
            # value in our readings at the end of this method)
            _log.debug("In capture data")
            timestamp_string = headers.get(headers_mod.DATE, None)

            data = message
            try:
                # 2.0 agents compatability layer makes sender = pubsub.compat
                # so we can do the proper thing when it is here
                _log.debug("message in capture_data {}".format(message))
                if sender == 'pubsub.compat':
                    # data = jsonapi.loads(message[0])
                    data = compat.unpack_legacy_message(headers, message)
                    _log.debug("data in capture_data {}".format(data))
                if isinstance(data, dict):
                    data = data
                elif isinstance(data, int) or \
                        isinstance(data, float) or \
                        isinstance(data, long):
                    data = data
                    # else:
                    #     data = data[0]
            except ValueError as e:
                log_message = "message for {topic} bad message string:" \
                              "{message_string}"
                _log.error(
                    log_message.format(topic=topic, message_string=message[0]))
                raise

            if topic_replace_list:
                if topic in self._topic_replace_map.keys():
                    topic = self._topic_replace_map[topic]
                else:
                    self._topic_replace_map[topic] = topic
                    temptopics = {}
                    for x in topic_replace_list:
                        if x['from'] in topic:
                            new_topic = temptopics.get(topic, topic)
                            temptopics[topic] = new_topic.replace(
                                x['from'], x['to'])

                    for k, v in temptopics.items():
                        self._topic_replace_map[k] = v
                    topic = self._topic_replace_map[topic]

            if gather_timing_data:
                add_timing_data_to_header(
                    headers, self.core.agent_uuid or self.core.identity,
                    "collected")

            payload = {'headers': headers, 'message': data}

            self._event_queue.put({
                'source': "forwarded",
                'topic': topic,
                'readings': [(timestamp_string, payload)]
            })

        @doc_inherit
        def publish_to_historian(self, to_publish_list):
            handled_records = []

            _log.debug("publish_to_historian number of items: {}".format(
                len(to_publish_list)))
            parsed = urlparse(self.core.address)
            next_dest = urlparse(destination_vip)
            current_time = self.timestamp()
            last_time = self._last_timeout
            _log.debug('Lasttime: {} currenttime: {}'.format(
                last_time, current_time))
            timeout_occurred = False
            if self._last_timeout:
                # if we failed we need to wait 60 seconds before we go on.
                if self.timestamp() < self._last_timeout + 60:
                    _log.debug('Not allowing send < 60 seconds from failure')
                    return
            if not self._target_platform:
                self.historian_setup()
            if not self._target_platform:
                _log.debug('Could not connect to target')
                return

            for vip_id in required_target_agents:
                try:
                    self._target_platform.vip.ping(vip_id).get()
                except Unreachable:
                    skip = "Skipping publish: Target platform not running " \
                           "required agent {}".format(vip_id)
                    _log.warn(skip)
                    self.vip.health.set_status(STATUS_BAD, skip)
                    return
                except Exception as e:
                    err = "Unhandled error publishing to target platform."
                    _log.error(err)
                    _log.error(traceback.format_exc())
                    self.vip.health.set_status(STATUS_BAD, err)
                    return

            for x in to_publish_list:
                topic = x['topic']
                value = x['value']
                # payload = jsonapi.loads(value)
                payload = value
                headers = payload['headers']
                headers['X-Forwarded'] = True
                try:
                    del headers['Origin']
                except KeyError:
                    pass
                try:
                    del headers['Destination']
                except KeyError:
                    pass

                if gather_timing_data:
                    add_timing_data_to_header(
                        headers, self.core.agent_uuid or self.core.identity,
                        "forwarded")

                if timeout_occurred:
                    _log.error(
                        'A timeout has occurred so breaking out of publishing')
                    break
                with gevent.Timeout(30):
                    try:
                        _log.debug('debugger: {} {} {}'.format(
                            topic, headers, payload))

                        self._target_platform.vip.pubsub.publish(
                            peer='pubsub',
                            topic=topic,
                            headers=headers,
                            message=payload['message']).get()
                    except gevent.Timeout:
                        _log.debug("Timeout occurred email should send!")
                        timeout_occurred = True
                        self._last_timeout = self.timestamp()
                        self._num_failures += 1
                        # Stop the current platform from attempting to
                        # connect
                        self._target_platform.core.stop()
                        self._target_platform = None
                        self.vip.health.set_status(STATUS_BAD,
                                                   "Timeout occured")
                    except Unreachable:
                        _log.error(
                            "Target not reachable. Wait till it's ready!")
                    except ZMQError as exc:
                        if exc.errno == ENOTSOCK:
                            # Stop the current platform from attempting to
                            # connect
                            _log.error(
                                "Target disconnected. Stopping target platform agent"
                            )
                            self._target_platform = None
                            self.vip.health.set_status(
                                STATUS_BAD, "Target platform disconnected")
                    except Exception as e:
                        err = "Unhandled error publishing to target platfom."
                        _log.error(err)
                        _log.error(traceback.format_exc())
                        self.vip.health.set_status(STATUS_BAD, err)
                        # Before returning lets mark any that weren't errors
                        # as sent.
                        self.report_handled(handled_records)
                        return
                    else:
                        handled_records.append(x)

            _log.debug("handled: {} number of items".format(
                len(to_publish_list)))
            self.report_handled(handled_records)

            if timeout_occurred:
                _log.debug('Sending alert from the ForwardHistorian')
                status = Status.from_json(self.vip.health.get_status())
                self.vip.health.send_alert(FORWARD_TIMEOUT_KEY, status)
            else:
                self.vip.health.set_status(
                    STATUS_GOOD,
                    "published {} items".format(len(to_publish_list)))

        @doc_inherit
        def historian_setup(self):
            _log.debug("Setting up to forward to {}".format(destination_vip))
            try:
                agent = build_agent(address=destination_vip,
                                    serverkey=destination_serverkey,
                                    publickey=self.core.publickey,
                                    secretkey=self.core.secretkey,
                                    enable_store=False)

            except gevent.Timeout:
                self.vip.health.set_status(STATUS_BAD,
                                           "Timeout in setup of agent")
                status = Status.from_json(self.vip.health.get_status_json())
                self.vip.health.send_alert(FORWARD_TIMEOUT_KEY, status)
            else:
                self._target_platform = agent

    return ForwardHistorian(backup_storage_limit_gb=backup_storage_limit_gb,
                            **kwargs)
Beispiel #16
0
def historian(config_path, **kwargs):
    config = utils.load_config(config_path)
    services_topic_list = config.get('services_topic_list', ['all'])
    custom_topic_list = config.get('custom_topic_list', [])
    topic_replace_list = config.get('topic_replace_list', [])
    source_vip = config.get('source-vip')
    hosts = KnownHostsStore()
    source_serverkey = hosts.serverkey(source_vip)
    if source_serverkey is None:
        _log.info(
            "Source serverkey not found in known hosts file, using config")
        source_serverkey = config['source-serverkey']
    identity = config.get('identity', kwargs.pop('identity', None))
    include_destination_in_header = config.get('include_destination_in_header',
                                               False)

    origin = config.get('origin', None)
    overwrite_origin = config.get('overwrite_origin', False)
    include_origin_in_header = config.get('include_origin_in_header', False)
    if 'all' in services_topic_list:
        services_topic_list = [
            topics.DRIVER_TOPIC_BASE, topics.LOGGER_BASE, topics.ACTUATOR,
            topics.ANALYSIS_TOPIC_BASE
        ]

    class DataPuller(Agent):
        '''This historian pulls data from another platform.
        '''
        def __init__(self, **kwargs):
            # will be available in both threads.
            self._topic_replace_map = {}
            self._num_failures = 0
            self._last_timeout = 0
            self._target_platform = None
            super(DataPuller, self).__init__(**kwargs)

        @Core.receiver("onstart")
        def starting_base(self, sender, **kwargs):
            self.puller_setup()
            '''
            Subscribes to the platform message bus on the actuator, record,
            datalogger, and device topics to capture data.
            '''
            def subscriber(subscription, callback_method):
                _log.debug("subscribing to {}".format(subscription))
                self._target_platform.vip.pubsub.subscribe(
                    peer='pubsub',
                    prefix=subscription,
                    callback=callback_method)

            _log.debug("Starting DataPuller")
            for topic_subscriptions in services_topic_list:
                subscriber(topic_subscriptions, self.on_message)

            for custom_topic in custom_topic_list:
                subscriber(custom_topic, self.on_message)

            self._started = True

        def timestamp(self):
            return time.mktime(datetime.datetime.now().timetuple())

        def on_message(self, peer, sender, bus, topic, headers, message):

            # Grab the timestamp string from the message (we use this as the
            # value in our readings at the end of this method)
            _log.debug("In capture data")
            timestamp_string = headers.get(headers_mod.DATE, None)

            data = message
            try:
                # 2.0 agents compatability layer makes sender = pubsub.compat
                # so we can do the proper thing when it is here
                _log.debug("message in capture_data {}".format(message))
                if sender == 'pubsub.compat':
                    # data = jsonapi.loads(message[0])
                    data = compat.unpack_legacy_message(headers, message)
                    _log.debug("data in capture_data {}".format(data))
                if isinstance(data, dict):
                    data = data
                elif isinstance(data, int) or \
                        isinstance(data, float) or \
                        isinstance(data, long):
                    data = data
                    # else:
                    #     data = data[0]
            except ValueError as e:
                log_message = "message for {topic} bad message string:" \
                              "{message_string}"
                _log.error(
                    log_message.format(topic=topic, message_string=message[0]))
                raise

            if topic_replace_list:
                if topic in self._topic_replace_map.keys():
                    topic = self._topic_replace_map[topic]
                else:
                    self._topic_replace_map[topic] = topic
                    temptopics = {}
                    for x in topic_replace_list:
                        if x['from'] in topic:
                            new_topic = temptopics.get(topic, topic)
                            temptopics[topic] = new_topic.replace(
                                x['from'], x['to'])

                    for k, v in temptopics.items():
                        self._topic_replace_map[k] = v
                    topic = self._topic_replace_map[topic]

            payload = {'headers': headers, 'topic': topic, 'message': data}
            self.publish_to_self(topic, payload)

        #             self._event_queue.put({'source': "forwarded",
        #                                    'topic': topic,
        #                                    'readings': [(timestamp_string, payload)]})

        def publish_to_self(self, topic, payload):
            handled_records = []

            parsed = urlparse(self.core.address)
            next_dest = urlparse(source_vip)
            current_time = self.timestamp()
            last_time = self._last_timeout
            _log.debug('Lasttime: {} currenttime: {}'.format(
                last_time, current_time))
            timeout_occurred = False
            if self._last_timeout:
                # if we failed we need to wait 60 seconds before we go on.
                if self.timestamp() < self._last_timeout + 60:
                    _log.debug('Not allowing send < 60 seconds from failure')
                    return
            if not self._target_platform:
                self.puller_setup()
            if not self._target_platform:
                _log.debug('Could not connect to target')
                return

            headers = payload['headers']
            headers['X-Forwarded'] = True
            try:
                del headers['Origin']
            except KeyError:
                pass
            try:
                del headers['Destination']
            except KeyError:
                pass

            print("should publish", topic, payload)

            self.vip.pubsub.publish(peer='pubsub',
                                    topic=topic,
                                    headers=headers,
                                    message=payload['message']).get(30)

        def puller_setup(self):
            _log.debug("Setting up to forward to {}".format(source_vip))
            try:
                agent = build_agent(address=source_vip,
                                    serverkey=source_serverkey,
                                    publickey=self.core.publickey,
                                    secretkey=self.core.secretkey,
                                    enable_store=False)
            except gevent.Timeout:
                self.vip.health.set_status(STATUS_BAD,
                                           "Timeout in setup of agent")
                status = Status.from_json(self.vip.health.get_status_json())
                self.vip.health.send_alert(FORWARD_TIMEOUT_KEY, status)
            else:
                self._target_platform = agent

    DataPuller.__name__ = 'DataPuller'
    return DataPuller(identity=identity, **kwargs)
Beispiel #17
0
def historian(config_path, **kwargs):
    config = utils.load_config(config_path)
    services_topic_list = config.get('services_topic_list', ['all'])
    custom_topic_list = config.get('custom_topic_list', [])
    topic_replace_list = config.get('topic_replace_list', [])
    source_vip = config.get('source-vip')
    hosts = KnownHostsStore()
    source_serverkey = hosts.serverkey(source_vip)
    if source_serverkey is None:
        _log.info("Source serverkey not found in known hosts file, using config")
        source_serverkey = config['source-serverkey']
    identity = config.get('identity', kwargs.pop('identity', None))
    include_destination_in_header = config.get('include_destination_in_header',
                                               False)

    origin = config.get('origin', None)
    overwrite_origin = config.get('overwrite_origin', False)
    include_origin_in_header = config.get('include_origin_in_header', False)
    if 'all' in services_topic_list:
        services_topic_list = [topics.DRIVER_TOPIC_BASE, topics.LOGGER_BASE,
                               topics.ACTUATOR, topics.ANALYSIS_TOPIC_BASE]

    class DataPuller(Agent):
        '''This historian pulls data from another platform.
        '''

        def __init__(self, **kwargs):
            # will be available in both threads.
            self._topic_replace_map = {}
            self._num_failures = 0
            self._last_timeout = 0
            self._target_platform = None
            super(DataPuller, self).__init__(**kwargs)

        @Core.receiver("onstart")
        def starting_base(self, sender, **kwargs):
            self.puller_setup()
            '''
            Subscribes to the platform message bus on the actuator, record,
            datalogger, and device topics to capture data.
            '''

            def subscriber(subscription, callback_method):
                _log.debug("subscribing to {}".format(subscription))
                self._target_platform.vip.pubsub.subscribe(peer='pubsub',
                                                           prefix=subscription,
                                                           callback=callback_method)

            _log.debug("Starting DataPuller")
            for topic_subscriptions in services_topic_list:
                subscriber(topic_subscriptions, self.on_message)

            for custom_topic in custom_topic_list:
                subscriber(custom_topic, self.on_message)

            self._started = True

        def timestamp(self):
            return time.mktime(datetime.datetime.now().timetuple())

        def on_message(self, peer, sender, bus, topic, headers, message):

            # Grab the timestamp string from the message (we use this as the
            # value in our readings at the end of this method)
            _log.debug("In capture data")
            timestamp_string = headers.get(headers_mod.DATE, None)

            data = message
            try:
                # 2.0 agents compatability layer makes sender = pubsub.compat
                # so we can do the proper thing when it is here
                _log.debug("message in capture_data {}".format(message))
                if sender == 'pubsub.compat':
                    # data = jsonapi.loads(message[0])
                    data = compat.unpack_legacy_message(headers, message)
                    _log.debug("data in capture_data {}".format(data))
                if isinstance(data, dict):
                    data = data
                elif isinstance(data, int) or \
                        isinstance(data, float) or \
                        isinstance(data, long):
                    data = data
                    # else:
                    #     data = data[0]
            except ValueError as e:
                log_message = "message for {topic} bad message string:" \
                              "{message_string}"
                _log.error(log_message.format(topic=topic,
                                              message_string=message[0]))
                raise

            if topic_replace_list:
                if topic in self._topic_replace_map.keys():
                    topic = self._topic_replace_map[topic]
                else:
                    self._topic_replace_map[topic] = topic
                    temptopics = {}
                    for x in topic_replace_list:
                        if x['from'] in topic:
                            new_topic = temptopics.get(topic, topic)
                            temptopics[topic] = new_topic.replace(
                                x['from'], x['to'])

                    for k, v in temptopics.items():
                        self._topic_replace_map[k] = v
                    topic = self._topic_replace_map[topic]

            payload = {'headers': headers, 'topic': topic, 'message': data}
            self.publish_to_self(topic, payload)

        #             self._event_queue.put({'source': "forwarded",
        #                                    'topic': topic,
        #                                    'readings': [(timestamp_string, payload)]})


        def publish_to_self(self, topic, payload):
            handled_records = []

            parsed = urlparse(self.core.address)
            next_dest = urlparse(source_vip)
            current_time = self.timestamp()
            last_time = self._last_timeout
            _log.debug('Lasttime: {} currenttime: {}'.format(last_time, current_time))
            timeout_occurred = False
            if self._last_timeout:
                # if we failed we need to wait 60 seconds before we go on.
                if self.timestamp() < self._last_timeout + 60:
                    _log.debug('Not allowing send < 60 seconds from failure')
                    return
            if not self._target_platform:
                self.puller_setup()
            if not self._target_platform:
                _log.debug('Could not connect to target')
                return

            headers = payload['headers']
            headers['X-Forwarded'] = True
            try:
                del headers['Origin']
            except KeyError:
                pass
            try:
                del headers['Destination']
            except KeyError:
                pass

            print("should publish", topic, payload)

            self.vip.pubsub.publish(
                peer='pubsub',
                topic=topic,
                headers=headers,
                message=payload['message']).get(30)

        def puller_setup(self):
            _log.debug("Setting up to forward to {}".format(source_vip))
            try:
                agent = build_agent(address=source_vip,
                                    serverkey=source_serverkey,
                                    publickey=self.core.publickey,
                                    secretkey=self.core.secretkey,
                                    enable_store=False)
            except gevent.Timeout:
                self.vip.health.set_status(
                    STATUS_BAD, "Timeout in setup of agent")
                status = Status.from_json(self.vip.health.get_status_json())
                self.vip.health.send_alert(FORWARD_TIMEOUT_KEY,
                                           status)
            else:
                self._target_platform = agent

    DataPuller.__name__ = 'DataPuller'
    return DataPuller(identity=identity,
                      **kwargs)