def historian(config_path, **kwargs): config = utils.load_config(config_path) services_topic_list = config.get('services_topic_list', [ topics.DRIVER_TOPIC_BASE, topics.LOGGER_BASE, topics.ACTUATOR, topics.ANALYSIS_TOPIC_BASE ]) custom_topic_list = config.get('custom_topic_list', []) topic_replace_list = config.get('topic_replace_list', []) destination_vip = config.get('destination-vip') destination_historian_identity = config.get( 'destination-historian-identity', 'platform.historian') backup_storage_limit_gb = config.get('backup_storage_limit_gb', None) gather_timing_data = config.get('gather_timing_data', False) hosts = KnownHostsStore() destination_serverkey = hosts.serverkey(destination_vip) if destination_serverkey is None: _log.info( "Destination serverkey not found in known hosts file, using config" ) destination_serverkey = config['destination-serverkey'] return DataMover(services_topic_list, custom_topic_list, topic_replace_list, destination_vip, destination_serverkey, destination_historian_identity, gather_timing_data, backup_storage_limit_gb=backup_storage_limit_gb, **kwargs)
def historian(config_path, **kwargs): config = utils.load_config(config_path) services_topic_list = config.get('services_topic_list', [ topics.DRIVER_TOPIC_BASE, topics.LOGGER_BASE, topics.ACTUATOR, topics.ANALYSIS_TOPIC_BASE ]) custom_topic_list = config.get('custom_topic_list', []) topic_replace_list = config.get('topic_replace_list', []) destination_vip = config.get('destination-vip') destination_historian_identity = config.get('destination-historian-identity', 'platform.historian') backup_storage_limit_gb = config.get('backup_storage_limit_gb', None) gather_timing_data = config.get('gather_timing_data', False) hosts = KnownHostsStore() destination_serverkey = hosts.serverkey(destination_vip) if destination_serverkey is None: _log.info("Destination serverkey not found in known hosts file, using config") destination_serverkey = config['destination-serverkey'] return DataMover(services_topic_list, custom_topic_list, topic_replace_list, destination_vip, destination_serverkey, destination_historian_identity, gather_timing_data, backup_storage_limit_gb=backup_storage_limit_gb, **kwargs)
def historian(config_path, **kwargs): config = utils.load_config(config_path) custom_topic_list = config.pop('custom_topic_list', []) topic_replace_list = config.pop('topic_replace_list', []) destination_vip = config.pop('destination-vip', None) service_topic_list = config.pop('service_topic_list', None) destination_serverkey = None try: destination_address = config.pop('destination-address') except KeyError: destination_address = None if service_topic_list is not None: w = "Deprecated service_topic_list. Use capture_device_data " \ "capture_log_data, capture_analysis_data or capture_record_data " \ "instead!" _log.warning(w) # Populate the new values for the kwargs based upon the old data. kwargs['capture_device_data'] = True if ( "device" in service_topic_list or "all" in service_topic_list) else False kwargs['capture_log_data'] = True if ( "datalogger" in service_topic_list or "all" in service_topic_list) else False kwargs['capture_record_data'] = True if ( "record" in service_topic_list or "all" in service_topic_list) else False kwargs['capture_analysis_data'] = True if ( "analysis" in service_topic_list or "all" in service_topic_list) else False if destination_vip: hosts = KnownHostsStore() destination_serverkey = hosts.serverkey(destination_vip) if destination_serverkey is None: _log.info( "Destination serverkey not found in known hosts file, using config" ) destination_serverkey = config.pop('destination-serverkey') else: config.pop('destination-serverkey', None) destination_messagebus = 'zmq' required_target_agents = config.pop('required_target_agents', []) cache_only = config.pop('cache_only', False) utils.update_kwargs_with_config(kwargs, config) return ForwardHistorian(destination_vip, destination_serverkey, custom_topic_list=custom_topic_list, topic_replace_list=topic_replace_list, required_target_agents=required_target_agents, cache_only=cache_only, destination_address=destination_address, **kwargs)
def get_keys(): """Gets keys from keystore and known-hosts store :returns: Keys for connecting to the platform :rtype: dict """ hosts = KnownHostsStore() serverkey = hosts.serverkey(get_address()) key_store = KeyStore() publickey = key_store.public secretkey = key_store.secret return {'publickey': publickey, 'secretkey': secretkey, 'serverkey': serverkey}
def get_keys(): """Gets keys from keystore and known-hosts store :returns: Keys for connecting to the platform :rtype: dict """ hosts = KnownHostsStore() serverkey = hosts.serverkey(get_address()) key_store = KeyStore() publickey = key_store.public secretkey = key_store.secret return {'publickey': publickey, 'secretkey': secretkey, 'serverkey': serverkey}
def historian(config_path, **kwargs): config = utils.load_config(config_path) destination_vip = config.get('destination-vip', None) assert destination_vip is not None hosts = KnownHostsStore() destination_serverkey = hosts.serverkey(destination_vip) if destination_serverkey is not None: config['destination-serverkey'] = destination_serverkey else: assert config.get('destination-serverkey') is not None _log.info("Destination serverkey not found in known hosts file, using config") utils.update_kwargs_with_config(kwargs, config) return DataMover(**kwargs)
def historian(config_path, **kwargs): config = utils.load_config(config_path) destination_vip = config.get('destination-vip', None) assert destination_vip is not None hosts = KnownHostsStore() serverkey = hosts.serverkey(destination_vip) if serverkey is not None: config['destination-serverkey'] = serverkey else: assert config.get('destination-serverkey') is not None _log.info("Destination serverkey not found in known hosts file, " "using config") utils.update_kwargs_with_config(kwargs, config) return DataMover(**kwargs)
def __init__(self, config_path, **kwargs): super(FailoverAgent, self).__init__(**kwargs) config = utils.load_config(config_path) # Get agent and remote ids agent_id = config["agent_id"] if agent_id == "primary": self.agent_id = "primary" self.remote_id = "secondary" elif agent_id == "secondary": self.agent_id = "secondary" self.remote_id = "primary" else: _log.error("agent_id must be either 'primary' or 'secondary'") # Modify ids if we're using the simple option # Defaults to true pending vc coordination use_simple = config.get("simple_behavior", True) if use_simple: self.agent_id = "simple_" + self.agent_id self.remote_id = "simple_" + self.remote_id self.remote_vip = config["remote_vip"] hosts = KnownHostsStore() self.remote_serverkey = hosts.serverkey(self.remote_vip) if self.remote_serverkey is None: self.remote_serverkey = config["remote_serverkey"] self.agent_vip_identity = config["agent_vip_identity"] self.heartbeat_period = config["heartbeat_period"] self.timeout = config["timeout"] self.vc_timeout = self.timeout self.remote_timeout = self.timeout self.agent_uuid = None self.heartbeat = None self.last_connected = None self._state = True, True self._state_machine = getattr(self, self.agent_id + '_state_machine')
def __init__(self, config_path, **kwargs): super(FailoverAgent, self).__init__(**kwargs) config = utils.load_config(config_path) # Get agent and remote ids agent_id = config["agent_id"] if agent_id == "primary": self.agent_id = "primary" self.remote_id = "secondary" elif agent_id == "secondary": self.agent_id = "secondary" self.remote_id = "primary" else: _log.error("agent_id must be either 'primary' or 'secondary'") # Modify ids if we're using the simple option # Defaults to true pending vc coordination use_simple = config.get("simple_behavior", True) if use_simple: self.agent_id = "simple_" + self.agent_id self.remote_id = "simple_" + self.remote_id self.remote_vip = config["remote_vip"] hosts = KnownHostsStore() self.remote_serverkey = hosts.serverkey(self.remote_vip) if self.remote_serverkey is None: self.remote_serverkey = config["remote_serverkey"] self.agent_vip_identity = config["agent_vip_identity"] self.heartbeat_period = config["heartbeat_period"] self.timeout = config["timeout"] self.vc_timeout = self.timeout self.remote_timeout = self.timeout self.agent_uuid = None self.heartbeat = None self.last_connected = None self._state = True, True self._state_machine = getattr(self, self.agent_id + '_state_machine')
def historian(config_path, **kwargs): config = utils.load_config(config_path) custom_topic_list = config.pop('custom_topic_list', []) topic_replace_list = config.pop('topic_replace_list', []) destination_vip = config.pop('destination-vip', None) service_topic_list = config.pop('service_topic_list', None) if service_topic_list is not None: w = "Deprecated service_topic_list. Use capture_device_data " \ "capture_log_data, capture_analysis_data or capture_record_data " \ "instead!" _log.warning(w) # Populate the new values for the kwargs based upon the old data. kwargs['capture_device_data'] = True if ("device" in service_topic_list or "all" in service_topic_list) else False kwargs['capture_log_data'] = True if ("datalogger" in service_topic_list or "all" in service_topic_list) else False kwargs['capture_record_data'] = True if ("record" in service_topic_list or "all" in service_topic_list) else False kwargs['capture_analysis_data'] = True if ("analysis" in service_topic_list or "all" in service_topic_list) else False hosts = KnownHostsStore() destination_serverkey = hosts.serverkey(destination_vip) if destination_serverkey is None: _log.info("Destination serverkey not found in known hosts file, using config") destination_serverkey = config.pop('destination-serverkey') else: config.pop('destination-serverkey', None) required_target_agents = config.pop('required_target_agents', []) cache_only = config.pop('cache_only', False) utils.update_kwargs_with_config(kwargs, config) return ForwardHistorian(destination_vip, destination_serverkey, custom_topic_list=custom_topic_list, topic_replace_list=topic_replace_list, required_target_agents=required_target_agents, cache_only=cache_only, **kwargs)
def connect_remote_platform( self, address, serverkey=None, agent_class=None ): """ Agent attempts to connect to a remote platform to exchange data. address must start with http, https, tcp, ampq, or ampqs or a ValueError will be raised If this function is successful it will return an instance of the `agent_class` parameter if not then this function will return None. If the address parameter begins with http or https TODO: use the known host functionality here the agent will attempt to use Discovery to find the values associated with it. Discovery should return either an rmq-address or a vip-address or both. In that situation the connection will be made using zmq. In the event that fails then rmq will be tried. If both fail then None is returned from this function. """ from volttron.platform.vip.agent.utils import build_agent from volttron.platform.vip.agent import Agent if agent_class is None: agent_class = Agent parsed_address = urlparse(address) _log.debug("Begining auth.connect_remote_platform: {}".format(address)) value = None if parsed_address.scheme == "tcp": # ZMQ connection hosts = KnownHostsStore() temp_serverkey = hosts.serverkey(address) if not temp_serverkey: _log.info( "Destination serverkey not found in known hosts file, " "using config" ) destination_serverkey = serverkey elif not serverkey: destination_serverkey = temp_serverkey else: if temp_serverkey != serverkey: raise ValueError( "server_key passed and known hosts serverkey do not " "" "match!" ) destination_serverkey = serverkey publickey, secretkey = ( self._core().publickey, self._core().secretkey, ) _log.debug( "Connecting using: %s", get_fq_identity(self._core().identity) ) value = build_agent( agent_class=agent_class, identity=get_fq_identity(self._core().identity), serverkey=destination_serverkey, publickey=publickey, secretkey=secretkey, message_bus="zmq", address=address, ) elif parsed_address.scheme in ("https", "http"): from volttron.platform.web import DiscoveryInfo from volttron.platform.web import DiscoveryError try: # TODO: Use known host instead of looking up for discovery # info if possible. # We need to discover which type of bus is at the other end. info = DiscoveryInfo.request_discovery_info(address) remote_identity = "{}.{}.{}".format( info.instance_name, get_platform_instance_name(), self._core().identity, ) # if the current message bus is zmq then we need # to connect a zmq on the remote, whether that be the # rmq router or proxy. Also note that we are using the # fully qualified # version of the identity because there will be conflicts if # volttron central has more than one platform.agent connecting if get_messagebus() == "zmq": if not info.vip_address or not info.serverkey: err = ( "Discovery from {} did not return serverkey " "and/or vip_address".format(address) ) raise ValueError(err) _log.debug( "Connecting using: %s", get_fq_identity(self._core().identity), ) # use fully qualified identity value = build_agent( identity=get_fq_identity(self._core().identity), address=info.vip_address, serverkey=info.serverkey, secretkey=self._core().secretkey, publickey=self._core().publickey, agent_class=agent_class, ) else: # we are on rmq messagebus # This is if both remote and local are rmq message buses. if info.messagebus_type == "rmq": _log.debug("Both remote and local are rmq messagebus.") fqid_local = get_fq_identity(self._core().identity) # Check if we already have the cert, if so use it # instead of requesting cert again remote_certs_dir = self.get_remote_certs_dir() remote_cert_name = "{}.{}".format( info.instance_name, fqid_local ) certfile = os.path.join( remote_certs_dir, remote_cert_name + ".crt" ) if os.path.exists(certfile): response = certfile else: response = self.request_cert( address, fqid_local, info ) if response is None: _log.error("there was no response from the server") value = None elif isinstance(response, tuple): if response[0] == "PENDING": _log.info( "Waiting for administrator to accept a " "CSR request." ) value = None # elif isinstance(response, dict): # response elif os.path.exists(response): # info = DiscoveryInfo.request_discovery_info( # address) # From the remote platforms perspective the # remote user name is # remoteinstance.localinstance.identity, # this is what we must # pass to the build_remote_connection_params # for a successful remote_rmq_user = get_fq_identity( fqid_local, info.instance_name ) _log.debug( "REMOTE RMQ USER IS: %s", remote_rmq_user ) remote_rmq_address = self._core().rmq_mgmt.build_remote_connection_param( remote_rmq_user, info.rmq_address, ssl_auth=True, cert_dir=self.get_remote_certs_dir(), ) value = build_agent( identity=fqid_local, address=remote_rmq_address, instance_name=info.instance_name, publickey=self._core().publickey, secretkey=self._core().secretkey, message_bus="rmq", enable_store=False, agent_class=agent_class, ) else: raise ValueError( "Unknown path through discovery process!" ) else: # TODO: cache the connection so we don't always have # to ping the server to connect. # This branch happens when the message bus is not # the same note # this writes to the agent-data directory of this # agent if the agent # is installed. if get_messagebus() == "rmq": if not os.path.exists("keystore.json"): with open("keystore.json", "w") as file_pointer: file_pointer.write( jsonapi.dumps( KeyStore.generate_keypair_dict() ) ) with open("keystore.json") as file_pointer: keypair = jsonapi.loads(file_pointer.read()) value = build_agent( agent_class=agent_class, identity=remote_identity, serverkey=info.serverkey, publickey=keypair.get("publickey"), secretkey=keypair.get("secretekey"), message_bus="zmq", address=info.vip_address, ) except DiscoveryError: _log.error( "Couldn't connect to %s or incorrect response returned " "response was %s", address, value, ) else: raise ValueError( "Invalid configuration found the address: {} has an invalid " "scheme".format(address) ) return value
def _get_serverkey_from_known_hosts(self): known_hosts_file = os.path.join(self.volttron_home, 'known_hosts') known_hosts = KnownHostsStore(known_hosts_file) return known_hosts.serverkey(self.address)
def historian(config_path, **kwargs): config = utils.load_config(config_path) services_topic_list = config.get('services_topic_list', ['all']) custom_topic_list = config.get('custom_topic_list', []) topic_replace_list = config.get('topic_replace_list', []) destination_vip = config.get('destination-vip') gather_timing_data = config.get('gather_timing_data', False) hosts = KnownHostsStore() destination_serverkey = hosts.serverkey(destination_vip) if destination_serverkey is None: _log.info("Destination serverkey not found in known hosts file, using config") destination_serverkey = config['destination-serverkey'] required_target_agents = config.get('required_target_agents', []) backup_storage_limit_gb = config.get('backup_storage_limit_gb', None) if 'all' in services_topic_list: services_topic_list = [topics.DRIVER_TOPIC_BASE, topics.LOGGER_BASE, topics.ACTUATOR, topics.ANALYSIS_TOPIC_BASE] class ForwardHistorian(BaseHistorian): """ This historian forwards data to another platform. """ def __init__(self, **kwargs): # will be available in both threads. self._topic_replace_map = {} self._num_failures = 0 self._last_timeout = 0 self._target_platform = None super(ForwardHistorian, self).__init__(**kwargs) @Core.receiver("onstart") def starting_base(self, sender, **kwargs): """ Subscribes to the platform message bus on the actuator, record, datalogger, and device topics to capture data. """ def subscriber(subscription, callback_method): _log.debug("subscribing to {}".format(subscription)) self.vip.pubsub.subscribe(peer='pubsub', prefix=subscription, callback=callback_method) _log.debug("Starting Forward historian") for topic_subscriptions in services_topic_list: subscriber(topic_subscriptions, self.capture_data) for custom_topic in custom_topic_list: subscriber(custom_topic, self.capture_data) self._started = True def timestamp(self): return time.mktime(datetime.datetime.now().timetuple()) def capture_data(self, peer, sender, bus, topic, headers, message): # Grab the timestamp string from the message (we use this as the # value in our readings at the end of this method) _log.debug("In capture data") timestamp_string = headers.get(headers_mod.DATE, None) data = message try: # 2.0 agents compatability layer makes sender = pubsub.compat # so we can do the proper thing when it is here _log.debug("message in capture_data {}".format(message)) if sender == 'pubsub.compat': # data = jsonapi.loads(message[0]) data = compat.unpack_legacy_message(headers, message) _log.debug("data in capture_data {}".format(data)) if isinstance(data, dict): data = data elif isinstance(data, int) or \ isinstance(data, float) or \ isinstance(data, long): data = data # else: # data = data[0] except ValueError as e: log_message = "message for {topic} bad message string:" \ "{message_string}" _log.error(log_message.format(topic=topic, message_string=message[0])) raise if topic_replace_list: if topic in self._topic_replace_map.keys(): topic = self._topic_replace_map[topic] else: self._topic_replace_map[topic] = topic temptopics = {} for x in topic_replace_list: if x['from'] in topic: new_topic = temptopics.get(topic, topic) temptopics[topic] = new_topic.replace( x['from'], x['to']) for k, v in temptopics.items(): self._topic_replace_map[k] = v topic = self._topic_replace_map[topic] if gather_timing_data: add_timing_data_to_header(headers, self.core.agent_uuid or self.core.identity, "collected") payload = {'headers': headers, 'message': data} self._event_queue.put({'source': "forwarded", 'topic': topic, 'readings': [(timestamp_string, payload)]}) @doc_inherit def publish_to_historian(self, to_publish_list): handled_records = [] _log.debug("publish_to_historian number of items: {}" .format(len(to_publish_list))) parsed = urlparse(self.core.address) next_dest = urlparse(destination_vip) current_time = self.timestamp() last_time = self._last_timeout _log.debug('Lasttime: {} currenttime: {}'.format(last_time, current_time)) timeout_occurred = False if self._last_timeout: # if we failed we need to wait 60 seconds before we go on. if self.timestamp() < self._last_timeout + 60: _log.debug('Not allowing send < 60 seconds from failure') return if not self._target_platform: self.historian_setup() if not self._target_platform: _log.debug('Could not connect to target') return for vip_id in required_target_agents: try: self._target_platform.vip.ping(vip_id).get() except Unreachable: skip = "Skipping publish: Target platform not running " \ "required agent {}".format(vip_id) _log.warn(skip) self.vip.health.set_status( STATUS_BAD, skip) return except Exception as e: err = "Unhandled error publishing to target platform." _log.error(err) _log.error(traceback.format_exc()) self.vip.health.set_status( STATUS_BAD, err) return for x in to_publish_list: topic = x['topic'] value = x['value'] # payload = jsonapi.loads(value) payload = value headers = payload['headers'] headers['X-Forwarded'] = True try: del headers['Origin'] except KeyError: pass try: del headers['Destination'] except KeyError: pass if gather_timing_data: add_timing_data_to_header(headers, self.core.agent_uuid or self.core.identity,"forwarded") if timeout_occurred: _log.error( 'A timeout has occurred so breaking out of publishing') break with gevent.Timeout(30): try: _log.debug('debugger: {} {} {}'.format(topic, headers, payload)) self._target_platform.vip.pubsub.publish( peer='pubsub', topic=topic, headers=headers, message=payload['message']).get() except gevent.Timeout: _log.debug("Timeout occurred email should send!") timeout_occurred = True self._last_timeout = self.timestamp() self._num_failures += 1 # Stop the current platform from attempting to # connect self._target_platform.core.stop() self._target_platform = None self.vip.health.set_status( STATUS_BAD, "Timeout occured") except Unreachable: _log.error("Target not reachable. Wait till it's ready!") except ZMQError as exc: if exc.errno == ENOTSOCK: # Stop the current platform from attempting to # connect _log.error("Target disconnected. Stopping target platform agent") self._target_platform = None self.vip.health.set_status( STATUS_BAD, "Target platform disconnected") except Exception as e: err = "Unhandled error publishing to target platfom." _log.error(err) _log.error(traceback.format_exc()) self.vip.health.set_status( STATUS_BAD, err) # Before returning lets mark any that weren't errors # as sent. self.report_handled(handled_records) return else: handled_records.append(x) _log.debug("handled: {} number of items".format( len(to_publish_list))) self.report_handled(handled_records) if timeout_occurred: _log.debug('Sending alert from the ForwardHistorian') status = Status.from_json(self.vip.health.get_status()) self.vip.health.send_alert(FORWARD_TIMEOUT_KEY, status) else: self.vip.health.set_status( STATUS_GOOD,"published {} items".format( len(to_publish_list))) @doc_inherit def historian_setup(self): _log.debug("Setting up to forward to {}".format(destination_vip)) try: agent = build_agent(address=destination_vip, serverkey=destination_serverkey, publickey=self.core.publickey, secretkey=self.core.secretkey, enable_store=False) except gevent.Timeout: self.vip.health.set_status( STATUS_BAD, "Timeout in setup of agent") status = Status.from_json(self.vip.health.get_status_json()) self.vip.health.send_alert(FORWARD_TIMEOUT_KEY, status) else: self._target_platform = agent return ForwardHistorian(backup_storage_limit_gb=backup_storage_limit_gb, **kwargs)
def _get_serverkey_from_known_hosts(self): known_hosts_file = os.path.join(self.volttron_home, 'known_hosts') known_hosts = KnownHostsStore(known_hosts_file) return known_hosts.serverkey(self.address)
def historian(config_path, **kwargs): config = utils.load_config(config_path) services_topic_list = config.get('services_topic_list', ['all']) custom_topic_list = config.get('custom_topic_list', []) topic_replace_list = config.get('topic_replace_list', []) destination_vip = config.get('destination-vip') gather_timing_data = config.get('gather_timing_data', False) hosts = KnownHostsStore() destination_serverkey = hosts.serverkey(destination_vip) if destination_serverkey is None: _log.info( "Destination serverkey not found in known hosts file, using config" ) destination_serverkey = config['destination-serverkey'] required_target_agents = config.get('required_target_agents', []) backup_storage_limit_gb = config.get('backup_storage_limit_gb', None) if 'all' in services_topic_list: services_topic_list = [ topics.DRIVER_TOPIC_BASE, topics.LOGGER_BASE, topics.ACTUATOR, topics.ANALYSIS_TOPIC_BASE ] class ForwardHistorian(BaseHistorian): """ This historian forwards data to another platform. """ def __init__(self, **kwargs): # will be available in both threads. self._topic_replace_map = {} self._num_failures = 0 self._last_timeout = 0 self._target_platform = None super(ForwardHistorian, self).__init__(**kwargs) @Core.receiver("onstart") def starting_base(self, sender, **kwargs): """ Subscribes to the platform message bus on the actuator, record, datalogger, and device topics to capture data. """ def subscriber(subscription, callback_method): _log.debug("subscribing to {}".format(subscription)) self.vip.pubsub.subscribe(peer='pubsub', prefix=subscription, callback=callback_method) _log.debug("Starting Forward historian") for topic_subscriptions in services_topic_list: subscriber(topic_subscriptions, self.capture_data) for custom_topic in custom_topic_list: subscriber(custom_topic, self.capture_data) self._started = True def timestamp(self): return time.mktime(datetime.datetime.now().timetuple()) def capture_data(self, peer, sender, bus, topic, headers, message): # Grab the timestamp string from the message (we use this as the # value in our readings at the end of this method) _log.debug("In capture data") timestamp_string = headers.get(headers_mod.DATE, None) data = message try: # 2.0 agents compatability layer makes sender = pubsub.compat # so we can do the proper thing when it is here _log.debug("message in capture_data {}".format(message)) if sender == 'pubsub.compat': # data = jsonapi.loads(message[0]) data = compat.unpack_legacy_message(headers, message) _log.debug("data in capture_data {}".format(data)) if isinstance(data, dict): data = data elif isinstance(data, int) or \ isinstance(data, float) or \ isinstance(data, long): data = data # else: # data = data[0] except ValueError as e: log_message = "message for {topic} bad message string:" \ "{message_string}" _log.error( log_message.format(topic=topic, message_string=message[0])) raise if topic_replace_list: if topic in self._topic_replace_map.keys(): topic = self._topic_replace_map[topic] else: self._topic_replace_map[topic] = topic temptopics = {} for x in topic_replace_list: if x['from'] in topic: new_topic = temptopics.get(topic, topic) temptopics[topic] = new_topic.replace( x['from'], x['to']) for k, v in temptopics.items(): self._topic_replace_map[k] = v topic = self._topic_replace_map[topic] if gather_timing_data: add_timing_data_to_header( headers, self.core.agent_uuid or self.core.identity, "collected") payload = {'headers': headers, 'message': data} self._event_queue.put({ 'source': "forwarded", 'topic': topic, 'readings': [(timestamp_string, payload)] }) @doc_inherit def publish_to_historian(self, to_publish_list): handled_records = [] _log.debug("publish_to_historian number of items: {}".format( len(to_publish_list))) parsed = urlparse(self.core.address) next_dest = urlparse(destination_vip) current_time = self.timestamp() last_time = self._last_timeout _log.debug('Lasttime: {} currenttime: {}'.format( last_time, current_time)) timeout_occurred = False if self._last_timeout: # if we failed we need to wait 60 seconds before we go on. if self.timestamp() < self._last_timeout + 60: _log.debug('Not allowing send < 60 seconds from failure') return if not self._target_platform: self.historian_setup() if not self._target_platform: _log.debug('Could not connect to target') return for vip_id in required_target_agents: try: self._target_platform.vip.ping(vip_id).get() except Unreachable: skip = "Skipping publish: Target platform not running " \ "required agent {}".format(vip_id) _log.warn(skip) self.vip.health.set_status(STATUS_BAD, skip) return except Exception as e: err = "Unhandled error publishing to target platform." _log.error(err) _log.error(traceback.format_exc()) self.vip.health.set_status(STATUS_BAD, err) return for x in to_publish_list: topic = x['topic'] value = x['value'] # payload = jsonapi.loads(value) payload = value headers = payload['headers'] headers['X-Forwarded'] = True try: del headers['Origin'] except KeyError: pass try: del headers['Destination'] except KeyError: pass if gather_timing_data: add_timing_data_to_header( headers, self.core.agent_uuid or self.core.identity, "forwarded") if timeout_occurred: _log.error( 'A timeout has occurred so breaking out of publishing') break with gevent.Timeout(30): try: _log.debug('debugger: {} {} {}'.format( topic, headers, payload)) self._target_platform.vip.pubsub.publish( peer='pubsub', topic=topic, headers=headers, message=payload['message']).get() except gevent.Timeout: _log.debug("Timeout occurred email should send!") timeout_occurred = True self._last_timeout = self.timestamp() self._num_failures += 1 # Stop the current platform from attempting to # connect self._target_platform.core.stop() self._target_platform = None self.vip.health.set_status(STATUS_BAD, "Timeout occured") except Unreachable: _log.error( "Target not reachable. Wait till it's ready!") except ZMQError as exc: if exc.errno == ENOTSOCK: # Stop the current platform from attempting to # connect _log.error( "Target disconnected. Stopping target platform agent" ) self._target_platform = None self.vip.health.set_status( STATUS_BAD, "Target platform disconnected") except Exception as e: err = "Unhandled error publishing to target platfom." _log.error(err) _log.error(traceback.format_exc()) self.vip.health.set_status(STATUS_BAD, err) # Before returning lets mark any that weren't errors # as sent. self.report_handled(handled_records) return else: handled_records.append(x) _log.debug("handled: {} number of items".format( len(to_publish_list))) self.report_handled(handled_records) if timeout_occurred: _log.debug('Sending alert from the ForwardHistorian') status = Status.from_json(self.vip.health.get_status()) self.vip.health.send_alert(FORWARD_TIMEOUT_KEY, status) else: self.vip.health.set_status( STATUS_GOOD, "published {} items".format(len(to_publish_list))) @doc_inherit def historian_setup(self): _log.debug("Setting up to forward to {}".format(destination_vip)) try: agent = build_agent(address=destination_vip, serverkey=destination_serverkey, publickey=self.core.publickey, secretkey=self.core.secretkey, enable_store=False) except gevent.Timeout: self.vip.health.set_status(STATUS_BAD, "Timeout in setup of agent") status = Status.from_json(self.vip.health.get_status_json()) self.vip.health.send_alert(FORWARD_TIMEOUT_KEY, status) else: self._target_platform = agent return ForwardHistorian(backup_storage_limit_gb=backup_storage_limit_gb, **kwargs)
def historian(config_path, **kwargs): config = utils.load_config(config_path) services_topic_list = config.get('services_topic_list', ['all']) custom_topic_list = config.get('custom_topic_list', []) topic_replace_list = config.get('topic_replace_list', []) source_vip = config.get('source-vip') hosts = KnownHostsStore() source_serverkey = hosts.serverkey(source_vip) if source_serverkey is None: _log.info( "Source serverkey not found in known hosts file, using config") source_serverkey = config['source-serverkey'] identity = config.get('identity', kwargs.pop('identity', None)) include_destination_in_header = config.get('include_destination_in_header', False) origin = config.get('origin', None) overwrite_origin = config.get('overwrite_origin', False) include_origin_in_header = config.get('include_origin_in_header', False) if 'all' in services_topic_list: services_topic_list = [ topics.DRIVER_TOPIC_BASE, topics.LOGGER_BASE, topics.ACTUATOR, topics.ANALYSIS_TOPIC_BASE ] class DataPuller(Agent): '''This historian pulls data from another platform. ''' def __init__(self, **kwargs): # will be available in both threads. self._topic_replace_map = {} self._num_failures = 0 self._last_timeout = 0 self._target_platform = None super(DataPuller, self).__init__(**kwargs) @Core.receiver("onstart") def starting_base(self, sender, **kwargs): self.puller_setup() ''' Subscribes to the platform message bus on the actuator, record, datalogger, and device topics to capture data. ''' def subscriber(subscription, callback_method): _log.debug("subscribing to {}".format(subscription)) self._target_platform.vip.pubsub.subscribe( peer='pubsub', prefix=subscription, callback=callback_method) _log.debug("Starting DataPuller") for topic_subscriptions in services_topic_list: subscriber(topic_subscriptions, self.on_message) for custom_topic in custom_topic_list: subscriber(custom_topic, self.on_message) self._started = True def timestamp(self): return time.mktime(datetime.datetime.now().timetuple()) def on_message(self, peer, sender, bus, topic, headers, message): # Grab the timestamp string from the message (we use this as the # value in our readings at the end of this method) _log.debug("In capture data") timestamp_string = headers.get(headers_mod.DATE, None) data = message try: # 2.0 agents compatability layer makes sender = pubsub.compat # so we can do the proper thing when it is here _log.debug("message in capture_data {}".format(message)) if sender == 'pubsub.compat': # data = jsonapi.loads(message[0]) data = compat.unpack_legacy_message(headers, message) _log.debug("data in capture_data {}".format(data)) if isinstance(data, dict): data = data elif isinstance(data, int) or \ isinstance(data, float) or \ isinstance(data, long): data = data # else: # data = data[0] except ValueError as e: log_message = "message for {topic} bad message string:" \ "{message_string}" _log.error( log_message.format(topic=topic, message_string=message[0])) raise if topic_replace_list: if topic in self._topic_replace_map.keys(): topic = self._topic_replace_map[topic] else: self._topic_replace_map[topic] = topic temptopics = {} for x in topic_replace_list: if x['from'] in topic: new_topic = temptopics.get(topic, topic) temptopics[topic] = new_topic.replace( x['from'], x['to']) for k, v in temptopics.items(): self._topic_replace_map[k] = v topic = self._topic_replace_map[topic] payload = {'headers': headers, 'topic': topic, 'message': data} self.publish_to_self(topic, payload) # self._event_queue.put({'source': "forwarded", # 'topic': topic, # 'readings': [(timestamp_string, payload)]}) def publish_to_self(self, topic, payload): handled_records = [] parsed = urlparse(self.core.address) next_dest = urlparse(source_vip) current_time = self.timestamp() last_time = self._last_timeout _log.debug('Lasttime: {} currenttime: {}'.format( last_time, current_time)) timeout_occurred = False if self._last_timeout: # if we failed we need to wait 60 seconds before we go on. if self.timestamp() < self._last_timeout + 60: _log.debug('Not allowing send < 60 seconds from failure') return if not self._target_platform: self.puller_setup() if not self._target_platform: _log.debug('Could not connect to target') return headers = payload['headers'] headers['X-Forwarded'] = True try: del headers['Origin'] except KeyError: pass try: del headers['Destination'] except KeyError: pass print("should publish", topic, payload) self.vip.pubsub.publish(peer='pubsub', topic=topic, headers=headers, message=payload['message']).get(30) def puller_setup(self): _log.debug("Setting up to forward to {}".format(source_vip)) try: agent = build_agent(address=source_vip, serverkey=source_serverkey, publickey=self.core.publickey, secretkey=self.core.secretkey, enable_store=False) except gevent.Timeout: self.vip.health.set_status(STATUS_BAD, "Timeout in setup of agent") status = Status.from_json(self.vip.health.get_status_json()) self.vip.health.send_alert(FORWARD_TIMEOUT_KEY, status) else: self._target_platform = agent DataPuller.__name__ = 'DataPuller' return DataPuller(identity=identity, **kwargs)
def historian(config_path, **kwargs): config = utils.load_config(config_path) services_topic_list = config.get('services_topic_list', ['all']) custom_topic_list = config.get('custom_topic_list', []) topic_replace_list = config.get('topic_replace_list', []) source_vip = config.get('source-vip') hosts = KnownHostsStore() source_serverkey = hosts.serverkey(source_vip) if source_serverkey is None: _log.info("Source serverkey not found in known hosts file, using config") source_serverkey = config['source-serverkey'] identity = config.get('identity', kwargs.pop('identity', None)) include_destination_in_header = config.get('include_destination_in_header', False) origin = config.get('origin', None) overwrite_origin = config.get('overwrite_origin', False) include_origin_in_header = config.get('include_origin_in_header', False) if 'all' in services_topic_list: services_topic_list = [topics.DRIVER_TOPIC_BASE, topics.LOGGER_BASE, topics.ACTUATOR, topics.ANALYSIS_TOPIC_BASE] class DataPuller(Agent): '''This historian pulls data from another platform. ''' def __init__(self, **kwargs): # will be available in both threads. self._topic_replace_map = {} self._num_failures = 0 self._last_timeout = 0 self._target_platform = None super(DataPuller, self).__init__(**kwargs) @Core.receiver("onstart") def starting_base(self, sender, **kwargs): self.puller_setup() ''' Subscribes to the platform message bus on the actuator, record, datalogger, and device topics to capture data. ''' def subscriber(subscription, callback_method): _log.debug("subscribing to {}".format(subscription)) self._target_platform.vip.pubsub.subscribe(peer='pubsub', prefix=subscription, callback=callback_method) _log.debug("Starting DataPuller") for topic_subscriptions in services_topic_list: subscriber(topic_subscriptions, self.on_message) for custom_topic in custom_topic_list: subscriber(custom_topic, self.on_message) self._started = True def timestamp(self): return time.mktime(datetime.datetime.now().timetuple()) def on_message(self, peer, sender, bus, topic, headers, message): # Grab the timestamp string from the message (we use this as the # value in our readings at the end of this method) _log.debug("In capture data") timestamp_string = headers.get(headers_mod.DATE, None) data = message try: # 2.0 agents compatability layer makes sender = pubsub.compat # so we can do the proper thing when it is here _log.debug("message in capture_data {}".format(message)) if sender == 'pubsub.compat': # data = jsonapi.loads(message[0]) data = compat.unpack_legacy_message(headers, message) _log.debug("data in capture_data {}".format(data)) if isinstance(data, dict): data = data elif isinstance(data, int) or \ isinstance(data, float) or \ isinstance(data, long): data = data # else: # data = data[0] except ValueError as e: log_message = "message for {topic} bad message string:" \ "{message_string}" _log.error(log_message.format(topic=topic, message_string=message[0])) raise if topic_replace_list: if topic in self._topic_replace_map.keys(): topic = self._topic_replace_map[topic] else: self._topic_replace_map[topic] = topic temptopics = {} for x in topic_replace_list: if x['from'] in topic: new_topic = temptopics.get(topic, topic) temptopics[topic] = new_topic.replace( x['from'], x['to']) for k, v in temptopics.items(): self._topic_replace_map[k] = v topic = self._topic_replace_map[topic] payload = {'headers': headers, 'topic': topic, 'message': data} self.publish_to_self(topic, payload) # self._event_queue.put({'source': "forwarded", # 'topic': topic, # 'readings': [(timestamp_string, payload)]}) def publish_to_self(self, topic, payload): handled_records = [] parsed = urlparse(self.core.address) next_dest = urlparse(source_vip) current_time = self.timestamp() last_time = self._last_timeout _log.debug('Lasttime: {} currenttime: {}'.format(last_time, current_time)) timeout_occurred = False if self._last_timeout: # if we failed we need to wait 60 seconds before we go on. if self.timestamp() < self._last_timeout + 60: _log.debug('Not allowing send < 60 seconds from failure') return if not self._target_platform: self.puller_setup() if not self._target_platform: _log.debug('Could not connect to target') return headers = payload['headers'] headers['X-Forwarded'] = True try: del headers['Origin'] except KeyError: pass try: del headers['Destination'] except KeyError: pass print("should publish", topic, payload) self.vip.pubsub.publish( peer='pubsub', topic=topic, headers=headers, message=payload['message']).get(30) def puller_setup(self): _log.debug("Setting up to forward to {}".format(source_vip)) try: agent = build_agent(address=source_vip, serverkey=source_serverkey, publickey=self.core.publickey, secretkey=self.core.secretkey, enable_store=False) except gevent.Timeout: self.vip.health.set_status( STATUS_BAD, "Timeout in setup of agent") status = Status.from_json(self.vip.health.get_status_json()) self.vip.health.send_alert(FORWARD_TIMEOUT_KEY, status) else: self._target_platform = agent DataPuller.__name__ = 'DataPuller' return DataPuller(identity=identity, **kwargs)