def historian(config_path, **kwargs):
config = utils.load_config(config_path)
services_topic_list = config.get('services_topic_list', [
topics.DRIVER_TOPIC_BASE, topics.LOGGER_BASE, topics.ACTUATOR,
topics.ANALYSIS_TOPIC_BASE
])
custom_topic_list = config.get('custom_topic_list', [])
topic_replace_list = config.get('topic_replace_list', [])
destination_vip = config.get('destination-vip')
destination_historian_identity = config.get(
'destination-historian-identity', 'platform.historian')
backup_storage_limit_gb = config.get('backup_storage_limit_gb', None)
gather_timing_data = config.get('gather_timing_data', False)
hosts = KnownHostsStore()
destination_serverkey = hosts.serverkey(destination_vip)
if destination_serverkey is None:
_log.info(
"Destination serverkey not found in known hosts file, using config"
)
destination_serverkey = config['destination-serverkey']
return DataMover(services_topic_list,
custom_topic_list,
topic_replace_list,
destination_vip,
destination_serverkey,
destination_historian_identity,
gather_timing_data,
backup_storage_limit_gb=backup_storage_limit_gb,
**kwargs)
def historian(config_path, **kwargs):
config = utils.load_config(config_path)
services_topic_list = config.get('services_topic_list', [
topics.DRIVER_TOPIC_BASE,
topics.LOGGER_BASE,
topics.ACTUATOR,
topics.ANALYSIS_TOPIC_BASE
])
custom_topic_list = config.get('custom_topic_list', [])
topic_replace_list = config.get('topic_replace_list', [])
destination_vip = config.get('destination-vip')
destination_historian_identity = config.get('destination-historian-identity',
'platform.historian')
backup_storage_limit_gb = config.get('backup_storage_limit_gb', None)
gather_timing_data = config.get('gather_timing_data', False)
hosts = KnownHostsStore()
destination_serverkey = hosts.serverkey(destination_vip)
if destination_serverkey is None:
_log.info("Destination serverkey not found in known hosts file, using config")
destination_serverkey = config['destination-serverkey']
return DataMover(services_topic_list,
custom_topic_list,
topic_replace_list,
destination_vip,
destination_serverkey,
destination_historian_identity,
gather_timing_data,
backup_storage_limit_gb=backup_storage_limit_gb,
**kwargs)
def historian(config_path, **kwargs):
config = utils.load_config(config_path)
custom_topic_list = config.pop('custom_topic_list', [])
topic_replace_list = config.pop('topic_replace_list', [])
destination_vip = config.pop('destination-vip', None)
service_topic_list = config.pop('service_topic_list', None)
destination_serverkey = None
try:
destination_address = config.pop('destination-address')
except KeyError:
destination_address = None
if service_topic_list is not None:
w = "Deprecated service_topic_list. Use capture_device_data " \
"capture_log_data, capture_analysis_data or capture_record_data " \
"instead!"
_log.warning(w)
# Populate the new values for the kwargs based upon the old data.
kwargs['capture_device_data'] = True if (
"device" in service_topic_list
or "all" in service_topic_list) else False
kwargs['capture_log_data'] = True if (
"datalogger" in service_topic_list
or "all" in service_topic_list) else False
kwargs['capture_record_data'] = True if (
"record" in service_topic_list
or "all" in service_topic_list) else False
kwargs['capture_analysis_data'] = True if (
"analysis" in service_topic_list
or "all" in service_topic_list) else False
if destination_vip:
hosts = KnownHostsStore()
destination_serverkey = hosts.serverkey(destination_vip)
if destination_serverkey is None:
_log.info(
"Destination serverkey not found in known hosts file, using config"
)
destination_serverkey = config.pop('destination-serverkey')
else:
config.pop('destination-serverkey', None)
destination_messagebus = 'zmq'
required_target_agents = config.pop('required_target_agents', [])
cache_only = config.pop('cache_only', False)
utils.update_kwargs_with_config(kwargs, config)
return ForwardHistorian(destination_vip,
destination_serverkey,
custom_topic_list=custom_topic_list,
topic_replace_list=topic_replace_list,
required_target_agents=required_target_agents,
cache_only=cache_only,
destination_address=destination_address,
**kwargs)
def get_keys():
"""Gets keys from keystore and known-hosts store
:returns: Keys for connecting to the platform
:rtype: dict
"""
hosts = KnownHostsStore()
serverkey = hosts.serverkey(get_address())
key_store = KeyStore()
publickey = key_store.public
secretkey = key_store.secret
return {'publickey': publickey,
'secretkey': secretkey,
'serverkey': serverkey}
def get_keys():
"""Gets keys from keystore and known-hosts store
:returns: Keys for connecting to the platform
:rtype: dict
"""
hosts = KnownHostsStore()
serverkey = hosts.serverkey(get_address())
key_store = KeyStore()
publickey = key_store.public
secretkey = key_store.secret
return {'publickey': publickey,
'secretkey': secretkey,
'serverkey': serverkey}
def historian(config_path, **kwargs):
config = utils.load_config(config_path)
destination_vip = config.get('destination-vip', None)
assert destination_vip is not None
hosts = KnownHostsStore()
destination_serverkey = hosts.serverkey(destination_vip)
if destination_serverkey is not None:
config['destination-serverkey'] = destination_serverkey
else:
assert config.get('destination-serverkey') is not None
_log.info("Destination serverkey not found in known hosts file, using config")
utils.update_kwargs_with_config(kwargs, config)
return DataMover(**kwargs)
def historian(config_path, **kwargs):
config = utils.load_config(config_path)
destination_vip = config.get('destination-vip', None)
assert destination_vip is not None
hosts = KnownHostsStore()
serverkey = hosts.serverkey(destination_vip)
if serverkey is not None:
config['destination-serverkey'] = serverkey
else:
assert config.get('destination-serverkey') is not None
_log.info("Destination serverkey not found in known hosts file, "
"using config")
utils.update_kwargs_with_config(kwargs, config)
return DataMover(**kwargs)
def __init__(self, config_path, **kwargs):
super(FailoverAgent, self).__init__(**kwargs)
config = utils.load_config(config_path)
# Get agent and remote ids
agent_id = config["agent_id"]
if agent_id == "primary":
self.agent_id = "primary"
self.remote_id = "secondary"
elif agent_id == "secondary":
self.agent_id = "secondary"
self.remote_id = "primary"
else:
_log.error("agent_id must be either 'primary' or 'secondary'")
# Modify ids if we're using the simple option
# Defaults to true pending vc coordination
use_simple = config.get("simple_behavior", True)
if use_simple:
self.agent_id = "simple_" + self.agent_id
self.remote_id = "simple_" + self.remote_id
self.remote_vip = config["remote_vip"]
hosts = KnownHostsStore()
self.remote_serverkey = hosts.serverkey(self.remote_vip)
if self.remote_serverkey is None:
self.remote_serverkey = config["remote_serverkey"]
self.agent_vip_identity = config["agent_vip_identity"]
self.heartbeat_period = config["heartbeat_period"]
self.timeout = config["timeout"]
self.vc_timeout = self.timeout
self.remote_timeout = self.timeout
self.agent_uuid = None
self.heartbeat = None
self.last_connected = None
self._state = True, True
self._state_machine = getattr(self, self.agent_id + '_state_machine')
def __init__(self, config_path, **kwargs):
super(FailoverAgent, self).__init__(**kwargs)
config = utils.load_config(config_path)
# Get agent and remote ids
agent_id = config["agent_id"]
if agent_id == "primary":
self.agent_id = "primary"
self.remote_id = "secondary"
elif agent_id == "secondary":
self.agent_id = "secondary"
self.remote_id = "primary"
else:
_log.error("agent_id must be either 'primary' or 'secondary'")
# Modify ids if we're using the simple option
# Defaults to true pending vc coordination
use_simple = config.get("simple_behavior", True)
if use_simple:
self.agent_id = "simple_" + self.agent_id
self.remote_id = "simple_" + self.remote_id
self.remote_vip = config["remote_vip"]
hosts = KnownHostsStore()
self.remote_serverkey = hosts.serverkey(self.remote_vip)
if self.remote_serverkey is None:
self.remote_serverkey = config["remote_serverkey"]
self.agent_vip_identity = config["agent_vip_identity"]
self.heartbeat_period = config["heartbeat_period"]
self.timeout = config["timeout"]
self.vc_timeout = self.timeout
self.remote_timeout = self.timeout
self.agent_uuid = None
self.heartbeat = None
self.last_connected = None
self._state = True, True
self._state_machine = getattr(self, self.agent_id + '_state_machine')
def historian(config_path, **kwargs):
config = utils.load_config(config_path)
custom_topic_list = config.pop('custom_topic_list', [])
topic_replace_list = config.pop('topic_replace_list', [])
destination_vip = config.pop('destination-vip', None)
service_topic_list = config.pop('service_topic_list', None)
if service_topic_list is not None:
w = "Deprecated service_topic_list. Use capture_device_data " \
"capture_log_data, capture_analysis_data or capture_record_data " \
"instead!"
_log.warning(w)
# Populate the new values for the kwargs based upon the old data.
kwargs['capture_device_data'] = True if ("device" in service_topic_list or "all" in service_topic_list) else False
kwargs['capture_log_data'] = True if ("datalogger" in service_topic_list or "all" in service_topic_list) else False
kwargs['capture_record_data'] = True if ("record" in service_topic_list or "all" in service_topic_list) else False
kwargs['capture_analysis_data'] = True if ("analysis" in service_topic_list or "all" in service_topic_list) else False
hosts = KnownHostsStore()
destination_serverkey = hosts.serverkey(destination_vip)
if destination_serverkey is None:
_log.info("Destination serverkey not found in known hosts file, using config")
destination_serverkey = config.pop('destination-serverkey')
else:
config.pop('destination-serverkey', None)
required_target_agents = config.pop('required_target_agents', [])
cache_only = config.pop('cache_only', False)
utils.update_kwargs_with_config(kwargs, config)
return ForwardHistorian(destination_vip, destination_serverkey,
custom_topic_list=custom_topic_list,
topic_replace_list=topic_replace_list,
required_target_agents=required_target_agents,
cache_only=cache_only,
**kwargs)
def connect_remote_platform(
self,
address,
serverkey=None,
agent_class=None
):
"""
Agent attempts to connect to a remote platform to exchange data.
address must start with http, https, tcp, ampq, or ampqs or a
ValueError will be
raised
If this function is successful it will return an instance of the
`agent_class`
parameter if not then this function will return None.
If the address parameter begins with http or https
TODO: use the known host functionality here
the agent will attempt to use Discovery to find the values
associated with it.
Discovery should return either an rmq-address or a vip-address or
both. In
that situation the connection will be made using zmq. In the event
that
fails then rmq will be tried. If both fail then None is returned
from this
function.
"""
from volttron.platform.vip.agent.utils import build_agent
from volttron.platform.vip.agent import Agent
if agent_class is None:
agent_class = Agent
parsed_address = urlparse(address)
_log.debug("Begining auth.connect_remote_platform: {}".format(address))
value = None
if parsed_address.scheme == "tcp":
# ZMQ connection
hosts = KnownHostsStore()
temp_serverkey = hosts.serverkey(address)
if not temp_serverkey:
_log.info(
"Destination serverkey not found in known hosts file, "
"using config"
)
destination_serverkey = serverkey
elif not serverkey:
destination_serverkey = temp_serverkey
else:
if temp_serverkey != serverkey:
raise ValueError(
"server_key passed and known hosts serverkey do not "
""
"match!"
)
destination_serverkey = serverkey
publickey, secretkey = (
self._core().publickey,
self._core().secretkey,
)
_log.debug(
"Connecting using: %s", get_fq_identity(self._core().identity)
)
value = build_agent(
agent_class=agent_class,
identity=get_fq_identity(self._core().identity),
serverkey=destination_serverkey,
publickey=publickey,
secretkey=secretkey,
message_bus="zmq",
address=address,
)
elif parsed_address.scheme in ("https", "http"):
from volttron.platform.web import DiscoveryInfo
from volttron.platform.web import DiscoveryError
try:
# TODO: Use known host instead of looking up for discovery
# info if possible.
# We need to discover which type of bus is at the other end.
info = DiscoveryInfo.request_discovery_info(address)
remote_identity = "{}.{}.{}".format(
info.instance_name,
get_platform_instance_name(),
self._core().identity,
)
# if the current message bus is zmq then we need
# to connect a zmq on the remote, whether that be the
# rmq router or proxy. Also note that we are using the
# fully qualified
# version of the identity because there will be conflicts if
# volttron central has more than one platform.agent connecting
if get_messagebus() == "zmq":
if not info.vip_address or not info.serverkey:
err = (
"Discovery from {} did not return serverkey "
"and/or vip_address".format(address)
)
raise ValueError(err)
_log.debug(
"Connecting using: %s",
get_fq_identity(self._core().identity),
)
# use fully qualified identity
value = build_agent(
identity=get_fq_identity(self._core().identity),
address=info.vip_address,
serverkey=info.serverkey,
secretkey=self._core().secretkey,
publickey=self._core().publickey,
agent_class=agent_class,
)
else: # we are on rmq messagebus
# This is if both remote and local are rmq message buses.
if info.messagebus_type == "rmq":
_log.debug("Both remote and local are rmq messagebus.")
fqid_local = get_fq_identity(self._core().identity)
# Check if we already have the cert, if so use it
# instead of requesting cert again
remote_certs_dir = self.get_remote_certs_dir()
remote_cert_name = "{}.{}".format(
info.instance_name, fqid_local
)
certfile = os.path.join(
remote_certs_dir, remote_cert_name + ".crt"
)
if os.path.exists(certfile):
response = certfile
else:
response = self.request_cert(
address, fqid_local, info
)
if response is None:
_log.error("there was no response from the server")
value = None
elif isinstance(response, tuple):
if response[0] == "PENDING":
_log.info(
"Waiting for administrator to accept a "
"CSR request."
)
value = None
# elif isinstance(response, dict):
# response
elif os.path.exists(response):
# info = DiscoveryInfo.request_discovery_info(
# address)
# From the remote platforms perspective the
# remote user name is
# remoteinstance.localinstance.identity,
# this is what we must
# pass to the build_remote_connection_params
# for a successful
remote_rmq_user = get_fq_identity(
fqid_local, info.instance_name
)
_log.debug(
"REMOTE RMQ USER IS: %s", remote_rmq_user
)
remote_rmq_address = self._core().rmq_mgmt.build_remote_connection_param(
remote_rmq_user,
info.rmq_address,
ssl_auth=True,
cert_dir=self.get_remote_certs_dir(),
)
value = build_agent(
identity=fqid_local,
address=remote_rmq_address,
instance_name=info.instance_name,
publickey=self._core().publickey,
secretkey=self._core().secretkey,
message_bus="rmq",
enable_store=False,
agent_class=agent_class,
)
else:
raise ValueError(
"Unknown path through discovery process!"
)
else:
# TODO: cache the connection so we don't always have
# to ping the server to connect.
# This branch happens when the message bus is not
# the same note
# this writes to the agent-data directory of this
# agent if the agent
# is installed.
if get_messagebus() == "rmq":
if not os.path.exists("keystore.json"):
with open("keystore.json", "w") as file_pointer:
file_pointer.write(
jsonapi.dumps(
KeyStore.generate_keypair_dict()
)
)
with open("keystore.json") as file_pointer:
keypair = jsonapi.loads(file_pointer.read())
value = build_agent(
agent_class=agent_class,
identity=remote_identity,
serverkey=info.serverkey,
publickey=keypair.get("publickey"),
secretkey=keypair.get("secretekey"),
message_bus="zmq",
address=info.vip_address,
)
except DiscoveryError:
_log.error(
"Couldn't connect to %s or incorrect response returned "
"response was %s",
address,
value,
)
else:
raise ValueError(
"Invalid configuration found the address: {} has an invalid "
"scheme".format(address)
)
return value
def _get_serverkey_from_known_hosts(self):
known_hosts_file = os.path.join(self.volttron_home, 'known_hosts')
known_hosts = KnownHostsStore(known_hosts_file)
return known_hosts.serverkey(self.address)
def historian(config_path, **kwargs):
config = utils.load_config(config_path)
services_topic_list = config.get('services_topic_list', ['all'])
custom_topic_list = config.get('custom_topic_list', [])
topic_replace_list = config.get('topic_replace_list', [])
destination_vip = config.get('destination-vip')
gather_timing_data = config.get('gather_timing_data', False)
hosts = KnownHostsStore()
destination_serverkey = hosts.serverkey(destination_vip)
if destination_serverkey is None:
_log.info("Destination serverkey not found in known hosts file, using config")
destination_serverkey = config['destination-serverkey']
required_target_agents = config.get('required_target_agents', [])
backup_storage_limit_gb = config.get('backup_storage_limit_gb', None)
if 'all' in services_topic_list:
services_topic_list = [topics.DRIVER_TOPIC_BASE, topics.LOGGER_BASE,
topics.ACTUATOR, topics.ANALYSIS_TOPIC_BASE]
class ForwardHistorian(BaseHistorian):
"""
This historian forwards data to another platform.
"""
def __init__(self, **kwargs):
# will be available in both threads.
self._topic_replace_map = {}
self._num_failures = 0
self._last_timeout = 0
self._target_platform = None
super(ForwardHistorian, self).__init__(**kwargs)
@Core.receiver("onstart")
def starting_base(self, sender, **kwargs):
"""
Subscribes to the platform message bus on the actuator, record,
datalogger, and device topics to capture data.
"""
def subscriber(subscription, callback_method):
_log.debug("subscribing to {}".format(subscription))
self.vip.pubsub.subscribe(peer='pubsub',
prefix=subscription,
callback=callback_method)
_log.debug("Starting Forward historian")
for topic_subscriptions in services_topic_list:
subscriber(topic_subscriptions, self.capture_data)
for custom_topic in custom_topic_list:
subscriber(custom_topic, self.capture_data)
self._started = True
def timestamp(self):
return time.mktime(datetime.datetime.now().timetuple())
def capture_data(self, peer, sender, bus, topic, headers, message):
# Grab the timestamp string from the message (we use this as the
# value in our readings at the end of this method)
_log.debug("In capture data")
timestamp_string = headers.get(headers_mod.DATE, None)
data = message
try:
# 2.0 agents compatability layer makes sender = pubsub.compat
# so we can do the proper thing when it is here
_log.debug("message in capture_data {}".format(message))
if sender == 'pubsub.compat':
# data = jsonapi.loads(message[0])
data = compat.unpack_legacy_message(headers, message)
_log.debug("data in capture_data {}".format(data))
if isinstance(data, dict):
data = data
elif isinstance(data, int) or \
isinstance(data, float) or \
isinstance(data, long):
data = data
# else:
# data = data[0]
except ValueError as e:
log_message = "message for {topic} bad message string:" \
"{message_string}"
_log.error(log_message.format(topic=topic,
message_string=message[0]))
raise
if topic_replace_list:
if topic in self._topic_replace_map.keys():
topic = self._topic_replace_map[topic]
else:
self._topic_replace_map[topic] = topic
temptopics = {}
for x in topic_replace_list:
if x['from'] in topic:
new_topic = temptopics.get(topic, topic)
temptopics[topic] = new_topic.replace(
x['from'], x['to'])
for k, v in temptopics.items():
self._topic_replace_map[k] = v
topic = self._topic_replace_map[topic]
if gather_timing_data:
add_timing_data_to_header(headers, self.core.agent_uuid or self.core.identity, "collected")
payload = {'headers': headers, 'message': data}
self._event_queue.put({'source': "forwarded",
'topic': topic,
'readings': [(timestamp_string, payload)]})
@doc_inherit
def publish_to_historian(self, to_publish_list):
handled_records = []
_log.debug("publish_to_historian number of items: {}"
.format(len(to_publish_list)))
parsed = urlparse(self.core.address)
next_dest = urlparse(destination_vip)
current_time = self.timestamp()
last_time = self._last_timeout
_log.debug('Lasttime: {} currenttime: {}'.format(last_time,
current_time))
timeout_occurred = False
if self._last_timeout:
# if we failed we need to wait 60 seconds before we go on.
if self.timestamp() < self._last_timeout + 60:
_log.debug('Not allowing send < 60 seconds from failure')
return
if not self._target_platform:
self.historian_setup()
if not self._target_platform:
_log.debug('Could not connect to target')
return
for vip_id in required_target_agents:
try:
self._target_platform.vip.ping(vip_id).get()
except Unreachable:
skip = "Skipping publish: Target platform not running " \
"required agent {}".format(vip_id)
_log.warn(skip)
self.vip.health.set_status(
STATUS_BAD, skip)
return
except Exception as e:
err = "Unhandled error publishing to target platform."
_log.error(err)
_log.error(traceback.format_exc())
self.vip.health.set_status(
STATUS_BAD, err)
return
for x in to_publish_list:
topic = x['topic']
value = x['value']
# payload = jsonapi.loads(value)
payload = value
headers = payload['headers']
headers['X-Forwarded'] = True
try:
del headers['Origin']
except KeyError:
pass
try:
del headers['Destination']
except KeyError:
pass
if gather_timing_data:
add_timing_data_to_header(headers, self.core.agent_uuid or self.core.identity,"forwarded")
if timeout_occurred:
_log.error(
'A timeout has occurred so breaking out of publishing')
break
with gevent.Timeout(30):
try:
_log.debug('debugger: {} {} {}'.format(topic,
headers,
payload))
self._target_platform.vip.pubsub.publish(
peer='pubsub',
topic=topic,
headers=headers,
message=payload['message']).get()
except gevent.Timeout:
_log.debug("Timeout occurred email should send!")
timeout_occurred = True
self._last_timeout = self.timestamp()
self._num_failures += 1
# Stop the current platform from attempting to
# connect
self._target_platform.core.stop()
self._target_platform = None
self.vip.health.set_status(
STATUS_BAD, "Timeout occured")
except Unreachable:
_log.error("Target not reachable. Wait till it's ready!")
except ZMQError as exc:
if exc.errno == ENOTSOCK:
# Stop the current platform from attempting to
# connect
_log.error("Target disconnected. Stopping target platform agent")
self._target_platform = None
self.vip.health.set_status(
STATUS_BAD, "Target platform disconnected")
except Exception as e:
err = "Unhandled error publishing to target platfom."
_log.error(err)
_log.error(traceback.format_exc())
self.vip.health.set_status(
STATUS_BAD, err)
# Before returning lets mark any that weren't errors
# as sent.
self.report_handled(handled_records)
return
else:
handled_records.append(x)
_log.debug("handled: {} number of items".format(
len(to_publish_list)))
self.report_handled(handled_records)
if timeout_occurred:
_log.debug('Sending alert from the ForwardHistorian')
status = Status.from_json(self.vip.health.get_status())
self.vip.health.send_alert(FORWARD_TIMEOUT_KEY,
status)
else:
self.vip.health.set_status(
STATUS_GOOD,"published {} items".format(
len(to_publish_list)))
@doc_inherit
def historian_setup(self):
_log.debug("Setting up to forward to {}".format(destination_vip))
try:
agent = build_agent(address=destination_vip,
serverkey=destination_serverkey,
publickey=self.core.publickey,
secretkey=self.core.secretkey,
enable_store=False)
except gevent.Timeout:
self.vip.health.set_status(
STATUS_BAD, "Timeout in setup of agent")
status = Status.from_json(self.vip.health.get_status_json())
self.vip.health.send_alert(FORWARD_TIMEOUT_KEY,
status)
else:
self._target_platform = agent
return ForwardHistorian(backup_storage_limit_gb=backup_storage_limit_gb,
**kwargs)
def _get_serverkey_from_known_hosts(self):
known_hosts_file = os.path.join(self.volttron_home, 'known_hosts')
known_hosts = KnownHostsStore(known_hosts_file)
return known_hosts.serverkey(self.address)
def historian(config_path, **kwargs):
config = utils.load_config(config_path)
services_topic_list = config.get('services_topic_list', ['all'])
custom_topic_list = config.get('custom_topic_list', [])
topic_replace_list = config.get('topic_replace_list', [])
destination_vip = config.get('destination-vip')
gather_timing_data = config.get('gather_timing_data', False)
hosts = KnownHostsStore()
destination_serverkey = hosts.serverkey(destination_vip)
if destination_serverkey is None:
_log.info(
"Destination serverkey not found in known hosts file, using config"
)
destination_serverkey = config['destination-serverkey']
required_target_agents = config.get('required_target_agents', [])
backup_storage_limit_gb = config.get('backup_storage_limit_gb', None)
if 'all' in services_topic_list:
services_topic_list = [
topics.DRIVER_TOPIC_BASE, topics.LOGGER_BASE, topics.ACTUATOR,
topics.ANALYSIS_TOPIC_BASE
]
class ForwardHistorian(BaseHistorian):
"""
This historian forwards data to another platform.
"""
def __init__(self, **kwargs):
# will be available in both threads.
self._topic_replace_map = {}
self._num_failures = 0
self._last_timeout = 0
self._target_platform = None
super(ForwardHistorian, self).__init__(**kwargs)
@Core.receiver("onstart")
def starting_base(self, sender, **kwargs):
"""
Subscribes to the platform message bus on the actuator, record,
datalogger, and device topics to capture data.
"""
def subscriber(subscription, callback_method):
_log.debug("subscribing to {}".format(subscription))
self.vip.pubsub.subscribe(peer='pubsub',
prefix=subscription,
callback=callback_method)
_log.debug("Starting Forward historian")
for topic_subscriptions in services_topic_list:
subscriber(topic_subscriptions, self.capture_data)
for custom_topic in custom_topic_list:
subscriber(custom_topic, self.capture_data)
self._started = True
def timestamp(self):
return time.mktime(datetime.datetime.now().timetuple())
def capture_data(self, peer, sender, bus, topic, headers, message):
# Grab the timestamp string from the message (we use this as the
# value in our readings at the end of this method)
_log.debug("In capture data")
timestamp_string = headers.get(headers_mod.DATE, None)
data = message
try:
# 2.0 agents compatability layer makes sender = pubsub.compat
# so we can do the proper thing when it is here
_log.debug("message in capture_data {}".format(message))
if sender == 'pubsub.compat':
# data = jsonapi.loads(message[0])
data = compat.unpack_legacy_message(headers, message)
_log.debug("data in capture_data {}".format(data))
if isinstance(data, dict):
data = data
elif isinstance(data, int) or \
isinstance(data, float) or \
isinstance(data, long):
data = data
# else:
# data = data[0]
except ValueError as e:
log_message = "message for {topic} bad message string:" \
"{message_string}"
_log.error(
log_message.format(topic=topic, message_string=message[0]))
raise
if topic_replace_list:
if topic in self._topic_replace_map.keys():
topic = self._topic_replace_map[topic]
else:
self._topic_replace_map[topic] = topic
temptopics = {}
for x in topic_replace_list:
if x['from'] in topic:
new_topic = temptopics.get(topic, topic)
temptopics[topic] = new_topic.replace(
x['from'], x['to'])
for k, v in temptopics.items():
self._topic_replace_map[k] = v
topic = self._topic_replace_map[topic]
if gather_timing_data:
add_timing_data_to_header(
headers, self.core.agent_uuid or self.core.identity,
"collected")
payload = {'headers': headers, 'message': data}
self._event_queue.put({
'source': "forwarded",
'topic': topic,
'readings': [(timestamp_string, payload)]
})
@doc_inherit
def publish_to_historian(self, to_publish_list):
handled_records = []
_log.debug("publish_to_historian number of items: {}".format(
len(to_publish_list)))
parsed = urlparse(self.core.address)
next_dest = urlparse(destination_vip)
current_time = self.timestamp()
last_time = self._last_timeout
_log.debug('Lasttime: {} currenttime: {}'.format(
last_time, current_time))
timeout_occurred = False
if self._last_timeout:
# if we failed we need to wait 60 seconds before we go on.
if self.timestamp() < self._last_timeout + 60:
_log.debug('Not allowing send < 60 seconds from failure')
return
if not self._target_platform:
self.historian_setup()
if not self._target_platform:
_log.debug('Could not connect to target')
return
for vip_id in required_target_agents:
try:
self._target_platform.vip.ping(vip_id).get()
except Unreachable:
skip = "Skipping publish: Target platform not running " \
"required agent {}".format(vip_id)
_log.warn(skip)
self.vip.health.set_status(STATUS_BAD, skip)
return
except Exception as e:
err = "Unhandled error publishing to target platform."
_log.error(err)
_log.error(traceback.format_exc())
self.vip.health.set_status(STATUS_BAD, err)
return
for x in to_publish_list:
topic = x['topic']
value = x['value']
# payload = jsonapi.loads(value)
payload = value
headers = payload['headers']
headers['X-Forwarded'] = True
try:
del headers['Origin']
except KeyError:
pass
try:
del headers['Destination']
except KeyError:
pass
if gather_timing_data:
add_timing_data_to_header(
headers, self.core.agent_uuid or self.core.identity,
"forwarded")
if timeout_occurred:
_log.error(
'A timeout has occurred so breaking out of publishing')
break
with gevent.Timeout(30):
try:
_log.debug('debugger: {} {} {}'.format(
topic, headers, payload))
self._target_platform.vip.pubsub.publish(
peer='pubsub',
topic=topic,
headers=headers,
message=payload['message']).get()
except gevent.Timeout:
_log.debug("Timeout occurred email should send!")
timeout_occurred = True
self._last_timeout = self.timestamp()
self._num_failures += 1
# Stop the current platform from attempting to
# connect
self._target_platform.core.stop()
self._target_platform = None
self.vip.health.set_status(STATUS_BAD,
"Timeout occured")
except Unreachable:
_log.error(
"Target not reachable. Wait till it's ready!")
except ZMQError as exc:
if exc.errno == ENOTSOCK:
# Stop the current platform from attempting to
# connect
_log.error(
"Target disconnected. Stopping target platform agent"
)
self._target_platform = None
self.vip.health.set_status(
STATUS_BAD, "Target platform disconnected")
except Exception as e:
err = "Unhandled error publishing to target platfom."
_log.error(err)
_log.error(traceback.format_exc())
self.vip.health.set_status(STATUS_BAD, err)
# Before returning lets mark any that weren't errors
# as sent.
self.report_handled(handled_records)
return
else:
handled_records.append(x)
_log.debug("handled: {} number of items".format(
len(to_publish_list)))
self.report_handled(handled_records)
if timeout_occurred:
_log.debug('Sending alert from the ForwardHistorian')
status = Status.from_json(self.vip.health.get_status())
self.vip.health.send_alert(FORWARD_TIMEOUT_KEY, status)
else:
self.vip.health.set_status(
STATUS_GOOD,
"published {} items".format(len(to_publish_list)))
@doc_inherit
def historian_setup(self):
_log.debug("Setting up to forward to {}".format(destination_vip))
try:
agent = build_agent(address=destination_vip,
serverkey=destination_serverkey,
publickey=self.core.publickey,
secretkey=self.core.secretkey,
enable_store=False)
except gevent.Timeout:
self.vip.health.set_status(STATUS_BAD,
"Timeout in setup of agent")
status = Status.from_json(self.vip.health.get_status_json())
self.vip.health.send_alert(FORWARD_TIMEOUT_KEY, status)
else:
self._target_platform = agent
return ForwardHistorian(backup_storage_limit_gb=backup_storage_limit_gb,
**kwargs)
def historian(config_path, **kwargs):
config = utils.load_config(config_path)
services_topic_list = config.get('services_topic_list', ['all'])
custom_topic_list = config.get('custom_topic_list', [])
topic_replace_list = config.get('topic_replace_list', [])
source_vip = config.get('source-vip')
hosts = KnownHostsStore()
source_serverkey = hosts.serverkey(source_vip)
if source_serverkey is None:
_log.info(
"Source serverkey not found in known hosts file, using config")
source_serverkey = config['source-serverkey']
identity = config.get('identity', kwargs.pop('identity', None))
include_destination_in_header = config.get('include_destination_in_header',
False)
origin = config.get('origin', None)
overwrite_origin = config.get('overwrite_origin', False)
include_origin_in_header = config.get('include_origin_in_header', False)
if 'all' in services_topic_list:
services_topic_list = [
topics.DRIVER_TOPIC_BASE, topics.LOGGER_BASE, topics.ACTUATOR,
topics.ANALYSIS_TOPIC_BASE
]
class DataPuller(Agent):
'''This historian pulls data from another platform.
'''
def __init__(self, **kwargs):
# will be available in both threads.
self._topic_replace_map = {}
self._num_failures = 0
self._last_timeout = 0
self._target_platform = None
super(DataPuller, self).__init__(**kwargs)
@Core.receiver("onstart")
def starting_base(self, sender, **kwargs):
self.puller_setup()
'''
Subscribes to the platform message bus on the actuator, record,
datalogger, and device topics to capture data.
'''
def subscriber(subscription, callback_method):
_log.debug("subscribing to {}".format(subscription))
self._target_platform.vip.pubsub.subscribe(
peer='pubsub',
prefix=subscription,
callback=callback_method)
_log.debug("Starting DataPuller")
for topic_subscriptions in services_topic_list:
subscriber(topic_subscriptions, self.on_message)
for custom_topic in custom_topic_list:
subscriber(custom_topic, self.on_message)
self._started = True
def timestamp(self):
return time.mktime(datetime.datetime.now().timetuple())
def on_message(self, peer, sender, bus, topic, headers, message):
# Grab the timestamp string from the message (we use this as the
# value in our readings at the end of this method)
_log.debug("In capture data")
timestamp_string = headers.get(headers_mod.DATE, None)
data = message
try:
# 2.0 agents compatability layer makes sender = pubsub.compat
# so we can do the proper thing when it is here
_log.debug("message in capture_data {}".format(message))
if sender == 'pubsub.compat':
# data = jsonapi.loads(message[0])
data = compat.unpack_legacy_message(headers, message)
_log.debug("data in capture_data {}".format(data))
if isinstance(data, dict):
data = data
elif isinstance(data, int) or \
isinstance(data, float) or \
isinstance(data, long):
data = data
# else:
# data = data[0]
except ValueError as e:
log_message = "message for {topic} bad message string:" \
"{message_string}"
_log.error(
log_message.format(topic=topic, message_string=message[0]))
raise
if topic_replace_list:
if topic in self._topic_replace_map.keys():
topic = self._topic_replace_map[topic]
else:
self._topic_replace_map[topic] = topic
temptopics = {}
for x in topic_replace_list:
if x['from'] in topic:
new_topic = temptopics.get(topic, topic)
temptopics[topic] = new_topic.replace(
x['from'], x['to'])
for k, v in temptopics.items():
self._topic_replace_map[k] = v
topic = self._topic_replace_map[topic]
payload = {'headers': headers, 'topic': topic, 'message': data}
self.publish_to_self(topic, payload)
# self._event_queue.put({'source': "forwarded",
# 'topic': topic,
# 'readings': [(timestamp_string, payload)]})
def publish_to_self(self, topic, payload):
handled_records = []
parsed = urlparse(self.core.address)
next_dest = urlparse(source_vip)
current_time = self.timestamp()
last_time = self._last_timeout
_log.debug('Lasttime: {} currenttime: {}'.format(
last_time, current_time))
timeout_occurred = False
if self._last_timeout:
# if we failed we need to wait 60 seconds before we go on.
if self.timestamp() < self._last_timeout + 60:
_log.debug('Not allowing send < 60 seconds from failure')
return
if not self._target_platform:
self.puller_setup()
if not self._target_platform:
_log.debug('Could not connect to target')
return
headers = payload['headers']
headers['X-Forwarded'] = True
try:
del headers['Origin']
except KeyError:
pass
try:
del headers['Destination']
except KeyError:
pass
print("should publish", topic, payload)
self.vip.pubsub.publish(peer='pubsub',
topic=topic,
headers=headers,
message=payload['message']).get(30)
def puller_setup(self):
_log.debug("Setting up to forward to {}".format(source_vip))
try:
agent = build_agent(address=source_vip,
serverkey=source_serverkey,
publickey=self.core.publickey,
secretkey=self.core.secretkey,
enable_store=False)
except gevent.Timeout:
self.vip.health.set_status(STATUS_BAD,
"Timeout in setup of agent")
status = Status.from_json(self.vip.health.get_status_json())
self.vip.health.send_alert(FORWARD_TIMEOUT_KEY, status)
else:
self._target_platform = agent
DataPuller.__name__ = 'DataPuller'
return DataPuller(identity=identity, **kwargs)
def historian(config_path, **kwargs):
config = utils.load_config(config_path)
services_topic_list = config.get('services_topic_list', ['all'])
custom_topic_list = config.get('custom_topic_list', [])
topic_replace_list = config.get('topic_replace_list', [])
source_vip = config.get('source-vip')
hosts = KnownHostsStore()
source_serverkey = hosts.serverkey(source_vip)
if source_serverkey is None:
_log.info("Source serverkey not found in known hosts file, using config")
source_serverkey = config['source-serverkey']
identity = config.get('identity', kwargs.pop('identity', None))
include_destination_in_header = config.get('include_destination_in_header',
False)
origin = config.get('origin', None)
overwrite_origin = config.get('overwrite_origin', False)
include_origin_in_header = config.get('include_origin_in_header', False)
if 'all' in services_topic_list:
services_topic_list = [topics.DRIVER_TOPIC_BASE, topics.LOGGER_BASE,
topics.ACTUATOR, topics.ANALYSIS_TOPIC_BASE]
class DataPuller(Agent):
'''This historian pulls data from another platform.
'''
def __init__(self, **kwargs):
# will be available in both threads.
self._topic_replace_map = {}
self._num_failures = 0
self._last_timeout = 0
self._target_platform = None
super(DataPuller, self).__init__(**kwargs)
@Core.receiver("onstart")
def starting_base(self, sender, **kwargs):
self.puller_setup()
'''
Subscribes to the platform message bus on the actuator, record,
datalogger, and device topics to capture data.
'''
def subscriber(subscription, callback_method):
_log.debug("subscribing to {}".format(subscription))
self._target_platform.vip.pubsub.subscribe(peer='pubsub',
prefix=subscription,
callback=callback_method)
_log.debug("Starting DataPuller")
for topic_subscriptions in services_topic_list:
subscriber(topic_subscriptions, self.on_message)
for custom_topic in custom_topic_list:
subscriber(custom_topic, self.on_message)
self._started = True
def timestamp(self):
return time.mktime(datetime.datetime.now().timetuple())
def on_message(self, peer, sender, bus, topic, headers, message):
# Grab the timestamp string from the message (we use this as the
# value in our readings at the end of this method)
_log.debug("In capture data")
timestamp_string = headers.get(headers_mod.DATE, None)
data = message
try:
# 2.0 agents compatability layer makes sender = pubsub.compat
# so we can do the proper thing when it is here
_log.debug("message in capture_data {}".format(message))
if sender == 'pubsub.compat':
# data = jsonapi.loads(message[0])
data = compat.unpack_legacy_message(headers, message)
_log.debug("data in capture_data {}".format(data))
if isinstance(data, dict):
data = data
elif isinstance(data, int) or \
isinstance(data, float) or \
isinstance(data, long):
data = data
# else:
# data = data[0]
except ValueError as e:
log_message = "message for {topic} bad message string:" \
"{message_string}"
_log.error(log_message.format(topic=topic,
message_string=message[0]))
raise
if topic_replace_list:
if topic in self._topic_replace_map.keys():
topic = self._topic_replace_map[topic]
else:
self._topic_replace_map[topic] = topic
temptopics = {}
for x in topic_replace_list:
if x['from'] in topic:
new_topic = temptopics.get(topic, topic)
temptopics[topic] = new_topic.replace(
x['from'], x['to'])
for k, v in temptopics.items():
self._topic_replace_map[k] = v
topic = self._topic_replace_map[topic]
payload = {'headers': headers, 'topic': topic, 'message': data}
self.publish_to_self(topic, payload)
# self._event_queue.put({'source': "forwarded",
# 'topic': topic,
# 'readings': [(timestamp_string, payload)]})
def publish_to_self(self, topic, payload):
handled_records = []
parsed = urlparse(self.core.address)
next_dest = urlparse(source_vip)
current_time = self.timestamp()
last_time = self._last_timeout
_log.debug('Lasttime: {} currenttime: {}'.format(last_time, current_time))
timeout_occurred = False
if self._last_timeout:
# if we failed we need to wait 60 seconds before we go on.
if self.timestamp() < self._last_timeout + 60:
_log.debug('Not allowing send < 60 seconds from failure')
return
if not self._target_platform:
self.puller_setup()
if not self._target_platform:
_log.debug('Could not connect to target')
return
headers = payload['headers']
headers['X-Forwarded'] = True
try:
del headers['Origin']
except KeyError:
pass
try:
del headers['Destination']
except KeyError:
pass
print("should publish", topic, payload)
self.vip.pubsub.publish(
peer='pubsub',
topic=topic,
headers=headers,
message=payload['message']).get(30)
def puller_setup(self):
_log.debug("Setting up to forward to {}".format(source_vip))
try:
agent = build_agent(address=source_vip,
serverkey=source_serverkey,
publickey=self.core.publickey,
secretkey=self.core.secretkey,
enable_store=False)
except gevent.Timeout:
self.vip.health.set_status(
STATUS_BAD, "Timeout in setup of agent")
status = Status.from_json(self.vip.health.get_status_json())
self.vip.health.send_alert(FORWARD_TIMEOUT_KEY,
status)
else:
self._target_platform = agent
DataPuller.__name__ = 'DataPuller'
return DataPuller(identity=identity,
**kwargs)