def historian_setup(self):
if self.rmq_to_rmq_comm:
_log.debug("Setting up to forward to {}".format(
self.destination_instance_name))
self._target_platform = None
else:
_log.debug("Setting up to forward to {}".format(
self.destination_vip))
try:
agent = build_agent(
address=self.destination_vip,
serverkey=self.destination_serverkey,
publickey=self.core.publickey,
secretkey=self.core.secretkey,
enable_store=False,
identity=self.remote_identity,
instance_name=self.destination_instance_name)
except gevent.Timeout:
self.vip.health.set_status(STATUS_BAD,
"Timeout in setup of agent")
try:
status = Status.from_json(self.vip.health.get_status())
self.vip.health.send_alert(DATAMOVER_TIMEOUT_KEY, status)
except KeyError:
_log.error("Error getting the health status")
else:
self._target_platform = agent
def setup(self):
"""
Read resource files and load list of valid tags, categories,
tags grouped by categories, list of reference tags and its parent.
:return:
"""
_log.debug("Setup of mongodb tagging agent")
err_message = ""
collections = []
db = None
try:
db = self._client.get_default_database()
collections = db.collection_names(include_system_collections=False)
_log.debug(collections)
except Exception as e:
err_message = "Unable to query list of existing tables from the " \
"database. Exception in init of tagging service: {}. " \
"Stopping tagging service agent".format(e.args)
collection = ""
try:
collection = self.tags_collection
if self.tags_collection in collections:
_log.info("{} collection exists. Assuming initial values have "
"been loaded".format(collection))
else:
self._init_tags(db)
self._init_category_tags(db)
collection = self.tag_refs_collection
if self.tag_refs_collection in collections:
_log.info("{} collection exists. Assuming initial values have "
"been loaded".format(collection))
else:
self._init_tag_refs(db)
collection = self.categories_collection
if self.categories_collection in collections:
_log.info("{} collection exists. Assuming initial values "
"have been loaded".format(collection))
else:
self._init_categories(db)
except Exception as e:
err_message = "Initialization of " + collection + \
" collection failed with exception: {}" \
"Stopping tagging service agent. ".format(e.args)
if err_message:
_log.error(err_message)
self.vip.health.set_status(
STATUS_BAD, "Initialization of tagging service "
"failed")
status = Status.from_json(self.vip.health.get_status_json())
# status.context = status.context + \
# " Exception: {}".format(e.args) + \
# " Stopping tagging service agent"
# _log.debug("status:{}".format(status))
self.vip.health.send_alert(TAGGING_SERVICE_SETUP_FAILED, status)
self.core.stop()
def setup(self):
"""
Read resource files and load list of valid tags, categories,
tags grouped by categories, list of reference tags and its parent.
:return:
"""
_log.debug("Setup of mongodb tagging agent")
err_message = ""
collections = []
db = None
try:
db = self._client.get_default_database()
collections = db.collection_names(include_system_collections=False)
_log.debug(collections)
except Exception as e:
err_message = "Unable to query list of existing tables from the " \
"database. Exception in init of tagging service: {}. " \
"Stopping tagging service agent".format(e.args)
collection = ""
try:
collection = self.tags_collection
if self.tags_collection in collections:
_log.info("{} collection exists. Assuming initial values have "
"been loaded".format(collection))
else:
self._init_tags(db)
self._init_category_tags(db)
collection = self.tag_refs_collection
if self.tag_refs_collection in collections:
_log.info("{} collection exists. Assuming initial values have "
"been loaded".format(collection))
else:
self._init_tag_refs(db)
collection = self.categories_collection
if self.categories_collection in collections:
_log.info("{} collection exists. Assuming initial values "
"have been loaded".format(collection))
else:
self._init_categories(db)
except Exception as e:
err_message = "Initialization of " + collection + \
" collection failed with exception: {}" \
"Stopping tagging service agent. ".format(e.args)
if err_message:
_log.error(err_message)
self.vip.health.set_status(STATUS_BAD,
"Initialization of tagging service "
"failed")
status = Status.from_json(self.vip.health.get_status_json())
# status.context = status.context + \
# " Exception: {}".format(e.args) + \
# " Stopping tagging service agent"
# _log.debug("status:{}".format(status))
self.vip.health.send_alert(TAGGING_SERVICE_SETUP_FAILED, status)
self.core.stop()
def historian_setup(self):
_log.debug("Setting up to forward to {}".format(self.destination_vip))
try:
agent = build_agent(address=self.destination_vip,
serverkey=self.destination_serverkey,
publickey=self.core.publickey,
secretkey=self.core.secretkey,
enable_store=False)
except gevent.Timeout:
self.vip.health.set_status(STATUS_BAD, "Timeout in setup of agent")
status = Status.from_json(self.vip.health.get_status())
self.vip.health.send_alert(DATAMOVER_TIMEOUT_KEY, status)
else:
self._target_platform = agent
def historian_setup(self):
try:
_log.debug(
"Setting up to forward to {}".format(destination_vip))
event = gevent.event.Event()
agent = Agent(address=destination_vip, enable_store=False)
agent.core.onstart.connect(lambda *a, **kw: event.set(), event)
gevent.spawn(agent.core.run)
event.wait(timeout=10)
self._target_platform = agent
except gevent.Timeout:
self.vip.health.set_status(STATUS_BAD,
"Timeout in setup of agent")
status = Status.from_json(self.vip.health.get_status())
self.vip.health.send_alert(FORWARD_TIMEOUT_KEY, status)
def historian_setup(self):
_log.debug("Setting up to forward to {}".format(self.destination_vip))
try:
agent = build_agent(address=self.destination_vip,
serverkey=self.destination_serverkey,
publickey=self.core.publickey,
secretkey=self.core.secretkey,
enable_store=False)
except gevent.Timeout:
self.vip.health.set_status(
STATUS_BAD, "Timeout in setup of agent")
status = Status.from_json(self.vip.health.get_status())
self.vip.health.send_alert(DATAMOVER_TIMEOUT_KEY,
status)
else:
self._target_platform = agent
def historian_setup(self):
try:
_log.debug(
"Setting up to forward to {}".format(destination_vip))
event = gevent.event.Event()
agent = Agent(address=destination_vip)
agent.core.onstart.connect(lambda *a, **kw: event.set(),
event)
gevent.spawn(agent.core.run)
event.wait(timeout=10)
self._target_platform = agent
except gevent.Timeout:
self.vip.health.set_status(
STATUS_BAD, "Timeout in setup of agent")
status = Status.from_json(self.vip.health.get_status())
self.vip.health.send_alert(FORWARD_TIMEOUT_KEY,
status)
def publish_to_historian(self, to_publish_list):
handled_records = []
_log.debug("publish_to_historian number of items: {}"
.format(len(to_publish_list)))
parsed = urlparse(self.core.address)
next_dest = urlparse(destination_vip)
current_time = self.timestamp()
last_time = self._last_timeout
_log.debug('Lasttime: {} currenttime: {}'.format(last_time,
current_time))
timeout_occurred = False
if self._last_timeout:
# if we failed we need to wait 60 seconds before we go on.
if self.timestamp() < self._last_timeout + 60:
_log.debug('Not allowing send < 60 seconds from failure')
return
if not self._target_platform:
self.historian_setup()
if not self._target_platform:
_log.debug('Could not connect to target')
return
for x in to_publish_list:
topic = x['topic']
value = x['value']
# payload = jsonapi.loads(value)
payload = value
headers = payload['headers']
headers['X-Forwarded'] = True
try:
del headers['Origin']
except KeyError:
pass
try:
del headers['Destination']
except KeyError:
pass
# if not headers.get('Origin', None)
# if overwrite_origin:
# if not include_origin_in_header:
# try:
# del headers['Origin']
# except KeyError:
# pass
# else:
# headers['Origin'] = origin
# else:
# headers['Origin'] = parsed.hostname
# headers['Destination'] = [next_dest.scheme +
# '://'+
# next_dest.hostname]
# else:
# headers['Destination'].append(next_dest.hostname)
if timeout_occurred:
_log.error(
'A timeout has occured so breaking out of publishing')
break
with gevent.Timeout(30):
try:
_log.debug('debugger: {} {} {}'.format(topic,
headers,
payload))
self._target_platform.vip.pubsub.publish(
peer='pubsub',
topic=topic,
headers=headers,
message=payload['message']).get()
except gevent.Timeout:
_log.debug("Timout occurred email should send!")
timeout_occurred = True
self._last_timeout = self.timestamp()
self._num_failures += 1
# Stop the current platform from attempting to
# connect
self._target_platform.core.stop()
self._target_platform = None
self.vip.health.set_status(
STATUS_BAD, "Timout occured")
except Exception as e:
_log.error(e)
else:
handled_records.append(x)
_log.debug("handled: {} number of items".format(
len(to_publish_list)))
self.report_handled(handled_records)
if timeout_occurred:
_log.debug('Sending alert from the ForwardHistorian')
status = Status.from_json(self.vip.health.get_status())
self.vip.health.send_alert(FORWARD_TIMEOUT_KEY,
status)
def setup(self):
"""
Read resource files and load list of valid tags, categories,
tags grouped by categories, list of reference tags and its
parent.
:return:
"""
_log.debug("Setup of sqlite tagging agent")
err_message = ""
table_names = []
try:
stmt = "SELECT name FROM sqlite_master " \
"WHERE type='table';"
name_list = self.sqlite_utils.select(stmt, None, fetch_all=True)
table_names = [name[0] for name in name_list]
_log.debug(table_names)
except Exception as e:
err_message = "Unable to query list of existing tables from the " \
"database. Exception: {}. Stopping tagging " \
"service agent".format(e.args)
table_name = ""
try:
table_name = self.tags_table
if self.tags_table in table_names:
_log.info("{} table exists. Assuming initial values have been "
"loaded".format(table_name))
else:
self._init_tags()
table_name = self.tag_refs_table
if self.tag_refs_table in table_names:
_log.info("{} table exists. Assuming initial values have been "
"loaded".format(table_name))
else:
self._init_tag_refs()
table_name = self.topic_tags_table
if self.topic_tags_table in table_names:
_log.info("{} table exists. Assuming initial values "
"have been loaded".format(table_name))
else:
self._init_topic_tags()
table_name = self.categories_table
if self.categories_table in table_names:
_log.info("{} table exists. Assuming initial values "
"have been loaded".format(table_name))
else:
self._init_categories()
table_name = self.category_tags_table
if self.category_tags_table in table_names:
_log.info("{} table exists. Assuming initial values "
"have been loaded".format(table_name))
else:
self._init_category_tags()
except Exception as e:
err_message = "Initialization of " + table_name + \
" table failed with exception: {}" \
"Stopping tagging service agent. ".format(str(e))
if err_message:
_log.error(err_message)
self.vip.health.set_status(
STATUS_BAD, "Initialization of tagging service "
"failed")
status = Status.from_json(self.vip.health.get_status_json())
self.vip.health.send_alert(TAGGING_SERVICE_SETUP_FAILED, status)
self.core.stop()
def publish_to_historian(self, to_publish_list):
handled_records = []
_log.debug("publish_to_historian number of items: {}"
.format(len(to_publish_list)))
parsed = urlparse(self.core.address)
next_dest = urlparse(destination_vip)
current_time = self.timestamp()
last_time = self._last_timeout
_log.debug('Lasttime: {} currenttime: {}'.format(last_time,
current_time))
timeout_occurred = False
if self._last_timeout:
# if we failed we need to wait 60 seconds before we go on.
if self.timestamp() < self._last_timeout + 60:
_log.debug('Not allowing send < 60 seconds from failure')
return
if not self._target_platform:
self.historian_setup()
if not self._target_platform:
_log.debug('Could not connect to target')
return
for vip_id in required_target_agents:
try:
self._target_platform.vip.ping(vip_id).get()
except Unreachable:
skip = "Skipping publish: Target platform not running " \
"required agent {}".format(vip_id)
_log.warn(skip)
self.vip.health.set_status(
STATUS_BAD, skip)
return
except Exception as e:
err = "Unhandled error publishing to target platform."
_log.error(err)
_log.error(traceback.format_exc())
self.vip.health.set_status(
STATUS_BAD, err)
return
for x in to_publish_list:
topic = x['topic']
value = x['value']
# payload = jsonapi.loads(value)
payload = value
headers = payload['headers']
headers['X-Forwarded'] = True
try:
del headers['Origin']
except KeyError:
pass
try:
del headers['Destination']
except KeyError:
pass
if gather_timing_data:
add_timing_data_to_header(headers, self.core.agent_uuid or self.core.identity,"forwarded")
if timeout_occurred:
_log.error(
'A timeout has occurred so breaking out of publishing')
break
with gevent.Timeout(30):
try:
_log.debug('debugger: {} {} {}'.format(topic,
headers,
payload))
self._target_platform.vip.pubsub.publish(
peer='pubsub',
topic=topic,
headers=headers,
message=payload['message']).get()
except gevent.Timeout:
_log.debug("Timeout occurred email should send!")
timeout_occurred = True
self._last_timeout = self.timestamp()
self._num_failures += 1
# Stop the current platform from attempting to
# connect
self._target_platform.core.stop()
self._target_platform = None
self.vip.health.set_status(
STATUS_BAD, "Timeout occured")
except Unreachable:
_log.error("Target not reachable. Wait till it's ready!")
except ZMQError as exc:
if exc.errno == ENOTSOCK:
# Stop the current platform from attempting to
# connect
_log.error("Target disconnected. Stopping target platform agent")
self._target_platform = None
self.vip.health.set_status(
STATUS_BAD, "Target platform disconnected")
except Exception as e:
err = "Unhandled error publishing to target platfom."
_log.error(err)
_log.error(traceback.format_exc())
self.vip.health.set_status(
STATUS_BAD, err)
# Before returning lets mark any that weren't errors
# as sent.
self.report_handled(handled_records)
return
else:
handled_records.append(x)
_log.debug("handled: {} number of items".format(
len(to_publish_list)))
self.report_handled(handled_records)
if timeout_occurred:
_log.debug('Sending alert from the ForwardHistorian')
status = Status.from_json(self.vip.health.get_status())
self.vip.health.send_alert(FORWARD_TIMEOUT_KEY,
status)
else:
self.vip.health.set_status(
STATUS_GOOD,"published {} items".format(
len(to_publish_list)))
def list_agents(self):
""" List the agents that are installed on the platform.
Note this does not take into account agents that are connected
with the instance, but only the ones that are installed and
have a uuid.
:return: A list of agents.
"""
agents = self.vip.rpc.call("control", "list_agents").get(timeout=30)
status_running = self.status_agents()
uuid_to_status = {}
# proc_info has a list of [startproc, endprox]
for a in agents:
pinfo = None
is_running = False
for uuid, name, proc_info in status_running:
if a['uuid'] == uuid:
is_running = proc_info[0] > 0 and proc_info[1] == None
pinfo = proc_info
break
uuid_to_status[a['uuid']] = {
'is_running': is_running,
'process_id': None,
'error_code': None,
'permissions': {
'can_stop': is_running,
'can_start': not is_running,
'can_restart': True,
'can_remove': True
}
}
if pinfo:
uuid_to_status[a['uuid']]['process_id'] = proc_info[0]
uuid_to_status[a['uuid']]['error_code'] = proc_info[1]
if 'volttroncentral' in a['name'] or \
'vcplatform' in a['name']:
uuid_to_status[a['uuid']]['permissions']['can_stop'] = False
uuid_to_status[a['uuid']]['permissions']['can_remove'] = False
# The default agent is stopped health looks like this.
uuid_to_status[a['uuid']]['health'] = {
'status': 'UNKNOWN',
'context': None,
'last_updated': None
}
if is_running:
identity = self.vip.rpc.call('control', 'agent_vip_identity',
a['uuid']).get(timeout=30)
status = self.vip.rpc.call(identity,
'health.get_status').get(timeout=30)
uuid_to_status[a['uuid']]['health'] = Status.from_json(
status).as_dict()
for a in agents:
if a['uuid'] in uuid_to_status.keys():
_log.debug('UPDATING STATUS OF: {}'.format(a['uuid']))
a.update(uuid_to_status[a['uuid']])
return agents
def get_health(self):
_log.debug("Getting health: {}".format(self.vip.health.get_status()))
return Status.from_json(self.vip.health.get_status()).as_dict()
def publish_to_historian(self, to_publish_list):
handled_records = []
_log.debug("publish_to_historian number of items: {}".format(
len(to_publish_list)))
parsed = urlparse(self.core.address)
next_dest = urlparse(destination_vip)
current_time = self.timestamp()
last_time = self._last_timeout
_log.debug('Lasttime: {} currenttime: {}'.format(
last_time, current_time))
timeout_occurred = False
if self._last_timeout:
# if we failed we need to wait 60 seconds before we go on.
if self.timestamp() < self._last_timeout + 60:
_log.debug('Not allowing send < 60 seconds from failure')
return
if not self._target_platform:
self.historian_setup()
if not self._target_platform:
_log.debug('Could not connect to target')
return
for vip_id in required_target_agents:
try:
self._target_platform.vip.ping(vip_id).get()
except Unreachable:
skip = "Skipping publish: Target platform not running " \
"required agent {}".format(vip_id)
_log.warn(skip)
self.vip.health.set_status(STATUS_BAD, skip)
return
except Exception as e:
err = "Unhandled error publishing to target platform."
_log.error(err)
_log.error(traceback.format_exc())
self.vip.health.set_status(STATUS_BAD, err)
return
for x in to_publish_list:
topic = x['topic']
value = x['value']
# payload = jsonapi.loads(value)
payload = value
headers = payload['headers']
headers['X-Forwarded'] = True
try:
del headers['Origin']
except KeyError:
pass
try:
del headers['Destination']
except KeyError:
pass
if gather_timing_data:
add_timing_data_to_header(
headers, self.core.agent_uuid or self.core.identity,
"forwarded")
if timeout_occurred:
_log.error(
'A timeout has occurred so breaking out of publishing')
break
with gevent.Timeout(30):
try:
_log.debug('debugger: {} {} {}'.format(
topic, headers, payload))
self._target_platform.vip.pubsub.publish(
peer='pubsub',
topic=topic,
headers=headers,
message=payload['message']).get()
except gevent.Timeout:
_log.debug("Timeout occurred email should send!")
timeout_occurred = True
self._last_timeout = self.timestamp()
self._num_failures += 1
# Stop the current platform from attempting to
# connect
self._target_platform.core.stop()
self._target_platform = None
self.vip.health.set_status(STATUS_BAD,
"Timeout occured")
except Unreachable:
_log.error(
"Target not reachable. Wait till it's ready!")
except ZMQError as exc:
if exc.errno == ENOTSOCK:
# Stop the current platform from attempting to
# connect
_log.error(
"Target disconnected. Stopping target platform agent"
)
self._target_platform = None
self.vip.health.set_status(
STATUS_BAD, "Target platform disconnected")
except Exception as e:
err = "Unhandled error publishing to target platfom."
_log.error(err)
_log.error(traceback.format_exc())
self.vip.health.set_status(STATUS_BAD, err)
# Before returning lets mark any that weren't errors
# as sent.
self.report_handled(handled_records)
return
else:
handled_records.append(x)
_log.debug("handled: {} number of items".format(
len(to_publish_list)))
self.report_handled(handled_records)
if timeout_occurred:
_log.debug('Sending alert from the ForwardHistorian')
status = Status.from_json(self.vip.health.get_status())
self.vip.health.send_alert(FORWARD_TIMEOUT_KEY, status)
else:
self.vip.health.set_status(
STATUS_GOOD,
"published {} items".format(len(to_publish_list)))
def setup(self):
"""
Read resource files and load list of valid tags, categories,
tags grouped by categories, list of reference tags and its
parent.
:return:
"""
_log.debug("Setup of sqlite tagging agent")
err_message = ""
table_names = []
try:
stmt = "SELECT name FROM sqlite_master " \
"WHERE type='table';"
name_list = self.sqlite_utils.select(stmt, None, fetch_all=True)
table_names = [name[0] for name in name_list]
_log.debug(table_names)
except Exception as e:
err_message = "Unable to query list of existing tables from the " \
"database. Exception: {}. Stopping tagging " \
"service agent".format(e.args)
table_name = ""
try:
table_name = self.tags_table
if self.tags_table in table_names:
_log.info("{} table exists. Assuming initial values have been "
"loaded".format(table_name))
else:
self._init_tags()
table_name = self.tag_refs_table
if self.tag_refs_table in table_names:
_log.info("{} table exists. Assuming initial values have been "
"loaded".format(table_name))
else:
self._init_tag_refs()
table_name = self.topic_tags_table
if self.topic_tags_table in table_names:
_log.info("{} table exists. Assuming initial values "
"have been loaded".format(table_name))
else:
self._init_topic_tags()
table_name = self.categories_table
if self.categories_table in table_names:
_log.info("{} table exists. Assuming initial values "
"have been loaded".format(table_name))
else:
self._init_categories()
table_name = self.category_tags_table
if self.category_tags_table in table_names:
_log.info("{} table exists. Assuming initial values "
"have been loaded".format(table_name))
else:
self._init_category_tags()
except Exception as e:
err_message = "Initialization of " + table_name + \
" table failed with exception: {}" \
"Stopping tagging service agent. ".format(e.args)
if err_message:
_log.error(err_message)
self.vip.health.set_status(STATUS_BAD,
"Initialization of tagging service "
"failed")
status = Status.from_json(self.vip.health.get_status_json())
self.vip.health.send_alert(TAGGING_SERVICE_SETUP_FAILED, status)
self.core.stop()
def list_agents(self):
""" List the agents that are installed on the platform.
Note this does not take into account agents that are connected
with the instance, but only the ones that are installed and
have a uuid.
:return: A list of agents.
"""
agents = self.vip.rpc.call("control", "list_agents").get(timeout=30)
status_running = self.status_agents()
uuid_to_status = {}
# proc_info has a list of [startproc, endprox]
for a in agents:
pinfo = None
is_running = False
for uuid, name, proc_info in status_running:
if a['uuid'] == uuid:
is_running = proc_info[0] > 0 and proc_info[1] == None
pinfo = proc_info
break
uuid_to_status[a['uuid']] = {
'is_running': is_running,
'process_id': None,
'error_code': None,
'permissions': {
'can_stop': is_running,
'can_start': not is_running,
'can_restart': True,
'can_remove': True
}
}
if pinfo:
uuid_to_status[a['uuid']]['process_id'] = proc_info[0]
uuid_to_status[a['uuid']]['error_code'] = proc_info[1]
if 'volttroncentral' in a['name'] or \
'vcplatform' in a['name']:
uuid_to_status[a['uuid']]['permissions']['can_stop'] = False
uuid_to_status[a['uuid']]['permissions']['can_remove'] = False
# The default agent is stopped health looks like this.
uuid_to_status[a['uuid']]['health'] = {
'status': 'UNKNOWN',
'context': None,
'last_updated': None
}
if is_running:
identity = self.vip.rpc.call('control', 'agent_vip_identity',
a['uuid']).get(timeout=30)
status = self.vip.rpc.call(identity,
'health.get_status').get(timeout=30)
uuid_to_status[a['uuid']]['health'] = Status.from_json(
status).as_dict()
for a in agents:
if a['uuid'] in uuid_to_status.keys():
_log.debug('UPDATING STATUS OF: {}'.format(a['uuid']))
a.update(uuid_to_status[a['uuid']])
return agents
def get_health(self):
_log.debug("Getting health: {}".format(self.vip.health.get_status()))
return Status.from_json(self.vip.health.get_status()).as_dict()
def publish_to_historian(self, to_publish_list):
handled_records = []
_log.debug("publish_to_historian number of items: {}".format(
len(to_publish_list)))
parsed = urlparse(self.core.address)
next_dest = urlparse(destination_vip)
current_time = self.timestamp()
last_time = self._last_timeout
_log.debug('Lasttime: {} currenttime: {}'.format(
last_time, current_time))
timeout_occurred = False
if self._last_timeout:
# if we failed we need to wait 60 seconds before we go on.
if self.timestamp() < self._last_timeout + 60:
_log.debug('Not allowing send < 60 seconds from failure')
return
if not self._target_platform:
self.historian_setup()
if not self._target_platform:
_log.debug('Could not connect to target')
return
for x in to_publish_list:
topic = x['topic']
value = x['value']
# payload = jsonapi.loads(value)
payload = value
headers = payload['headers']
headers['X-Forwarded'] = True
try:
del headers['Origin']
except KeyError:
pass
try:
del headers['Destination']
except KeyError:
pass
# if not headers.get('Origin', None)
# if overwrite_origin:
# if not include_origin_in_header:
# try:
# del headers['Origin']
# except KeyError:
# pass
# else:
# headers['Origin'] = origin
# else:
# headers['Origin'] = parsed.hostname
# headers['Destination'] = [next_dest.scheme +
# '://'+
# next_dest.hostname]
# else:
# headers['Destination'].append(next_dest.hostname)
if timeout_occurred:
_log.error(
'A timeout has occured so breaking out of publishing')
break
with gevent.Timeout(30):
try:
_log.debug('debugger: {} {} {}'.format(
topic, headers, payload))
self._target_platform.vip.pubsub.publish(
peer='pubsub',
topic=topic,
headers=headers,
message=payload['message']).get()
except gevent.Timeout:
_log.debug("Timout occurred email should send!")
timeout_occurred = True
self._last_timeout = self.timestamp()
self._num_failures += 1
# Stop the current platform from attempting to
# connect
self._target_platform.core.stop()
self._target_platform = None
self.vip.health.set_status(STATUS_BAD,
"Timout occured")
except Exception as e:
_log.error(e)
else:
handled_records.append(x)
_log.debug("handled: {} number of items".format(
len(to_publish_list)))
self.report_handled(handled_records)
if timeout_occurred:
_log.debug('Sending alert from the ForwardHistorian')
status = Status.from_json(self.vip.health.get_status())
self.vip.health.send_alert(FORWARD_TIMEOUT_KEY, status)