def send_stat_event(self):
""" Send AMQP Event for drop and pass metrics """
message_dropped = '{} event dropped since {}'.format(
self.drop_event_count, self.beat_interval)
message_passed = '{} event passed since {}'.format(
self.pass_event_count, self.beat_interval)
event = forger(connector='Engine',
connector_name='engine',
event_type='check',
source_type='resource',
resource=self.amqp_queue + '_data',
state=0,
state_type=1,
output=message_dropped,
perf_data_array=[{
'metric': 'pass_event',
'value': self.pass_event_count,
'type': 'GAUGE'
}, {
'metric': 'drop_event',
'value': self.drop_event_count,
'type': 'GAUGE'
}])
self.logger.debug(message_dropped)
self.logger.debug(message_passed)
try:
self.beat_amqp_publisher.canopsis_event(event)
except Exception as e:
self.logger.exception("Unable to send stat event")
self.drop_event_count = 0
self.pass_event_count = 0
def get_event(self, entity, event_type='check', **kwargs):
"""Get an event from an entity.
:param dict entity: entity to convert to an event.
:param str event_type: specific event_type. Default check.
:param dict kwargs: additional fields to put in the event.
:rtype: dict
"""
kwargs['event_type'] = event_type
# In some cases, name is present but is component in fact
if 'name' in entity:
if 'component' not in entity:
entity['component'] = entity['name']
entity.pop('name')
# fill kwargs with entity values
for field in entity:
kwargs[field] = entity[field]
# forge the event
result = forger(**kwargs)
return result
def send_stat_event(self):
""" Send AMQP Event for drop and pass metrics """
message_dropped = '{} event dropped since {}'.format(
self.drop_event_count,
self.beat_interval
)
message_passed = '{} event passed since {}'.format(
self.pass_event_count,
self.beat_interval
)
event = forger(
connector='Engine',
connector_name='engine',
event_type='check',
source_type='resource',
resource=self.amqp_queue + '_data',
state=0,
state_type=1,
output=message_dropped,
perf_data_array=[
{'metric': 'pass_event',
'value': self.pass_event_count,
'type': 'GAUGE'},
{'metric': 'drop_event',
'value': self.drop_event_count,
'type': 'GAUGE'}
])
self.logger.debug(message_dropped)
self.logger.debug(message_passed)
publish(publisher=self.amqp, event=event)
self.drop_event_count = 0
self.pass_event_count = 0
def done_action():
"""
Trigger done action.
For json payload, see doc/docs/fr/guide_developpeur/apis/v2/alerts.md
:rtype: dict
"""
dico = request.json
if dico is None or not isinstance(dico, dict) or len(dico) <= 0:
return gen_json_error({'description': 'wrong done dict'},
HTTP_ERROR)
author = dico.get(am.AUTHOR)
event = forger(event_type=Check.EVENT_TYPE,
author=author,
connector=dico.get('connector'),
connector_name=dico.get('connector_name'),
component=dico.get('component'),
output=dico.get('comment'))
if dico.get('source_type', None) == 'resource':
event['resource'] = dico['resource']
event['source_type'] = 'resource'
ws.logger.debug('Received done action: {}'.format(event))
entity_id = am.context_manager.get_id(event)
retour = am.execute_task('alerts.useraction.done',
event=event,
author=author,
entity_id=entity_id)
return gen_json(retour)
def publish_statduration_event(self,
timestamp,
duration_name,
duration_value,
entity,
alarm,
author=None):
"""
Publish a statduration event on amqp.
:param int timestamp: the time at which the event occurs
:param str duration_name: the name of the duration to add
:param str duration_value: the value of the duration
:param dict entity: the entity
:param dict alarm: the alarm
:param Optional[str] author: the author of the event that triggered the
statistic event.
"""
if not self.send_events:
return
component = alarm.get(Event.COMPONENT)
resource = alarm.get(Event.RESOURCE)
# AmqpPublisher.canopsis needs the component and resource of the event
# to be unicode strings.
try:
component = component.decode('utf-8')
except (UnicodeError, AttributeError):
pass
try:
resource = resource.decode('utf-8')
except (UnicodeError, AttributeError):
pass
# Although only the alarm's value is needed by statsng, the alarm needs
# to have the same structure as a full alarm for compatibility with the
# go engines.
full_alarm = {'v': alarm}
event = forger(
connector="canopsis",
connector_name="engine",
event_type=StatEvents.statduration,
source_type=Event.RESOURCE if resource else Event.COMPONENT,
component=component,
resource=resource,
timestamp=timestamp,
author=author,
stat_name=duration_name,
duration=duration_value,
current_alarm=full_alarm,
current_entity=entity)
self.amqp_pub.canopsis_event(event)
def test_01(self):
event = forger(
connector='unittest',
source_type='component',
connector_name='test1',
event_type='log'
)
rk = get_routingkey(event)
print(rk)
print(event)
def pre_run(self):
self.ack_event = forger(
connector="Engine",
connector_name=self.etype,
event_type="perf",
source_type='component',
resource='ack',
state=0,
state_type=1,
)
self.beat()
def pre_run(self):
self.ack_event = forger(
connector="Engine",
connector_name=self.etype,
event_type="perf",
source_type='component',
resource='ack',
state=0,
state_type=1,
)
self.beat()
def get_event(self, entity, **kwargs):
"""Get an event from an entity.
:param dict entity: entity to convert to an event.
:param dict kwargs: additional fields to put in the event.
"""
# fill kwargs with entity values
for field in entity:
kwargs[field] = entity[field]
# forge the event
result = forger(**kwargs)
return result
def get_event(self, *args, **kwargs):
"""Get vertice event.
:param args: event forging args.
:param kwargs: event forging kwargs.
"""
result = forger(
event_type=Check.EVENT_TYPE,
source_type=self.type,
component=self.id,
*args, **kwargs
)
return result
def publish_states(self):
stats_event = forger(
connector="engine",
connector_name="engine",
event_type="perf",
source_type="resource",
resource="Engine_stats",
state=0,
perf_data_array=self.perf_data_array,
)
self.logger.debug("Publishing {}".format(stats_event))
publish(publisher=self.amqp, event=stats_event)
def get_event(self, entity, event_type='check', **kwargs):
"""Get an event from an entity.
:param dict entity: entity to convert to an event.
:param str event_type: specific event_type. Default check.
:param dict kwargs: additional fields to put in the event.
:rtype: dict
"""
kwargs['event_type'] = event_type
# fill kwargs with entity values
for field in entity:
kwargs[field] = entity[field]
# forge the event
result = forger(**kwargs)
return result
def publish_statstateinterval_event(self, timestamp, state_name,
state_duration, state_value, entity,
alarm):
"""
Publish a statstateinterval event on amqp.
:param int timestamp: the time at which the event occurs
:param str state_name: the name of the state
:param str state_duration: the time spent in this state
:param str state_value: the value of the state
:param dict entity: the entity
:param dict alarm: the alarm
"""
if not self.send_events:
return
component = alarm.get(Event.COMPONENT)
resource = alarm.get(Event.RESOURCE)
# AmqpPublisher.canopsis needs the component and resource of the event
# to be unicode strings.
try:
component = component.decode('utf-8')
except (UnicodeError, AttributeError):
pass
try:
resource = resource.decode('utf-8')
except (UnicodeError, AttributeError):
pass
event = forger(
connector="canopsis",
connector_name="engine",
event_type=StatEvents.statstateinterval,
source_type=Event.RESOURCE if resource else Event.COMPONENT,
component=component,
resource=resource,
timestamp=timestamp,
state_name=state_name,
state_duration=state_duration,
state_value=state_value,
alarm=alarm,
entity=entity)
self.amqp_pub.canopsis_event(event)
def consume_dispatcher(self, event, *args, **kargs):
self.logger.debug("Start metrics consolidation")
t_serie = event.copy()
self.logger.debug('\n\n\n\n----->serie: {}'.format(t_serie))
if not t_serie:
# Show error message
self.logger.error('No record found.')
# Test Settings
_from = 1425394522
_to = 1425402296
perf_data_array = []
_, points = self.fetch(t_serie, _from, _to)
# This method allow us to update an metric or a list of metrics
self.manager.put(metric_id=t_serie['_id'], points=points)
# Publish the consolidation metrics
# metric_name = 'metric_name' # Change the value with UI data
for t, v in points:
#c_event['timestamp'] = t
perf_data_array.append(
{
'metric': t_serie['_id'], 'value': v,
'unit': t_serie['_id'], 'min': None,
'max': None, 'warn': None, 'crit': None, 'type': 'GAUGE'
}
)
conso_event = forger(
timestamp=t,
component='conso',
connector='Engine',
connector_name='consolidation',
event_type='perf',
source_type='component',
perf_data_array=perf_data_array
)
self.logger.debug('Publishing {}'.format(conso_event))
publish(publisher=self.amqp, event=conso_event)
perf_data_array = [] # reset the perf_data_array data
# Update crecords informations
event_id = t_serie['_id']
self.crecord_task_complete(event_id)
def send_perfdata(self, uuid, time, updated, deleted):
"""Send stat about the import through a perfdata event.
:param str uuid: the import uuid
:param float time: the execution time of the import
:param int updated: the number of updated entities during the import
:param int deleted: the number of deleted entities during the import
"""
# define the state according to the duration of the import
if time > self._thd_crit_s:
state = ST_WARNING
elif time > self._thd_warn_s:
state = ST_MINOR
else:
state = ST_INFO
perf_data_array[0]["value"] = time
perf_data_array[1]["value"] = updated
perf_data_array[2]["value"] = deleted
output = "execution : {0} sec, updated ent :"\
" {1}, deleted ent : {2}".format(time, updated, deleted)
self.logger.critical("AMQP queue = {0}".format(self.amqp_queue))
# create a perfdata event
event = forger(
connector="Taskhandler",
connector_name=self.etype,
component=uuid,
event_type="check",
source_type="resource",
resource="task_importctx/report",
state=state,
state_type=1,
output=output,
perf_data_array=perf_data_array
)
try:
self.work_amqp_publisher.canopsis_event(event)
except Exception as e:
self.logger.exception("Unable to send event")
def publish_event(self, display_name, computed_state, output, _id):
"""
Publish an event watcher on amqp.
TODO: move that elsewhere (not specific to watchers)
:param display_name: watcher display_name
:param computed_state: watcher state
:param output: watcher output
"""
event = forger(connector="canopsis",
connector_name="engine",
event_type="watcher",
source_type="component",
component=_id,
state=computed_state,
output=output,
perf_data_array=[],
display_name=display_name)
self.amqp_pub.canopsis_event(event)
def set_derogation_state(self, derogation, active):
dactive = derogation.get('active', False)
name = derogation.get('crecord_name', None)
notify = False
state = 0
if active:
if not dactive:
self.logger.info("%s (%s) is now active" % (
derogation['crecord_name'], derogation['_id'])
)
self.storage.update(derogation['_id'], {'active': True})
notify = True
else:
if dactive:
self.logger.info("%s (%s) is now inactive" % (
derogation['crecord_name'], derogation['_id'])
)
self.storage.update(derogation['_id'], {'active': False})
notify = True
if notify:
if active:
output = "Derogation '%s' is now active" % name
state = 1
else:
output = "Derogation '%s' is now inactive" % name
event = forger(
connector="Engine",
connector_name="engine",
event_type="log",
source_type="component",
component=self.etype,
state=state,
output=output,
long_output=derogation.get('description', None)
)
publish(publisher=self.amqp, event=event)
def get_event(self, entity, event_type='check', **kwargs):
"""Get an event from an entity.
:param dict entity: entity to convert to an event.
:param str event_type: specific event_type. Default check.
:param dict kwargs: additional fields to put in the event.
:rtype: dict
"""
# keys from entity that should not be in event
delete_keys = [
"_id", "depends", "impact", "type", "measurements", "infos",
"last_state_change"
]
kwargs['event_type'] = event_type
# In some cases, name is present but is component in fact
if 'name' in entity:
if 'component' not in entity:
entity['component'] = entity['name']
entity.pop('name')
# remove key that should not be in the event
tmp = entity.copy()
for key in delete_keys:
try:
tmp.pop(key)
except KeyError:
pass
# fill kwargs with entity values
for field in tmp:
kwargs[field] = tmp[field]
# forge the event
result = forger(**kwargs)
return result
def publish_event(self, display_name, computed_state, output, _id):
"""
Publish an event watcher on amqp.
TODO: move that elsewhere (not specific to watchers)
:param display_name: watcher display_name
:param computed_state: watcher state
:param output: watcher output
"""
event = forger(
connector="canopsis",
connector_name="engine",
event_type="watcher",
source_type="component",
component=_id,
state=computed_state,
output=output,
perf_data_array=[],
display_name=display_name)
self.amqp_pub.canopsis_event(event)
def get_event(self, entity, event_type='check', **kwargs):
"""Get an event from an entity.
:param dict entity: entity to convert to an event.
:param str event_type: specific event_type. Default check.
:param dict kwargs: additional fields to put in the event.
:rtype: dict
"""
# keys from entity that should not be in event
delete_keys = [
"_id", "depends", "impact", "type", "measurements", "infos",
"last_state_change"
]
kwargs['event_type'] = event_type
# In some cases, name is present but is component in fact
if 'name' in entity:
if 'component' not in entity:
entity['component'] = entity['name']
entity.pop('name')
# remove key that should not be in the event
tmp = entity.copy()
for key in delete_keys:
try:
tmp.pop(key)
except KeyError:
pass
# fill kwargs with entity values
for field in tmp:
kwargs[field] = tmp[field]
# forge the event
result = forger(**kwargs)
return result
def create_watcher(self, watcher):
"""
Create a watcher and return its id.
:param Dict[str, Any] watcher:
:rtype: str
:raises: CollectionError if the creation fails.
"""
watcher = self.check_watcher_fields(watcher)
if '_id' not in watcher:
watcher['_id'] = str(uuid.uuid4())
wid = self.__collection.insert(watcher)
event = forger(
connector="watcher",
connector_name="watcher",
event_type="updatewatcher",
source_type="component",
component=wid)
self.amqp_pub.canopsis_event(event)
return wid
def update_watcher_by_id(self, watcher, wid):
"""
Update a watcher given its id.
Return a boolean if the operation is successful.
:param str wid: the id of the watcher.
:param Dict[str, Any] watcher:
:rtype: bool
:raises: CollectionError if the update fails.
"""
watcher = self.check_watcher_fields(watcher)
resp = self.__collection.update(query={'_id': wid, "type": "watcher"},
document=watcher)
event = forger(
connector="watcher",
connector_name="watcher",
event_type="updatewatcher",
source_type="component",
component=wid)
self.amqp_pub.canopsis_event(event)
return self.__collection.is_successfull(resp)
def done_action():
"""
Trigger done action.
For json payload, see doc/docs/fr/guide_developpeur/apis/v2/alerts.md
:rtype: dict
"""
dico = request.json
if dico is None or not isinstance(dico, dict) or len(dico) <= 0:
return gen_json_error(
{'description': 'wrong done dict'}, HTTP_ERROR)
author = dico.get(am.AUTHOR)
event = forger(
event_type=Check.EVENT_TYPE,
author=author,
connector=dico.get('connector'),
connector_name=dico.get('connector_name'),
component=dico.get('component'),
output=dico.get('comment')
)
if dico.get('source_type', None) == 'resource':
event['resource'] = dico['resource']
event['source_type'] = 'resource'
ws.logger.debug('Received done action: {}'.format(event))
entity_id = am.context_manager.get_id(event)
retour = am.execute_task(
'alerts.useraction.done',
event=event,
author=author,
entity_id=entity_id
)
return gen_json(retour)
def event(self):
# Get state information form aggregation
states, state, ack_count, wstate_for_ack, infobagot = self.getState()
information = None
if state == -1 and ack_count == -1:
state = 0
information = u'No event matched by selector {}'.format(
self.display_name
)
# Build output
total = 0
total_error = 0
worst_state_with_ack = 0
for s in states:
states[s] = int(states[s])
total += states[s]
if s > 0:
total_error += states[s]
# Computed state when all events are not ack
computed_state = wstate_for_ack
# Is selector produced event acknowleged
if ack_count >= (total_error + infobagot) and ack_count > 0:
send_ack = True
# All matched event were acknowleged,
# then user chose what kind of state to use. it is either:
# The worst state computed from all events
# matched that are ack or not ack
if self.state_when_all_ack == 'worststate':
if state != -1:
computed_state = state
else:
computed_state = 0
# A defined state set by user
if self.state_when_all_ack in self.states_labels:
computed_state = self.states_labels[self.state_when_all_ack]
else:
send_ack = False
# Display purpose
if ack_count == -1:
ack_count = 0
self.logger.debug(u' + state: {}'.format(state))
perf_data_array = []
self.logger.debug(u' + total: {}'.format(total))
# Create perfdata array and generate output data
output = self.output_tpl.replace('[TOTAL]', str(total))
# Producing metrics for ack
if ack_count:
perf_data_array.append({
"metric": 'cps_sel_ack',
"value": ack_count,
"max": total
})
output = output.replace('[ACK]', str(ack_count))
output_data = {}
# Metrics and output computation
for i in [0, 1, 2, 3]:
value = 0
if i in states:
value = states[i]
metric = u'cps_sel_state_{}'.format(i)
output = output.replace(self.template_replace[i], str(value))
output_data[metric] = value
perf_data_array.append({
"metric": metric,
"value": value,
"max": total
})
self.logger.info(u'metric {} : {}'.format(metric, value))
perf_data_array.append({
"metric": "cps_sel_total",
"value": total
})
output_data['total'] = total
# Build Event
event = forger(
connector="canopsis",
connector_name="engine",
event_type="selector",
source_type="component",
component=self.display_name,
state=computed_state,
output=output,
perf_data_array=perf_data_array,
display_name=self.display_name
)
# Build RK
rk = get_routingkey(event)
return (rk, event, send_ack)
def import_context(self, uuid):
"""Import a new context.
:param uuid: the uuid of the import to process
:return type: a tuple (updated entities, deleted entities)
"""
file_ = ImportKey.IMPORT_FILE.format(uuid)
fd = open(file_, 'r')
# In case the previous import failed and/or raise an exception, we\
# clean now
self.clean_attributes()
start = time.time()
self.__superficial_check(fd)
end = time.time()
self.logger.debug("Import {0} : superficial"
" check {1}.".format(uuid,
execution_time(end - start)))
start = time.time()
self.entities_to_update = self.__get_entities_to_update(file_)
end = time.time()
self.logger.debug("Import {0} : get_entities_to"
"_update {1}.".format(uuid,
execution_time(end - start)))
# create/update watcher tracker
watcher_tracker = set()
# Process cis list
start = time.time()
for ci in ijson.items(fd, "{0}.item".format(self.K_CIS)):
self.logger.debug("Current ci : {0}".format(ci))
if ci[self.K_ACTION] == self.A_DELETE:
self.__a_delete_entity(ci)
elif ci[self.K_ACTION] == self.A_CREATE:
if ci.get(self.K_TYPE, '') == 'watcher':
watcher_tracker.add(ci.get(self.K_ID))
self.__a_create_entity(ci)
elif ci[self.K_ACTION] == self.A_UPDATE:
if ci.get(self.K_TYPE, '') == 'watcher':
watcher_tracker.add(ci.get(self.K_ID))
self.__a_update_entity(ci)
elif ci[self.K_ACTION] == self.A_SET:
self.__a_set_entity(ci)
elif ci[self.K_ACTION] == self.A_DISABLE:
self.__a_disable_entity(ci)
elif ci[self.K_ACTION] == self.A_ENABLE:
self.__a_enable_entity(ci)
else:
raise ValueError("The action {0} is not recognized\n".format(
ci[self.K_ACTION]))
end = time.time()
self.logger.debug("Import {0} : update cis {1}.".format(
uuid, execution_time(end - start)))
fd.seek(0)
# Process link list
start = time.time()
for link in ijson.items(fd, "{0}.item".format(self.K_LINKS)):
self.logger.debug("Current link : {0}".format(link))
if link[self.K_ACTION] == self.A_DELETE:
self.__a_delete_link(link)
elif link[self.K_ACTION] == self.A_CREATE:
self.__a_create_link(link)
elif link[self.K_ACTION] == self.A_UPDATE:
self.__a_update_link(link)
elif link[self.K_ACTION] == self.A_DISABLE:
self.__a_disable_link(link)
elif link[self.K_ACTION] == self.A_ENABLE:
self.__a_enable_link(link)
else:
raise ValueError("The action {0} is not recognized".format(
link[self.K_ACTION]))
end = time.time()
self.logger.debug("Import {0} : update links"
" {1}.".format(uuid, execution_time(end - start)))
for id_ in self.update:
if id_ in self.delete:
desc = "The entity {0} to be deleted is updated in "\
"the same import. Update aborted.".format(id_)
raise ValueError(desc)
updated_entities = len(self.update)
deleted_entities = len(self.delete)
for entity in self.update.values():
entity[self.K_IMPACT] = list(entity[self.K_IMPACT])
entity[self.K_DEPENDS] = list(entity[self.K_DEPENDS])
start = time.time()
self._put_entities(self.update.values())
# send updatewatcher event
for _id in watcher_tracker:
event = forger(connector="watcher",
connector_name="watcher",
event_type="updatewatcher",
source_type="component",
component=_id)
self.amqp_pub.canopsis_event(event)
end = time.time()
self.logger.debug("Import {0} : push updated"
" entities {1}.".format(uuid,
execution_time(end - start)))
start = time.time()
self._delete_entities(self.delete)
end = time.time()
self.logger.debug("Import {0} : delete entities"
" {1}.".format(uuid, execution_time(end - start)))
self.clean_attributes()
return updated_entities, deleted_entities
def on_collectd_event(self, body, msg):
start = time()
error = False
prog = re.compile('^(PUTVAL) ("(.+)"|([^\s]+)) (interval=.+) ([^\s]+)$')
match = re.match(prog, body)
if match:
if match.group(3):
cnode = match.group(3)
else:
cnode = match.group(4)
collectd_info = [match.group(1),cnode,match.group(5),match.group(6)]
self.logger.debug(body)
action = collectd_info[0]
self.logger.debug(" + Action: %s" % action)
if len(collectd_info) == 4 and action == "PUTVAL":
cnode = collectd_info[1].split("/")
component = cnode[0]
resource = cnode[1]
metric = cnode[2]
options = collectd_info[2]
values = collectd_info[3]
self.logger.debug(" + Options: %s" % options)
self.logger.debug(" + Component: %s" % component)
self.logger.debug(" + Resource: %s" % resource)
self.logger.debug(" + Metric: %s" % metric)
self.logger.debug(" + Raw Values: %s" % values)
values = values.split(":")
perf_data_array = []
ctype = None
try:
## Know metric
ctype = types[metric]
except:
try:
ctype = types[metric.split('-')[0]]
metric = metric.split('-')[1]
except Exception as err:
self.logger.error(
"Invalid format '%s' (%s)" % (body, err))
return None
try:
timestamp = int(float(values[0]))
values = values[1:]
self.logger.debug(" + Timestamp: %s" % timestamp)
self.logger.debug(" + Values: %s" % values)
except Exception as err:
self.logger.error(
"Impossible to get timestamp or values (%s)" % err)
return None
self.logger.debug(" + metric: %s" % metric)
self.logger.debug(" + ctype: %s" % ctype)
if ctype:
try:
i = 0
for value in values:
name = ctype[i]['name']
unit = ctype[i]['unit']
vmin = ctype[i]['min']
vmax = ctype[i]['max']
if vmin == 'U':
vmin = None
if vmax == 'U':
vmax = None
if name == "value":
name = metric
if metric != name:
name = "%s-%s" % (metric, name)
data_type = ctype[i]['type']
value = float(value)
self.logger.debug(" + %s" % name)
self.logger.debug(
" -> %s (%s)" % (value, data_type))
i += 1
perf_data_array.append(
{
'metric': name, 'value': value,
'type': data_type, 'unit': unit,
'min': vmin, 'max': vmax
}
)
except Exception as err:
self.logger.error(
"Impossible to parse values '%s' (%s)" % (
values, err))
if perf_data_array:
self.logger.debug(
' + perf_data_array: %s', perf_data_array)
event = forger(
connector='collectd',
connector_name='collectd2event',
component=component,
resource=resource,
timestamp=timestamp,
source_type='resource',
event_type='perf',
perf_data_array=perf_data_array
)
self.logger.debug("Send Event: %s" % event)
## send event on amqp
publish(publisher=self.amqp, event=event)
else:
error = True
self.logger.error("Invalid collectd Action (%s)" % body)
else:
self.logger.error("Invalid collectd Message (%s)" % body)
if error:
self.counter_error += 1
self.counter_event += 1
self.counter_worktime += time() - start
def _beat(self):
now = int(time())
if self.last_stat + 60 <= now:
self.logger.debug(" + Send stats")
self.last_stat = now
evt_per_sec = 0
sec_per_evt = 0
if self.counter_event:
evt_per_sec = float(self.counter_event) / self.beat_interval
self.logger.debug(" + %0.2f event(s)/seconds" % evt_per_sec)
if self.counter_worktime and self.counter_event:
sec_per_evt = self.counter_worktime / self.counter_event
self.logger.debug(" + %0.5f seconds/event" % sec_per_evt)
# Submit event
if self.send_stats_event and self.counter_event != 0:
state = 0
if sec_per_evt > self.thd_warn_sec_per_evt:
state = 1
if sec_per_evt > self.thd_crit_sec_per_evt:
state = 2
perf_data_array = [
{
'retention': self.perfdata_retention,
'metric': 'cps_evt_per_sec',
'value': round(evt_per_sec, 2), 'unit': 'evt'},
{
'retention': self.perfdata_retention,
'metric': 'cps_sec_per_evt',
'value': round(sec_per_evt, 5), 'unit': 's',
'warn': self.thd_warn_sec_per_evt,
'crit': self.thd_crit_sec_per_evt}
]
self.logger.debug(" + State: {0}".format(state))
event = forger(
connector="Engine",
connector_name="engine",
event_type="check",
source_type="resource",
resource=self.amqp_queue,
state=state,
state_type=1,
output="%0.2f evt/sec, %0.5f sec/evt" % (
evt_per_sec, sec_per_evt),
perf_data_array=perf_data_array
)
publish(event=event, publisher=self.amqp)
self.counter_error = 0
self.counter_event = 0
self.counter_worktime = 0
try:
self.beat()
except Exception as err:
self.logger.error("Beat raise exception: {0}".format(err))
self.logger.error(print_exc())
finally:
self.beat_lock = False
def work(self, event, *args, **kargs):
logevent = None
ackremove = False
state = event.get('state', 0)
state_type = event.get('state_type', 1)
if event['event_type'] == 'ackremove':
# remove ack from event
# Ack remove information exists when ack is just removed
# And deleted if event is ack again
rk = event['ref_rk']
self.events_collection.update(
{'_id': rk},
{
'$set': {
'ack_remove': {
'author': event['author'],
'comment': event['output'],
'timestamp': time()
}
},
'$unset': {
'ack': '',
'ticket_declared_author': '',
'ticket_declared_date': '',
'ticket': '',
'ticket_date': ''
}
}
)
ackremove = True
# If event is of type ack, then ack reference event
if event['event_type'] == 'ack':
self.logger.debug('Ack event found, will proceed ack.')
rk = event.get('referer', event.get('ref_rk', None))
if not rk:
self.logger.error(
'Cannot get acknowledged event, missing referer or ref_rk'
)
return event
for comment in self.comments:
if comment['comment'] in event['output']:
# An ack comment is contained into a defined comment
# Then let save referer key to the comment
# Set referer rk to last update date
self.objects_backend.update(
{'_id': comment['_id']},
{"$addToSet": {'referer_event_rks': {'rk': rk}}},
upsert=True)
self.logger.info(
'Added a referer rk to the comment {}'.format(
comment['comment']
)
)
ack_info = {
'timestamp': event['timestamp'],
'ackts': int(time()),
'rk': rk,
'author': event['author'],
'comment': event['output']
}
# add rk to acknowledged rks
response = self.stbackend.find_and_modify(
query={'rk': rk, 'solved': False},
update={'$set': ack_info},
upsert=True,
full_response=True,
new=True
)
self.logger.debug(
u'Updating event {} with author {} and comment {}'.format(
rk,
ack_info['author'],
ack_info['comment']
)
)
ack_info['isAck'] = True
# Useless information for event ack data
del ack_info['ackts']
# clean eventual previous ack remove information
self.events_collection.update(
{
'_id': rk
}, {
'$set': {
'ack': ack_info,
},
'$unset': {
'ack_remove': '',
}
}
)
if not response['lastErrorObject']['updatedExisting']:
record = response['value']
# Emit an event log
referer_event = self.storage.find_one(
mfilter={'_id': rk},
namespace='events'
)
if referer_event:
referer_event = referer_event.dump()
logevent = forger(
connector="Engine",
connector_name=self.etype,
event_type="log",
source_type=referer_event['source_type'],
component=referer_event['component'],
resource=referer_event.get('resource', None),
state=0,
state_type=1,
ref_rk=event['rk'],
output=u'Event {0} acknowledged by {1}'.format(
rk, event['author']),
long_output=event['output'],
perf_data_array=[
{
'metric': 'ack_delay',
'value': record['ackts'] - record['timestamp'],
'unit': 's'
}
]
)
# Now update counters
ackhost = is_host_acknowledged(event)
# Cast response to ! 0|1
cvalues = int(not ackhost)
alerts_event = forger(
connector="Engine",
connector_name=self.etype,
event_type="perf",
source_type="component",
component="__canopsis__",
perf_data_array=[
{
'metric': 'cps_alerts_ack',
'value': cvalues,
'type': 'COUNTER'
},
{
'metric': 'cps_alerts_not_ack',
'value': -1,
'type': 'COUNTER'
}
]
)
publish(
publisher=self.amqp,
event=alerts_event
)
self.logger.debug('Ack internal metric sent.')
for hostgroup in event.get('hostgroups', []):
alerts_event = forger(
connector="Engine",
connector_name=self.etype,
event_type="perf",
source_type="resource",
component="__canopsis__",
resource=hostgroup,
perf_data_array=[
{
'metric': 'cps_alerts_ack',
'value': cvalues,
'type': 'COUNTER'
},
{
'metric': 'cps_alerts_not_ack',
'value': -1,
'type': 'COUNTER'
}
]
)
publish(publisher=self.amqp, event=alerts_event)
self.logger.debug('Reloading ack cache')
self.reload_ack_cache()
# If event is acknowledged, and went back to normal, remove the ack
# This test concerns most of case
# And could not perform query for each event
elif state == 0 and state_type == 1:
solvedts = int(time())
if event['rk'] in self.cache_acks:
self.logger.debug(
'Ack exists for this event, and has to be recovered.'
)
# We have an ack to process for this event
query = {
'rk': event['rk'],
'solved': False,
'ackts': {'$gt': -1}
}
ack = self.stbackend.find_one(query)
if ack:
self.stbackend.update(
query,
{
'$set': {
'solved': True,
'solvedts': solvedts
}
}
)
logevent = forger(
connector="Engine",
connector_name=self.etype,
event_type="log",
source_type=event['source_type'],
component=event['component'],
resource=event.get('resource', None),
state=0,
state_type=1,
ref_rk=event['rk'],
output=u'Acknowledgement removed for event {0}'.format(
event['rk']),
long_output=u'Everything went back to normal',
perf_data_array=[
{
'metric': 'ack_solved_delay',
'value': solvedts - ack['ackts'],
'unit': 's'
}
]
)
logevent['acknowledged_connector'] = event['connector']
logevent['acknowledged_source'] = event['connector_name']
logevent['acknowledged_at'] = ack['ackts']
logevent['solved_at'] = solvedts
# If the event is in problem state,
# update the solved state of acknowledgement
elif ackremove or (state != 0 and state_type == 1):
self.logger.debug('Alert on event, preparing ACK statement.')
self.stbackend.find_and_modify(
query={'rk': event['rk'], 'solved': True},
update={'$set': {
'solved': False,
'solvedts': -1,
'ackts': -1,
'timestamp': -1,
'author': '',
'comment': ''
}}
)
if logevent:
publish(
publisher=self.amqp, event=logevent,
exchange=self.acknowledge_on
)
return event
def init_tests(self):
self.evt_ok1 = forger(
event_type='check',
connector='gutamaya',
connector_name='imperial',
source_type='resource',
component='test_weatherfilter',
resource='ok1',
state=0,
output='cutter'
)
self.evt_ok2 = forger(
event_type='check',
connector='gutamaya',
connector_name='imperial',
source_type='resource',
component='test_weatherfilter',
resource='ok2',
state=0,
output='cutter'
)
self.evt_paused = forger(
event_type='check',
connector='gutamaya',
connector_name='imperial',
source_type='resource',
component='test_weatherfilter',
resource='paused',
state=0,
output='cutter'
)
self.evt_paused_id = 'paused/test_weatherfilter'
self.evt_maintenance = forger(
event_type='check',
connector='faulcon',
connector_name='delacy',
source_type='resource',
component='test_weatherfilter',
resource='maintenance',
state=0,
output='cobraMKIII'
)
self.evt_maintenance_id = 'maintenance/test_weatherfilter'
now = int(time())
self.pb_paused = {
'name': 'gutamaya_imperial_interdictor',
'author': 'cmdr',
'filter_': {'_id': self.evt_paused_id},
'rrule': None,
'tstart': now,
'tstop': now + 60 * 60,
'type_': 'pause'
}
self.pb_maintenance = {
'name': 'cobramkIII',
'author': 'cmdr',
'filter_': {'_id': self.evt_maintenance_id},
'rrule': None,
'tstart': now,
'tstop': now + 60 * 60,
'type_': 'maintenance'
}
self.watcher_ok = WatcherModel(
"weatherfilter_ok",
{
"_id": {
"$in": [
"ok1/test_weatherfilter",
"ok2/test_weatherfilter"
]
}
},
"only_ok"
)
self.watcher_partial_pause = WatcherModel(
"weatherfilter_partial_pause",
{
"_id": {
"$in": [
"paused/test_weatherfilter",
"ok1/test_weatherfilter"
]
}
},
"partialy_paused"
)
self.watcher_full_pause = WatcherModel(
"weatherfilter_full_pause",
{
"_id": {
"$in": [
"maintenance/test_weatherfilter",
"paused/test_weatherfilter"
]
}
},
"paused_maintenance"
)
self.watcher_full_pause_maintenance = WatcherModel(
"watcherfilter_full_pause_maintenance",
{
"_id": {
"$in": [
"maintenance/test_weatherfilter",
]
}
},
"maintenance"
)
self.watcher_mixed = WatcherModel(
"weatherfilter_mixed",
{
"_id": {
"$in": [
"maintenance/test_weatherfilter",
"paused/test_weatherfilter",
"ok1/test_weatherfilter"
]
}
},
"mixed"
)
def work(self, event, *args, **kwargs):
# If the event is a downtime event,
# add entry to the downtime collection
if event['event_type'] == 'downtime':
self.logger.debug(
'Event downtime received: {0}'.format(event['rk']))
# Build entry, so we know there is a downtime on the component
record = Record({
'_expire': event['start'] + event['duration'],
'connector': event['connector'],
'source': event['connector_name'],
'component': event['component'],
'resource': event.get('resource', None),
'start': event['start'],
'end': event['end'],
'fixed': event['fixed'],
'timestamp': event['entry'],
'author': event['author'],
'comment': event['output']
})
# Save record, and log the action
record.save(self.storage)
logevent = forger(
connector="Engine",
connector_name=self.etype,
event_type="log",
source_type=event['source_type'],
component=event['component'],
resource=event.get('resource', None),
state=0,
state_type=1,
output=u'Downtime scheduled by {0} from {1} to {2}'.format(
event['author'],
event['start'],
event['end']
),
long_output=event['output']
)
logevent['downtime_connector'] = event['connector']
logevent['downtime_source'] = event['connector_name']
publish(publisher=self.amqp, event=logevent)
# Set downtime for events already in database
self.evt_backend.update(
{
'connector': event['connector'],
'connector_name': event['connector_name'],
'component': event['component'],
'resource': event.get('resource', None)
},
{
'$set': {
'downtime': True
}
},
multi=True
)
# Takes care of the new downtime
self.cdowntime.reload(delta_beat=self.beat_interval)
# For every other case, check if the event is in downtime
else:
event['downtime'] = False
if (self.cdowntime.is_downtime(
event.get('component', ''),
event.get('resource', ''))):
event['downtime'] = True
self.logger.debug(
'Received event: {0}, and set downtime to {1}'.format(
event['rk'],
event['downtime']))
return event
def _beat(self):
now = int(time())
if self.last_stat + 60 <= now:
self.logger.debug(" + Send stats")
self.last_stat = now
evt_per_sec = 0
sec_per_evt = 0
if self.counter_event:
evt_per_sec = float(self.counter_event) / self.beat_interval
self.logger.debug(" + %0.2f event(s)/seconds", evt_per_sec)
if self.counter_worktime and self.counter_event:
sec_per_evt = self.counter_worktime / self.counter_event
self.logger.debug(" + %0.5f seconds/event", sec_per_evt)
# Submit event
if self.send_stats_event and self.counter_event != 0:
state = 0
if sec_per_evt > self.thd_warn_sec_per_evt:
state = 1
if sec_per_evt > self.thd_crit_sec_per_evt:
state = 2
perf_data_array = [{
'retention': self.perfdata_retention,
'metric': 'cps_evt_per_sec',
'value': round(evt_per_sec, 2),
'unit': 'evt'
}, {
'retention': self.perfdata_retention,
'metric': 'cps_sec_per_evt',
'value': round(sec_per_evt, 5),
'unit': 's',
'warn': self.thd_warn_sec_per_evt,
'crit': self.thd_crit_sec_per_evt
}]
self.logger.debug(" + State: {0}".format(state))
event = forger(connector="Engine",
connector_name="engine",
event_type="check",
source_type="resource",
resource=self.amqp_queue,
state=state,
state_type=1,
output="%0.2f evt/sec, %0.5f sec/evt" %
(evt_per_sec, sec_per_evt),
perf_data_array=perf_data_array)
try:
self.beat_amqp_publisher.canopsis_event(event)
except Exception as e:
self.logger.exception("Unable to send perfdata")
self.counter_error = 0
self.counter_event = 0
self.counter_worktime = 0
try:
self.beat()
except Exception as err:
self.logger.error("Beat raise exception: {0}".format(err))
self.logger.error(print_exc())
def prepare_event(
self,
display_name,
sla_measures,
output,
sla_state,
alerts_percent,
alerts_duration,
avail_duration,
timewindow_dict,
now
):
perf_data_array = []
# Compute metrics to publish
for state in self.states:
perf_data_array.append({
'metric': 'cps_pct_by_{}'.format(state),
'value': round(sla_measures[state] * 100.0, 2),
'max': 100
})
availability = (1.0 - alerts_percent) * 100.0
perf_data_array.append({
'metric': 'cps_avail',
'value': round(availability, 2),
'max': 100,
SLIDING_TIME: True
})
perf_data_array.append({
'metric': 'cps_avail_duration',
'value': avail_duration,
SLIDING_TIME: True
})
perf_data_array.append({
'metric': 'cps_alerts_duration',
'value': alerts_duration,
SLIDING_TIME: True
})
period_options = {
timewindow_dict['durationType']: timewindow_dict['value']
}
self.logger.debug(u'period options {}, now {}'.format(
period_options,
now
))
period = Period(**period_options)
periodic_timestamp = period.round_timestamp(now, next_period=True)
self.logger.debug(u'periodic timestamp {}'.format(periodic_timestamp))
event = forger(
connector='sla',
connector_name='engine',
event_type='sla',
source_type='resource',
component=display_name,
resource='sla',
state=sla_state,
output=output,
perf_data_array=perf_data_array,
display_name=display_name,
timestamp=periodic_timestamp
)
self.logger.info(u'publishing sla {}, states {}'.format(
display_name,
sla_measures
))
self.logger.debug(u'event : {}'.format(pp.pformat(event)))
return event
def prepare_event(
self,
display_name,
sla_measures,
output,
sla_state,
alerts_percent,
alerts_duration,
avail_duration,
timewindow_dict,
now
):
perf_data_array = []
# Compute metrics to publish
for state in self.states:
perf_data_array.append({
'metric': 'cps_pct_by_{}'.format(state),
'value': round(sla_measures[state] * 100.0, 2),
'max': 100
})
availability = (1.0 - alerts_percent) * 100.0
perf_data_array.append({
'metric': 'cps_avail',
'value': round(availability, 2),
'max': 100,
SLIDING_TIME: True
})
perf_data_array.append({
'metric': 'cps_avail_duration',
'value': avail_duration,
SLIDING_TIME: True
})
perf_data_array.append({
'metric': 'cps_alerts_duration',
'value': alerts_duration,
SLIDING_TIME: True
})
period_options = {
timewindow_dict['durationType']: timewindow_dict['value']
}
self.logger.debug(u'period options {}, now {}'.format(
period_options,
now
))
period = Period(**period_options)
periodic_timestamp = period.round_timestamp(now, next_period=True)
self.logger.debug(u'periodic timestamp {}'.format(periodic_timestamp))
event = forger(
connector='sla',
connector_name='engine',
event_type='sla',
source_type='resource',
component=display_name,
resource='sla',
state=sla_state,
output=output,
perf_data_array=perf_data_array,
display_name=display_name,
timestamp=periodic_timestamp
)
self.logger.info(u'publishing sla {}, states {}'.format(
display_name,
sla_measures
))
self.logger.debug(u'event : {}'.format(pp.pformat(event)))
return event
def init_tests(self):
"""
Create basic objects that will be manipulated.
"""
self.context_url = '{}/api/v2/context'.format(self.URL_BASE)
self.event1 = forger(event_type='check',
connector='cap_kirk',
connector_name='spock',
source_type='resource',
component='mc_coy',
resource='uhura',
state=2,
output='NCC_1701')
self.event2 = forger(event_type='check',
connector='cap_kirk',
connector_name='spock',
source_type='resource',
component='zulu',
resource='chekov',
state=3,
output='NCC_1701-B')
# Unlinked event
self.event3 = forger(event_type='check',
connector='picard',
connector_name='ricker',
source_type='resource',
component='laforge',
resource='worf',
state=1,
output='NCC_1701-D')
# Retrieve futur event id
get_entity_id = '{}/api/v2/context_graph/get_id/'.format(self.URL_BASE)
self.event1_id = self._send(url=get_entity_id,
method=Method.post,
data=dumps(self.event1)).json()
self.event2_id = self._send(url=get_entity_id,
method=Method.post,
data=dumps(self.event2)).json()
self.event3_id = self._send(url=get_entity_id,
method=Method.post,
data=dumps(self.event3)).json()
# Simple watcher (to insert)
self.watcher_1 = {
"description": "first gen",
"display_name": "st 1",
"enable": True,
"mfilter":
dumps({'_id': {
'$in': [self.event1_id, self.event2_id]
}}),
"_id": "watcher_first_gen"
}
# Simple watcher (to insert)
self.watcher_2 = {
"description": "pikes crew",
"display_name": "st 1 - pilot",
"enable": True,
"mfilter": dumps({'_id': self.event1_id}),
"_id": "watcher_pikes_crew"
}
self.watcher_3 = {
"description": "next gen",
"display_name": "st 2",
"enable": True,
"mfilter": dumps({'_id': {
'$in': [self.event3_id]
}}),
"_id": "watcher_next_gen"
}
now = int(time())
self.pbehavior1 = {
'name': 'imagine',
'author': 'lennon',
'filter_': {
'_id': self.event1_id
},
'rrule': None,
'tstart': now,
'tstop': now + 60 * 60,
'type_': 'pause'
}
self.pbehavior_watcher1 = {
'name': 'buck',
'author': 'mira',
'filter_': {
'_id': 'watcher_first_gen'
},
'rrule': None,
'tstart': now,
'tstop': now + 60 * 60,
'type_': 'pause'
}
def init_tests(self):
self.evt_ok1 = forger(event_type='check',
connector='gutamaya',
connector_name='imperial',
source_type='resource',
component='test_weatherfilter',
resource='ok1',
state=0,
output='cutter')
self.evt_ok2 = forger(event_type='check',
connector='gutamaya',
connector_name='imperial',
source_type='resource',
component='test_weatherfilter',
resource='ok2',
state=0,
output='cutter')
self.evt_paused = forger(event_type='check',
connector='gutamaya',
connector_name='imperial',
source_type='resource',
component='test_weatherfilter',
resource='paused',
state=0,
output='cutter')
self.evt_paused_id = 'paused/test_weatherfilter'
self.evt_maintenance = forger(event_type='check',
connector='faulcon',
connector_name='delacy',
source_type='resource',
component='test_weatherfilter',
resource='maintenance',
state=0,
output='cobraMKIII')
self.evt_maintenance_id = 'maintenance/test_weatherfilter'
now = int(time())
self.pb_paused = {
'name': 'gutamaya_imperial_interdictor',
'author': 'cmdr',
'filter_': {
'_id': self.evt_paused_id
},
'rrule': None,
'tstart': now,
'tstop': now + 60 * 60,
'type_': 'pause'
}
self.pb_maintenance = {
'name': 'cobramkIII',
'author': 'cmdr',
'filter_': {
'_id': self.evt_maintenance_id
},
'rrule': None,
'tstart': now,
'tstop': now + 60 * 60,
'type_': 'maintenance'
}
self.watcher_ok = WatcherModel("weatherfilter_ok", {
"_id": {
"$in": ["ok1/test_weatherfilter", "ok2/test_weatherfilter"]
}
}, "only_ok")
self.watcher_partial_pause = WatcherModel(
"weatherfilter_partial_pause", {
"_id": {
"$in":
["paused/test_weatherfilter", "ok1/test_weatherfilter"]
}
}, "partialy_paused")
self.watcher_full_pause = WatcherModel(
"weatherfilter_full_pause", {
"_id": {
"$in": [
"maintenance/test_weatherfilter",
"paused/test_weatherfilter"
]
}
}, "paused_maintenance")
self.watcher_full_pause_maintenance = WatcherModel(
"watcherfilter_full_pause_maintenance",
{"_id": {
"$in": [
"maintenance/test_weatherfilter",
]
}}, "maintenance")
self.watcher_mixed = WatcherModel(
"weatherfilter_mixed", {
"_id": {
"$in": [
"maintenance/test_weatherfilter",
"paused/test_weatherfilter", "ok1/test_weatherfilter"
]
}
}, "mixed")
def work(self, event, *args, **kargs):
logevent = None
ackremove = False
state = event.get('state', 0)
state_type = event.get('state_type', 1)
if event['event_type'] == 'ackremove':
# remove ack from event
# Ack remove information exists when ack is just removed
# And deleted if event is ack again
rk = event['ref_rk']
self.events_collection.update(
{'_id': rk},
{
'$set': {
'ack_remove': {
'author': event['author'],
'comment': event['output'],
'timestamp': time()
},
'ack': ''
},
'$unset': {
'ticket_declared_author': '',
'ticket_declared_date': '',
'ticket': '',
'ticket_date': ''
}
}
)
ackremove = True
# If event is of type ack, then ack reference event
if event['event_type'] == 'ack':
self.logger.debug(u'Ack event found, will proceed ack.')
rk = event.get('referer', event.get('ref_rk', None))
if event.get("source_type") == "component" and\
event.get("ack_resources") in [True, "true", "True"]:
# fetch not ok component's resources
component = event.get("component")
sub_res_query = {"component": component,
"state": {"$ne": "0"},
"source_type": "resource"}
result_cur = self.events_collection.find(sub_res_query)
for resource in result_cur:
sub_ack_event = {
"ref_rk": resource.get("_id"),
"author": event.get("author"),
"output": event.get("output"),
"authkey": event.get("authkey"),
"connector": resource.get("connector"),
"connector_name": resource.get("connector_name"),
"event_type": "ack",
"source_type": "resource",
"component": component,
"resource": resource.get("resource")
}
if "ticket" in event:
sub_ack_event["ticket"] = event.get("ticket")
try:
self.work_amqp_publisher.canopsis_event(
sub_ack_event,
exchange_name=self.acknowledge_on)
except Exception as e:
self.logger.exception("Unable to send ack event")
author = event['author']
self.logger.debug(dumps(event, indent=2))
if not rk:
self.logger.error(
'Cannot get acknowledged event, missing referer or ref_rk'
)
return event
for comment in self.comments:
if comment['comment'] in event['output']:
# An ack comment is contained into a defined comment
# Then let save referer key to the comment
# Set referer rk to last update date
self.objects_backend.update(
{'_id': comment['_id']},
{"$addToSet": {'referer_event_rks': {'rk': rk}}},
upsert=True)
self.logger.info(
'Added a referer rk to the comment {}'.format(
comment['comment']
)
)
ackts = int(time())
ack_info = {
'timestamp': event['timestamp'],
'ackts': ackts,
'rk': rk,
'author': author,
'comment': event['output']
}
# add rk to acknowledged rks
response = self.stbackend.find_and_modify(
query={'rk': rk, 'solved': False},
update={'$set': ack_info},
upsert=True,
full_response=True,
new=True
)
self.logger.debug(
u'Updating event {} with author {} and comment {}'.format(
rk,
author,
ack_info['comment']
)
)
ack_info['isAck'] = True
# Useless information for event ack data
del ack_info['ackts']
# clean eventual previous ack remove information
self.events_collection.update(
{
'_id': rk
}, {
'$set': {
'ack': ack_info,
},
'$unset': {
'ack_remove': '',
}
}
)
# When an ack status is changed
# Emit an event log
referer_event = self.storage.find_one(
mfilter={'_id': rk},
namespace='events'
)
if referer_event:
referer_event = referer_event.dump()
# Duration between event last state and acknolegement date
duration = ackts - referer_event.get(
'last_state_change', event['timestamp']
)
logevent = forger(
connector="Engine",
connector_name=self.etype,
event_type="log",
source_type=referer_event['source_type'],
component=referer_event['component'],
resource=referer_event.get('resource', None),
state=0,
state_type=1,
ref_rk=event['rk'],
output=u'Event {0} acknowledged by {1}'.format(
rk, author),
long_output=event['output'],
)
# Now update counters
ackhost = is_host_acknowledged(event)
# Cast response to ! 0|1
cvalues = int(not ackhost)
ack_event = deepcopy(self.ack_event)
ack_event['component'] = author
ack_event['perf_data_array'] = [
{
'metric': 'alerts_by_host',
'value': cvalues,
'type': 'COUNTER'
},
{
'metric': 'alerts_count{}'.format(
self.get_metric_name_adp(event)
),
'value': 1,
'type': 'COUNTER'
},
{
'metric': 'delay',
'value': duration,
'type': 'COUNTER'
}
]
try:
self.work_amqp_publisher.canopsis_event(
ack_event,
exchange_name=self.acknowledge_on)
except Exception as e:
self.logger.exception("Unable to send ack event")
self.logger.debug(u'Ack internal metric sent. {}'.format(
dumps(ack_event['perf_data_array'], indent=2)
))
for hostgroup in event.get('hostgroups', []):
ack_event = deepcopy(self.ack_event)
ack_event['perf_data_array'] = [
{
'metric': 'alerts',
'value': cvalues,
'type': 'COUNTER'
}
]
try:
self.work_amqp_publisher.canopsis_event(
ack_event,
exchange_name=self.acknowledge_on)
except Exception as e:
self.logger.exception("Unable to send ack event")
self.logger.debug(u'Reloading ack cache')
self.reload_ack_cache()
# If event is acknowledged, and went back to normal, remove the ack
# This test concerns most of case
# And could not perform query for each event
elif state == 0 and state_type == 1:
solvedts = int(time())
if event['rk'] in self.cache_acks:
self.logger.debug(
'Ack exists for this event, and has to be recovered.'
)
# We have an ack to process for this event
query = {
'rk': event['rk'],
'solved': False,
'ackts': {'$gt': -1}
}
ack = self.stbackend.find_one(query)
if ack:
ackts = ack['ackts']
self.stbackend.update(
query,
{
'$set': {
'solved': True,
'solvedts': solvedts
}
}
)
logevent = forger(
connector="Engine",
connector_name=self.etype,
event_type="log",
source_type=event['source_type'],
component=event['component'],
resource=event.get('resource', None),
state=0,
state_type=1,
ref_rk=event['rk'],
output=u'Acknowledgement removed for event {0}'.format(
event['rk']),
long_output=u'Everything went back to normal'
)
logevent['acknowledged_connector'] = event['connector']
logevent['acknowledged_source'] = event['connector_name']
logevent['acknowledged_at'] = ackts
logevent['solved_at'] = solvedts
# Metric for solved alarms
ack_event = deepcopy(self.ack_event)
ack_event['component'] = 'solved_alert'
ack_event['perf_data_array'] = [
{
'metric': 'delay',
'value': solvedts - ackts,
'unit': 's'
},
{
'metric': 'count',
'value': 1,
'type': 'COUNTER'
}
]
try:
self.work_amqp_publisher.canopsis_event(
ack_event,
exchange_name=self.acknowledge_on)
except Exception as e:
self.logger.exception("Unable to send ack event")
# If the event is in problem state,
# update the solved state of acknowledgement
elif ackremove or (state != 0 and state_type == 1):
self.logger.debug(u'Alert on event, preparing ACK statement.')
self.stbackend.find_and_modify(
query={'rk': event['rk'], 'solved': True},
update={'$set': {
'solved': False,
'solvedts': -1,
'ackts': -1,
'timestamp': -1,
'author': '',
'comment': ''
}}
)
if logevent:
self.logger.debug(u'publishing log event {}'.format(
dumps(logevent, indent=2)
))
try:
self.work_amqp_publisher.canopsis_event(
logevent,
exchange_name=self.acknowledge_on)
except Exception as e:
self.logger.exception("Unable to send log event")
return event
def work(self, event, *args, **kargs):
logevent = None
ackremove = False
state = event.get('state', 0)
state_type = event.get('state_type', 1)
if event['event_type'] == 'ackremove':
# remove ack from event
# Ack remove information exists when ack is just removed
# And deleted if event is ack again
rk = event['ref_rk']
self.events_collection.update({'_id': rk}, {
'$set': {
'ack_remove': {
'author': event['author'],
'comment': event['output'],
'timestamp': time()
},
'ack': ''
},
'$unset': {
'ticket_declared_author': '',
'ticket_declared_date': '',
'ticket': '',
'ticket_date': ''
}
})
ackremove = True
# If event is of type ack, then ack reference event
if event['event_type'] == 'ack':
self.logger.debug(u'Ack event found, will proceed ack.')
rk = event.get('referer', event.get('ref_rk', None))
if event.get("source_type") == "component" and\
event.get("ack_resources") in [True, "true", "True"]:
# fetch not ok component's resources
component = event.get("component")
sub_res_query = {
"component": component,
"state": {
"$ne": "0"
},
"source_type": "resource"
}
result_cur = self.events_collection.find(sub_res_query)
for resource in result_cur:
sub_ack_event = {
"ref_rk": resource.get("_id"),
"author": event.get("author"),
"output": event.get("output"),
"authkey": event.get("authkey"),
"connector": resource.get("connector"),
"connector_name": resource.get("connector_name"),
"event_type": "ack",
"source_type": "resource",
"component": component,
"resource": resource.get("resource")
}
if "ticket" in event:
sub_ack_event["ticket"] = event.get("ticket")
try:
self.work_amqp_publisher.canopsis_event(
sub_ack_event, exchange_name=self.acknowledge_on)
except Exception as e:
self.logger.exception("Unable to send ack event")
author = event['author']
self.logger.debug(dumps(event, indent=2))
if not rk:
self.logger.error(
'Cannot get acknowledged event, missing referer or ref_rk')
return event
for comment in self.comments:
if comment['comment'] in event['output']:
# An ack comment is contained into a defined comment
# Then let save referer key to the comment
# Set referer rk to last update date
self.objects_backend.update(
{'_id': comment['_id']},
{"$addToSet": {
'referer_event_rks': {
'rk': rk
}
}},
upsert=True)
self.logger.info(
'Added a referer rk to the comment {}'.format(
comment['comment']))
ackts = int(time())
ack_info = {
'timestamp': event['timestamp'],
'ackts': ackts,
'rk': rk,
'author': author,
'comment': event['output']
}
# add rk to acknowledged rks
response = self.stbackend.find_and_modify(
query={
'rk': rk,
'solved': False
},
update={'$set': ack_info},
upsert=True,
full_response=True,
new=True)
self.logger.debug(
u'Updating event {} with author {} and comment {}'.format(
rk, author, ack_info['comment']))
ack_info['isAck'] = True
# Useless information for event ack data
del ack_info['ackts']
# clean eventual previous ack remove information
self.events_collection.update({'_id': rk}, {
'$set': {
'ack': ack_info,
},
'$unset': {
'ack_remove': '',
}
})
# When an ack status is changed
# Emit an event log
referer_event = self.storage.find_one(mfilter={'_id': rk},
namespace='events')
if referer_event:
referer_event = referer_event.dump()
# Duration between event last state and acknolegement date
duration = ackts - referer_event.get('last_state_change',
event['timestamp'])
logevent = forger(
connector="Engine",
connector_name=self.etype,
event_type="log",
source_type=referer_event['source_type'],
component=referer_event['component'],
resource=referer_event.get('resource', None),
state=0,
state_type=1,
ref_rk=event['rk'],
output=u'Event {0} acknowledged by {1}'.format(rk, author),
long_output=event['output'],
)
# Now update counters
ackhost = is_host_acknowledged(event)
# Cast response to ! 0|1
cvalues = int(not ackhost)
ack_event = deepcopy(self.ack_event)
ack_event['component'] = author
ack_event['perf_data_array'] = [{
'metric': 'alerts_by_host',
'value': cvalues,
'type': 'COUNTER'
}, {
'metric':
'alerts_count{}'.format(self.get_metric_name_adp(event)),
'value':
1,
'type':
'COUNTER'
}, {
'metric': 'delay',
'value': duration,
'type': 'COUNTER'
}]
try:
self.work_amqp_publisher.canopsis_event(
ack_event, exchange_name=self.acknowledge_on)
except Exception as e:
self.logger.exception("Unable to send ack event")
self.logger.debug(u'Ack internal metric sent. {}'.format(
dumps(ack_event['perf_data_array'], indent=2)))
for hostgroup in event.get('hostgroups', []):
ack_event = deepcopy(self.ack_event)
ack_event['perf_data_array'] = [{
'metric': 'alerts',
'value': cvalues,
'type': 'COUNTER'
}]
try:
self.work_amqp_publisher.canopsis_event(
ack_event, exchange_name=self.acknowledge_on)
except Exception as e:
self.logger.exception("Unable to send ack event")
self.logger.debug(u'Reloading ack cache')
self.reload_ack_cache()
# If event is acknowledged, and went back to normal, remove the ack
# This test concerns most of case
# And could not perform query for each event
elif state == 0 and state_type == 1:
solvedts = int(time())
if event['rk'] in self.cache_acks:
self.logger.debug(
'Ack exists for this event, and has to be recovered.')
# We have an ack to process for this event
query = {
'rk': event['rk'],
'solved': False,
'ackts': {
'$gt': -1
}
}
ack = self.stbackend.find_one(query)
if ack:
ackts = ack['ackts']
self.stbackend.update(
query,
{'$set': {
'solved': True,
'solvedts': solvedts
}})
logevent = forger(
connector="Engine",
connector_name=self.etype,
event_type="log",
source_type=event['source_type'],
component=event['component'],
resource=event.get('resource', None),
state=0,
state_type=1,
ref_rk=event['rk'],
output=u'Acknowledgement removed for event {0}'.format(
event['rk']),
long_output=u'Everything went back to normal')
logevent['acknowledged_connector'] = event['connector']
logevent['acknowledged_source'] = event['connector_name']
logevent['acknowledged_at'] = ackts
logevent['solved_at'] = solvedts
# Metric for solved alarms
ack_event = deepcopy(self.ack_event)
ack_event['component'] = 'solved_alert'
ack_event['perf_data_array'] = [{
'metric': 'delay',
'value': solvedts - ackts,
'unit': 's'
}, {
'metric': 'count',
'value': 1,
'type': 'COUNTER'
}]
try:
self.work_amqp_publisher.canopsis_event(
ack_event, exchange_name=self.acknowledge_on)
except Exception as e:
self.logger.exception("Unable to send ack event")
# If the event is in problem state,
# update the solved state of acknowledgement
elif ackremove or (state != 0 and state_type == 1):
self.logger.debug(u'Alert on event, preparing ACK statement.')
self.stbackend.find_and_modify(query={
'rk': event['rk'],
'solved': True
},
update={
'$set': {
'solved': False,
'solvedts': -1,
'ackts': -1,
'timestamp': -1,
'author': '',
'comment': ''
}
})
if logevent:
self.logger.debug(u'publishing log event {}'.format(
dumps(logevent, indent=2)))
try:
self.work_amqp_publisher.canopsis_event(
logevent, exchange_name=self.acknowledge_on)
except Exception as e:
self.logger.exception("Unable to send log event")
return event
def work(self, event, *args, **kargs):
logevent = None
ackremove = False
state = event.get("state", 0)
state_type = event.get("state_type", 1)
if event["event_type"] == "ackremove":
# remove ack from event
# Ack remove information exists when ack is just removed
# And deleted if event is ack again
rk = event["ref_rk"]
self.events_collection.update(
{"_id": rk},
{
"$set": {
"ack_remove": {"author": event["author"], "comment": event["output"], "timestamp": time()}
},
"$unset": {
"ack": "",
"ticket_declared_author": "",
"ticket_declared_date": "",
"ticket": "",
"ticket_date": "",
},
},
)
ackremove = True
# If event is of type ack, then ack reference event
if event["event_type"] == "ack":
self.logger.debug(u"Ack event found, will proceed ack.")
rk = event.get("referer", event.get("ref_rk", None))
author = event["author"]
self.logger.debug(dumps(event, indent=2))
if not rk:
self.logger.error("Cannot get acknowledged event, missing referer or ref_rk")
return event
for comment in self.comments:
if comment["comment"] in event["output"]:
# An ack comment is contained into a defined comment
# Then let save referer key to the comment
# Set referer rk to last update date
self.objects_backend.update(
{"_id": comment["_id"]}, {"$addToSet": {"referer_event_rks": {"rk": rk}}}, upsert=True
)
self.logger.info("Added a referer rk to the comment {}".format(comment["comment"]))
ackts = int(time())
ack_info = {
"timestamp": event["timestamp"],
"ackts": ackts,
"rk": rk,
"author": author,
"comment": event["output"],
}
# add rk to acknowledged rks
response = self.stbackend.find_and_modify(
query={"rk": rk, "solved": False}, update={"$set": ack_info}, upsert=True, full_response=True, new=True
)
self.logger.debug(
u"Updating event {} with author {} and comment {}".format(rk, author, ack_info["comment"])
)
ack_info["isAck"] = True
# Useless information for event ack data
del ack_info["ackts"]
# clean eventual previous ack remove information
self.events_collection.update({"_id": rk}, {"$set": {"ack": ack_info}, "$unset": {"ack_remove": ""}})
# When an ack status is changed
# Emit an event log
referer_event = self.storage.find_one(mfilter={"_id": rk}, namespace="events")
if referer_event:
referer_event = referer_event.dump()
# Duration between event last state and acknolegement date
duration = ackts - referer_event.get("last_state_change", event["timestamp"])
logevent = forger(
connector="Engine",
connector_name=self.etype,
event_type="log",
source_type=referer_event["source_type"],
component=referer_event["component"],
resource=referer_event.get("resource", None),
state=0,
state_type=1,
ref_rk=event["rk"],
output=u"Event {0} acknowledged by {1}".format(rk, author),
long_output=event["output"],
)
# Now update counters
ackhost = is_host_acknowledged(event)
# Cast response to ! 0|1
cvalues = int(not ackhost)
ack_event = deepcopy(self.ack_event)
ack_event["component"] = author
ack_event["perf_data_array"] = [
{"metric": "alerts_by_host", "value": cvalues, "type": "COUNTER"},
{"metric": "alerts_count{}".format(self.get_metric_name_adp(event)), "value": 1, "type": "COUNTER"},
{"metric": "delay", "value": duration, "type": "COUNTER"},
]
publish(publisher=self.amqp, event=ack_event, exchange=self.acknowledge_on)
self.logger.debug(u"Ack internal metric sent. {}".format(dumps(ack_event["perf_data_array"], indent=2)))
for hostgroup in event.get("hostgroups", []):
ack_event = deepcopy(self.ack_event)
ack_event["perf_data_array"] = [{"metric": "alerts", "value": cvalues, "type": "COUNTER"}]
publish(publisher=self.amqp, event=ack_event, exchange=self.acknowledge_on)
self.logger.debug(u"Reloading ack cache")
self.reload_ack_cache()
# If event is acknowledged, and went back to normal, remove the ack
# This test concerns most of case
# And could not perform query for each event
elif state == 0 and state_type == 1:
solvedts = int(time())
if event["rk"] in self.cache_acks:
self.logger.debug("Ack exists for this event, and has to be recovered.")
# We have an ack to process for this event
query = {"rk": event["rk"], "solved": False, "ackts": {"$gt": -1}}
ack = self.stbackend.find_one(query)
if ack:
ackts = ack["ackts"]
self.stbackend.update(query, {"$set": {"solved": True, "solvedts": solvedts}})
logevent = forger(
connector="Engine",
connector_name=self.etype,
event_type="log",
source_type=event["source_type"],
component=event["component"],
resource=event.get("resource", None),
state=0,
state_type=1,
ref_rk=event["rk"],
output=u"Acknowledgement removed for event {0}".format(event["rk"]),
long_output=u"Everything went back to normal",
)
logevent["acknowledged_connector"] = event["connector"]
logevent["acknowledged_source"] = event["connector_name"]
logevent["acknowledged_at"] = ackts
logevent["solved_at"] = solvedts
# Metric for solved alarms
ack_event = deepcopy(self.ack_event)
ack_event["component"] = "solved_alert"
ack_event["perf_data_array"] = [
{"metric": "delay", "value": solvedts - ackts, "unit": "s"},
{"metric": "count", "value": 1, "type": "COUNTER"},
]
publish(publisher=self.amqp, event=ack_event, exchange=self.acknowledge_on)
# If the event is in problem state,
# update the solved state of acknowledgement
elif ackremove or (state != 0 and state_type == 1):
self.logger.debug(u"Alert on event, preparing ACK statement.")
self.stbackend.find_and_modify(
query={"rk": event["rk"], "solved": True},
update={
"$set": {"solved": False, "solvedts": -1, "ackts": -1, "timestamp": -1, "author": "", "comment": ""}
},
)
if logevent:
self.logger.debug(u"publishing log event {}".format(dumps(logevent, indent=2)))
publish(publisher=self.amqp, event=logevent, exchange=self.acknowledge_on)
return event
def init_tests(self):
"""
Create basic objects that will be manipulated.
"""
self.context_url = '{}/api/v2/context'.format(self.URL_BASE)
self.event1 = forger(
event_type='check',
connector='cap_kirk',
connector_name='spock',
source_type='resource',
component='mc_coy',
resource='uhura',
state=2,
output='NCC_1701'
)
self.event2 = forger(
event_type='check',
connector='cap_kirk',
connector_name='spock',
source_type='resource',
component='zulu',
resource='chekov',
state=3,
output='NCC_1701-B'
)
# Unlinked event
self.event3 = forger(
event_type='check',
connector='picard',
connector_name='ricker',
source_type='resource',
component='laforge',
resource='worf',
state=1,
output='NCC_1701-D'
)
# Retrieve futur event id
get_entity_id = '{}/api/v2/context_graph/get_id/'.format(self.URL_BASE)
self.event1_id = self._send(url=get_entity_id,
method=Method.post,
data=dumps(self.event1)).json()
self.event2_id = self._send(url=get_entity_id,
method=Method.post,
data=dumps(self.event2)).json()
self.event3_id = self._send(url=get_entity_id,
method=Method.post,
data=dumps(self.event3)).json()
# Simple watcher (to insert)
self.watcher_1 = {
"description": "first gen",
"display_name": "st 1",
"enable": True,
"mfilter": dumps({'_id': {'$in': [self.event1_id, self.event2_id]}}),
"_id": "watcher_first_gen"
}
# Simple watcher (to insert)
self.watcher_2 = {
"description": "pikes crew",
"display_name": "st 1 - pilot",
"enable": True,
"mfilter": dumps({'_id': self.event1_id}),
"_id": "watcher_pikes_crew"
}
self.watcher_3 = {
"description": "next gen",
"display_name": "st 2",
"enable": True,
"mfilter": dumps({'_id': {'$in': [self.event3_id]}}),
"_id": "watcher_next_gen"
}
now = int(time())
self.pbehavior1 = {
'name': 'imagine',
'author': 'lennon',
'filter_': {'_id': self.event1_id},
'rrule': None,
'tstart': now,
'tstop': now + 60 * 60,
'type_': 'pause'
}
self.pbehavior_watcher1 = {
'name': 'buck',
'author': 'mira',
'filter_': {'_id': 'watcher_first_gen'},
'rrule': None,
'tstart': now,
'tstop': now + 60 * 60,
'type_': 'pause'
}
