def email_failed_logins_alarm(now=None, failed_threshold=None, **dead_for):
"""
raise alert about failed `Citrix` logon events and send it via email
:arg int failed_threshold: the number of failed logons that will trigger the
alert
By default, this will be retrieved from the dynamic preference
`Failed logon events count alert threshold
<../../../admin/dynamic_preferences/globalpreferencemodel/?q=logon_alert_threshold>`__
See the :func:`email_dead_borgs_alert` for details about the other arguments.
This alert used the subscription at `Citrix logon alert
<../../../admin/ssl_cert_tracker/subscription/?q=Citrix+logon+alert>`__.
"""
if failed_threshold is None:
failed_threshold = get_preference(
'citrusborglogon__logon_alert_threshold')
if not dead_for:
time_delta = get_preference('citrusborglogon__logon_alert_after')
else:
time_delta = base_utils.MomentOfTime.time_delta(**dead_for)
now = base_utils.MomentOfTime.now(now)
data = raise_failed_logins_alarm(now=now,
time_delta=time_delta,
failed_threshold=failed_threshold)
if not data:
return (
'there were less than {} failed logon events between'
' {:%a %b %d, %Y %H:%M %Z} and {:%a %b %d, %Y %H:%M %Z}'.format(
failed_threshold, timezone.localtime(value=now),
timezone.localtime(now - time_delta)))
try:
return base_utils.borgs_are_hailing(
data=data,
subscription=base_utils.get_subscription('Citrix logon alert'),
logger=LOGGER,
time_delta=time_delta,
failed_threshold=failed_threshold)
except Exception as error:
raise error
def get_session():
"""
prepare and return a :class:`<requests.Session>` instance with all the
trimmings
#TODO: not useful until the orionsdk package is update =d to version 0.0.8
or higher
"""
session = requests.Session()
session.timeout = (get_preference('orionserverconn__orion_conn_timeout'),
get_preference('orionserverconn__orion_read_timeout'))
session.verify = get_preference('orionserverconn__orion_verify_ssl_cert')
retry = Retry(
total=get_preference('orionserverconn__orion_retry'),
read=get_preference('orionserverconn__orion_retry'),
connect=get_preference('orionserverconn__orion_retry'),
backoff_factor=get_preference('orionserverconn__orion_backoff_factor'),
status_forcelist=(500, 502, 504))
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https:/', adapter)
return session
def invoke_report_events_by_bot(report_interval=None, report_level=None):
"""
invoke tasks for mailing the events by bot reports
:arg `object` report_interval:
the time interval for which the report is calculated
it is either a :class:`datetime.timedelta` instance or a
:class:`dict` suitable for constructing a :class:`datetime.timedelta`
instance. it defaults to ``None`` and when it is ``None``,
the value is picked from
:class:`citrus_borg.dynamic_preferences_registry.ExchangeReportInterval`
:arg str report_level: similar to a log level, defaults to ``None``
when ``None``, the value is picked from
:class:`citrus_borg.dynamic_preferences_registry"""\
""".ExchangeDefaultErrorLevel`
:returns: the bots for which the report tasks have been invoked
:rtype: str
"""
if report_interval is None:
report_interval = get_preference('exchange__report_interval')
if report_level is None:
report_level = get_preference('exchange__report_level')
bots = list(
base_utils.get_base_queryset('mail_collector.mailhost',
enabled=True).values_list('host_name',
flat=True))
group(
report_events_by_bot.s(bot, report_interval, report_level)
for bot in bots)()
group(report_failed_events_by_bot.s(bot, report_interval)
for bot in bots)()
LOG.info('launched report tasks for exchange events by bot for bots: %s',
', '.join(bots))
def delete_expired_entries(data_source=None):
"""
delete rows marked as expired from the :class:`model
<django.db.models.Model>` defined by the `data_source` argument
This task assumes that the :class:`model <django.db.models.Model>`
defined by the data_source argument has a :class:`bool` attribute named
`is_expired`.
The task will actually delete entries only if the user preference defined
in :class:
`citrus_borg.dynamic_preferences_registry.LdapDeleteExpiredProbeLogEntries`
is so configured.
:arg str data_source: the name of a :class:`model
<django.db.models.Model>` in the `app_label.modelname` format
By default this argument will point to the
:class:`ldap_probe.models.LdapProbeLog` model
:returns: the name of the data source and the number of deleted rows
:raises:
:exc:`p_soc-auto.base.utils.UnknownDataTargetError` if the
`data_source` argument cannot be resolved to a valid
:class:`django.db.models.Model`
:exc:`exceptions.AttributeError` if the
:class:`django.db.models.Model` resolved from the `data_source`
argument doesn't have an `is_expired` attribute
"""
LOG.debug('kick delete_expired_entries')
if not get_preference('ldapprobe__ldap_delete_expired'):
LOG.info('the application is not configured to delete expired rows')
return
if data_source is None:
data_source = 'ldap_probe.LdapProbeLog'
try:
data_source = apps.get_model(data_source)
except utils.UnknownDataTargetError:
LOG.exception('Cannot delete entries for non-existent model %s',
data_source)
raise
if not hasattr(data_source, 'is_expired'):
raise AttributeError(
f'There are no expired rows in {data_source._meta.model_name}.'
' It does not have an is_expired field')
count_deleted = data_source.objects.filter(is_expired=True).all().delete()
LOG.info('Deleted %s expired rows from %s',
count_deleted, data_source._meta.verbose_name)
def _annotate_orion_url(cls, queryset=None):
return queryset.annotate(
orion_anchor=cls.sql_orion_anchor_field).annotate(
orion_url=Concat(
Value(get_preference('orionserverconn__orion_server_url')),
F('node__details_url'),
output_field=URLField()
)
)
def get_ux_alarms( # pylint: disable=too-many-arguments
now=None, group_by=GroupBy.NONE,
time_delta=get_preference('citrusborgux__ux_alert_interval'),
ux_alert_threshold=get_preference('citrusborgux__ux_alert_threshold'),
annotate_url=True, annotate_details_url=True, **details):
"""
get the user experience alert data
"""
queryset = raise_ux_alarm(
now=now, group_by=group_by, time_delta=time_delta,
ux_alert_threshold=ux_alert_threshold)
if annotate_url:
queryset = base_utils.url_annotate(queryset)
if annotate_details_url:
queryset = base_utils.details_url_annotate(queryset, **details)
return queryset
def __init__(
self, *args,
verify=get_preference('orionserverconn__orion_verify_ssl_cert')):
"""
override the parent constructor to prefer defaults specific to the
target server
"""
if not args:
args = DST_DEFAULTS
super().__init__(*args, verify=verify)
def perf_warn(self):
"""
flag for considering if an instance of this class must trigger a
performance warning
:returns: `True` if there is a timing from the probe above the warn
threshold, and ldap_perf_raise_all is `True`.
:rtype: bool
"""
return get_preference('ldapprobe__ldap_perf_raise_all')\
and self._perf_level(self.node_perf_bucket.avg_warn_threshold)
def raise_ldap_probe_failed_alert(instance_pk=None, subscription=None):
"""
raise an email alert for a failed instance of the
:class:`ldap_probe.models.LdapProbeLog` model
:arg int instance_pk: the primary key of the instance
:arg str subscription: the value of the :attr:`subscription
<p_soc_auto_base.models.Subscription.subscription>` attribute
used to retrieve the
:class:`p_soc_auto_base.models.Subscription` instance required
for raising this alert via email
"""
if subscription is None:
subscription = get_preference('ldapprobe__ldap_error_subscription')
return _raise_ldap_alert(
instance_pk=instance_pk,
subscription=Subscription.get_subscription(subscription),
level=get_preference('commonalertargs__error_level'))
def query(cls, orion_query, **params):
"""
query the `Orion` server
:arg str query: the query string
:arg dict params: the query params
:returns: the data that matches the query
:rtype: dict
"""
# need to configure the SESSION object here because the
# user configurable settings will not work if used at the
# module level
SESSION.auth = (get_preference('orionserverconn__orion_user'),
get_preference('orionserverconn__orion_password'))
SESSION.verify = get_preference(
'orionserverconn__orion_verify_ssl_cert')
response = SESSION.post(
'{}/Query'.format(
get_preference('orionserverconn__orion_rest_url')),
data=json.dumps(dict(query=orion_query, parameters=params),
default=serialize_custom_json),
timeout=(get_preference('orionserverconn__orion_conn_timeout'),
get_preference('orionserverconn__orion_read_timeout')))
response.raise_for_status()
return response.json()['results']
def login_states_by_site_host_hour(
now=None,
time_delta=get_preference(
'citrusborgevents__ignore_events_older_than'),
site=None,
host_name=None):
"""
:returns: a :class:`django.db.models.query.QuerySet` based on the
:class:`citru_borg.models.WinlogbeatHost` `annotated
<https://docs.djangoproject.com/en/2.2/ref/models/querysets/#annotate>`__
with all the `aggregations
<https://docs.djangoproject.com/en/2.2/topics/db/aggregation/#aggregation>`__
extracted from the :class:`citrus_borg.models.WinlogEvent` model
calculated on a per hour basis for the interval defined by the
`now` and `time_delta` arguments
:arg datetime.datetime now: the initial moment
By default the initial moment is the value returned by
:meth:`django.utils.timezone.now`
:arg datetime.timedelta time_delta: the time interval to consider
By default, this will be retrieved from the dynamic preference
`Ignore events created older than
<../../../admin/dynamic_preferences/globalpreferencemodel/?q=ignore_events_older_than>`__
:arg str site: if present, filter the data that will be returned by `site`,
otherwise return data for all known sites
:arg str host_name: if present, filter the data that will be returned by
'host_name`, otherwise return data for all known hosts
Be careful when using the `site` and `host_name` arguments together.
It is possible to come up with a combination of values for which
no data can be returned (one filters by a host that lives at a site
that was filtered out already). This function considers such a
combination valid and will happily return an empty
:class:`django.db.models.query.QuerySet` when invoked with such a
combination
:raises: :exc:`Exception` when failing
"""
try:
queryset = _by_site_host_hour(now, time_delta, site, host_name)
except Exception as error:
raise error
return queryset.order_by('site__site', 'host_name', '-hour')
def raise_site_not_configured_for_bot():
"""
email alerts if there are exchange bots with mis-configured site info
"""
data = models.MailHost.objects.filter(
Q(site__isnull=True) | Q(site__site__iexact='site.not.exist')).\
exclude(host_name__iexact='host.not.exist')
if data.exists():
data = base_utils.url_annotate(data)
if not data and not get_preference('exchange__empty_alerts'):
LOG.info('all exchange bots are properly configured')
return
if Email.send_email(
data=data,
subscription=Subscription.get_subscription('Exchange bot no site'),
level=get_preference('exchange__server_error')):
LOG.info('emailed alert for mis-configured Exchange bots')
LOG.warning('cannot email alert for mis-configured Exchange bots')
def orion_node_url(self):
"""
absolute `SolarWinds Orion <https://www.solarwinds.com/solutions/orion>`__
`URL` for the network node serving this :class:`SslCertificate` instance
"""
orion_node = OrionNode.objects.filter(orion_id=self.orion_id)
if orion_node.exists():
orion_node = orion_node.values('node_caption', 'details_url')[0]
return '<a href="%s%s">%s on Orion</>' % (
get_preference('orionserverconn__orion_server_url'),
orion_node.get('details_url'), orion_node.get('node_caption'))
return 'acquired outside the Orion infrastructure'
def expire_events():
"""
task that marks :class:`citrus_borg.models.WinlogEvent` instances as
expired; it will also delete `expired` instances if so configured
This task is configured by:
* `Mark Events As Expired If Older Than
<../../../admin/dynamic_preferences/globalpreferencemodel/?q=expire_events_older_than>`__
This preference is defined by
:class:`citrus_borg.dynamic_preferences_registry.ExpireEvents`
* `Delete Expired Events
<../../../admin/dynamic_preferences/globalpreferencemodel/?q=delete_expired_events>`__
The preference is defined by
:class:`citrus_borg.dynamic_preferences_registry.DeleteExpireEvents`
:returns: a status string about how many events were handles by the task,
the types of operations performed, and the threshold used for marking
events as expired (see `Mark Events As Expired If Older Than
<../../../admin/dynamic_preferences/globalpreferencemodel/?q=expire_events_older_than>`__)
:rtype: str
"""
expired = WinlogEvent.objects.filter(
created_on__lt=timezone.now() -
get_preference('citrusborgevents__expire_events_older_than')).update(
is_expired=True)
if get_preference('citrusborgevents__delete_expired_events'):
WinlogEvent.objects.filter(is_expired=True).all().delete()
return 'deleted %s events accumulated over the last %s' % (
expired,
get_preference('citrusborgevents__expire_events_older_than'))
return 'expired %s events accumulated over the last %s' % (
expired, get_preference('citrusborgevents__expire_events_older_than'))
def save_citrix_login_event(borg):
"""
Save data received from a citrix bot.
:param borg: Message received from bot.
:raises: Any exception raised during
"""
def reraise(msg):
LOG.exception('%s %s.', msg, borg)
# This function should only be called from inside an except block
# If an Exception is not being handled it will cause a RuntimeError
raise # pylint: disable=misplaced-bare-raise
event_host = WinlogbeatHost.get_or_create_from_borg(borg)
event_broker = KnownBrokeringDevice.get_or_create_from_borg(borg)
try:
event_source = AllowedEventSource.objects.get(
source_name=borg.event_source)
except AllowedEventSource.DoesNotExist:
reraise('Cannot match event source for event')
try:
windows_log = WindowsLog.objects.get(log_name=borg.windows_log)
except WindowsLog.DoesNotExist:
reraise('Cannot match windows log info for event')
user = get_or_create_user(get_preference('citrusborgcommon__service_user'))
winlogevent = WinlogEvent(
source_host=event_host,
record_number=borg.record_number,
event_source=event_source,
windows_log=windows_log,
event_state=borg.borg_message.state,
xml_broker=event_broker,
event_test_result=borg.borg_message.test_result,
storefront_connection_duration
=borg.borg_message.storefront_connection_duration,
receiver_startup_duration=borg.borg_message.receiver_startup_duration,
connection_achieved_duration
=borg.borg_message.connection_achieved_duration,
logon_achieved_duration=borg.borg_message.logon_achieved_duration,
logoff_achieved_duration=borg.borg_message.logoff_achieved_duration,
failure_reason=borg.borg_message.failure_reason,
failure_details=borg.borg_message.failure_details,
raw_message=borg.borg_message.raw_message,
event_id=borg.event_id,
timestamp=borg.timestamp,
created_by=user, updated_by=user)
winlogevent.save()
LOG.info('saved event: %s', winlogevent.uuid)
def email_failed_login_sites_report(now=None, send_no_news=False,
**reporting_period):
"""
spawn an instance of the :func:`email_failed_login_site_report` task for the
`site` and `borg_name` combination.
If `site` and/or `borg_name` are `None`, spawn an instance of the
:func:`email_failed_login_site_report` task for each `enabled` `site` and
`borg_name` known to the system
:arg str site: if `None`, spawn tasks for all sites
:arg str borg_name: the short host name of the bot host
If `None`, spawn a task for each `borg_name` on the `site`.
Note that it is possible to pick `site` and `borg_name` combinations
that will result in no data being generated.
In most cases this `task` is used to spawn an instance of the
:func:`email_login_ux_summary` task for each `enabled` bot on each
`enabled` site known to the system.
:arg now: see :func:`email_dead_borgs_alert`
:arg bool send_no_news: see :func:`email_dead_borgs_alert`
:arg dict reporting_period: see the `dead_for` argument of the
:func:`email_dead_borgs_alert` task
If this argument is not present, the reporting interval is picked from
dynamic preference `Logon events reporting period
<../../../admin/dynamic_preferences/globalpreferencemodel/"""\
"""?q=logon_report_period>`__
"""
if not reporting_period:
time_delta = get_preference('citrusborglogon__logon_report_period')
else:
time_delta = base_utils.MomentOfTime.time_delta(**reporting_period)
borg_names = WinlogbeatHost.active\
.order_by('host_name').values_list('host_name', flat=True)
group(email_failed_login_site_report.s(now, time_delta,
send_no_news, borg_name) for
borg_name in borg_names)()
LOG.info('bootstrapped failed login reports for %s', borg_names)
def report_failed_events_by_bot(bot, report_interval):
"""
send out report for failed events for a bot over a given duration measured
back from the moment of the call via email
:arg str bot: the host (short) name of the bot to report on
:arg `object` report_interval: the time interval for which the report is
calculated. it is either a :class:`datetime.timedelta` instance or a
:class:`dict` suitable for constructing a :classL`datetime.timedelta`
instance, for instance {'hours': 1, 'seconds': 1}
:returns: the result of the email send operation
:rtype: str
:raises:
:exc:`Exception` if an exception was thrown while sending the alert
"""
subscription = base_utils.get_subscription(
'Exchange Failed Send Receive By Bot')
data = queries.dead_bodies(
data_source='mail_collector.mailbotmessage',
filter_exp='event__event_registered_on__gte',
not_seen_after=base_utils.MomentOfTime.past(
time_delta=report_interval),
event__source_host__host_name=bot,
event__event_status__iexact='fail').\
order_by('-mail_message_identifier', 'event__event_type_sort')
try:
ret = base_utils.borgs_are_hailing(
data=data, subscription=subscription, logger=LOGGER,
time_delta=report_interval,
level=get_preference('exchange__server_error'), bot=bot)
except Exception as error:
raise error
if ret:
return (
'emailed exchange failed send receive events report for bot %s'
% bot)
return (
'could not email exchange failed send receive events report for bot %s'
% bot)
def orion_node_url(self):
"""
:returns: an instance property with the `URL` of the
`Django Admin change` form for the
:class:`orion_integration.models.OrionNode` instance representing
the host
"""
orion_node = OrionNode.objects.filter(orion_id=self.orion_id)
if orion_node.exists():
orion_node = orion_node.values('node_caption', 'details_url')[0]
return '<a href="%s%s">%s on Orion</>' % (
get_preference('orionserverconn__orion_server_url'),
orion_node.get('details_url'), orion_node.get('node_caption'))
return 'acquired outside the Orion infrastructure'
def get_dead_brokers(now=None, time_delta=None):
"""
get the `Citrix` session hosts that have not serviced any requests during
the interval defined by the arguments
This function is very similar to :func:`get_dead_bots`.
The only differences are:
* The data is pulled from the
:class:`citrus_borg.models.KnownnBrokeringDevice`
* The default value for the `time_delta` argument is picked from the
dynamic preference `Session host not seen alert threshold
<../../../admin/dynamic_preferences/globalpreferencemodel/?q=dead_session_host_after>`__
"""
if now is None:
now = timezone.now()
if time_delta is None:
time_delta = get_preference('citrusborgnode__dead_session_host_after')
if not isinstance(now, datetime.datetime):
raise TypeError('%s type invalid for %s' % (type(now), now))
if not isinstance(time_delta, datetime.timedelta):
raise TypeError('%s type invalid for %s' %
(type(time_delta), time_delta))
live_brokers = list(
WinlogEvent.objects.filter(
created_on__gt=now - time_delta,
event_state__iexact='successful').distinct().values_list(
'xml_broker__broker_name', flat=True))
all_brokers = list(
KnownBrokeringDevice.objects.filter(enabled=True).values_list(
'broker_name', flat=True))
dead_brokers = set(all_brokers).symmetric_difference(set(live_brokers))
dead_brokers = KnownBrokeringDevice.objects.\
filter(broker_name__in=list(dead_brokers))
if dead_brokers.exists():
dead_brokers = dead_brokers.order_by('last_seen')
return dead_brokers
def get_queryset(self):
"""
override :meth:`django.db.models.Manager.get_queryset`
See `Modifying a manager's initial QuerySet
<https://docs.djangoproject.com/en/2.2/topics/db/managers/#modifying-a-manager-s-initial-queryset>`__
in the `Django` docs.
:returns: a :class:`django.db.models.query.QuerySet` with the
:class:`OrionNode` instances that are `Windows domain
controllers`
"""
return super().get_queryset().\
filter(program_application_type__icontains=get_preference(
'orionfilters__domaincontroller')).\
exclude(enabled=False)
def get_queryset(self):
"""
override :meth:`django.db.models.Manager.get_queryset`
See `Modifying a manager's initial QuerySet
<https://docs.djangoproject.com/en/2.2/topics/db/managers/#modifying-a-manager-s-initial-queryset>`__
in the `Django` docs.
:returns: a :class:`django.db.models.query.QuerySet` with the
:class:`OrionNode` instances that map to `Orion` nodes tagged as
`Cerner-CST` nodes
"""
return super().get_queryset().filter(
program_application__exact=get_preference(
'orionfilters__cerner_cst'))
def report_failed_events_by_site(site, report_interval):
"""
send out report with failed events for a site via email
:arg str site: the host (short) name of the bot to report on
:arg `object` report_interval: the time interval for which the report is
calculated. it is either a :class:`datetime.timedelta` instance or a
:class:`dict` suitable for constructing a :class:`datetime.timedelta`
instance
:returns: the result of the email send operation
:rtype: str
:raises:
:exc:`Exception` if an exception was thrown while sending the alert
"""
subscription = Subscription.get_subscription(
'Exchange Failed Send Receive By Site')
data = queries.dead_bodies(
data_source='mail_collector.mailbotmessage',
filter_exp='event__event_registered_on__gte',
not_seen_after=base_utils.MomentOfTime.past(
time_delta=report_interval),
event__source_host__site__site=site,
event__event_status__iexact='fail').\
order_by('-mail_message_identifier', 'event__event_type_sort')
if Email.send_email(data=data,
subscription=subscription,
time_delta=report_interval,
level=get_preference('exchange__server_error'),
site=site):
LOG.info(
'emailed exchange failed send receive events report for site'
' %s', site)
return
LOG.warning(
'could not email exchange failed send receive events report for'
' site %s', site)
def get_logins_by_event_state_borg_hour(now=None, time_delta=None):
"""
get the number of failed events and successful events during the interval
defined by the arguments for each monitoring site aggregated by hour
:arg datetime.datetime now: the initial moment
By default the initial moment is the value returned by
:meth:`django.utils.timezone.now`
:arg datetime.timedelta time_delta: the time interval to consider
By default, this will be retrieved from the dynamic preference
`Dead if not seen for more than
<../../../admin/dynamic_preferences/globalpreferencemodel/?q=dead_after>`__
:returns: a :class:`django.db.models.query.QuerySet` based on the
:class:`citrus_borg.models.WinlogbeatHost` model
"""
if now is None:
now = timezone.now()
if time_delta is None:
time_delta = get_preference('citrusborgcommon__dead_after')
if not isinstance(now, datetime.datetime):
raise TypeError('%s type invalid for %s' % (type(now), now))
if not isinstance(time_delta, datetime.timedelta):
raise TypeError('%s type invalid for %s' %
(type(time_delta), time_delta))
return WinlogbeatHost.objects.\
filter(winlogevent__created_on__gt=now - time_delta).\
annotate(hour=TruncHour('winlogevent__created_on')).values('hour').\
annotate(
failed_events=Count(
'winlogevent__event_state',
filter=Q(winlogevent__event_state__iexact='failed'))).\
annotate(
successful_events=Count(
'winlogevent__event_state',
filter=Q(winlogevent__event_state__iexact='successful'))).\
order_by('-hour', 'site__site')
def get_dead_sites(now=None, time_delta=None):
"""
ge the remote sites from where no `ControlUp` events have been delivered
during the interval defined by the arguments
This function is very similar to :func:`get_dead_bots`.
The only differences are:
* The data is pulled from the
:class:`citrus_borg.models.BorgSite`
* The default value for the `time_delta` argument is picked from the
dynamic preference `Site not seen alert threshold
<../../../admin/dynamic_preferences/globalpreferencemodel/?q=dead_site_after>`__
"""
if now is None:
now = timezone.now()
if time_delta is None:
time_delta = get_preference('citrusborgnode__dead_site_after')
if not isinstance(now, datetime.datetime):
raise TypeError('%s type invalid for %s' % (type(now), now))
if not isinstance(time_delta, datetime.timedelta):
raise TypeError('%s type invalid for %s' %
(type(time_delta), time_delta))
live_sites = list(
WinlogEvent.objects.filter(created_on__gt=now -
time_delta).distinct().values_list(
'source_host__site__site', flat=True))
all_sites = list(
BorgSite.objects.filter(enabled=True).values_list('site', flat=True))
dead_sites = set(all_sites).symmetric_difference(set(live_sites))
dead_sites = BorgSite.objects.filter(site__in=list(dead_sites)).distinct()
if dead_sites.exists():
dead_sites = dead_sites.order_by('winlogbeathost__last_seen')
return dead_sites
def email_dead_servers_report(now=None, send_no_news=False, **dead_for):
"""
generate and email reports about `Citrix` application servers that have not
service any requests during the time interval defined by arguments `now` and
`dead_for` via email
This task is almost identical to :func:`email_dead_borgs_alert` with the
exception of the default value for the :class:`datetime.timedelta` object
created when `dead_for` is not present.
This value is picked from dynamic preference `Reporting period for dead nodes
<../../../admin/dynamic_preferences/globalpreferencemodel/?q=node_forgotten_after>`__
This task used the subscription at `Missing Citrix farm hosts
<../../../admin/ssl_cert_tracker/subscription/?q=Missing+Citrix+farm+hosts>`__
to render the emails being sent.
"""
now = base_utils.MomentOfTime.now(now)
if not dead_for:
time_delta = get_preference('citrusborgnode__node_forgotten_after')
else:
time_delta = base_utils.MomentOfTime.time_delta(**dead_for)
data = get_dead_brokers(now=now, time_delta=time_delta)
if not data and send_no_news:
return (
'all known Cerner session servers were active between'
' {:%a %b %d, %Y %H:%M %Z} and {:%a %b %d, %Y %H:%M %Z}'.format(
timezone.localtime(value=now),
timezone.localtime(now - time_delta)))
try:
return base_utils.borgs_are_hailing(
data=data,
subscription=base_utils.get_subscription(
'Missing Citrix farm hosts'),
logger=LOGGER,
time_delta=time_delta)
except Exception as error:
raise error
def email_dead_sites_report(now=None, send_no_news=False, **dead_for):
"""
generate and email reports about remote sites where`Citrix` monitoring bots
have not been seen within the time interval defined by arguments `now` and
`dead_for` via email
This task is almost identical to :func:`email_dead_borgs_alert` with the
exception of the default value for the :class:`datetime.timedelta` object
created when `dead_for` is not present.
This value is picked from dynamic preference `Reporting period for dead nodes
<../../../admin/dynamic_preferences/globalpreferencemodel/?q=node_forgotten_after>`__
This task used the subscription at `Dead Citrix client sites
<../../../admin/ssl_cert_tracker/subscription/?q=Dead+Citrix+client+sites>`__
to render the emails being sent.
"""
now = base_utils.MomentOfTime.now(now)
if not dead_for:
time_delta = get_preference('citrusborgnode__node_forgotten_after')
else:
time_delta = base_utils.MomentOfTime.time_delta(**dead_for)
data = get_dead_sites(now=now, time_delta=time_delta)
if not data and send_no_news:
return (
'at least one monitoring bot on each site was active between'
' {:%a %b %d, %Y %H:%M %Z} and {:%a %b %d, %Y %H:%M %Z}'.format(
timezone.localtime(value=now),
timezone.localtime(now - time_delta)))
try:
return base_utils.borgs_are_hailing(
data=data,
subscription=base_utils.get_subscription(
'Dead Citrix client sites'),
logger=LOGGER,
time_delta=time_delta)
except Exception as error:
raise error
def invoke_raise_citrix_slow_alert(sender, instance, *args, **kwargs):
"""
evaluate whether the :class:`citrus_borg.models.WinlogEvent` instance
is reporting slower than expected timings and invoke the `Celery task` that
is responsible for dispatching the alert
"""
alertable_timings = [
instance.storefront_connection_duration,
instance.logon_achieved_duration, instance.receiver_startup_duration,
instance.connection_achieved_duration,
instance.receiver_startup_duration
]
# TODO alternately just check if the test passed, if it did there should be
# timings if not is it worth alerting?
# remove None values
alertable_timings = [d for d in alertable_timings if d]
threshold = get_preference('citrusborgux__ux_alert_threshold')
if any(timing > threshold for timing in alertable_timings):
LOG.info('Slowdown on %s', instance.source_host.host_name)
raise_citrix_slow_alert.delay(instance.id, threshold.total_seconds())
def raise_failed_event_by_mail(event_pk):
"""
send an alert email for failed events
:arg int event_pk:
the primary key of the :class:`mail_collector.models.MailBotLogEvent`
instance with the failed event
:returns: the result of the email operation
:rtype: str
:raises: :exc:`Exception` if an error is thrown by the email send op
"""
data = models.MailBotLogEvent.objects.filter(pk=event_pk)
subscription = base_utils.get_subscription('Exchange Client Error')
# let's cache some data to avoid evaluating the queryset multiple times
data_extract = data.values(
'uuid', 'event_type',
'source_host__site__site', 'source_host__host_name')[0]
try:
ret = base_utils.borgs_are_hailing(
data=data, subscription=subscription, logger=LOGGER,
level=get_preference('exchange__server_error'),
event_type=data_extract.get('event_type'),
site=data_extract.get('source_host__site__site'),
bot=data_extract.get('source_host__host_name'))
except Exception as error:
raise error
if ret:
return 'raised email alert for event %s' % str(
data_extract.get('uuid'))
return 'could not raise email alert for event %s' % str(
data_extract.get('uuid'))
def email_dead_borgs_report(now=None, send_no_news=False, **dead_for):
"""
generate and email reports about `Citrix` monitoring bots that have not
been seen within the time interval defined by arguments `now` and
`dead_for` via email
This task is almost identical to :func:`email_dead_borgs_alert` with the
exception of the default value for the :class:`datetime.timedelta` object
created when `dead_for` is not present. This value is picked from dynamic
preference `Reporting period for dead nodes
<../../../admin/dynamic_preferences/globalpreferencemodel/"""\
"""?q=node_forgotten_after>`__.
This task used the subscription at `Dead Citrix monitoring bots
<../../../admin/p_soc_auto_base/subscription/"""\
"""?q=Dead+Citrix+monitoring+bots>`__ to render the emails being sent.
"""
now = base_utils.MomentOfTime.now(now)
if not dead_for:
time_delta = get_preference('citrusborgnode__node_forgotten_after')
else:
time_delta = base_utils.MomentOfTime.time_delta(**dead_for)
data = get_dead_bots(now=now, time_delta=time_delta)
if not data and send_no_news:
return (
'all monitoring bots were active between'
' {:%a %b %d, %Y %H:%M %Z} and {:%a %b %d, %Y %H:%M %Z}'.
format(timezone.localtime(value=now),
timezone.localtime(now - time_delta))
)
return Email.send_email(
data=data,
subscription=Subscription.get_subscription(
'Dead Citrix monitoring bots'),
time_delta=time_delta)
def maintain_ad_orion_nodes():
"""
synchronize :class:`ldap_probe.models.OrionADNode` with
:class:`orion_integration.models.OrionDomainControllerNode`
This is needed because `AD` `Orion` nodes may change outside this
application
"""
known_nodes_model = apps.get_model('ldap_probe.orionadnode')
new_nodes_model = apps.get_model(
'orion_integration.oriondomaincontrollernode')
known_node_ids = known_nodes_model.objects.values_list(
'node_id', flat=True)
new_nodes = new_nodes_model.objects.exclude(id__in=known_node_ids)
if not new_nodes.exists():
LOG.debug('did not find any unknown AD nodes in Orion')
return
service_user = get_or_create_user(
name=get_preference('ldapprobe__service_user'))
ldap_bind_cred = apps.get_model('ldap_probe.ldapbindcred').default()
for node in new_nodes:
new_ad_orion_node = known_nodes_model(
node=node, ldap_bind_cred=ldap_bind_cred,
created_by=service_user, updated_by=service_user)
try:
new_ad_orion_node.save()
except Exception as error:
LOG.exception(error)
raise error
LOG.info('created %s AD nodes from Orion', new_nodes.count())
