def dispatch_dupe_nodes_reports():
"""
`Celery task` for generating a report with duplicate `AD` nodes
on the `Orion` server
:returns: a :class:`str` object interpreting the `return` of
:meth:`ssl_cert_tracker.lib.Email.send`
"""
LOG.debug('invoking the duplicate ad nodes in orion report')
try:
data = apps.get_model('ldap_probe.orionadnode').\
report_duplicate_nodes()
except Exception as error:
LOG.exception('invoking the duplicate ad nodes in orion report raises'
' error %s', str(error))
raise error
subscription = Subscription.get_subscription(
get_preference('ldapprobe__ldap_orion_dupes_ad_nodes_subscription'))
info_level = get_preference('commonalertargs__info_level')
if Email.send_email(data=data, subscription=subscription,
level=info_level):
LOG.info('dispatched the duplicate ad nodes in orion report')
return
LOG.warning('could not dispatch the duplicate ad nodes in orion report')
def dispatch_non_orion_ad_nodes_report():
"""
`Celery task` for generating the report about `AD` network nodes
not defined in `Orion`
:returns: information about the arguments used to call the task and the
result of :meth:`ssl_cert_tracker.lib.Email.send`
:rtype: str
:raises: :exc:`Exception` to allow `Celery` to deal with
errors
"""
LOG.debug('invoking the non orion ad nodes report')
try:
data = apps.get_model('ldap_probe.nonorionadnode').active
except Exception as error:
LOG.exception('invoking the non orion ad nodes report raises error %s',
str(error))
raise error
subscription = Subscription.get_subscription(
get_preference('ldapprobe__ldap_non_orion_ad_nodes_subscription'))
warn_level = get_preference('commonalertargs__warn_level')
if Email.send_email(data=data, subscription=subscription, level=warn_level):
LOG.info('dispatched the non orion ad nodes report')
return
LOG.warning('could not dispatch the non orion ad nodes report')
def email_failed_login_site_report(now, reporting_period, send_no_news, host):
"""
prepare and email a report with the `Citrix` failed logins details for the
bot combination specified in `host`
:arg tuple host: the bot for which the report is prepared
:arg now: see :func:`email_dead_borgs_alert`
:arg bool send_no_news: see :func:`email_dead_borgs_alert`
:arg dict reporting_period: see the `dead_for` argument of the
:func:`email_dead_borgs_alert` task
This report uses the subscription at `Citrix Failed Logins per Report
<../../../admin/p_soc_auto_base/subscription/"""\
"""?q=Citrix+Failed+Logins+per+Report>`__ to render the emails being sent.
"""
now = base_utils.MomentOfTime.now(now)
site = WinlogbeatHost.objects.get(host_name__exact=host).site
data = get_failed_events(
now=now, time_delta=reporting_period, site=site, host_name=host)
if not data and send_no_news:
LOG.info('there were no failed logon events on the %s bot in %s between'
' %s and %s', host, site,
timezone.localtime(value=now).isoformat(),
timezone.localtime(now - reporting_period).isoformat())
return 0 # Did not send email.
return Email.send_email(
data=data,
subscription=Subscription.get_subscription(
'Citrix Failed Logins per Site Report'),
time_delta=reporting_period, site=site, host_name=host)
def report_mail_between_domains(only_fails=False, subscription=None):
"""
task to run reports about mail between domains
:arg str subscription:
the email subscription. default: 'Mail Verification Report'
:arg bool only_fails: only report the fails, default: ``False``
:returns: the result of the email send operation
:rtype: str
:raises: :exc:`Exception` if an exception was thrown while sending the alert
"""
if subscription is None:
subscription = 'Mail Verification Report'
subscription = Subscription.get_subscription(subscription)
queryset = models.MailBetweenDomains.active.filter(is_expired=False)
if only_fails:
queryset = queryset.filter(status__iexact='FAILED')
if Email.send_email(data=queryset, subscription=subscription):
LOG.info('emailed report for mail between domains verification')
return
LOG.warning('could not email report for mail between domains verification')
def email_failed_logins_report(now=None, send_no_news=False, **dead_for):
"""
generate and email a report with all the failed `Citrix` logon events
See the :func:`email_dead_borgs_alert` for details about the arguments to
this task.
This report uses the subscription at `Citrix Failed Logins Report
<../../../admin/p_soc_auto_base/subscription/"""\
"""?q=Citrix+Failed+Logins+Report>`__.
"""
if not dead_for:
time_delta = get_preference('citrusborglogon__logon_report_period')
else:
time_delta = base_utils.MomentOfTime.time_delta(**dead_for)
now = base_utils.MomentOfTime.now(now)
data = get_failed_events(now=now, time_delta=time_delta)
if not data and send_no_news:
return (
'there were no failed logon events between'
' {:%a %b %d, %Y %H:%M %Z} and {:%a %b %d, %Y %H:%M %Z}'.
format(timezone.localtime(value=now),
timezone.localtime(now - time_delta))
)
return Email.send_email(
data=data,
subscription=Subscription.get_subscription(
'Citrix Failed Logins Report'),
time_delta=time_delta)
def email_login_ux_summary(now, time_delta, site_host_args):
"""
generate and send a report with login event state counts and user experience
stats for the site and host combination specified by `site_host_args` over
the interval defined by `now` and `time_delta`
:arg tuple site_host_args: the `site` and `host_name`
:arg datetime.datetime now: the initial moment for calculating the data in
the report
:arg datetime.timedelta time_delta: the reporting interval used for
calculating the data in the report
This report used the subscription at `Citrix logon event and ux summary
<../../../admin/p_soc_auto_base/subscription/"""\
"""?q=Citrix+logon+event+and+ux+summary>`__ to render the emails being sent.
"""
return Email.send_email(
data=login_states_by_site_host_hour(
now=now, time_delta=time_delta,
site=site_host_args[0], host_name=site_host_args[1]),
subscription=Subscription.get_subscription(
'Citrix logon event and ux summary'),
time_delta=time_delta,
site=site_host_args[0], host_name=site_host_args[1])
def email_ux_alarm(
now, time_delta, send_no_news, ux_alert_threshold, host):
"""
raise user experience alert and send by email for each `host` in
`host_names`
:arg str host: the name of the host to get alerts for.
See the :func:`email_us_alarms` task for details about the other arguments.
This alert used the subscription at `Citrix UX Alert
<../../../admin/p_soc_auto_base/subscription/?q=Citrix+UX+Alert>`__.
"""
now = base_utils.MomentOfTime.now(now)
site = WinlogbeatHost.objects.get(host_name__exact=host).site
data = raise_ux_alarm(now=now, time_delta=time_delta,
ux_alert_threshold=ux_alert_threshold,
host_name=host, site=site)
if not data and send_no_news:
LOG.info('Citrix response times on %s bot in %s were better than %s'
' between %s and %s', host, site, ux_alert_threshold,
timezone.localtime(value=now).isoformat(),
timezone.localtime(now - time_delta).isoformat())
return 0 # did not send email
return Email.send_email(
data=data, subscription=Subscription.get_subscription('Citrix UX Alert'),
time_delta=time_delta, ux_alert_threshold=ux_alert_threshold,
host_name=host, site=site)
def email_borg_login_summary_report(now=None, **dead_for):
"""
prepare and email a report with all the logon events generated by the
`Citrix` bots
The report includes events that occurred during the interval defined by the
arguments `now` and `dead_for`. The report includes counts for `failed` and
`successful` events.
The default value for `dead_for` is the dynamic preference
`Ignore events created older than
<../../../admin/dynamic_preferences/globalpreferencemodel/"""\
"""?q=ignore_events_older_than>`__.
This task used the subscription at `Citrix logon event summary
<../../../admin/p_soc_auto_base/subscription/"""\
"""?q=Citrix+logon+event+summary>`__ to render the emails being sent.
"""
if not dead_for:
time_delta = get_preference(
'citrusborgevents__ignore_events_older_than')
else:
time_delta = base_utils.MomentOfTime.time_delta(**dead_for)
return Email.send_email(
data=get_logins_by_event_state_borg_hour(
now=base_utils.MomentOfTime.now(now), time_delta=time_delta),
subscription=Subscription.get_subscription(
'Citrix logon event summary'),
time_delta=time_delta)
def email_dead_servers_report(now=None, send_no_news=False, **dead_for):
"""
generate and email reports about `Citrix` application servers that have not
service any requests during the time interval defined by arguments `now`
and `dead_for` via email.
This task is almost identical to :func:`email_dead_servers_alert`.
This task used the subscription at `Missing Citrix farm hosts
<../../../admin/p_soc_auto_base/subscription/"""\
"""?q=Missing+Citrix+farm+hosts>`__ to render the emails being sent.
"""
now = base_utils.MomentOfTime.now(now)
if not dead_for:
time_delta = get_preference('citrusborgnode__node_forgotten_after')
else:
time_delta = base_utils.MomentOfTime.time_delta(**dead_for)
data = get_dead_brokers(now=now, time_delta=time_delta)
if not data and send_no_news:
return (
'all known Cerner session servers were active between'
' {:%a %b %d, %Y %H:%M %Z} and {:%a %b %d, %Y %H:%M %Z}'.
format(timezone.localtime(value=now),
timezone.localtime(now - time_delta))
)
return Email.send_email(
data=data,
subscription=Subscription.get_subscription(
'Missing Citrix farm hosts'),
time_delta=time_delta)
def email_ssl_report():
"""
task to send `SSL` reports via email
In this task, the data contains all the valid `SSL` certificates ordered
by the number of days left until they expire.
We could make this a more abstract task but then:
* We would have to use :mod:`pickle` to serialize a `Django`
:class:`django.db.models.query.QuerySet` to a `Celery task
<https://docs.celeryproject.org/en/latest/userguide/tasks.html#tasks>`__.
This is not the default and it does have security implications
* We would also have to train the users on how to configure `Periodic tasks
<https://docs.celeryproject.org/en/latest/userguide/periodic-tasks.html#periodic-tasks>`__
using `django-celery-beat
<https://docs.celeryproject.org/en/latest/userguide/periodic-tasks.html#using-custom-scheduler-classes>`__
* See :func:`mail_collector.tasks.bring_out_your_dead` for an example about
a complex, abstract task. The user also has to understand how to write
`JSON <https://www.json.org/>`__ data by hand in order to configure
different `Periodic tasks` wrapped around this task by hand
This, and all the other tasks, dealing with `SSL` reports and alerts follow
the recommended patterns for basic `Celery` tasks.
"""
return Email.send_email(
data=expires_in(),
subscription=Subscription.get_subscription('SSl Report'))
def raise_failed_event_by_mail(event_pk):
"""
send an alert email for failed events
:arg int event_pk:
the primary key of the :class:`mail_collector.models.MailBotLogEvent`
instance with the failed event
:returns: the result of the email operation
:rtype: str
:raises: :exc:`Exception` if an error is thrown by the email send op
"""
data = models.MailBotLogEvent.objects.filter(pk=event_pk)
subscription = Subscription.get_subscription('Exchange Client Error')
# let's cache some data to avoid evaluating the queryset multiple times
data_extract = data.values('uuid', 'event_type', 'source_host__site__site',
'source_host__host_name')[0]
if Email.send_email(data=data,
subscription=subscription,
level=get_preference('commonalertargs__error_level'),
event_type=data_extract.get('event_type'),
site=data_extract.get('source_host__site__site'),
bot=data_extract.get('source_host__host_name')):
LOG.info('raised email alert for event %s', data_extract.get('uuid'))
return
LOG.warning('could not raise email alert for event %s',
data_extract.get('uuid'))
def email_expired_ssl_report():
"""
task to send reports about expired `SSL` by email
"""
return Email.send_email(
data=has_expired(), subscription=Subscription.get_subscription('Expired SSl Report'),
expired=True)
def email_invalid_ssl_report():
"""
task to send reports about `SSL` certificates that are not yet valid by
email
"""
return Email.send_email(
data=is_not_yet_valid(),
subscription=Subscription.get_subscription(subscription='Invalid SSl Report'),
invalid=True)
def dead_mail_sites(subscription, time_delta_pref=None, level=None):
"""
task for site related alerts via email
:returns:
the return depends on whether the systems has detected alerts and
on the value of the
:class:`citrus_borg.dynamic_preferences_registry.ExchangeEmptyAlerts`
dynamic setting
* if there are alerts, this task will return the result of the email
send op
* otherwise the task will check the value of the
:class:`citrus_borg.dynamic_preferences_registry.ExchangeEmptyAlerts`
dynamic setting.
* if the value is ``False``, the task will not send any emails
and it will return this information
* otherwise the task will send an email saying that there are
no alerts and return the result of the email send op
:rtype: str
:raises: :exc:`Exception` if an exception was thrown while sending the alert
"""
if level is None:
level = get_preference('exchange__default_level')
subscription = Subscription.get_subscription(subscription)
if time_delta_pref is None:
time_delta = get_preference('exchange__default_error')
else:
time_delta = get_preference(time_delta_pref)
not_seen_after = base_utils.MomentOfTime.past(time_delta=time_delta)
data = queries.dead_mail_sites(not_seen_after=not_seen_after)
if not data and not get_preference('exchange__empty_alerts'):
LOG.info('no %s data found for %s', level, subscription.subscription)
return
if Email.send_email(data=data,
subscription=subscription,
time_delta=time_delta,
level=level):
LOG.info('emailed data for %s', data.model._meta.verbose_name_plural)
return
LOG.warning('could not email data for %s',
data.model._meta.verbose_name_plural)
def raise_citrix_slow_alert(event_id, threshold_secs):
"""
Raises an alert for slow Citrix timings.
:param event_id: The id of the WinlogEvent with slow timings.
:param threshold_secs: The threshold used, in seconds.
:return: 1 if an email is sent, 0 otherwise.
"""
data = WinlogEvent.active.filter(pk=event_id)
return Email.send_email(
data=data,
subscription=Subscription.get_subscription('Citrix Slow Alert'),
ux_alert_threshold=timezone.timedelta(seconds=threshold_secs))
def dispatch_ldap_error_report(**time_delta_args):
"""
`Celery task` for generating `AD` services monitoring error reports
:arg time_delta_args: optional named arguments that are used to
initialize a :class:`datetime.duration` object
If not present, the method will use the period defined by the
:class:`citrus_borg.dynamic_preferences_registry.LdapReportPeriod`
user preference
:returns: information about the arguments used to call the task and the
result of :meth:`ssl_cert_tracker.lib.Email.send`
:rtype: str
:raises: :exc:`Exception` to allow `Celery` to deal with
errors
"""
LOG.debug(
'invoking ldap error report with time_delta_args: %s',
time_delta_args)
if time_delta_args:
time_delta = timezone.timedelta(**time_delta_args)
else:
time_delta = get_preference('ldapprobe__ldap_reports_period')
now = timezone.now()
try:
data = apps.get_model('ldap_probe.ldapprobelog').\
error_report(time_delta)
except Exception as error:
LOG.exception(
('invoking ldap error report with time_delta_args: %s'
' raises error %s'), time_delta_args, error)
raise error
subscription = Subscription.get_subscription(
get_preference('ldapprobe__ldap_error_report_subscription'))
error_level = get_preference('commonalertargs__error_level')
if Email.send_email(data=data, subscription=subscription, level=error_level,
now=now, time_delta=time_delta):
LOG.info('dispatched LDAP error report with time_delta_args: %s',
time_delta_args)
return
LOG.warning('could not dispatch LDAP error report with time_delta_args: %s',
time_delta_args)
def failure_cluster_check(sender, instance, *args, **kwargs):
"""
Send an alert if there has been a cluster of failed winlogevents, as defined
by the appropriate preferences: citrusborgux__cluster_event_ids.,
citrusborgux__cluster_length, citrusborgux__cluster_size. See preference
definitions for more details.
"""
failure_ids = get_int_list_preference('citrusborgux__cluster_event_ids')
if instance.event_id not in failure_ids:
return
recent_failures = WinlogEvent.active.filter(
timestamp__gte=instance.timestamp -
get_preference('citrusborgux__cluster_length'),
timestamp__lte=instance.timestamp,
event_id__in=failure_ids,
cluster__isnull=True)
recent_failures_count = recent_failures.count()
LOG.debug('there have been %d failures recently', recent_failures_count)
if recent_failures_count >= get_preference('citrusborgux__cluster_size'):
default_user = get_or_create_user()
new_cluster = EventCluster(created_by=default_user,
updated_by=default_user)
new_cluster.save()
new_cluster.winlogevent_set.add(*list(recent_failures))
# TODO could this be done on the server side?
# Note that this count includes the cluster we just created, hence <=
if (len([
cluster for cluster in EventCluster.active.all()
if cluster.end_time > timezone.now() -
get_preference('citrusborgux__backoff_time')
]) <= get_preference('citrusborgux__backoff_limit')):
Email.send_email(
None,
Subscription.get_subscription('Citrix Cluster Alert'),
False,
start_time=new_cluster.start_time,
end_time=new_cluster.end_time.astimezone(
timezone.get_current_timezone()).time(),
bots=new_cluster.winlogevent_set.all())
LOG.debug('sent cluster email')
else:
new_cluster.enabled = False
new_cluster.save()
LOG.info('Cluster created, but alert skipped due to frequency.')
def email_ssl_expires_in_days_report(lt_days): # pylint: disable=invalid-name
"""
task to send email alerts about `SSL` certificated that will expire soon
:arg int lt_days: the alarm trigger
Raise an alert for each `SSL` certificate that will expired in fewer
days than the number provided by this argument.
"""
return Email.send_email(
data=expires_in(lt_days=lt_days),
subscription=Subscription.get_subscription(subscription='SSl Report'),
expires_in_less_than=lt_days)
def email_dead_servers_alert(now=None, send_no_news=None, **dead_for):
"""
send out alerts about `Citrix` application servers that have not
service any requests during the time interval defined by arguments `now`
and `dead_for` via email.
This task is almost identical to :func:`email_dead_borgs_alert` with the
exception that we are considering application servers instead of bots and
the default value for `dead_for` is the dynamic preference
`Reporting period for dead nodes
<../../../admin/dynamic_preferences/globalpreferencemodel/"""\
"""?q=node_forgotten_after>`__.
This task used the subscription at `Missing Citrix farm hosts
<../../../admin/p_soc_auto_base/subscription/"""\
"""?q=Missing+Citrix+farm+hosts>`__ to render the emails being sent.
"""
now = base_utils.MomentOfTime.now(now)
if not dead_for:
time_delta = get_preference('citrusborgnode__node_forgotten_after')
else:
time_delta = base_utils.MomentOfTime.time_delta(**dead_for)
if send_no_news is None:
send_no_news = get_preference('citrusborgcommon__send_no_news')
if not isinstance(send_no_news, bool):
raise TypeError(
'object {} type {} is not valid. must be boolean'.
format(send_no_news, type(send_no_news))
)
data = get_dead_brokers(now=now, time_delta=time_delta)
if not data and send_no_news:
return (
'all known Cerner session servers were active between'
' {:%a %b %d, %Y %H:%M %Z} and {:%a %b %d, %Y %H:%M %Z}'.
format(timezone.localtime(value=now),
timezone.localtime(now - time_delta))
)
return Email.send_email(
data=data,
subscription=Subscription.get_subscription(
'Missing Citrix farm hosts'),
time_delta=time_delta)
def email_dead_sites_alert(now=None, send_no_news=None, **dead_for):
"""
send out alerts about remote sites where all the `Citrix` monitoring bots
that have not been seen within the time interval defined by arguments `now`
and `dead_for` via email.
This task is almost identical to :func:`email_dead_borgs_alert` with the
exception that we are looking at sites instead of bots, and that the default
value for `dead_for` is the dynamic preference
`Site not seen alert threshold
<../../../admin/dynamic_preferences/globalpreferencemodel/"""\
"""?q=dead_site_after>`__.
This task used the subscription at `Dead Citrix client sites
<../../../admin/p_soc_auto_base/subscription/"""\
"""?q=Dead+Citrix+client+sites>`__ to render the emails being sent.
"""
now = base_utils.MomentOfTime.now(now)
if not dead_for:
time_delta = get_preference('citrusborgnode__dead_site_after')
else:
time_delta = base_utils.MomentOfTime.time_delta(**dead_for)
if send_no_news is None:
send_no_news = get_preference('citrusborgcommon__send_no_news')
if not isinstance(send_no_news, bool):
raise TypeError(
'object {} type {} is not valid. must be boolean'.
format(send_no_news, type(send_no_news))
)
data = get_dead_sites(now=now, time_delta=time_delta)
if not data and send_no_news:
return (
'at least one monitoring bot on each site was active between'
' {:%a %b %d, %Y %H:%M %Z} and {:%a %b %d, %Y %H:%M %Z}'.
format(timezone.localtime(value=now),
timezone.localtime(now - time_delta))
)
return Email.send_email(
data=data,
subscription=Subscription.get_subscription(
'Dead Citrix client sites'),
time_delta=time_delta)
def report_failed_events_by_site(site, report_interval):
"""
send out report with failed events for a site via email
:arg str site: the host (short) name of the bot to report on
:arg `object` report_interval: the time interval for which the report is
calculated. it is either a :class:`datetime.timedelta` instance or a
:class:`dict` suitable for constructing a :class:`datetime.timedelta`
instance
:returns: the result of the email send operation
:rtype: str
:raises:
:exc:`Exception` if an exception was thrown while sending the alert
"""
subscription = Subscription.get_subscription(
'Exchange Failed Send Receive By Site')
data = queries.dead_bodies(
data_source='mail_collector.mailbotmessage',
filter_exp='event__event_registered_on__gte',
not_seen_after=base_utils.MomentOfTime.past(
time_delta=report_interval),
event__source_host__site__site=site,
event__event_status__iexact='fail').\
order_by('-mail_message_identifier', 'event__event_type_sort')
if Email.send_email(data=data,
subscription=subscription,
time_delta=report_interval,
level=get_preference('exchange__server_error'),
site=site):
LOG.info(
'emailed exchange failed send receive events report for site'
' %s', site)
return
LOG.warning(
'could not email exchange failed send receive events report for'
' site %s', site)
def email_failed_login_alarm(now=None, failed_threshold=None, **dead_for):
"""
raise alert about failed `Citrix` logon events and send it via email
:arg int failed_threshold: the number of failed logons that will trigger the
alert
By default, this will be retrieved from the dynamic preference
`Failed logon events count alert threshold
<../../../admin/dynamic_preferences/globalpreferencemodel/"""\
"""?q=logon_alert_threshold>`__
See the :func:`email_dead_borgs_alert` for details about other arguments.
This alert used the subscription at `Citrix logon alert
<../../../admin/p_soc_auto_base/subscription/?q=Citrix+logon+alert>`__.
"""
if failed_threshold is None:
failed_threshold = get_preference(
'citrusborglogon__logon_alert_threshold')
if not dead_for:
time_delta = get_preference('citrusborglogon__logon_alert_after')
else:
time_delta = base_utils.MomentOfTime.time_delta(**dead_for)
now = base_utils.MomentOfTime.now(now)
data = raise_failed_logins_alarm(
now=now, time_delta=time_delta,
failed_threshold=failed_threshold)
if not data:
return (
'there were less than {} failed logon events between'
' {:%a %b %d, %Y %H:%M %Z} and {:%a %b %d, %Y %H:%M %Z}'.
format(failed_threshold, timezone.localtime(value=now),
timezone.localtime(now - time_delta))
)
return Email.send_email(
data=data, subscription=Subscription.get_subscription(
'Citrix logon alert'),
time_delta=time_delta,
failed_threshold=failed_threshold)
def report_events_by_bot(bot, report_interval, report_level):
"""
send out report for events for a bot over a given duration measured
back from the moment of the call via email
:arg str bot: the host (short) name of the bot to report on
:arg `object` report_interval: the time interval for which the report is
calculated. it is either a :class:`datetime.timedelta` instance or a
:class:`dict` suitable for constructing a :class:`datetime.timedelta`
instance
:arg str report_level: INFO|WARN|ERROR pre-pended to the email subject line
:returns: the result of the email send operation
:rtype: str
:raises:
:exc:`Exception` if an exception was thrown while sending the alert
"""
subscription = Subscription.get_subscription(
'Exchange Send Receive By Bot')
data = queries.dead_bodies(
data_source='mail_collector.mailbotmessage',
filter_exp='event__event_registered_on__gte',
not_seen_after=base_utils.MomentOfTime.past(
time_delta=report_interval),
event__source_host__host_name=bot).\
order_by('-mail_message_identifier', 'event__event_type_sort')
if Email.send_email(data=data,
subscription=subscription,
time_delta=report_interval,
level=report_level,
bot=bot):
LOG.info('emailed exchange send receive events report for bot %s', bot)
return
LOG.warning(
'could not email exchange send receive events report for bot %s', bot)
def raise_ldap_probe_failed_alert(instance_pk=None, subscription=None):
"""
raise an email alert for a failed instance of the
:class:`ldap_probe.models.LdapProbeLog` model
:arg int instance_pk: the primary key of the instance
:arg str subscription: the value of the :attr:`subscription
<p_soc_auto_base.models.Subscription.subscription>` attribute
used to retrieve the
:class:`p_soc_auto_base.models.Subscription` instance required
for raising this alert via email
"""
if subscription is None:
subscription = get_preference('ldapprobe__ldap_error_subscription')
return _raise_ldap_alert(
instance_pk=instance_pk,
subscription=Subscription.get_subscription(subscription),
level=get_preference('commonalertargs__error_level'))
def raise_site_not_configured_for_bot():
"""
email alerts if there are exchange bots with mis-configured site info
"""
data = models.MailHost.objects.filter(
Q(site__isnull=True) | Q(site__site__iexact='site.not.exist')).\
exclude(host_name__iexact='host.not.exist')
if data.exists():
data = base_utils.url_annotate(data)
if not data and not get_preference('exchange__empty_alerts'):
LOG.info('all exchange bots are properly configured')
return
if Email.send_email(
data=data,
subscription=Subscription.get_subscription('Exchange bot no site'),
level=get_preference('exchange__server_error')):
LOG.info('emailed alert for mis-configured Exchange bots')
LOG.warning('cannot email alert for mis-configured Exchange bots')
def email_dead_borgs_report(now=None, send_no_news=False, **dead_for):
"""
generate and email reports about `Citrix` monitoring bots that have not
been seen within the time interval defined by arguments `now` and
`dead_for` via email
This task is almost identical to :func:`email_dead_borgs_alert` with the
exception of the default value for the :class:`datetime.timedelta` object
created when `dead_for` is not present. This value is picked from dynamic
preference `Reporting period for dead nodes
<../../../admin/dynamic_preferences/globalpreferencemodel/"""\
"""?q=node_forgotten_after>`__.
This task used the subscription at `Dead Citrix monitoring bots
<../../../admin/p_soc_auto_base/subscription/"""\
"""?q=Dead+Citrix+monitoring+bots>`__ to render the emails being sent.
"""
now = base_utils.MomentOfTime.now(now)
if not dead_for:
time_delta = get_preference('citrusborgnode__node_forgotten_after')
else:
time_delta = base_utils.MomentOfTime.time_delta(**dead_for)
data = get_dead_bots(now=now, time_delta=time_delta)
if not data and send_no_news:
return (
'all monitoring bots were active between'
' {:%a %b %d, %Y %H:%M %Z} and {:%a %b %d, %Y %H:%M %Z}'.
format(timezone.localtime(value=now),
timezone.localtime(now - time_delta))
)
return Email.send_email(
data=data,
subscription=Subscription.get_subscription(
'Dead Citrix monitoring bots'),
time_delta=time_delta)
def email_failed_ux_report(now=None, send_no_news=False,
ux_threshold_seconds=None, **dead_for):
"""
prepare and email a report with all the `Citrix` logon timings that do not
satisfy the user response time threshold
:arg now: see :func:`email_dead_borgs_alert`
:arg bool send_no_news: see :func:`email_dead_borgs_alert`
:arg datetime.timedelta ux_alert_threshold: the threshold for triggering
a user experience alert
By default the dynamic preference
`Maximum acceptable response time for citrix events
<../../../admin/dynamic_preferences/globalpreferencemodel/"""\
"""?q=ux_alert_threshold>`__
:arg dict dead_for: see the `dead_for` argument of the
:func:`email_dead_borgs_alert` task
The default is the dynamic preference `User experience reporting period
<../../../admin/dynamic_preferences/globalpreferencemodel/"""\
"""?q=ux_reporitng_period>`__
This report uses the subscription at
`Citrix Failed UX Event Components Report
<../../../admin/p_soc_auto_base/subscription/"""\
"""?q=Citrix+Failed+UX+Event+Components+Report>`__ to render the emails
being sent.
"""
if not dead_for:
time_delta = get_preference('citrusborgux__ux_reporting_period')
else:
time_delta = base_utils.MomentOfTime.time_delta(**dead_for)
if ux_threshold_seconds is None:
ux_alert_threshold = get_preference(
'citrusborgux__ux_alert_threshold')
else:
ux_alert_threshold = base_utils.MomentOfTime.time_delta(
time_delta=None, seconds=ux_threshold_seconds)
now = base_utils.MomentOfTime.now(now)
data = get_failed_ux_events(
now=now, time_delta=time_delta, ux_alert_threshold=ux_alert_threshold)
if not data and send_no_news:
return (
'there were no response time logon event components'
' longer than {:%S} seconds between'
' {:%a %b %d, %Y %H:%M %Z} and {:%a %b %d, %Y %H:%M %Z}'.
format(ux_alert_threshold,
timezone.localtime(value=now),
timezone.localtime(now - time_delta))
)
return Email.send_email(
data=data,
subscription=Subscription.get_subscription(
'Citrix Failed UX Event Components Report'),
time_delta=time_delta,
ux_alert_threshold=ux_alert_threshold)
def dispatch_ldap_report(data_source, anon, perf_filter, **time_delta_args):
"""
`Celery task` for generating `AD` services monitoring summary reports
:arg str data_source: the name of the `Django model` to be used as a
data source
We are passing the model using its name because the default
task serializer (`JSON`) is not capable of handling Python classes.
If we use the name of the model, we can resolve the class inside
the task.
:arg bool anon: flag used to decide the type of the `AD` probes;
probes that executed anonymous bind calls or probes that
executed non anonymous bind calls
:arg str perf_filter: apply filters for performance degradation if
this argument is provided
See :meth:`ldap_probe.models.BaseADNode.report_probe_aggregates`
:arg time_delta_args: optional named arguments that are used to
initialize a :class:`datetime.timedelta` object
If not present, the method will use the period defined by the
:class:`citrus_borg.dynamic_preferences_registry.LdapReportPeriod`
user preference
:returns: information about the arguments used to call the task and the
result of :meth:`ssl_cert_tracker.lib.Email.send`
:rtype: str
"""
LOG.debug(
('invoking ldap probes report with data_source: %s, anon: %s,'
' perf_filter: %s, time_delta_args: %s'),
data_source, anon, perf_filter, time_delta_args)
try:
# TODO why is report_probe_aggregates supplying now, time_delta, etc
now, time_delta, subscription, data, perf_filter = \
apps.get_model(data_source).\
report_probe_aggregates(
anon=anon, perf_filter=perf_filter, **time_delta_args)
except Exception as error:
LOG.exception(
('invoking ldap probes report with data_source: %s, anon: %s,'
' perf_filter: %s, time_delta_args: %s raises error %s'),
data_source, anon, perf_filter, time_delta_args, str(error))
raise error
subscription = Subscription.get_subscription(subscription)
full = 'full bind' in subscription.subscription.lower()
orion = 'non orion' not in subscription.subscription.lower()
if Email.send_email(
data=data, subscription=subscription,
level=get_preference('commonalertargs__info_level'), now=now,
time_delta=time_delta, full=full, orion=orion,
perf_filter=perf_filter):
LOG.info('dispatched LDAP probes report with data_source: %s, anon: %s,'
' perf_filter: %s, time_delta_args: %s',
anon, data_source, perf_filter, time_delta_args)
return
LOG.warning('could not dispatch LDAP probes report with data_source: %s,'
' anon: %s, perf_filter: %s, time_delta_args: %s',
data_source, anon, perf_filter, time_delta_args)
def dispatch_ldap_perf_report(
data_source, bucket, anon, level, **time_delta_args):
"""
task that generates a performance degradation report via email for the
arguments used to invoke it
Under normal circumstances, this task will always be invoked via
a `Celery group()
<https://docs.celeryproject.org/en/latest/userguide/canvas.html#groups>`__
call.
:arg str data_source: the named of the model containing information
about `AD` nodes using the 'app_lable.model_name' convention
:arg str bucket: the value of the
:attr:`ldap_probe.models.ADNodePerfBucket.performance_bucket` field;
this will be used to retrieve the applicable thresholds from the
:class:`ldap_probe.models.ADNodePerfBucket` instance
:arg bool anon: look for full bind or anonymous bind probe data
:arg str level: the performance degradation level
:arg time_delta_args: optional named arguments that are used to
initialize a :class:`datetime.timedelta` object
If not present, the method will use the period defined by the
:class:`citrus_borg.dynamic_preferences_registry.LdapReportPeriod`
user preference
:returns: information about the arguments used to call the task and the
result of :meth:`ssl_cert_tracker.lib.Email.send`
:rtype: str
:raises:
:exc:`Exception` if the data for the report cannot be generated of
the email cannot be sent.
If the send operation raises an :exc:`smtplib.SMTPConnectError`,
the task execution will be retried up to 3 times before the
exception is allowed to propagate.
"""
LOG.debug(
('invoking ldap probes report with data_source: %s, bucket: %s,'
' anon: %s, level: %s, time_delta_args: %s'),
data_source, bucket, anon, level, time_delta_args)
try:
(now, time_delta, subscription, data, threshold, no_nodes) = \
apps.get_model(data_source).report_perf_degradation(
bucket=bucket, anon=anon, level=level, **time_delta_args
)
except Exception as error:
LOG.exception(
('invoking ldap probes report with data_source: %s,'
' bucket: %s, anon: %s, level: %s, time_delta_args: %s'
' raises error %s'),
data_source, bucket, anon, level, time_delta_args, error)
raise error
if no_nodes:
LOG.warning('there are no AD network nodes for %s', bucket)
if not get_preference('ldapprobe__ldap_perf_send_good_news') and not data:
LOG.info('there is no performance degradation for %s', bucket)
return
subscription = Subscription.get_subscription(subscription)
full = 'full bind' in subscription.subscription.lower()
orion = 'non orion' not in subscription.subscription.lower()
threshold = utils.show_milliseconds(threshold)
if Email.send_email(
data=data, subscription=subscription, level=level, now=now,
time_delta=time_delta, full=full, orion=orion, bucket=bucket,
threshold=threshold):
LOG.info('dispatched LDAP probes performance degradation report with '
'data_source: %s, anon: %s, bucket: %s, level: %s, '
'time_delta_args: %s',
data_source, anon, bucket, level, time_delta_args)
return
LOG.warning('could not dispatch LDAP probes performance degradation report'
' with data_source: %s, anon: %s, bucket: %s, level: %s,'
' time_delta_args: %s',
data_source, anon, bucket, level, time_delta_args)
def bring_out_your_dead(data_source,
filter_exp,
subscription,
url_annotate=False,
level=None,
filter_pref=None,
**base_filters):
"""
generic task to raise email alerts about
:ref:`Mail Collector Application` entities that have been in an
abnormal state for a given duration measured going back from the
current moment
:arg str data_source: the reference to a
:class:`django.db.models.Model` in the form of 'app_label.model_name'
:arg str filter_exp: a django filter lhs expression (field_name__lookup);
it is actually geared to deal with datetime fields
:arg str subscription: the reference to the
:class:`p_soc_auto_base.models.Subscription` instance to be used
for rendering the email alert
:arg bool url_annotate:
extend the queryset with the entity URL; default ``False``
:arg str level: INFO|WARN|ERROR to add to the subject line of the email
alert; default ``None``
:arg `object` filter_pref: either a
:class:`django.utils.timezone.timedelta` instance or a :class:`dict`
suitable as argument for constructing a
:class:`django.utils.timezone.timedelta` like {'days': ``float``,
'hours': ``float``, 'seconds': ``float``}; default ``None``.
when ``None`` the value is picked up from the
:class:`citrus_borg.dynamic_preferences_registry.ExchangeDefaultError`
dynamic setting
:arg \*\*base_filters: additional django lookup style arguments to be
applied to the queryset
:returns: the result of the email send operation
:rtype: str
:raises:
:exc:`TypeError` if filter_pref cannot be cast to
``datetime.timedelta``
:exc:`Exception` if an exception was thrown while sending the alert
example::
qs=dead_bodies('mail_collector.mailhost','exchange_last_seen__lte',
not_seen_after={'minutes': 1}, enabled=True)
"""
if level is None:
level = get_preference('exchange__default_level')
subscription = Subscription.get_subscription(subscription)
if filter_pref is None:
filter_pref = get_preference('exchange__default_error')
if not isinstance(filter_pref, dict):
filter_pref = get_preference(filter_pref)
if not isinstance(filter_pref, timezone.timedelta):
raise TypeError(
'Invalid object type %s. Must be datetime.timedelta.' %
type(filter_pref))
not_seen_after = base_utils.MomentOfTime.past(time_delta=filter_pref)
data = queries.dead_bodies(data_source,
filter_exp,
not_seen_after=not_seen_after,
url_annotate=url_annotate,
**base_filters)
if not data and not get_preference('exchange__empty_alerts'):
LOG.info('no %s data found for %s', level, subscription.subscription)
return
if Email.send_email(data=data,
subscription=subscription,
time_delta=filter_pref,
level=level):
LOG.info('emailed data for %s', data.model._meta.verbose_name_plural)
return
LOG.warning('could not email data for %s',
data.model._meta.verbose_name_plural)
