def dead_bodies(data_source,
filter_exp,
not_seen_after=None,
url_annotate=False,
**base_filters):
"""
:returns: :class:`django.db.models.Queryset` objects that are filtered on
:class:`django.db.models.DateTimeField` fields
:arg data_source: the data source used to populate the queryset
Either a :class:`django.db.models.Queryset` object or a :class:`str`
in 'app_label.modelname' format
:arg str filter_exp: the field and lookup to use for filtering
For example, 'last_updated__lte' will filter on a field named
last_updated using a less than or equal lookup
:arg not_seen_after:
the time interval required to buid the filter parameter value
It can be a :class:`datetime.timedelta` object or a :class:`dict`
that can be used to construct a :class:`datetime.timedelta` object.
Default: ``None``. When ``None``, the value is picked up from
:class:`citrus_borg.dynamic_properties_registry.ExchangeServerError`.
:arg bool url_annotate: do we also add the absolute url for each object?
:arg \*\*base_filters: optional django lookups for the queryset
:raises:
:exc:`TypeError` if the not_seen_after argument is of types other
than :class:`datetime.timedelta` object, :class:`dict`
that can be used to construct a :class:`datetime.timedelta` object,
or ``None``
"""
if not_seen_after is None:
not_seen_after = MomentOfTime.past(
time_delta=get_preference('exchange__server_error'))
if isinstance(not_seen_after, dict):
not_seen_after = MomentOfTime.past(**not_seen_after)
if not isinstance(not_seen_after, datetime.datetime):
raise TypeError('Invalid object type %s, was expecting datetime' %
type(not_seen_after))
if not isinstance(data_source, QuerySet):
queryset = get_base_queryset(data_source, **base_filters)
queryset = queryset.filter(**{filter_exp: not_seen_after})
if url_annotate:
queryset = _url_annotate(queryset)
return queryset
def dead_mail_sites(not_seen_after=None):
"""
:returns:
:class:`django.db.models.Queryset` objects based on
:class:`mail_collector.models.MailHost` that
represent sites from where exchange requests have not
been observed for the duration specified in the argument
This query function must filter on an aggregated annotation.
There may be multiple bots on any site and the site can only be
considered down if all its bots are down. We must filter against the
bot most recently seen for each site.
:param not_seen_after: the time interval required to buid the filter
parameter value
It can be a :class:`datetime.timedelta` object or a :class:`dict`
that can be used to construct a :class:`datetime.timedelta` object.
Default: ``None``. When ``None``, the value is picked up from
:class:`citrus_borg.dynamic_properties_registry.ExchangeServerError`
:raises:
:exc:`TypeError` if the not_seen_after argument is of types other
than :class:`datetime.timedelta` object, :class:`dict`
that can be used to construct a :class:`datetime.timedelta` object,
or ``None``
"""
if not_seen_after is None:
not_seen_after = MomentOfTime.past(time_delta=not_seen_after)
if isinstance(not_seen_after, dict):
not_seen_after = MomentOfTime.past(**not_seen_after)
if not isinstance(not_seen_after, datetime.datetime):
raise TypeError('Invalid object type %s, was expecting datetime' %
type(not_seen_after))
queryset = get_base_queryset('mail_collector.mailhost', enabled=True)
queryset = queryset.values('site__site').\
annotate(most_recent=Max('excgh_last_seen')).\
filter(most_recent__lte=not_seen_after).\
order_by('site__site', '-most_recent')
return queryset
def invoke_report_events_by_site(report_interval=None, report_level=None):
"""
invoke tasks for mailing the events by site reports
:arg `object` report_interval:
the time interval for which the report is calculated
it is either a :class:`datetime.timedelta` instance or a
:class:`dict` suitable for constructing a :class:`datetime.timedelta`
instance. it defaults to ``None`` and when it is ``None``,
the value is picked from
:class:`citrus_borg.dynamic_preferences_registry.ExchangeReportInterval`
:arg str report_level: similar to a log level, defaults to ``None``
when ``None``, the value is picked from
:class:`citrus_borg.dynamic_preferences_registry."""\
"""ExchangeDefaultErrorLevel`
:returns: the sites for which the report tasks have been invoked
:rtype: str
"""
if report_interval is None:
report_interval = get_preference('exchange__report_interval')
if report_level is None:
report_level = get_preference('exchange__report_level')
sites = list(
base_utils.get_base_queryset('mail_collector.mailsite',
enabled=True).values_list('site',
flat=True))
group(
report_events_by_site.s(site, report_interval, report_level)
for site in sites)()
group(
report_failed_events_by_site.s(site, report_interval)
for site in sites)()
LOG.info('launched report tasks for exchange events by site for sites: %s',
', '.join(sites))
def get_ssl_base_queryset(app_label,
model_name,
url_annotate=True,
issuer_url_annotate=True):
"""
:returns:
a :class:`django.db.models.query.QuerySet` based on the
:class:`ssl_cert_tracker.models.SslCertificate` and `annotated
<https://docs.djangoproject.com/en/2.2/ref/models/querysets/"""\
"""#annotate>`__
with the absolute path of the `URL` for the
:class:`Django admin <django.contrib.admin.ModelAdmin>` instance based
on the related entry in
:class:`ssl_cert_tracker.models.SslCertificateIssuer`
The annotation is present in a field named `url_issuer`.
This function cannot be abstracted to generate annotations for one or more
foreign key fields because the `annotation` names cannot be passed as
variables
"""
queryset = utils.get_base_queryset(f'{app_label}.{model_name}',
enabled=True)
if url_annotate:
queryset = utils.url_annotate(queryset)
if issuer_url_annotate \
and app_label == 'ssl_cert_tracker' \
and model_name == 'sslcertificate':
queryset = queryset.\
annotate(url_issuer_id=Cast('issuer__id', TextField())).\
annotate(url_issuer=Concat(
Value(settings.SERVER_PROTO), Value('://'),
Value(socket.getfqdn()),
Value(':'), Value(settings.SERVER_PORT),
Value('/admin/'),
Value(app_label), Value('/sslcertificateissuer/'),
F('url_issuer_id'), Value('/change/'),
output_field=TextField()))
return queryset
def bootstrap_ad_probes(data_sources=None):
"""
launch a separate probing task for each `AD` controller extracted
from the arguments
This task uses the `Celery Canvas Group primitive
<https://docs.celeryproject.org/en/latest/userguide/canvas.html#groups>`__
to launch multiple parallel instances of the :meth:`probe_ad_controller`
task.
:arg data_sources: the name(s) of the :class:`Django model(s)
<django.db.models.Model>` that store `AD` controller information
This item in the argument must be represented using the
`app_label.model_name` convention. The argument can be a
:class:`list` or a :class:`tuple`, or a :class:`str` that separates
each entry using a comma (,) character.
By default this argument points to the models defined under
:class:`ldap_probe.models.OrionADNode` and
:class:`ldap_probe.models.NonOrionADNode`
:returns: a `CR` separated string containing the name of each data
source and the number of `AD` controllers defined under said source
:rtype: str
"""
if data_sources is None:
data_sources = 'ldap_probe.OrionADNode,ldap_probe.NonOrionADNode'
if not isinstance(data_sources, (list, tuple)):
data_sources = data_sources.split(',')
for data_source in data_sources:
pk_list = utils.get_pk_list(
utils.get_base_queryset(data_source, enabled=True))
group(probe_ad_controller.s(data_source, ad_pk) for ad_pk in pk_list)()
LOG.info('Will probe %s AD controllers from %s.', len(pk_list),
data_source)