def handle(self, *args, **options):
oservice = options['service']
if not oservice:
services = Service.objects.all()
else:
services = [oservice]
if options['list_services']:
print('available services')
for s in services:
print(' ', s.name, '(', s.url, ')')
print(' type', s.service_type.name)
print(' running on', s.host.name, s.host.ip)
print(' active:', s.active)
if s.last_check:
print(' last check:', s.last_check)
else:
print(' not checked yet')
print(' ')
return
c = CollectorAPI()
for s in services:
try:
self.run_check(s,
collector=c,
since=options['since'],
until=options['until'],
force_check=options['force_check'],
format=options['format'])
except Exception as err:
log.error("Cannot collect from %s: %s", s, err, exc_info=err)
if options['halt_on_errors']:
raise
if not options['do_not_clear']:
log.info("Clearing old data")
c.clear_old_data()
if options['emit_notifications']:
log.info("Processing notifications for %s", options['until'])
s = Service.objects.first()
interval = s.check_interval
now = datetime.utcnow().replace(tzinfo=pytz.utc)
notifications_check = now - interval
c.emit_notifications() #notifications_check)
def test_notifications_api(self):
capi = CollectorAPI()
start = datetime.utcnow().replace(tzinfo=pytz.utc)
start_aligned = align_period_start(start, self.service.check_interval)
end_aligned = start_aligned + self.service.check_interval
# for (metric_name, field_opt, use_service,
# use_resource, use_label, use_ows_service,
# minimum, maximum, thresholds,) in thresholds:
notifications_config = (
'geonode is not working', 'detects when requests are not handled',
(
(
'request.count',
'min_value',
False,
False,
False,
False,
0,
10,
None,
'Number of handled requests is lower than',
),
(
'response.time',
'max_value',
False,
False,
False,
False,
500,
None,
None,
'Response time is higher than',
),
))
nc = NotificationCheck.create(*notifications_config)
self.assertTrue(nc.definitions.all().count() == 2)
user = self.u2
pwd = self.passwd2
self.client.login(username=user.username, password=pwd)
for nc in NotificationCheck.objects.all():
notifications_config_url = reverse(
'monitoring:api_user_notification_config', args=(nc.id, ))
nc_form = nc.get_user_form()
self.assertTrue(nc_form)
self.assertTrue(nc_form.fields.keys())
vals = [1000000, 100000]
data = {'emails': []}
data['emails'] = '\n'.join(data['emails'])
idx = 0
for fname, field in nc_form.fields.items():
if fname in self.reserved_fields:
continue
data[fname] = vals[idx]
idx += 1
resp = self.client.post(notifications_config_url, data)
self.assertEqual(resp.status_code, 400)
vals = [7, 600]
data = {
'emails':
'\n'.join([self.u.email, self.u2.email, '*****@*****.**'])
}
idx = 0
for fname, field in nc_form.fields.items():
if fname in self.reserved_fields:
continue
data[fname] = vals[idx]
idx += 1
# data['emails'] = '\n'.join(data['emails'])
resp = self.client.post(notifications_config_url, data)
nc.refresh_from_db()
self.assertEqual(resp.status_code, 200, resp)
_emails = data['emails'].split('\n')[-1:]
_users = data['emails'].split('\n')[:-1]
self.assertEqual(set([u.email for u in nc.get_users()]),
set(_users))
self.assertEqual(set([email for email in nc.get_emails()]),
set(_emails))
metric_rq_count = Metric.objects.get(name='request.count')
metric_rq_time = Metric.objects.get(name='response.time')
MetricValue.add(metric_rq_count,
start_aligned,
end_aligned,
self.service,
label="Count",
value_raw=0,
value_num=0,
value=0)
MetricValue.add(metric_rq_time,
start_aligned,
end_aligned,
self.service,
label="Count",
value_raw=700,
value_num=700,
value=700)
nc = NotificationCheck.objects.get()
self.assertTrue(len(nc.get_emails()) > 0)
self.assertTrue(len(nc.get_users()) > 0)
self.assertEqual(nc.last_send, None)
self.assertTrue(nc.can_send)
self.assertEqual(len(mail.outbox), 0)
# make sure inactive will not trigger anything
nc.active = False
nc.save()
capi.emit_notifications(start)
self.assertEqual(len(mail.outbox), 0)
nc.active = True
nc.save()
capi.emit_notifications(start)
self.assertTrue(nc.receivers.all().count() > 0)
self.assertEqual(len(mail.outbox), nc.receivers.all().count())
nc.refresh_from_db()
notifications_url = reverse('monitoring:api_user_notifications')
nresp = self.client.get(notifications_url)
self.assertEqual(nresp.status_code, 200)
ndata = json.loads(nresp.content)
self.assertEqual(
set([n['id'] for n in ndata['data']]),
set(NotificationCheck.objects.all().values_list('id', flat=True)))
self.assertTrue(isinstance(nc.last_send, datetime))
self.assertFalse(nc.can_send)
mail.outbox = []
self.assertEqual(len(mail.outbox), 0)
capi.emit_notifications(start)
self.assertEqual(len(mail.outbox), 0)
nc.last_send = start - nc.grace_period
nc.save()
self.assertTrue(nc.can_send)
mail.outbox = []
self.assertEqual(len(mail.outbox), 0)
capi.emit_notifications(start)
self.assertEqual(len(mail.outbox), nc.receivers.all().count())
def collect_metric(**options):
from geonode.celery_app import app
from geonode.tasks.tasks import memcache_lock
from geonode.monitoring.models import Service
from geonode.monitoring.collector import CollectorAPI
_start_time = None
_end_time = None
# The cache key consists of the task name and the MD5 digest
# of the name.
name = b'collect_metric'
hexdigest = md5(name).hexdigest()
lock_id = '{0}-lock-{1}'.format(name, hexdigest)
_start_time = datetime.utcnow().isoformat()
log.info('[{}] Collecting Metrics - started @ {}'.format(
lock_id, _start_time))
with memcache_lock(lock_id, app.oid) as acquired:
if acquired:
oservice = options['service']
if not oservice:
services = Service.objects.all()
else:
services = [oservice]
if options['list_services']:
print('available services')
for s in services:
print(' ', s.name, '(', s.url, ')')
print(' type', s.service_type.name)
print(' running on', s.host.name, s.host.ip)
print(' active:', s.active)
if s.last_check:
print(' last check:', s.last_check)
else:
print(' not checked yet')
print(' ')
return
c = CollectorAPI()
for s in services:
try:
run_check(s,
collector=c,
since=options['since'],
until=options['until'],
force_check=options['force_check'],
format=options['format'])
except Exception as err:
log.error("Cannot collect from %s: %s",
s,
err,
exc_info=err)
if options['halt_on_errors']:
raise
if not options['do_not_clear']:
log.debug("Clearing old data")
c.clear_old_data()
if options['emit_notifications']:
log.debug("Processing notifications for %s", options['until'])
# s = Service.objects.first()
# interval = s.check_interval
# now = datetime.utcnow().replace(tzinfo=pytz.utc)
# notifications_check = now - interval
c.emit_notifications() # notifications_check))
_end_time = datetime.utcnow().isoformat()
log.info('[{}] Collecting Metrics - finished @ {}'.format(
lock_id, _end_time))
return (_start_time, _end_time)
def collect_metric(**options):
from geonode.tasks.tasks import memcache_lock
from geonode.monitoring.models import Service
from geonode.monitoring.collector import CollectorAPI
_start_time = None
_end_time = None
# The cache key consists of the task name and the MD5 digest
# of the name.
name = b'collect_metric'
hexdigest = md5(name).hexdigest()
lock_id = f'{name.decode()}-lock-{hexdigest}'
_start_time = _end_time = datetime.utcnow().isoformat()
log.info('[{}] Collecting Metrics - started @ {}'.format(
lock_id, _start_time))
lock = memcache_lock(lock_id)
if lock.acquire(blocking=False) is True:
log.info('[{}] Collecting Metrics - [...acquired lock] @ {}'.format(
lock_id, _start_time))
try:
oservice = options['service']
if not oservice:
services = Service.objects.all()
else:
services = [oservice]
if options['list_services']:
print('available services')
for s in services:
print(' ', s.name, '(', s.url, ')')
print(' type', s.service_type.name)
print(' running on', s.host.name, s.host.ip)
print(' active:', s.active)
if s.last_check:
print(' last check:', s.last_check)
else:
print(' not checked yet')
print(' ')
return
c = CollectorAPI()
for s in services:
try:
run_check(s,
collector=c,
since=options['since'],
until=options['until'],
force_check=options['force_check'],
format=options['format'])
except Exception as e:
log.warning(e)
if not options['do_not_clear']:
log.info("Clearing old data")
c.clear_old_data()
if options['emit_notifications']:
log.info("Processing notifications for %s", options['until'])
# s = Service.objects.first()
# interval = s.check_interval
# now = datetime.utcnow().replace(tzinfo=pytz.utc)
# notifications_check = now - interval
c.emit_notifications() # notifications_check))
_end_time = datetime.utcnow().isoformat()
log.info('[{}] Collecting Metrics - finished @ {}'.format(
lock_id, _end_time))
except Exception as e:
log.info('[{}] Collecting Metrics - errored @ {}'.format(
lock_id, _end_time))
log.exception(e)
finally:
lock.release()
log.info('[{}] Collecting Metrics - exit @ {}'.format(lock_id, _end_time))
return (_start_time, _end_time)
def test_notifications_api(self):
capi = CollectorAPI()
start = datetime.utcnow().replace(tzinfo=pytz.utc)
start_aligned = align_period_start(start, self.service.check_interval)
end_aligned = start_aligned + self.service.check_interval
# for (metric_name, field_opt, use_service,
# use_resource, use_label, use_ows_service,
# minimum, maximum, thresholds,) in thresholds:
notifications_config = ('geonode is not working',
'detects when requests are not handled',
(('request.count', 'min_value', False, False,
False, False, 0, 10, None, 'Number of handled requests is lower than',),
('response.time', 'max_value', False, False,
False, False, 500, None, None, 'Response time is higher than',),))
nc = NotificationCheck.create(*notifications_config)
self.assertTrue(nc.definitions.all().count() == 2)
user = self.u2
pwd = self.passwd2
self.client.login(username=user.username, password=pwd)
for nc in NotificationCheck.objects.all():
notifications_config_url = reverse(
'monitoring:api_user_notification_config', args=(nc.id,))
nc_form = nc.get_user_form()
self.assertTrue(nc_form)
self.assertTrue(nc_form.fields.keys())
vals = [1000000, 100000]
data = {'emails': []}
data['emails'] = '\n'.join(data['emails'])
idx = 0
for fname, field in nc_form.fields.items():
if fname in self.reserved_fields:
continue
data[fname] = vals[idx]
idx += 1
resp = self.client.post(notifications_config_url, data)
self.assertEqual(resp.status_code, 400)
vals = [7, 600]
data = {'emails': '\n'.join(
[self.u.email,
self.u2.email,
'*****@*****.**'])}
idx = 0
for fname, field in nc_form.fields.items():
if fname in self.reserved_fields:
continue
data[fname] = vals[idx]
idx += 1
# data['emails'] = '\n'.join(data['emails'])
resp = self.client.post(notifications_config_url, data)
nc.refresh_from_db()
self.assertEqual(resp.status_code, 200, resp)
_emails = data['emails'].split('\n')[-1:]
_users = data['emails'].split('\n')[:-1]
self.assertEqual(
set([u.email for u in nc.get_users()]),
set(_users))
self.assertEqual(
set([email for email in nc.get_emails()]),
set(_emails))
metric_rq_count = Metric.objects.get(name='request.count')
metric_rq_time = Metric.objects.get(name='response.time')
MetricValue.add(metric_rq_count,
start_aligned,
end_aligned,
self.service,
label="Count",
value_raw=0,
value_num=0,
value=0)
MetricValue.add(metric_rq_time,
start_aligned,
end_aligned,
self.service,
label="Count",
value_raw=700,
value_num=700,
value=700)
nc = NotificationCheck.objects.get()
self.assertTrue(len(nc.get_emails()) > 0)
self.assertTrue(len(nc.get_users()) > 0)
self.assertEqual(nc.last_send, None)
self.assertTrue(nc.can_send)
self.assertEqual(len(mail.outbox), 0)
# make sure inactive will not trigger anything
nc.active = False
nc.save()
capi.emit_notifications(start)
self.assertEqual(len(mail.outbox), 0)
nc.active = True
nc.save()
capi.emit_notifications(start)
self.assertTrue(nc.receivers.all().count() > 0)
self.assertEqual(len(mail.outbox), nc.receivers.all().count())
nc.refresh_from_db()
notifications_url = reverse('monitoring:api_user_notifications')
nresp = self.client.get(notifications_url)
self.assertEqual(nresp.status_code, 200)
ndata = json.loads(nresp.content)
self.assertEqual(set([n['id'] for n in ndata['data']]),
set(NotificationCheck.objects.all().values_list('id', flat=True)))
self.assertTrue(isinstance(nc.last_send, datetime))
self.assertFalse(nc.can_send)
mail.outbox = []
self.assertEqual(len(mail.outbox), 0)
capi.emit_notifications(start)
self.assertEqual(len(mail.outbox), 0)
nc.last_send = start - nc.grace_period
nc.save()
self.assertTrue(nc.can_send)
mail.outbox = []
self.assertEqual(len(mail.outbox), 0)
capi.emit_notifications(start)
self.assertEqual(len(mail.outbox), nc.receivers.all().count())