def test_service_handlers(self):
"""
Test if we can calculate metrics
"""
self.client.force_login(self.u)
self.assertTrue(get_user(self.client).is_authenticated())
_l = file_upload(
os.path.join(gisdata.VECTOR_DATA,
"san_andres_y_providencia_poi.shp"),
name="san_andres_y_providencia_poi",
user=self.u,
overwrite=True,
)
for idx, _l in enumerate(Layer.objects.all()):
for inum in range(0, idx + 1):
self.client.get(reverse('layer_detail', args=(_l.alternate, )),
**{"HTTP_USER_AGENT": self.ua})
# Works only with Postgres
requests = RequestEvent.objects.all()
c = CollectorAPI()
q = requests.order_by('created')
c.process_requests(self.service, requests,
q.first().created,
q.last().created)
interval = self.service.check_interval
now = datetime.utcnow().replace(tzinfo=pytz.utc)
valid_from = now - (2 * interval)
valid_to = now
self.assertTrue(isinstance(valid_from, datetime))
self.assertTrue(isinstance(valid_to, datetime))
self.assertTrue(isinstance(interval, timedelta))
# Works only with Postgres
metrics = c.get_metrics_for(metric_name='request.ip',
valid_from=valid_from,
valid_to=valid_to,
interval=interval)
self.assertIsNotNone(metrics)
class Command(BaseCommand):
"""
Run collecting for monitoring
"""
def add_arguments(self, parser):
parser.add_argument('-m',
'--list-metrics',
dest='list_metrics',
action='store_true',
default=False,
help=_("Show list of metrics"))
parser.add_argument('-l',
'--list-labels',
dest='list_labels',
action='store_true',
default=False,
help=_("Show list of labels for metric"))
parser.add_argument('-r',
'--list-resources',
dest='list_resources',
action='store_true',
default=False,
help=_("Show list of resources for metric"))
parser.add_argument(
'-s',
'--since',
dest='since',
default=None,
type=parse_datetime,
help=_(
"Process data since specific timestamp (YYYY-MM-DD HH:MM:SS format). "
"If not provided, last sync will be used."))
parser.add_argument(
'-u',
'--until',
dest='until',
default=None,
type=parse_datetime,
help=_(
"Process data until specific timestamp (YYYY-MM-DD HH:MM:SS format). "
"If not provided, now will be used."))
parser.add_argument(
'-i',
'--interval',
dest='interval',
default=60,
type=int,
help=_("Data aggregation interval in seconds (default: 60)"))
parser.add_argument('metric_name',
default=None,
nargs="?",
help=_("Metric name"))
parser.add_argument(
'-rr',
'--for-resource',
dest='resource',
type=TypeChecks.resource_type,
help=
_("Show data for specific resource in resource_type=resource_name format"
))
parser.add_argument('-ss',
'--service',
dest='service',
type=TypeChecks.service_type,
help=_("Show data for specific resource"))
parser.add_argument('-ll',
'--label',
dest='label',
type=TypeChecks.label_type,
help=_("Show data for specific label"))
@timeout_decorator.timeout(LOCAL_TIMEOUT)
def handle(self, *args, **options):
# Exit early if MONITORING_ENABLED=False
if not settings.MONITORING_ENABLED:
return
self.collector = CollectorAPI()
if options['list_metrics']:
self.list_metrics()
return
interval = timedelta(seconds=options['interval'])
metric_names = options['metric_name']
resource = options['resource']
# service = options['service']
label = options['label']
if not metric_names:
raise CommandError("No metric name")
if isinstance(metric_names, str):
metric_names = [metric_names]
for m in metric_names:
if options['list_labels']:
self.list_labels(m)
elif options['list_resources']:
self.list_resources(m)
else:
self.show_metrics(m,
options['since'],
options['until'],
interval,
resource=resource,
label=label)
@timeout_decorator.timeout(LOCAL_TIMEOUT)
def list_labels(self, metric, resource=None):
labels = self.collector.get_labels_for_metric(metric,
resource=resource)
print(f'Labels for metric {metric}')
for label in labels:
print(' ', *label)
@timeout_decorator.timeout(LOCAL_TIMEOUT)
def list_resources(self, metric):
resources = self.collector.get_resources_for_metric(metric)
print(f'Resources for metric {metric}')
for res in resources:
print(' ', '='.join(res))
@timeout_decorator.timeout(LOCAL_TIMEOUT)
def show_metrics(self,
metric,
since,
until,
interval,
resource=None,
label=None,
service=None):
print(f'Monitoring Metric values for {metric}')
if service:
print(f' for service: {service} ')
if resource:
print(f' for resource: {resource.type}={resource.name} ')
if label:
print(f' for label: {label.name} label')
utc = pytz.utc
since = since.replace(tzinfo=utc) if since else None
until = until.replace(tzinfo=utc) if until else None
data = self.collector.get_metrics_for(metric,
valid_from=since,
valid_to=until,
interval=interval,
resource=resource,
label=label,
service=service)
print(f" since {data['input_valid_from'].strftime(TIMESTAMP_OUTPUT)} "
f"until {data['input_valid_to'].strftime(TIMESTAMP_OUTPUT)}\n")
for row in data['data']:
val = None
if row['data']:
val = row['data'][0]['val']
print(' ', row['valid_to'].strftime(TIMESTAMP_OUTPUT), '->',
'' if not val else val)
@timeout_decorator.timeout(LOCAL_TIMEOUT)
def list_metrics(self):
_metrics = self.collector.get_metric_names()
for stype, metrics in _metrics:
print('service type', stype.name)
for m in metrics:
print(f' {m.name}[{m.type}]')
class Command(BaseCommand):
"""
Run collecting for monitoring
"""
def add_arguments(self, parser):
parser.add_argument('-m', '--list-metrics', dest='list_metrics', action='store_true', default=False,
help=_("Show list of metrics"))
parser.add_argument('-l', '--list-labels', dest='list_labels', action='store_true', default=False,
help=_("Show list of labels for metric"))
parser.add_argument('-r', '--list-resources', dest='list_resources', action='store_true', default=False,
help=_("Show list of resources for metric"))
parser.add_argument('-s', '--since', dest='since', default=None, type=parse_datetime,
help=_("Process data since specific timestamp (YYYY-MM-DD HH:MM:SS format). If not provided, last sync will be used."))
parser.add_argument('-u', '--until', dest='until', default=None, type=parse_datetime,
help=_("Process data until specific timestamp (YYYY-MM-DD HH:MM:SS format). If not provided, now will be used."))
parser.add_argument('-i', '--interval', dest='interval', default=60, type=int,
help=_("Data aggregation interval in seconds (default: 60)"))
parser.add_argument('metric_name', default=None, nargs="?", help=_("Metric name"))
parser.add_argument('-rr', '--for-resource', dest='resource', type=TypeChecks.resource_type,
help=_("Show data for specific resource in resource_type=resource_name format"))
parser.add_argument('-ss', '--service', dest='service', type=TypeChecks.service_type,
help=_("Show data for specific resource"))
parser.add_argument('-ll', '--label', dest='label', type=TypeChecks.label_type,
help=_("Show data for specific label"))
@timeout_decorator.timeout(LOCAL_TIMEOUT)
def handle(self, *args, **options):
self.collector = CollectorAPI()
if options['list_metrics']:
self.list_metrics()
return
interval = timedelta(seconds=options['interval'])
metric_names = options['metric_name']
resource = options['resource']
service = options['service']
label = options['label']
if not metric_names:
raise CommandError("No metric name")
if isinstance(metric_names, types.StringTypes):
metric_names = [metric_names]
for m in metric_names:
#def get_metrics_for(self, metric_name, valid_from=None, valid_to=None, interval=None, service=None, label=None, resource=None):
if options['list_labels']:
self.list_labels(m)
elif options['list_resources']:
self.list_resources(m)
else:
self.show_metrics(m, options['since'], options['until'], interval, resource=resource, label=label)
@timeout_decorator.timeout(LOCAL_TIMEOUT)
def list_labels(self, metric, resource=None):
labels = self.collector.get_labels_for_metric(metric, resource=resource)
print('Labels for metric {}'.format(metric))
for label in labels:
print(' ', *label)
@timeout_decorator.timeout(LOCAL_TIMEOUT)
def list_resources(self, metric):
resources = self.collector.get_resources_for_metric(metric)
print('Resources for metric {}'.format(metric))
for res in resources:
print(' ', '='.join(res))
@timeout_decorator.timeout(LOCAL_TIMEOUT)
def show_metrics(self, metric, since, until, interval, resource=None, label=None, service=None):
print('Monitoring Metric values for {}'.format(metric))
if service:
print(' for service: {} '.format(service))
if resource:
print(' for resource: {}={} '.format(resource.type, resource.name))
if label:
print(' for label: {} label'.format(label.name))
utc = pytz.utc
now = datetime.utcnow().replace(tzinfo=utc)
since = since.replace(tzinfo=utc) if since else None
until = until.replace(tzinfo=utc) if until else None
data = self.collector.get_metrics_for(metric,
valid_from=since,
valid_to=until,
interval=interval,
resource=resource,
label=label,
service=service)
print(' since {} until {}\n'.format(data['input_valid_from'].strftime(TIMESTAMP_OUTPUT),
data['input_valid_to'].strftime(TIMESTAMP_OUTPUT)))
for row in data['data']:
val = None
if row['data']:
val = row['data'][0]['val']
print(' ', row['valid_to'].strftime(TIMESTAMP_OUTPUT), '->', '' if not val else val)
@timeout_decorator.timeout(LOCAL_TIMEOUT)
def list_metrics(self):
_metrics = self.collector.get_metric_names()
for stype, metrics in _metrics:
print('service type', stype.name)
for m in metrics:
print(' {}[{}]'.format(m.name, m.type))