def process_host_geoserver(self, service, data, valid_from, valid_to):
"""
Generates mertic values for system-level measurements
"""
desc_re = re.compile(r'\[(\w+)\]')
def get_iface_name(row):
desc = row['description']
m = desc_re.search(desc)
if m is None:
return
return m.groups()[0]
def get_network_rate(row, value, metric_defaults,
metric_name, valid_to):
iface_label = get_iface_name(row)
if not iface_label:
try:
log.debug('no label', metric_name, row.get('description'))
except BaseException:
pass
return
rate = self._calculate_rate(
metric_name, iface_label, value, valid_to)
if rate is None:
try:
log.debug('no rate for', metric_name)
except BaseException:
pass
return
mdata = {'value': rate,
'value_raw': rate,
'value_num': rate,
'label': iface_label,
'metric': '{}.rate'.format(metric_name)}
mdata.update(metric_defaults)
log.debug(MetricValue.add(**mdata))
def get_mem_label(*args):
return 'B'
# gs metric -> monitoring metric name, label function, postproc
# function
GS_METRIC_MAP = dict((('SYSTEM_UPTIME', ('uptime', None, None,),),
('SYSTEM_AVERAGE_LOAD', ('load.1m', None, None,),),
('CPU_LOAD', ('cpu.usage.percent', None, None,),),
('MEMORY_USED',
('mem.usage.percent', get_mem_label, None,),),
('MEMORY_TOTAL', ('mem.all', get_mem_label, None,),),
('MEMORY_FREE', ('mem.free', get_mem_label, None,),),
('NETWORK_INTERFACE_SEND', ('network.out',
get_iface_name, get_network_rate),),
('NETWORK_INTERFACE_RECEIVED', ('network.in',
get_iface_name, get_network_rate),),
('NETWORK_INTERFACES_SEND',
('network.out', None, get_network_rate),),
('NETWORK_INTERFACES_RECEIVED',
('network.in', None, get_network_rate),),
)
)
utc = pytz.utc
collected_at = datetime.utcnow().replace(tzinfo=utc)
valid_from = align_period_start(collected_at, service.check_interval)
valid_to = align_period_end(collected_at, service.check_interval)
mdefaults = {'valid_from': valid_from,
'valid_to': valid_to,
'resource': None,
'samples_count': 1,
'service': service}
metrics = [m[0] for m in GS_METRIC_MAP.values()]
MetricValue.objects.filter(service_metric__metric__name__in=metrics,
valid_from=valid_from,
valid_to=valid_to,
service=service)\
.delete()
for metric_data in data:
map_data = GS_METRIC_MAP.get(metric_data['name'])
if not map_data:
continue
metric_name, label_function, processing_function = map_data
if metric_name is None:
continue
value = metric_data['value']
if isinstance(value, (str, unicode,)):
value = value.replace(',', '.')
mdata = {'value': value,
'value_raw': value,
'value_num': value,
'label': label_function(metric_data) if callable(label_function) else None,
'metric': metric_name}
mdata.update(mdefaults)
log.debug(MetricValue.add(**mdata))
if callable(processing_function):
processing_function(
metric_data,
value,
mdefaults,
metric_name,
valid_to)
def process_host_geonode(self, service, data, valid_from, valid_to):
"""
Generates mertic values for system-level measurements
"""
utc = pytz.utc
import dateutil.parser
collected_at = parse_datetime(dateutil.parser.parse(data['timestamp'])
.strftime("%Y-%m-%d %H:%M:%S")).replace(tzinfo=utc)
valid_from = align_period_start(collected_at, service.check_interval)
valid_to = align_period_end(collected_at, service.check_interval)
mdefaults = {'valid_from': valid_from,
'valid_to': valid_to,
'resource': None,
'samples_count': 1,
'service': service}
MetricValue.objects.filter(service_metric__metric__name__in=('network.in', 'network.out'),
valid_from=valid_from,
valid_to=valid_to,
service=service)\
.delete()
for ifname, ifdata in data['data']['network'].iteritems():
for tx_label, tx_value in ifdata['traffic'].items():
mdata = {'value': tx_value,
'value_raw': tx_value,
'value_num': tx_value,
'label': ifname,
'metric': 'network.{}'.format(tx_label)}
mdata.update(mdefaults)
rate = self._calculate_rate(
mdata['metric'], ifname, tx_value, valid_to)
log.debug(MetricValue.add(**mdata))
if rate:
mdata['metric'] = '{}.rate'.format(mdata['metric'])
mdata['value'] = rate
mdata['value_num'] = rate
mdata['value_raw'] = rate
log.debug(MetricValue.add(**mdata))
ldata = data['data']['load']
llabel = ['1', '5', '15']
memory_info = data['data']['memory']
mkeys = [m.name[len('mem.'):]
for m in service.get_metrics() if m.name.startswith('mem.')]
for mkey in mkeys:
mdata = memory_info.get(mkey)
if not mdata:
continue
mdata = {'value': mdata,
'value_raw': mdata,
'value_num': mdata,
'metric': 'mem.{}'.format(mkey),
'label': 'B',
}
mdata.update(mdefaults)
MetricValue.objects.filter(service_metric__metric__name=mdata['metric'],
valid_from=mdata['valid_from'],
valid_to=mdata['valid_to'],
label__name='MB',
service=service)\
.delete()
log.debug(MetricValue.add(**mdata))
MetricValue.objects.filter(service_metric__metric__name__in=('storage.total', 'storage.used', 'storage.free',),
valid_from=valid_from,
valid_to=valid_to,
service=service)\
.delete()
for df in data['data']['disks']:
# dev = df['device']
total = df['total']
used = df['used']
free = df['free']
# free_pct = df['percent']
mount = df['mountpoint']
for metric, val in (('storage.total', total,),
('storage.used', used,),
('storage.free', free,),):
mdata = {'value': val,
'value_raw': val,
'value_num': val,
'metric': metric,
'label': mount,
}
mdata.update(mdefaults)
log.debug(MetricValue.add(**mdata))
if ldata:
for lidx, l in enumerate(ldata):
mdata = {'value': l,
'value_raw': l,
'value_num': l,
'metric': 'load.{}m'.format(llabel[lidx]),
'label': 'Value',
}
mdata.update(mdefaults)
MetricValue.objects.filter(service_metric__metric__name=mdata['metric'],
valid_from=mdata['valid_from'],
valid_to=mdata['valid_to'],
label__name='Value',
service=service)\
.delete()
log.debug(MetricValue.add(**mdata))
uptime = data['data'].get('uptime')
if uptime is not None:
mdata = {'value': uptime,
'value_raw': uptime,
'value_num': uptime,
'metric': 'uptime',
'label': 'Seconds'}
mdata.update(mdefaults)
MetricValue.objects.filter(service_metric__metric__name=mdata['metric'],
valid_from=mdata['valid_from'],
valid_to=mdata['valid_to'],
label__name=mdata['label'],
service=service)\
.delete()
log.debug(MetricValue.add(**mdata))
if data['data'].get('cpu'):
_l = data['data']['cpu']['usage']
mdata = {'value': _l,
'value_raw': _l,
'value_num': _l,
'metric': 'cpu.usage',
'label': 'Seconds',
}
mdata.update(mdefaults)
MetricValue.objects.filter(service_metric__metric__name=mdata['metric'],
valid_from=mdata['valid_from'],
valid_to=mdata['valid_to'],
label__name=mdata['label'],
service=service)\
.delete()
log.debug(MetricValue.add(**mdata))
rate = self._calculate_rate(
mdata['metric'],
mdata['label'],
mdata['value'],
mdata['valid_to'])
if rate:
rate_data = mdata.copy()
rate_data['metric'] = '{}.rate'.format(mdata['metric'])
rate_data['value'] = rate
rate_data['value_num'] = rate
rate_data['value_raw'] = rate
log.debug(MetricValue.add(**rate_data))
percent = self._calculate_percent(
mdata['metric'],
mdata['label'],
mdata['value'],
mdata['valid_to'])
if percent:
percent_data = mdata.copy()
percent_data['metric'] = '{}.percent'.format(mdata['metric'])
percent_data['value'] = percent
percent_data['value_num'] = percent
percent_data['value_raw'] = percent
percent_data['label'] = 'Value'
log.debug(MetricValue.add(**percent_data))
mdata.update(mdefaults)
log.debug(MetricValue.add(**mdata))
