def process_host_geoserver(self, service, data, valid_from, valid_to): """ Generates mertic values for system-level measurements """ desc_re = re.compile(r'\[(\w+)\]') def get_iface_name(row): desc = row['description'] m = desc_re.search(desc) if m is None: return return m.groups()[0] def get_network_rate(row, value, metric_defaults, metric_name, valid_to): iface_label = get_iface_name(row) if not iface_label: try: log.debug('no label', metric_name, row.get('description')) except BaseException: pass return rate = self._calculate_rate( metric_name, iface_label, value, valid_to) if rate is None: try: log.debug('no rate for', metric_name) except BaseException: pass return mdata = {'value': rate, 'value_raw': rate, 'value_num': rate, 'label': iface_label, 'metric': '{}.rate'.format(metric_name)} mdata.update(metric_defaults) log.debug(MetricValue.add(**mdata)) def get_mem_label(*args): return 'B' # gs metric -> monitoring metric name, label function, postproc # function GS_METRIC_MAP = dict((('SYSTEM_UPTIME', ('uptime', None, None,),), ('SYSTEM_AVERAGE_LOAD', ('load.1m', None, None,),), ('CPU_LOAD', ('cpu.usage.percent', None, None,),), ('MEMORY_USED', ('mem.usage.percent', get_mem_label, None,),), ('MEMORY_TOTAL', ('mem.all', get_mem_label, None,),), ('MEMORY_FREE', ('mem.free', get_mem_label, None,),), ('NETWORK_INTERFACE_SEND', ('network.out', get_iface_name, get_network_rate),), ('NETWORK_INTERFACE_RECEIVED', ('network.in', get_iface_name, get_network_rate),), ('NETWORK_INTERFACES_SEND', ('network.out', None, get_network_rate),), ('NETWORK_INTERFACES_RECEIVED', ('network.in', None, get_network_rate),), ) ) utc = pytz.utc collected_at = datetime.utcnow().replace(tzinfo=utc) valid_from = align_period_start(collected_at, service.check_interval) valid_to = align_period_end(collected_at, service.check_interval) mdefaults = {'valid_from': valid_from, 'valid_to': valid_to, 'resource': None, 'samples_count': 1, 'service': service} metrics = [m[0] for m in GS_METRIC_MAP.values()] MetricValue.objects.filter(service_metric__metric__name__in=metrics, valid_from=valid_from, valid_to=valid_to, service=service)\ .delete() for metric_data in data: map_data = GS_METRIC_MAP.get(metric_data['name']) if not map_data: continue metric_name, label_function, processing_function = map_data if metric_name is None: continue value = metric_data['value'] if isinstance(value, (str, unicode,)): value = value.replace(',', '.') mdata = {'value': value, 'value_raw': value, 'value_num': value, 'label': label_function(metric_data) if callable(label_function) else None, 'metric': metric_name} mdata.update(mdefaults) log.debug(MetricValue.add(**mdata)) if callable(processing_function): processing_function( metric_data, value, mdefaults, metric_name, valid_to)
def process_host_geonode(self, service, data, valid_from, valid_to): """ Generates mertic values for system-level measurements """ utc = pytz.utc import dateutil.parser collected_at = parse_datetime(dateutil.parser.parse(data['timestamp']) .strftime("%Y-%m-%d %H:%M:%S")).replace(tzinfo=utc) valid_from = align_period_start(collected_at, service.check_interval) valid_to = align_period_end(collected_at, service.check_interval) mdefaults = {'valid_from': valid_from, 'valid_to': valid_to, 'resource': None, 'samples_count': 1, 'service': service} MetricValue.objects.filter(service_metric__metric__name__in=('network.in', 'network.out'), valid_from=valid_from, valid_to=valid_to, service=service)\ .delete() for ifname, ifdata in data['data']['network'].iteritems(): for tx_label, tx_value in ifdata['traffic'].items(): mdata = {'value': tx_value, 'value_raw': tx_value, 'value_num': tx_value, 'label': ifname, 'metric': 'network.{}'.format(tx_label)} mdata.update(mdefaults) rate = self._calculate_rate( mdata['metric'], ifname, tx_value, valid_to) log.debug(MetricValue.add(**mdata)) if rate: mdata['metric'] = '{}.rate'.format(mdata['metric']) mdata['value'] = rate mdata['value_num'] = rate mdata['value_raw'] = rate log.debug(MetricValue.add(**mdata)) ldata = data['data']['load'] llabel = ['1', '5', '15'] memory_info = data['data']['memory'] mkeys = [m.name[len('mem.'):] for m in service.get_metrics() if m.name.startswith('mem.')] for mkey in mkeys: mdata = memory_info.get(mkey) if not mdata: continue mdata = {'value': mdata, 'value_raw': mdata, 'value_num': mdata, 'metric': 'mem.{}'.format(mkey), 'label': 'B', } mdata.update(mdefaults) MetricValue.objects.filter(service_metric__metric__name=mdata['metric'], valid_from=mdata['valid_from'], valid_to=mdata['valid_to'], label__name='MB', service=service)\ .delete() log.debug(MetricValue.add(**mdata)) MetricValue.objects.filter(service_metric__metric__name__in=('storage.total', 'storage.used', 'storage.free',), valid_from=valid_from, valid_to=valid_to, service=service)\ .delete() for df in data['data']['disks']: # dev = df['device'] total = df['total'] used = df['used'] free = df['free'] # free_pct = df['percent'] mount = df['mountpoint'] for metric, val in (('storage.total', total,), ('storage.used', used,), ('storage.free', free,),): mdata = {'value': val, 'value_raw': val, 'value_num': val, 'metric': metric, 'label': mount, } mdata.update(mdefaults) log.debug(MetricValue.add(**mdata)) if ldata: for lidx, l in enumerate(ldata): mdata = {'value': l, 'value_raw': l, 'value_num': l, 'metric': 'load.{}m'.format(llabel[lidx]), 'label': 'Value', } mdata.update(mdefaults) MetricValue.objects.filter(service_metric__metric__name=mdata['metric'], valid_from=mdata['valid_from'], valid_to=mdata['valid_to'], label__name='Value', service=service)\ .delete() log.debug(MetricValue.add(**mdata)) uptime = data['data'].get('uptime') if uptime is not None: mdata = {'value': uptime, 'value_raw': uptime, 'value_num': uptime, 'metric': 'uptime', 'label': 'Seconds'} mdata.update(mdefaults) MetricValue.objects.filter(service_metric__metric__name=mdata['metric'], valid_from=mdata['valid_from'], valid_to=mdata['valid_to'], label__name=mdata['label'], service=service)\ .delete() log.debug(MetricValue.add(**mdata)) if data['data'].get('cpu'): _l = data['data']['cpu']['usage'] mdata = {'value': _l, 'value_raw': _l, 'value_num': _l, 'metric': 'cpu.usage', 'label': 'Seconds', } mdata.update(mdefaults) MetricValue.objects.filter(service_metric__metric__name=mdata['metric'], valid_from=mdata['valid_from'], valid_to=mdata['valid_to'], label__name=mdata['label'], service=service)\ .delete() log.debug(MetricValue.add(**mdata)) rate = self._calculate_rate( mdata['metric'], mdata['label'], mdata['value'], mdata['valid_to']) if rate: rate_data = mdata.copy() rate_data['metric'] = '{}.rate'.format(mdata['metric']) rate_data['value'] = rate rate_data['value_num'] = rate rate_data['value_raw'] = rate log.debug(MetricValue.add(**rate_data)) percent = self._calculate_percent( mdata['metric'], mdata['label'], mdata['value'], mdata['valid_to']) if percent: percent_data = mdata.copy() percent_data['metric'] = '{}.percent'.format(mdata['metric']) percent_data['value'] = percent percent_data['value_num'] = percent percent_data['value_raw'] = percent percent_data['label'] = 'Value' log.debug(MetricValue.add(**percent_data)) mdata.update(mdefaults) log.debug(MetricValue.add(**mdata))