class MemoryDataSource(DataSource): ''' Monitor memory usage via psutil. ''' def __init__(self): DataSource.__init__(self) x = virtual_memory( ) # as per psutil docs: first call will give rubbish self._keys = [ a for a in dir(x) if not a.startswith('_') and \ not callable(getattr(x,a)) ] self._results = ResultsContainer() self._log = logging.getLogger('mm') def __call__(self): sample = virtual_memory() self._results.add_result( dict([(k, getattr(sample, k)) for k in self._keys])) @property def name(self): return 'mem' def metadata(self): self._results.drop_first() self._log.info("Mem summary: {}".format(self._results.summary(mean))) return self._results.all() def show_status(self): pass
class CPUDataSource(DataSource): ''' Monitor CPU usage (via psutil module) ''' def __init__(self): DataSource.__init__(self) self._results = ResultsContainer() cpulist = cpu_times_percent(percpu=True) # as per psutil docs: first call gives rubbish self._log = logging.getLogger('mm') x = cpulist.pop(0) self._keys = [ a for a in dir(x) if not a.startswith('_') and \ not callable(getattr(x,a)) ] def __call__(self): sample = cpu_times_percent(percpu=True) result = { "cpu{}_{}".format(i,k):getattr(sample[i],k) for i in range(len(sample)) \ for k in self._keys } result['idle'] = mean([ getattr(x, 'idle') for x in sample ]) self._results.add_result(result) @property def name(self): return 'cpu' def metadata(self): self._results.drop_first() self._log.info("CPU summary: {}".format(self._results.summary(mean))) return self._results.all() def show_status(self): self._log.info("CPU idle: {}".format(self._results.last_result('idle')))
class NetIfDataSource(DataSource): ''' Monitor network interface counters. Can be constructed with one or more names (strings) of network interfaces, or nothing to monitor all interfaces. The psutil call just yields current counter values; internally we keep last sample and only store differences. ''' def __init__(self, *nics_of_interest): DataSource.__init__(self) x = self._lastsample = net_io_counters(pernic=True) # as per psutil docs: first call will give rubbish if not nics_of_interest: self._nics = list(x.keys()) else: self._nics = [ n for n in x.keys() if n in nics_of_interest ] if not self._nics: raise ConfigurationError("Bad interface names specified for netstat monitor: {}".format(', '.join(nics_of_interest))) d1 = list(self._nics)[0] self._keys = [ a for a in dir(x[d1]) if not a.startswith('_') and \ not callable(getattr(x[d1],a)) ] self._results = ResultsContainer() self._log = logging.getLogger('mm') def __call__(self): sample = net_io_counters(pernic=True) rd = { '_'.join((n,k)):_compute_diff_with_wrap(getattr(sample[n], k), \ getattr(self._lastsample[n], k)) \ for k in self._keys for n in self._nics } self._results.add_result(rd) @property def name(self): return 'netstat' def metadata(self): self._results.drop_first() self._log.info("Netstat summary: {}".format(self._results.summary(mean))) return self._results.all() def show_status(self): pass
class IODataSource(DataSource): ''' Monitor disk IO counters via psutil. The psutil call just yields the current counter values; internally we keep last sample and only store differences. ''' def __init__(self): DataSource.__init__(self) x = self._lastsample = disk_io_counters( perdisk=True) # as per psutil docs: first call will give rubbish self._disks = x.keys() self._results = ResultsContainer() self._log = logging.getLogger('mm') d1 = list(self._disks)[0] self._keys = [ a for a in dir(x[d1]) if not a.startswith('_') and \ not callable(getattr(x[d1],a)) ] def __call__(self): sample = disk_io_counters(perdisk=True) rd = { '_'.join((d,k)):_compute_diff_with_wrap(getattr(sample[d], k), \ getattr(self._lastsample[d], k)) \ for k in self._keys for d in self._disks } self._lastsample = sample self._results.add_result(rd) @property def name(self): return 'io' def metadata(self): self._results.drop_first() self._log.info("IO summary: {}".format(self._results.summary(mean))) return self._results.all() def show_status(self): pass
class PcapReceiverSource(DataSource): ''' Just receive data on a pcap device and collect timestamps. ''' def __init__(self, interface, decode): DataSource.__init__(self) self._interface = interface self._results = ResultsContainer() self._log = logging.getLogger('mm') self._name = "pcaprecv_{}".format(self._interface) self._num_recv = 0 self._do_decode = decode xfilter = 'udp or icmp' self._setup_port(interface, xfilter) self._monfut = asyncio.Future() @property def name(self): return self._name def _setup_port(self, ifname, filterstr): p = pcapffi.PcapLiveDevice.create(ifname) p.snaplen = 128 p.set_promiscuous(True) p.set_timeout(10) # choose the "best" timestamp available: # highest number up to 3 (don't use unsynced adapter stamps) stamptypes = [ t for t in p.list_tstamp_types() if t <= pcapffi.PcapTstampType.Adapter ] if len(stamptypes): beststamp = max(stamptypes) try: p.set_tstamp_type(beststamp) stval = pcapffi.PcapTstampType(beststamp) self._log.info("Set timestamp type to {}".format(stval.name)) except: self._log.warn( "Couldn't set timestamp type to the advertised value {}". format(stval.name)) try: p.tstamp_precision = pcapffi.PcapTstampPrecision.Nano self._log.info("Using nanosecond timestamp precision.") except: self._log.info("Using microsecond timestamp precision.") if sys.platform == 'linux': try: p.set_immediate_mode(True) except: pass w = p.activate() if w != 0: wval = pcapffi.PcapWarning(w) self._log.warn("Warning on activation: {}".format(wval.name)) p.blocking = False p.set_direction(pcapffi.PcapDirection.InOut) p.set_filter(filterstr) self._pcap = p asyncio.get_event_loop().add_reader(p.fd, self._packet_arrival_callback) def __call__(self): '''Don't do anything when we get called!''' pass def _packet_arrival_callback(self): p = self._pcap.recv_packet_or_none() if p is None: return ts = p.timestamp if self._do_decode: pkt = decode_packet(self._pcap.dlt, p.raw) self._results.add_result(ts) return if pkt.has_header(Arp) and pkt[Arp].operation == ArpOperation.Reply: a = pkt[Arp] self._arp_queue.put_nowait((a.senderhwaddr, a.senderprotoaddr)) return elif pkt.has_header(ICMP): if (pkt[ICMP].icmptype in \ (ICMPType.EchoReply,ICMPType.EchoRequest) and \ pkt[ICMP].icmpdata.identifier == self._pktident): seq = pkt[ICMP].icmpdata.sequence direction = ProbeDirection.Outgoing if pkt[ICMP].icmptype == ICMPType.EchoReply: direction = ProbeDirection.Incoming # ignore Echo Reply if src addr doesn't match # our intended dest if pkt[IPv4].src != self._dest: return self._probe_queue.put_nowait( (ts, seq, pkt[IPv4].src, pkt[IPv4].ttl, direction)) return elif pkt[ICMP].icmptype == ICMPType.TimeExceeded: p = self._probehelper.reconstruct_carcass( pkt[ICMP].icmpdata.data) origttl = self._infer_orig_ttl(pkt[IPv4].ttl) if p.has_header(self._probehelper.klass): # ttl stuffed into ipid of previously sent # pkt is unreliable seq, ident = self._probehelper.decode_carcass(p) if ident == self._pktident and p[IPv4].dst == self._dest: self._probe_queue.put_nowait( (ts, seq, pkt[IPv4].src, origttl, ProbeDirection.Incoming)) return # identify our outgoing TCP or UDP probe packet. ICMP is caught # in prevous elif elif pkt.has_header(self._probehelper.klass): seq, ident = self._probehelper.decode_carcass(pkt) origttl = pkt[IPv4].ttl if ident == self._pktident: self._probe_queue.put_nowait( (ts, seq, pkt[IPv4].src, origttl, ProbeDirection.Outgoing)) return def _get_pcap_stats(self): if self._pcap is None: return self._pcapstats s = self._pcap.stats() self._pcapstats = { 'recv': s.ps_recv, 'pcapdrop': s.ps_drop, 'ifdrop': s.ps_ifdrop } return self._pcapstats def cleanup(self): s = self._get_pcap_stats() self._log.info("Closing {}: {}".format(self._interface, s)) self._pcap.close() self._pcap = None def metadata(self): xmeta = {} xmeta['libpcap_stats'] = self._get_pcap_stats() xmeta['pkts_received'] = self._num_recv xmeta['tstamps'] = self._results.all() return xmeta def show_status(self): pass