def set_up_mocks(self, su=None):
self.mox.StubOutWithMock(dirutil, 'safe_mkdtemp')
dirutil.safe_mkdtemp().AndReturn('/tmp/test')
self.mox.StubOutWithMock(log, 'init')
log.init('/tmp/test/current_run').AndReturn(0)
self.mox.StubOutWithMock(CommandUtil, 'execute_and_get_output')
stub = CommandUtil.execute_and_get_output(['git','remote', '-v'])
stub.AndReturn((0, dedent("""origin https://git.twitter.biz/science (fetch)
origin https://git.twitter.biz/science (push)""")))
stub2 = CommandUtil.execute_and_get_output(['git','rev-parse', '--abbrev-ref', 'HEAD'])
stub2.AndReturn((0,"test_br"))
self.mox.StubOutWithMock(psutil, 'cpu_percent')
psutil.cpu_percent(interval=1).AndReturn(1.0)
self.mox.StubOutWithMock(psutil, 'network_io_counters')
psutil.network_io_counters().AndReturn("1000,10000,1000")
self.mox.StubOutWithMock(psutil, 'NUM_CPUS')
psutil.NUM_CPUS = 5
self.mox.StubOutWithMock(socket, 'gethostname')
socket.gethostname().AndReturn("localhost")
self.mox.StubOutWithMock(socket, 'gethostbyname')
socket.gethostbyname("localhost").AndReturn("localhost")
self.mox.StubOutWithMock(sys, 'exit')
sys.exit(0).AndReturn(0)
self.mox.ReplayAll()
def threadLoop(self):
pnic_before = psutil.network_io_counters(pernic=True)
if not pnic_before.has_key(self.config.network_interface):
# print "Couldn't find the network interface %s" % (self.config.network_interface)
self.config.network_interface = None
for i in pnic_before.keys():
if i != "lo":
self.config.network_interface = i
break
if self.config.network_interface == None:
return
# print "Using %s instead" % (self.config.network_interface)
stats_before = pnic_before[self.config.network_interface]
while not self.terminate:
time.sleep(1)
pnic_after = psutil.network_io_counters(pernic=True)
stats_after = pnic_after[self.config.network_interface]
# format bytes to string
bytesSent = byte_to_bit(stats_after.bytes_sent - stats_before.bytes_sent) #toKbps(stats_after.bytes_sent - stats_before.bytes_sent) / 1
bytesReceived = byte_to_bit(stats_after.bytes_recv - stats_before.bytes_recv) #toKbps(stats_after.bytes_recv - stats_before.bytes_recv) / 1
# store on a circular list
self.sent.append(bytesSent)
self.recv.append(bytesReceived)
stats_before = stats_after
def network_all():
tot_before = psutil.network_io_counters()
pnic_before = psutil.network_io_counters(pernic=True)
# sleep some time
time.sleep(1)
tot_after = psutil.network_io_counters()
pnic_after = psutil.network_io_counters(pernic=True)
nic_names = pnic_after.keys()
network = {}
for name in nic_names:
stats_before = pnic_before[name]
stats_after = pnic_after[name]
network[name] = {
"bytes_sent": stats_after.bytes_sent,
"bytes_recv": stats_after.bytes_recv,
"bytes_sent_last_sec": stats_after.bytes_sent - stats_before.bytes_sent,
"bytes_recv_last_sec": stats_after.bytes_recv - stats_before.bytes_recv,
}
network["total"] = {
"bytes_sent": tot_after.bytes_sent,
"bytes_recv": tot_after.bytes_recv,
"bytes_sent_last_sec": tot_after.bytes_sent - tot_before.bytes_sent,
"bytes_recv_last_sec": tot_after.bytes_recv - tot_before.bytes_recv,
}
return network
def set_up_mocks(self, su=None):
self.mox.StubOutWithMock(dirutil, 'safe_mkdtemp')
dirutil.safe_mkdtemp().AndReturn('/tmp/test')
self.mox.StubOutWithMock(log, 'init')
log.init('/tmp/test/current_run').AndReturn(0)
self.mox.StubOutWithMock(CommandUtil, 'execute_and_get_output')
stub = CommandUtil.execute_and_get_output(['git', 'remote', '-v'])
stub.AndReturn(
(0,
dedent("""origin https://git.twitter.biz/science (fetch)
origin https://git.twitter.biz/science (push)""")))
stub2 = CommandUtil.execute_and_get_output(
['git', 'rev-parse', '--abbrev-ref', 'HEAD'])
stub2.AndReturn((0, "test_br"))
self.mox.StubOutWithMock(psutil, 'cpu_percent')
psutil.cpu_percent(interval=1).AndReturn(1.0)
self.mox.StubOutWithMock(psutil, 'network_io_counters')
psutil.network_io_counters().AndReturn("1000,10000,1000")
self.mox.StubOutWithMock(psutil, 'NUM_CPUS')
psutil.NUM_CPUS = 5
self.mox.StubOutWithMock(socket, 'gethostname')
socket.gethostname().AndReturn("localhost")
self.mox.StubOutWithMock(socket, 'gethostbyname')
socket.gethostbyname("localhost").AndReturn("localhost")
self.mox.StubOutWithMock(sys, 'exit')
sys.exit(0).AndReturn(0)
self.mox.ReplayAll()
def get_status():
net_before = psutil.network_io_counters(pernic=True)[NETWORK]
time.sleep(.5)
net_after = psutil.network_io_counters(pernic=True)[NETWORK]
STATUS = json.dumps(
{
"hostname":platform.node(),
"dist":platform.dist(),
"results":
{
"cpu":
{
"load":os.getloadavg(),
"percent": psutil.cpu_percent(interval=.5, percpu=True)
},
"memory":psutil.virtual_memory(),
"swap":psutil.swap_memory(),
"disk":psutil.disk_usage(DISK),
"network":
{
"before":net_before,
"after":net_after
},
}
}
)
return STATUS
def getSI(val=None):
"""
Récupération de données système ajoutées dans uns liste
Arguments:
val -- métriques à acquérir
"""
if val == None:
val = dict()
val["time"] = []
val["mem"] = []
val["swap"] = []
val["io_read"] = []
val["io_write"] = []
val["net_sent"] = []
val["net_recv"] = []
val["cpu"] = []
val["time"] += [time.time()]
val["mem"] += [psutil.virtual_memory()[2]]
val["swap"] += [psutil.swap_memory()[3]]
val["io_read"] += [psutil.disk_io_counters(perdisk=False)[2]]
val["io_write"] += [psutil.disk_io_counters(perdisk=False)[3]]
val["net_sent"] += [psutil.network_io_counters(pernic=False)[0]]
val["net_recv"] += [psutil.network_io_counters(pernic=False)[1]]
val["cpu"] += [psutil.cpu_percent(interval=0.8)]
return val
def threadLoop(self):
pnic_before = psutil.network_io_counters(pernic=True)
if not pnic_before.has_key(self.config.network_interface):
# print "Couldn't find the network interface %s" % (self.config.network_interface)
self.config.network_interface = None
for i in pnic_before.keys():
if i != "lo":
self.config.network_interface = i
break
if self.config.network_interface == None:
return
# print "Using %s instead" % (self.config.network_interface)
stats_before = pnic_before[self.config.network_interface]
while not self.terminate:
time.sleep(1)
pnic_after = psutil.network_io_counters(pernic=True)
stats_after = pnic_after[self.config.network_interface]
# format bytes to string
bytesSent = byte_to_bit(stats_after.bytes_sent - stats_before.bytes_sent) #toKbps(stats_after.bytes_sent - stats_before.bytes_sent) / 1
bytesReceived = byte_to_bit(stats_after.bytes_recv - stats_before.bytes_recv) #toKbps(stats_after.bytes_recv - stats_before.bytes_recv) / 1
# store on a circular list
self.sent.append(bytesSent)
self.recv.append(bytesReceived)
stats_before = stats_after
def sysStat(sleep=5):
fname=''
for x in list(time.localtime(time.time())[:]):
fname=fname+str(x)
fname=fname+'.xls'
filelog = open(fname,'w')
count = 1
print 'Cou \t CPU \t Mem \t Sent \t Recv'
filelog.write('Count \t CPU \t Mem \t Sent \t Recv\n')
while True:
netSentStart = psutil.network_io_counters()[0]
netRecvStart = psutil.network_io_counters()[1]
time.sleep(1)
netSentEnd = psutil.network_io_counters()[0]
netRecvEnd = psutil.network_io_counters()[1]
netSent = (netSentEnd - netSentStart)/1024
netRecv = (netRecvEnd - netRecvStart)/1024
time.sleep(sleep-1)
cpu = psutil.cpu_percent()
memory = psutil.virtual_memory()[2]
message = str(count) + '\t' + '% 4.1f'%cpu +'\t'+'% 4.1f'%memory +'\t' + '%.2f'%netSent + '\t' + '%.2f'%netRecv
print message
filelog.write(message+'\n')
count+=1
def timer_thread():
while True:
netcount_before = psutil.network_io_counters()
time.sleep(1)
netcount_after = psutil.network_io_counters()
glb.network_send_speed = netcount_after.bytes_sent - netcount_before.bytes_sent
glb.network_recv_speed = netcount_after.bytes_recv - netcount_before.bytes_recv
def get_network_ststus(self):
network_io_ststus = []
network_stats_list = psutil.network_io_counters()
network_in_bytes = int(network_stats_list.bytes_recv)
network_out_bytes = int(network_stats_list.bytes_sent)
network_in_packets = int(network_stats_list.packets_recv)
network_out_packets = int(network_stats_list.packets_sent)
# print network_out_bytes
time.sleep(2)
network_stats_list = psutil.network_io_counters()
network_in_bytes_new = int(network_stats_list.bytes_recv)
network_out_bytes_new = int(network_stats_list.bytes_sent)
network_in_packets_new = int(network_stats_list.packets_recv)
network_out_packets_new = int(network_stats_list.packets_sent)
# print network_out_bytes_new
network_in_bytes_ststus = int(network_in_bytes_new) - int(network_in_bytes)
network_out_bytes_ststus = int(network_out_bytes_new) - int(network_out_bytes)
# network_in_packets_ststus = int(network_in_packets_new) - int(network_in_packets)
# network_out_packets_ststus = int(network_out_packets_new) - int(network_out_packets)
network_io_ststus.append([network_in_bytes_ststus,network_out_bytes_ststus])
return network_io_ststus
def network_all():
tot_before = psutil.network_io_counters()
pnic_before = psutil.network_io_counters(pernic=True)
# sleep some time
time.sleep(1)
tot_after = psutil.network_io_counters()
pnic_after = psutil.network_io_counters(pernic=True)
nic_names = pnic_after.keys()
network= {}
for name in nic_names:
stats_before = pnic_before[name]
stats_after = pnic_after[name]
network[name] = {'bytes_sent': stats_after.bytes_sent,
'bytes_recv': stats_after.bytes_recv,
'bytes_sent_last_sec':stats_after.bytes_sent - stats_before.bytes_sent ,
'bytes_recv_last_sec': stats_after.bytes_recv - stats_before.bytes_recv }
network['total'] = {'bytes_sent':tot_after.bytes_sent,
'bytes_recv':tot_after.bytes_recv,
'bytes_sent_last_sec': tot_after.bytes_sent - tot_before.bytes_sent,
'bytes_recv_last_sec': tot_after.bytes_recv - tot_before.bytes_recv}
return network
def network_monitor(self, f):
"""
Packets sent or received
"""
sent = ps.network_io_counters()[0]
received = ps.network_io_counters()[1]
print("Network: Sent: " +str(sent)+ ", Received: " + str(received))
f.write(str(sent)+","+str(received)+"\n")
def get_net_iostat(nic=None): if not nic: if 'net_io_counters' in dir(ps): return ps.net_io_counters(pernic=False) return ps.network_io_counters(pernic=False) if 'net_io_counters' in dir(ps): return ps.net_io_counters(pernic=False) return ps.network_io_counters(pernic=True).get(nic)
def _network_monitor(self):
"""
Packets sent or received
"""
sent = ps.network_io_counters()[0]
received = ps.network_io_counters()[1]
self._print("Sent: " +str(sent)+ ", Received: " + str(received))
self.file.write(str(sent)+";"+str(received)+"\n")
def rrdtool_insert2(self):
for keys in psutil.network_io_counters(pernic=True):
if keys == 'eth0':
sent=psutil.network_io_counters(pernic=True)[keys][0]
recv=psutil.network_io_counters(pernic=True)[keys][1]
#up=rrdtool.updatev('rest.rrd','N:%d:%d' % (sent,recv))
up=rrdtool.updatev(self.rrd_file, 'N:%d:%d' % (recv,sent))
# print up
print "sent: %f recv: %f" % (sent, recv)
def poll(interval):
"""Retrieve raw stats within an interval window."""
tot_before = psutil.network_io_counters()
pnic_before = psutil.network_io_counters(pernic=True)
# sleep some time
time.sleep(interval)
tot_after = psutil.network_io_counters()
pnic_after = psutil.network_io_counters(pernic=True)
return (tot_before, tot_after, pnic_before, pnic_after)
def poll(interval):
"""Retrieve raw stats within an interval window."""
tot_before = psutil.network_io_counters()
pnic_before = psutil.network_io_counters(pernic=True)
# sleep some time
time.sleep(interval)
tot_after = psutil.network_io_counters()
pnic_after = psutil.network_io_counters(pernic=True)
return (tot_before, tot_after, pnic_before, pnic_after)
def pick_speed_in_secs(secs):
value = psutil.network_io_counters(pernic=True)
sub = value['eth0'][0]
down = value['eth0'][1]
time.sleep(secs)
value = psutil.network_io_counters(pernic=True)
actual_sub = int(((value['eth0'][0] - sub) * 0.0009765625 ) / secs)
actual_down = int(((value['eth0'][1] - down) * 0.0009765625) / secs)
return [actual_sub,actual_down]
def getInfo():
#print("Process list: ")
#print psutil.get_process_list()
print "cpu usage: "
print psutil.cpu_percent(1)
print "Memory Usage: "
print psutil.virtual_memory()
print "Disk Usage: "
print psutil.disk_usage('/')
print "Network info: "
print psutil.network_io_counters(True)
def getInfo():
#print("Process list: ")
#print psutil.get_process_list()
print "cpu usage: "
print psutil.cpu_percent(1)
print "Memory Usage: "
print psutil.virtual_memory()
print "Disk Usage: "
print psutil.disk_usage('/')
print "Network info: "
print psutil.network_io_counters(True)
def io_stats():
def to_kilos(x):
return round(x / (self.interval * 1000.0), 1)
d = [
psutil.disk_io_counters().read_bytes,
psutil.disk_io_counters().write_bytes
]
n = [
psutil.network_io_counters().bytes_recv,
psutil.network_io_counters().bytes_sent
]
return map(to_kilos, d), map(to_kilos, n)
def poll(interval):
"""Retrieve raw stats within an interval window."""
tot_before = psutil.network_io_counters()
pnic_before = psutil.network_io_counters(pernic=True)
# sleep some time
time.sleep(interval)
tot_after = psutil.network_io_counters()
pnic_after = psutil.network_io_counters(pernic=True)
# get cpu state
cpu_state = getCPUstate(interval)
# get memory
memory_state = getMemorystate()
return (tot_before, tot_after, pnic_before, pnic_after,cpu_state,memory_state)
def tstateup(self):
(netul, netdl) = psutil.network_io_counters(pernic=1)["wlan0"][:2]
lastnettime = time.time()
while 1:
# sysfs power stats
if os.path.exists("/sys/class/power_supply/battery/capacity"):
self.faststate['cpu_bcap'] = int(
open("/sys/class/power_supply/battery/capacity",
"r").read().strip())
if os.path.exists("/sys/class/power_supply/battery/current_now"):
self.faststate['cpu_bcur'] = int(
open("/sys/class/power_supply/battery/current_now",
"r").read().strip()) / 1000
if os.path.exists("/sys/class/power_supply/ac/current_now"):
self.faststate['cpu_accur'] = int(
open("/sys/class/power_supply/ac/current_now",
"r").read().strip()) / 1000
if os.path.exists("/sys/class/power_supply/battery/status"):
self.faststate['cpu_bstate'] = open(
"/sys/class/power_supply/battery/status",
"r").read().strip()
# iwconfig networking stats
self.piwconfig = subprocess.Popen(["/sbin/iwconfig", "wlan0"],
stdout=subprocess.PIPE)
iwconfig_out, err = self.piwconfig.communicate()
match = self.brre.search(iwconfig_out)
if match: self.faststate['wifi_br'] = match.group(1)
else: self.faststate['wifi_br'] = "n/a"
match = self.lqre.search(iwconfig_out)
if match:
self.faststate['wifi_lq'] = match.group(1)
self.faststate['wifi_sl'] = match.group(2)
else:
self.faststate['wifi_lq'] = "n/a"
self.faststate['wifi_sl'] = "n/a"
# cpu load
self.faststate['cpu'] = psutil.cpu_percent()
# current bandwidth
(_netul,
_netdl) = psutil.network_io_counters(pernic=1)["wlan0"][:2]
_nettime = time.time()
_dnettime = _nettime - lastnettime
self.faststate['ul'] = ((_netul - netul) / _dnettime) / 1000000
self.faststate['dl'] = ((_netdl - netdl) / _dnettime) / 1000000
(netul, netdl, lastnettime) = (_netul, _netdl, _nettime)
self.stateupload()
time.sleep(1)
def check_usage():
status = dict()
status["cpuusage"] = int(psutil.cpu_percent(interval=1))
status["memufree"] = int(psutil.virtual_memory().free)
status["swapfree"] = int(psutil.swap_memory().free)
status["netusage"] = str(psutil.network_io_counters())
response = requests.post("http://54.86.187.108:3000/sendNodeStatusUpdate", status)
def getNET(): # devuelve un arreglo con la cantidad de bits enviados, la cantidad de bits recibidos, la cantidad de paquetes enviados y la cantidad de paquetes recibidos por las interfaces de red
try:
getNET = psutil.network_io_counters(pernic=False)
miNET = str(getNET[0]) + "," + str(getNET[1]) + "," + str(getNET[2]) + "," + str(getNET[3])
return miNET
except:
return ""
def get_data(self):
"""Gets the system's resource usage and sends an alert when it becomes too high."""
now = time.time()
# We have to aggregate usage so we'll pull everything regardless of the params
network_io = psutil.network_io_counters()
time_diff = now - self.last_check
ram_usage = psutil.phymem_usage()
current_usage = {
'bytes_sent': (network_io.bytes_sent - self.total_net_io.bytes_sent) / time_diff,
'bytes_recv': (network_io.bytes_recv - self.total_net_io.bytes_recv) / time_diff,
'ram': ram_usage.percent,
'cpu': psutil.cpu_percent(interval=None),
}
# Add in disk usage data
for disk in self.disks:
usage = psutil.disk_usage(disk)
disk_name = 'disk_{}'.format(disk)
current_usage[disk_name] = usage.percent
self.last_check = now
self.total_net_io = network_io
return current_usage
def network(self):
ifaces = {}
try:
netw = psutil.net_io_counters(pernic=True)
except AttributeError:
# old psutil version
netw = psutil.network_io_counters(pernic=True)
for iface in netw:
ip, mac = self.ifaceinfo(iface)
ifaces[iface] = {}
ifaces[iface]["ip"] = ip
ifaces[iface]["MAC"] = mac
command = "/usr/sbin/ethtool {0}".format(iface)
out, err, rc = self.runcomm(command)
speed = "Unknown"
for line in out:
if "Speed" not in line:
continue
speed = line.split(":")[1].strip()
ifaces[iface]["speed"] = speed
return ifaces
def _net_io_counters(self, pernic=False):
# Smooth out differences in psutil versions: network_io_counters was
# replaced
try:
nic_counters = psutil.network_io_counters(pernic=pernic)
except AttributeError:
nic_counters = psutil.net_io_counters(pernic=pernic)
nic_stats = []
for nic, counters in nic_counters.iteritems():
stats = {'nic': nic}
# psutil 0.4.1 does not capture drops/errors/etc. grab those
ifconfig = Popen(['ifconfig', nic], stdout=PIPE, stderr=PIPE)
out, __ = ifconfig.communicate()
for line in out.splitlines():
line = line.strip()
# we only care about RX rates for now
if not line.startswith('RX packets'):
continue
# now we have the RX packets line
parts = line.split()
for part in parts[2:]: # skip "RX" and "packets"
try:
name, value = part.split(':')
value = int(value)
except ValueError:
continue
stats[name] = value
stats.update(counters._asdict())
nic_stats.append(stats)
return nic_stats
def UpdateStats(cpu, memory, net_in, net_out):
mem_history = []
cpu_history = []
net_in_history = []
net_out_history = []
buffer_len = 120
window = buffer_len / 2
# to baseline the cpu call
psutil.cpu_percent(interval=1)
# TODO: remove outliers
while True:
vmem = psutil.virtual_memory()
mem_history.append(vmem.percent / 100)
mem_history = mem_history[-buffer_len:]
l = sorted(mem_history, reverse=True)[:window]
memory.value = sum(l) / len(l)
cpu_history.append(psutil.cpu_percent(0) / 100)
cpu_history = cpu_history[-buffer_len:]
l = sorted(cpu_history, reverse=True)[:window]
cpu.value = sum(l) / len(l)
net = psutil.network_io_counters()
net_in_history.append(net.bytes_recv)
net_in_history = net_in_history[-buffer_len:]
bytes = [y-x for x,y in pairwise(net_in_history)]
if bytes:
l = sorted(bytes, reverse=True)[:window]
net_in.value = sum(l) / len(l)
net_out_history.append(net.bytes_sent)
net_out_history = net_out_history[-buffer_len:]
bytes = [y-x for x,y in pairwise(net_out_history)]
if bytes:
l = sorted(bytes, reverse=True)[:window]
net_out.value = sum(l) / len(l)
time.sleep(10)
def _get_network(self):
retval = {}
try:
data = self._to_data(psutil.network_io_counters(pernic=True))
if not data.get('errin') and not self.is_win():
# Get manualy errors
try:
f = open("/proc/net/dev", "r")
lines = f.readlines()
for line in lines[2:]:
colon = line.find(':')
assert colon > 0, line
name = line[:colon].strip()
fields = line[colon+1:].strip().split()
# Do not set counter or gauge for this values
data[name]['errir'] = int(fields[2])
data[name]['errout'] = int(fields[10])
f.close()
except:
pass
retval = self.counters(data, 'network')
except:
pass
return retval
def runinsert():
cpu_usage = psutil.cpu_percent()
mem_usage = psutil.phymem_usage().percent
disk_usage = psutil.disk_usage('/').percent
virtual = psutil.virtmem_usage().percent
network = simplejson.dumps(psutil.network_io_counters(True))
conn = sqlite3.connect('stats.db')
cur = conn.cursor()
cur.execute(
"DELETE FROM stats WHERE timestamp < datetime('now','localtime','-7 days')"
)
# cur.execute("DELETE FROM stats WHERE timestamp < datetime('now', 'localtime','-1 hour')")
conn.commit()
cur.execute(
"INSERT INTO stats (cpu, memory, disk, virtual, network) VALUES (?, ?, ?, ?, ? )",
(cpu_usage, mem_usage, disk_usage, virtual, network))
# DUBUG
# cur.execute("SELECT * FROM stats")
# print(cur.fetchall())
conn.commit()
conn.close()
def _net_io_counters(self, pernic=False):
nic_stats = []
nic_counters = psutil.network_io_counters(pernic=pernic)
for nic, counters in nic_counters.iteritems():
stats = {'nic': nic}
# psutil 0.4.1 does not capture drops/errors/etc. grab those
ifconfig = Popen(['ifconfig', nic], stdout=PIPE, stderr=PIPE)
out, __ = ifconfig.communicate()
for line in out.splitlines():
line = line.strip()
# we only care about RX rates for now
if not line.startswith('RX packets'):
continue
# now we have the RX packets line
parts = line.split()
for part in parts[2:]: # skip "RX" and "packets"
try:
name, value = part.split(':')
value = int(value)
except ValueError:
continue
stats[name] = value
stats.update(counters._asdict())
nic_stats.append(stats)
return nic_stats
def get_host_info():
"""Return a ``dict`` with system's information
`Example of its output <https://gist.github.com/turicas/2891134>`_
"""
memory_info = dict(psutil.virtual_memory()._asdict())
swap_info = dict(psutil.swap_memory()._asdict())
info_per_nic = psutil.network_io_counters(pernic=True)
network_info = {}
for key, value in info_per_nic.iteritems():
network_info[key] = {'bytes sent': value.bytes_sent,
'bytes received': value.bytes_recv,
'packets sent': value.packets_sent,
'packets received': value.packets_recv,}
partitions = psutil.disk_partitions()
storage_info = {}
for partition in partitions:
disk_usage = psutil.disk_usage(partition.mountpoint)
storage_info[partition.device] = {'mount point': partition.mountpoint,
'file system': partition.fstype,
'total bytes': disk_usage.total,
'total used bytes': disk_usage.used,
'total free bytes': disk_usage.free,
'percent used': disk_usage.percent,}
return {'memory': memory_info,
'swap': swap_info,
'cpu': {'number of cpus': psutil.NUM_CPUS,
'cpu percent': psutil.cpu_percent(),},
'network': {'interfaces': network_info,},
'storage': storage_info,
'uptime': time() - psutil.BOOT_TIME,}
def GET(self):
render = web.template.render('templates/')
device = self.inet_dev # change this to reflect your network device
cpus = psutil.NUM_CPUS
if self.convert:
up = HumanReadable(psutil.network_io_counters(pernic=True)[device].bytes_sent).bytes2human()
down = HumanReadable(psutil.network_io_counters(pernic=True)[device].bytes_recv).bytes2human()
phymem = HumanReadable(psutil.TOTAL_PHYMEM).bytes2human()
else:
up = psutil.network_io_counters(pernic=True)[device].bytes_sent
down = psutil.network_io_counters(pernic=True)[device].bytes_recv
phymem = psutil.TOTAL_PHYMEM
disks_name = self.get_disk_usage().keys()
disks = self.get_disk_usage()
return render.index(up, down, cpus, phymem, disks, disks_name)
def network_stats():
network_bytes = []
if not common.check_config_sections(['networking', 'interfaces']):
return network_bytes
if hasattr(psutil, 'network_io_counters'):
counters = psutil.network_io_counters(pernic=True)
else:
counters = psutil.net_io_counters(True)
for interface in common.CONFIG['networking']['interfaces']:
counter = counters.get(interface, None)
if not counter:
common.process_exception('cannot find counters for interface %s. skip..' % interface)
continue
date = common.now()
mb_rcv = common.b_to_mb(counter.bytes_recv)
mb_sent = common.b_to_mb(counter.bytes_sent)
logging.info ("NET date: %s interface: %s recv: %s sent: %s", date, interface, mb_rcv, mb_sent, )
network_bytes.extend([
{"date": date, "t": "NET-RCV", "d1": common.HOSTNAME, "d2": interface, "V": mb_rcv},
{"date": date, "t": "NET-SENT", "d1": common.HOSTNAME, "d2": interface, "V": mb_sent},
])
return network_bytes
def _prep_first_run(self):
'''this will prime the needed buffers to present valid data when math is needed'''
try:
#create the first counter values to do math against for network, disk and swap
try:
net_io = psutil.net_io_counters()
except AttributeError:
net_io = psutil.network_io_counters()
for i in range(len(net_io)):
self.buffers[net_io._fields[i]] = net_io[i]
disk_io = psutil.disk_io_counters()
for i in range(len(disk_io)):
self.buffers[disk_io._fields[i]] = disk_io[i]
swap_io = psutil.swap_memory()
for i in range(len(swap_io)):
if swap_io._fields[i] == 'sin' or swap_io._fields[i] == 'sout':
self.buffers[swap_io._fields[i]] = swap_io[i]
#then we sleep so the math represents 1 minute intervals when we do it next
self.logger.debug("sleeping...")
time.sleep(60)
self.first_run = False
self.logger.debug("The pump is primed")
return True
except Exception, e:
self.logger.exception(e)
raise e
def action():
import statsd
stats = statsd.StatsClient()
pipe = stats.pipeline()
counters = psutil.network_io_counters(True)
pattern = None
if j.application.config.exists('nic.pattern'):
pattern = j.application.config.getStr('nic.pattern')
for nic, stat in counters.iteritems():
if pattern and j.codetools.regex.match(pattern, nic) == False:
continue
result = dict()
bytes_sent, bytes_recv, packets_sent, packets_recv, errin, errout, dropin, dropout = stat
result['kbytes_sent'] = int(round(bytes_sent / 1024.0, 0))
result['kbytes_recv'] = int(round(bytes_recv / 1024.0, 0))
result['packets_sent'] = packets_sent
result['packets_recv'] = packets_recv
result['errin'] = errin
result['errout'] = errout
result['dropin'] = dropin
result['dropout'] = dropout
for key, value in result.iteritems():
pipe.gauge(
"%s_%s_nic_%s_%s" %
(j.application.whoAmI.gid, j.application.whoAmI.nid, nic, key),
value)
pipe.send()
def _prep_first_run(self):
'''this will prime the needed buffers to present valid data when math is needed'''
try:
#create the first counter values to do math against for network, disk and swap
try:
net_io = psutil.net_io_counters()
except AttributeError:
net_io = psutil.network_io_counters()
for i in range(len(net_io)):
self.buffers[net_io._fields[i]] = net_io[i]
disk_io = psutil.disk_io_counters()
for i in range(len(disk_io)):
self.buffers[disk_io._fields[i]] = disk_io[i]
swap_io = psutil.swap_memory()
for i in range(len(swap_io)):
if swap_io._fields[i] == 'sin' or swap_io._fields[i] == 'sout':
self.buffers[swap_io._fields[i]] = swap_io[i]
#then we sleep so the math represents 1 minute intervals when we do it next
self.logger.debug("sleeping...")
time.sleep(60)
self.first_run = False
self.logger.debug("The pump is primed")
return True
except Exception, e:
self.logger.exception(e)
raise e
def dump_tolog(cCpu, cSensor, cData):
logFile = open("data/ForensicLog.txt", 'a+')
cPhyMem = psutil.phymem_usage()
cVirMem = psutil.virtmem_usage()
cNetTraffic = psutil.network_io_counters(pernic=True)
cPartitionInfo = psutil.disk_partitions()
cActiveProcess = ''
logFile.write('*===================================================*' +
'\n')
logFile.write('|Forensic Activity Logged at: ' +
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + '\n')
logFile.write('*===================================================*' +
'\n')
logFile.write('SYSCOND: MALFUNCTION | CPU LOAD: ' + cCpu +
'% | SENSOR: ' + cSensor + ' | READ: ' + cData + '\n')
logFile.write('PHYSICAL MEMORY:\n' + str(cPhyMem) + '\n\n')
logFile.write('VIRTUAL MEMORY :\n' + str(cVirMem) + '\n\n')
logFile.write('NETWORK STATUS :\n' + str(cNetTraffic) + '\n\n')
logFile.write('MOUNTED DISKS :\n' + str(cPartitionInfo) + '\n\n')
logFile.write('PROCESS LIST :\n')
for proc in psutil.process_iter():
logFile.write(str(proc) + ',')
logFile.write('\n')
logFile.close()
return
def query_system_status():
"""Query global mem, CPU, disk, and network (if present)"""
uptime = int(time.time() - psutil.BOOT_TIME)
uptime = str(datetime.timedelta(seconds=uptime))
try:
used_mem = psutil.virtmem_usage().percent
used_cpu = psutil.cpu_percent(
percpu=False) #N.B Only looking at one CPU
used_disk = psutil.disk_usage('/').percent
total_network = psutil.network_io_counters(pernic=True)
network_sent, network_rcvd = -1, -1
if network_interface in total_network:
network_sent = round(
total_network[network_interface].bytes_sent / 1024.0 / 1024.0,
2)
network_rcvd = round(
total_network[network_interface].bytes_recv / 1024.0 / 1024.0,
2)
except psutil.AccessDenied:
loggging.error("Access denied when trying to query PID %d" % p)
return None
return {
'used_mem': used_mem,
'used_cpu': used_cpu,
'used_disk': used_disk,
'network_sent': network_sent,
'network_rcvd': network_rcvd,
'uptime': uptime
}
def run_network():
sys.path.append(os.path.split(os.path.dirname(__file__))[0]+'/lib')
push = __import__('pushdata')
value_rate= __import__('record_rate')
raw_netstats = str(psutil.network_io_counters())
split_netstats = raw_netstats.split('(')[-1].split(')')[0]
check_type = 'system'
net_list = [x.strip() for x in split_netstats.split(',')]
jsondata=push.JonSon()
jsondata.create_data()
rate=value_rate.ValueRate()
try:
for index in range(0, len(net_list)):
timestamp = int(datetime.datetime.now().strftime("%s"))
mytype = 'net_'+net_list[index].split('=')[0]
myvalue = net_list[index].split('=')[1]
net_rate=rate.record_value_rate(mytype, myvalue, timestamp)
jsondata.gen_data(mytype, timestamp, net_rate, push.hostname, check_type, cluster_name)
jsondata.put_json()
jsondata.truncate_data()
except Exception as e:
push = __import__('pushdata')
push.print_error(__name__ , (e))
pass
def _get_host_network_io_rate(self, seconds):
net_recv_bytes = self.host_stats['net_recv_bytes']
net_sent_bytes = self.host_stats['net_sent_bytes']
prev_recv_bytes = net_recv_bytes[-1] if net_recv_bytes else 0
prev_sent_bytes = net_sent_bytes[-1] if net_sent_bytes else 0
net_ios = None
if hasattr(psutil, 'net_io_counters'):
net_ios = psutil.net_io_counters(True)
elif hasattr(psutil, 'network_io_counters'):
net_ios = psutil.network_io_counters(True)
recv_bytes = 0
sent_bytes = 0
for key in set(self.nics() +
self.wlans()) & set(net_ios.iterkeys()):
recv_bytes = recv_bytes + net_ios[key].bytes_recv
sent_bytes = sent_bytes + net_ios[key].bytes_sent
rx_rate = int(float(recv_bytes - prev_recv_bytes) / seconds + 0.5)
tx_rate = int(float(sent_bytes - prev_sent_bytes) / seconds + 0.5)
self.host_stats['net_recv_rate'].append(rx_rate)
self.host_stats['net_sent_rate'].append(tx_rate)
self.host_stats['net_recv_bytes'].append(recv_bytes)
self.host_stats['net_sent_bytes'].append(sent_bytes)
def UpdateStats(cpu, memory, net_in, net_out):
mem_history = []
cpu_history = []
net_in_history = []
net_out_history = []
buffer_len = 120
# to baseline the cpu call
psutil.cpu_percent(interval=1)
# TODO: remove outliers
while True:
vmem = psutil.virtual_memory()
mem_history.append(vmem.percent / 100)
mem_history = mem_history[-buffer_len:]
memory.value = sum(mem_history) / len(mem_history)
cpu_history.append(psutil.cpu_percent(0) / 100)
cpu_history = cpu_history[-buffer_len:]
cpu.value = sum(cpu_history) / len(cpu_history)
net = psutil.network_io_counters()
net_in_history.append(net.bytes_rec)
net_in_history = net_in_history[-buffer_len:]
bytes = [y - x for x, y in pairwise(net_in_history)]
if bytes:
net_in.value = sum(bytes) / len(bytes)
net_out_history.append(net.bytes_sent)
net_out_history = net_out_history[-buffer_len:]
bytes = [y - x for x, y in pairwise(net_out_history)]
if bytes:
net_out.value = sum(bytes) / len(bytes)
time.sleep(10)
def UpdateStats(cpu, memory, net_in, net_out):
mem_history = []
cpu_history = []
net_in_history = []
net_out_history = []
buffer_len = 120
# to baseline the cpu call
psutil.cpu_percent(interval=1)
# TODO: remove outliers
while True:
vmem = psutil.virtual_memory()
mem_history.append(vmem.percent / 100)
mem_history = mem_history[-buffer_len:]
memory.value = sum(mem_history) / len(mem_history)
cpu_history.append(psutil.cpu_percent(0) / 100)
cpu_history = cpu_history[-buffer_len:]
cpu.value = sum(cpu_history) / len(cpu_history)
net = psutil.network_io_counters()
net_in_history.append(net.bytes_rec)
net_in_history = net_in_history[-buffer_len:]
bytes = [y-x for x,y in pairwise(net_in_history)]
if bytes:
net_in.value = sum(bytes) / len(bytes)
net_out_history.append(net.bytes_sent)
net_out_history = net_out_history[-buffer_len:]
bytes = [y-x for x,y in pairwise(net_out_history)]
if bytes:
net_out.value = sum(bytes) / len(bytes)
time.sleep(10)
def init_self(self):
# print "init_self"
nets = psutil.network_io_counters(pernic=True)
# {'lo' : iostat(bytes_sent=799953745, bytes_recv=799953745 , packets_sent=453698 , packets_recv=453698 , errin=0, errout=0, dropin=0, dropout=0),
# 'eth0': iostat(bytes_sent=734324837, bytes_recv=4163935363, packets_sent=3605828, packets_recv=4096685, errin=0, errout=0, dropin=0, dropout=0)}
self.net_devices = []
for net in sorted(nets):
netdata = nets[net]
ifdata = netifaces.ifaddresses(net)
#print net, ifdata,"\n"
#print net, "LINK",netifaces.AF_LINK,ifdata[netifaces.AF_LINK],ifdata[netifaces.AF_LINK][0],ifdata[netifaces.AF_LINK][0]['addr'],"\n"
#print net, "INET",netifaces.AF_INET,ifdata[netifaces.AF_INET],ifdata[netifaces.AF_INET][0],ifdata[netifaces.AF_INET][0]['addr'],"\n"
mac = "00:00:00:00:00:00"
if netifaces.AF_LINK in ifdata:
mac = ifdata[netifaces.AF_LINK][0]['addr']
ip = "0.0.0.0"
if netifaces.AF_INET in ifdata:
ip = ifdata[netifaces.AF_INET][0]['addr']
nd = Network_devices(net, netdata.bytes_sent, netdata.bytes_recv,
netdata.packets_sent, netdata.packets_recv,
netdata.errin, netdata.errout, netdata.dropin,
netdata.dropout, mac, ip)
self.net_devices.append(nd)
def _get_host_network_io_rate(self, seconds):
net_recv_bytes = self.host_stats['net_recv_bytes']
net_sent_bytes = self.host_stats['net_sent_bytes']
prev_recv_bytes = net_recv_bytes[-1] if net_recv_bytes else 0
prev_sent_bytes = net_sent_bytes[-1] if net_sent_bytes else 0
net_ios = None
if hasattr(psutil, 'net_io_counters'):
net_ios = psutil.net_io_counters(True)
elif hasattr(psutil, 'network_io_counters'):
net_ios = psutil.network_io_counters(True)
recv_bytes = 0
sent_bytes = 0
for key in set(self.nics() + self.wlans()) & set(net_ios.keys()):
recv_bytes = recv_bytes + net_ios[key].bytes_recv
sent_bytes = sent_bytes + net_ios[key].bytes_sent
rx_rate = int(float(recv_bytes - prev_recv_bytes) / seconds + 0.5)
tx_rate = int(float(sent_bytes - prev_sent_bytes) / seconds + 0.5)
self.host_stats['net_recv_rate'].append(rx_rate)
self.host_stats['net_sent_rate'].append(tx_rate)
self.host_stats['net_recv_bytes'].append(recv_bytes)
self.host_stats['net_sent_bytes'].append(sent_bytes)
def loadMon(duration,interval,emulationID,emulationName,emuStartTime):
HOMEPATH= Library.getHomepath()
emulationName=str(emulationName)
interval=int(interval)
'''
starting cpu monitoring in the loop
'''
iterationsNo=int(duration)/int(interval)
try:
f = open(HOMEPATH+"/logs/"+str(emulationID)+"-"+str(emulationName)+"-res"+"_"+str(emuStartTime)+".csv", 'a')
f.write(emulationName+";\nCountdown;Time;CPU(%);MEM(%);IOread(bytes);IOwrite(bytes);NET(bytes_sent)\n")
#start time
initTime=time.time()
while iterationsNo !=0:
CPU=str(psutil.cpu_percent(interval, False))
#MEM=str(psutil.virtual_memory().percent)
MEM=str(psutil.avail_virtmem())
IOr=str(psutil.disk_io_counters().read_time)
IOw=str(psutil.disk_io_counters().write_time)
NET=str(psutil.network_io_counters(False).bytes_sent)
#print (emulationName+";\nTime;CPU(%);MEM(%);IOread(bytes);IOwrite(bytes);NET(bytes_sent)\n"+str(time.time())+";"+CPU+";"+MEM+";"+IOr+";"+IOw+";"+NET)
probeTime=time.time()-initTime
timeStamp=dt.now()
f.write(str(int(probeTime))+";"+str(timeStamp.strftime("%Y-%m-%d %H:%M:%S.%f"))+";"+CPU+";"+MEM+";"+IOr+";"+IOw+";"+NET+"\n")
iterationsNo=iterationsNo-1
except Exception,e:
print "Unable to create log file\nError: ",e
def action():
import psutil
import os
import statsd
statscl = statsd.StatsClient()
pipe = statscl.pipeline()
results={}
val=psutil.cpu_percent()
results["cpu.percent"]=val
results["cpu.promile"]=val * 10 # we store promile to have more percision
cput= psutil.cpu_times()
for key in cput.__dict__.keys():
val=cput.__dict__[key]
results["cpu.time.%s"%(key)]=val
counter=psutil.network_io_counters(False)
bytes_sent, bytes_recv, packets_sent, packets_recv, errin, errout, dropin, dropout=counter
results["network.kbytes.recv"]=round(bytes_recv/1024.0,0)
results["network.kbytes.send"]=round(bytes_sent/1024.0,0)
results["network.packets.recv"]=packets_recv
results["network.packets.send"]=packets_sent
results["network.error.in"]=errin
results["network.error.out"]=errout
results["network.drop.in"]=dropin
results["network.drop.out"]=dropout
avg1min, avg5min, avg15min = os.getloadavg()
results["load.avg1min"] = int(avg1min * 100)
results["load.avg5min"] = int(avg5min * 100)
results["load.avg15min"] = int(avg15min * 100)
memory=psutil.virtual_memory()
results["memory.used"]=round((memory.used - memory.cached)/1024.0/1024.0,2)
results["memory.cached"]=round(memory.cached/1024.0/1024.0,2)
results["memory.free"]=round(memory.total/1024.0/1024.0,2) - results['memory.used'] - results['memory.cached']
results["memory.percent"]=memory.percent
total,used,free,percent,sin,sout=psutil.virtmem_usage()
results["swap.free"]=round(free/1024.0/1024.0,2)
results["swap.used"]=round(used/1024.0/1024.0,2)
results["swap.percent"]=percent
stat = j.system.fs.fileGetContents('/proc/stat')
stats = dict()
for line in stat.splitlines():
_, key, value = re.split("^(\w+)\s", line)
stats[key] = value
num_ctx_switches = int(stats['ctxt'])
results["cpu.num_ctx_switches"]=num_ctx_switches
for key, value in results.iteritems():
pipe.gauge("%s_%s_%s" % (j.application.whoAmI.gid, j.application.whoAmI.nid, key), int(round(value)))
pipe.send()
return results
def _get_interfaces():
try:
io_counters = psutil.net_io_counters(pernic=True)
except AttributeError:
io_counters = psutil.network_io_counters(pernic=True)
for interface, data in io_counters.items():
if data:
yield interface, data.bytes_recv, data.bytes_sent
def check_usage():
status = dict()
status["cpuusage"] = int(psutil.cpu_percent(interval=1))
status["memufree"] = int(psutil.virtual_memory().free)
status["swapfree"] = int(psutil.swap_memory().free)
status["netusage"] = str(psutil.network_io_counters())
response = requests.post("http://54.86.187.108:3000/sendNodeStatusUpdate",
status)
def monitor_network(sleep=5):
"Monitor network activity"
while True:
data = {'timestamp': timestamp()}
net = psutil.network_io_counters(pernic=True)
data.update(get_srv_data(net))
yield data
time.sleep(sleep)
def collect(self):
"""
Collect network interface stats.
"""
# Initialize results
results = {}
if os.access(self.PROC, os.R_OK):
# Open File
file = open(self.PROC)
# Build Regular Expression
greed = ''
if self.config['greedy'].lower() == 'true' :
greed = '[0-9]+'
exp = '^(?:\s*)((?:%s)%s):(?:\s*)(?P<rx_bytes>\d+)(?:\s*)(?P<rx_packets>\w+)(?:\s*)(?P<rx_errors>\d+)(?:\s*)(?P<rx_drop>\d+)(?:\s*)(?P<rx_fifo>\d+)(?:\s*)(?P<rx_frame>\d+)(?:\s*)(?P<rx_compressed>\d+)(?:\s*)(?P<rx_multicast>\d+)(?:\s*)(?P<tx_bytes>\d+)(?:\s*)(?P<tx_packets>\w+)(?:\s*)(?P<tx_errors>\d+)(?:\s*)(?P<tx_drop>\d+)(?:\s*)(?P<tx_fifo>\d+)(?:\s*)(?P<tx_frame>\d+)(?:\s*)(?P<tx_compressed>\d+)(?:\s*)(?P<tx_multicast>\d+)(?:.*)$' % (( '|'.join(self.config['interfaces']) ), greed)
reg = re.compile(exp)
# Match Interfaces
for line in file:
match = reg.match(line)
if match:
device = match.group(1)
results[device] = match.groupdict()
# Close File
file.close()
elif psutil:
network_stats = psutil.network_io_counters(True)
for device in network_stats.keys():
results[device] = {}
results[device]['rx_bytes'] = network_stats[device].bytes_recv
results[device]['tx_bytes'] = network_stats[device].bytes_sent
results[device]['rx_packets'] = network_stats[device].packets_recv
results[device]['tx_packets'] = network_stats[device].packets_sent
for device in results:
stats = results[device]
for s,v in stats.items():
# Get Metric Name
metric_name = '.'.join([device, s])
# Get Metric Value
metric_value = self.derivative(metric_name, long(v), self.MAX_VALUES[s])
# Convert rx_bytes and tx_bytes
if s == 'rx_bytes' or s == 'tx_bytes':
convertor = diamond.convertor.binary(value = metric_value, unit = 'byte')
for u in self.config['byte_unit']:
# Public Converted Metric
self.publish(metric_name.replace('bytes', u), convertor.get(unit = u))
else:
# Publish Metric Derivative
self.publish(metric_name, metric_value)
return None
def collect_netinfo(dirname):
logging.info("Collecting network info")
logging.info("Collecting network I/O counters")
with open('%s/network-io-counters' % (dirname), 'w') as outfile:
outfile.write(''.join(_pretty(
psutil.network_io_counters(pernic=True))))
logging.info("Collecting netstat information")
with open('%s/netstat' % (dirname), 'w') as outfile:
subprocess.call(['netstat', '-tulpn'], stdout=outfile, stderr=outfile)
def network_io_counters():
'''
Return network I/O statisitics.
CLI Example::
salt '*' ps.network_io_counters
'''
return dict(psutil.network_io_counters()._asdict())
def _get_bytes(interface):
try:
io_counters = psutil.net_io_counters(pernic=True)
except AttributeError:
io_counters = psutil.network_io_counters(pernic=True)
if_io = io_counters.get(interface)
if not if_io:
return None
return if_io.bytes_recv, if_io.bytes_sent
def collect_network_io(self,whitelist=[]):
"""
TODO refactor, as it is essentially copy-paste of disk_io
"""
stats=psutil.network_io_counters(pernic=True)
for entry,stat in stats.iteritems():
if not whitelist or entry in whitelist:
for k,v in stat._asdict().iteritems():
self.add_data("nic-%s.%s"%(entry.replace(" ","_"),k),v)
def sysState(self):
"""
Get system stats
"""
val = dict()
val["time"] = time.time()
val["net_load_loc"] = self.data["net_load_loc"]
val["net_load_in"] = self.data["net_load_in"]
val["net_load_out"] = self.data["net_load_out"]
res_net = psutil.network_io_counters(pernic=True)
# bytes_sent, bytes_recv, packets_sent, packets_recv, errin, errout, dropin, dropout
if self.dev in res_net.keys():
val["net"] = res_net[self.dev]
# print val["net"]
else:
val["net"] = psutil.network_io_counters(pernic=False)
return val