def main(expr):
vars = {'free': psutil.avail_virtmem(),
'available': psutil.avail_virtmem(),
'pfree': psutil.avail_phymem(),
'buffers': psutil.phymem_buffers(),
'total': psutil.total_virtmem(),
'cached': psutil.cached_phymem(),
}
return eval(expr, {}, vars)
def _computeMaximumBufferSize(nodes):
"""number of nodes.
This actually depends on the system mcvine is running on.
If we are running on one machine that has shared memory, we need
to set here nodes to the number of mpi instances.
If we are running on a cluster of nodes without shared memory,
we should not need to divide the maximum buffer size by nodes.
"""
import psutil
memsize = min(psutil.TOTAL_PHYMEM / 2,
(psutil.avail_phymem() + psutil.avail_virtmem()) * 0.7)
memsize = int(memsize)
from mcni.neutron_storage.idfneutron import ndblsperneutron
bytesperdble = 8
minsize = MINIMUM_BUFFER_SIZE
n = int(memsize/nodes/ndblsperneutron/bytesperdble/minsize) \
*MINIMUM_BUFFER_SIZE
if n < minsize:
raise RuntimeError, "Not enough memory"
return n
def mem(self, widget, data=None) : print ' \n DETAILS OF PHYSICAL AND VIRTUAL MEMORY\n ' print 'Total physical memory (in bytes) - ' print psutil.TOTAL_PHYMEM print 'Available physical memory (in bytes) - ' print psutil.avail_phymem() print 'Used physical memory (in bytes) - ' print psutil.used_phymem() print 'Total virtual memory (in bytes) - ' print psutil.total_virtmem() print 'Available virtual memory (in bytes) - ' print psutil.avail_virtmem() print 'Used virtual memory (in bytes) - ' print psutil.used_virtmem() print 'Total cached memory (in bytes) - ' print psutil.cached_phymem()
def _computeMaximumBufferSize(nodes):
"""number of nodes.
This actually depends on the system mcvine is running on.
If we are running on one machine that has shared memory, we need
to set here nodes to the number of mpi instances.
If we are running on a cluster of nodes without shared memory,
we should not need to divide the maximum buffer size by nodes.
"""
import psutil
memsize = min(psutil.TOTAL_PHYMEM/2, (psutil.avail_phymem() + psutil.avail_virtmem())*0.7)
memsize = int(memsize)
from mcni.neutron_storage.idfneutron import ndblsperneutron
bytesperdble = 8
minsize = MINIMUM_BUFFER_SIZE
n = int(memsize/nodes/ndblsperneutron/bytesperdble/minsize) \
*MINIMUM_BUFFER_SIZE
if n<minsize:
raise RuntimeError, "Not enough memory"
return n
def getStat(self):
return {'memTotal': psutil.TOTAL_PHYMEM,
'memFree': psutil.avail_phymem(),
'swapTotal': psutil.total_virtmem(),
'swapFree': psutil.avail_virtmem(),
'buffers': psutil.phymem_buffers(),
'cached': psutil.cached_phymem()}
def calculate_resizing_tofitmemory(x_size,y_size,n_slices,byte):
"""
Predicts the percentage (between 0 and 1) to resize the image to fit the memory,
giving the following information:
x_size, y_size: image size
n_slices: number of slices
byte: bytes allocated for each pixel sample
"""
imagesize = x_size * y_size * n_slices * byte * 28
# USING LIBSIGAR
#import sigar
#sg = sigar.open()
#ram_free = sg.mem().actual_free()
#ram_total = sg.mem().total()
#swap_free = sg.swap().free()
#sg.close()
# USING PSUTIL
import psutil
try:
if (psutil.version_info>=(0,6,0)):
ram_free = psutil.virtual_memory().available
ram_total = psutil.virtual_memory().total
swap_free = psutil.swap_memory().free
else:
ram_free = psutil.phymem_usage().free + psutil.cached_phymem() + psutil.phymem_buffers()
ram_total = psutil.phymem_usage().total
swap_free = psutil.virtmem_usage().free
except:
print "Exception! psutil version < 0.3 (not recommended)"
ram_total = psutil.TOTAL_PHYMEM # this is for psutil < 0.3
ram_free = 0.8 * psutil.TOTAL_PHYMEM
swap_free = psutil.avail_virtmem()
print "RAM_FREE=", ram_free
print "RAM_TOTAL=", ram_total
if (sys.platform == 'win32'):
if (platform.architecture()[0] == '32bit'):
if ram_free>1400000000:
ram_free=1400000000
if ram_total>1400000000:
ram_total=1400000000
if (sys.platform == 'linux2'):
if (platform.architecture()[0] == '32bit'):
if ram_free>3500000000:
ram_free=3500000000
if ram_total>3500000000:
ram_total=3500000000
if (swap_free>ram_total):
swap_free=ram_total
resize = (float((ram_free+0.5*swap_free)/imagesize))
resize=math.sqrt(resize) # this gives the "resize" for each axis x and y
if (resize>1):
resize=1
return round(resize,2)
def loadMon(duration,interval,emulationID,emulationName,emuStartTime):
HOMEPATH= Library.getHomepath()
emulationName=str(emulationName)
interval=int(interval)
'''
starting cpu monitoring in the loop
'''
iterationsNo=int(duration)/int(interval)
try:
f = open(HOMEPATH+"/logs/"+str(emulationID)+"-"+str(emulationName)+"-res"+"_"+str(emuStartTime)+".csv", 'a')
f.write(emulationName+";\nCountdown;Time;CPU(%);MEM(%);IOread(bytes);IOwrite(bytes);NET(bytes_sent)\n")
#start time
initTime=time.time()
while iterationsNo !=0:
CPU=str(psutil.cpu_percent(interval, False))
#MEM=str(psutil.virtual_memory().percent)
MEM=str(psutil.avail_virtmem())
IOr=str(psutil.disk_io_counters().read_time)
IOw=str(psutil.disk_io_counters().write_time)
NET=str(psutil.network_io_counters(False).bytes_sent)
#print (emulationName+";\nTime;CPU(%);MEM(%);IOread(bytes);IOwrite(bytes);NET(bytes_sent)\n"+str(time.time())+";"+CPU+";"+MEM+";"+IOr+";"+IOw+";"+NET)
probeTime=time.time()-initTime
timeStamp=dt.now()
f.write(str(int(probeTime))+";"+str(timeStamp.strftime("%Y-%m-%d %H:%M:%S.%f"))+";"+CPU+";"+MEM+";"+IOr+";"+IOw+";"+NET+"\n")
iterationsNo=iterationsNo-1
except Exception,e:
print "Unable to create log file\nError: ",e
def calculate_resizing_tofitmemory(x_size,y_size,n_slices,byte):
"""
Predicts the percentage (between 0 and 1) to resize the image to fit the memory,
giving the following information:
x_size, y_size: image size
n_slices: number of slices
byte: bytes allocated for each pixel sample
"""
imagesize = x_size * y_size * n_slices * byte * 28
# USING LIBSIGAR
#import sigar
#sg = sigar.open()
#ram_free = sg.mem().actual_free()
#ram_total = sg.mem().total()
#swap_free = sg.swap().free()
#sg.close()
# USING PSUTIL
import psutil
try:
if (psutil.version_info>=(0,6,0)):
ram_free = psutil.virtual_memory().available
ram_total = psutil.virtual_memory().total
swap_free = psutil.swap_memory().free
else:
ram_free = psutil.phymem_usage().free + psutil.cached_phymem() + psutil.phymem_buffers()
ram_total = psutil.phymem_usage().total
swap_free = psutil.virtmem_usage().free
except:
print "Exception! psutil version < 0.3 (not recommended)"
ram_total = psutil.TOTAL_PHYMEM # this is for psutil < 0.3
ram_free = 0.8 * psutil.TOTAL_PHYMEM
swap_free = psutil.avail_virtmem()
print "RAM_FREE=", ram_free
print "RAM_TOTAL=", ram_total
if (sys.platform == 'win32'):
if (platform.architecture()[0] == '32bit'):
if ram_free>1400000000:
ram_free=1400000000
if ram_total>1400000000:
ram_total=1400000000
if (sys.platform == 'linux2'):
if (platform.architecture()[0] == '32bit'):
if ram_free>3500000000:
ram_free=3500000000
if ram_total>3500000000:
ram_total=3500000000
if (swap_free>ram_total):
swap_free=ram_total
resize = (float((ram_free+0.5*swap_free)/imagesize))
resize=math.sqrt(resize) # this gives the "resize" for each axis x and y
if (resize>1):
resize=1
return round(resize,2)
def memInfo():
logging.debug("Mem info")
logging.debug(psutil.virtual_memory())
logging.debug(psutil.swap_memory())
logging.debug(psutil.phymem_usage())
logging.debug(psutil.avail_virtmem())
logging.debug(psutil.avail_phymem())
def generate(self, **kwargs):
result = { 'mem_phys_total': psutil.TOTAL_PHYMEM,
'mem_phys_avail': psutil.avail_phymem(),
'mem_phys_used' : psutil.used_phymem(),
'mem_virt_total': psutil.total_virtmem(),
'mem_virt_avail': psutil.avail_virtmem(),
'mem_virt_used' : psutil.used_virtmem(),
'cpu_cores' : self._get_total_cores(),
}
return result
def generate(self, **kwargs):
result = {
'mem_phys_total': psutil.TOTAL_PHYMEM,
'mem_phys_avail': psutil.avail_phymem(),
'mem_phys_used': psutil.used_phymem(),
'mem_virt_total': psutil.total_virtmem(),
'mem_virt_avail': psutil.avail_virtmem(),
'mem_virt_used': psutil.used_virtmem(),
'cpu_cores': self._get_total_cores(),
}
return result
def get_host_info():
"""Return a ``dict`` with system's information
`Example of its output <https://gist.github.com/gists/2891134>`_
"""
memory_usage = psutil.phymem_usage()
cached_memory = psutil.cached_phymem()
buffered_memory = psutil.phymem_buffers()
real_used = memory_usage.used - buffered_memory - cached_memory
real_free = memory_usage.total - real_used
percent = 100 * (float(memory_usage.used) / memory_usage.total)
real_percent = 100 * (float(real_used) / memory_usage.total)
virtual_used = psutil.used_virtmem()
virtual_free = psutil.avail_virtmem()
virtual_total = virtual_used + virtual_free
info_per_nic = psutil.network_io_counters(pernic=True)
network_info = {}
for key, value in info_per_nic.iteritems():
network_info[key] = {'bytes sent': value.bytes_sent,
'bytes received': value.bytes_recv,
'packets sent': value.packets_sent,
'packets received': value.packets_recv,}
partitions = psutil.disk_partitions()
storage_info = {}
for partition in partitions:
disk_usage = psutil.disk_usage(partition.mountpoint)
storage_info[partition.device] = {'mount point': partition.mountpoint,
'file system': partition.fstype,
'total bytes': disk_usage.total,
'total used bytes': disk_usage.used,
'total free bytes': disk_usage.free,
'percent used': disk_usage.percent,}
return {'memory': {'free': memory_usage.free,
'total': memory_usage.total,
'used': memory_usage.used,
'cached': cached_memory,
'buffers': buffered_memory,
'real used': real_used,
'real free': real_free,
'percent': percent,
'real percent': real_percent,
'total virtual': virtual_total,
'used virtual': virtual_used,
'free virtual': virtual_free,},
'cpu': {'number of cpus': psutil.NUM_CPUS,
'cpu percent': psutil.cpu_percent(),},
'network': {'interfaces': network_info,},
'storage': storage_info,
'uptime': time() - psutil.BOOT_TIME,}
def on_global(self, msg=None):
global_cpu = psutil.get_system_cpu_times()
global_cpu['percent'] = psutil.cpu_percent()
global_mem = {
'used_physical': psutil.used_phymem(),
'used_virtual': psutil.used_virtmem(),
'available_physical': psutil.avail_phymem(),
'available_virtual': psutil.avail_virtmem(),
}
result = {
'hostnome': socket.gethostname(),
'cpu': global_cpu,
'mem': global_mem,
}
return {'global': result}
def test_avail_virtmem(self):
# This test is not particularly accurate and may fail if the OS is
# consuming memory for other applications.
# We just want to test that the difference between psutil result
# and sysctl's is not too high.
p = subprocess.Popen("sysctl vm.vmtotal", shell=1, stdout=subprocess.PIPE)
result = p.communicate()[0].strip()
if sys.version_info >= (3,):
result = str(result, sys.stdout.encoding)
_, sysctl_avail_virtmem = parse_sysctl_vmtotal(result)
psutil_avail_virtmem = psutil.avail_virtmem()
difference = abs(sysctl_avail_virtmem - psutil_avail_virtmem)
# let's assume the test is failed if difference is > 0.5 MB
if difference > (0.5 * 2**20):
self.fail("sysctl=%s; psutil=%s; difference=%s;" %(
sysctl_avail_virtmem, psutil_avail_virtmem, difference))
def collect(self):
return dict(
cpu_times=psutil.cpu_times()._asdict(),
mem=dict(
total_phymem=psutil.TOTAL_PHYMEM,
avail_phymem=psutil.avail_phymem(),
avail_virtmem=psutil.avail_virtmem(),
cached_phymem=psutil.cached_phymem(),
phymem_buffers=psutil.phymem_buffers(),
total_virtmem=psutil.total_virtmem(),
used_phymem=psutil.used_phymem(),
used_virtmem=psutil.used_virtmem(),
),
processes=list(self._processes()),
net=dict(ifaces=self._net_dev(), proto=self._net_proto()),
io=self._io_stats(),
fs=dict(self._fs_usage()),
fh=self._file_handles(),
)
def test2(self):
"Instrument: _getBufferSize"
instrument = Instrument('t')
# for lower values of ncount, buffer_size=ncount/mpisize/DEFAULT_NUMBER_SIM_LOOPS
instrument.inventory.ncount = ncount = 1e3
self.assertEqual(instrument._getBufferSize(),
ncount / DEFAULT_NUMBER_SIM_LOOPS)
instrument.inventory.ncount = ncount = 2e3
self.assertEqual(instrument._getBufferSize(),
ncount / DEFAULT_NUMBER_SIM_LOOPS)
instrument.mpiSize = mpiSize = 10
self.assertEqual(instrument._getBufferSize(),
int(ncount / mpiSize / DEFAULT_NUMBER_SIM_LOOPS))
# for higher values, buffer_size is set by memory limit
import psutil
temp = min(psutil.TOTAL_PHYMEM / 2,
(psutil.avail_phymem() + psutil.avail_virtmem()) * 0.7)
temp = int(temp)
from mcni.neutron_storage.idfneutron import ndblsperneutron
max = int(temp / ndblsperneutron / 8 / 100 / mpiSize) * 100
instrument.inventory.ncount = ncount = 1e9
self.assertEqual(instrument._getBufferSize(), max)
instrument.inventory.ncount = ncount = 2e9
self.assertEqual(instrument._getBufferSize(), max)
# if user set a too high number for buffer size, it is ignored
instrument.inventory.ncount = ncount = 2e9
instrument.inventory.buffer_size = 2e9
self.assertEqual(instrument._getBufferSize(), max)
# if user set a too low value for buffer_size, a warning would be issued
instrument.inventory.ncount = 1e8
instrument.inventory.buffer_size = 100
instrument._getBufferSize()
return
def refresh(self):
logging.debug("Host.refresh(): Refreshing all host entries")
self.refreshVirtServiceSpecs()
self.__machine_type=platform.machine()
self.__network_name=platform.node()
self.__processor=platform.processor()
self.__python_version=platform.python_version()
self.__linuxKernel_version=platform.release()
#self.__name=platform.system()
dist_info=platform.linux_distribution(distname='', version='', id='', supported_dists=('SuSE', 'debian', 'redhat'), full_distribution_name=1)
self.__distroName=dist_info[0]
self.__distroVersion=dist_info[1]
self.__distroCode=dist_info[2]
self.__vm_manager_pid=os.getpid()
self.__freePhyMem=psutil.avail_phymem()/1048576
self.__freeVirtMem=psutil.avail_virtmem()/1048576
self.__totalVirtMem=psutil.total_virtmem()/1048576
self.__usedPhyMem=psutil.used_phymem()/1048576
self.__manager_process=psutil.Process(self.__vm_manager_pid)
self.__virt_process=psutil.Process(int(self.__virtProcessPid))
self.__processUptime=datetime.datetime.fromtimestamp(self.__manager_process.create_time).strftime("%Y-%m-%d %H:%M")
#self.__processOwner=self.__manager_process.username
#self.__numOfProcessThreads=self.__manager_process.get_num_threads()
#self.__numOfVirtProcessThreads=self.__virt_process.get_num_threads()
#for thread in self.__virt_process.get_threads():
# self.__virtProcessThreadIDs.append(thread[0])
#for thread in self.__manager_process.get_threads():
# self.__processThreadIDs.append(thread[0])
self.__virtProcessCpuUsage=self.__virt_process.get_cpu_percent()
self.__proccessCpuUsage=self.__manager_process.get_cpu_percent()
mem=self.__virt_process.get_memory_info()
self.__virtProcessMemUsage=mem[1]/1048576
mem=self.__manager_process.get_memory_info()
self.__processMemUsage=mem[1]/1048576
#self.__processOpenedFilesSpecs=self.__manager_process.get_open_files()
#self.__virtProcessOpenedFilesSpecs=self.__virt_process.get_open_files()
#self.__processConnectionsSpecs=self.__manager_process.get_connections()
#self.__virtProcessConnectionsSpecs=self.__virt_process.get_connections()
self.__cpuUsage=psutil.cpu_percent()
def report_mem(prev_values=None,msg=None):
""" Show status of phy/virt mem, and if given previous values,
report differences
requires the psutil module - avail in apt-get, but I used pip
"""
if msg is None: msg=''
else: msg += ': '
pm = psutil.avail_phymem()
vm = psutil.avail_virtmem()
tim = seconds()
print('{msg}{pm:.2g} GB phys mem, {vm:.2g} GB virt mem avail'
.format(msg=msg, pm=pm/1e9, vm=vm/1e9)),
if prev_values is None:
print
else:
print('- dt={dt:.2g}s, used {pm:.2g} GB phys, {vm:.2g} GB virt'
.format(pm=(prev_values[0] - pm)/1e9,
vm=(prev_values[1] - vm)/1e9, dt = tim-prev_values[2]))
return((pm,vm,tim))
def report_mem(prev_values=None, msg=None):
""" Show status of phy/virt mem, and if given previous values,
report differences
requires the psutil module - avail in apt-get, but I used pip
"""
if msg is None: msg = ''
else: msg += ': '
pm = psutil.avail_phymem()
vm = psutil.avail_virtmem()
tim = seconds()
print('{msg}{pm:.2g} GB phys mem, {vm:.2g} GB virt mem avail'.format(
msg=msg, pm=pm / 1e9, vm=vm / 1e9)),
if prev_values is None:
print
else:
print('- dt={dt:.2g}s, used {pm:.2g} GB phys, {vm:.2g} GB virt'.
format(pm=(prev_values[0] - pm) / 1e9,
vm=(prev_values[1] - vm) / 1e9,
dt=tim - prev_values[2]))
return ((pm, vm, tim))
def loadMon(duration, interval, emulationID, emulationName, emuStartTime):
HOMEPATH = Library.getHomepath()
emulationName = str(emulationName)
interval = int(interval)
'''
starting cpu monitoring in the loop
'''
iterationsNo = int(duration) / int(interval)
try:
f = open(
HOMEPATH + "/logs/" + str(emulationID) + "-" + str(emulationName) +
"-res" + "_" + str(emuStartTime) + ".csv", 'a')
f.write(
emulationName +
";\nCountdown;Time;CPU(%);MEM(%);IOread(bytes);IOwrite(bytes);NET(bytes_sent)\n"
)
#start time
initTime = time.time()
while iterationsNo != 0:
CPU = str(psutil.cpu_percent(interval, False))
#MEM=str(psutil.virtual_memory().percent)
MEM = str(psutil.avail_virtmem())
IOr = str(psutil.disk_io_counters().read_time)
IOw = str(psutil.disk_io_counters().write_time)
NET = str(psutil.network_io_counters(False).bytes_sent)
#print (emulationName+";\nTime;CPU(%);MEM(%);IOread(bytes);IOwrite(bytes);NET(bytes_sent)\n"+str(time.time())+";"+CPU+";"+MEM+";"+IOr+";"+IOw+";"+NET)
probeTime = time.time() - initTime
timeStamp = dt.now()
f.write(
str(int(probeTime)) + ";" +
str(timeStamp.strftime("%Y-%m-%d %H:%M:%S.%f")) + ";" + CPU +
";" + MEM + ";" + IOr + ";" + IOw + ";" + NET + "\n")
iterationsNo = iterationsNo - 1
except Exception, e:
print "Unable to create log file\nError: ", e
def collect(self):
return dict(
cpu_times=psutil.cpu_times()._asdict(),
mem=dict(
total_phymem=psutil.TOTAL_PHYMEM,
avail_phymem=psutil.avail_phymem(),
avail_virtmem=psutil.avail_virtmem(),
cached_phymem=psutil.cached_phymem(),
phymem_buffers=psutil.phymem_buffers(),
total_virtmem=psutil.total_virtmem(),
used_phymem=psutil.used_phymem(),
used_virtmem=psutil.used_virtmem(),
),
processes=list(self._processes()),
net=dict(
ifaces=self._net_dev(),
proto=self._net_proto()
),
io=self._io_stats(),
fs=dict(self._fs_usage()),
fh=self._file_handles(),
)
def test2(self):
"Instrument: _getBufferSize"
instrument = Instrument('t')
# for lower values of ncount, buffer_size=ncount/mpisize/DEFAULT_NUMBER_SIM_LOOPS
instrument.inventory.ncount = ncount = 1e3
self.assertEqual(instrument._getBufferSize(), ncount/DEFAULT_NUMBER_SIM_LOOPS)
instrument.inventory.ncount = ncount = 2e3
self.assertEqual(instrument._getBufferSize(), ncount/DEFAULT_NUMBER_SIM_LOOPS)
instrument.mpiSize = mpiSize = 10
self.assertEqual(instrument._getBufferSize(), int(ncount/mpiSize/DEFAULT_NUMBER_SIM_LOOPS))
# for higher values, buffer_size is set by memory limit
import psutil
temp = min(psutil.TOTAL_PHYMEM/2, (psutil.avail_phymem() + psutil.avail_virtmem())*0.7)
temp = int(temp)
from mcni.neutron_storage.idfneutron import ndblsperneutron
max = int(temp/ndblsperneutron/8/100/mpiSize) * 100
instrument.inventory.ncount = ncount = 1e9
self.assertEqual(instrument._getBufferSize(), max)
instrument.inventory.ncount = ncount = 2e9
self.assertEqual(instrument._getBufferSize(), max)
# if user set a too high number for buffer size, it is ignored
instrument.inventory.ncount = ncount = 2e9
instrument.inventory.buffer_size = 2e9
self.assertEqual(instrument._getBufferSize(), max)
# if user set a too low value for buffer_size, a warning would be issued
instrument.inventory.ncount = 1e8
instrument.inventory.buffer_size = 100
instrument._getBufferSize()
return
def hello_world_app(environ,start_response):
message=""
status = '200 ok'
headers = [('Content-type', 'html; charset=utf-8')]
start_response(status,headers)
#1 psutil parameter
boot_time = datetime.datetime.fromtimestamp(psutil.boot_time()).strftime("%Y-%m-%d %H:%M:%S")
print("\nBOOT TIME:", boot_time)
#2 psutil parameter
cpu_util = psutil.cpu_percent(interval=1, percpu=True)
i=1
print("\nCPU UTILIZATION:")
for cpu in cpu_util:
print("CPU {} : {}% ---".format(i, cpu))
i+=1
#3 psutil parameter
mem = psutil.virtual_memory()
availablemem=psutil.avail_virtmem()
#4 psutil parameter
usedmem=psutil.used_phymem()
#5 psutil parameter
usedper=mem.percent
THRESHOLD = 100 * 1024 * 1024 # 100MB
message +="<TABLE border=10> <TR> <TD> BOOT TIME</TD><TD>"+boot_time+"</TD></TR>"
#message +="<TR><TD>CPU UTILIZATION</TD><TD><TABLE BORDER =2><TR><TD>CPU1</TD><TD>VAL</TD></TR><TR><TD>CPU2</TD><TD>VAL</TD></TR><TR><TD>CPU3</TD><TD>VAL</TD></TR></TABLE></TD></TR>"
message +="<TR> <TD> AVAILABLE MEMORY </TD><TD>"+str(availablemem)+"</TD></TR>"
message +="<TR> <TD> USED MEMORY </TD><TD>"+str(usedmem)+"</TD></TR>"
message +="<TR> <TD> USED PERCENT </TD><TD>"+str(usedper)+"</TD></TR></TABLE>"
return[bytes(message,'utf-8')]
def run(self):
speed_avg = psutil.network_io_counters(pernic=True)
seguir = True
while seguir:
#calculate time generate stats
start = time.time()
#calculate stats
# speed network #
network_actual = pick_speed_in_secs(2)
network_avg = pick_speed_avg(speed_avg);
speed_avg = network_avg[2];
# pack all #
try:
temperature = int(int(os.popen("cat /sys/class/thermal/thermal_zone0/temp").read())/1000)
except:
temperature = 40
data = {
"network_down": network_actual[1],#
"network_up": network_actual[0],#
"network_avg_down": network_avg[1],#
"network_avg_up": network_avg[0],#
"cache": psutil.cached_phymem(),#
"buffer": psutil.phymem_buffers(),#
"used": psutil.used_phymem(),#
"swap_total": psutil.total_virtmem(),#
"swap_used": psutil.used_virtmem(),#
"hdd_use_": psutil.disk_usage('/')[3],
"hdd_use_home": psutil.disk_usage('/home')[3],
"cpu_use": psutil.cpu_percent(interval=1),#
"cpu_mhz": int(os.popen("cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq").read()[:-3]),
"temp": temperature
}
data_string = json.dumps(data)
#print 'ENCODED:', data_string
temp = time.localtime(start)
datatime = str(temp[2])+"/"+str(temp[1])+"/"+str(temp[0])+" "+str(temp[3])+"/"+str(temp[4])+"/"+str(temp[5])
if (os.path.exists(url_file_location) != True):
data = {
"ava": psutil.avail_phymem(),#
"swap_ava": psutil.avail_virtmem(),#
}
data_string_2 = json.dumps(data)
tmp = True
else:
tmp = False
f=open(str(url_file_location),"a")
if (tmp == True):
f.write('"data":'+data_string_2)
f.write(', '+'"'+datatime+'":'+data_string)
f.close()
del data
del data_string
del temp
del datatime
del network_actual
del temperature
time.sleep(generate_time - int(time.time() - start))
def Scheduler():
try:
#Connect to database
#conn = MySQLdb.connect (host = "localhost", user = "******", passwd = "mdat", db = "mdat")
db_path = os.path.join(app_folder, 'databases', 'storage.db')
con = sqlite3.connect(db_path)
con.row_factory = sqlite3.Row
c=con.cursor()
#Check lock
sys_info_sql = 'SELECT * FROM systems WHERE id = "1"'
c.execute(sys_info_sql)
except:
print "Error (1) with database in Scheduler:", sys.exc_info()[0]
con.close()
gc.collect()
return 0
row = c.fetchone()
locked = row['locked']
default_process = row['default_process']
max_process = row['max_process']
if (locked == 'F'): #Lock is free: run analysis
try:
#Lock
locked_sql = 'UPDATE systems SET locked = "T" WHERE id = "1"'
with con:
c.execute(locked_sql)
except:
print "Error (2) with database in Scheduler:", sys.exc_info()[0]
con.close()
gc.collect()
return 0
#Read datasets
c.execute('SELECT id, status FROM results ORDER BY id ASC')
results = c.fetchall()
#Count the number of running processes
running_process=0
for result in results:
if (result['status']=='Running'):
running_process = running_process + 1
#Running process < default, continue
if (running_process < default_process):
remaining_process=0
for result in results:
if (result['status']=='Waiting'):
#Take dataset id
result_id = result['id']
#Start, set status to Running
try:
start_time = datetime.now()
result_update_sql = 'UPDATE results SET status = "Running", start_time = "' + str(start_time) + '" WHERE id = "' + str(result_id) + '"'
with con:
c.execute(result_update_sql)
except:
print "Error (3) with database in Scheduler:", sys.exc_info()[0]
con.close()
gc.collect()
break
remaining_process = remaining_process + 1
try:
executer_path = os.path.join(app_folder, 'modules', 'Executer.py')
cmd = 'python ' + executer_path + ' ' + app_folder + ' ' + str(result_id) + ' &'
os.system(cmd)
except:
print "Error (3) with analyzing process in Scheduler:", sys.exc_info()[0]
con.close()
gc.collect()
break
#Check number of remaining allowed processes
if (remaining_process >= default_process - running_process):
avail_phymem = psutil.avail_phymem()
avail_virtmem = psutil.avail_virtmem()
if (avail_phymem < 500000000) | (avail_virtmem < 1000000000) | (remaining_process >= max_process - running_process):
break
#Unlock and Halt
try:
unlocked_sql = 'UPDATE systems SET locked = "F" WHERE id = "1"'
with con:
c.execute(unlocked_sql)
except:
print "Error (4) with database in Scheduler:", sys.exc_info()[0]
con.close()
gc.collect()
return 0
con.close()
gc.collect()
return 0
else: #Lock is not free: halt
con.close()
gc.collect()
return 0
def check_mem():
server = 'localhost'
port = 2003
## hostname needed for the graphite tree
hostname = socket.gethostname()
## build the tree up a bit
tree = hostname+".memory."
## build a dictionary of the values we want to graph
try:
mem_dict = {"total_physical_memory": psutil.phymem_usage().total, 'free_physical_memory':psutil.phymem_usage().free, "used_physical_memory": psutil.phymem_usage().used, "total_virtual_memory":psutil.total_virtmem(), "free_virtual_memory": psutil.avail_virtmem(), "used_virtual_memory": psutil.used_virtmem()}
## Iterate the dictionary and write out the value to graphite for each key
for key in mem_dict:
w2g(server, port, tree+key, mem_dict[key])
except Exception as e: ## clean up errors a bit
print e
exit(1)
# -*- coding: utf-8 -*-
"""
Created on Tue Sep 3 09:21:57 2013
@author: ozdemircili
"""
import psutil
psutil.phymem_usage()
psutil.virtmem_usage()
psutil.cached_phymem()
psutil.phymem_buffers()
psutil.avail_phymem()
psutil.used_phymem()
psutil.total_virtmem()
psutil.avail_virtmem()
psutil.used_virtmem()
psutil.cpu_percent()
#CPU
from __future__ import print_function
with open('/proc/cpuinfo') as f:
for line in f:
#Formatting the output!!
if line.strip():
if line.rstrip('\n').startswith('model name'):
model_name = line.rstrip('\n').split(':')[1]
def test_avail_virtmem(self):
x = psutil.avail_virtmem()
self.assertTrue(isinstance(x, (int, long)))
self.assertTrue(x >= 0)
def _store_resources(self, store=True):
"""Looks at the resources usage and store the data locally.
store (bool): if False, run the method but do not store the
resulting values - useful for initializing the
previous values
"""
logger.debug("ResourceService._store_resources")
# We use the precise time to compute the delta
now = time.time()
delta = now - self._last_saved_time
self._last_saved_time = now
now = int(now)
data = {}
# CPU
cpu_times = self._get_cpu_times()
data["cpu"] = dict((x, int(round((cpu_times[x] -
self._prev_cpu_times[x])
/ delta * 100.0)))
for x in cpu_times)
data["cpu"]["num_cpu"] = psutil.NUM_CPUS
self._prev_cpu_times = cpu_times
# Memory. We differentiate from old and deprecated (< 0.3.0)
# methods to the new ones. Remove the differentiation when we
# drop the support for Ubuntu 11.10 (which ships 0.2.1).
ram_cached = psutil.cached_phymem()
ram_buffers = psutil.phymem_buffers()
if psutil_version < (0, 3, 0):
data["memory"] = {
"ram_total": psutil.TOTAL_PHYMEM / B_TO_MB,
"ram_available": psutil.avail_phymem() / B_TO_MB,
"ram_cached": ram_cached / B_TO_MB,
"ram_buffers": ram_buffers / B_TO_MB,
"ram_used": (psutil.used_phymem() - ram_cached - ram_buffers)
/ B_TO_MB,
"swap_total": psutil.total_virtmem() / B_TO_MB,
"swap_available": psutil.avail_virtmem() / B_TO_MB,
"swap_used": psutil.used_virtmem() / B_TO_MB,
}
else:
phymem = psutil.phymem_usage()
virtmem = psutil.virtmem_usage()
data["memory"] = {
"ram_total": phymem.total / B_TO_MB,
"ram_available": phymem.free / B_TO_MB,
"ram_cached": ram_cached / B_TO_MB,
"ram_buffers": ram_buffers / B_TO_MB,
"ram_used": (phymem.used - ram_cached - ram_buffers) / B_TO_MB,
"swap_total": virtmem.total / B_TO_MB,
"swap_available": virtmem.free / B_TO_MB,
"swap_used": virtmem.used / B_TO_MB,
}
data["services"] = {}
# Details of our services
for service in self._local_services:
dic = {"autorestart": self._will_restart[service],
"running": True}
proc = self._procs[service]
# If we don't have a previously found process for the
# service, we find it
if proc is None:
proc = self._find_proc(service)
# If we still do not find it, there is no process
if proc is None:
dic["running"] = False
# We have a process, but maybe it has been shut down
elif not proc.is_running():
# If so, let us find the new one
proc = self._find_proc(service)
# If there is no new one, continue
if proc is None:
dic["running"] = False
# If the process is not running, we have nothing to do.
if not dic["running"]:
data["services"][str(service)] = dic
continue
try:
dic["since"] = self._last_saved_time - proc.create_time
dic["resident"], dic["virtual"] = \
(x / 1048576 for x in proc.get_memory_info())
cpu_times = proc.get_cpu_times()
dic["user"] = int(
round((cpu_times[0] -
self._services_prev_cpu_times[service][0])
/ delta * 100))
dic["sys"] = int(
round((cpu_times[1] -
self._services_prev_cpu_times[service][1])
/ delta * 100))
self._services_prev_cpu_times[service] = cpu_times
try:
dic["threads"] = proc.get_num_threads()
except AttributeError:
dic["threads"] = 0 # 0 = Not implemented
self._procs[service] = proc
except psutil.error.NoSuchProcess:
# Shut down while we operated?
dic = {"autorestart": self._will_restart[service],
"running": False}
data["services"][str(service)] = dic
if store:
if len(self._local_store) >= 5000: # almost 7 hours
self._local_store = self._local_store[1:]
self._local_store.append((now, data))
return True
def getMemPerc(cls):
return "%03d" % int(100.0 * (1.0 - float(psutil.avail_virtmem()) / float(psutil.total_virtmem()))) + \
"m% "
"""
Created on Tue Sep 3 09:21:57 2013
@author: ozdemircili
"""
import psutil
psutil.phymem_usage()
psutil.virtmem_usage()
psutil.cached_phymem()
psutil.phymem_buffers()
psutil.avail_phymem()
psutil.used_phymem()
psutil.total_virtmem()
psutil.avail_virtmem()
psutil.used_virtmem()
psutil.cpu_percent()
#CPU
from __future__ import print_function
with open('/proc/cpuinfo') as f:
for line in f:
#Formatting the output!!
if line.strip():
if line.rstrip('\n').startswith('model name'):
model_name = line.rstrip('\n').split(':')[1]
def _store_resources(self, store=True):
"""Looks at the resources usage and store the data locally.
store (bool): if False, run the method but do not store the
resulting values - useful for initializing the
previous values
"""
logger.debug("ResourceService._store_resources")
# We use the precise time to compute the delta
now = time.time()
delta = now - self._last_saved_time
self._last_saved_time = now
now = int(now)
data = {}
# CPU
cpu_times = self._get_cpu_times()
data["cpu"] = dict((x, int(round((cpu_times[x] -
self._prev_cpu_times[x])
/ delta * 100.0)))
for x in cpu_times)
data["cpu"]["num_cpu"] = psutil.NUM_CPUS
self._prev_cpu_times = cpu_times
# Memory. We differentiate from old and deprecated (< 0.3.0)
# methods to the new ones. Remove the differentiation when we
# drop the support for Ubuntu 11.10 (which ships 0.2.1).
ram_cached = psutil.cached_phymem()
ram_buffers = psutil.phymem_buffers()
if psutil_version < (0, 3, 0):
data["memory"] = {
"ram_total": psutil.TOTAL_PHYMEM / B_TO_MB,
"ram_available": psutil.avail_phymem() / B_TO_MB,
"ram_cached": ram_cached / B_TO_MB,
"ram_buffers": ram_buffers / B_TO_MB,
"ram_used": (psutil.used_phymem() - ram_cached - ram_buffers)
/ B_TO_MB,
"swap_total": psutil.total_virtmem() / B_TO_MB,
"swap_available": psutil.avail_virtmem() / B_TO_MB,
"swap_used": psutil.used_virtmem() / B_TO_MB,
}
else:
phymem = psutil.phymem_usage()
virtmem = psutil.virtmem_usage()
data["memory"] = {
"ram_total": phymem.total / B_TO_MB,
"ram_available": phymem.free / B_TO_MB,
"ram_cached": ram_cached / B_TO_MB,
"ram_buffers": ram_buffers / B_TO_MB,
"ram_used": (phymem.used - ram_cached - ram_buffers) / B_TO_MB,
"swap_total": virtmem.total / B_TO_MB,
"swap_available": virtmem.free / B_TO_MB,
"swap_used": virtmem.used / B_TO_MB,
}
data["services"] = {}
# Details of our services
for service in self._local_services:
dic = {"autorestart": self._will_restart[service],
"running": True}
proc = self._procs[service]
# If we don't have a previously found process for the
# service, we find it
if proc is None:
proc = self._find_proc(service)
# If we still do not find it, there is no process
if proc is None:
dic["running"] = False
# We have a process, but maybe it has been shut down
elif not proc.is_running():
# If so, let us find the new one
proc = self._find_proc(service)
# If there is no new one, continue
if proc is None:
dic["running"] = False
# If the process is not running, we have nothing to do.
if not dic["running"]:
data["services"][str(service)] = dic
continue
try:
dic["since"] = self._last_saved_time - proc.create_time
dic["resident"], dic["virtual"] = \
(x / 1048576 for x in proc.get_memory_info())
cpu_times = proc.get_cpu_times()
dic["user"] = int(
round((cpu_times[0] -
self._services_prev_cpu_times[service][0])
/ delta * 100))
dic["sys"] = int(
round((cpu_times[1] -
self._services_prev_cpu_times[service][1])
/ delta * 100))
self._services_prev_cpu_times[service] = cpu_times
try:
dic["threads"] = proc.get_num_threads()
except AttributeError:
dic["threads"] = 0 # 0 = Not implemented
self._procs[service] = proc
except psutil.error.NoSuchProcess:
# Shut down while we operated?
dic = {"autorestart": self._will_restart[service],
"running": False}
data["services"][str(service)] = dic
if store:
if len(self._local_store) >= 5000: # almost 7 hours
self._local_store = self._local_store[1:]
self._local_store.append((now, data))
return True
def test_avail_virtmem(self):
x = psutil.avail_virtmem()
self.assertTrue(isinstance(x, (int, long)))
self.assertTrue(x >= 0)
