def calculate_resizing_tofitmemory(x_size,y_size,n_slices,byte):
"""
Predicts the percentage (between 0 and 1) to resize the image to fit the memory,
giving the following information:
x_size, y_size: image size
n_slices: number of slices
byte: bytes allocated for each pixel sample
"""
imagesize = x_size * y_size * n_slices * byte * 28
# USING LIBSIGAR
#import sigar
#sg = sigar.open()
#ram_free = sg.mem().actual_free()
#ram_total = sg.mem().total()
#swap_free = sg.swap().free()
#sg.close()
# USING PSUTIL
import psutil
try:
if (psutil.version_info>=(0,6,0)):
ram_free = psutil.virtual_memory().available
ram_total = psutil.virtual_memory().total
swap_free = psutil.swap_memory().free
else:
ram_free = psutil.phymem_usage().free + psutil.cached_phymem() + psutil.phymem_buffers()
ram_total = psutil.phymem_usage().total
swap_free = psutil.virtmem_usage().free
except:
print "Exception! psutil version < 0.3 (not recommended)"
ram_total = psutil.TOTAL_PHYMEM # this is for psutil < 0.3
ram_free = 0.8 * psutil.TOTAL_PHYMEM
swap_free = psutil.avail_virtmem()
print "RAM_FREE=", ram_free
print "RAM_TOTAL=", ram_total
if (sys.platform == 'win32'):
if (platform.architecture()[0] == '32bit'):
if ram_free>1400000000:
ram_free=1400000000
if ram_total>1400000000:
ram_total=1400000000
if (sys.platform == 'linux2'):
if (platform.architecture()[0] == '32bit'):
if ram_free>3500000000:
ram_free=3500000000
if ram_total>3500000000:
ram_total=3500000000
if (swap_free>ram_total):
swap_free=ram_total
resize = (float((ram_free+0.5*swap_free)/imagesize))
resize=math.sqrt(resize) # this gives the "resize" for each axis x and y
if (resize>1):
resize=1
return round(resize,2)
def get_mem_stats():
'''return mem stats'''
if hasattr(psutil, 'cached_phymem') and hasattr(psutil, 'phymem_buffers'):
cachemem = psutil.cached_phymem() + psutil.phymem_buffers()
else:
cachemem = -1
phymem = psutil.phymem_usage()
mem = {
'cache': cachemem,
'total': phymem.total,
'used': phymem.used,
'free': phymem.free,
'percent': phymem.percent
}
virtmem = psutil.virtmem_usage()
memswap = {
'total': virtmem.total,
'used': virtmem.used,
'free': virtmem.free,
'percent': virtmem.percent
}
return {"cache": cachemem, "mem": mem, "swap": memswap}
def getStat(self):
return {'memTotal': psutil.TOTAL_PHYMEM,
'memFree': psutil.avail_phymem(),
'swapTotal': psutil.total_virtmem(),
'swapFree': psutil.avail_virtmem(),
'buffers': psutil.phymem_buffers(),
'cached': psutil.cached_phymem()}
def calculate_resizing_tofitmemory(x_size,y_size,n_slices,byte):
"""
Predicts the percentage (between 0 and 1) to resize the image to fit the memory,
giving the following information:
x_size, y_size: image size
n_slices: number of slices
byte: bytes allocated for each pixel sample
"""
imagesize = x_size * y_size * n_slices * byte * 28
# USING LIBSIGAR
#import sigar
#sg = sigar.open()
#ram_free = sg.mem().actual_free()
#ram_total = sg.mem().total()
#swap_free = sg.swap().free()
#sg.close()
# USING PSUTIL
import psutil
try:
if (psutil.version_info>=(0,6,0)):
ram_free = psutil.virtual_memory().available
ram_total = psutil.virtual_memory().total
swap_free = psutil.swap_memory().free
else:
ram_free = psutil.phymem_usage().free + psutil.cached_phymem() + psutil.phymem_buffers()
ram_total = psutil.phymem_usage().total
swap_free = psutil.virtmem_usage().free
except:
print "Exception! psutil version < 0.3 (not recommended)"
ram_total = psutil.TOTAL_PHYMEM # this is for psutil < 0.3
ram_free = 0.8 * psutil.TOTAL_PHYMEM
swap_free = psutil.avail_virtmem()
print "RAM_FREE=", ram_free
print "RAM_TOTAL=", ram_total
if (sys.platform == 'win32'):
if (platform.architecture()[0] == '32bit'):
if ram_free>1400000000:
ram_free=1400000000
if ram_total>1400000000:
ram_total=1400000000
if (sys.platform == 'linux2'):
if (platform.architecture()[0] == '32bit'):
if ram_free>3500000000:
ram_free=3500000000
if ram_total>3500000000:
ram_total=3500000000
if (swap_free>ram_total):
swap_free=ram_total
resize = (float((ram_free+0.5*swap_free)/imagesize))
resize=math.sqrt(resize) # this gives the "resize" for each axis x and y
if (resize>1):
resize=1
return round(resize,2)
def get_mem_stats():
'''return mem stats'''
if hasattr(psutil, 'cached_phymem') and hasattr(psutil, 'phymem_buffers'):
cachemem = psutil.cached_phymem() + psutil.phymem_buffers()
else:
cachemem = -1
phymem = psutil.phymem_usage()
mem = {
'cache': cachemem,
'total': phymem.total,
'used': phymem.used,
'free': phymem.free,
'percent': phymem.percent
}
virtmem = psutil.virtmem_usage()
memswap = {
'total': virtmem.total,
'used': virtmem.used,
'free': virtmem.free,
'percent': virtmem.percent
}
return {
"cache": cachemem,
"mem": mem,
"swap": memswap
}
def mem(self, widget, data=None) : print ' \n DETAILS OF PHYSICAL AND VIRTUAL MEMORY\n ' print 'Total physical memory (in bytes) - ' print psutil.TOTAL_PHYMEM print 'Available physical memory (in bytes) - ' print psutil.avail_phymem() print 'Used physical memory (in bytes) - ' print psutil.used_phymem() print 'Total virtual memory (in bytes) - ' print psutil.total_virtmem() print 'Available virtual memory (in bytes) - ' print psutil.avail_virtmem() print 'Used virtual memory (in bytes) - ' print psutil.used_virtmem() print 'Total cached memory (in bytes) - ' print psutil.cached_phymem()
def update(self):
phys_mem = psutil.phymem_usage()
self.mem_usage = {'total': phys_mem.total, 'used': phys_mem.used,
'free': phys_mem.free, 'percent': phys_mem.percent,
'buffers': psutil.phymem_buffers(),
'cached': psutil.cached_phymem()}
virt_mem = psutil.virtmem_usage()
self.swap_usage = {'total': virt_mem.total, 'used': virt_mem.used, 'free': virt_mem.free, 'percent': virt_mem.percent}
def cached_physical_memory():
'''
Return the amount cached memory.
CLI Example::
salt '*' ps.cached_physical_memory
'''
return psutil.cached_phymem()
def cached_physical_memory():
'''
Return the amount cached memory.
CLI Example::
salt '*' ps.cached_physical_memory
'''
return psutil.cached_phymem()
def main(expr):
vars = {'free': psutil.avail_virtmem(),
'available': psutil.avail_virtmem(),
'pfree': psutil.avail_phymem(),
'buffers': psutil.phymem_buffers(),
'total': psutil.total_virtmem(),
'cached': psutil.cached_phymem(),
}
return eval(expr, {}, vars)
def stat(self): ds = dict() ds['pt'] = int(psutil.phymem_usage().total / 1024) ds['pf'] = int(psutil.phymem_usage().free / 1024) ds['pu'] = int(psutil.phymem_usage().used / 1024) ds['pb'] = int(psutil.phymem_buffers() / 1024) ds['pc'] = int(psutil.cached_phymem() / 1024) ds['st'] = int(psutil.virtmem_usage().total / 1024) ds['su'] = int(psutil.virtmem_usage().used / 1024) return ds
def test_cached_phymem(self):
# test psutil.cached_phymem against "cached" column of free
# command line utility
p = subprocess.Popen("free", shell=1, stdout=subprocess.PIPE)
output = p.communicate()[0].strip()
if sys.version_info >= (3,):
output = str(output, sys.stdout.encoding)
free_cmem = int(output.split("\n")[1].split()[6])
psutil_cmem = psutil.cached_phymem() / 1024
self.assertEqual(free_cmem, psutil_cmem)
def test_cached_phymem(self):
# test psutil.cached_phymem against "cached" column of free
# command line utility
p = subprocess.Popen("free", shell=1, stdout=subprocess.PIPE)
output = p.communicate()[0].strip()
if PY3:
output = str(output, sys.stdout.encoding)
free_cmem = int(output.split('\n')[1].split()[6])
psutil_cmem = psutil.cached_phymem() / 1024
self.assertEqual(free_cmem, psutil_cmem)
def stat(self):
ds = dict()
ds['pt'] = int(psutil.phymem_usage().total / 1024)
ds['pf'] = int(psutil.phymem_usage().free / 1024)
ds['pu'] = int(psutil.phymem_usage().used / 1024)
ds['pb'] = int(psutil.phymem_buffers() / 1024)
ds['pc'] = int(psutil.cached_phymem() / 1024)
ds['st'] = int(psutil.virtmem_usage().total / 1024)
ds['su'] = int(psutil.virtmem_usage().used / 1024)
return ds
def usedmemory(self):
import psutil
from distutils.version import StrictVersion
# print sizes in human readable (MB)
MB = 1024 * 1024
if StrictVersion(psutil.__version__) < '0.4': # deprecated
total_mem = psutil.avail_phymem() + \
psutil.cached_phymem() + psutil.phymem_buffers()
return "%d of %d" % (psutil.used_phymem() / MB, total_mem / MB)
else:
# I don't care about cached memory..
return "%d of %d" % \
((psutil.phymem_usage().used -
psutil.cached_phymem() -
psutil.phymem_buffers()) / MB,
psutil.phymem_usage().total / MB)
def cached_physical_memory():
'''
Return the amount cached memory.
CLI Example:
.. code-block:: bash
salt '*' ps.cached_physical_memory
'''
return psutil.cached_phymem()
def memorypercent(self):
import psutil
from distutils.version import StrictVersion
if StrictVersion(psutil.__version__) < '0.4': # deprecated
total_mem = psutil.avail_phymem() + \
psutil.cached_phymem() + psutil.phymem_buffers()
return psutil.used_phymem() * 100 / total_mem
else:
return int(psutil.phymem_usage().percent)
def cached_physical_memory():
'''
Return the amount cached memory.
CLI Example:
.. code-block:: bash
salt '*' ps.cached_physical_memory
'''
return psutil.cached_phymem()
def __init__(self):
self.memfree = int(psutil.avail_phymem()) / (1024 * 1024)
self.memused = int(psutil.used_phymem()) / (1024 * 1024)
self.memcached = int(psutil.cached_phymem()) / (1024 * 1024)
self.memtotal = self.memfree + self.memused
self.memload = psutil.phymem_usage()[3]
self.load = os.getloadavg()
self.uptime = ProcParser().getuptime()
self.coresload = psutil.cpu_percent(0.7, True)
self.time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
statvfs = os.statvfs('/')
self.hddtotal = statvfs.f_frsize * statvfs.f_blocks / 1024 / 1024 # Size of filesystem in bytes
self.hddfree = statvfs.f_frsize * statvfs.f_bfree / 1024 / 1024 # Actual number of free bytes
self.hddused = self.hddtotal - self.hddfree
def _get_mem(self):
retval = {}
try:
if hasattr(psutil, 'virtual_memory'):
mem = psutil.virtual_memory()
if hasattr(mem, 'active'):
retval['active'] = self.to_gb(mem.active)
if hasattr(mem, 'inactive'):
retval['inactive'] = self.to_gb(mem.inactive)
if hasattr(mem, 'buffers'):
retval['buffers'] = self.to_gb(mem.buffers)
if hasattr(mem, 'cached'):
retval['cached'] = self.to_gb(mem.cached)
if hasattr(mem, 'shared'):
retval['shared'] = self.to_gb(mem.shared)
else:
mem = psutil.phymem_usage()
retval['cached'] = self.to_gb(psutil.cached_phymem())
retval['buffers'] = self.to_gb(psutil.phymem_buffers())
if not self.is_win():
try:
f = open('/proc/meminfo', 'r')
for line in f:
if line.startswith('Active:'):
retval['active'] = self.to_gb(int(line.split()[1]) * 1024)
if line.startswith('Inactive:'):
retval['inactive'] = self.to_gb(int(line.split()[1]) * 1024)
if line.startswith('Buffers:'):
retval['buffers'] = self.to_gb(int(line.split()[1]) * 1024)
if line.startswith('Cached:'):
retval['cached'] = self.to_gb(int(line.split()[1]) * 1024)
if line.startswith('Shared:'):
retval['shared'] = self.to_gb(int(line.split()[1]) * 1024)
f.close()
except:
pass
retval['total'] = self.to_gb(mem.total)
retval['used'] = self.to_gb(mem.used)
retval['free'] = self.to_gb(mem.free)
retval['percent'] = mem.percent
except:
pass
return retval
def get_host_info():
"""Return a ``dict`` with system's information
`Example of its output <https://gist.github.com/gists/2891134>`_
"""
memory_usage = psutil.phymem_usage()
cached_memory = psutil.cached_phymem()
buffered_memory = psutil.phymem_buffers()
real_used = memory_usage.used - buffered_memory - cached_memory
real_free = memory_usage.total - real_used
percent = 100 * (float(memory_usage.used) / memory_usage.total)
real_percent = 100 * (float(real_used) / memory_usage.total)
virtual_used = psutil.used_virtmem()
virtual_free = psutil.avail_virtmem()
virtual_total = virtual_used + virtual_free
info_per_nic = psutil.network_io_counters(pernic=True)
network_info = {}
for key, value in info_per_nic.iteritems():
network_info[key] = {'bytes sent': value.bytes_sent,
'bytes received': value.bytes_recv,
'packets sent': value.packets_sent,
'packets received': value.packets_recv,}
partitions = psutil.disk_partitions()
storage_info = {}
for partition in partitions:
disk_usage = psutil.disk_usage(partition.mountpoint)
storage_info[partition.device] = {'mount point': partition.mountpoint,
'file system': partition.fstype,
'total bytes': disk_usage.total,
'total used bytes': disk_usage.used,
'total free bytes': disk_usage.free,
'percent used': disk_usage.percent,}
return {'memory': {'free': memory_usage.free,
'total': memory_usage.total,
'used': memory_usage.used,
'cached': cached_memory,
'buffers': buffered_memory,
'real used': real_used,
'real free': real_free,
'percent': percent,
'real percent': real_percent,
'total virtual': virtual_total,
'used virtual': virtual_used,
'free virtual': virtual_free,},
'cpu': {'number of cpus': psutil.NUM_CPUS,
'cpu percent': psutil.cpu_percent(),},
'network': {'interfaces': network_info,},
'storage': storage_info,
'uptime': time() - psutil.BOOT_TIME,}
def getAvailableMemory(settings):
if settings.nopsutil:
return 0
import psutil
cacheusage=0
if 'linux' in settings.OSTYPE.lower():
cacheusage = psutil.cached_phymem()
memusage = `psutil.phymem_usage()`.split(",")
freemem = long(memusage[2].split("free=")[-1])+long(cacheusage)
percentfree = float(memusage[3].split("percent=")[-1].split(")")[0])
avram = (freemem/1000000000)
return avram
def cached_physical_memory():
'''
.. deprecated:: Helium
Use :mod:`ps.virtual_memory <salt.modules.ps.virtual_memory>` instead.
Return the amount cached memory.
CLI Example:
.. code-block:: bash
salt '*' ps.cached_physical_memory
'''
salt.utils.warn_until(
'Helium', '\'ps.cached_physical_memory\' is deprecated. Please use'
'\'ps.virtual_memory\' instead. This functionality will'
'be removed in Salt {version}.')
return psutil.cached_phymem()
def collect(self):
return dict(
cpu_times=psutil.cpu_times()._asdict(),
mem=dict(
total_phymem=psutil.TOTAL_PHYMEM,
avail_phymem=psutil.avail_phymem(),
avail_virtmem=psutil.avail_virtmem(),
cached_phymem=psutil.cached_phymem(),
phymem_buffers=psutil.phymem_buffers(),
total_virtmem=psutil.total_virtmem(),
used_phymem=psutil.used_phymem(),
used_virtmem=psutil.used_virtmem(),
),
processes=list(self._processes()),
net=dict(ifaces=self._net_dev(), proto=self._net_proto()),
io=self._io_stats(),
fs=dict(self._fs_usage()),
fh=self._file_handles(),
)
def update(self, cur_time):
self.counter += 1L
row = []
row.append(self.counter)
row.append('%.2f' % (cur_time - self.get_creation_time()))
pymem = psutil.phymem_usage()
vmem = psutil.virtmem_usage()
row.append('%.2f' % (atof(pymem[1]) / (1024.0 * 1024.0)))
row.append('%.2f' % pymem[3])
row.append('%.2f' % (atof(vmem[1]) / (1024.0 * 1024.0)))
row.append('%.2f' % vmem[3])
# cached memory and physical memory buffers on the system
if self.linux == True:
row.append('%d' % (psutil.cached_phymem() / (1024.0 * 1024.0)))
row.append('%d' % (psutil.phymem_buffers() / (1024.0 * 1024.0)))
self.writer.writerow(row)
def _collect_and_write(self,
process,
writer,
ts=None,
iostat=None,
netstat=None):
import psutil
if ts is None:
ts = time.time()
cpu_percent = process.get_cpu_percent()
cpu_times = process.get_cpu_times()
mem_info = process.get_memory_info()
data = [
int(ts), process.pid, process.name, cpu_percent, cpu_times.user,
cpu_times.system,
process.get_memory_percent(), mem_info.rss, mem_info.vms
]
box_data = None
if iostat and netstat:
box_cpu = psutil.cpu_times_percent()
io = psutil.disk_io_counters()
netio = psutil.net_io_counters()
box_mem = psutil.phymem_usage()
box_data = [
box_cpu.user, box_cpu.system, box_cpu.idle,
io.read_bytes - iostat.read_bytes,
io.write_bytes - iostat.write_bytes,
netio.bytes_recv - netstat.bytes_recv,
netio.bytes_sent - netstat.bytes_sent,
box_mem.used, box_mem.free,
(psutil.used_phymem() - psutil.cached_phymem()), box_mem.total
]
else:
box_data = [0, 0, 0, 0, 0, 0, 0, 0, 0]
data.extend(box_data)
# write data
print >> writer, "\t".join([str(s) for s in data])
# process children
for child in process.get_children():
self._collect_and_write(child, writer, ts=ts)
pass
def cached_physical_memory():
'''
.. deprecated:: Helium
Use :mod:`ps.virtual_memory <salt.modules.ps.virtual_memory>` instead.
Return the amount cached memory.
CLI Example:
.. code-block:: bash
salt '*' ps.cached_physical_memory
'''
salt.utils.warn_until(
'Helium',
'\'ps.cached_physical_memory\' is deprecated. Please use'
'\'ps.virtual_memory\' instead. This functionality will'
'be removed in Salt {version}.'
)
return psutil.cached_phymem()
def cached_physical_memory():
"""
.. deprecated:: Helium
Use :mod:`ps.virtual_memory <salt.modules.ps.virtual_memory>` instead.
Return the amount cached memory.
CLI Example:
.. code-block:: bash
salt '*' ps.cached_physical_memory
"""
salt.utils.warn_until(
"Helium",
"'ps.cached_physical_memory' is deprecated. Please use"
"'ps.virtual_memory' instead. This functionality will"
"be removed in Salt {version}.",
)
return psutil.cached_phymem()
def __call__(self):
status = []
status.append(('num cpus', psutil.NUM_CPUS))
status.append(('cpu used', '%.2f%%' % psutil.cpu_percent()))
total_phy = psutil.TOTAL_PHYMEM
used_phy = psutil.used_phymem()
status.append(('total phy mem', ConvertBytes(total_phy)))
status.append(('used phy mem', ConvertBytes(used_phy)))
status.append(('free phy mem', ConvertBytes(total_phy - used_phy)))
try:
status.append(('buffers', ConvertBytes(psutil.phymem_buffers())))
status.append(('cached', ConvertBytes(psutil.cached_phymem())))
except:
pass
total_virt = psutil.total_virtmem()
used_virt = psutil.used_virtmem()
status.append(('total virt mem', ConvertBytes(total_virt)))
status.append(('used virt mem', ConvertBytes(used_virt)))
status.append(('free virt mem', ConvertBytes(total_virt - used_virt)))
return status
def collect(self):
return dict(
cpu_times=psutil.cpu_times()._asdict(),
mem=dict(
total_phymem=psutil.TOTAL_PHYMEM,
avail_phymem=psutil.avail_phymem(),
avail_virtmem=psutil.avail_virtmem(),
cached_phymem=psutil.cached_phymem(),
phymem_buffers=psutil.phymem_buffers(),
total_virtmem=psutil.total_virtmem(),
used_phymem=psutil.used_phymem(),
used_virtmem=psutil.used_virtmem(),
),
processes=list(self._processes()),
net=dict(
ifaces=self._net_dev(),
proto=self._net_proto()
),
io=self._io_stats(),
fs=dict(self._fs_usage()),
fh=self._file_handles(),
)
def cached_physical_memory(*args, **kwarg):
"""
def cached_physical_memory(*args, **kwarg) -> Return the amount cached memory.
@return int:
"""
return psutil.cached_phymem()
'mem': int(p.get_memory_percent() * 10)
}
except:
continue
all_process_info.append(process_dict)
result['processes'] = all_process_info
if psutil.version_info >= (5, 0, 1):
mem_info = psutil.virtual_memory()
result['mem_used'] = mem_info.used
result['mem_total'] = mem_info.total
else:
result['mem_used'] = psutil.used_phymem() - psutil.phymem_buffers(
) - psutil.cached_phymem()
result['mem_total'] = psutil.TOTAL_PHYMEM
result['disk_used'] = du.used
result['disk_total'] = du.total
if psutil.version_info >= (5, 0, 1):
interfaces = psutil.net_io_counters(pernic=True)
else:
interfaces = psutil.network_io_counters(pernic=True)
if 'tunl0' in interfaces: # ignore tunl0 interface
del interfaces['tunl0']
result['ifaces'] = interfaces
def cached_phymem():
'''
Return the amount cached memory.
'''
return psutil.cached_phymem()
def test_cached_phymem(self):
x = psutil.cached_phymem()
self.assertTrue(isinstance(x, (int, long)))
self.assertTrue(x >= 0)
def _get_memory_used():
memory_used = psutil.phymem_usage().used - psutil.cached_phymem()
return memory_used / MEGABYTES
def cached_phymem():
'''
Return the amount cached memory.
'''
return psutil.cached_phymem()
def run(self):
speed_avg = psutil.network_io_counters(pernic=True)
seguir = True
while seguir:
#calculate time generate stats
start = time.time()
#calculate stats
# speed network #
network_actual = pick_speed_in_secs(2)
network_avg = pick_speed_avg(speed_avg);
speed_avg = network_avg[2];
# pack all #
try:
temperature = int(int(os.popen("cat /sys/class/thermal/thermal_zone0/temp").read())/1000)
except:
temperature = 40
data = {
"network_down": network_actual[1],#
"network_up": network_actual[0],#
"network_avg_down": network_avg[1],#
"network_avg_up": network_avg[0],#
"cache": psutil.cached_phymem(),#
"buffer": psutil.phymem_buffers(),#
"used": psutil.used_phymem(),#
"swap_total": psutil.total_virtmem(),#
"swap_used": psutil.used_virtmem(),#
"hdd_use_": psutil.disk_usage('/')[3],
"hdd_use_home": psutil.disk_usage('/home')[3],
"cpu_use": psutil.cpu_percent(interval=1),#
"cpu_mhz": int(os.popen("cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq").read()[:-3]),
"temp": temperature
}
data_string = json.dumps(data)
#print 'ENCODED:', data_string
temp = time.localtime(start)
datatime = str(temp[2])+"/"+str(temp[1])+"/"+str(temp[0])+" "+str(temp[3])+"/"+str(temp[4])+"/"+str(temp[5])
if (os.path.exists(url_file_location) != True):
data = {
"ava": psutil.avail_phymem(),#
"swap_ava": psutil.avail_virtmem(),#
}
data_string_2 = json.dumps(data)
tmp = True
else:
tmp = False
f=open(str(url_file_location),"a")
if (tmp == True):
f.write('"data":'+data_string_2)
f.write(', '+'"'+datatime+'":'+data_string)
f.close()
del data
del data_string
del temp
del datatime
del network_actual
del temperature
time.sleep(generate_time - int(time.time() - start))
def request_data(self):
'''
(0,'cpu_percent'),
(1,'phymem_usage_total'),
(2,'phymem_usage_available'),
(3,'phymem_usage_percent'),
(4,'phymem_usage_used'),
(5,'phymem_usage_free'),
(6,'phymem_usage_active'),
(7,'phymem_usage_inactive'),
(8,'phymem_usage_buffers'),
(9,'phymem_usage_cached'),
(10,'swap_memory_total'),
(11,'swap_memory_used'),
(12,'swap_memory_free'),
(13,'swap_memory_percent'),
(14,'swap_memory_sin'),
(15,'swap_memory_sout'),
(16,'cached_phymem'),
(17,'disk_usage_systemdisk'),
'''
data = {}
for key, item in self.variables.iteritems():
if item['inf_id'] == 0:
# cpu_percent
value = psutil.cpu_percent()
elif item['inf_id'] == 1:
# phymem_usage_total
value = psutil.phymem_usage().total
elif item['inf_id'] == 2:
#phymem_usage_available
value = psutil.phymem_usage().available
elif item['inf_id'] == 3:
#phymem_usage_percent
value = psutil.phymem_usage().percent
elif item['inf_id'] == 4:
#phymem_usage_used
value = psutil.phymem_usage().used
elif item['inf_id'] == 5:
#phymem_usage_free
value = psutil.phymem_usage().free
elif item['inf_id'] == 6:
#phymem_usage_active
value = psutil.phymem_usage().active
elif item['inf_id'] == 7:
#phymem_usage_inactive
value = psutil.phymem_usage().inactive
elif item['inf_id'] == 8:
#phymem_usage_buffers
value = psutil.phymem_usage().buffers
elif item['inf_id'] == 9:
#phymem_usage_cached
value = psutil.phymem_usage().cached
elif item['inf_id'] == 10:
#swap_memory_total
value = psutil.swap_memory().total
elif item['inf_id'] == 11:
#swap_memory_used
value = psutil.swap_memory().used
elif item['inf_id'] == 12:
#swap_memory_free
value = psutil.swap_memory().free
elif item['inf_id'] == 13:
#swap_memory_percent
value = psutil.swap_memory().percent
elif item['inf_id'] == 14:
#swap_memory_sin
value = psutil.swap_memory().sin
elif item['inf_id'] == 15:
#swap_memory_sout
value = psutil.swap_memory().sout
elif item['inf_id'] == 16:
#cached_phymem
value = psutil.cached_phymem()
elif item['inf_id'] == 17:
#disk_usage_systemdisk
value = psutil.disk_usage('/')
elif item['inf_id'] == 18:
#disk_usage_systemdisk
value = psutil.disk_usage(item['param'])
else:
value = 0
data[key] = value
return data
def cached_physical_memory(*args, **kwarg):
"""
def cached_physical_memory(*args, **kwarg) -> Return the amount cached memory.
@return int:
"""
return psutil.cached_phymem()
def updateScreen(self):
dt = datetime.datetime.now()
with self.context():
self._text[1] = dt.strftime('%A, %b %d, %Y').center(27)
self._text[2] = dt.strftime('%I:%M:%S %p').center(27)
self._text[3] = ('CPU:%4.1f%% RAM:%6.2f%%' % (psutil.cpu_percent(), (psutil.used_phymem()-psutil.cached_phymem())/psutil.TOTAL_PHYMEM*100)).center(27)
self._text[4] = ('%i processes' % len(psutil.get_pid_list())).center(27)
def _store_resources(self, store=True):
"""Looks at the resources usage and store the data locally.
store (bool): if False, run the method but do not store the
resulting values - useful for initializing the
previous values
"""
logger.debug("ResourceService._store_resources")
# We use the precise time to compute the delta
now = time.time()
delta = now - self._last_saved_time
self._last_saved_time = now
now = int(now)
data = {}
# CPU
cpu_times = self._get_cpu_times()
data["cpu"] = dict((x, int(round((cpu_times[x] -
self._prev_cpu_times[x])
/ delta * 100.0)))
for x in cpu_times)
data["cpu"]["num_cpu"] = psutil.NUM_CPUS
self._prev_cpu_times = cpu_times
# Memory. We differentiate from old and deprecated (< 0.3.0)
# methods to the new ones. Remove the differentiation when we
# drop the support for Ubuntu 11.10 (which ships 0.2.1).
ram_cached = psutil.cached_phymem()
ram_buffers = psutil.phymem_buffers()
if psutil_version < (0, 3, 0):
data["memory"] = {
"ram_total": psutil.TOTAL_PHYMEM / B_TO_MB,
"ram_available": psutil.avail_phymem() / B_TO_MB,
"ram_cached": ram_cached / B_TO_MB,
"ram_buffers": ram_buffers / B_TO_MB,
"ram_used": (psutil.used_phymem() - ram_cached - ram_buffers)
/ B_TO_MB,
"swap_total": psutil.total_virtmem() / B_TO_MB,
"swap_available": psutil.avail_virtmem() / B_TO_MB,
"swap_used": psutil.used_virtmem() / B_TO_MB,
}
else:
phymem = psutil.phymem_usage()
virtmem = psutil.virtmem_usage()
data["memory"] = {
"ram_total": phymem.total / B_TO_MB,
"ram_available": phymem.free / B_TO_MB,
"ram_cached": ram_cached / B_TO_MB,
"ram_buffers": ram_buffers / B_TO_MB,
"ram_used": (phymem.used - ram_cached - ram_buffers) / B_TO_MB,
"swap_total": virtmem.total / B_TO_MB,
"swap_available": virtmem.free / B_TO_MB,
"swap_used": virtmem.used / B_TO_MB,
}
data["services"] = {}
# Details of our services
for service in self._local_services:
dic = {"autorestart": self._will_restart[service],
"running": True}
proc = self._procs[service]
# If we don't have a previously found process for the
# service, we find it
if proc is None:
proc = self._find_proc(service)
# If we still do not find it, there is no process
if proc is None:
dic["running"] = False
# We have a process, but maybe it has been shut down
elif not proc.is_running():
# If so, let us find the new one
proc = self._find_proc(service)
# If there is no new one, continue
if proc is None:
dic["running"] = False
# If the process is not running, we have nothing to do.
if not dic["running"]:
data["services"][str(service)] = dic
continue
try:
dic["since"] = self._last_saved_time - proc.create_time
dic["resident"], dic["virtual"] = \
(x / 1048576 for x in proc.get_memory_info())
cpu_times = proc.get_cpu_times()
dic["user"] = int(
round((cpu_times[0] -
self._services_prev_cpu_times[service][0])
/ delta * 100))
dic["sys"] = int(
round((cpu_times[1] -
self._services_prev_cpu_times[service][1])
/ delta * 100))
self._services_prev_cpu_times[service] = cpu_times
try:
dic["threads"] = proc.get_num_threads()
except AttributeError:
dic["threads"] = 0 # 0 = Not implemented
self._procs[service] = proc
except psutil.error.NoSuchProcess:
# Shut down while we operated?
dic = {"autorestart": self._will_restart[service],
"running": False}
data["services"][str(service)] = dic
if store:
if len(self._local_store) >= 5000: # almost 7 hours
self._local_store = self._local_store[1:]
self._local_store.append((now, data))
return True
def test_cached_phymem(self):
x = psutil.cached_phymem()
self.assertTrue(isinstance(x, (int, long)))
self.assertTrue(x >= 0)
def _store_resources(self, store=True):
"""Looks at the resources usage and store the data locally.
store (bool): if False, run the method but do not store the
resulting values - useful for initializing the
previous values
"""
logger.debug("ResourceService._store_resources")
# We use the precise time to compute the delta
now = time.time()
delta = now - self._last_saved_time
self._last_saved_time = now
now = int(now)
data = {}
# CPU
cpu_times = self._get_cpu_times()
data["cpu"] = dict((x, int(round((cpu_times[x] -
self._prev_cpu_times[x])
/ delta * 100.0)))
for x in cpu_times)
data["cpu"]["num_cpu"] = psutil.NUM_CPUS
self._prev_cpu_times = cpu_times
# Memory. We differentiate from old and deprecated (< 0.3.0)
# methods to the new ones. Remove the differentiation when we
# drop the support for Ubuntu 11.10 (which ships 0.2.1).
ram_cached = psutil.cached_phymem()
ram_buffers = psutil.phymem_buffers()
if psutil_version < (0, 3, 0):
data["memory"] = {
"ram_total": psutil.TOTAL_PHYMEM / B_TO_MB,
"ram_available": psutil.avail_phymem() / B_TO_MB,
"ram_cached": ram_cached / B_TO_MB,
"ram_buffers": ram_buffers / B_TO_MB,
"ram_used": (psutil.used_phymem() - ram_cached - ram_buffers)
/ B_TO_MB,
"swap_total": psutil.total_virtmem() / B_TO_MB,
"swap_available": psutil.avail_virtmem() / B_TO_MB,
"swap_used": psutil.used_virtmem() / B_TO_MB,
}
else:
phymem = psutil.phymem_usage()
virtmem = psutil.virtmem_usage()
data["memory"] = {
"ram_total": phymem.total / B_TO_MB,
"ram_available": phymem.free / B_TO_MB,
"ram_cached": ram_cached / B_TO_MB,
"ram_buffers": ram_buffers / B_TO_MB,
"ram_used": (phymem.used - ram_cached - ram_buffers) / B_TO_MB,
"swap_total": virtmem.total / B_TO_MB,
"swap_available": virtmem.free / B_TO_MB,
"swap_used": virtmem.used / B_TO_MB,
}
data["services"] = {}
# Details of our services
for service in self._local_services:
dic = {"autorestart": self._will_restart[service],
"running": True}
proc = self._procs[service]
# If we don't have a previously found process for the
# service, we find it
if proc is None:
proc = self._find_proc(service)
# If we still do not find it, there is no process
if proc is None:
dic["running"] = False
# We have a process, but maybe it has been shut down
elif not proc.is_running():
# If so, let us find the new one
proc = self._find_proc(service)
# If there is no new one, continue
if proc is None:
dic["running"] = False
# If the process is not running, we have nothing to do.
if not dic["running"]:
data["services"][str(service)] = dic
continue
try:
dic["since"] = self._last_saved_time - proc.create_time
dic["resident"], dic["virtual"] = \
(x / 1048576 for x in proc.get_memory_info())
cpu_times = proc.get_cpu_times()
dic["user"] = int(
round((cpu_times[0] -
self._services_prev_cpu_times[service][0])
/ delta * 100))
dic["sys"] = int(
round((cpu_times[1] -
self._services_prev_cpu_times[service][1])
/ delta * 100))
self._services_prev_cpu_times[service] = cpu_times
try:
dic["threads"] = proc.get_num_threads()
except AttributeError:
dic["threads"] = 0 # 0 = Not implemented
self._procs[service] = proc
except psutil.error.NoSuchProcess:
# Shut down while we operated?
dic = {"autorestart": self._will_restart[service],
"running": False}
data["services"][str(service)] = dic
if store:
if len(self._local_store) >= 5000: # almost 7 hours
self._local_store = self._local_store[1:]
self._local_store.append((now, data))
return True
if __name__ == "__main__":
print "Starting metAMOS pipeline"
if settings.threads < 1:
settings.threads = 1
settings = utils.initConfig(settings.kmer, settings.threads, settings.rundir, settings.taxa_level, settings.VERBOSE, settings.OUTPUT_ONLY)
# add krona to system path
currPath = os.environ["PATH"]
if utils.Settings.KRONA not in currPath:
os.environ["PATH"]="%s:%s"%(utils.Settings.KRONA, currPath)
# check for memory/cpu
if not nopsutil and lowmem == False:
cacheusage=0
if 'linux' in utils.Settings.OSTYPE.lower():
cacheusage = psutil.cached_phymem()
memusage = `psutil.phymem_usage()`.split(",")
freemem = long(memusage[2].split("free=")[-1])+long(cacheusage)
percentfree = float(memusage[3].split("percent=")[-1].split(")")[0])
avram = (freemem/1000000000)
print "[Available RAM: %d GB]"%(avram)
lowmem= False
nofcpblast = False
if avram <= 32:
print utils.WARNING_YELLOW+"\tThere is *%d GB of RAM currently available on this machine, suggested minimum of 32 GB"%(avram)+utils.ENDC
print utils.WARNING_YELLOW+"\t*Enabling low MEM mode, might slow down some steps in pipeline"+utils.ENDC
print utils.WARNING_YELLOW+"\t*If more RAM is available than what is listed above, please close down other programs and restart runPipeline"+utils.ENDC
lowmem= True
else:
print utils.OK_GREEN+"\t*ok"+utils.ENDC
numcpus = psutil.NUM_CPUS
)
parser.add_argument(
"-c",
type=int,
default=256,
help='critical level',
)
args = parser.parse_args()
#have to use avail_phymem to support psutils v 0.5 in debian wheezy.
try:
physmem = psutil.virtual_memory().available / MB #psutils >= 0.6
except AttributeError: # have to use old api
physmem = (( psutil.avail_phymem() +
psutil.phymem_buffers() +
psutil.cached_phymem()
) / MB
)
if args.c >= args.w:
print ('UNKNOWN: critical level must be less than warning level')
sys.exit(3)
if physmem > args.w:
print ("OK - %#s megabytes of physical memory available." % physmem)
sys.exit(0)
elif physmem > args.c:
print ("WARNING - %#s megabytes of physical memory available." %
physmem)
sys.exit(1)
elif physmem <= args.c:
print ("CRITICAL - %#s megabytes of physical memory available." %
