def main(expr): vars = {'free': psutil.avail_virtmem(), 'available': psutil.avail_virtmem(), 'pfree': psutil.avail_phymem(), 'buffers': psutil.phymem_buffers(), 'total': psutil.total_virtmem(), 'cached': psutil.cached_phymem(), } return eval(expr, {}, vars)
def _computeMaximumBufferSize(nodes): """number of nodes. This actually depends on the system mcvine is running on. If we are running on one machine that has shared memory, we need to set here nodes to the number of mpi instances. If we are running on a cluster of nodes without shared memory, we should not need to divide the maximum buffer size by nodes. """ import psutil memsize = min(psutil.TOTAL_PHYMEM / 2, (psutil.avail_phymem() + psutil.avail_virtmem()) * 0.7) memsize = int(memsize) from mcni.neutron_storage.idfneutron import ndblsperneutron bytesperdble = 8 minsize = MINIMUM_BUFFER_SIZE n = int(memsize/nodes/ndblsperneutron/bytesperdble/minsize) \ *MINIMUM_BUFFER_SIZE if n < minsize: raise RuntimeError, "Not enough memory" return n
def mem(self, widget, data=None) : print ' \n DETAILS OF PHYSICAL AND VIRTUAL MEMORY\n ' print 'Total physical memory (in bytes) - ' print psutil.TOTAL_PHYMEM print 'Available physical memory (in bytes) - ' print psutil.avail_phymem() print 'Used physical memory (in bytes) - ' print psutil.used_phymem() print 'Total virtual memory (in bytes) - ' print psutil.total_virtmem() print 'Available virtual memory (in bytes) - ' print psutil.avail_virtmem() print 'Used virtual memory (in bytes) - ' print psutil.used_virtmem() print 'Total cached memory (in bytes) - ' print psutil.cached_phymem()
def _computeMaximumBufferSize(nodes): """number of nodes. This actually depends on the system mcvine is running on. If we are running on one machine that has shared memory, we need to set here nodes to the number of mpi instances. If we are running on a cluster of nodes without shared memory, we should not need to divide the maximum buffer size by nodes. """ import psutil memsize = min(psutil.TOTAL_PHYMEM/2, (psutil.avail_phymem() + psutil.avail_virtmem())*0.7) memsize = int(memsize) from mcni.neutron_storage.idfneutron import ndblsperneutron bytesperdble = 8 minsize = MINIMUM_BUFFER_SIZE n = int(memsize/nodes/ndblsperneutron/bytesperdble/minsize) \ *MINIMUM_BUFFER_SIZE if n<minsize: raise RuntimeError, "Not enough memory" return n
def getStat(self): return {'memTotal': psutil.TOTAL_PHYMEM, 'memFree': psutil.avail_phymem(), 'swapTotal': psutil.total_virtmem(), 'swapFree': psutil.avail_virtmem(), 'buffers': psutil.phymem_buffers(), 'cached': psutil.cached_phymem()}
def calculate_resizing_tofitmemory(x_size,y_size,n_slices,byte): """ Predicts the percentage (between 0 and 1) to resize the image to fit the memory, giving the following information: x_size, y_size: image size n_slices: number of slices byte: bytes allocated for each pixel sample """ imagesize = x_size * y_size * n_slices * byte * 28 # USING LIBSIGAR #import sigar #sg = sigar.open() #ram_free = sg.mem().actual_free() #ram_total = sg.mem().total() #swap_free = sg.swap().free() #sg.close() # USING PSUTIL import psutil try: if (psutil.version_info>=(0,6,0)): ram_free = psutil.virtual_memory().available ram_total = psutil.virtual_memory().total swap_free = psutil.swap_memory().free else: ram_free = psutil.phymem_usage().free + psutil.cached_phymem() + psutil.phymem_buffers() ram_total = psutil.phymem_usage().total swap_free = psutil.virtmem_usage().free except: print "Exception! psutil version < 0.3 (not recommended)" ram_total = psutil.TOTAL_PHYMEM # this is for psutil < 0.3 ram_free = 0.8 * psutil.TOTAL_PHYMEM swap_free = psutil.avail_virtmem() print "RAM_FREE=", ram_free print "RAM_TOTAL=", ram_total if (sys.platform == 'win32'): if (platform.architecture()[0] == '32bit'): if ram_free>1400000000: ram_free=1400000000 if ram_total>1400000000: ram_total=1400000000 if (sys.platform == 'linux2'): if (platform.architecture()[0] == '32bit'): if ram_free>3500000000: ram_free=3500000000 if ram_total>3500000000: ram_total=3500000000 if (swap_free>ram_total): swap_free=ram_total resize = (float((ram_free+0.5*swap_free)/imagesize)) resize=math.sqrt(resize) # this gives the "resize" for each axis x and y if (resize>1): resize=1 return round(resize,2)
def loadMon(duration,interval,emulationID,emulationName,emuStartTime): HOMEPATH= Library.getHomepath() emulationName=str(emulationName) interval=int(interval) ''' starting cpu monitoring in the loop ''' iterationsNo=int(duration)/int(interval) try: f = open(HOMEPATH+"/logs/"+str(emulationID)+"-"+str(emulationName)+"-res"+"_"+str(emuStartTime)+".csv", 'a') f.write(emulationName+";\nCountdown;Time;CPU(%);MEM(%);IOread(bytes);IOwrite(bytes);NET(bytes_sent)\n") #start time initTime=time.time() while iterationsNo !=0: CPU=str(psutil.cpu_percent(interval, False)) #MEM=str(psutil.virtual_memory().percent) MEM=str(psutil.avail_virtmem()) IOr=str(psutil.disk_io_counters().read_time) IOw=str(psutil.disk_io_counters().write_time) NET=str(psutil.network_io_counters(False).bytes_sent) #print (emulationName+";\nTime;CPU(%);MEM(%);IOread(bytes);IOwrite(bytes);NET(bytes_sent)\n"+str(time.time())+";"+CPU+";"+MEM+";"+IOr+";"+IOw+";"+NET) probeTime=time.time()-initTime timeStamp=dt.now() f.write(str(int(probeTime))+";"+str(timeStamp.strftime("%Y-%m-%d %H:%M:%S.%f"))+";"+CPU+";"+MEM+";"+IOr+";"+IOw+";"+NET+"\n") iterationsNo=iterationsNo-1 except Exception,e: print "Unable to create log file\nError: ",e
def memInfo(): logging.debug("Mem info") logging.debug(psutil.virtual_memory()) logging.debug(psutil.swap_memory()) logging.debug(psutil.phymem_usage()) logging.debug(psutil.avail_virtmem()) logging.debug(psutil.avail_phymem())
def generate(self, **kwargs): result = { 'mem_phys_total': psutil.TOTAL_PHYMEM, 'mem_phys_avail': psutil.avail_phymem(), 'mem_phys_used' : psutil.used_phymem(), 'mem_virt_total': psutil.total_virtmem(), 'mem_virt_avail': psutil.avail_virtmem(), 'mem_virt_used' : psutil.used_virtmem(), 'cpu_cores' : self._get_total_cores(), } return result
def generate(self, **kwargs): result = { 'mem_phys_total': psutil.TOTAL_PHYMEM, 'mem_phys_avail': psutil.avail_phymem(), 'mem_phys_used': psutil.used_phymem(), 'mem_virt_total': psutil.total_virtmem(), 'mem_virt_avail': psutil.avail_virtmem(), 'mem_virt_used': psutil.used_virtmem(), 'cpu_cores': self._get_total_cores(), } return result
def get_host_info(): """Return a ``dict`` with system's information `Example of its output <https://gist.github.com/gists/2891134>`_ """ memory_usage = psutil.phymem_usage() cached_memory = psutil.cached_phymem() buffered_memory = psutil.phymem_buffers() real_used = memory_usage.used - buffered_memory - cached_memory real_free = memory_usage.total - real_used percent = 100 * (float(memory_usage.used) / memory_usage.total) real_percent = 100 * (float(real_used) / memory_usage.total) virtual_used = psutil.used_virtmem() virtual_free = psutil.avail_virtmem() virtual_total = virtual_used + virtual_free info_per_nic = psutil.network_io_counters(pernic=True) network_info = {} for key, value in info_per_nic.iteritems(): network_info[key] = {'bytes sent': value.bytes_sent, 'bytes received': value.bytes_recv, 'packets sent': value.packets_sent, 'packets received': value.packets_recv,} partitions = psutil.disk_partitions() storage_info = {} for partition in partitions: disk_usage = psutil.disk_usage(partition.mountpoint) storage_info[partition.device] = {'mount point': partition.mountpoint, 'file system': partition.fstype, 'total bytes': disk_usage.total, 'total used bytes': disk_usage.used, 'total free bytes': disk_usage.free, 'percent used': disk_usage.percent,} return {'memory': {'free': memory_usage.free, 'total': memory_usage.total, 'used': memory_usage.used, 'cached': cached_memory, 'buffers': buffered_memory, 'real used': real_used, 'real free': real_free, 'percent': percent, 'real percent': real_percent, 'total virtual': virtual_total, 'used virtual': virtual_used, 'free virtual': virtual_free,}, 'cpu': {'number of cpus': psutil.NUM_CPUS, 'cpu percent': psutil.cpu_percent(),}, 'network': {'interfaces': network_info,}, 'storage': storage_info, 'uptime': time() - psutil.BOOT_TIME,}
def on_global(self, msg=None): global_cpu = psutil.get_system_cpu_times() global_cpu['percent'] = psutil.cpu_percent() global_mem = { 'used_physical': psutil.used_phymem(), 'used_virtual': psutil.used_virtmem(), 'available_physical': psutil.avail_phymem(), 'available_virtual': psutil.avail_virtmem(), } result = { 'hostnome': socket.gethostname(), 'cpu': global_cpu, 'mem': global_mem, } return {'global': result}
def test_avail_virtmem(self): # This test is not particularly accurate and may fail if the OS is # consuming memory for other applications. # We just want to test that the difference between psutil result # and sysctl's is not too high. p = subprocess.Popen("sysctl vm.vmtotal", shell=1, stdout=subprocess.PIPE) result = p.communicate()[0].strip() if sys.version_info >= (3,): result = str(result, sys.stdout.encoding) _, sysctl_avail_virtmem = parse_sysctl_vmtotal(result) psutil_avail_virtmem = psutil.avail_virtmem() difference = abs(sysctl_avail_virtmem - psutil_avail_virtmem) # let's assume the test is failed if difference is > 0.5 MB if difference > (0.5 * 2**20): self.fail("sysctl=%s; psutil=%s; difference=%s;" %( sysctl_avail_virtmem, psutil_avail_virtmem, difference))
def collect(self): return dict( cpu_times=psutil.cpu_times()._asdict(), mem=dict( total_phymem=psutil.TOTAL_PHYMEM, avail_phymem=psutil.avail_phymem(), avail_virtmem=psutil.avail_virtmem(), cached_phymem=psutil.cached_phymem(), phymem_buffers=psutil.phymem_buffers(), total_virtmem=psutil.total_virtmem(), used_phymem=psutil.used_phymem(), used_virtmem=psutil.used_virtmem(), ), processes=list(self._processes()), net=dict(ifaces=self._net_dev(), proto=self._net_proto()), io=self._io_stats(), fs=dict(self._fs_usage()), fh=self._file_handles(), )
def test2(self): "Instrument: _getBufferSize" instrument = Instrument('t') # for lower values of ncount, buffer_size=ncount/mpisize/DEFAULT_NUMBER_SIM_LOOPS instrument.inventory.ncount = ncount = 1e3 self.assertEqual(instrument._getBufferSize(), ncount / DEFAULT_NUMBER_SIM_LOOPS) instrument.inventory.ncount = ncount = 2e3 self.assertEqual(instrument._getBufferSize(), ncount / DEFAULT_NUMBER_SIM_LOOPS) instrument.mpiSize = mpiSize = 10 self.assertEqual(instrument._getBufferSize(), int(ncount / mpiSize / DEFAULT_NUMBER_SIM_LOOPS)) # for higher values, buffer_size is set by memory limit import psutil temp = min(psutil.TOTAL_PHYMEM / 2, (psutil.avail_phymem() + psutil.avail_virtmem()) * 0.7) temp = int(temp) from mcni.neutron_storage.idfneutron import ndblsperneutron max = int(temp / ndblsperneutron / 8 / 100 / mpiSize) * 100 instrument.inventory.ncount = ncount = 1e9 self.assertEqual(instrument._getBufferSize(), max) instrument.inventory.ncount = ncount = 2e9 self.assertEqual(instrument._getBufferSize(), max) # if user set a too high number for buffer size, it is ignored instrument.inventory.ncount = ncount = 2e9 instrument.inventory.buffer_size = 2e9 self.assertEqual(instrument._getBufferSize(), max) # if user set a too low value for buffer_size, a warning would be issued instrument.inventory.ncount = 1e8 instrument.inventory.buffer_size = 100 instrument._getBufferSize() return
def refresh(self): logging.debug("Host.refresh(): Refreshing all host entries") self.refreshVirtServiceSpecs() self.__machine_type=platform.machine() self.__network_name=platform.node() self.__processor=platform.processor() self.__python_version=platform.python_version() self.__linuxKernel_version=platform.release() #self.__name=platform.system() dist_info=platform.linux_distribution(distname='', version='', id='', supported_dists=('SuSE', 'debian', 'redhat'), full_distribution_name=1) self.__distroName=dist_info[0] self.__distroVersion=dist_info[1] self.__distroCode=dist_info[2] self.__vm_manager_pid=os.getpid() self.__freePhyMem=psutil.avail_phymem()/1048576 self.__freeVirtMem=psutil.avail_virtmem()/1048576 self.__totalVirtMem=psutil.total_virtmem()/1048576 self.__usedPhyMem=psutil.used_phymem()/1048576 self.__manager_process=psutil.Process(self.__vm_manager_pid) self.__virt_process=psutil.Process(int(self.__virtProcessPid)) self.__processUptime=datetime.datetime.fromtimestamp(self.__manager_process.create_time).strftime("%Y-%m-%d %H:%M") #self.__processOwner=self.__manager_process.username #self.__numOfProcessThreads=self.__manager_process.get_num_threads() #self.__numOfVirtProcessThreads=self.__virt_process.get_num_threads() #for thread in self.__virt_process.get_threads(): # self.__virtProcessThreadIDs.append(thread[0]) #for thread in self.__manager_process.get_threads(): # self.__processThreadIDs.append(thread[0]) self.__virtProcessCpuUsage=self.__virt_process.get_cpu_percent() self.__proccessCpuUsage=self.__manager_process.get_cpu_percent() mem=self.__virt_process.get_memory_info() self.__virtProcessMemUsage=mem[1]/1048576 mem=self.__manager_process.get_memory_info() self.__processMemUsage=mem[1]/1048576 #self.__processOpenedFilesSpecs=self.__manager_process.get_open_files() #self.__virtProcessOpenedFilesSpecs=self.__virt_process.get_open_files() #self.__processConnectionsSpecs=self.__manager_process.get_connections() #self.__virtProcessConnectionsSpecs=self.__virt_process.get_connections() self.__cpuUsage=psutil.cpu_percent()
def report_mem(prev_values=None,msg=None): """ Show status of phy/virt mem, and if given previous values, report differences requires the psutil module - avail in apt-get, but I used pip """ if msg is None: msg='' else: msg += ': ' pm = psutil.avail_phymem() vm = psutil.avail_virtmem() tim = seconds() print('{msg}{pm:.2g} GB phys mem, {vm:.2g} GB virt mem avail' .format(msg=msg, pm=pm/1e9, vm=vm/1e9)), if prev_values is None: print else: print('- dt={dt:.2g}s, used {pm:.2g} GB phys, {vm:.2g} GB virt' .format(pm=(prev_values[0] - pm)/1e9, vm=(prev_values[1] - vm)/1e9, dt = tim-prev_values[2])) return((pm,vm,tim))
def report_mem(prev_values=None, msg=None): """ Show status of phy/virt mem, and if given previous values, report differences requires the psutil module - avail in apt-get, but I used pip """ if msg is None: msg = '' else: msg += ': ' pm = psutil.avail_phymem() vm = psutil.avail_virtmem() tim = seconds() print('{msg}{pm:.2g} GB phys mem, {vm:.2g} GB virt mem avail'.format( msg=msg, pm=pm / 1e9, vm=vm / 1e9)), if prev_values is None: print else: print('- dt={dt:.2g}s, used {pm:.2g} GB phys, {vm:.2g} GB virt'. format(pm=(prev_values[0] - pm) / 1e9, vm=(prev_values[1] - vm) / 1e9, dt=tim - prev_values[2])) return ((pm, vm, tim))
def loadMon(duration, interval, emulationID, emulationName, emuStartTime): HOMEPATH = Library.getHomepath() emulationName = str(emulationName) interval = int(interval) ''' starting cpu monitoring in the loop ''' iterationsNo = int(duration) / int(interval) try: f = open( HOMEPATH + "/logs/" + str(emulationID) + "-" + str(emulationName) + "-res" + "_" + str(emuStartTime) + ".csv", 'a') f.write( emulationName + ";\nCountdown;Time;CPU(%);MEM(%);IOread(bytes);IOwrite(bytes);NET(bytes_sent)\n" ) #start time initTime = time.time() while iterationsNo != 0: CPU = str(psutil.cpu_percent(interval, False)) #MEM=str(psutil.virtual_memory().percent) MEM = str(psutil.avail_virtmem()) IOr = str(psutil.disk_io_counters().read_time) IOw = str(psutil.disk_io_counters().write_time) NET = str(psutil.network_io_counters(False).bytes_sent) #print (emulationName+";\nTime;CPU(%);MEM(%);IOread(bytes);IOwrite(bytes);NET(bytes_sent)\n"+str(time.time())+";"+CPU+";"+MEM+";"+IOr+";"+IOw+";"+NET) probeTime = time.time() - initTime timeStamp = dt.now() f.write( str(int(probeTime)) + ";" + str(timeStamp.strftime("%Y-%m-%d %H:%M:%S.%f")) + ";" + CPU + ";" + MEM + ";" + IOr + ";" + IOw + ";" + NET + "\n") iterationsNo = iterationsNo - 1 except Exception, e: print "Unable to create log file\nError: ", e
def collect(self): return dict( cpu_times=psutil.cpu_times()._asdict(), mem=dict( total_phymem=psutil.TOTAL_PHYMEM, avail_phymem=psutil.avail_phymem(), avail_virtmem=psutil.avail_virtmem(), cached_phymem=psutil.cached_phymem(), phymem_buffers=psutil.phymem_buffers(), total_virtmem=psutil.total_virtmem(), used_phymem=psutil.used_phymem(), used_virtmem=psutil.used_virtmem(), ), processes=list(self._processes()), net=dict( ifaces=self._net_dev(), proto=self._net_proto() ), io=self._io_stats(), fs=dict(self._fs_usage()), fh=self._file_handles(), )
def test2(self): "Instrument: _getBufferSize" instrument = Instrument('t') # for lower values of ncount, buffer_size=ncount/mpisize/DEFAULT_NUMBER_SIM_LOOPS instrument.inventory.ncount = ncount = 1e3 self.assertEqual(instrument._getBufferSize(), ncount/DEFAULT_NUMBER_SIM_LOOPS) instrument.inventory.ncount = ncount = 2e3 self.assertEqual(instrument._getBufferSize(), ncount/DEFAULT_NUMBER_SIM_LOOPS) instrument.mpiSize = mpiSize = 10 self.assertEqual(instrument._getBufferSize(), int(ncount/mpiSize/DEFAULT_NUMBER_SIM_LOOPS)) # for higher values, buffer_size is set by memory limit import psutil temp = min(psutil.TOTAL_PHYMEM/2, (psutil.avail_phymem() + psutil.avail_virtmem())*0.7) temp = int(temp) from mcni.neutron_storage.idfneutron import ndblsperneutron max = int(temp/ndblsperneutron/8/100/mpiSize) * 100 instrument.inventory.ncount = ncount = 1e9 self.assertEqual(instrument._getBufferSize(), max) instrument.inventory.ncount = ncount = 2e9 self.assertEqual(instrument._getBufferSize(), max) # if user set a too high number for buffer size, it is ignored instrument.inventory.ncount = ncount = 2e9 instrument.inventory.buffer_size = 2e9 self.assertEqual(instrument._getBufferSize(), max) # if user set a too low value for buffer_size, a warning would be issued instrument.inventory.ncount = 1e8 instrument.inventory.buffer_size = 100 instrument._getBufferSize() return
def hello_world_app(environ,start_response): message="" status = '200 ok' headers = [('Content-type', 'html; charset=utf-8')] start_response(status,headers) #1 psutil parameter boot_time = datetime.datetime.fromtimestamp(psutil.boot_time()).strftime("%Y-%m-%d %H:%M:%S") print("\nBOOT TIME:", boot_time) #2 psutil parameter cpu_util = psutil.cpu_percent(interval=1, percpu=True) i=1 print("\nCPU UTILIZATION:") for cpu in cpu_util: print("CPU {} : {}% ---".format(i, cpu)) i+=1 #3 psutil parameter mem = psutil.virtual_memory() availablemem=psutil.avail_virtmem() #4 psutil parameter usedmem=psutil.used_phymem() #5 psutil parameter usedper=mem.percent THRESHOLD = 100 * 1024 * 1024 # 100MB message +="<TABLE border=10> <TR> <TD> BOOT TIME</TD><TD>"+boot_time+"</TD></TR>" #message +="<TR><TD>CPU UTILIZATION</TD><TD><TABLE BORDER =2><TR><TD>CPU1</TD><TD>VAL</TD></TR><TR><TD>CPU2</TD><TD>VAL</TD></TR><TR><TD>CPU3</TD><TD>VAL</TD></TR></TABLE></TD></TR>" message +="<TR> <TD> AVAILABLE MEMORY </TD><TD>"+str(availablemem)+"</TD></TR>" message +="<TR> <TD> USED MEMORY </TD><TD>"+str(usedmem)+"</TD></TR>" message +="<TR> <TD> USED PERCENT </TD><TD>"+str(usedper)+"</TD></TR></TABLE>" return[bytes(message,'utf-8')]
def run(self): speed_avg = psutil.network_io_counters(pernic=True) seguir = True while seguir: #calculate time generate stats start = time.time() #calculate stats # speed network # network_actual = pick_speed_in_secs(2) network_avg = pick_speed_avg(speed_avg); speed_avg = network_avg[2]; # pack all # try: temperature = int(int(os.popen("cat /sys/class/thermal/thermal_zone0/temp").read())/1000) except: temperature = 40 data = { "network_down": network_actual[1],# "network_up": network_actual[0],# "network_avg_down": network_avg[1],# "network_avg_up": network_avg[0],# "cache": psutil.cached_phymem(),# "buffer": psutil.phymem_buffers(),# "used": psutil.used_phymem(),# "swap_total": psutil.total_virtmem(),# "swap_used": psutil.used_virtmem(),# "hdd_use_": psutil.disk_usage('/')[3], "hdd_use_home": psutil.disk_usage('/home')[3], "cpu_use": psutil.cpu_percent(interval=1),# "cpu_mhz": int(os.popen("cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq").read()[:-3]), "temp": temperature } data_string = json.dumps(data) #print 'ENCODED:', data_string temp = time.localtime(start) datatime = str(temp[2])+"/"+str(temp[1])+"/"+str(temp[0])+" "+str(temp[3])+"/"+str(temp[4])+"/"+str(temp[5]) if (os.path.exists(url_file_location) != True): data = { "ava": psutil.avail_phymem(),# "swap_ava": psutil.avail_virtmem(),# } data_string_2 = json.dumps(data) tmp = True else: tmp = False f=open(str(url_file_location),"a") if (tmp == True): f.write('"data":'+data_string_2) f.write(', '+'"'+datatime+'":'+data_string) f.close() del data del data_string del temp del datatime del network_actual del temperature time.sleep(generate_time - int(time.time() - start))
def Scheduler(): try: #Connect to database #conn = MySQLdb.connect (host = "localhost", user = "******", passwd = "mdat", db = "mdat") db_path = os.path.join(app_folder, 'databases', 'storage.db') con = sqlite3.connect(db_path) con.row_factory = sqlite3.Row c=con.cursor() #Check lock sys_info_sql = 'SELECT * FROM systems WHERE id = "1"' c.execute(sys_info_sql) except: print "Error (1) with database in Scheduler:", sys.exc_info()[0] con.close() gc.collect() return 0 row = c.fetchone() locked = row['locked'] default_process = row['default_process'] max_process = row['max_process'] if (locked == 'F'): #Lock is free: run analysis try: #Lock locked_sql = 'UPDATE systems SET locked = "T" WHERE id = "1"' with con: c.execute(locked_sql) except: print "Error (2) with database in Scheduler:", sys.exc_info()[0] con.close() gc.collect() return 0 #Read datasets c.execute('SELECT id, status FROM results ORDER BY id ASC') results = c.fetchall() #Count the number of running processes running_process=0 for result in results: if (result['status']=='Running'): running_process = running_process + 1 #Running process < default, continue if (running_process < default_process): remaining_process=0 for result in results: if (result['status']=='Waiting'): #Take dataset id result_id = result['id'] #Start, set status to Running try: start_time = datetime.now() result_update_sql = 'UPDATE results SET status = "Running", start_time = "' + str(start_time) + '" WHERE id = "' + str(result_id) + '"' with con: c.execute(result_update_sql) except: print "Error (3) with database in Scheduler:", sys.exc_info()[0] con.close() gc.collect() break remaining_process = remaining_process + 1 try: executer_path = os.path.join(app_folder, 'modules', 'Executer.py') cmd = 'python ' + executer_path + ' ' + app_folder + ' ' + str(result_id) + ' &' os.system(cmd) except: print "Error (3) with analyzing process in Scheduler:", sys.exc_info()[0] con.close() gc.collect() break #Check number of remaining allowed processes if (remaining_process >= default_process - running_process): avail_phymem = psutil.avail_phymem() avail_virtmem = psutil.avail_virtmem() if (avail_phymem < 500000000) | (avail_virtmem < 1000000000) | (remaining_process >= max_process - running_process): break #Unlock and Halt try: unlocked_sql = 'UPDATE systems SET locked = "F" WHERE id = "1"' with con: c.execute(unlocked_sql) except: print "Error (4) with database in Scheduler:", sys.exc_info()[0] con.close() gc.collect() return 0 con.close() gc.collect() return 0 else: #Lock is not free: halt con.close() gc.collect() return 0
def check_mem(): server = 'localhost' port = 2003 ## hostname needed for the graphite tree hostname = socket.gethostname() ## build the tree up a bit tree = hostname+".memory." ## build a dictionary of the values we want to graph try: mem_dict = {"total_physical_memory": psutil.phymem_usage().total, 'free_physical_memory':psutil.phymem_usage().free, "used_physical_memory": psutil.phymem_usage().used, "total_virtual_memory":psutil.total_virtmem(), "free_virtual_memory": psutil.avail_virtmem(), "used_virtual_memory": psutil.used_virtmem()} ## Iterate the dictionary and write out the value to graphite for each key for key in mem_dict: w2g(server, port, tree+key, mem_dict[key]) except Exception as e: ## clean up errors a bit print e exit(1)
# -*- coding: utf-8 -*- """ Created on Tue Sep 3 09:21:57 2013 @author: ozdemircili """ import psutil psutil.phymem_usage() psutil.virtmem_usage() psutil.cached_phymem() psutil.phymem_buffers() psutil.avail_phymem() psutil.used_phymem() psutil.total_virtmem() psutil.avail_virtmem() psutil.used_virtmem() psutil.cpu_percent() #CPU from __future__ import print_function with open('/proc/cpuinfo') as f: for line in f: #Formatting the output!! if line.strip(): if line.rstrip('\n').startswith('model name'): model_name = line.rstrip('\n').split(':')[1]
def test_avail_virtmem(self): x = psutil.avail_virtmem() self.assertTrue(isinstance(x, (int, long))) self.assertTrue(x >= 0)
def _store_resources(self, store=True): """Looks at the resources usage and store the data locally. store (bool): if False, run the method but do not store the resulting values - useful for initializing the previous values """ logger.debug("ResourceService._store_resources") # We use the precise time to compute the delta now = time.time() delta = now - self._last_saved_time self._last_saved_time = now now = int(now) data = {} # CPU cpu_times = self._get_cpu_times() data["cpu"] = dict((x, int(round((cpu_times[x] - self._prev_cpu_times[x]) / delta * 100.0))) for x in cpu_times) data["cpu"]["num_cpu"] = psutil.NUM_CPUS self._prev_cpu_times = cpu_times # Memory. We differentiate from old and deprecated (< 0.3.0) # methods to the new ones. Remove the differentiation when we # drop the support for Ubuntu 11.10 (which ships 0.2.1). ram_cached = psutil.cached_phymem() ram_buffers = psutil.phymem_buffers() if psutil_version < (0, 3, 0): data["memory"] = { "ram_total": psutil.TOTAL_PHYMEM / B_TO_MB, "ram_available": psutil.avail_phymem() / B_TO_MB, "ram_cached": ram_cached / B_TO_MB, "ram_buffers": ram_buffers / B_TO_MB, "ram_used": (psutil.used_phymem() - ram_cached - ram_buffers) / B_TO_MB, "swap_total": psutil.total_virtmem() / B_TO_MB, "swap_available": psutil.avail_virtmem() / B_TO_MB, "swap_used": psutil.used_virtmem() / B_TO_MB, } else: phymem = psutil.phymem_usage() virtmem = psutil.virtmem_usage() data["memory"] = { "ram_total": phymem.total / B_TO_MB, "ram_available": phymem.free / B_TO_MB, "ram_cached": ram_cached / B_TO_MB, "ram_buffers": ram_buffers / B_TO_MB, "ram_used": (phymem.used - ram_cached - ram_buffers) / B_TO_MB, "swap_total": virtmem.total / B_TO_MB, "swap_available": virtmem.free / B_TO_MB, "swap_used": virtmem.used / B_TO_MB, } data["services"] = {} # Details of our services for service in self._local_services: dic = {"autorestart": self._will_restart[service], "running": True} proc = self._procs[service] # If we don't have a previously found process for the # service, we find it if proc is None: proc = self._find_proc(service) # If we still do not find it, there is no process if proc is None: dic["running"] = False # We have a process, but maybe it has been shut down elif not proc.is_running(): # If so, let us find the new one proc = self._find_proc(service) # If there is no new one, continue if proc is None: dic["running"] = False # If the process is not running, we have nothing to do. if not dic["running"]: data["services"][str(service)] = dic continue try: dic["since"] = self._last_saved_time - proc.create_time dic["resident"], dic["virtual"] = \ (x / 1048576 for x in proc.get_memory_info()) cpu_times = proc.get_cpu_times() dic["user"] = int( round((cpu_times[0] - self._services_prev_cpu_times[service][0]) / delta * 100)) dic["sys"] = int( round((cpu_times[1] - self._services_prev_cpu_times[service][1]) / delta * 100)) self._services_prev_cpu_times[service] = cpu_times try: dic["threads"] = proc.get_num_threads() except AttributeError: dic["threads"] = 0 # 0 = Not implemented self._procs[service] = proc except psutil.error.NoSuchProcess: # Shut down while we operated? dic = {"autorestart": self._will_restart[service], "running": False} data["services"][str(service)] = dic if store: if len(self._local_store) >= 5000: # almost 7 hours self._local_store = self._local_store[1:] self._local_store.append((now, data)) return True
def getMemPerc(cls): return "%03d" % int(100.0 * (1.0 - float(psutil.avail_virtmem()) / float(psutil.total_virtmem()))) + \ "m% "
""" Created on Tue Sep 3 09:21:57 2013 @author: ozdemircili """ import psutil psutil.phymem_usage() psutil.virtmem_usage() psutil.cached_phymem() psutil.phymem_buffers() psutil.avail_phymem() psutil.used_phymem() psutil.total_virtmem() psutil.avail_virtmem() psutil.used_virtmem() psutil.cpu_percent() #CPU from __future__ import print_function with open('/proc/cpuinfo') as f: for line in f: #Formatting the output!! if line.strip(): if line.rstrip('\n').startswith('model name'): model_name = line.rstrip('\n').split(':')[1]