Beispiel #1
1
def meminfo():
	mem = int((psutil.virtual_memory().active/psutil.virtual_memory().total)*100)
	if mem >= 90:
		return ("MEM: {0}%".format(mem), RED)
	elif mem <= 10:
		return ("MEM: {0}%".format(mem), BLUE)
	else:
		return ("MEM: {0}%".format(mem), GREEN)
Beispiel #2
0
def CatchMEM(RunNumb, Interval, TimeStart):

	"""
	Funzione per catturare i dati della Memoria

	Cattura le percentuali di utilizzo della memoria e della swap medie nell'intervallo richiesto
	 e le impacchetta per formare la risposta.

	:param RunNumb: Il numero della Run corrente (al momento in cui è stato richiesto il lavoro al server)
	:param Interval: Intervallo tra un campionamento e l'altro.
	:param TimeStart: Tempo di avvio di questo campionamento, lo restituisco in uscita per poter calcolare la latenza.
	:returns: Una tupla contenente i dati rilevati nel Interval.
	"""
	#selettori per andare a selezionare la percentuale di memoria utilizzata nelle rispettive liste ritornate da psutil
	SelPerc = 3

	Average_MEM, Average_SWP = 0, 0
	for i in range(Interval):
		#Average_MEM += psutil.phymem_usage()[SelPerc]/(Interval+1)
		#Average_SWP += psutil.virtmem_usage()[SelPerc]/(Interval+1)
		Average_MEM += psutil.virtual_memory().percent/(Interval+1)
		Average_SWP += psutil.swap_memory().percent/(Interval+1)
		time.sleep(1)

	Average_MEM += psutil.virtual_memory().percent/(Interval+1)
	Average_SWP += psutil.swap_memory().percent/(Interval+1)
	ProbeManager.PutResult([ProbeID, RunNumb, "MEM", Average_MEM, Average_SWP, TimeStart])
Beispiel #3
0
    def still_checkin(self, hostname, ip_addr, port, load, data_dir, status="OK", max_tasks=2, cur_tasks=0):
        ###
        # still_checkin : Check to see if the still entry already exists in the database, if it does update the timestamp, port, data_dir, and load.
        #                 If does not exist then go ahead and create an entry.
        ###
        s = self.Session()

        if s.query(Still).filter(Still.hostname == hostname).count() > 0:  # Check if the still already exists, if so just update the time
            still = s.query(Still).filter(Still.hostname == hostname).one()
            still.last_checkin = datetime.datetime.now()
            still.status = status
            # print("STILL_CHECKIN, test mode, setting load = 0, change back before release")
            # still.current_load = 0
            still.current_load = psutil.cpu_percent()
            still.number_of_cores = psutil.cpu_count()
            still.free_memory = round(psutil.virtual_memory().free / (1024 ** 3), 2)
            still.total_memory = round(psutil.virtual_memory().total / (1024 ** 3), 2)
            still.data_dir = data_dir
            still.port = port
            still.max_num_of_tasks = max_tasks
            still.cur_num_of_tasks = cur_tasks
            s.add(still)
        else:  # Still doesn't exist, lets add it
            still = Still(hostname=hostname, ip_addr=ip_addr, port=port, current_load=load,
                          data_dir=data_dir, status=status, max_num_of_tasks=max_tasks, cur_num_of_tasks=cur_tasks)
            s.add(still)

        s.commit()
        s.close()
        return 0
 def run(self):
     while self.RUN:
         cpu = psutil.cpu_percent(interval=None)
         ram = (psutil.virtual_memory()[0] - psutil.virtual_memory()[1]) / psutil.virtual_memory()[0] * 100
         
         update_sensors.save_system_reading(cpu, ram)
         time.sleep(settings.how_often_to_check_system)
Beispiel #5
0
def getSystemInfo():
	"""
	Get a dictionary with some system/server info

	return -- ["unix", "connectedUsers", "webServer", "cpuUsage", "totalMemory", "usedMemory", "loadAverage"]
	"""

	data = {}

	# Get if server is running under unix/nt
	data["unix"] = runningUnderUnix()

	# General stats
	data["connectedUsers"] = len(glob.tokens.tokens)
	data["webServer"] = glob.conf.config["server"]["server"]
	data["cpuUsage"] = psutil.cpu_percent()
	data["totalMemory"] = "{0:.2f}".format(psutil.virtual_memory()[0]/1074000000)
	data["usedMemory"] = "{0:.2f}".format(psutil.virtual_memory()[3]/1074000000)

	# Unix only stats
	if (data["unix"] == True):
		data["loadAverage"] = os.getloadavg()
	else:
		data["loadAverage"] = (0,0,0)

	return data
    def cpustat(self):
        tags="node:%s %s"%(self.nodename,self.tags)

        val=int(psutil.cpu_percent())
        measurement="cpu_perc"
        key="%s.%s"%(self.nodename,measurement)
        self.measure(key,measurement,tags,val,aggrkey=measurement)

        val=int(psutil.virtual_memory()[1]/1024/1024)
        measurement="mem_free_mb"
        key="%s.%s"%(self.nodename,measurement)
        self.measure(key,measurement,tags,val,aggrkey=measurement)

        val=int(psutil.swap_memory()[3])
        measurement="mem_virt_perc"
        key="%s.%s"%(self.nodename,measurement)
        self.measure(key,measurement,tags,val,aggrkey=measurement)

        val=int(psutil.virtual_memory()[2])
        measurement="mem_phy_perc"
        key="%s.%s"%(self.nodename,measurement)
        self.measure(key,measurement,tags,val,aggrkey=measurement)

        res=psutil.cpu_times_percent()
        names=["cputimeperc_user","cputimeperc_nice","cputimeperc_system","cputimeperc_idle","cputimeperc_iowait","cputimeperc_irq","cputimeperc_softirq","steal","guest","guest_nice"]
        for i in range(len(names)):
            if names[i].startswith("cputime"):
                val=int(res[i])
                measurement=names[i]
                key="%s.%s"%(self.nodename,measurement)
                self.measure(key,measurement,tags,val,aggrkey=measurement)
Beispiel #7
0
def sysinfo():
    boot_start = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(psutil.boot_time()))
    time.sleep(0.5)
    cpu_usage = psutil.cpu_percent()
    ram = int(psutil.virtual_memory().total / (1027 * 1024))
    ram_percent = psutil.virtual_memory().percent
    swap = int(psutil.swap_memory().total / (1027 * 1024))
    swap_percent = psutil.swap_memory().percent
    net_sent = psutil.net_io_counters().bytes_sent
    net_recv = psutil.net_io_counters().bytes_recv
    net_spkg = psutil.net_io_counters().packets_sent
    net_rpkg = psutil.net_io_counters().packets_recv
    sysinfo_string = ""
    if __name__ == "__main__":
        bfh = r'%'
        print(" \033[1;32m开机时间:%s\033[1;m" % boot_start)
        print(" \033[1;32m当前CPU使用率:%s%s\033[1;m" % (cpu_usage, bfh))
        print(" \033[1;32m物理内存:%dM\t使用率:%s%s\033[1;m" % (ram, ram_percent, bfh))
        print(" \033[1;32mSwap内存:%dM\t使用率:%s%s\033[1;m" % (swap, swap_percent, bfh))
        print(" \033[1;32m发送:%d Byte\t发送包数:%d个\033[1;m" % (net_sent, net_spkg))
        print(" \033[1;32m接收:%d Byte\t接收包数:%d个\033[1;m" % (net_recv, net_rpkg))
        for i in psutil.disk_partitions():
            print(" \033[1;32m盘符: %s 挂载点: %s 使用率: %s%s\033[1;m" % (i[0], i[1], psutil.disk_usage(i[1])[3], bfh))

        sysinfo_string = '开机时间: ' + boot_start + '\n'
        sysinfo_string += '当前CPU使用率: ' + str(cpu_usage) + '%\n'
        sysinfo_string += '物理内存: ' + str(ram) + 'M\t使用率: ' + str(ram_percent) + '%\t'
        sysinfo_string += 'Swap内存: ' + str(swap) + 'M\t使用率: ' + str(swap_percent) + '%\n'
        return sysinfo_string

    else:
        file = open("sysinfo.log", "a")
        file.write("CPU:%s   \tRAM:%s\tnet_recv:%d\tNet_sent:%d\r\n" % (cpu_usage, ram_percent, net_recv, net_sent))
        file.flush()
        file.close()
Beispiel #8
0
def ajax_sysinfo():
    sys_info['mem_usage_percent'] = psutil.virtual_memory().percent
    sys_info['mem_total'] = psutil.virtual_memory().total
    sys_info['mem_used'] = psutil.virtual_memory().used
    sys_info['cpu_util'] = psutil.cpu_percent(percpu=True, interval=None)
    #data = jsonify(results=["1", "2", "3"], name="Punit")
    return jsonify(sysinfo=sys_info)
Beispiel #9
0
def calculate_resizing_tofitmemory(x_size,y_size,n_slices,byte):
    """
    Predicts the percentage (between 0 and 1) to resize the image to fit the memory, 
    giving the following information:
        x_size, y_size: image size
        n_slices: number of slices
        byte: bytes allocated for each pixel sample
    """
    imagesize = x_size * y_size * n_slices * byte * 28 
    
    #  USING LIBSIGAR
    #import sigar
    #sg = sigar.open()
    #ram_free = sg.mem().actual_free()
    #ram_total = sg.mem().total()
    #swap_free = sg.swap().free()
    #sg.close()

    # USING PSUTIL  
    import psutil

    try:
        if (psutil.version_info>=(0,6,0)):
            ram_free = psutil.virtual_memory().available
            ram_total = psutil.virtual_memory().total
            swap_free = psutil.swap_memory().free
        else:
            ram_free = psutil.phymem_usage().free + psutil.cached_phymem() + psutil.phymem_buffers()
            ram_total = psutil.phymem_usage().total
            swap_free = psutil.virtmem_usage().free
    except:
        print "Exception! psutil version < 0.3 (not recommended)"
        ram_total = psutil.TOTAL_PHYMEM  # this is for psutil < 0.3
        ram_free = 0.8 * psutil.TOTAL_PHYMEM 
        swap_free = psutil.avail_virtmem()
                    
    print "RAM_FREE=", ram_free
    print "RAM_TOTAL=", ram_total

    if (sys.platform == 'win32'):
        if (platform.architecture()[0] == '32bit'):
            if ram_free>1400000000:
                ram_free=1400000000
            if ram_total>1400000000:
                ram_total=1400000000

    if (sys.platform == 'linux2'):
        if (platform.architecture()[0] == '32bit'):
            if ram_free>3500000000:
                ram_free=3500000000
            if ram_total>3500000000:
                ram_total=3500000000

    if (swap_free>ram_total):
        swap_free=ram_total
    resize = (float((ram_free+0.5*swap_free)/imagesize))      
    resize=math.sqrt(resize)  # this gives the "resize" for each axis x and y
    if (resize>1): 
        resize=1
    return round(resize,2)
Beispiel #10
0
def performance(attribute):
    """ A little smater routing system.
    """

    data = None

    if attribute == 'system':
        data = { 'system': platform.system() }
    elif attribute == 'processor':
        data = { 'processor': platform.processor() }
    elif attribute in ['cpu_count', 'cpucount'] :
        data = { 'cpu_count': psutil.cpu_count() }
    elif attribute == 'machine':
        data = { 'machine': platform.machine() }
    elif attribute in ['virtual_mem', 'virtualmem']:
        data = { 'virtual_mem': psutil.virtual_memory().total }
    elif attribute in ['virtual_mem_gb', 'virtualmemgb']:
        data = { 'virtual_mem_gb': psutil.virtual_memory().total / (1024.0 ** 3) }
    elif attribute == 'all':
        data = {
            'system': platform.system(),
            'processor': platform.processor(),
            'cpu_count': psutil.cpu_count(),
            'machine': platform.machine(),
            'virtual_mem': psutil.virtual_memory().total,
            'virtual_mem_gb': psutil.virtual_memory().total / (1024.0 ** 3),
        }

    packet = json.dumps(data)
    resp = Response(packet, status=200, mimetype='application/json')

    return(resp)
Beispiel #11
0
    def get_mem_swap(self,):
        """
        Get memory and swap usage
        """
        try:
            # psutil >= 0.6.0
            phymem = psutil.virtual_memory()
            buffers = psutil.virtual_memory().buffers
            cached = psutil.virtual_memory().cached
            vmem = psutil.swap_memory()
        except AttributeError:
            # psutil > 0.4.0 and < 0.6.0
            phymem = psutil.phymem_usage()
            buffers = getattr(psutil, 'phymem_buffers', lambda: 0)()
            cached = getattr(psutil, 'cached_phymem', lambda: 0)()
            vmem = psutil.virtmem_usage()

        mem_used = phymem.total - (phymem.free + buffers + cached)
        return (
            phymem.percent,
            mem_used,
            phymem.total,
            vmem.percent,
            vmem.used,
            vmem.total)
Beispiel #12
0
    def __init__(self, interval, history_len):
        """

        :param interval: Collect statistics according to this interval.
        :param history_len: Use this many to compute avg/max statistics.
        """

        self.interval = interval
        self.history_len = history_len
        try:
            import psutil  # @UnresolvedImport @Reimport
        except:
            self._available = False
        else:
            self._available = True

            self.cpu = Collect('cpu', lambda: psutil.cpu_percent(interval=0),
                               interval, history_len)

            try:
                # new in 0.8
                psutil.virtual_memory().percent
                get_mem = lambda: psutil.virtual_memory().percent
            except:
                get_mem = lambda: psutil.phymem_usage().percent

            self.mem = Collect('mem', get_mem, interval, history_len)
            try:
                # new in 0.8
                psutil.swap_memory().percent
                get_mem = lambda: psutil.swap_memory().percent
            except:
                get_mem = lambda: psutil.virtmem_usage().percent

            self.swap_mem = Collect('swap', get_mem, interval, history_len)
Beispiel #13
0
def Sysinfo():  
    Hostname = platform.node()
    Sys_version = platform.platform()
    Boot_Start = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(psutil.boot_time()))    
    time.sleep(0.5)  
    Cpu_usage = psutil.cpu_percent()  
    RAM = int(psutil.virtual_memory().total/(1027*1024))  
    RAM_percent = psutil.virtual_memory().percent  
    Swap = int(psutil.swap_memory().total/(1027*1024))  
    Swap_percent = psutil.swap_memory().percent  
    Net_sent = psutil.net_io_counters().bytes_sent  
    Net_recv = psutil.net_io_counters().bytes_recv  
    Net_spkg = psutil.net_io_counters().packets_sent  
    Net_rpkg = psutil.net_io_counters().packets_recv  
    BFH = r'%'
    print " \033[1;32m主机名: %s\033[1;m" % Hostname  
    print " \033[1;32m系统版本: %s\033[1;m" % Sys_version  
    print " \033[1;32m开机时间:%s\033[1;m"  % Boot_Start  
    print " \033[1;32m当前CPU使用率:%s%s\033[1;m" % (Cpu_usage,BFH)  
    print " \033[1;32m物理内存:%dM\t使用率:%s%s\033[1;m" % (RAM,RAM_percent,BFH)  
    print " \033[1;32mSwap内存:%dM\t使用率:%s%s\033[1;m" % (Swap,Swap_percent,BFH)  
    print " \033[1;32m发送:%d Byte\t发送包数:%d个\033[1;m" % (Net_sent,Net_spkg)  
    print " \033[1;32m接收:%d Byte\t接收包数:%d个\033[1;m" % (Net_recv,Net_rpkg)  
  
    for i in psutil.disk_partitions():  
        print " \033[1;32m盘符: %s 挂载点: %s 使用率: %s%s\033[1;m" % (i[0],i[1],psutil.disk_usage(i[1])[3],BFH)  
    for key in psutil.net_if_addrs().keys():  
        print " \033[1;32m网卡: %s IP: %s\033[1;m" % (key,psutil.net_if_addrs()[key][0][1])  
Beispiel #14
0
 def __init__(self):
     super(NumpyDevice, self).__init__()
     self.device_info = DeviceInfo(
         desc="Python Numpy", memsize=virtual_memory().total,
         memalign=8, version=numpy.__version__,
         device_type="Hybrid",
         max_work_group_size=None, max_work_item_sizes=None,
         local_memsize=virtual_memory().total)
 def  check_mem(self):
   #  meminfo=psutil.virtual_memory()
      mem_total=psutil.virtual_memory().total
      mem_free=psutil.virtual_memory().free
  #    mem_free_percent= float(  "%.2f" % (float(mem_free)/float(mem_total))  )
      mem_free_percent= float(  "%.1f" % (float(mem_free)/float(mem_total) *100 )  )
      meminfo=[mem_total,mem_free,mem_free_percent]
      return  meminfo
Beispiel #16
0
    def memInfo(self):
        mem={"total":'', "used":'', "threshhold":False}
        mem['total'] = psutil.virtual_memory().total
        mem['used'] = psutil.virtual_memory().used

        if (mem['total'] - mem['used']) <= conf.memLimit:
            mem["threshhold"]=True
        return mem
 def update(self):
     """Get the latest system information."""
     import psutil
     if self.type == 'disk_use_percent':
         self._state = psutil.disk_usage(self.argument).percent
     elif self.type == 'disk_use':
         self._state = round(psutil.disk_usage(self.argument).used /
                             1024**3, 1)
     elif self.type == 'disk_free':
         self._state = round(psutil.disk_usage(self.argument).free /
                             1024**3, 1)
     elif self.type == 'memory_use_percent':
         self._state = psutil.virtual_memory().percent
     elif self.type == 'memory_use':
         self._state = round((psutil.virtual_memory().total -
                              psutil.virtual_memory().available) /
                             1024**2, 1)
     elif self.type == 'memory_free':
         self._state = round(psutil.virtual_memory().available / 1024**2, 1)
     elif self.type == 'swap_use_percent':
         self._state = psutil.swap_memory().percent
     elif self.type == 'swap_use':
         self._state = round(psutil.swap_memory().used / 1024**3, 1)
     elif self.type == 'swap_free':
         self._state = round(psutil.swap_memory().free / 1024**3, 1)
     elif self.type == 'processor_use':
         self._state = round(psutil.cpu_percent(interval=None))
     elif self.type == 'process':
         if any(self.argument in l.name() for l in psutil.process_iter()):
             self._state = STATE_ON
         else:
             self._state = STATE_OFF
     elif self.type == 'network_out' or self.type == 'network_in':
         counters = psutil.net_io_counters(pernic=True)
         if self.argument in counters:
             counter = counters[self.argument][IO_COUNTER[self.type]]
             self._state = round(counter / 1024**2, 1)
         else:
             self._state = STATE_UNKNOWN
     elif self.type == 'packets_out' or self.type == 'packets_in':
         counters = psutil.net_io_counters(pernic=True)
         if self.argument in counters:
             self._state = counters[self.argument][IO_COUNTER[self.type]]
         else:
             self._state = STATE_UNKNOWN
     elif self.type == 'ipv4_address' or self.type == 'ipv6_address':
         addresses = psutil.net_if_addrs()
         if self.argument in addresses:
             self._state = addresses[self.argument][IF_ADDRS[self.type]][1]
         else:
             self._state = STATE_UNKNOWN
     elif self.type == 'last_boot':
         self._state = dt_util.as_local(
             dt_util.utc_from_timestamp(psutil.boot_time())
         ).date().isoformat()
     elif self.type == 'since_last_boot':
         self._state = dt_util.utcnow() - dt_util.utc_from_timestamp(
             psutil.boot_time())
Beispiel #18
0
def MemCheck(pod, thresh_fraction=0.75, thresh_factor=20.0, swap_thresh_fraction=0.5, return_status=False):



    # Check whether psutil is available (as it is only for unix systems)
    try:
        import psutil
    except:
        if pod['verbose']:print '['+pod['id']+'] Library psutil not available (is this a Windows system?); unable to check RAM, proceeding regardless.'

    # Start infinite loop
    wait_initial = True
    wait_count = 0
    while True:
        mem_wait = False

        # Assess how much RAM is free
        mem_stats = psutil.virtual_memory()
        mem_usage = 1.0-(float(mem_stats[1])/float(mem_stats[0])) #float(mem_stats[2]) / 100.0

        # Also, require wait if the amount of RAM free is more than some multiple the size of the current file
        mem_free = float(psutil.virtual_memory()[4])
        if thresh_factor!=None:
            if ( float(thresh_factor) * float(pod['in_fitspath_size']) )>mem_free:
                if pod['in_fitspath_size']!=None:
                    if (pod['in_fitspath_size']*thresh_factor)>(0.75):
                        thresh_fraction = 0.25
                    else:
                        mem_wait = True

        # Require wait if less than some fraction of RAM is free
        if thresh_fraction!=None:
            if mem_usage>=float(thresh_fraction):
                mem_wait = True

        # Also, require that some fraction of the swap is free
        swap_stats = psutil.swap_memory()
        swap_usage = float(swap_stats[3]) / 100.0
        if swap_usage>swap_thresh_fraction:
             mem_wait = True

        # If process has waited loads of times, progress regardless
        if wait_count>=20:
            mem_wait = False

        # If required, conduct wait for a semi-random period of time; otherwise, break
        if mem_wait==True:
            if wait_initial:
                if pod['verbose']: print '['+pod['id']+'] Waiting for necessary RAM to free up before continuing processing.'
                wait_initial = False
            wait_count += 1
            time.sleep(10.0+(5.0*np.random.rand()))
        else:
            break

        # Return memory status
        if return_status:
            return mem_stats
Beispiel #19
0
def get_memory_info():
	total= int(psutil.virtual_memory().total/1024/1024)
	free=int(psutil.virtual_memory().free/1024/1024)
	used= "Memory percent used {0}%".format(int(psutil.virtual_memory().percent))
	if used>90:
		_waninginfo='Waring Waring Waring \n\n Host_IP:{0} \n Total:{1}M \n Free:{2}M \n {3}'.format(localip,int(total),int(free),used)
		Sendmail(content=_waninginfo,subject="Memory usage rate is high").send()
	else:
		pass
Beispiel #20
0
def system_info(mounted_point='/'):
	return dict(
			hostname=socket.gethostname().split('.')[0],
			psutil_version='.'.join(map(str, ps.version_info)),
			mem_total=ps.virtual_memory().total/M,
			mem_used=ps.virtual_memory().used/M,
			disk_total=ps.disk_usage(mounted_point).total/M,
			disk_used=ps.disk_usage(mounted_point).used/M
		)
Beispiel #21
0
def getHostStats():
    return jsonify(
        {
            'total_usage': psutil.cpu_percent(percpu=False),
            'cpu_usages': psutil.cpu_percent(percpu=True),
            'memory_total': psutil.virtual_memory().total,
            'memory_used': psutil.virtual_memory().used,
            'timestamp': datetime.datetime.now().isoformat()
        }
    )
Beispiel #22
0
 def update_sysinfo(self):
     cpu_times = psutil.cpu_percent(percpu=True)
     
     for i, cpu_time in enumerate(cpu_times):
         self.cpu_usages[i] = float(cpu_time / 100)
         
     self.total_ram = psutil.virtual_memory().total
     self.used_ram = psutil.virtual_memory().total - psutil.virtual_memory().available
     
     self.uptime = time.time() - psutil.boot_time()
Beispiel #23
0
def total_memoria():
	'''
		Retorna a quantidade total de memoria contida no host.
	'''
	return json.dumps({
			"total_mem_b": '%.3f' % (psutil.virtual_memory()[0]),
			"total_mem_k": '%.3f' % (psutil.virtual_memory()[0] / 1024.),
			"total_mem_m": '%.3f' % (psutil.virtual_memory()[0] / 1048576.),
			"total_mem_g": '%.3f' % (psutil.virtual_memory()[0] / 1073741824.)
		})
Beispiel #24
0
def check_memory_usage(bytes_needed, confirm):
    if bytes_needed > psutil.virtual_memory().available:
        if bytes_needed < (psutil.virtual_memory().available + psutil.swap_memory().free):
            text = "Action requires %s, you have enough swap memory available but it will make your computer slower, do you want to continue?" % (
            filesize_format(bytes_needed),)
            return confirm("Memory usage issue", text)
        else:
            text = "Action requires %s, you do not have enough swap memory available, do you want try anyway?" % (
            filesize_format(bytes_needed),)
            return confirm("Memory usage issue", text)
    return True
Beispiel #25
0
def pi_info():
    # issue #1 is caused somewhere here,
    # documentation for psutil says the following call should be what I'm looking for,
    # but always returns 100 percent...
    # see issue #1 comments
    cpu_usage = psutil.cpu_percent(interval=1)
    ram_total = psutil.virtual_memory().total / 1024
    ram_used = psutil.virtual_memory().used / 1024
    return {"cpu_usage": cpu_usage,
            "ram_total": ram_total,
            "ram_used": ram_used}
Beispiel #26
0
def get_memory():
    '''
    获取本地内存的总共、使用、剩余空间量
    '''
    memorys = {}
    token = psutil.virtual_memory()
    memorys["total"] = psutil.virtual_memory().total
    memorys["used"] = psutil.virtual_memory().used
    memorys["free"] = psutil.virtual_memory().free
    Mem = formatMem(memorys)
    return Mem
Beispiel #27
0
 def __init__(self):
     temp = os.popen("vcgencmd measure_temp").readline()
     self.cpuTemp = float(temp.replace("temp=", "").replace("'C\n", ""))
     self.totalRAM = self.b2k(p.virtual_memory().total)
     self.usedRAM = self.b2k(p.virtual_memory().used)
     self.availableRAM = self.b2k(p.virtual_memory().available)
     self.totalDiskSpace = self.b2k(p.disk_usage("/").total)
     self.usedDiskSpace = self.b2k(p.disk_usage("/").used)
     self.freeDiskSpace = self.b2k(p.disk_usage("/").free)
     self.cpuUsage = p.cpu_percent(interval=1)
     self.createJSON()
def check_dic_usage():
    '''查看CPU耗用资源的问题'''
    total_list = []
    start_time = time.time()
    combine_bigram_remove_freq_1_filename = os.path.join(PATH, 'cuted_linguistic_stample', 'combine_bigram_remove_freq_1.txt')
    print psutil.virtual_memory()[2]
    with codecs.open(combine_bigram_remove_freq_1_filename, encoding='utf-8') as f:
        temp_list =f.read()
        print psutil.virtual_memory()[2]
        end_time = time.time()
        print end_time-start_time
def getInfo():
        #print("Process list: ")
        #print psutil.get_process_list() 
        print "cpu usage: "
        print psutil.cpu_percent(1)
        print "Memory Usage: "
        print psutil.virtual_memory()
        print "Disk Usage: "
        print psutil.disk_usage('/')
        print "Network info: "
        print psutil.network_io_counters(True)
Beispiel #30
0
def stats():
	data = "Boot time : "
	data += str(psutil.boot_time())
	data += '<br />'
	data += "Memory : "
	data += str(psutil.virtual_memory()[1])
	data += " / "
	data += str(psutil.virtual_memory()[0])
	data += '<br />'
	data += "CPU : "
	data += str(psutil.cpu_percent(interval=1, percpu=True))
	return data
Beispiel #31
0
def get_ram_total():
    return int(psutil.virtual_memory().total)
Beispiel #32
0
def memory():
    memory = psutil.virtual_memory()
    swap = psutil.swap_memory()
    return memory.total, memory.total - (
            memory.free + memory.inactive), memory.free + memory.inactive, swap.total, swap.used, swap.free, memory.percent
Beispiel #33
0
def get_system_status() -> None:
    message("Memory usage: {} %".format(psutil.virtual_memory().percent))
    message("CPU usage: {} %".format(psutil.cpu_percent()))
 def get_virtual_memory_percent(self):
     return psutil.virtual_memory()._asdict()['percent']
Beispiel #35
0
def get_total_mem_usage():
    sysmem = psutil.virtual_memory()
    return sysmem.total
Beispiel #36
0
 def get_mem_metrics():
     """Get memory percentage metric."""
     return "mem_metrics", psutil.virtual_memory().percent
Beispiel #37
0
def get_available_memory():
    sysvmem = psutil.virtual_memory()
    return sysvmem.available
Beispiel #38
0
def collect_env_info():
    """
    Returns:
        str - a table contains important information about the environment
    """
    data = []
    data.append(("sys.platform", sys.platform))
    data.append(("Python", sys.version.replace("\n", "")))
    data.append(("Tensorpack", __git_version__))
    data.append(("Numpy", np.__version__))

    data.append(("TensorFlow", tfv1.VERSION + "/" + tfv1.GIT_VERSION))
    data.append(("TF Compiler Version", tfv1.COMPILER_VERSION))
    has_cuda = tf.test.is_built_with_cuda()
    data.append(("TF CUDA support", has_cuda))

    try:
        from tensorflow.python.framework import test_util
        data.append(("TF MKL support", test_util.IsMklEnabled()))
    except Exception:
        pass

    try:
        from tensorflow.python.framework import test_util
        data.append(("TF XLA support", test_util.is_xla_enabled()))
    except Exception:
        pass

    if has_cuda:
        data.append(("Nvidia Driver", find_library("nvidia-ml")))
        data.append(("CUDA", find_library("cudart")))
        data.append(("CUDNN", find_library("cudnn")))
        data.append(("NCCL", find_library("nccl")))

        # List devices with NVML
        data.append(("CUDA_VISIBLE_DEVICES",
                     os.environ.get("CUDA_VISIBLE_DEVICES", str(None))))
        try:
            devs = defaultdict(list)
            with NVMLContext() as ctx:
                for idx, dev in enumerate(ctx.devices()):
                    devs[dev.name()].append(str(idx))

            for devname, devids in devs.items():
                data.append(("GPU " + ",".join(devids), devname))
        except Exception:
            data.append(("GPU", "Not found with NVML"))

    vram = psutil.virtual_memory()
    data.append(
        ("Free RAM", "{:.2f}/{:.2f} GB".format(vram.available / 1024**3,
                                               vram.total / 1024**3)))
    data.append(("CPU Count", psutil.cpu_count()))

    # Other important dependencies:
    try:
        import horovod
        data.append(("horovod", horovod.__version__))
    except ImportError:
        pass

    try:
        import cv2
        data.append(("cv2", cv2.__version__))
    except ImportError:
        pass

    import msgpack
    data.append(("msgpack", ".".join([str(x) for x in msgpack.version])))

    has_prctl = True
    try:
        import prctl
        _ = prctl.set_pdeathsig  # noqa
    except Exception:
        has_prctl = False
    data.append(("python-prctl", has_prctl))

    return tabulate(data)
Beispiel #39
0
    a = int(cpuP)
    b = str(cpuS)
    c = str(cpuF)

    b1 = cpuS[0]
    b2 = cpuS[1]
    b3 = cpuS[2]
    b4 = cpuS[3]

    c1 = cpuF[0]
    c2 = cpuF[1]
    c3 = cpuF[2]
    s = "#" * int(a)

    memory = psutil.virtual_memory()[-3]
    ss = int(memory)

    g = "#" * int(ss)

    disk = psutil.disk_usage(path)[-1]
    r = int(disk)

    f = "#" * int(r)
    print("              ---------------------------------------------------------------------- ")
    print("             {'If you see nothing on your screen after clicking please press enter!'}")
    print("              ---------------------------------------------------------------------- ")
    print("")
    print("                               [  CPU Percentage: " + str(cpuP) + "%")
    print("                               |-----------------------------------")
    print("                               |  CPU Stats: ")
Beispiel #40
0
def getMemoUsage():
    return str(psutil.virtual_memory().percent)
import cloudpickle
import py_entitymatching as em

pbar = ProgressBar()
pbar.register()

import os

datapath = '../datasets'

stopwords = get_stop_words(os.path.join(datapath, 'stopwords'))
#stopwords.extend(['and', 'in', 'the', 'my', 'me', 'to', 'you', 'i', 'andre', 'from', 'a', 'of', 'the', 'version', 'love', 'live', 'la', 'mix', 'album', 'dont'])
stopwords = list(set(stopwords))

print("Mem. usage before reading:{0} (GB)".format(
    psutil.virtual_memory().used / 1e9))
A = pd.read_csv('../datasets/msd.csv')
B = pd.read_csv('../datasets/msd.csv')
print("Mem. usage after reading:{0} (GB)".format(psutil.virtual_memory().used /
                                                 1e9))

#stopWords = list(get_stop_words())

memUsageBefore = psutil.virtual_memory().used / 1e9
timeBefore = time.time()
C = downsample_dk(A,
                  B,
                  'id',
                  'id',
                  100000,
                  1,
Beispiel #42
0
def get_ram_usage():
    return int(psutil.virtual_memory().total - psutil.virtual_memory().available)
Beispiel #43
0
def thermald_thread():
    # prevent LEECO from undervoltage
    BATT_PERC_OFF = 100  #10 if LEON else 3

    health_timeout = int(1000 * 2.5 *
                         DT_TRML)  # 2.5x the expected health frequency

    # now loop
    thermal_sock = messaging.pub_sock('thermal')
    health_sock = messaging.sub_sock('health', timeout=health_timeout)
    location_sock = messaging.sub_sock('gpsLocation')

    ignition = False
    fan_speed = 0
    count = 0

    off_ts = None
    started_ts = None
    started_seen = False
    thermal_status = ThermalStatus.green
    thermal_status_prev = ThermalStatus.green
    usb_power = True
    usb_power_prev = True

    network_type = NetworkType.none
    network_strength = NetworkStrength.unknown

    current_filter = FirstOrderFilter(0., CURRENT_TAU, DT_TRML)
    cpu_temp_filter = FirstOrderFilter(0., CPU_TEMP_TAU, DT_TRML)
    health_prev = None
    fw_version_match_prev = True
    time_valid_prev = True
    should_start_prev = False
    handle_fan = None
    is_uno = False
    has_relay = False

    params = Params()
    pm = PowerMonitoring()
    no_panda_cnt = 0

    IsOpenpilotViewEnabled = 0

    do_uninstall = 0
    accepted_terms = 0
    completed_training = 0
    panda_signature = 0

    ts_last_ip = 0
    ip_addr = '255.255.255.255'

    # sound trigger
    sound_trigger = 1

    env = dict(os.environ)
    env['LD_LIBRARY_PATH'] = mediaplayer

    getoff_alert = Params().get('OpkrEnableGetoffAlert') == b'1'
    OpkrAutoShutdown = params.get_OpkrAutoShutdown()

    while 1:
        ts = sec_since_boot()
        health = messaging.recv_sock(health_sock, wait=True)
        location = messaging.recv_sock(location_sock)
        location = location.gpsLocation if location else None
        msg = read_thermal()

        if health is not None:
            usb_power = health.health.usbPowerMode != log.HealthData.UsbPowerMode.client

            # If we lose connection to the panda, wait 5 seconds before going offroad
            if health.health.hwType == log.HealthData.HwType.unknown:
                no_panda_cnt += 1
                if no_panda_cnt > DISCONNECT_TIMEOUT / DT_TRML:
                    if ignition:
                        cloudlog.error("Lost panda connection while onroad")
                    ignition = False
            else:
                no_panda_cnt = 0
                ignition = health.health.ignitionLine or health.health.ignitionCan

            # Setup fan handler on first connect to panda
            if handle_fan is None and health.health.hwType != log.HealthData.HwType.unknown:
                is_uno = health.health.hwType == log.HealthData.HwType.uno
                has_relay = health.health.hwType in [
                    log.HealthData.HwType.blackPanda,
                    log.HealthData.HwType.uno, log.HealthData.HwType.dos
                ]

                if is_uno or not ANDROID:
                    cloudlog.info("Setting up UNO fan handler")
                    handle_fan = handle_fan_uno
                else:
                    cloudlog.info("Setting up EON fan handler")
                    setup_eon_fan()
                    handle_fan = handle_fan_eon

            # Handle disconnect
            if health_prev is not None:
                if health.health.hwType == log.HealthData.HwType.unknown and \
                  health_prev.health.hwType != log.HealthData.HwType.unknown:
                    params.panda_disconnect()
            health_prev = health
        elif ignition == False or IsOpenpilotViewEnabled:
            IsOpenpilotViewEnabled = int(params.get("IsOpenpilotViewEnabled"))
            ignition = IsOpenpilotViewEnabled

        # get_network_type is an expensive call. update every 10s
        if (count % int(10. / DT_TRML)) == 0:
            try:
                network_type = get_network_type()
                network_strength = get_network_strength(network_type)
            except Exception:
                cloudlog.exception("Error getting network status")

        msg.thermal.freeSpace = get_available_percent(default=100.0) / 100.0
        msg.thermal.memUsedPercent = int(round(
            psutil.virtual_memory().percent))
        msg.thermal.cpuPerc = int(round(psutil.cpu_percent()))
        msg.thermal.networkType = network_type
        msg.thermal.networkStrength = network_strength
        msg.thermal.batteryPercent = get_battery_capacity()
        msg.thermal.batteryStatus = get_battery_status()
        msg.thermal.batteryCurrent = get_battery_current()
        msg.thermal.batteryVoltage = get_battery_voltage()
        msg.thermal.usbOnline = get_usb_present()

        # Fake battery levels on uno for frame
        if is_uno:
            msg.thermal.batteryPercent = 100
            msg.thermal.batteryStatus = "Charging"

        # update ip every 10 seconds
        ts = sec_since_boot()
        if ts - ts_last_ip >= 10.:
            try:
                result = subprocess.check_output(["ifconfig", "wlan0"],
                                                 encoding='utf8')  # pylint: disable=unexpected-keyword-arg
                ip_addr = re.findall(r"inet addr:((\d+\.){3}\d+)",
                                     result)[0][0]
            except:
                ip_addr = 'N/A'
            ts_last_ip = ts
        msg.thermal.ipAddr = ip_addr

        current_filter.update(msg.thermal.batteryCurrent / 1e6)

        # TODO: add car battery voltage check
        max_cpu_temp = cpu_temp_filter.update(
            max(msg.thermal.cpu0, msg.thermal.cpu1, msg.thermal.cpu2,
                msg.thermal.cpu3) / 10.0)

        max_comp_temp = max(max_cpu_temp, msg.thermal.mem / 10.,
                            msg.thermal.gpu / 10.)
        bat_temp = msg.thermal.bat / 1000.

        if handle_fan is not None:
            fan_speed = handle_fan(max_cpu_temp, bat_temp, fan_speed, ignition)
            msg.thermal.fanSpeed = fan_speed

        # If device is offroad we want to cool down before going onroad
        # since going onroad increases load and can make temps go over 107
        # We only do this if there is a relay that prevents the car from faulting
        if max_cpu_temp > 107. or bat_temp >= 63. or (has_relay and
                                                      (started_ts is None)
                                                      and max_cpu_temp > 70.0):
            # onroad not allowed
            thermal_status = ThermalStatus.danger
        elif max_comp_temp > 96.0 or bat_temp > 60.:
            # hysteresis between onroad not allowed and engage not allowed
            thermal_status = clip(thermal_status, ThermalStatus.red,
                                  ThermalStatus.danger)
        elif max_cpu_temp > 94.0:
            # hysteresis between engage not allowed and uploader not allowed
            thermal_status = clip(thermal_status, ThermalStatus.yellow,
                                  ThermalStatus.red)
        elif max_cpu_temp > 80.0:
            # uploader not allowed
            thermal_status = ThermalStatus.yellow
        elif max_cpu_temp > 75.0:
            # hysteresis between uploader not allowed and all good
            thermal_status = clip(thermal_status, ThermalStatus.green,
                                  ThermalStatus.yellow)
        else:
            # all good
            thermal_status = ThermalStatus.green

        # **** starting logic ****

        # Check for last update time and display alerts if needed
        now = datetime.datetime.utcnow()

        # show invalid date/time alert
        time_valid = now.year >= 2019
        if time_valid and not time_valid_prev:
            params.delete("Offroad_InvalidTime")
        if not time_valid and time_valid_prev:
            put_nonblocking("Offroad_InvalidTime",
                            json.dumps(OFFROAD_ALERTS["Offroad_InvalidTime"]))
        time_valid_prev = time_valid

        # Show update prompt
        """
    try:
      last_update = datetime.datetime.fromisoformat(params.get("LastUpdateTime", encoding='utf8'))
    except (TypeError, ValueError):
      last_update = now
    dt = now - last_update

    update_failed_count = params.get("UpdateFailedCount")
    update_failed_count = 0 if update_failed_count is None else int(update_failed_count)

    if dt.days > DAYS_NO_CONNECTIVITY_MAX and update_failed_count > 1:
      if current_connectivity_alert != "expired":
        current_connectivity_alert = "expired"
        params.delete("Offroad_ConnectivityNeededPrompt")
        put_nonblocking("Offroad_ConnectivityNeeded", json.dumps(OFFROAD_ALERTS["Offroad_ConnectivityNeeded"]))
    elif dt.days > DAYS_NO_CONNECTIVITY_PROMPT:
      remaining_time = str(max(DAYS_NO_CONNECTIVITY_MAX - dt.days, 0))
      if current_connectivity_alert != "prompt" + remaining_time:
        current_connectivity_alert = "prompt" + remaining_time
        alert_connectivity_prompt = copy.copy(OFFROAD_ALERTS["Offroad_ConnectivityNeededPrompt"])
        alert_connectivity_prompt["text"] += remaining_time + " days."
        params.delete("Offroad_ConnectivityNeeded")
        put_nonblocking("Offroad_ConnectivityNeededPrompt", json.dumps(alert_connectivity_prompt))
    elif current_connectivity_alert is not None:
      current_connectivity_alert = None
      params.delete("Offroad_ConnectivityNeeded")
      params.delete("Offroad_ConnectivityNeededPrompt")
    """

        do_uninstall = params.get("DoUninstall") == b"1"
        accepted_terms = params.get("HasAcceptedTerms") == terms_version
        completed_training = params.get(
            "CompletedTrainingVersion") == training_version

        panda_signature = params.get("PandaFirmware")
        fw_version_match = (panda_signature is None) or (
            panda_signature == FW_SIGNATURE
        )  # don't show alert is no panda is connected (None)

        should_start = ignition

        # with 2% left, we killall, otherwise the phone will take a long time to boot
        should_start = should_start and msg.thermal.freeSpace > 0.02

        # confirm we have completed training and aren't uninstalling
        should_start = should_start and accepted_terms and completed_training and (
            not do_uninstall)

        # check for firmware mismatch
        should_start = should_start and fw_version_match

        # check if system time is valid
        should_start = should_start and time_valid

        # don't start while taking snapshot
        if not should_start_prev:
            is_viewing_driver = params.get("IsDriverViewEnabled") == b"1"
            is_taking_snapshot = params.get("IsTakingSnapshot") == b"1"
            should_start = should_start and (not is_taking_snapshot) and (
                not is_viewing_driver)

        if fw_version_match and not fw_version_match_prev:
            params.delete("Offroad_PandaFirmwareMismatch")
        if not fw_version_match and fw_version_match_prev:
            put_nonblocking(
                "Offroad_PandaFirmwareMismatch",
                json.dumps(OFFROAD_ALERTS["Offroad_PandaFirmwareMismatch"]))

        # if any CPU gets above 107 or the battery gets above 63, kill all processes
        # controls will warn with CPU above 95 or battery above 60
        if thermal_status >= ThermalStatus.danger:
            should_start = False
            if thermal_status_prev < ThermalStatus.danger:
                put_nonblocking(
                    "Offroad_TemperatureTooHigh",
                    json.dumps(OFFROAD_ALERTS["Offroad_TemperatureTooHigh"]))
        else:
            if thermal_status_prev >= ThermalStatus.danger:
                params.delete("Offroad_TemperatureTooHigh")

        if should_start:
            if not should_start_prev:
                params.delete("IsOffroad")

            off_ts = None
            if started_ts is None:
                started_ts = sec_since_boot()
                started_seen = True
                os.system(
                    'echo performance > /sys/class/devfreq/soc:qcom,cpubw/governor'
                )
        else:
            if should_start_prev or (count == 0):
                put_nonblocking("IsOffroad", "1")

            started_ts = None
            if off_ts is None:
                off_ts = sec_since_boot()
                os.system(
                    'echo powersave > /sys/class/devfreq/soc:qcom,cpubw/governor'
                )

            if sound_trigger == 1 and msg.thermal.batteryStatus == "Discharging" and started_seen and (
                    sec_since_boot() - off_ts) > 1 and getoff_alert:
                subprocess.Popen([
                    mediaplayer + 'mediaplayer',
                    '/data/openpilot/selfdrive/assets/sounds/eondetach.wav'
                ],
                                 shell=False,
                                 stdin=None,
                                 stdout=None,
                                 stderr=None,
                                 env=env,
                                 close_fds=True)
                sound_trigger = 0

            # shutdown if the battery gets lower than 3%, it's discharging, we aren't running for
            # more than a minute but we were running
            if msg.thermal.batteryPercent <= BATT_PERC_OFF and msg.thermal.batteryStatus == "Discharging" and \
               started_seen and OpkrAutoShutdown and (sec_since_boot() - off_ts) > OpkrAutoShutdown:
                os.system('LD_LIBRARY_PATH="" svc power shutdown')

        prebuiltlet = Params().get('PutPrebuiltOn') == b'1'
        if not os.path.isfile(prebuiltfile) and prebuiltlet:
            os.system("cd /data/openpilot; touch prebuilt")
        elif os.path.isfile(prebuiltfile) and not prebuiltlet:
            os.system("cd /data/openpilot; rm -f prebuilt")

        # Offroad power monitoring
        pm.calculate(health, msg)
        msg.thermal.offroadPowerUsage = pm.get_power_used()

        msg.thermal.chargingError = current_filter.x > 0. and msg.thermal.batteryPercent < 90  # if current is positive, then battery is being discharged
        msg.thermal.started = started_ts is not None
        msg.thermal.startedTs = int(1e9 * (started_ts or 0))

        msg.thermal.thermalStatus = thermal_status
        thermal_sock.send(msg.to_bytes())

        if usb_power_prev and not usb_power:
            put_nonblocking(
                "Offroad_ChargeDisabled",
                json.dumps(OFFROAD_ALERTS["Offroad_ChargeDisabled"]))
        elif usb_power and not usb_power_prev:
            params.delete("Offroad_ChargeDisabled")

        thermal_status_prev = thermal_status
        usb_power_prev = usb_power
        fw_version_match_prev = fw_version_match
        should_start_prev = should_start

        if usb_power:
            pm.charging_ctrl(msg, ts, 80, 70)

        # report to server once per minute
        if (count % int(60. / DT_TRML)) == 0:
            cloudlog.event("STATUS_PACKET",
                           count=count,
                           health=(health.to_dict() if health else None),
                           location=(location.to_dict() if location else None),
                           thermal=msg.to_dict())

        count += 1
data = '离离原上草,一岁一枯荣'.encode('gbk')
data2 = '离离原上草,一岁一枯荣'.encode('utf-8')
data3 = '最新の主要ニュース'.encode('euc-jp')
print(chardet.detect(data))
print(chardet.detect(data2))
print(chardet.detect(data3))

print(psutil.cpu_count())  # CPU逻辑数量
print(psutil.cpu_count(logical=False))  # CPU物理核心
print(psutil.cpu_times())  # CPU的用户、系统、空闲时间

# for x in range(10):
#     print(psutil.cpu_percent(interval=1, percpu=True))
print('内存:')
print(psutil.virtual_memory())  # 获取内存信息
print(psutil.swap_memory())
print('磁盘:')
print(psutil.disk_partitions())  # 磁盘分区信息
print(psutil.disk_usage('/'))  # 磁盘使用情况
print(psutil.disk_io_counters())  # 磁盘ID
print('网络:')
print(psutil.net_io_counters())  # 获取网络读写字节/包的个数
print(psutil.net_if_addrs())  # 网络接口信息
print(psutil.net_if_stats())  # 网络接口状态
print(psutil.net_connections())  # 获取当前网络连接信息
print('进程:')
print(psutil.pids())  # 获取所哟进程
p = psutil.Process(2128)
print(p.name())  # 获取指定进程
# print(p.exe())  # 进程的exe路径,需要对应权限
def get_memory():
    Mem = psutil.virtual_memory()
    return int(Mem.total / 1024.0), int(Mem.used / 1024.0)
Beispiel #46
0
def get_ram_usage_pct():
    return psutil.virtual_memory().percent
Beispiel #47
0
def available_memory() -> float:
    """System memory in MB"""
    import psutil
    return psutil.virtual_memory().available / 2 ** 20
Beispiel #48
0
def get_mem_info():
    swap_memory = psutil.swap_memory()  # 获取swap分区信息
    total_memory = psutil.virtual_memory().total  # 获取内存总数

    return total_memory
Beispiel #49
0
    def extract_raw(self, cutoff=None):
        """Extract .tif sequence from .raw file.
        cutoff -- number of images extracted.
        Edit:
        Mar 14, 2022 -- Initial commit."""
        # read info from info_file
        folder, file = os.path.split(self.file)
        info_file = os.path.join(folder, "RawImageInfo.txt")
        if os.path.exists(info_file):
            fps, h, w = self.read_raw_image_info(info_file)
        else:
            print("Info file missing!")

        # create output folders, skip if both folders exist
        save_folder = folder
        out_raw_folder = os.path.join(save_folder, 'raw')
        out_8_folder = os.path.join(save_folder, '8-bit')
        if os.path.exists(out_raw_folder) and os.path.exists(out_8_folder):
            print(time.asctime() + " // {} tif folders exists, skipping".format(self.file))
            return None
        if os.path.exists(out_raw_folder) == False:
            os.makedirs(out_raw_folder)
        if os.path.exists(out_8_folder) == False:
            os.makedirs(out_8_folder)

        # check disk
        if self._disk_capacity_check() == False:
            raise SystemError("No enough disk capacity!")

        # calculate buffer size based on available memory, use half of it
        file_size = os.path.getsize(self.file)
        avail_mem = psutil.virtual_memory().available
        unit_size = (h * w + 2) * 2 # the size of a single unit in bytes (frame# + image)
        buffer_size = ((avail_mem // 2) // unit_size) * unit_size # integer number of units
        remaining_size = file_size

        # print("Memory information:")
        # print("Available memory: {:.1f} G".format(avail_mem / 2 ** 30))
        # print("File size: {:.1f} G".format(file_size / 2 ** 30))
        # print("Buffer size: {:.1f} G".format(buffer_size / 2 ** 30))

        # read the binary files, in partitions if needed
        num = 0
        while remaining_size > 0:
            # load np.array from buffer
            if remaining_size > buffer_size:
                read_size = buffer_size
            else:
                read_size = remaining_size
            with open(self.file, "rb") as f:
                f.seek(-remaining_size, 2) # offset bin file reading, counting remaining size from EOF
                bytestring = f.read(read_size)
                a = np.frombuffer(bytestring, dtype=np.uint16)
            remaining_size -= read_size

            # manipulate the np.array
            assert(a.shape[0] % (h*w+2) == 0)
            num_images = a.shape[0] // (h*w+2)
            img_in_row = a.reshape(num_images, h*w+2)
            labels = img_in_row[:, :2] # not in use currently
            images = img_in_row[:, 2:]
            images_reshape = images.reshape(num_images, h, w)

            # save the image sequence
            for label, img in zip(labels, images_reshape):
                # num = label[0] + label[1] * 2 ** 16 + 1 # convert image label to uint32 to match the info in StagePosition.txt
                io.imsave(os.path.join(save_folder, 'raw', '{:05d}.tif'.format(num)), img, check_contrast=False)
                io.imsave(os.path.join(save_folder, '8-bit', '{:05d}.tif'.format(num)), to8bit(img), check_contrast=False)

                # with open(os.path.join(out_raw_folder, 'log.txt'), 'a') as f:
                #     f.write(time.asctime() + ' // frame {:05d}'.format(num))
                # with open(os.path.join(out_8_folder, 'log.txt'), 'a') as f:
                #     f.write(time.asctime() + ' // frame {:05d}'.format(num))
                num += 1
                if cutoff is not None:
                    if num > cutoff:
                        return None
Beispiel #50
0
def get_percent_mem_usage():
    sysmem = psutil.virtual_memory()
    return sysmem.percent