def get_n_splits(ctrl, n_mc,n_meas): import os, math import numpy as np if not(ctrl.isfield('n_para')): ctrl.n_para = 1 # no paralell computing mem_bytes = float(os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')) / 2 n_split = int(math.ceil((8 * n_mc * n_meas *3 * ctrl.n_para) / mem_bytes)); n_split =3 if n_split > 1: n_tmp = math.floor(n_meas/n_split) split_start = np.zeros([n_split]) split_end = np.ones([n_split]) * n_tmp s_part = np.ones([n_split]) * n_tmp for t in xrange(1,n_split): split_start[t] = split_end[t-1] if t == n_split-1: split_end[t] = n_meas else: split_end[t] = split_end[t-1] + n_tmp else: split_start = np.zeros(1); split_end = np.ones(1) * n_meas; s_part = np.ones(1) * n_meas; if ctrl.isSetTrue('debug'): print split_start print split_end print s_part return (n_split,split_start,split_end,s_part)
def check_options(parsed_args): # section of code that checks the infolder entry if os.path.isdir(parsed_args.infolder): infolder = parsed_args.infolder else: print "The folder %s does not exist." % parsed_args.infolder sys.exit() # if the directory that the user specifies does not exist, then the program makes it for them. if not os.path.isdir(parsed_args.outfolder): os.makedirs(parsed_args.outfolder) outfolder = parsed_args.outfolder if outfolder[-1] != '/': outfolder = outfolder + '/' # Check the filter file if parsed_args.filter == 'NONE' or os.path.exists(parsed_args.filter): filter_file = parsed_args.filter else: print "The file %s does not exist." % parsed_args.filter sys.exit() # section of code that deals determining the number of CPU cores that will be used by the program if parsed_args.num_proc > os.sysconf("SC_NPROCESSORS_CONF"): num_proc = os.sysconf("SC_NPROCESSORS_CONF") elif parsed_args.num_proc < 1: num_proc = 1 else: num_proc = int(parsed_args.num_proc) do_protein = parsed_args.protein return infolder, outfolder, filter_file, num_proc, do_protein
def check_options(parsed_args): # section of code that deals determining the number of CPU cores that will be used by the program if parsed_args.num_proc > os.sysconf("SC_NPROCESSORS_CONF"): num_proc = os.sysconf("SC_NPROCESSORS_CONF") elif parsed_args.num_proc < 1: num_proc = 1 else: num_proc = parsed_args.num_proc if os.path.exists(parsed_args.infile): infile = parsed_args.infile else: print "The file %s does not exist." % parsed_args.infile sys.exit() if os.path.exists(parsed_args.query): query = parsed_args.query else: print "The file %s does not exist." % parsed_args.query sys.exit() # if the directory that the user specifies does not exist, then the program makes it for them. folder = parsed_args.folder if not os.path.isdir(parsed_args.folder): os.makedirs(parsed_args.folder) # should offer some sanity here for the input, but it is missing right now. call it a to do item e_val = parsed_args.eval return infile, query, folder, num_proc, e_val
def ctl_socket_handler_stats(self, data, fd): if not 'stats' in self.__dict__: self.stats = { "start_up_timestamp": self.start_up_timestamp } self.stats_last_update = 0 self.ram_usage = 0 if 'SC_PAGESIZE' in os.sysconf_names: self.stats_pagesize = os.sysconf('SC_PAGESIZE') elif 'SC_PAGE_SIZE' in os.sysconf_names: self.stats_pagesize = os.sysconf('SC_PAGE_SIZE') elif '_SC_PAGESIZE' in os.sysconf_names: self.stats_pagesize = os.sysconf('_SC_PAGESIZE') else: self.stats_pagesize = 4096 self.stats["infeeds"] = sum(1 for x in self.feeds if x.startswith('infeed-')) self.stats["outfeeds"] = sum(1 for x in self.feeds if x.startswith('outfeed-')) self.stats["plugins"] = len(self.plugins) if time.time() - self.stats_last_update > 5: if self.stats_ramfile != None: self.stats_ramfile.seek(0) self.ram_usage = int(self.stats_ramfile.read().split(' ')[1]) * self.stats_pagesize st = os.statvfs(os.getcwd()) self.stats["groups"] = os.stat('groups').st_nlink - 2 self.stats["articles"] = sum(1 for x in os.listdir('articles')) - os.stat('articles').st_nlink + 2 self.stats["cpu"] = 15 self.stats["ram"] = self.ram_usage self.stats["disk_free"] = st.f_bavail * st.f_frsize self.stats["disk_used"] = (st.f_blocks - st.f_bfree) * st.f_frsize self.stats_last_update = time.time() return self.stats
def run(setupconfig): yield "1,installing requiered software" os.system("pacman -Sy reflector --noconfirm --needed 2> /dev/null > /dev/null") yield "10,fetching mirrorlist" # get the 50 >most up to date< servers and sort them by download speed os.system("reflector --verbose -l 50 -p http --sort rate --save list.txt 2>/dev/null > /dev/null") yield "33,partitioning disk" # Calculate Partition layout: # # /dev/sdx1 = data = TOTALSIZE - SIZE OF RAM # /dev/sdx2 = swap = SIZE OF RAM # This should work for most cases cmd = str("blockdev --getsize64 /dev/" + setupconfig.disk) size = int(subprocess.check_output(cmd, shell=True).decode()) ramsize = int(os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")) / 1000 / 1000 data = (size / 1000 / 1000) - ramsize Disks().part_disk(setupconfig, data) yield "60,creating filesystems" # Create FS Disks().makefs(setupconfig) yield "80,mounting disk" # mount disk Disks().mount(setupconfig) yield "100,done!"
def get_php_fpm_memory(php_fpm_total_ram_slice=0.4): # Taken from http://stackoverflow.com/a/28161352 mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') # e.g. 4015976448 mem_megs = mem_bytes/(1024.**2) # e.g. 3.74 LOGGER.info("Total memory available in the system: %sM", '%d' % mem_megs) return mem_megs * php_fpm_total_ram_slice
def cpu_count(): try: return multiprocessing.cpu_count() except NotImplementedError: pass count = int(os.environ.get('NUMBER_OF_PROCESSORS', 0)) if count > 0: return count try: if 'SC_NPROCESSORS_ONLN' in os.sysconf_names: count = os.sysconf('SC_NPROCESSORS_ONLN') elif 'SC_NPROCESSORS_CONF' in os.sysconf_names: count = os.sysconf('SC_NPROCESSORS_CONF') if count > 0: return cpu_count except AttributeError: pass count = 1 # unable to detect number of CPUs return count
def get_system_memory(): '''Attempt to determine total system memory If amount cannot be determined, emits warning and returns -1. ''' # MacOS X doesn't support sysconf('SC_PHYS_PAGES') if platform.system() == 'Darwin': try: out = subprocess.check_output(['sysctl', 'hw.memsize'], universal_newlines=True) except subprocess.CalledProcessError as exc: log.warning('Cannot determine system memory, sysctl failed with %s', exc.output) return -1 # output of sysctl is 'hw.memsize: #'. Strip the prefix. hit = re.match(r'^hw.memsize: ([0-9]+)$', out) if not hit: log.warning('Cannot determine system memory, unable to parse sysctl output.') return -1 return int(hit.group(1)) else: try: return os.sysconf('SC_PHYS_PAGES') * os.sysconf('SC_PAGESIZE') except ValueError: log.warning('Unable to determine number of CPU cores (sysconf failed).') return -1
def __init__(self, name): """Initialization of procparser object. Takes one argument name for the parser. Initializes the name, pagesize from system, hertz from the system, proclist, procdata and data parsed from the procps subsystem of Linux. """ self.name = name self.statfld = ['pid', 'comm', 'state', 'ppid', 'pgrp', 'session', 'tty_nr', 'tpgid', 'flags', 'minflt', 'cminflt', 'majflt', 'cmajflt', 'utime', 'stime', 'cutime', 'cstime', 'priority', 'nice', 'num_threads', 'itrealvalue', 'starttime', 'vsize', 'rss', 'rsslim', 'startcode', 'endcode', 'startstack', 'kstkesp', 'kstkeip', 'signal', 'blocked', 'sigignore', 'sigcatch', 'wchan', 'nswap', 'cnswap', 'exit_signal', 'processor', 'rt_priority', 'policy', 'delayacct_blkio_ticks', 'guest_time', 'cguest_time'] # Get the pagesize from the system. self.pagesize = os.sysconf(os.sysconf_names['SC_PAGE_SIZE']) # Get the ticks/sec from the system. self.hertz = os.sysconf(os.sysconf_names['SC_CLK_TCK']) self.proclist = None self.procdata = None for field in self.fields: self.data[field] = None
def determine_threads(options): '''Return optimum number of upload threads''' cores = os.sysconf('SC_NPROCESSORS_ONLN') memory = os.sysconf('SC_PHYS_PAGES') * os.sysconf('SC_PAGESIZE') if options.compress == 'lzma': # Keep this in sync with compression level in backends/common.py # Memory usage according to man xz(1) mem_per_thread = 186 * 1024 ** 2 else: # Only check LZMA memory usage mem_per_thread = 0 if cores == -1 or memory == -1: log.warn("Can't determine number of cores, using 2 upload threads.") return 1 elif 2 * cores * mem_per_thread > (memory / 2): threads = min(int((memory / 2) // mem_per_thread), 10) if threads > 0: log.info('Using %d upload threads (memory limited).', threads) else: log.warn('Warning: compression will require %d MiB memory ' '(%d%% of total system memory', mem_per_thread / 1024 ** 2, mem_per_thread * 100 / memory) threads = 1 return threads else: threads = min(2 * cores, 10) log.info("Using %d upload threads.", threads) return threads
def calculate_memory_for_sort(): """Calculate available memory for ``samtools sort`` function. If there is enough memory, no temp files are created. **Enough** is defined as at least 1G per CPU. Returns ------- sort_memory: str or None String to use directly with *-m* option in sort, or None. """ avail_memory = None if HAVE_PSUTIL == True: mem = virtual_memory() avail_memory = mem.total else: try: avail_memory = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') # e.g. 4015976448 except ValueError: logging.error("If you're in Mac OS you need to have the psutil Python library.") raise SystemExit avail_cpu = multiprocessing.cpu_count() sort_memory = avail_memory / avail_cpu / 1024 ** 2 # samtools default from documentation is 768M, to be conservative # only use -m option when there is more than 1G per CPU. if sort_memory < 1024: sort_memory = None else: sort_memory = "%sG" % (int(floor(sort_memory / 1024))) return sort_memory
def jobs(self): """ Find the amount of cpu cores to set the default amount of tasks executed in parallel. At runtime the options can be obtained from :py:const:`waflib.Options.options` :: from waflib.Options import options njobs = options.jobs :return: the amount of cpu cores :rtype: int """ count = int(os.environ.get('JOBS', 0)) if count < 1: if 'NUMBER_OF_PROCESSORS' in os.environ: # on Windows, use the NUMBER_OF_PROCESSORS environment variable count = int(os.environ.get('NUMBER_OF_PROCESSORS', 1)) else: # on everything else, first try the POSIX sysconf values if hasattr(os, 'sysconf_names'): if 'SC_NPROCESSORS_ONLN' in os.sysconf_names: count = int(os.sysconf('SC_NPROCESSORS_ONLN')) elif 'SC_NPROCESSORS_CONF' in os.sysconf_names: count = int(os.sysconf('SC_NPROCESSORS_CONF')) elif os.name not in ('nt', 'java'): tmp = self.cmd_and_log(['sysctl', '-n', 'hw.ncpu']) if re.match('^[0-9]+$', tmp): count = int(tmp) if count < 1: count = 1 elif count > 1024: count = 1024 return count
def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None): # those lines are copied from distutils.ccompiler.CCompiler directly macros, objects, extra_postargs, pp_opts, build = self._setup_compile(output_dir, macros, include_dirs, sources, depends, extra_postargs) cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) # parallel code N = 2*multiprocessing.cpu_count()# number of parallel compilations try: # On Unix-like platforms attempt to obtain the total memory in the # machine and limit the number of parallel jobs to the number of Gbs # of RAM (to avoid killing smaller platforms like the Pi) mem = os.sysconf('SC_PHYS_PAGES') * os.sysconf('SC_PAGE_SIZE') # bytes except (AttributeError, ValueError): # Couldn't query RAM; don't limit parallelism (it's probably a well # equipped Windows / Mac OS X box) pass else: mem = max(1, int(round(mem / 1024 ** 3))) # convert to Gb N = min(mem, N) def _single_compile(obj): try: src, ext = build[obj] except KeyError: return newcc_args = cc_args if _platform == "darwin": if src.endswith('.cpp'): newcc_args = cc_args + ["-stdlib=libc++"] self._compile(obj, src, ext, newcc_args, extra_postargs, pp_opts) # convert to list, imap is evaluated on-demand pool = multiprocessing.pool.ThreadPool(N) list(pool.imap(_single_compile,objects)) return objects
def jobs(self): """ Amount of threads to use """ count = int(os.environ.get('JOBS', 0)) if count < 1: if sys.platform == 'win32': # on Windows, use the NUMBER_OF_PROCESSORS environment variable count = int(os.environ.get('NUMBER_OF_PROCESSORS', 1)) else: # on everything else, first try the POSIX sysconf values if hasattr(os, 'sysconf_names'): if 'SC_NPROCESSORS_ONLN' in os.sysconf_names: count = int(os.sysconf('SC_NPROCESSORS_ONLN')) elif 'SC_NPROCESSORS_CONF' in os.sysconf_names: count = int(os.sysconf('SC_NPROCESSORS_CONF')) elif os.name != 'java': tmp = self.cmd_and_log(['sysctl', '-n', 'hw.ncpu']) if re.match('^[0-9]+$', tmp): count = int(tmp) if count < 1: count = 1 elif count > 1024: count = 1024 return count
def check_options(parsed_args): # I'm not issuing a message that hey, this file is there and will be overwritten, the program will just overwrite. # TODO: check that the folder that this is saved in exists if os.path.isdir(parsed_args.infolder): infolder = parsed_args.infolder else: print "The infolder %s does not exist." % parsed_args.infolder sys.exit() outfile = parsed_args.outfile # Check the gene block file if os.path.exists(parsed_args.gene_block_file): gene_block_file = parsed_args.gene_block_file else: print "The gene_block file %s does not exist." % parsed_args.gene_block_file sys.exit() # section of code that deals determining the number of CPU cores that will be used by the program if parsed_args.num_proc > os.sysconf("SC_NPROCESSORS_CONF"): num_proc = os.sysconf("SC_NPROCESSORS_CONF") elif parsed_args.num_proc < 1: num_proc = 1 else: num_proc = int(parsed_args.num_proc) # I am not checking this option here, just leave it as default for the time being. refrence_list = [parsed_args.refrence] quiet = parsed_args.quiet return infolder, outfile, gene_block_file, num_proc, refrence_list, quiet
def __init__(self, log_d): self.log_d = log_d self.mem_gib = (os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') ) / (1024.**3) # self.ncpu = multiprocessing.cpu_count() #Using this will parameter will likely run out of memory and swap self.ncpu = int(self.mem_gib // 16) # Allocate 16 GB ram per concurrent instance. If not enough ram, start only 1 instance if (self.ncpu == 0): self.ncpu = 1 self.parser = Parser(self.log_d)
def get_max_redis_memory(): total_virtual_mem = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')/(pow(1024, 2)) max_memory = int(total_virtual_mem * 0.05) # Max memory for redis is 5% of virtual memory if max_memory < 50: return 50 else: return max_memory
def get_system_info(): """Return system hz use SC_CLK_TCK.""" ncpus = float(os.sysconf(os.sysconf_names['SC_NPROCESSORS_ONLN'])) ticks = float(os.sysconf(os.sysconf_names['SC_CLK_TCK'])) if ticks == -1: ticks = 100 return (ticks, ncpus)
def get_total_memory(self): """ Get machine total memory. """ try: return os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") except: return None
def virtual_memory(): # we could have done this with kstat, but imho this is good enough total = os.sysconf('SC_PHYS_PAGES') * PAGE_SIZE # note: there's no difference on Solaris free = avail = os.sysconf('SC_AVPHYS_PAGES') * PAGE_SIZE used = total - free percent = usage_percent(used, total, _round=1) return svmem(total, avail, percent, used, free)
def fetch_ram(): lob.output_y("Ram") mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') mem_gib = mem_bytes/(1024.**3) print("{} G".format(mem_gib)) r.hset(mac, "Ram", mem_gib)
def __init__(self, pid=None, window=0.3): self.CLOCKTICK = os.sysconf('SC_CLK_TCK') self.PAGESIZE = os.sysconf('SC_PAGESIZE') self.window = window if not pid: self.pid = str(os.getpid()) else: self.pid = str(pid) self.allconns = self._ConnTable()
def is_s5orig(): # returns True if the TS is running on the initially released S5 sequencer if is_TsVm(): try: memTotalGb = os.sysconf('SC_PAGE_SIZE')*os.sysconf('SC_PHYS_PAGES')/(1024*1024*1024) return memTotalGb < 70 except: logger.error("Unable to determine system memory") return False
def check_mem(): """ Check host has more than 10GB memory """ mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') LOG.debug('Memory: %s' % str(mem_bytes)) if mem_bytes < 10737418240: LOG.error('Not enough memory: %d' % mem_bytes) else: LOG.info('Memory ok: %d' % mem_bytes)
def getProcessData(pid): try: items = open("/proc/%d/stat" % pid).read().split() return { "cpu": formatCPU(float(int(items[13]) + int(items[14])) / os.sysconf("SC_CLK_TCK")), "rss": formatRSS(int(items[23]) * os.sysconf("SC_PAGE_SIZE")) } except: return { "cpu": "N/A", "rss": "N/A" }
def get_hypervisor_memory(reserved): """Get physical available memory for hypervisor in MiB -> float arguments: reserved - MiB reserved for OS """ memory_bytes = sysconf('SC_PAGE_SIZE') * sysconf('SC_PHYS_PAGES') return float(memory_bytes) / 1024.0**2.0 - reserved
def getpagesize(): if _getpagesize: return _getpagesize() else: try: return sysconf("SC_PAGE_SIZE") except ValueError: # Irix 5.3 has _SC_PAGESIZE, but not _SC_PAGE_SIZE return sysconf("SC_PAGESIZE")
def memory_bytes(self): """ Total installed memory of this RunManager """ try: return os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') except ValueError: # Fallback to sysctl when os.sysconf('SC_PHYS_PAGES') fails on OS X return int(check_output(['sysctl', '-n', 'hw.memsize']).strip())
def get_free_memory_os(): free_memory_Kb = 0 try: free_memory_Kb = (os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_AVPHYS_PAGES')) / 1024.0 print 'The strict free memory available determined is ' + str(round((free_memory_Kb / (1024.0 ** 2)), 1)) + ' Gb' except Exception as e: print e print 'WARNING: it was not possible to determine the free memory available!' return free_memory_Kb
def virtualMemory(self): """ """ PAGE_SIZE = os.sysconf('SC_PAGE_SIZE') total = os.sysconf('SC_PHYS_PAGES') * PAGE_SIZE free = avail = os.sysconf('SC_AVPHYS_PAGES') * PAGE_SIZE used = total - free percent = self.usagePercent(used, total) svmem = namedtuple('svmem', ['total', 'available', 'percent', 'used', 'free']) return svmem(total, avail, percent, used, free)
import stem.util.connection import stem.util.enum import stem.util.str_tools from stem.util import log try: # unavailable on windows (#19823) import pwd IS_PWD_AVAILABLE = True except ImportError: IS_PWD_AVAILABLE = False # os.sysconf is only defined on unix try: CLOCK_TICKS = os.sysconf(os.sysconf_names['SC_CLK_TCK']) except AttributeError: CLOCK_TICKS = None IS_LITTLE_ENDIAN = sys.byteorder == 'little' ENCODED_ADDR = {} # cache of encoded ips to their decoded version Stat = stem.util.enum.Enum(('COMMAND', 'command'), ('CPU_UTIME', 'utime'), ('CPU_STIME', 'stime'), ('START_TIME', 'start time')) @functools.lru_cache() def is_available(): """ Checks if proc information is available on this platform.
def online_count(): """ Return Number of Online cpus in the system """ return os.sysconf('SC_NPROCESSORS_ONLN')
# uncompyle6 version 3.2.4 # Python bytecode 2.7 (62211) # Decompiled from: Python 2.7.15 (v2.7.15:ca079a3ea3, Apr 30 2018, 16:30:26) [MSC v.1500 64 bit (AMD64)] # Embedded file name: popen2 import os, sys, warnings warnings.warn('The popen2 module is deprecated. Use the subprocess module.', DeprecationWarning, stacklevel=2) __all__ = [ 'popen2', 'popen3', 'popen4'] try: MAXFD = os.sysconf('SC_OPEN_MAX') except (AttributeError, ValueError): MAXFD = 256 _active = [] def _cleanup(): for inst in _active[:]: if inst.poll(_deadstate=sys.maxint) >= 0: try: _active.remove(inst) except ValueError: pass class Popen3: sts = -1 def __init__(self, cmd, capturestderr=False, bufsize=-1): _cleanup() self.cmd = cmd p2cread, p2cwrite = os.pipe()
#!/usr/bin env python3 import subprocess import os import sys import time ITERTIME = 1 iter=0 # waiting iterations jiffy = float(os.sysconf(os.sysconf_names['SC_CLK_TCK'])) cmdline = sys.argv[1:] print(cmdline) ncores = os.cpu_count() nprocs = int(os.environ.get('OMP_NUM_THREADS', os.cpu_count())) print("procs: {}\t cores: {}".format(nprocs, ncores)) proc = subprocess.Popen(cmdline) tzero = time.clock_gettime(time.CLOCK_REALTIME) print(proc.pid) procdir = os.path.join("/proc", str(proc.pid)) # utime for a threadcd tutlist = {} corelist = [0.0]*nprocs corearr = [0.0]*ncores with open('/proc/uptime', 'r') as f:
def memAvail(p=0.8): mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_AVPHYS_PAGES') # e.g. 4015976448 mem_gib = mem_bytes/(1024.**3) # e.g. 3.74 return int(mem_gib*p)
def get_cpu_tick_per_second(): return os.sysconf('SC_CLK_TCK')
def test_os_sysconf(self): os = self.posix assert os.sysconf(self.sysconf_value) == self.sysconf_result assert os.sysconf(self.sysconf_name) == self.sysconf_result assert os.sysconf_names[self.sysconf_name] == self.sysconf_value
class ForkAwareLocal(threading.local): def __init__(self): register_after_fork(self, lambda obj: obj.__dict__.clear()) def __reduce__(self): return type(self), () # # Close fds except those specified # try: MAXFD = os.sysconf("SC_OPEN_MAX") except Exception: MAXFD = 256 def close_all_fds_except(fds): fds = list(fds) + [-1, MAXFD] fds.sort() assert fds[-1] == MAXFD, 'fd too large' for i in range(len(fds) - 1): os.closerange(fds[i] + 1, fds[i + 1]) # # Close sys.stdin and replace stdin with os.devnull #
import os import time import socket from resource import getrusage, RUSAGE_SELF from twisted.application.service import Service from twisted.internet.task import LoopingCall from carbon.conf import settings stats = {} HOSTNAME = socket.gethostname().replace('.', '_') PAGESIZE = os.sysconf('SC_PAGESIZE') rusage = getrusage(RUSAGE_SELF) lastUsage = rusage.ru_utime + rusage.ru_stime lastUsageTime = time.time() # NOTE: Referencing settings in this *top level scope* will # give you *defaults* only. Probably not what you wanted. # TODO(chrismd) refactor the graphite metrics hierarchy to be cleaner, # more consistent, and make room for frontend metrics. #metric_prefix = "Graphite.backend.%(program)s.%(instance)s." % settings def increment(stat, increase=1): try: stats[stat] += increase except KeyError: stats[stat] = increase
def get_workers(): procs = os.sysconf('SC_NPROCESSORS_ONLN') if procs > 0: return procs * 2 + 1 else: return 3
def get_total_mem(self): # Get total memory in bytes and divide by 1024**2 to get the value in MB. return os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') / (1024 **2)
def online_cpus(): return frozenset(range(os.sysconf('SC_NPROCESSORS_ONLN')))
def numCPUs(): if not hasattr(os, "sysconf"): raise RuntimeError("No sysconf detected.") return os.sysconf("SC_NPROCESSORS_ONLN")
line = file_.readline() if line == b"": break if not line.endswith(b"\n"): file_.seek(fpos) break yield line[:-1] def _parse_send(line): message_id, send_time = line.split(b",", 1) send_time = int(send_time) return message_id, send_time def _parse_receive(line): message_id, send_time, receive_time = line.split(b",", 2) send_time = int(send_time) receive_time = int(receive_time) return message_id, send_time, receive_time _join = _plano.join _ticks_per_ms = _os.sysconf(_os.sysconf_names["SC_CLK_TCK"]) / 1000 _page_size = _resource.getpagesize()
GCC = 'gcc' CPP = os.environ.get('CPP', 'cpp') try: CPUCOUNT = int(os.environ.get('CPUCOUNT', -1)) except: CPUCOUNT = -1 if CPUCOUNT < 1: try: import multiprocessing CPUCOUNT = multiprocessing.cpu_count() except: try: CPUCOUNT = int(os.sysconf('SC_NPROCESSORS_ONLN')) except: CPUCOUNT = 1 binary_list = [] # this is used for reporting (at the end of the build) # the server configuration report = {} report['kernel'] = False report['execinfo'] = False report['ifaddrs'] = False report['locking'] = False report['event'] = False report['timer'] = False report['filemonitor'] = False
import errno import functools import os from collections import namedtuple from psutil import _common from psutil import _psposix from psutil._common import conn_tmap, usage_percent, isfile_strict import _psutil_osx as cext import _psutil_posix __extra__all__ = [] # --- constants PAGESIZE = os.sysconf("SC_PAGE_SIZE") # http://students.mimuw.edu.pl/lxr/source/include/net/tcp_states.h TCP_STATUSES = { cext.TCPS_ESTABLISHED: _common.CONN_ESTABLISHED, cext.TCPS_SYN_SENT: _common.CONN_SYN_SENT, cext.TCPS_SYN_RECEIVED: _common.CONN_SYN_RECV, cext.TCPS_FIN_WAIT_1: _common.CONN_FIN_WAIT1, cext.TCPS_FIN_WAIT_2: _common.CONN_FIN_WAIT2, cext.TCPS_TIME_WAIT: _common.CONN_TIME_WAIT, cext.TCPS_CLOSED: _common.CONN_CLOSE, cext.TCPS_CLOSE_WAIT: _common.CONN_CLOSE_WAIT, cext.TCPS_LAST_ACK: _common.CONN_LAST_ACK, cext.TCPS_LISTEN: _common.CONN_LISTEN, cext.TCPS_CLOSING: _common.CONN_CLOSING, cext.PSUTIL_CONN_NONE: _common.CONN_NONE,
so_ext = '.dylib' elif sys.platform.startswith('win'): so_ext = '.dll' else: raise OSError('Unknown platform') libname_so = libname + so_ext return ctypes.CDLL(os.path.join(os.path.dirname(__file__), libname_so)) else: _loaderpath = os.path.dirname(__file__) return numpy.ctypeslib.load_library(libname, _loaderpath) #Fixme, the standard resouce module gives wrong number when objects are released #see http://fa.bianp.net/blog/2013/different-ways-to-get-memory-consumption-or-lessons-learned-from-memory_profiler/#fn:1 #or use slow functions as memory_profiler._get_memory did CLOCK_TICKS = os.sysconf("SC_CLK_TCK") PAGESIZE = os.sysconf("SC_PAGE_SIZE") def current_memory(): '''Return the size of used memory and allocated virtual memory (in MB)''' #import resource #return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000 if sys.platform.startswith('linux'): with open("/proc/%s/statm" % os.getpid()) as f: vms, rss = [int(x) * PAGESIZE for x in f.readline().split()[:2]] return rss / 1e6, vms / 1e6 else: return 0, 0
def cpu_count(): try: import multiprocessing # pylint: disable=import-outside-toplevel return multiprocessing.cpu_count() except ImportError: return int(os.sysconf('SC_NPROCESSORS_ONLN'))
def _get_cpu_nums(): return os.sysconf("SC_NPROCESSORS_ONLN")
sys.excepthook = sys.__excepthook__ if issubclass(etype, KeyboardInterrupt): pass elif issubclass(etype, IOError) and value.errno == errno.EPIPE: pass else: sys.__excepthook__(etype, value, tb) sys.excepthook = std_exceptions # # Define some global variables # PAGESIZE = os.sysconf("SC_PAGE_SIZE") / 1024 #KiB our_pid = os.getpid() have_pss = 0 have_swap_pss = 0 class Proc: def __init__(self): uname = os.uname() if uname[0] == "FreeBSD": self.proc = '/compat/linux/proc' else: self.proc = '/proc' def path(self, *args):
object.__setattr__(self, key, value) states = _StateEnum() # noqa: E305 states.STOPPED = states.State() states.STARTING = states.State() states.STARTED = states.State() states.STOPPING = states.State() states.EXITING = states.State() try: import fcntl except ImportError: max_files = 0 else: try: max_files = os.sysconf('SC_OPEN_MAX') except AttributeError: max_files = 1024 class Bus(object): """Process state-machine and messenger for HTTP site deployment. All listeners for a given channel are guaranteed to be called even if others at the same channel fail. Each failure is logged, but execution proceeds on to the next listener. The only way to stop all processing from inside a listener is to raise SystemExit and stop the whole server. """ states = states
def number_of_cpus(): """ Number of virtual or physical CPUs on this system, i.e. user/real as output by time(1) when called with an optimally scaling userspace-only program Return -1 if ncpus cannot be detected taken from: http://stackoverflow.com/questions/1006289/how-to-find-out-the-number-of-cpus-in-python """ import os, re, subprocess # Python 2.6+ try: import multiprocessing return multiprocessing.cpu_count() except (ImportError, NotImplementedError): pass # POSIX try: res = int(os.sysconf('SC_NPROCESSORS_ONLN')) if res > 0: return res except (AttributeError, ValueError): pass # Windows try: res = int(os.environ['NUMBER_OF_PROCESSORS']) if res > 0: return res except (KeyError, ValueError): pass # jython try: from java.lang import Runtime runtime = Runtime.getRuntime() res = runtime.availableProcessors() if res > 0: return res except ImportError: pass # BSD try: sysctl = subprocess.Popen(['sysctl', '-n', 'hw.ncpu'], stdout=subprocess.PIPE) scStdout = sysctl.communicate()[0] res = int(scStdout) if res > 0: return res except (OSError, ValueError): pass # Linux try: res = open('/proc/cpuinfo').read().count('processor\t:') if res > 0: return res except IOError: pass # Solaris try: pseudoDevices = os.listdir('/devices/pseudo/') expr = re.compile('^cpuid@[0-9]+$') res = 0 for pd in pseudoDevices: if expr.match(pd) is not None: res += 1 if res > 0: return res except OSError: pass # Other UNIXes (heuristic) try: try: dmesg = open('/var/run/dmesg.boot').read() except IOError: dmesgProcess = subprocess.Popen(['dmesg'], stdout=subprocess.PIPE) dmesg = dmesgProcess.communicate()[0] res = 0 while '\ncpu' + str(res) + ':' in dmesg: res += 1 if res > 0: return res except OSError: pass return -1
def __get_page_size(): try: return os.sysconf("SC_PAGE_SIZE") # FIX_ME except Exception: return PSEUDO_PAGE_SIZE
def determineNumberOfCPUs(debug=False): """ Number of virtual or physical CPUs on this system, i.e. user/real as output by time(1) when called with an optimally scaling userspace-only program""" # Python 2.6+ try: import multiprocessing return multiprocessing.cpu_count() except (ImportError,NotImplementedError): pass # POSIX try: res = int(os.sysconf('SC_NPROCESSORS_ONLN')) if res > 0: return res except (AttributeError,ValueError): pass # Windows try: res = int(os.environ['NUMBER_OF_PROCESSORS']) if res > 0: return res except (KeyError, ValueError): pass # jython try: from java.lang import Runtime runtime = Runtime.getRuntime() res = runtime.availableProcessors() if res > 0: return res except ImportError: pass # BSD try: sysctl = subprocess.Popen(['sysctl', '-n', 'hw.ncpu'], stdout=subprocess.PIPE) scStdout = sysctl.communicate()[0] res = int(scStdout) if res > 0: return res except (OSError, ValueError): pass # Linux try: res = open('/proc/cpuinfo').read().count('processor\t:') if res > 0: return res except IOError: pass # Solaris try: pseudoDevices = os.listdir('/devices/pseudo/') expr = re.compile('^cpuid@[0-9]+$') res = 0 for pd in pseudoDevices: if expr.match(pd) != None: res += 1 if res > 0: return res except OSError: pass # Other UNIXes (heuristic) try: try: dmesg = open('/var/run/dmesg.boot').read() except IOError: dmesgProcess = subprocess.Popen(['dmesg'], stdout=subprocess.PIPE) dmesg = dmesgProcess.communicate()[0] res = 0 while '\ncpu' + str(res) + ':' in dmesg: res += 1 if res > 0: return res except OSError: pass if debug: raise Exception('Can not determine number of CPUs on this system') else: return 1
def parser_code(): parser = argparse.ArgumentParser(description="This program will be used to remove spurious results from a BLAST search organized by gene block.") parser.add_argument("-i", "--infolder", dest="infolder", default='./blast_parse_1/', metavar="DIRECTORY", help="A folder that contains the gene block BLAST results.") parser.add_argument("-o", "--outfolder", dest="outfolder", metavar="DIRECTORY", default='./optimized_gene_block_1/', help="Folder where the filtered results will be stored. Default is the folder './optimized_gene_block_1/'.") parser.add_argument("-f", "--filter", dest="filter", default='', metavar="FILE", help="A file that contains the gene blocks that are under investigation. All others will be omitted from analysis an results.") parser.add_argument("-n", "--num_proc", dest="num_proc", metavar="INT", default = os.sysconf("SC_NPROCESSORS_CONF"), type=int, help="Number of processors that you want this script to run on. The default is every CPU that the system has.") parser.add_argument("-e", "--eval", dest="eval", default='1e-10', metavar="FLOAT", type=float, help="Use this option to change the eval for the BLAST search that is permitted. Useful if you would like to investigate what altering the eval threshold will do to your results.") parser.add_argument("-g", "--max_gap", dest="max_gap", metavar="INT", default = 500, type=int, help="Largest allowable gap to be considered a gene block by the analysis.") parser.add_argument("-q", "--quiet", dest="quiet", action="store_true", default=False, help="Suppresses most program text outputs.") return parser.parse_args()
sys.exit(1) if not try_compile(compiler=args.cxx, source='''\ #include <boost/version.hpp> #if BOOST_VERSION < 105500 #error Boost version too low #endif '''): print('Installed boost version too old. Please update {}.'.format(pkgname("boost-devel"))) sys.exit(1) defines = ' '.join(['-D' + d for d in defines]) globals().update(vars(args)) total_memory = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') link_pool_depth = max(int(total_memory / 7e9), 1) build_modes = modes if args.mode == 'all' else [args.mode] build_artifacts = all_artifacts if not args.artifacts else args.artifacts status = subprocess.call("./SCYLLA-VERSION-GEN") if status != 0: print('Version file generation failed') sys.exit(1) file = open('build/SCYLLA-VERSION-FILE', 'r') scylla_version = file.read().strip() file = open('build/SCYLLA-RELEASE-FILE', 'r') scylla_release = file.read().strip()
def total_count(): """ Return Number of Total cpus in the system including offline cpus """ return os.sysconf('SC_NPROCESSORS_CONF')
all_epochs = {10, 15, 20, 50} # , 100, 500} all_batchsize = {64, 128, 256, 512} # , 1024, 2048} all_mu = {0} all_sigma = {0.1, 0.01, 0.001} all_conv1depth = {20, 60, 200} all_conv2depth = {100, 500} all_conv3depth = {100, 500} all_fc1size = {100, 200, 400, 1000} all_fc2size = {84, 116, 200, 500} all_conv1keepprob = {0.8, 0.9, 0.95, 1.0} all_conv2keepprob = {0.8, 0.9, 0.95, 1.0} all_conv3keepprob = {0.8, 0.9, 0.95, 1.0} all_learningrate = {0.001} # {0.01, 0.001, 0.0001} Other learning rates didn't work well. numcores = ncpus = os.sysconf("SC_NPROCESSORS_ONLN") combo = list(product(all_epochs, all_batchsize, all_mu, all_sigma, all_conv1depth, all_conv2depth, all_conv3depth, all_fc1size, all_fc2size, all_conv1keepprob, all_conv2keepprob, all_conv3keepprob, all_learningrate)) shuffle(combo) def busy_or_not(): if (os.getloadavg()[0] / numcores > 0.9): return 1 else: return 0 def running_or_not(): proc = subprocess.Popen("nvidia-smi", stdout=subprocess.PIPE) output = proc.stdout.read() output = "".join(map(chr, output)) if output.find('No running processes found') == -1:
# # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. """Various utility functions, and a utility class.""" from __future__ import absolute_import from __future__ import unicode_literals from dnf.cli.format import format_number from dnf.i18n import _ import logging import os import time _USER_HZ = os.sysconf(os.sysconf_names['SC_CLK_TCK']) logger = logging.getLogger('dnf') def jiffies_to_seconds(jiffies): """Convert a number of jiffies to seconds. How many jiffies are in a second is system-dependent, e.g. 100 jiffies = 1 second is common. :param jiffies: a number of jiffies :return: the equivalent number of seconds """ return int(jiffies) / _USER_HZ def seconds_to_ui_time(seconds): """Return a human-readable string representation of the length of a time interval given in seconds.
"""Memory watchdog: periodically read the memory usage of the main test process and print it out, until terminated.""" import os import sys import time try: page_size = os.sysconf('SC_PAGESIZE') except (ValueError, AttributeError): try: page_size = os.sysconf('SC_PAGE_SIZE') except (ValueError, AttributeError): page_size = 4096 while True: sys.stdin.seek(0) statm = sys.stdin.read() data = int(statm.split()[5]) sys.stdout.write(' ... process data size: {data:.1f}G\n'.format(data= data * page_size / 1024 ** 3)) sys.stdout.flush() time.sleep(1)