def get_n_splits(ctrl, n_mc,n_meas):
import os, math
import numpy as np
if not(ctrl.isfield('n_para')):
ctrl.n_para = 1 # no paralell computing
mem_bytes = float(os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')) / 2
n_split = int(math.ceil((8 * n_mc * n_meas *3 * ctrl.n_para) / mem_bytes));
n_split =3
if n_split > 1:
n_tmp = math.floor(n_meas/n_split)
split_start = np.zeros([n_split])
split_end = np.ones([n_split]) * n_tmp
s_part = np.ones([n_split]) * n_tmp
for t in xrange(1,n_split):
split_start[t] = split_end[t-1]
if t == n_split-1:
split_end[t] = n_meas
else:
split_end[t] = split_end[t-1] + n_tmp
else:
split_start = np.zeros(1);
split_end = np.ones(1) * n_meas;
s_part = np.ones(1) * n_meas;
if ctrl.isSetTrue('debug'):
print split_start
print split_end
print s_part
return (n_split,split_start,split_end,s_part)
def check_options(parsed_args):
# section of code that checks the infolder entry
if os.path.isdir(parsed_args.infolder):
infolder = parsed_args.infolder
else:
print "The folder %s does not exist." % parsed_args.infolder
sys.exit()
# if the directory that the user specifies does not exist, then the program makes it for them.
if not os.path.isdir(parsed_args.outfolder):
os.makedirs(parsed_args.outfolder)
outfolder = parsed_args.outfolder
if outfolder[-1] != '/':
outfolder = outfolder + '/'
# Check the filter file
if parsed_args.filter == 'NONE' or os.path.exists(parsed_args.filter):
filter_file = parsed_args.filter
else:
print "The file %s does not exist." % parsed_args.filter
sys.exit()
# section of code that deals determining the number of CPU cores that will be used by the program
if parsed_args.num_proc > os.sysconf("SC_NPROCESSORS_CONF"):
num_proc = os.sysconf("SC_NPROCESSORS_CONF")
elif parsed_args.num_proc < 1:
num_proc = 1
else:
num_proc = int(parsed_args.num_proc)
do_protein = parsed_args.protein
return infolder, outfolder, filter_file, num_proc, do_protein
def check_options(parsed_args):
# section of code that deals determining the number of CPU cores that will be used by the program
if parsed_args.num_proc > os.sysconf("SC_NPROCESSORS_CONF"):
num_proc = os.sysconf("SC_NPROCESSORS_CONF")
elif parsed_args.num_proc < 1:
num_proc = 1
else:
num_proc = parsed_args.num_proc
if os.path.exists(parsed_args.infile):
infile = parsed_args.infile
else:
print "The file %s does not exist." % parsed_args.infile
sys.exit()
if os.path.exists(parsed_args.query):
query = parsed_args.query
else:
print "The file %s does not exist." % parsed_args.query
sys.exit()
# if the directory that the user specifies does not exist, then the program makes it for them.
folder = parsed_args.folder
if not os.path.isdir(parsed_args.folder):
os.makedirs(parsed_args.folder)
# should offer some sanity here for the input, but it is missing right now. call it a to do item
e_val = parsed_args.eval
return infile, query, folder, num_proc, e_val
def ctl_socket_handler_stats(self, data, fd):
if not 'stats' in self.__dict__:
self.stats = { "start_up_timestamp": self.start_up_timestamp }
self.stats_last_update = 0
self.ram_usage = 0
if 'SC_PAGESIZE' in os.sysconf_names:
self.stats_pagesize = os.sysconf('SC_PAGESIZE')
elif 'SC_PAGE_SIZE' in os.sysconf_names:
self.stats_pagesize = os.sysconf('SC_PAGE_SIZE')
elif '_SC_PAGESIZE' in os.sysconf_names:
self.stats_pagesize = os.sysconf('_SC_PAGESIZE')
else:
self.stats_pagesize = 4096
self.stats["infeeds"] = sum(1 for x in self.feeds if x.startswith('infeed-'))
self.stats["outfeeds"] = sum(1 for x in self.feeds if x.startswith('outfeed-'))
self.stats["plugins"] = len(self.plugins)
if time.time() - self.stats_last_update > 5:
if self.stats_ramfile != None:
self.stats_ramfile.seek(0)
self.ram_usage = int(self.stats_ramfile.read().split(' ')[1]) * self.stats_pagesize
st = os.statvfs(os.getcwd())
self.stats["groups"] = os.stat('groups').st_nlink - 2
self.stats["articles"] = sum(1 for x in os.listdir('articles')) - os.stat('articles').st_nlink + 2
self.stats["cpu"] = 15
self.stats["ram"] = self.ram_usage
self.stats["disk_free"] = st.f_bavail * st.f_frsize
self.stats["disk_used"] = (st.f_blocks - st.f_bfree) * st.f_frsize
self.stats_last_update = time.time()
return self.stats
def run(setupconfig):
yield "1,installing requiered software"
os.system("pacman -Sy reflector --noconfirm --needed 2> /dev/null > /dev/null")
yield "10,fetching mirrorlist"
# get the 50 >most up to date< servers and sort them by download speed
os.system("reflector --verbose -l 50 -p http --sort rate --save list.txt 2>/dev/null > /dev/null")
yield "33,partitioning disk"
# Calculate Partition layout:
#
# /dev/sdx1 = data = TOTALSIZE - SIZE OF RAM
# /dev/sdx2 = swap = SIZE OF RAM
# This should work for most cases
cmd = str("blockdev --getsize64 /dev/" + setupconfig.disk)
size = int(subprocess.check_output(cmd, shell=True).decode())
ramsize = int(os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")) / 1000 / 1000
data = (size / 1000 / 1000) - ramsize
Disks().part_disk(setupconfig, data)
yield "60,creating filesystems"
# Create FS
Disks().makefs(setupconfig)
yield "80,mounting disk"
# mount disk
Disks().mount(setupconfig)
yield "100,done!"
def get_php_fpm_memory(php_fpm_total_ram_slice=0.4):
# Taken from http://stackoverflow.com/a/28161352
mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') # e.g. 4015976448
mem_megs = mem_bytes/(1024.**2) # e.g. 3.74
LOGGER.info("Total memory available in the system: %sM", '%d' % mem_megs)
return mem_megs * php_fpm_total_ram_slice
def cpu_count():
try:
return multiprocessing.cpu_count()
except NotImplementedError:
pass
count = int(os.environ.get('NUMBER_OF_PROCESSORS', 0))
if count > 0:
return count
try:
if 'SC_NPROCESSORS_ONLN' in os.sysconf_names:
count = os.sysconf('SC_NPROCESSORS_ONLN')
elif 'SC_NPROCESSORS_CONF' in os.sysconf_names:
count = os.sysconf('SC_NPROCESSORS_CONF')
if count > 0:
return cpu_count
except AttributeError:
pass
count = 1 # unable to detect number of CPUs
return count
def get_system_memory():
'''Attempt to determine total system memory
If amount cannot be determined, emits warning and
returns -1.
'''
# MacOS X doesn't support sysconf('SC_PHYS_PAGES')
if platform.system() == 'Darwin':
try:
out = subprocess.check_output(['sysctl', 'hw.memsize'],
universal_newlines=True)
except subprocess.CalledProcessError as exc:
log.warning('Cannot determine system memory, sysctl failed with %s',
exc.output)
return -1
# output of sysctl is 'hw.memsize: #'. Strip the prefix.
hit = re.match(r'^hw.memsize: ([0-9]+)$', out)
if not hit:
log.warning('Cannot determine system memory, unable to parse sysctl output.')
return -1
return int(hit.group(1))
else:
try:
return os.sysconf('SC_PHYS_PAGES') * os.sysconf('SC_PAGESIZE')
except ValueError:
log.warning('Unable to determine number of CPU cores (sysconf failed).')
return -1
def __init__(self, name):
"""Initialization of procparser object.
Takes one argument name for the parser. Initializes the name,
pagesize from system, hertz from the system, proclist, procdata
and data parsed from the procps subsystem of Linux.
"""
self.name = name
self.statfld = ['pid', 'comm', 'state', 'ppid', 'pgrp',
'session', 'tty_nr', 'tpgid', 'flags',
'minflt', 'cminflt', 'majflt', 'cmajflt',
'utime', 'stime', 'cutime', 'cstime',
'priority', 'nice', 'num_threads',
'itrealvalue', 'starttime', 'vsize', 'rss',
'rsslim', 'startcode', 'endcode',
'startstack', 'kstkesp', 'kstkeip', 'signal',
'blocked', 'sigignore', 'sigcatch', 'wchan',
'nswap', 'cnswap', 'exit_signal', 'processor',
'rt_priority', 'policy',
'delayacct_blkio_ticks', 'guest_time',
'cguest_time']
# Get the pagesize from the system.
self.pagesize = os.sysconf(os.sysconf_names['SC_PAGE_SIZE'])
# Get the ticks/sec from the system.
self.hertz = os.sysconf(os.sysconf_names['SC_CLK_TCK'])
self.proclist = None
self.procdata = None
for field in self.fields:
self.data[field] = None
def determine_threads(options):
'''Return optimum number of upload threads'''
cores = os.sysconf('SC_NPROCESSORS_ONLN')
memory = os.sysconf('SC_PHYS_PAGES') * os.sysconf('SC_PAGESIZE')
if options.compress == 'lzma':
# Keep this in sync with compression level in backends/common.py
# Memory usage according to man xz(1)
mem_per_thread = 186 * 1024 ** 2
else:
# Only check LZMA memory usage
mem_per_thread = 0
if cores == -1 or memory == -1:
log.warn("Can't determine number of cores, using 2 upload threads.")
return 1
elif 2 * cores * mem_per_thread > (memory / 2):
threads = min(int((memory / 2) // mem_per_thread), 10)
if threads > 0:
log.info('Using %d upload threads (memory limited).', threads)
else:
log.warn('Warning: compression will require %d MiB memory '
'(%d%% of total system memory', mem_per_thread / 1024 ** 2,
mem_per_thread * 100 / memory)
threads = 1
return threads
else:
threads = min(2 * cores, 10)
log.info("Using %d upload threads.", threads)
return threads
def calculate_memory_for_sort():
"""Calculate available memory for ``samtools sort`` function.
If there is enough memory, no temp files are created. **Enough**
is defined as at least 1G per CPU.
Returns
-------
sort_memory: str or None
String to use directly with *-m* option in sort, or None.
"""
avail_memory = None
if HAVE_PSUTIL == True:
mem = virtual_memory()
avail_memory = mem.total
else:
try:
avail_memory = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') # e.g. 4015976448
except ValueError:
logging.error("If you're in Mac OS you need to have the psutil Python library.")
raise SystemExit
avail_cpu = multiprocessing.cpu_count()
sort_memory = avail_memory / avail_cpu / 1024 ** 2
# samtools default from documentation is 768M, to be conservative
# only use -m option when there is more than 1G per CPU.
if sort_memory < 1024:
sort_memory = None
else:
sort_memory = "%sG" % (int(floor(sort_memory / 1024)))
return sort_memory
def jobs(self):
"""
Find the amount of cpu cores to set the default amount of tasks executed in parallel. At
runtime the options can be obtained from :py:const:`waflib.Options.options` ::
from waflib.Options import options
njobs = options.jobs
:return: the amount of cpu cores
:rtype: int
"""
count = int(os.environ.get('JOBS', 0))
if count < 1:
if 'NUMBER_OF_PROCESSORS' in os.environ:
# on Windows, use the NUMBER_OF_PROCESSORS environment variable
count = int(os.environ.get('NUMBER_OF_PROCESSORS', 1))
else:
# on everything else, first try the POSIX sysconf values
if hasattr(os, 'sysconf_names'):
if 'SC_NPROCESSORS_ONLN' in os.sysconf_names:
count = int(os.sysconf('SC_NPROCESSORS_ONLN'))
elif 'SC_NPROCESSORS_CONF' in os.sysconf_names:
count = int(os.sysconf('SC_NPROCESSORS_CONF'))
elif os.name not in ('nt', 'java'):
tmp = self.cmd_and_log(['sysctl', '-n', 'hw.ncpu'])
if re.match('^[0-9]+$', tmp):
count = int(tmp)
if count < 1:
count = 1
elif count > 1024:
count = 1024
return count
def parallelCCompile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None):
# those lines are copied from distutils.ccompiler.CCompiler directly
macros, objects, extra_postargs, pp_opts, build = self._setup_compile(output_dir, macros, include_dirs, sources, depends, extra_postargs)
cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
# parallel code
N = 2*multiprocessing.cpu_count()# number of parallel compilations
try:
# On Unix-like platforms attempt to obtain the total memory in the
# machine and limit the number of parallel jobs to the number of Gbs
# of RAM (to avoid killing smaller platforms like the Pi)
mem = os.sysconf('SC_PHYS_PAGES') * os.sysconf('SC_PAGE_SIZE') # bytes
except (AttributeError, ValueError):
# Couldn't query RAM; don't limit parallelism (it's probably a well
# equipped Windows / Mac OS X box)
pass
else:
mem = max(1, int(round(mem / 1024 ** 3))) # convert to Gb
N = min(mem, N)
def _single_compile(obj):
try: src, ext = build[obj]
except KeyError: return
newcc_args = cc_args
if _platform == "darwin":
if src.endswith('.cpp'):
newcc_args = cc_args + ["-stdlib=libc++"]
self._compile(obj, src, ext, newcc_args, extra_postargs, pp_opts)
# convert to list, imap is evaluated on-demand
pool = multiprocessing.pool.ThreadPool(N)
list(pool.imap(_single_compile,objects))
return objects
def jobs(self):
"""
Amount of threads to use
"""
count = int(os.environ.get('JOBS', 0))
if count < 1:
if sys.platform == 'win32':
# on Windows, use the NUMBER_OF_PROCESSORS environment variable
count = int(os.environ.get('NUMBER_OF_PROCESSORS', 1))
else:
# on everything else, first try the POSIX sysconf values
if hasattr(os, 'sysconf_names'):
if 'SC_NPROCESSORS_ONLN' in os.sysconf_names:
count = int(os.sysconf('SC_NPROCESSORS_ONLN'))
elif 'SC_NPROCESSORS_CONF' in os.sysconf_names:
count = int(os.sysconf('SC_NPROCESSORS_CONF'))
elif os.name != 'java':
tmp = self.cmd_and_log(['sysctl', '-n', 'hw.ncpu'])
if re.match('^[0-9]+$', tmp):
count = int(tmp)
if count < 1:
count = 1
elif count > 1024:
count = 1024
return count
def check_options(parsed_args):
# I'm not issuing a message that hey, this file is there and will be overwritten, the program will just overwrite.
# TODO: check that the folder that this is saved in exists
if os.path.isdir(parsed_args.infolder):
infolder = parsed_args.infolder
else:
print "The infolder %s does not exist." % parsed_args.infolder
sys.exit()
outfile = parsed_args.outfile
# Check the gene block file
if os.path.exists(parsed_args.gene_block_file):
gene_block_file = parsed_args.gene_block_file
else:
print "The gene_block file %s does not exist." % parsed_args.gene_block_file
sys.exit()
# section of code that deals determining the number of CPU cores that will be used by the program
if parsed_args.num_proc > os.sysconf("SC_NPROCESSORS_CONF"):
num_proc = os.sysconf("SC_NPROCESSORS_CONF")
elif parsed_args.num_proc < 1:
num_proc = 1
else:
num_proc = int(parsed_args.num_proc)
# I am not checking this option here, just leave it as default for the time being.
refrence_list = [parsed_args.refrence]
quiet = parsed_args.quiet
return infolder, outfile, gene_block_file, num_proc, refrence_list, quiet
def __init__(self, log_d):
self.log_d = log_d
self.mem_gib = (os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') ) / (1024.**3)
# self.ncpu = multiprocessing.cpu_count() #Using this will parameter will likely run out of memory and swap
self.ncpu = int(self.mem_gib // 16) # Allocate 16 GB ram per concurrent instance. If not enough ram, start only 1 instance
if (self.ncpu == 0):
self.ncpu = 1
self.parser = Parser(self.log_d)
def get_max_redis_memory():
total_virtual_mem = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')/(pow(1024, 2))
max_memory = int(total_virtual_mem * 0.05) # Max memory for redis is 5% of virtual memory
if max_memory < 50:
return 50
else:
return max_memory
def get_system_info():
"""Return system hz use SC_CLK_TCK."""
ncpus = float(os.sysconf(os.sysconf_names['SC_NPROCESSORS_ONLN']))
ticks = float(os.sysconf(os.sysconf_names['SC_CLK_TCK']))
if ticks == -1:
ticks = 100
return (ticks, ncpus)
def get_total_memory(self):
"""
Get machine total memory.
"""
try:
return os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE")
except:
return None
def virtual_memory():
# we could have done this with kstat, but imho this is good enough
total = os.sysconf('SC_PHYS_PAGES') * PAGE_SIZE
# note: there's no difference on Solaris
free = avail = os.sysconf('SC_AVPHYS_PAGES') * PAGE_SIZE
used = total - free
percent = usage_percent(used, total, _round=1)
return svmem(total, avail, percent, used, free)
def fetch_ram():
lob.output_y("Ram")
mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
mem_gib = mem_bytes/(1024.**3)
print("{} G".format(mem_gib))
r.hset(mac, "Ram", mem_gib)
def __init__(self, pid=None, window=0.3):
self.CLOCKTICK = os.sysconf('SC_CLK_TCK')
self.PAGESIZE = os.sysconf('SC_PAGESIZE')
self.window = window
if not pid:
self.pid = str(os.getpid())
else:
self.pid = str(pid)
self.allconns = self._ConnTable()
def is_s5orig():
# returns True if the TS is running on the initially released S5 sequencer
if is_TsVm():
try:
memTotalGb = os.sysconf('SC_PAGE_SIZE')*os.sysconf('SC_PHYS_PAGES')/(1024*1024*1024)
return memTotalGb < 70
except:
logger.error("Unable to determine system memory")
return False
def check_mem():
""" Check host has more than 10GB memory """
mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
LOG.debug('Memory: %s' % str(mem_bytes))
if mem_bytes < 10737418240:
LOG.error('Not enough memory: %d' % mem_bytes)
else:
LOG.info('Memory ok: %d' % mem_bytes)
def getProcessData(pid):
try:
items = open("/proc/%d/stat" % pid).read().split()
return { "cpu": formatCPU(float(int(items[13]) + int(items[14])) / os.sysconf("SC_CLK_TCK")),
"rss": formatRSS(int(items[23]) * os.sysconf("SC_PAGE_SIZE")) }
except:
return { "cpu": "N/A",
"rss": "N/A" }
def get_hypervisor_memory(reserved):
"""Get physical available memory for hypervisor in MiB -> float
arguments:
reserved - MiB reserved for OS
"""
memory_bytes = sysconf('SC_PAGE_SIZE') * sysconf('SC_PHYS_PAGES')
return float(memory_bytes) / 1024.0**2.0 - reserved
def getpagesize():
if _getpagesize:
return _getpagesize()
else:
try:
return sysconf("SC_PAGE_SIZE")
except ValueError:
# Irix 5.3 has _SC_PAGESIZE, but not _SC_PAGE_SIZE
return sysconf("SC_PAGESIZE")
def memory_bytes(self):
"""
Total installed memory of this RunManager
"""
try:
return os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
except ValueError:
# Fallback to sysctl when os.sysconf('SC_PHYS_PAGES') fails on OS X
return int(check_output(['sysctl', '-n', 'hw.memsize']).strip())
def get_free_memory_os():
free_memory_Kb = 0
try:
free_memory_Kb = (os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_AVPHYS_PAGES')) / 1024.0
print 'The strict free memory available determined is ' + str(round((free_memory_Kb / (1024.0 ** 2)), 1)) + ' Gb'
except Exception as e:
print e
print 'WARNING: it was not possible to determine the free memory available!'
return free_memory_Kb
def virtualMemory(self):
"""
"""
PAGE_SIZE = os.sysconf('SC_PAGE_SIZE')
total = os.sysconf('SC_PHYS_PAGES') * PAGE_SIZE
free = avail = os.sysconf('SC_AVPHYS_PAGES') * PAGE_SIZE
used = total - free
percent = self.usagePercent(used, total)
svmem = namedtuple('svmem', ['total', 'available', 'percent', 'used', 'free'])
return svmem(total, avail, percent, used, free)
import stem.util.connection
import stem.util.enum
import stem.util.str_tools
from stem.util import log
try:
# unavailable on windows (#19823)
import pwd
IS_PWD_AVAILABLE = True
except ImportError:
IS_PWD_AVAILABLE = False
# os.sysconf is only defined on unix
try:
CLOCK_TICKS = os.sysconf(os.sysconf_names['SC_CLK_TCK'])
except AttributeError:
CLOCK_TICKS = None
IS_LITTLE_ENDIAN = sys.byteorder == 'little'
ENCODED_ADDR = {} # cache of encoded ips to their decoded version
Stat = stem.util.enum.Enum(('COMMAND', 'command'), ('CPU_UTIME', 'utime'),
('CPU_STIME', 'stime'),
('START_TIME', 'start time'))
@functools.lru_cache()
def is_available():
"""
Checks if proc information is available on this platform.
def online_count():
"""
Return Number of Online cpus in the system
"""
return os.sysconf('SC_NPROCESSORS_ONLN')
# uncompyle6 version 3.2.4
# Python bytecode 2.7 (62211)
# Decompiled from: Python 2.7.15 (v2.7.15:ca079a3ea3, Apr 30 2018, 16:30:26) [MSC v.1500 64 bit (AMD64)]
# Embedded file name: popen2
import os, sys, warnings
warnings.warn('The popen2 module is deprecated. Use the subprocess module.', DeprecationWarning, stacklevel=2)
__all__ = [
'popen2', 'popen3', 'popen4']
try:
MAXFD = os.sysconf('SC_OPEN_MAX')
except (AttributeError, ValueError):
MAXFD = 256
_active = []
def _cleanup():
for inst in _active[:]:
if inst.poll(_deadstate=sys.maxint) >= 0:
try:
_active.remove(inst)
except ValueError:
pass
class Popen3:
sts = -1
def __init__(self, cmd, capturestderr=False, bufsize=-1):
_cleanup()
self.cmd = cmd
p2cread, p2cwrite = os.pipe()
#!/usr/bin env python3
import subprocess
import os
import sys
import time
ITERTIME = 1
iter=0 # waiting iterations
jiffy = float(os.sysconf(os.sysconf_names['SC_CLK_TCK']))
cmdline = sys.argv[1:]
print(cmdline)
ncores = os.cpu_count()
nprocs = int(os.environ.get('OMP_NUM_THREADS', os.cpu_count()))
print("procs: {}\t cores: {}".format(nprocs, ncores))
proc = subprocess.Popen(cmdline)
tzero = time.clock_gettime(time.CLOCK_REALTIME)
print(proc.pid)
procdir = os.path.join("/proc", str(proc.pid))
# utime for a threadcd
tutlist = {}
corelist = [0.0]*nprocs
corearr = [0.0]*ncores
with open('/proc/uptime', 'r') as f:
def memAvail(p=0.8):
mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_AVPHYS_PAGES') # e.g. 4015976448
mem_gib = mem_bytes/(1024.**3) # e.g. 3.74
return int(mem_gib*p)
def get_cpu_tick_per_second():
return os.sysconf('SC_CLK_TCK')
def test_os_sysconf(self):
os = self.posix
assert os.sysconf(self.sysconf_value) == self.sysconf_result
assert os.sysconf(self.sysconf_name) == self.sysconf_result
assert os.sysconf_names[self.sysconf_name] == self.sysconf_value
class ForkAwareLocal(threading.local):
def __init__(self):
register_after_fork(self, lambda obj: obj.__dict__.clear())
def __reduce__(self):
return type(self), ()
#
# Close fds except those specified
#
try:
MAXFD = os.sysconf("SC_OPEN_MAX")
except Exception:
MAXFD = 256
def close_all_fds_except(fds):
fds = list(fds) + [-1, MAXFD]
fds.sort()
assert fds[-1] == MAXFD, 'fd too large'
for i in range(len(fds) - 1):
os.closerange(fds[i] + 1, fds[i + 1])
#
# Close sys.stdin and replace stdin with os.devnull
#
import os
import time
import socket
from resource import getrusage, RUSAGE_SELF
from twisted.application.service import Service
from twisted.internet.task import LoopingCall
from carbon.conf import settings
stats = {}
HOSTNAME = socket.gethostname().replace('.', '_')
PAGESIZE = os.sysconf('SC_PAGESIZE')
rusage = getrusage(RUSAGE_SELF)
lastUsage = rusage.ru_utime + rusage.ru_stime
lastUsageTime = time.time()
# NOTE: Referencing settings in this *top level scope* will
# give you *defaults* only. Probably not what you wanted.
# TODO(chrismd) refactor the graphite metrics hierarchy to be cleaner,
# more consistent, and make room for frontend metrics.
#metric_prefix = "Graphite.backend.%(program)s.%(instance)s." % settings
def increment(stat, increase=1):
try:
stats[stat] += increase
except KeyError:
stats[stat] = increase
def get_workers():
procs = os.sysconf('SC_NPROCESSORS_ONLN')
if procs > 0:
return procs * 2 + 1
else:
return 3
def get_total_mem(self):
# Get total memory in bytes and divide by 1024**2 to get the value in MB.
return os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') / (1024
**2)
def online_cpus():
return frozenset(range(os.sysconf('SC_NPROCESSORS_ONLN')))
def numCPUs():
if not hasattr(os, "sysconf"):
raise RuntimeError("No sysconf detected.")
return os.sysconf("SC_NPROCESSORS_ONLN")
line = file_.readline()
if line == b"":
break
if not line.endswith(b"\n"):
file_.seek(fpos)
break
yield line[:-1]
def _parse_send(line):
message_id, send_time = line.split(b",", 1)
send_time = int(send_time)
return message_id, send_time
def _parse_receive(line):
message_id, send_time, receive_time = line.split(b",", 2)
send_time = int(send_time)
receive_time = int(receive_time)
return message_id, send_time, receive_time
_join = _plano.join
_ticks_per_ms = _os.sysconf(_os.sysconf_names["SC_CLK_TCK"]) / 1000
_page_size = _resource.getpagesize()
GCC = 'gcc'
CPP = os.environ.get('CPP', 'cpp')
try:
CPUCOUNT = int(os.environ.get('CPUCOUNT', -1))
except:
CPUCOUNT = -1
if CPUCOUNT < 1:
try:
import multiprocessing
CPUCOUNT = multiprocessing.cpu_count()
except:
try:
CPUCOUNT = int(os.sysconf('SC_NPROCESSORS_ONLN'))
except:
CPUCOUNT = 1
binary_list = []
# this is used for reporting (at the end of the build)
# the server configuration
report = {}
report['kernel'] = False
report['execinfo'] = False
report['ifaddrs'] = False
report['locking'] = False
report['event'] = False
report['timer'] = False
report['filemonitor'] = False
import errno
import functools
import os
from collections import namedtuple
from psutil import _common
from psutil import _psposix
from psutil._common import conn_tmap, usage_percent, isfile_strict
import _psutil_osx as cext
import _psutil_posix
__extra__all__ = []
# --- constants
PAGESIZE = os.sysconf("SC_PAGE_SIZE")
# http://students.mimuw.edu.pl/lxr/source/include/net/tcp_states.h
TCP_STATUSES = {
cext.TCPS_ESTABLISHED: _common.CONN_ESTABLISHED,
cext.TCPS_SYN_SENT: _common.CONN_SYN_SENT,
cext.TCPS_SYN_RECEIVED: _common.CONN_SYN_RECV,
cext.TCPS_FIN_WAIT_1: _common.CONN_FIN_WAIT1,
cext.TCPS_FIN_WAIT_2: _common.CONN_FIN_WAIT2,
cext.TCPS_TIME_WAIT: _common.CONN_TIME_WAIT,
cext.TCPS_CLOSED: _common.CONN_CLOSE,
cext.TCPS_CLOSE_WAIT: _common.CONN_CLOSE_WAIT,
cext.TCPS_LAST_ACK: _common.CONN_LAST_ACK,
cext.TCPS_LISTEN: _common.CONN_LISTEN,
cext.TCPS_CLOSING: _common.CONN_CLOSING,
cext.PSUTIL_CONN_NONE: _common.CONN_NONE,
so_ext = '.dylib'
elif sys.platform.startswith('win'):
so_ext = '.dll'
else:
raise OSError('Unknown platform')
libname_so = libname + so_ext
return ctypes.CDLL(os.path.join(os.path.dirname(__file__), libname_so))
else:
_loaderpath = os.path.dirname(__file__)
return numpy.ctypeslib.load_library(libname, _loaderpath)
#Fixme, the standard resouce module gives wrong number when objects are released
#see http://fa.bianp.net/blog/2013/different-ways-to-get-memory-consumption-or-lessons-learned-from-memory_profiler/#fn:1
#or use slow functions as memory_profiler._get_memory did
CLOCK_TICKS = os.sysconf("SC_CLK_TCK")
PAGESIZE = os.sysconf("SC_PAGE_SIZE")
def current_memory():
'''Return the size of used memory and allocated virtual memory (in MB)'''
#import resource
#return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
if sys.platform.startswith('linux'):
with open("/proc/%s/statm" % os.getpid()) as f:
vms, rss = [int(x) * PAGESIZE for x in f.readline().split()[:2]]
return rss / 1e6, vms / 1e6
else:
return 0, 0
def cpu_count():
try:
import multiprocessing # pylint: disable=import-outside-toplevel
return multiprocessing.cpu_count()
except ImportError:
return int(os.sysconf('SC_NPROCESSORS_ONLN'))
def _get_cpu_nums():
return os.sysconf("SC_NPROCESSORS_ONLN")
sys.excepthook = sys.__excepthook__
if issubclass(etype, KeyboardInterrupt):
pass
elif issubclass(etype, IOError) and value.errno == errno.EPIPE:
pass
else:
sys.__excepthook__(etype, value, tb)
sys.excepthook = std_exceptions
#
# Define some global variables
#
PAGESIZE = os.sysconf("SC_PAGE_SIZE") / 1024 #KiB
our_pid = os.getpid()
have_pss = 0
have_swap_pss = 0
class Proc:
def __init__(self):
uname = os.uname()
if uname[0] == "FreeBSD":
self.proc = '/compat/linux/proc'
else:
self.proc = '/proc'
def path(self, *args):
object.__setattr__(self, key, value)
states = _StateEnum() # noqa: E305
states.STOPPED = states.State()
states.STARTING = states.State()
states.STARTED = states.State()
states.STOPPING = states.State()
states.EXITING = states.State()
try:
import fcntl
except ImportError:
max_files = 0
else:
try:
max_files = os.sysconf('SC_OPEN_MAX')
except AttributeError:
max_files = 1024
class Bus(object):
"""Process state-machine and messenger for HTTP site deployment.
All listeners for a given channel are guaranteed to be called even
if others at the same channel fail. Each failure is logged, but
execution proceeds on to the next listener. The only way to stop all
processing from inside a listener is to raise SystemExit and stop the
whole server.
"""
states = states
def number_of_cpus():
"""
Number of virtual or physical CPUs on this system, i.e.
user/real as output by time(1) when called with an optimally scaling userspace-only program
Return -1 if ncpus cannot be detected
taken from:
http://stackoverflow.com/questions/1006289/how-to-find-out-the-number-of-cpus-in-python
"""
import os, re, subprocess
# Python 2.6+
try:
import multiprocessing
return multiprocessing.cpu_count()
except (ImportError, NotImplementedError):
pass
# POSIX
try:
res = int(os.sysconf('SC_NPROCESSORS_ONLN'))
if res > 0: return res
except (AttributeError, ValueError):
pass
# Windows
try:
res = int(os.environ['NUMBER_OF_PROCESSORS'])
if res > 0: return res
except (KeyError, ValueError):
pass
# jython
try:
from java.lang import Runtime
runtime = Runtime.getRuntime()
res = runtime.availableProcessors()
if res > 0: return res
except ImportError:
pass
# BSD
try:
sysctl = subprocess.Popen(['sysctl', '-n', 'hw.ncpu'],
stdout=subprocess.PIPE)
scStdout = sysctl.communicate()[0]
res = int(scStdout)
if res > 0: return res
except (OSError, ValueError):
pass
# Linux
try:
res = open('/proc/cpuinfo').read().count('processor\t:')
if res > 0: return res
except IOError:
pass
# Solaris
try:
pseudoDevices = os.listdir('/devices/pseudo/')
expr = re.compile('^cpuid@[0-9]+$')
res = 0
for pd in pseudoDevices:
if expr.match(pd) is not None:
res += 1
if res > 0: return res
except OSError:
pass
# Other UNIXes (heuristic)
try:
try:
dmesg = open('/var/run/dmesg.boot').read()
except IOError:
dmesgProcess = subprocess.Popen(['dmesg'], stdout=subprocess.PIPE)
dmesg = dmesgProcess.communicate()[0]
res = 0
while '\ncpu' + str(res) + ':' in dmesg:
res += 1
if res > 0: return res
except OSError:
pass
return -1
def __get_page_size():
try:
return os.sysconf("SC_PAGE_SIZE") # FIX_ME
except Exception:
return PSEUDO_PAGE_SIZE
def determineNumberOfCPUs(debug=False):
""" Number of virtual or physical CPUs on this system, i.e.
user/real as output by time(1) when called with an optimally scaling userspace-only program"""
# Python 2.6+
try:
import multiprocessing
return multiprocessing.cpu_count()
except (ImportError,NotImplementedError):
pass
# POSIX
try:
res = int(os.sysconf('SC_NPROCESSORS_ONLN'))
if res > 0:
return res
except (AttributeError,ValueError):
pass
# Windows
try:
res = int(os.environ['NUMBER_OF_PROCESSORS'])
if res > 0:
return res
except (KeyError, ValueError):
pass
# jython
try:
from java.lang import Runtime
runtime = Runtime.getRuntime()
res = runtime.availableProcessors()
if res > 0:
return res
except ImportError:
pass
# BSD
try:
sysctl = subprocess.Popen(['sysctl', '-n', 'hw.ncpu'], stdout=subprocess.PIPE)
scStdout = sysctl.communicate()[0]
res = int(scStdout)
if res > 0:
return res
except (OSError, ValueError):
pass
# Linux
try:
res = open('/proc/cpuinfo').read().count('processor\t:')
if res > 0:
return res
except IOError:
pass
# Solaris
try:
pseudoDevices = os.listdir('/devices/pseudo/')
expr = re.compile('^cpuid@[0-9]+$')
res = 0
for pd in pseudoDevices:
if expr.match(pd) != None:
res += 1
if res > 0:
return res
except OSError:
pass
# Other UNIXes (heuristic)
try:
try:
dmesg = open('/var/run/dmesg.boot').read()
except IOError:
dmesgProcess = subprocess.Popen(['dmesg'], stdout=subprocess.PIPE)
dmesg = dmesgProcess.communicate()[0]
res = 0
while '\ncpu' + str(res) + ':' in dmesg:
res += 1
if res > 0:
return res
except OSError:
pass
if debug:
raise Exception('Can not determine number of CPUs on this system')
else:
return 1
def parser_code():
parser = argparse.ArgumentParser(description="This program will be used to remove spurious results from a BLAST search organized by gene block.")
parser.add_argument("-i", "--infolder", dest="infolder", default='./blast_parse_1/', metavar="DIRECTORY",
help="A folder that contains the gene block BLAST results.")
parser.add_argument("-o", "--outfolder", dest="outfolder", metavar="DIRECTORY", default='./optimized_gene_block_1/',
help="Folder where the filtered results will be stored. Default is the folder './optimized_gene_block_1/'.")
parser.add_argument("-f", "--filter", dest="filter", default='', metavar="FILE",
help="A file that contains the gene blocks that are under investigation. All others will be omitted from analysis an results.")
parser.add_argument("-n", "--num_proc", dest="num_proc", metavar="INT", default = os.sysconf("SC_NPROCESSORS_CONF"), type=int,
help="Number of processors that you want this script to run on. The default is every CPU that the system has.")
parser.add_argument("-e", "--eval", dest="eval", default='1e-10', metavar="FLOAT", type=float,
help="Use this option to change the eval for the BLAST search that is permitted. Useful if you would like to investigate what altering the eval threshold will do to your results.")
parser.add_argument("-g", "--max_gap", dest="max_gap", metavar="INT", default = 500, type=int,
help="Largest allowable gap to be considered a gene block by the analysis.")
parser.add_argument("-q", "--quiet", dest="quiet", action="store_true", default=False,
help="Suppresses most program text outputs.")
return parser.parse_args()
sys.exit(1)
if not try_compile(compiler=args.cxx, source='''\
#include <boost/version.hpp>
#if BOOST_VERSION < 105500
#error Boost version too low
#endif
'''):
print('Installed boost version too old. Please update {}.'.format(pkgname("boost-devel")))
sys.exit(1)
defines = ' '.join(['-D' + d for d in defines])
globals().update(vars(args))
total_memory = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
link_pool_depth = max(int(total_memory / 7e9), 1)
build_modes = modes if args.mode == 'all' else [args.mode]
build_artifacts = all_artifacts if not args.artifacts else args.artifacts
status = subprocess.call("./SCYLLA-VERSION-GEN")
if status != 0:
print('Version file generation failed')
sys.exit(1)
file = open('build/SCYLLA-VERSION-FILE', 'r')
scylla_version = file.read().strip()
file = open('build/SCYLLA-RELEASE-FILE', 'r')
scylla_release = file.read().strip()
def total_count():
"""
Return Number of Total cpus in the system including offline cpus
"""
return os.sysconf('SC_NPROCESSORS_CONF')
all_epochs = {10, 15, 20, 50} # , 100, 500}
all_batchsize = {64, 128, 256, 512} # , 1024, 2048}
all_mu = {0}
all_sigma = {0.1, 0.01, 0.001}
all_conv1depth = {20, 60, 200}
all_conv2depth = {100, 500}
all_conv3depth = {100, 500}
all_fc1size = {100, 200, 400, 1000}
all_fc2size = {84, 116, 200, 500}
all_conv1keepprob = {0.8, 0.9, 0.95, 1.0}
all_conv2keepprob = {0.8, 0.9, 0.95, 1.0}
all_conv3keepprob = {0.8, 0.9, 0.95, 1.0}
all_learningrate = {0.001} # {0.01, 0.001, 0.0001} Other learning rates didn't work well.
numcores = ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
combo = list(product(all_epochs, all_batchsize, all_mu, all_sigma, all_conv1depth, all_conv2depth, all_conv3depth, all_fc1size, all_fc2size, all_conv1keepprob, all_conv2keepprob, all_conv3keepprob, all_learningrate))
shuffle(combo)
def busy_or_not():
if (os.getloadavg()[0] / numcores > 0.9):
return 1
else:
return 0
def running_or_not():
proc = subprocess.Popen("nvidia-smi", stdout=subprocess.PIPE)
output = proc.stdout.read()
output = "".join(map(chr, output))
if output.find('No running processes found') == -1:
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
"""Various utility functions, and a utility class."""
from __future__ import absolute_import
from __future__ import unicode_literals
from dnf.cli.format import format_number
from dnf.i18n import _
import logging
import os
import time
_USER_HZ = os.sysconf(os.sysconf_names['SC_CLK_TCK'])
logger = logging.getLogger('dnf')
def jiffies_to_seconds(jiffies):
"""Convert a number of jiffies to seconds. How many jiffies are in a second
is system-dependent, e.g. 100 jiffies = 1 second is common.
:param jiffies: a number of jiffies
:return: the equivalent number of seconds
"""
return int(jiffies) / _USER_HZ
def seconds_to_ui_time(seconds):
"""Return a human-readable string representation of the length of
a time interval given in seconds.
"""Memory watchdog: periodically read the memory usage of the main test process
and print it out, until terminated."""
import os
import sys
import time
try:
page_size = os.sysconf('SC_PAGESIZE')
except (ValueError, AttributeError):
try:
page_size = os.sysconf('SC_PAGE_SIZE')
except (ValueError, AttributeError):
page_size = 4096
while True:
sys.stdin.seek(0)
statm = sys.stdin.read()
data = int(statm.split()[5])
sys.stdout.write(' ... process data size: {data:.1f}G\n'.format(data=
data * page_size / 1024 ** 3))
sys.stdout.flush()
time.sleep(1)