def raise_nofile(nofile_atleast: int = 4096) -> Tuple[int, int]:
"""
sets nofile soft limit to at least 4096, useful for running matlplotlib/seaborn on
parallel executing plot generators vs. Ubuntu 16.04 default ulimit -n 1024 or OS X El Captian 256
temporary setting extinguishing with Python session.
"""
# %% (0) what is current ulimit -n setting?
soft, ohard = res.getrlimit(res.RLIMIT_NOFILE)
hard = ohard
# %% (1) increase limit (soft and even hard) if needed
if soft < nofile_atleast:
soft = nofile_atleast
if hard < soft:
hard = soft
print('setting soft & hard ulimit -n {} {}'.format(soft, hard))
try:
res.setrlimit(res.RLIMIT_NOFILE, (soft, hard))
except (ValueError, res.error):
try:
hard = soft
print(
'trouble with max limit, retrying with soft,hard {},{}'.format(soft, hard))
res.setrlimit(res.RLIMIT_NOFILE, (soft, hard))
except Exception:
print('failed to set ulimit, giving up')
soft, hard = res.getrlimit(res.RLIMIT_NOFILE)
return soft, hard
def testClushConfigSetRlimit(self):
"""test CLI.Config.ClushConfig (setrlimit)"""
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
hard2 = min(32768, hard)
f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
f.write("""
[Main]
fanout: 42
connect_timeout: 14
command_timeout: 0
history_size: 100
color: auto
fd_max: %d
verbosity: 1
""" % hard2)
f.flush()
parser = OptionParser("dummy")
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options, filename=f.name)
self.assert_(config != None)
display = Display(options, config)
self.assert_(display != None)
# force a lower soft limit
resource.setrlimit(resource.RLIMIT_NOFILE, (hard2/2, hard))
# max_fdlimit should increase soft limit again
set_fdlimit(config.fd_max, display)
# verify
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
self.assertEqual(soft, hard2)
f.close()
def __init__(self, max_threads=None):
'''
If max_threads is not supplied, calculates a reasonable value based
on system resource limits.
'''
self.active_requests = set()
if not max_threads:
# man getrlimit: "RLIMIT_NPROC The maximum number of processes (or,
# more precisely on Linux, threads) that can be created for the
# real user ID of the calling process."
try:
import resource
rlimit_nproc = resource.getrlimit(resource.RLIMIT_NPROC)[0]
rlimit_nofile = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
max_threads = min(rlimit_nofile // 10, rlimit_nproc // 2)
# resource.RLIM_INFINITY == -1 which can result in max_threads == 0
if max_threads <= 0 or max_threads > 5000:
max_threads = 5000
self.logger.info(
"max_threads=%s (rlimit_nproc=%s, rlimit_nofile=%s)",
max_threads, rlimit_nproc, rlimit_nofile)
except Exception as e:
self.logger.warn(
"unable to calculate optimal number of threads based "
"on resource limits due to %s", e)
max_threads = 100
self.logger.info("max_threads=%s", max_threads)
self.max_threads = max_threads
self.pool = concurrent.futures.ThreadPoolExecutor(max_threads)
def testClushConfigSetRlimitValueError(self):
"""test CLI.Config.ClushConfig (setrlimit ValueError)"""
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
f.write(dedent("""
[Main]
fanout: 42
connect_timeout: 14
command_timeout: 0
history_size: 100
color: auto
# Use wrong fd_max value to generate ValueError
fd_max: -1
verbosity: 1"""))
f.flush()
parser = OptionParser("dummy")
parser.install_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options, filename=f.name)
f.close()
display = Display(options, config)
class TestException(Exception): pass
def mock_vprint_err(level, message):
if message.startswith('Warning: Failed to set max open files'):
raise TestException()
display.vprint_err = mock_vprint_err
self.assertRaises(TestException, set_fdlimit, config.fd_max, display)
soft2, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
self.assertEqual(soft, soft2)
def __set_max_open_files(self):
# Let's check to see how our max open files(ulimit -n) setting is
mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE)
log.info(
'Current values for max open files soft/hard setting: '
'{0}/{1}'.format(
mof_s, mof_h
)
)
# Let's grab, from the configuration file, the value to raise max open
# files to
mof_c = self.opts['max_open_files']
if mof_c > mof_h:
# The configured value is higher than what's allowed
log.warning(
'The value for the \'max_open_files\' setting, {0}, is higher '
'than what the user running salt is allowed to raise to, {1}. '
'Defaulting to {1}.'.format(mof_c, mof_h)
)
mof_c = mof_h
if mof_s < mof_c:
# There's room to raise the value. Raise it!
log.warning('Raising max open files value to {0}'.format(mof_c))
resource.setrlimit(resource.RLIMIT_NOFILE, (mof_c, mof_h))
mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE)
log.warning(
'New values for max open files soft/hard values: '
'{0}/{1}'.format(mof_s, mof_h)
)
def __init__(
self,
application,
environ=None,
bindAddress=None,
umask=None,
multiplexed=False,
debug=False,
roles=(FCGI_RESPONDER,),
forceCGI=False,
timeout=None,
**kw
):
"""
environ, if present, must be a dictionary-like object. Its
contents will be copied into application's environ. Useful
for passing application-specific variables.
bindAddress, if present, must either be a string or a 2-tuple. If
present, run() will open its own listening socket. You would use
this if you wanted to run your application as an 'external' FastCGI
app. (i.e. the webserver would no longer be responsible for starting
your app) If a string, it will be interpreted as a filename and a UNIX
socket will be opened. If a tuple, the first element, a string,
is the interface name/IP to bind to, and the second element (an int)
is the port number.
"""
BaseFCGIServer.__init__(
self,
application,
environ=environ,
multithreaded=False,
multiprocess=True,
bindAddress=bindAddress,
umask=umask,
multiplexed=multiplexed,
debug=debug,
roles=roles,
forceCGI=forceCGI,
)
for key in ("multithreaded", "multiprocess", "jobClass", "jobArgs"):
if kw.has_key(key):
del kw[key]
PreforkServer.__init__(self, jobClass=self._connectionClass, jobArgs=(self, timeout), **kw)
try:
import resource
# Attempt to glean the maximum number of connections
# from the OS.
try:
maxProcs = resource.getrlimit(resource.RLIMIT_NPROC)[0]
maxConns = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
maxConns = min(maxConns, maxProcs)
except AttributeError:
maxConns = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
except ImportError:
maxConns = 100 # Just some made up number.
maxReqs = maxConns
self.capability = {FCGI_MAX_CONNS: maxConns, FCGI_MAX_REQS: maxReqs, FCGI_MPXS_CONNS: 0}
def process_limit(self):
# If our parent changed sucide
if self.ppid != os.getppid():
_logger.info("Worker (%s) Parent changed", self.pid)
self.alive = False
# check for lifetime
if self.request_count >= self.request_max:
_logger.info("Worker (%d) max request (%s) reached.", self.pid, self.request_count)
self.alive = False
# Reset the worker if it consumes too much memory (e.g. caused by a memory leak).
rss, vms = memory_info(psutil.Process(os.getpid()))
if vms > config['limit_memory_soft']:
_logger.info('Worker (%d) virtual memory limit (%s) reached.', self.pid, vms)
self.alive = False # Commit suicide after the request.
# VMS and RLIMIT_AS are the same thing: virtual memory, a.k.a. address space
soft, hard = resource.getrlimit(resource.RLIMIT_AS)
resource.setrlimit(resource.RLIMIT_AS, (config['limit_memory_hard'], hard))
# SIGXCPU (exceeded CPU time) signal handler will raise an exception.
r = resource.getrusage(resource.RUSAGE_SELF)
cpu_time = r.ru_utime + r.ru_stime
def time_expired(n, stack):
_logger.info('Worker (%d) CPU time limit (%s) reached.', self.pid, config['limit_time_cpu'])
# We dont suicide in such case
raise Exception('CPU time limit exceeded.')
signal.signal(signal.SIGXCPU, time_expired)
soft, hard = resource.getrlimit(resource.RLIMIT_CPU)
resource.setrlimit(resource.RLIMIT_CPU, (cpu_time + config['limit_time_cpu'], hard))
def test_rlimits():
for k, v in sorted(resource.__dict__.items()):
if k.startswith('RLIMIT_'):
val = resource.getrlimit(v)
logging.info('%s: %s', k, val)
limits = resource.getrlimit(resource.RLIMIT_NOFILE)
maxfiles = min(limits[1], 8192)
logging.info('Trying to open %d files..', maxfiles)
if real_run():
i = 0
try:
# list is needed to keep files open (prevent GC)
handles = []
for i in range(maxfiles):
fd = open('/tmp/file-{}'.format(i), 'w')
fd.write('1')
handles.append(fd)
if i > 0 and i % 1000 == 0:
logging.debug('Opened %d files', i)
except IOError:
logging.exception('Could open %s files', i)
for i in range(maxfiles):
try:
os.unlink('/tmp/file-{}'.format(i))
except:
pass
def limit_resource(): megs=1000000 rsrc = resource.RLIMIT_AS soft, hard = resource.getrlimit(rsrc) print 'Soft limit starts as :', soft resource.setrlimit(rsrc, (8*megs*1024, 8*megs*1024)) #limit to 8 Gigabytes soft, hard = resource.getrlimit(rsrc) print 'Soft limit changed to :', soft
def run(configs, optimal=True, final_config=None, final_config_builder=None,
timeout=None):
options, extra_args = parse_args()
plan_file = options.plan_file
# Time limits are either positive values in seconds or -1 (unlimited).
soft_time_limit, hard_time_limit = resource.getrlimit(resource.RLIMIT_CPU)
print 'External time limit:', hard_time_limit
if (hard_time_limit >= 0 and timeout is not None and
timeout != hard_time_limit):
sys.stderr.write("The externally set timeout (%d) differs from the one "
"in the portfolio file (%d). Is this expected?\n" %
(hard_time_limit, timeout))
# Prefer limits in the order: externally set, from portfolio file, default.
if hard_time_limit >= 0:
timeout = hard_time_limit
elif timeout is None:
sys.stderr.write("No timeout has been set for the portfolio so we take "
"the default of %ds.\n" % DEFAULT_TIMEOUT)
timeout = DEFAULT_TIMEOUT
print 'Internal time limit:', timeout
# Memory limits are either positive values in Bytes or -1 (unlimited).
soft_mem_limit, hard_mem_limit = resource.getrlimit(resource.RLIMIT_AS)
print 'External memory limit:', hard_mem_limit
memory = hard_mem_limit - BYTES_FOR_PYTHON
# Do not limit memory if the previous limit was very low or unlimited.
if memory < 0:
memory = None
print 'Internal memory limit:', memory
assert len(extra_args) == 3, extra_args
sas_file = extra_args.pop(0)
assert extra_args[0] in ["unit", "nonunit"], extra_args
unitcost = extra_args.pop(0)
assert extra_args[0][-1] in ["1", "2", "4"], extra_args
planner = extra_args.pop(0)
safe_unlink("plan_numbers_and_cost")
remaining_time_at_start = float(timeout)
try:
for line in open("elapsed.time"):
if line.strip():
remaining_time_at_start -= float(line)
except EnvironmentError:
print "WARNING! elapsed_time not found -- assuming full time available."
print "remaining time at start: %s" % remaining_time_at_start
if optimal:
exitcodes = run_opt(configs, planner, sas_file, plan_file,
remaining_time_at_start, memory)
else:
exitcodes = run_sat(configs, unitcost, planner, sas_file, plan_file,
final_config, final_config_builder,
remaining_time_at_start, memory)
sys.exit(_generate_exitcode(exitcodes))
def set_resource_utilization_limits(self):
if not self.memory_limit or self.memory_limit == 0:
return
# get the soft and hard limits for the heap limit - this is default
soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_AS)
resource.setrlimit(resource.RLIMIT_AS, (self.memory_limit, hard_limit))
soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_DATA)
resource.setrlimit(resource.RLIMIT_DATA, (self.memory_limit, hard_limit))
def _run(self):
try:
lim = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (lim[1], lim[1]))
lim = resource.getrlimit(resource.RLIMIT_NOFILE)
print " maximum number of open files set to", lim[0]
except Exception, e:
print " failed to raise the maximum number of open files:", str(e)
def prepare_subprocess(): # create a new session for the spawned subprocess using os.setsid, # so we can later kill it and all children on timeout, taken from http://stackoverflow.com/questions/4789837/how-to-terminate-a-python-subprocess-launched-with-shell-true os.setsid() # Limit the size of files created during execution resource.setrlimit(resource.RLIMIT_NOFILE,(128,128)) if fileseeklimit is not None: resource.setrlimit(resource.RLIMIT_FSIZE,(fileseeklimitbytes, fileseeklimitbytes)) if resource.getrlimit(resource.RLIMIT_FSIZE) != (fileseeklimitbytes, fileseeklimitbytes): raise ValueError(resource.getrlimit(resource.RLIMIT_FSIZE))
def _prevent_core_dump(cls):
"""Prevent the process from generating a core dump."""
try:
# Try to get the current limit
resource.getrlimit(resource.RLIMIT_CORE)
except ValueError:
# System doesn't support the RLIMIT_CORE resource limit
return
else:
# Set the soft and hard limits for core dump size to zero
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
def run(portfolio, executable, sas_file, plan_manager):
attributes = get_portfolio_attributes(portfolio)
configs = attributes["CONFIGS"]
optimal = attributes["OPTIMAL"]
final_config = attributes.get("FINAL_CONFIG")
final_config_builder = attributes.get("FINAL_CONFIG_BUILDER")
timeout = attributes.get("TIMEOUT")
# Time limits are either positive values in seconds or -1 (unlimited).
soft_time_limit, hard_time_limit = resource.getrlimit(resource.RLIMIT_CPU)
print("External time limits: %d, %d" % (soft_time_limit, hard_time_limit))
external_time_limit = None
if soft_time_limit != resource.RLIM_INFINITY:
external_time_limit = soft_time_limit
elif hard_time_limit != resource.RLIM_INFINITY:
external_time_limit = hard_time_limit
if (external_time_limit is not None and
timeout is not None and
timeout != external_time_limit):
print("The externally set timeout (%d) differs from the one "
"in the portfolio file (%d). Is this expected?" %
(external_time_limit, timeout), file=sys.stderr)
# Prefer limits in the order: external soft limit, external hard limit,
# from portfolio file, default.
if external_time_limit is not None:
timeout = external_time_limit
elif timeout is None:
print("No timeout has been set for the portfolio so we take "
"the default of %ds." % DEFAULT_TIMEOUT, file=sys.stderr)
timeout = DEFAULT_TIMEOUT
print("Internal time limit: %d" % timeout)
# Memory limits are either positive values in Bytes or -1 (unlimited).
soft_mem_limit, hard_mem_limit = resource.getrlimit(resource.RLIMIT_AS)
print("External memory limits: %d, %d" % (soft_mem_limit, hard_mem_limit))
if hard_mem_limit == resource.RLIM_INFINITY:
memory = None
else:
memory = hard_mem_limit
print("Internal memory limit: %s" % memory)
remaining_time_at_start = float(timeout) - get_elapsed_time()
print("remaining time at start: %.2f" % remaining_time_at_start)
if optimal:
exitcodes = run_opt(configs, executable, sas_file, plan_manager,
remaining_time_at_start, memory)
else:
exitcodes = run_sat(configs, executable, sas_file, plan_manager,
final_config, final_config_builder,
remaining_time_at_start, memory)
exitcode = generate_exitcode(exitcodes)
if exitcode != 0:
raise subprocess.CalledProcessError(exitcode, ["run-portfolio", portfolio])
def setUp(self):
limits = resource.getrlimit(resource.RLIMIT_NOFILE)
print 'NOFILE rlimit:', limits
resource.setrlimit(resource.RLIMIT_NOFILE, (10, limits[1]))
print 'NOFILE rlimit:', resource.getrlimit(resource.RLIMIT_NOFILE)
self.mypath = os.path.dirname(__file__)
self.srcCat = afwTable.SourceCatalog.readFits(
os.path.join(self.mypath, "v695833-e0-c000.xy.fits"))
# The .xy.fits file has sources in the range ~ [0,2000],[0,4500]
self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(2048, 4612)) # approximate
def setProcessLimits():
# Read the config file
config = ConfigParser.ConfigParser()
config.read('config.cfg')
sandboxingConfig = ConfigSectionMap(config, "SandboxingParams")
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (int(sandboxingConfig['nfile']), hard))
soft, hard = resource.getrlimit(resource.RLIMIT_NPROC)
resource.setrlimit(resource.RLIMIT_NPROC, (int(sandboxingConfig['nproc']), hard))
soft, hard = resource.getrlimit(resource.RLIMIT_STACK)
resource.setrlimit(resource.RLIMIT_STACK, (int(sandboxingConfig['stacksize']), hard))
def main():
global __file__
__file__ = os.path.abspath(__file__)
print __file__
if os.path.islink(__file__):
print True
__file__ = getattr(os, 'readlink', lambda x: x)(__file__)
print os.getpid()
print os.getppid()
print resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (8192, -1))
print resource.getrlimit(resource.RLIMIT_NOFILE)
def _setup_resources():
"""Attempt to increase resource limits up to hard limits.
This allows us to avoid out of file handle limits where we can
move beyond the soft limit up to the hard limit.
"""
target_procs = 10240
cur_proc, max_proc = resource.getrlimit(resource.RLIMIT_NPROC)
target_proc = min(max_proc, target_procs) if max_proc > 0 else target_procs
resource.setrlimit(resource.RLIMIT_NPROC, (max(cur_proc, target_proc), max_proc))
cur_hdls, max_hdls = resource.getrlimit(resource.RLIMIT_NOFILE)
target_hdls = min(max_hdls, target_procs) if max_hdls > 0 else target_procs
resource.setrlimit(resource.RLIMIT_NOFILE, (max(cur_hdls, target_hdls), max_hdls))
def set_nixMem(args) :
if args.nixMem is not None :
import resource # this only exists on Unix/Linux based systems
rsrc = resource.RLIMIT_AS
soft, hard = resource.getrlimit(rsrc)
print('Soft limit starts as :', soft)
print('Hard limit starts as :', hard)
resource.setrlimit(rsrc, (args.nixMem * 1048576, hard)) #limit
soft, hard = resource.getrlimit(rsrc)
print('Soft limit changed to :', soft)
print('Hard limit changed to :', hard)
def Main(Bytes = None):
#Kilobytes = Bytes/1000.0
print 'Changing Memory Limit To : ' + str(Bytes) + " Bytes"
MemoryLimitBytes = resource.RLIMIT_AS
soft, hard = resource.getrlimit(MemoryLimitBytes)
print 'Soft limit starts as :', soft
resource.setrlimit(MemoryLimitBytes, (Bytes, hard)) #limit to one kilobyte
soft, hard = resource.getrlimit(MemoryLimitBytes)
print 'Soft limit changed to :', soft
def prevent_core_dump():
"""Prevent this process from generating a core dump."""
core_resource = resource.RLIMIT_CORE
try:
resource.getrlimit(core_resource)
except ValueError as ex:
raise DaemonError(
'Unable to limit core dump size: '
'system does not support RLIMIT_CORE resource limit') from ex
# Set hard & soft limits to 0, i.e. no core dump at all
resource.setrlimit(core_resource, (0, 0))
def limit_info(line):
rsrc = resource.RLIMIT_AS
soft, hard = resource.getrlimit(rsrc)
print 'Soft limit starts as:', soft
print 'Hard limit starts as:', hard
if line:
limit_mb = int(line)
limit_kb = int(line)*1024
print 'Setting limit to %s Mb' % limit_mb
resource.setrlimit(rsrc, (limit_kb, hard)) #limit to one kilobyte
soft, hard = resource.getrlimit(rsrc)
print 'Soft limit is now:', soft
print 'Hard limit is now:', hard
def allowCoreDumps():
""" Allow core dump generating by defining infinite core dump size.
:Return:
previous settings used by operating system
"""
settings = CoreDumpSettings()
settings.oldRlimitCore = getrlimit(RLIMIT_CORE)
settings.oldRlimitFsize = getrlimit(RLIMIT_FSIZE)
#infinite limits
setrlimit(RLIMIT_CORE, (-1, -1))
setrlimit(RLIMIT_FSIZE, (-1, -1))
return settings
def execute(self, fname, needToLimitTime=False):
if os.path.isfile(options.solver) is not True:
print "Error: Cannot find STP executable. Searched in: '%s'" % \
options.solver
print "Error code 300"
exit(300)
#construct command
command = ""
if not options.novalgrind and random.randint(0, 10) == 0:
command += "valgrind -q --leak-check=full --error-exitcode=173 "
command += options.solver
command += self.random_options()
#if options.verbose == False:
# command += "--verb 0 "
command += "-p " #yes, print counterexample
command += options.extra_options + " "
command += fname
print "Executing: %s " % command
#print time limit
if options.verbose:
print "CPU limit of parent (pid %d)" % os.getpid(), resource.getrlimit(resource.RLIMIT_CPU)
#if need time limit, then limit
if needToLimitTime:
p = subprocess.Popen(command.rsplit(), stdout=subprocess.PIPE, preexec_fn=setlimits)
else:
p = subprocess.Popen(command.rsplit(), stdout=subprocess.PIPE)
#print time limit after child startup
if options.verbose:
print "CPU limit of parent (pid %d) after startup of child" % \
os.getpid(), resource.getrlimit(resource.RLIMIT_CPU)
#Get solver output
consoleOutput, err = p.communicate()
if options.verbose:
print "CPU limit of parent (pid %d) after child finished executing" % \
os.getpid(), resource.getrlimit(resource.RLIMIT_CPU)
if p.returncode == 173:
print "Valgrind is indicating an error!"
print err
print consoleOutput
exit(-1)
return consoleOutput
def set_max_fds(maxfds=8180):
nfd,tot = resource.getrlimit(resource.RLIMIT_NOFILE)
# if we want fewer than the maximum num of FDs, then just return
if maxfds < nfd:
return maxfds
try:
resource.setrlimit(resource.RLIMIT_NOFILE, (maxfds, tot))
except:
pass
nfd,tot = resource.getrlimit(resource.RLIMIT_NOFILE)
return nfd
def bump_nofile_limit():
from twisted.python import log
log.msg("Open files limit: %d" % resource.getrlimit(resource.RLIMIT_NOFILE)[0])
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
values_to_try = [v for v in [hard, 100000, 10000] if v > soft]
for new_soft in values_to_try:
try:
resource.setrlimit(resource.RLIMIT_NOFILE, (new_soft, hard))
except ValueError:
continue
else:
log.msg("Open files limit increased from %d to %d" % (soft, new_soft))
break
else:
log.msg("Can't bump open files limit")
def _set_a_process_limit(resource_name, desired_limit, description):
which_resource = getattr(resource, resource_name)
(soft, hard) = resource.getrlimit(which_resource)
def set_resource(attempted_value):
log_verbose("Trying setrlimit(resource.%s, (%d, %d))" %
(resource_name, attempted_value, hard))
resource.setrlimit(which_resource, (attempted_value, hard))
log_info("Setting OS limit on %s for process (desire up to %d)..."
"\n\t Current limit values: soft = %d, hard = %d" %
(description, desired_limit, soft, hard))
_negotiate_process_limit(set_resource, desired_limit, soft, hard)
log_info("Resulting OS limit on %s for process: " % description +
"soft = %d, hard = %d" % resource.getrlimit(which_resource))
def pre_request(worker, req):
import os
import psutil
import resource
import signal
# VMS and RLIMIT_AS are the same thing: virtual memory, a.k.a. address space
rss, vms = psutil.Process(os.getpid()).get_memory_info()
soft, hard = resource.getrlimit(resource.RLIMIT_AS)
resource.setrlimit(resource.RLIMIT_AS, (config['virtual_memory_limit'], hard))
r = resource.getrusage(resource.RUSAGE_SELF)
cpu_time = r.ru_utime + r.ru_stime
signal.signal(signal.SIGXCPU, time_expired)
soft, hard = resource.getrlimit(resource.RLIMIT_CPU)
resource.setrlimit(resource.RLIMIT_CPU, (cpu_time + config['cpu_time_limit'], hard))
def limit_resources():
_, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
# XXX warn if too few compared to max_wokers?
resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
_, hard = resource.getrlimit(resource.RLIMIT_AS)
rlimit_as = int(config.read('System', 'RLIMIT_AS_gigabytes'))
if rlimit_as == 0:
return
LOGGER.info('Setting RLIMIT_AS per configuration to %d gigabytes', rlimit_as)
rlimit_as *= 1024 * 1024 * 1024
if hard > 0 and rlimit_as > hard:
LOGGER.error('RLIMIT_AS limited to %d bytes by system limit', hard)
rlimit_as = hard
resource.setrlimit(resource.RLIMIT_AS, (rlimit_as, hard))
def set_max_memory(maxsize):
soft, hard = resource.getrlimit(resource.RLIMIT_AS)
resource.setrlimit(resource.RLIMIT_AS, (maxsize, hard))
def get_file_limits():
return resource.getrlimit(resource.RLIMIT_NOFILE)
def main():
shell.check_python()
config = shell.get_config(False)
shell.log_shadowsocks_version()
daemon.daemon_exec(config)
try:
import resource
logging.info(
'current process RLIMIT_NOFILE resource: soft %d hard %d' %
resource.getrlimit(resource.RLIMIT_NOFILE))
except ImportError:
pass
if config['port_password']:
pass
else:
config['port_password'] = {}
server_port = config['server_port']
if type(server_port) == list:
for a_server_port in server_port:
config['port_password'][a_server_port] = config['password']
else:
config['port_password'][str(server_port)] = config['password']
if not config.get('dns_ipv6', False):
asyncdns.IPV6_CONNECTION_SUPPORT = False
if config.get('manager_address', 0):
logging.info('entering manager mode')
manager.run(config)
return
tcp_servers = []
udp_servers = []
dns_resolver = asyncdns.DNSResolver()
if int(config['workers']) > 1:
stat_counter_dict = None
else:
stat_counter_dict = {}
port_password = config['port_password']
config_password = config.get('password', 'm')
del config['port_password']
for port, password_obfs in port_password.items():
method = config["method"]
protocol = config.get("protocol", 'origin')
protocol_param = config.get("protocol_param", '')
obfs = config.get("obfs", 'plain')
obfs_param = config.get("obfs_param", '')
bind = config.get("out_bind", '')
bindv6 = config.get("out_bindv6", '')
if type(password_obfs) == list:
password = password_obfs[0]
obfs = common.to_str(password_obfs[1])
if len(password_obfs) > 2:
protocol = common.to_str(password_obfs[2])
elif type(password_obfs) == dict:
password = password_obfs.get('password', config_password)
method = common.to_str(password_obfs.get('method', method))
protocol = common.to_str(password_obfs.get('protocol', protocol))
protocol_param = common.to_str(
password_obfs.get('protocol_param', protocol_param))
obfs = common.to_str(password_obfs.get('obfs', obfs))
obfs_param = common.to_str(
password_obfs.get('obfs_param', obfs_param))
bind = password_obfs.get('out_bind', bind)
bindv6 = password_obfs.get('out_bindv6', bindv6)
else:
password = password_obfs
a_config = config.copy()
ipv6_ok = False
logging.info(
"server start with protocol[%s] password [%s] method [%s] obfs [%s] obfs_param [%s]"
% (protocol, password, method, obfs, obfs_param))
if 'server_ipv6' in a_config:
try:
if len(a_config['server_ipv6']
) > 2 and a_config['server_ipv6'][
0] == "[" and a_config['server_ipv6'][-1] == "]":
a_config['server_ipv6'] = a_config['server_ipv6'][1:-1]
a_config['server_port'] = int(port)
a_config['password'] = password
a_config['method'] = method
a_config['protocol'] = protocol
a_config['protocol_param'] = protocol_param
a_config['obfs'] = obfs
a_config['obfs_param'] = obfs_param
a_config['out_bind'] = bind
a_config['out_bindv6'] = bindv6
a_config['server'] = a_config['server_ipv6']
logging.info("starting server at [%s]:%d" %
(a_config['server'], int(port)))
tcp_servers.append(
tcprelay.TCPRelay(a_config,
dns_resolver,
False,
stat_counter=stat_counter_dict))
udp_servers.append(
udprelay.UDPRelay(a_config,
dns_resolver,
False,
stat_counter=stat_counter_dict))
if a_config['server_ipv6'] == b"::":
ipv6_ok = True
except Exception as e:
shell.print_exception(e)
try:
a_config = config.copy()
a_config['server_port'] = int(port)
a_config['password'] = password
a_config['method'] = method
a_config['protocol'] = protocol
a_config['protocol_param'] = protocol_param
a_config['obfs'] = obfs
a_config['obfs_param'] = obfs_param
a_config['out_bind'] = bind
a_config['out_bindv6'] = bindv6
logging.info("starting server at %s:%d" %
(a_config['server'], int(port)))
tcp_servers.append(
tcprelay.TCPRelay(a_config,
dns_resolver,
False,
stat_counter=stat_counter_dict))
udp_servers.append(
udprelay.UDPRelay(a_config,
dns_resolver,
False,
stat_counter=stat_counter_dict))
except Exception as e:
if not ipv6_ok:
shell.print_exception(e)
def run_server():
def child_handler(signum, _):
logging.warn('received SIGQUIT, doing graceful shutting down..')
list(
map(lambda s: s.close(next_tick=True),
tcp_servers + udp_servers))
signal.signal(getattr(signal, 'SIGQUIT', signal.SIGTERM),
child_handler)
def int_handler(signum, _):
sys.exit(1)
signal.signal(signal.SIGINT, int_handler)
try:
loop = eventloop.EventLoop()
dns_resolver.add_to_loop(loop)
list(map(lambda s: s.add_to_loop(loop), tcp_servers + udp_servers))
daemon.set_user(config.get('user', None))
loop.run()
except Exception as e:
shell.print_exception(e)
sys.exit(1)
if int(config['workers']) > 1:
if os.name == 'posix':
children = []
is_child = False
for i in range(0, int(config['workers'])):
r = os.fork()
if r == 0:
logging.info('worker started')
is_child = True
run_server()
break
else:
children.append(r)
if not is_child:
def handler(signum, _):
for pid in children:
try:
os.kill(pid, signum)
os.waitpid(pid, 0)
except OSError: # child may already exited
pass
sys.exit()
signal.signal(signal.SIGTERM, handler)
signal.signal(signal.SIGQUIT, handler)
signal.signal(signal.SIGINT, handler)
# master
for a_tcp_server in tcp_servers:
a_tcp_server.close()
for a_udp_server in udp_servers:
a_udp_server.close()
dns_resolver.close()
for child in children:
os.waitpid(child, 0)
else:
logging.warn('worker is only available on Unix/Linux')
run_server()
else:
run_server()
from functools import partial
from mmcv.runner import get_dist_info
from mmcv.parallel import collate
from torch.utils.data import DataLoader
from .sampler import GroupSampler, DistributedGroupSampler, DistributedSampler
# https://github.com/pytorch/pytorch/issues/973
import resource
rlimit = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (4096, rlimit[1]))
def build_dataloader(dataset,
imgs_per_gpu,
workers_per_gpu,
num_gpus=1,
dist=True,
**kwargs):
shuffle = kwargs.get('shuffle', True)
if dist:
rank, world_size = get_dist_info()
if shuffle:
sampler = DistributedGroupSampler(dataset, imgs_per_gpu,
world_size, rank)
else:
sampler = DistributedSampler(
dataset, world_size, rank, shuffle=False)
batch_size = imgs_per_gpu
num_workers = workers_per_gpu
"""Early initialization for ansible-test before most other imports have been performed."""
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import resource
from .constants import (
SOFT_RLIMIT_NOFILE,
)
CURRENT_RLIMIT_NOFILE = resource.getrlimit(resource.RLIMIT_NOFILE)
DESIRED_RLIMIT_NOFILE = (SOFT_RLIMIT_NOFILE, CURRENT_RLIMIT_NOFILE[1])
if DESIRED_RLIMIT_NOFILE < CURRENT_RLIMIT_NOFILE:
resource.setrlimit(resource.RLIMIT_NOFILE, DESIRED_RLIMIT_NOFILE)
CURRENT_RLIMIT_NOFILE = DESIRED_RLIMIT_NOFILE
def main():
"""Do the real work"""
# Parse remaining variables
network_map = options["input"]
# network_mapset = network_map.split('@')[0]
network = network_map.split("@")[1] if len(network_map.split("@")) > 1 else None
suffix = options["suffix"]
layer = options["layer"]
corridor_tolerance = options["corridor_tolerance"]
cores = options["cores"]
where = None if options["where"] == "" else options["where"]
weights = options["weights"].split(",")
s_flag = flags["s"]
d_flag = flags["d"]
r_flag = flags["r"]
ulimit = resource.getrlimit(resource.RLIMIT_NOFILE)
net_hist_str = (
grass.read_command("v.info", map=network_map, flags="h")
.split("\n")[0]
.split(": ")[1]
)
dist_cmd_dict = task.cmdstring_to_tuple(net_hist_str)
dist_prefix = dist_cmd_dict[1]["prefix"]
# network_prefix = dist_cmd_dict[1]['prefix']
# print(where)
# in_vertices = dist_cmd_dict[1]['input']
# Check if db-connection for edge map exists
con = vect.vector_db(network_map)[int(layer)]
if not con:
grass.fatal(
"Database connection for map {} \
is not defined for layer {}.".format(
network, layer
)
)
# Check if required columns exist and are of required type
required_columns = ["con_id_u", "from_p", "to_p", "cd_u"]
if weights:
required_columns += weights
in_columns = vect.vector_columns(network_map, layer=layer)
missing_columns = np.setdiff1d(required_columns, in_columns.keys())
if missing_columns:
grass.fatal(
"Cannot find the following reqired/requested \
column(s) {} in vector map \
{}.".format(
", ".join(missing_columns), network
)
)
#
weight_types = []
# Check properly if column is numeric
for col in required_columns:
if in_columns[col]["type"] not in ["INTEGER", "DOUBLE PRECISION", "REAL"]:
grass.fatal(
"Column {} is of type {}. \
Only numeric types (integer, \
real or double precision) \
allowed!".format(
col, in_columns[col]["type"]
)
)
if col in weights:
weight_types.append(in_columns[col]["type"])
# Extract necessary informartion on edges from attribute table of
# edge map
table_io = StringIO(
unicode(
grass.read_command(
"v.db.select",
flags="c",
map=network_map,
columns=required_columns,
separator=",",
where=where,
)
)
)
try:
table_extract = np.genfromtxt(
table_io, delimiter=",", dtype=None, names=required_columns
)
except:
grass.fatal("No edges selected to compute corridors for...")
# Output result of where-clause and exit (if requested)
if s_flag:
print(table_extract)
# grass.message("con_id_u|from_p|to_p")
# for fid in $selected_edges_ud:
# message_text = $(echo $table_extract | tr ' ' '\n' |
# tr ',' ' ' | awk -v FID=$fid '{if($1==FID) print $1 "|" $2 "|"
# $3}' | head -n 1)
# grass.message(message_text)
sys.exit(0)
# Get unique identifiers for the selected undirected edges
selected_patches = np.unique(
np.append(table_extract["from_p"], table_extract["to_p"])
)
selected_edges = np.unique(table_extract["con_id_u"])
# activate z-flag if more maps have to be aggregated than ulimit
z_flag = None if len(selected_edges) < ulimit else "z"
# Check if cost distance raster maps exist
pattern = "{}_patch_*_cost_dist".format(dist_prefix)
patchmaps = (
grass.read_command("g.list", pattern=pattern, type="raster")
.rstrip("\n")
.split("\n")
)
for patch in selected_patches:
# Check if cost distance raster maps exist
patchmap = "{}_patch_{}_cost_dist".format(dist_prefix, patch)
if patchmap not in patchmaps:
grass.fatal("Cannot find raster map {}.".format(patchmap))
# Create mapcalculator expressions for cost distance corridors,
# assigning distance values
corridormaps = {}
if d_flag:
pattern = "{}_corridor_*_cost_dist".format(dist_prefix)
corridor_base = "dist"
else:
pattern = "{}_corridor_[0-9]+$".format(dist_prefix)
corridor_base = "id"
corridormaps[corridor_base] = (
grass.read_command("g.list", flags="e", pattern=pattern, type="raster")
.rstrip("\n")
.split("\n")
)
for weight in weights:
pattern = "{}_corridor_[0-9]+_{}".format(dist_prefix, weight)
corridormaps[weight] = (
grass.read_command("g.list", flags="e", pattern=pattern, type="raster")
.rstrip("\n")
.split("\n")
)
# Setup GRASS modules for raster processing
mapcalc = Module("r.mapcalc", quiet=True, run_=False)
reclass = Module("r.reclass", rules="-", quiet=True, run_=False)
recode = Module("r.recode", rules="-", quiet=True, run_=False)
# Setup paralel module queue if parallel processing is requested
# print(weight_types)
if cores > 1:
mapcalc_queue = ParallelModuleQueue(nprocs=cores)
if "INTEGER" in weight_types:
reclass_queue = ParallelModuleQueue(nprocs=cores)
if "REAL" in weight_types or "DOUBLE PRECISION" in weight_types:
recode_queue = ParallelModuleQueue(nprocs=cores)
corridor_list = []
for edge_id in selected_edges:
edge = table_extract[table_extract["con_id_u"] == edge_id][0]
# print(e.dtype.names)
if d_flag:
corridor = "{}_corridor_{}_cost_dist".format(dist_prefix, edge_id)
# corridor_list.append(corridor)
mc_expression = "{prefix}_corridor_{CON_ID}_cost_dist=if( \
({prefix}_patch_{FROM_P}_cost_dist+ \
{prefix}_patch_{TO_P}_cost_dist) - \
(({prefix}_patch_{FROM_P}_cost_dist+ \
{prefix}_patch_{TO_P}_cost_dist) * \
{cor_tolerance}/100.0)<= \
({prefix}_patch_{FROM_P}_cost_dist + \
{prefix}_patch_{TO_P}_cost_dist), \
({prefix}_patch_{FROM_P}_cost_dist+ \
{prefix}_patch_{TO_P}_cost_dist), \
null())".format(
prefix=dist_prefix,
CON_ID=edge["con_id_u"],
FROM_P=edge["from_p"],
TO_P=edge["to_p"],
cor_tolerance=corridor_tolerance,
)
else:
corridor = "{}_corridor_{}".format(dist_prefix, edge["con_id_u"])
# corridor_list.append(corridor)
# Create mapcalculator expressions for cost distance
# corridors, assigning connection IDs for reclassification
mc_expression = "{prefix}_corridor_{CON_ID}=if( \
({prefix}_patch_{FROM_P}_cost_dist+ \
{prefix}_patch_{TO_P}_cost_dist)- \
(({prefix}_patch_{FROM_P}_cost_dist+ \
{prefix}_patch_{TO_P}_cost_dist)* \
{cor_tolerance}/100.0)<={CD}, \
{CON_ID}, null())".format(
prefix=dist_prefix,
CON_ID=edge["con_id_u"],
FROM_P=edge["from_p"],
TO_P=edge["to_p"],
CD=edge["cd_u"],
cor_tolerance=corridor_tolerance,
)
corridor_list.append(corridor)
# print(corridor)
# print(corridormaps)
if r_flag or corridor not in corridormaps[corridor_base]:
new_mapcalc = copy.deepcopy(mapcalc)
if cores > 1:
calc = new_mapcalc(expression=mc_expression)
mapcalc_queue.put(calc)
else:
calc = new_mapcalc(expression=mc_expression, region="intersect")
calc.run()
for weight in weights:
if r_flag or corridor not in corridormaps[weight]:
in_map = corridor
out_map = "{}_{}".format(in_map, weight)
if in_columns[weight]["type"] == "INTEGER":
new_reclass = copy.deepcopy(reclass)
reclass_rule = "{} = {}".format(edge["con_id_u"], edge[weight])
rcl = new_reclass(input=in_map, output=out_map, stdin_=reclass_rule)
if cores > 1:
reclass_queue.put(rcl)
else:
rcl.run()
if in_columns[weight]["type"] in ["REAL", "DOUBLE PRECISION"]:
new_recode = copy.deepcopy(recode)
recode_rule = "{0}:{0}:{1}:{1}".format(
edge["con_id_u"], edge[weight]
)
rco = new_recode(input=in_map, output=out_map, stdin_=recode_rule)
if cores > 1:
recode_queue.put(rco)
else:
rco.run()
if cores > 1:
mapcalc_queue.wait()
if "INTEGER" in weight_types:
reclass_queue.wait()
if "REAL" in weight_types or "DOUBLE PRECISION" in weight_types:
recode_queue.wait()
grass.verbose("Aggregating corridor maps...")
if d_flag:
grass.run_command(
"r.series",
flags=z_flag,
quiet=True,
input=",".join(corridor_list),
output="{}_corridors_min_cost_dist_{}".format(dist_prefix, suffix),
method="minimum",
)
else:
# Summarize corridors
if not weights:
print(",".join(corridor_list))
output_map = "{}_corridors_count_{}".format(dist_prefix, suffix)
grass.run_command(
"r.series",
flags=z_flag,
quiet=True,
input=",".join(corridor_list),
output=output_map,
method="count",
)
write_raster_history(output_map)
else:
# Weight corridors according to user requested weights
for weight in weights:
# Generate corridor map list
corridor_map_list = (cm + "_{}".format(weight) for cm in corridor_list)
output_map = "{}_corridors_{}_sum_{}".format(
dist_prefix, weight, suffix
)
# Summarize corridors using r.series
grass.run_command(
"r.series",
flags=z_flag,
quiet=True,
input=corridor_map_list,
output=output_map,
method="sum",
)
write_raster_history(output_map)
platform.cores_logical
])
else:
cases = filter_ordered(
[platform.cores_physical, platform.cores_logical])
ret.extend([((name, nthread), ) for nthread in cases])
if basic not in ret:
warning("skipping `%s`; perhaps you've set OMP_NUM_THREADS to a "
"non-standard value while attempting autotuning in "
"`max` mode?" % dict(basic))
return ret
options = {
'squeezer': 4,
'blocksize-l0': (8, 16, 24, 32, 64, 96, 128),
'blocksize-l1': (8, 16, 32),
'stack_limit': resource.getrlimit(resource.RLIMIT_STACK)[0] / 4
}
"""Autotuning options."""
def log(msg):
perf("AutoTuner: %s" % msg)
def warning(msg):
_warning("AutoTuner: %s" % msg)
def cl_main():
# parse args
arg_parser = argparse.ArgumentParser(
description="SACAD (recursive tool) v%s.%s" %
(sacad.__version__, __doc__),
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
arg_parser.add_argument(
"lib_dir", help="Music library directory to recursively analyze")
arg_parser.add_argument("size", type=int, help="Target image size")
arg_parser.add_argument("cover_pattern",
help="""Cover image path pattern.
{artist} and {album} are replaced by their tag value.
You can set an absolute path, otherwise destination directory is relative to the audio files.
Use single character '%s' to embed JPEG into audio files."""
% (EMBEDDED_ALBUM_ART_SYMBOL))
arg_parser.add_argument(
"-i",
"--ignore-existing",
action="store_true",
default=False,
help=
"Ignore existing covers and force search and download for all files")
arg_parser.add_argument(
"-f",
"--full-scan",
action="store_true",
default=False,
help="""Enable scanning of all audio files in each directory.
By default the scanner will assume all audio files in a single directory are part of
the same album, and only read metadata for the first file.
Enable this if your files are organized in a way than allows files for different
albums to be in the same directory level.
WARNING: This will make the initial scan much slower."""
)
sacad.setup_common_args(arg_parser)
arg_parser.add_argument("-v",
"--verbose",
action="store_true",
default=False,
dest="verbose",
help="Enable verbose output")
args = arg_parser.parse_args()
if args.cover_pattern == EMBEDDED_ALBUM_ART_SYMBOL:
args.format = "jpg"
else:
args.format = os.path.splitext(args.cover_pattern)[1][1:].lower()
try:
args.format = sacad.SUPPORTED_IMG_FORMATS[args.format]
except KeyError:
print(
"Unable to guess image format from extension, or unknown format: %s"
% (args.format))
exit(1)
# setup logger
if not args.verbose:
logging.getLogger("sacad_r").setLevel(logging.WARNING)
logging.getLogger().setLevel(logging.ERROR)
logging.getLogger("asyncio").setLevel(logging.CRITICAL + 1)
fmt = "%(name)s: %(message)s"
else:
logging.getLogger("sacad_r").setLevel(logging.DEBUG)
logging.getLogger().setLevel(logging.DEBUG)
logging.getLogger("asyncio").setLevel(logging.WARNING)
fmt = "%(asctime)s %(levelname)s [%(name)s] %(message)s"
logging_formatter = colored_logging.ColoredFormatter(fmt=fmt)
logging_handler = logging.StreamHandler()
logging_handler.setFormatter(logging_formatter)
logging.getLogger().addHandler(logging_handler)
# bump nofile ulimit
try:
soft_lim, hard_lim = resource.getrlimit(resource.RLIMIT_NOFILE)
if ((soft_lim != resource.RLIM_INFINITY) and
((soft_lim < hard_lim) or (hard_lim == resource.RLIM_INFINITY))):
resource.setrlimit(resource.RLIMIT_NOFILE, (hard_lim, hard_lim))
logging.getLogger().debug(
"Max open files count set from %u to %u" %
(soft_lim, hard_lim))
except (AttributeError, OSError):
# not supported on system
pass
# do the job
work = analyze_lib(args.lib_dir,
args.cover_pattern,
ignore_existing=args.ignore_existing,
full_scan=args.full_scan,
all_formats=args.preserve_format)
get_covers(work, args)
def start(self):
"""
Start daemonization process.
"""
# If pidfile already exists, we should read pid from there; to overwrite it, if locking
# will fail, because locking attempt somehow purges the file contents.
if os.path.isfile(self.pid):
with open(self.pid, "r") as old_pidfile:
old_pid = old_pidfile.read()
# Create a lockfile so that only one instance of this daemon is running at any time.
try:
lockfile = open(self.pid, "w")
except IOError:
print("Unable to create the pidfile.")
sys.exit(1)
try:
# Try to get an exclusive lock on the file. This will fail if another process has the file
# locked.
fcntl.flock(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
print("Unable to lock on the pidfile.")
# We need to overwrite the pidfile if we got here.
with open(self.pid, "w") as pidfile:
pidfile.write(old_pid)
sys.exit(1)
# skip fork if foreground is specified
if not self.foreground:
# Fork, creating a new process for the child.
process_id = os.fork()
if process_id < 0:
# Fork error. Exit badly.
sys.exit(1)
elif process_id != 0:
# This is the parent process. Exit.
sys.exit(0)
# This is the child process. Continue.
# Stop listening for signals that the parent process receives.
# This is done by getting a new process id.
# setpgrp() is an alternative to setsid().
# setsid puts the process in a new parent group and detaches its controlling terminal.
process_id = os.setsid()
if process_id == -1:
# Uh oh, there was a problem.
sys.exit(1)
# Add lockfile to self.keep_fds.
self.keep_fds.append(lockfile.fileno())
# Close all file descriptors, except the ones mentioned in self.keep_fds.
devnull = "/dev/null"
if hasattr(os, "devnull"):
# Python has set os.devnull on this system, use it instead as it might be different
# than /dev/null.
devnull = os.devnull
if self.auto_close_fds:
for fd in range(3,
resource.getrlimit(resource.RLIMIT_NOFILE)[0]):
if fd not in self.keep_fds:
try:
os.close(fd)
except OSError:
pass
devnull_fd = os.open(devnull, os.O_RDWR)
os.dup2(devnull_fd, 0)
os.dup2(devnull_fd, 1)
os.dup2(devnull_fd, 2)
if self.logger is None:
# Initialize logging.
self.logger = logging.getLogger(self.app)
self.logger.setLevel(logging.DEBUG)
# Display log messages only on defined handlers.
self.logger.propagate = False
# Initialize syslog.
# It will correctly work on OS X, Linux and FreeBSD.
if sys.platform == "darwin":
syslog_address = "/var/run/syslog"
else:
syslog_address = "/dev/log"
# We will continue with syslog initialization only if actually have such capabilities
# on the machine we are running this.
if os.path.exists(syslog_address):
syslog = handlers.SysLogHandler(syslog_address)
if self.verbose:
syslog.setLevel(logging.DEBUG)
else:
syslog.setLevel(logging.INFO)
# Try to mimic to normal syslog messages.
formatter = logging.Formatter(
"%(asctime)s %(name)s: %(message)s", "%b %e %H:%M:%S")
syslog.setFormatter(formatter)
self.logger.addHandler(syslog)
# Set umask to default to safe file permissions when running as a root daemon. 027 is an
# octal number which we are typing as 0o27 for Python3 compatibility.
os.umask(0o27)
# Change to a known directory. If this isn't done, starting a daemon in a subdirectory that
# needs to be deleted results in "directory busy" errors.
os.chdir(self.chdir)
# Execute privileged action
privileged_action_result = self.privileged_action()
if not privileged_action_result:
privileged_action_result = []
# Change owner of pid file, it's required because pid file will be removed at exit.
uid, gid = -1, -1
if self.group:
try:
gid = grp.getgrnam(self.group).gr_gid
except KeyError:
self.logger.error("Group {0} not found".format(self.group))
sys.exit(1)
if self.user:
try:
uid = pwd.getpwnam(self.user).pw_uid
except KeyError:
self.logger.error("User {0} not found.".format(self.user))
sys.exit(1)
if uid != -1 or gid != -1:
os.chown(self.pid, uid, gid)
# Change gid
if self.group:
try:
os.setgid(gid)
except OSError:
self.logger.error("Unable to change gid.")
sys.exit(1)
# Change uid
if self.user:
try:
uid = pwd.getpwnam(self.user).pw_uid
except KeyError:
self.logger.error("User {0} not found.".format(self.user))
sys.exit(1)
try:
os.setuid(uid)
except OSError:
self.logger.error("Unable to change uid.")
sys.exit(1)
try:
lockfile.write("%s" % (os.getpid()))
lockfile.flush()
except IOError:
self.logger.error("Unable to write pid to the pidfile.")
print("Unable to write pid to the pidfile.")
sys.exit(1)
# Set custom action on SIGTERM.
signal.signal(signal.SIGTERM, self.sigterm)
atexit.register(self.exit)
self.logger.warn("Starting daemon.")
self.action(*privileged_action_result)
def _rlimit_nofile():
return resource.getrlimit(resource.RLIMIT_NOFILE)
print HOME_PAGE_EXAMPLE_ROW % {
"link": urllib.quote(query),
"query": cgi.escape(query).replace(" ", " "),
"text": cgi.escape(text),
}
print HOME_PAGE_TABLE_END
print HOME_PAGE_END
sys.exit(0)
query = fs.getvalue("q", "").strip()
start = int(fs.getvalue("start", 0))
num = int(fs.getvalue("num", PER_PAGE))
max_computation = int(fs.getvalue("comp", MAX_COMPUTATION))
# Shell out to the find-exec binary to get results
soft, hard = resource.getrlimit(resource.RLIMIT_CPU)
if soft == -1 or soft > 30: soft = 30
resource.setrlimit(resource.RLIMIT_CPU, (soft, hard))
soft, hard = resource.getrlimit(resource.RLIMIT_AS)
if hard == -1 or hard > 2048 * 1024 * 1024: hard = 2048 * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, (hard, hard))
proc = subprocess.Popen(
[binary, index, query],
preexec_fn=lambda: signal.signal(signal.SIGPIPE, signal.SIG_DFL),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
print RESULT_PAGE_BEGIN % {"query": cgi.escape(query, quote=True)}
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import resource
import sys
from k2.version import __version__
# import sys
# sys.path.insert(0, os.path.abspath('.'))
# ZMT-5229: We are sometimes hitting the recursion limit when generating the documentation.
# The default recursion limit is 1000. Lets double it and hope for the best.
sys.setrecursionlimit(2000)
# Lets also double the stack size limit to account for the new recursion limit.
rlimit = resource.getrlimit(resource.RLIMIT_STACK)
resource.setrlimit(resource.RLIMIT_STACK, (rlimit[0] * 2, rlimit[1]))
# -- General configuration ------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.ifconfig',
'sphinx.ext.viewcode', 'sphinxcontrib.plantuml',
'sphinxprettysearchresults'
class TestHoleModule(object):
try:
# on Unix we can manipulate our limits: http://docs.python.org/2/library/resource.html
import resource
soft_max_open_files, hard_max_open_files = resource.getrlimit(
resource.RLIMIT_NOFILE)
except ImportError:
pass
@staticmethod
@pytest.fixture()
def universe():
return mda.Universe(MULTIPDB_HOLE)
@pytest.mark.skipif(
rlimits_missing,
reason="Test skipped because platform does not allow setting rlimits")
def test_hole_module_fd_closure(self, universe, tmpdir):
"""test open file descriptors are closed (MDAnalysisTests.analysis.test_hole.TestHoleModule): Issue 129"""
# If Issue 129 isn't resolved, this function will produce an OSError on
# the system, and cause many other tests to fail as well.
#
# Successful test takes ~10 s, failure ~2 s.
# Hasten failure by setting "ulimit -n 64" (can't go too low because of open modules etc...)
import resource
# ----- temporary hack -----
# on Mac OS X (on Travis) we run out of open file descriptors
# before even starting this test (see
# https://github.com/MDAnalysis/mdanalysis/pull/901#issuecomment-231938093);
# if this issue is solved by #363 then revert the following
# hack:
#
import platform
if platform.platform() == "Darwin":
max_open_files = 512
else:
max_open_files = 64
#
# --------------------------
resource.setrlimit(resource.RLIMIT_NOFILE,
(max_open_files, self.hard_max_open_files))
with tmpdir.as_cwd():
try:
H = hole2.HoleAnalysis(universe, cvect=[0, 1, 0], sample=20.0)
finally:
self._restore_rlimits()
# pretty unlikely that the code will get through 2 rounds if the MDA
# issue 129 isn't fixed, although this depends on the file descriptor
# open limit for the machine in question
try:
for i in range(2):
# will typically get an OSError for too many files being open after
# about 2 seconds if issue 129 isn't resolved
H.run()
except OSError as err:
if err.errno == errno.EMFILE:
raise pytest.fail(
"hole2.HoleAnalysis does not close file descriptors (Issue 129)"
)
raise
finally:
# make sure to restore open file limit !!
self._restore_rlimits()
def _restore_rlimits(self):
try:
import resource
resource.setrlimit(
resource.RLIMIT_NOFILE,
(self.soft_max_open_files, self.hard_max_open_files))
except ImportError:
pass
def limit_memory(maxsize):
'''
限制内存使用，设置可使用的总内存值
'''
soft, hard = resource.getrlimit(resource.RLIMIT_AS)
resource.setrlimit(resource.RLIMIT_AS, (maxsize, hard))
def set_max_runtime(seconds: int) -> None:
soft, hard = resource.getrlimit(resource.RLIMIT_CPU)
resource.setrlimit(resource.RLIMIT_CPU, (seconds, hard))
signal.signal(signal.SIGXCPU, time_exceeded)
def __init__(
self,
bento_bundle_path: str = Provide[BentoMLContainer.bundle_path],
outbound_host: str = Provide[BentoMLContainer.forward_host],
outbound_port: int = Provide[BentoMLContainer.forward_port],
outbound_workers: int = Provide[BentoMLContainer.api_server_workers],
mb_max_batch_size: int = Provide[
BentoMLContainer.config.bento_server.microbatch.max_batch_size
],
mb_max_latency: int = Provide[
BentoMLContainer.config.bento_server.microbatch.max_latency
],
max_request_size: int = Provide[
BentoMLContainer.config.bento_server.max_request_size
],
outbound_unix_socket: str = None,
enable_access_control: bool = Provide[
BentoMLContainer.config.bento_server.cors.enabled
],
access_control_allow_origin: Optional[str] = Provide[
BentoMLContainer.config.bento_server.cors.access_control_allow_origin
],
access_control_options: Optional["ResourceOptions"] = Provide[
BentoMLContainer.access_control_options
],
timeout: int = Provide[BentoMLContainer.config.bento_server.timeout],
tracer=Provide[BentoMLContainer.tracer],
):
self._conn: Optional["BaseConnector"] = None
self._client: Optional["ClientSession"] = None
self.outbound_unix_socket = outbound_unix_socket
self.outbound_host = outbound_host
self.outbound_port = outbound_port
self.outbound_workers = outbound_workers
self.mb_max_batch_size = mb_max_batch_size
self.mb_max_latency = mb_max_latency
self.batch_handlers = dict()
self._outbound_sema = None # the semaphore to limit outbound connections
self._cleanup_tasks = None
self.max_request_size = max_request_size
self.tracer = tracer
self.enable_access_control = enable_access_control
self.access_control_allow_origin = access_control_allow_origin
self.access_control_options = access_control_options
self.bento_service_metadata_pb = load_bento_service_metadata(bento_bundle_path)
self.setup_routes_from_pb(self.bento_service_metadata_pb)
self.timeout = timeout
if psutil.POSIX:
import resource
self.CONNECTION_LIMIT = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
else:
self.CONNECTION_LIMIT = 1024
logger.info(
"Your system nofile limit is %d, which means each instance of microbatch "
"service is able to hold this number of connections at same time. "
"You can increase the number of file descriptors for the server process, "
"or launch more microbatch instances to accept more concurrent connection.",
self.CONNECTION_LIMIT,
)
def start_daemon(proc, pid_fd=None):
"""
start_daemon(proc, pid_fd = None) -> exit code
Start a daemon process. Caller must pass a function, proc(), with
prototype looks below:
def proc():
return <integer>
Please make sure the return code of proc() follows Win32 system
error code standard.
If pid_fd is not None, it should be a valid file object. The
file object should point to a lock file, so we can write real daemon
PID there.
"""
import resource
maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
if maxfd == resource.RLIM_INFINITY:
maxfd = MAXFD
# Make sure stdin, stdout and stderr are closed.
os.close(STDIN_FD)
os.open(NULL_TO, os.O_RDWR)
os.dup2(STDIN_FD, STDOUT_FD)
os.dup2(STDIN_FD, STDERR_FD)
try:
pid = os.fork()
except OSError:
msg = "start_daemon(): Failed on fork()"
write_log(STDERR_FD, None, ERROR_PROC_NOT_FOUND, msg)
raise ICAException(ERROR_PROC_NOT_FOUND, msg)
if pid == 0:
os.setsid()
# TODO Shall we ignore SIGHUP?
# import signal
# signal.signal(signal.SIGHUP, signal.SIG_IGN)
# TODO Not sure if it should be added. Ignoring child exit
# signal can take load off icadaemon. However it looks like it's
# supported only on Linux.
# signal.signal(signal.SIGCHLD, signal.SIG_IGN)
try:
pid = os.fork()
except OSError:
msg = "start_daemon(): Failed on fork(), second time"
write_log(STDERR_FD, None, ERROR_PROC_NOT_FOUND, msg)
raise ICAException(ERROR_PROC_NOT_FOUND, msg)
if pid == 0:
os.chdir(WORKDIR)
os.umask(UMASK)
proc_params = "Daemon is running: pid:%d,uid:%d,euid:%d,gid:%d,egid:%d" % (
os.getpid(), os.getuid(), os.geteuid(), os.getgid(),
os.getegid())
# Use ERR level to make sure the pid information is always
# shown. In FreeBSD 8.2, the INFO level message does not go
# to /var/log/message by default.
syslog.syslog(syslog.LOG_ERR, proc_params)
if pid_fd is not None:
if type(pid_fd) is type(0):
os.write(pid_fd, "%d\n" % os.getpid())
os.fsync(pid_fd)
else:
pid_fd.write("%d\n" % os.getpid())
pid_fd.flush()
os.fsync(pid_fd.fileno())
# Start specific function.
try:
ret = proc()
except Exception:
import StringIO
import traceback
ret = ERROR_BAD_ENVIRONMENT
exception_strfd = StringIO.StringIO()
traceback.print_exc(file=exception_strfd)
msg = "FATAL: Daemon got unhandled exception."
write_log(STDERR_FD, None, ret, msg)
for each_line in exception_strfd.getvalue().split("\n"):
write_log(STDERR_FD, None, ret, each_line)
msg = "FATAL: Traceback printed. Exit gracefully."
write_log(STDERR_FD, None, ret, msg)
if ret != ERROR_SUCCESS:
msg = "FATAL: proc() exit with code: %d" % ret
write_log(STDERR_FD, None, ret, msg)
os.exit(ret) # We should do cleanup here.
else:
os._exit(ERROR_SUCCESS)
else:
os._exit(ERROR_SUCCESS)
def set_max_runtime(seconds):
# Install the signal handler and set a resource limit
soft, hard = resource.getrlimit(resource.RLIMIT_CPU)
resource.setrlimit(resource.RLIMIT_CPU, (seconds, hard))
signal.signal(signal.SIGXCPU, time_exceeded) #程序运行时，SIGXCPU信号在时间过期时被生成，然后执行清理并退出。
def fit(self, X, Y):
import sklearn.svm
# Calculate the size of the kernel cache (in MB) for sklearn's LibSVM. The cache size is
# calculated as 2/3 of the available memory (which is calculated as the memory limit minus
# the used memory)
try:
# Retrieve memory limits imposed on the process
soft, hard = resource.getrlimit(resource.RLIMIT_AS)
if soft > 0:
# Convert limit to units of megabytes
soft /= 1024 * 1024
# Retrieve memory used by this process
maxrss = resource.getrusage(resource.RUSAGE_SELF)[2] / 1024
# In MacOS, the MaxRSS output of resource.getrusage in bytes; on other platforms,
# it's in kilobytes
if sys.platform == 'darwin':
maxrss = maxrss / 1024
cache_size = (soft - maxrss) / 1.5
if cache_size < 0:
cache_size = 200
else:
cache_size = 200
except Exception:
cache_size = 200
self.C = float(self.C)
if self.degree is None:
self.degree = 3
else:
self.degree = int(self.degree)
if self.gamma is None:
self.gamma = 0.0
else:
self.gamma = float(self.gamma)
if self.coef0 is None:
self.coef0 = 0.0
else:
self.coef0 = float(self.coef0)
self.tol = float(self.tol)
self.max_iter = float(self.max_iter)
self.shrinking = check_for_bool(self.shrinking)
if check_none(self.class_weight):
self.class_weight = None
self.estimator = sklearn.svm.SVC(C=self.C,
kernel=self.kernel,
degree=self.degree,
gamma=self.gamma,
coef0=self.coef0,
shrinking=self.shrinking,
tol=self.tol,
class_weight=self.class_weight,
max_iter=self.max_iter,
random_state=self.random_state,
cache_size=cache_size,
decision_function_shape='ovr')
self.estimator.fit(X, Y)
return self
def _setLimits(self):
self.logger.debug('Setting rlimits')
for limit in (resource.RLIMIT_NPROC, resource.RLIMIT_NOFILE):
soft, hard = resource.getrlimit(resource.RLIMIT_NPROC)
resource.setrlimit(resource.RLIMIT_NPROC, (hard, hard))
def test_max_open_files(self):
with TestsLoggingHandler() as handler:
logmsg_dbg = (
'DEBUG:This salt-master instance has accepted {0} minion keys.'
)
logmsg_chk = (
'{0}:The number of accepted minion keys({1}) should be lower '
'than 1/4 of the max open files soft setting({2}). According '
'to the system\'s hard limit, there\'s still a margin of {3} '
'to raise the salt\'s max_open_files setting. Please consider '
'raising this value.')
logmsg_crash = (
'{0}:The number of accepted minion keys({1}) should be lower '
'than 1/4 of the max open files soft setting({2}). '
'salt-master will crash pretty soon! According to the '
'system\'s hard limit, there\'s still a margin of {3} to '
'raise the salt\'s max_open_files setting. Please consider '
'raising this value.')
mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE)
tempdir = tempfile.mkdtemp(prefix='fake-keys')
keys_dir = os.path.join(tempdir, 'minions')
os.makedirs(keys_dir)
mof_test = 256
resource.setrlimit(resource.RLIMIT_NOFILE, (mof_test, mof_h))
try:
prev = 0
for newmax, level in ((24, None), (66, 'INFO'),
(127, 'WARNING'), (196, 'CRITICAL')):
for n in range(prev, newmax):
kpath = os.path.join(keys_dir, str(n))
with salt.utils.fopen(kpath, 'w') as fp_:
fp_.write(str(n))
opts = {'max_open_files': newmax, 'pki_dir': tempdir}
check_max_open_files(opts)
if level is None:
# No log message is triggered, only the DEBUG one which
# tells us how many minion keys were accepted.
self.assertEqual([logmsg_dbg.format(newmax)],
handler.messages)
else:
self.assertIn(logmsg_dbg.format(newmax),
handler.messages)
self.assertIn(
logmsg_chk.format(
level,
newmax,
mof_test,
mof_h - newmax,
), handler.messages)
handler.clear()
prev = newmax
newmax = mof_test
for n in range(prev, newmax):
kpath = os.path.join(keys_dir, str(n))
with salt.utils.fopen(kpath, 'w') as fp_:
fp_.write(str(n))
opts = {'max_open_files': newmax, 'pki_dir': tempdir}
check_max_open_files(opts)
self.assertIn(logmsg_dbg.format(newmax), handler.messages)
self.assertIn(
logmsg_crash.format(
'CRITICAL',
newmax,
mof_test,
mof_h - newmax,
), handler.messages)
handler.clear()
except IOError as err:
if err.errno == 24:
# Too many open files
self.skipTest('We\'ve hit the max open files setting')
raise
finally:
shutil.rmtree(tempdir)
resource.setrlimit(resource.RLIMIT_NOFILE, (mof_s, mof_h))
enc = preferred_encoding
if isosx:
enc = 'utf-8'
for i in range(1, len(sys.argv)):
if not isinstance(sys.argv[i], unicode):
sys.argv[i] = sys.argv[i].decode(enc, 'replace')
#
# Ensure that the max number of open files is at least 1024
if iswindows:
# See https://msdn.microsoft.com/en-us/library/6e3b887c.aspx
if hasattr(winutil, 'setmaxstdio'):
winutil.setmaxstdio(max(1024, winutil.getmaxstdio()))
else:
import resource
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
if soft < 1024:
try:
resource.setrlimit(resource.RLIMIT_NOFILE,
(min(1024, hard), hard))
except Exception:
if DEBUG:
import traceback
traceback.print_exc()
#
# Setup resources
import calibre.utils.resources as resources
resources
#
def open_file_limit():
return resource.getrlimit(resource.RLIMIT_NOFILE)[0]
except OSError:
pass
else:
if curMTime > lastMTime:
netlist = loadList(filename)
lastMTime = curMTime
line = sys.stdin.readline()
if __name__ == '__main__':
# We appear to end up with superfluous FDs, including pipes from other
# instances, forked from PowerDNS. This can keep us and others from
# exiting as the fd never gets closed. Close all fds we don't need.
try:
import resource
maxfds = resource.getrlimit(resource.RLIMIT_NOFILE)[1] + 1
# OS-X reports 9223372036854775808. That's a lot of fds to close
if maxfds > 1024:
maxfds = 1024
except:
maxfds = 256
import os
for fd in range(3, maxfds):
try:
os.close(fd)
except:
pass
main()
def main(infile, outfile):
try:
boot_time = time.time()
split_index = read_int(infile)
if split_index == -1: # for unit tests
sys.exit(-1)
version = utf8_deserializer.loads(infile)
if version != "%d.%d" % sys.version_info[:2]:
raise Exception((
"Python in worker has different version %s than that in " +
"driver %s, PySpark cannot run with different minor versions. "
+ "Please check environment variables PYSPARK_PYTHON and " +
"PYSPARK_DRIVER_PYTHON are correctly set.") %
("%d.%d" % sys.version_info[:2], version))
# read inputs only for a barrier task
isBarrier = read_bool(infile)
boundPort = read_int(infile)
secret = UTF8Deserializer().loads(infile)
# set up memory limits
memory_limit_mb = int(
os.environ.get('PYSPARK_EXECUTOR_MEMORY_MB', "-1"))
if memory_limit_mb > 0 and has_resource_module:
total_memory = resource.RLIMIT_AS
try:
(soft_limit, hard_limit) = resource.getrlimit(total_memory)
msg = "Current mem limits: {0} of max {1}\n".format(
soft_limit, hard_limit)
print(msg, file=sys.stderr)
# convert to bytes
new_limit = memory_limit_mb * 1024 * 1024
if soft_limit == resource.RLIM_INFINITY or new_limit < soft_limit:
msg = "Setting mem limits to {0} of max {1}\n".format(
new_limit, new_limit)
print(msg, file=sys.stderr)
resource.setrlimit(total_memory, (new_limit, new_limit))
except (resource.error, OSError, ValueError) as e:
# not all systems support resource limits, so warn instead of failing
print("WARN: Failed to set memory limit: {0}\n".format(e),
file=sys.stderr)
# initialize global state
taskContext = None
if isBarrier:
taskContext = BarrierTaskContext._getOrCreate()
BarrierTaskContext._initialize(boundPort, secret)
# Set the task context instance here, so we can get it by TaskContext.get for
# both TaskContext and BarrierTaskContext
TaskContext._setTaskContext(taskContext)
else:
taskContext = TaskContext._getOrCreate()
# read inputs for TaskContext info
taskContext._stageId = read_int(infile)
taskContext._partitionId = read_int(infile)
taskContext._attemptNumber = read_int(infile)
taskContext._taskAttemptId = read_long(infile)
taskContext._resources = {}
for r in range(read_int(infile)):
key = utf8_deserializer.loads(infile)
name = utf8_deserializer.loads(infile)
addresses = []
taskContext._resources = {}
for a in range(read_int(infile)):
addresses.append(utf8_deserializer.loads(infile))
taskContext._resources[key] = ResourceInformation(name, addresses)
taskContext._localProperties = dict()
for i in range(read_int(infile)):
k = utf8_deserializer.loads(infile)
v = utf8_deserializer.loads(infile)
taskContext._localProperties[k] = v
shuffle.MemoryBytesSpilled = 0
shuffle.DiskBytesSpilled = 0
_accumulatorRegistry.clear()
# fetch name of workdir
spark_files_dir = utf8_deserializer.loads(infile)
SparkFiles._root_directory = spark_files_dir
SparkFiles._is_running_on_worker = True
# fetch names of includes (*.zip and *.egg files) and construct PYTHONPATH
add_path(
spark_files_dir) # *.py files that were added will be copied here
num_python_includes = read_int(infile)
for _ in range(num_python_includes):
filename = utf8_deserializer.loads(infile)
add_path(os.path.join(spark_files_dir, filename))
if sys.version > '3':
import importlib
importlib.invalidate_caches()
# fetch names and values of broadcast variables
needs_broadcast_decryption_server = read_bool(infile)
num_broadcast_variables = read_int(infile)
if needs_broadcast_decryption_server:
# read the decrypted data from a server in the jvm
port = read_int(infile)
auth_secret = utf8_deserializer.loads(infile)
(broadcast_sock_file,
_) = local_connect_and_auth(port, auth_secret)
for _ in range(num_broadcast_variables):
bid = read_long(infile)
if bid >= 0:
if needs_broadcast_decryption_server:
read_bid = read_long(broadcast_sock_file)
assert (read_bid == bid)
_broadcastRegistry[bid] = \
Broadcast(sock_file=broadcast_sock_file)
else:
path = utf8_deserializer.loads(infile)
_broadcastRegistry[bid] = Broadcast(path=path)
else:
bid = -bid - 1
_broadcastRegistry.pop(bid)
if needs_broadcast_decryption_server:
broadcast_sock_file.write(b'1')
broadcast_sock_file.close()
_accumulatorRegistry.clear()
eval_type = read_int(infile)
if eval_type == PythonEvalType.NON_UDF:
func, profiler, deserializer, serializer = read_command(
pickleSer, infile)
else:
func, profiler, deserializer, serializer = read_udfs(
pickleSer, infile, eval_type)
init_time = time.time()
def process():
iterator = deserializer.load_stream(infile)
out_iter = func(split_index, iterator)
try:
serializer.dump_stream(out_iter, outfile)
finally:
if hasattr(out_iter, 'close'):
out_iter.close()
if profiler:
profiler.profile(process)
else:
process()
# Reset task context to None. This is a guard code to avoid residual context when worker
# reuse.
TaskContext._setTaskContext(None)
BarrierTaskContext._setTaskContext(None)
except BaseException:
try:
exc_info = traceback.format_exc()
if isinstance(exc_info, bytes):
# exc_info may contains other encoding bytes, replace the invalid bytes and convert
# it back to utf-8 again
exc_info = exc_info.decode("utf-8", "replace").encode("utf-8")
else:
exc_info = exc_info.encode("utf-8")
write_int(SpecialLengths.PYTHON_EXCEPTION_THROWN, outfile)
write_with_length(exc_info, outfile)
except IOError:
# JVM close the socket
pass
except BaseException:
# Write the error to stderr if it happened while serializing
print("PySpark worker failed with exception:", file=sys.stderr)
print(traceback.format_exc(), file=sys.stderr)
sys.exit(-1)
finish_time = time.time()
report_times(outfile, boot_time, init_time, finish_time)
write_long(shuffle.MemoryBytesSpilled, outfile)
write_long(shuffle.DiskBytesSpilled, outfile)
# Mark the beginning of the accumulators section of the output
write_int(SpecialLengths.END_OF_DATA_SECTION, outfile)
write_int(len(_accumulatorRegistry), outfile)
for (aid, accum) in _accumulatorRegistry.items():
pickleSer._write_with_length((aid, accum._value), outfile)
# check end of stream
if read_int(infile) == SpecialLengths.END_OF_STREAM:
write_int(SpecialLengths.END_OF_STREAM, outfile)
else:
# write a different value to tell JVM to not reuse this worker
write_int(SpecialLengths.END_OF_DATA_SECTION, outfile)
sys.exit(-1)
def main(host, port, http_port, bokeh_port, show, _bokeh, bokeh_whitelist,
prefix, use_xheaders, pid_file):
if pid_file:
with open(pid_file, 'w') as f:
f.write(str(os.getpid()))
def del_pid_file():
if os.path.exists(pid_file):
os.remove(pid_file)
atexit.register(del_pid_file)
if sys.platform.startswith('linux'):
import resource # module fails importing on Windows
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
limit = max(soft, hard // 2)
resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
given_host = host
host = host or get_ip()
if ':' in host and port == 8786:
host, port = host.rsplit(':', 1)
port = int(port)
ip = socket.gethostbyname(host)
loop = IOLoop.current()
logger.info('-' * 47)
services = {('http', http_port): HTTPScheduler}
if _bokeh:
from distributed.bokeh.scheduler import BokehScheduler
services[('bokeh', 8788)] = BokehScheduler
scheduler = Scheduler(ip=ip, loop=loop, services=services)
scheduler.start(port)
bokeh_proc = None
if _bokeh:
try:
from distributed.bokeh.application import BokehWebInterface
bokeh_proc = BokehWebInterface(host=host,
http_port=http_port,
tcp_port=port,
bokeh_port=bokeh_port,
bokeh_whitelist=bokeh_whitelist,
show=show,
prefix=prefix,
use_xheaders=use_xheaders,
quiet=False)
except ImportError:
logger.info("Please install Bokeh to get Web UI")
except Exception as e:
logger.warn("Could not start Bokeh web UI", exc_info=True)
logger.info('-' * 47)
try:
loop.start()
loop.close()
finally:
scheduler.stop()
if bokeh_proc:
bokeh_proc.close()
logger.info("End scheduler at %s:%d", ip, port)
def __init__(self,
application,
environ=None,
bindAddress=None,
umask=None,
multiplexed=False,
debug=False,
roles=(FCGI_RESPONDER, ),
forceCGI=False,
timeout=None,
**kw):
"""
environ, if present, must be a dictionary-like object. Its
contents will be copied into application's environ. Useful
for passing application-specific variables.
bindAddress, if present, must either be a string or a 2-tuple. If
present, run() will open its own listening socket. You would use
this if you wanted to run your application as an 'external' FastCGI
app. (i.e. the webserver would no longer be responsible for starting
your app) If a string, it will be interpreted as a filename and a UNIX
socket will be opened. If a tuple, the first element, a string,
is the interface name/IP to bind to, and the second element (an int)
is the port number.
"""
BaseFCGIServer.__init__(self,
application,
environ=environ,
multithreaded=False,
multiprocess=True,
bindAddress=bindAddress,
umask=umask,
multiplexed=multiplexed,
debug=debug,
roles=roles,
forceCGI=forceCGI)
for key in ('multithreaded', 'multiprocess', 'jobClass', 'jobArgs'):
if key in kw:
del kw[key]
PreforkServer.__init__(self,
jobClass=self._connectionClass,
jobArgs=(self, timeout),
**kw)
try:
import resource
# Attempt to glean the maximum number of connections
# from the OS.
try:
maxProcs = resource.getrlimit(resource.RLIMIT_NPROC)[0]
maxConns = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
maxConns = min(maxConns, maxProcs)
except AttributeError:
maxConns = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
except ImportError:
maxConns = 100 # Just some made up number.
maxReqs = maxConns
self.capability = {
FCGI_MAX_CONNS: maxConns,
FCGI_MAX_REQS: maxReqs,
FCGI_MPXS_CONNS: 0
}
import importlib
import pytest
import syncopy
from syncopy import __acme__
import syncopy.tests.test_packagesetup as setupTestModule
# If dask is available, either launch a SLURM cluster on a cluster node or
# create a `LocalCluster` object if tests are run on a single machine. If dask
# is not installed, return a dummy None-valued cluster object (tests will be
# skipped anyway)
if __acme__:
import dask.distributed as dd
import resource
from acme.dask_helpers import esi_cluster_setup
from syncopy.tests.misc import is_slurm_node
if max(resource.getrlimit(resource.RLIMIT_NOFILE)) < 1024:
msg = "Not enough open file descriptors allowed. Consider increasing " +\
"the limit using, e.g., `ulimit -Sn 1024`"
raise ValueError(msg)
if is_slurm_node():
cluster = esi_cluster_setup(partition="8GB", n_jobs=10,
timeout=360, interactive=False,
start_client=False)
else:
cluster = dd.LocalCluster(n_workers=2)
else:
cluster = None
# Set up a pytest fixture `testcluster` that uses the constructed cluster object
@pytest.fixture
def testcluster():
def rlimitTestAndSet(name, limit):
"Helper function to set rlimits"
soft, hard = getrlimit(name)
if soft < limit:
hardLimit = hard if limit < hard else limit
setrlimit(name, (limit, hardLimit))
# print out usage if incorrect arguments are passed
if not vars(args):
parser.print_help()
parser.exit(1)
percent_limit = args.percent_limit
# Pull the max open fd's and currently open fd's
file_nr = open("/proc/sys/fs/file-nr").read().strip()
open_fd = file_nr.split()[0]
max_fd = file_nr.split()[-1]
percentage_max_fd = round(float(open_fd) / float(max_fd) * 100, 2)
if percentage_max_fd > percent_limit:
print "Total open file descriptors have reached", percentage_max_fd, "% of the system max (", open_fd, " of ", max_fd, ")"
# get the soft limit and store it, to use as backup
default_limit = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]
for pid in pids:
try:
pid = pid.strip()
soft_limit = default_limit
limits = open("/proc/" + pid + "/limits").readlines() # get the limits for the proc
for line in limits:
if line.startswith("Max open files"):
soft_limit = int(line.split()[3]) # splitting the line on spaces we want the 4th value
break
# used os.walk() to get the names in the /proc/[pid]/fd dir
fds = []
for (dirpath, dirname, filenames) in os.walk("/proc/" + pid + "/fd/"):
