def setup(self):
require_user('root')
self._nofile = resource.getrlimit(resource.RLIMIT_NOFILE)
soft, hard = self._nofile
new_limit = (min(soft, RESPAWNS / 2), min(hard, RESPAWNS / 2))
resource.setrlimit(resource.RLIMIT_NOFILE, new_limit)
self._socket_fd_count = count_socket_fds()
def preexec_fn():
streams = [sys.stdin]
if self.close_child_stdout:
streams.append(sys.stdout)
if self.close_child_stderr:
streams.append(sys.stderr)
self._null_streams(streams)
os.setsid()
for limit, value in self.rlimits.items():
res = getattr(resource, 'RLIMIT_%s' % limit.upper(), None)
if res is None:
raise ValueError('unknown rlimit "%s"' % limit)
# TODO(petef): support hard/soft limits
resource.setrlimit(res, (value, value))
if self.gid:
try:
os.setgid(self.gid)
except OverflowError:
if not ctypes:
raise
# versions of python < 2.6.2 don't manage unsigned int for
# groups like on osx or fedora
os.setgid(-ctypes.c_int(-self.gid).value)
if self.uid:
os.setuid(self.uid)
def _suppress_core_files(self):
"""Try to prevent core files from being created.
Returns previous ulimit if successful, else None.
"""
if sys.platform == 'darwin':
# Check if the 'Crash Reporter' on OSX was configured
# in 'Developer' mode and warn that it will get triggered
# when it is.
#
# This assumes that this context manager is used in tests
# that might trigger the next manager.
value = subprocess.Popen(['/usr/bin/defaults', 'read',
'com.apple.CrashReporter', 'DialogType'],
stdout=subprocess.PIPE).communicate()[0]
if value.strip() == b'developer':
print "this tests triggers the Crash Reporter, that is intentional"
sys.stdout.flush()
try:
import resource
old_limit = resource.getrlimit(resource.RLIMIT_CORE)
resource.setrlimit(resource.RLIMIT_CORE, (0,0))
return old_limit
except (ImportError, ValueError, resource.error):
return None
def test_configurations_signed_data(self):
# Use a limit of ~4GiB
limit = 4000 * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, (limit, limit))
cs = ParamSklearnClassifier.get_hyperparameter_search_space(
dataset_properties={'signed': True})
print(cs)
for i in range(10):
config = cs.sample_configuration()
config._populate_values()
if config['classifier:passive_aggressive:n_iter'] is not None:
config._values['classifier:passive_aggressive:n_iter'] = 5
if config['classifier:sgd:n_iter'] is not None:
config._values['classifier:sgd:n_iter'] = 5
X_train, Y_train, X_test, Y_test = get_dataset(dataset='digits')
cls = ParamSklearnClassifier(config, random_state=1)
print(config)
try:
cls.fit(X_train, Y_train)
X_test_ = X_test.copy()
predictions = cls.predict(X_test)
self.assertIsInstance(predictions, np.ndarray)
predicted_probabiliets = cls.predict_proba(X_test_)
self.assertIsInstance(predicted_probabiliets, np.ndarray)
except ValueError as e:
if "Floating-point under-/overflow occurred at epoch" in \
e.args[0] or \
"removed all features" in e.args[0] or \
"all features are discarded" in e.args[0]:
continue
else:
print(config)
print(traceback.format_exc())
raise e
except RuntimeWarning as e:
if "invalid value encountered in sqrt" in e.args[0]:
continue
elif "divide by zero encountered in" in e.args[0]:
continue
elif "invalid value encountered in divide" in e.args[0]:
continue
elif "invalid value encountered in true_divide" in e.args[0]:
continue
else:
print(config)
print(traceback.format_exc())
raise e
except UserWarning as e:
if "FastICA did not converge" in e.args[0]:
continue
else:
print(config)
print(traceback.format_exc())
raise e
except MemoryError as e:
continue
def init_tests(self):
""" Initialize testing infrastructure - sockets, resource limits, etc. """
# Init Twisted factory.
self.server_factory = network.TestServerFactory(controller = self)
#self.client_factory = TestClientFactory(controller = self)
ports = sorted(self.test_ports)
log.notice("Binding to test ports: %s", ", ".join(map(str, ports)))
# Sort to try privileged ports first, since sets have no
# guaranteed ordering.
for port in ports:
reactor.listenTCP(port, self.server_factory)
# Set RLIMIT_NOFILE to its hard limit; we want to be able to
# use as many file descriptors as the system will allow.
# NOTE: Your soft/hard limits are inherited from the root user!
# The root user does NOT always have unlimited file descriptors.
# Take this into account when editing /etc/security/limits.conf.
(soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
log.verbose1("RLIMIT_NOFILE: soft = %d, hard = %d", soft, hard)
if soft < hard:
log.debug("Increasing RLIMIT_NOFILE soft limit to %d.", hard)
resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
log.debug("Initializing test threads.")
# TODO: Configure me!
scheduler_class = getattr(scheduler, config.scheduler)
self.scheduler = scheduler_class(controller = self,
max_pending_factor = config.max_pending_factor,
export_interval = config.export_interval)
T = threading.Thread
self.schedule_thread = T(target = Controller.test_schedule_thread,
name = "Scheduler", args = (self,))
self.watchdog_thread = T(target = Controller.watchdog,
name = "Watchdog", args = (self,))
def main():
args = parse_args()
program = args.program
if not os.path.isabs(program):
# program uses a relative path: try to find the absolute path
# to the executable
if sys.version_info >= (3, 3):
import shutil
program_abs = shutil.which(program)
else:
import distutils.spawn
program_abs = distutils.spawn.find_executable(program)
if program_abs:
program = program_abs
for arg_name, rlimit in RESOURCES:
value = getattr(args, arg_name)
if value is None:
continue
try:
resource.setrlimit(rlimit, (value, value))
except ValueError as exc:
print("%s: failed to set the %s resource limit: %s"
% (USAGE_PROGRAM, arg_name.upper(), exc),
file=sys.stderr)
sys.exit(1)
try:
os.execv(program, [program] + args.program_args)
except Exception as exc:
print("%s: failed to execute %s: %s"
% (USAGE_PROGRAM, program, exc),
file=sys.stderr)
sys.exit(1)
def _restore_rlimits(self):
try:
import resource
resource.setrlimit(resource.RLIMIT_NOFILE, (self.soft_max_open_files, self.hard_max_open_files))
except ImportError:
pass
def set_limit(kind, soft_limit, hard_limit=None):
hard_limit = hard_limit or soft_limit
try:
resource.setrlimit(kind, (soft_limit, hard_limit))
except (OSError, ValueError), err:
sys.stderr.write('Resource limit for %s could not be set to %s (%s)\n' %
(kind, (soft_limit, hard_limit), err))
def process_limit(self):
# If our parent changed sucide
if self.ppid != os.getppid():
_logger.info("Worker (%s) Parent changed", self.pid)
self.alive = False
# check for lifetime
if self.request_count >= self.request_max:
_logger.info("Worker (%d) max request (%s) reached.", self.pid, self.request_count)
self.alive = False
# Reset the worker if it consumes too much memory (e.g. caused by a memory leak).
rss, vms = memory_info(psutil.Process(os.getpid()))
if vms > config['limit_memory_soft']:
_logger.info('Worker (%d) virtual memory limit (%s) reached.', self.pid, vms)
self.alive = False # Commit suicide after the request.
# VMS and RLIMIT_AS are the same thing: virtual memory, a.k.a. address space
soft, hard = resource.getrlimit(resource.RLIMIT_AS)
resource.setrlimit(resource.RLIMIT_AS, (config['limit_memory_hard'], hard))
# SIGXCPU (exceeded CPU time) signal handler will raise an exception.
r = resource.getrusage(resource.RUSAGE_SELF)
cpu_time = r.ru_utime + r.ru_stime
def time_expired(n, stack):
_logger.info('Worker (%d) CPU time limit (%s) reached.', self.pid, config['limit_time_cpu'])
# We dont suicide in such case
raise Exception('CPU time limit exceeded.')
signal.signal(signal.SIGXCPU, time_expired)
soft, hard = resource.getrlimit(resource.RLIMIT_CPU)
resource.setrlimit(resource.RLIMIT_CPU, (cpu_time + config['limit_time_cpu'], hard))
def collate(self, restart=False):
import resource as res
# Set the stacksize to be unlimited
res.setrlimit(res.RLIMIT_STACK, (res.RLIM_INFINITY, res.RLIM_INFINITY))
restart_path = os.path.join(self.run_path, 'RESTART')
nc_files = [os.path.join(self.run_path, f)
for f in os.listdir(self.run_path)
if f.endswith('.nc.0000')]
if restart:
restart_files = [os.path.join(restart_path, f)
for f in os.listdir(restart_path)
if f.endswith('.nc.0000')]
nc_files.extend(restart_files)
mppnc_path = os.path.join(self.bin_path, 'mppnccombine')
for f in nc_files:
cmd = [mppnc_path, '-r', '-64', f]
sp.Popen(cmd).wait()
if self.archive_group:
self.regroup()
def set_open_files_limit(desired_limit):
"""
On POSIX systems, set the open files limit to the desired number, unless
it is already equal to or higher than that.
Setting a high limit enables Flintrock to launch or interact with really
large clusters.
Background discussion: https://github.com/nchammas/flintrock/issues/81
"""
soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
if soft_limit < desired_limit:
if desired_limit > hard_limit:
warnings.warn(
"Flintrock cannot set the open files limit to {desired} "
"because the OS hard limit is {hard}. Going with {hard}. "
"You may have problems launching or interacting with "
"really large clusters."
.format(
desired=desired_limit,
hard=hard_limit),
category=RuntimeWarning,
stacklevel=2)
resource.setrlimit(
resource.RLIMIT_NOFILE,
(min(desired_limit, hard_limit), hard_limit))
def setUp(self):
self.origFileLimit = resource.getrlimit(resource.RLIMIT_NOFILE)
newlimit = (self.OPEN_FILE_LIMIT, self.origFileLimit[1])
if newlimit[0] > newlimit[1]:
self.skipTest('Require %d open files, only %d available' % newlimit)
resource.setrlimit(resource.RLIMIT_NOFILE, newlimit)
super(test_schema03, self).setUp()
def launch_downloader_daemon():
# Increase the maximum file descriptor count (to the max)
# NOTE: the info logging is REQUIRED for some unknown reason, if it is not
# done here, no further logging can be done in the daemon and it gets stuck.
try:
import resource
logging.info('Increasing file descriptor count limit in Downloader')
resource.setrlimit(resource.RLIMIT_NOFILE, (10240, -1))
except ValueError:
logging.warn('setrlimit failed.')
# Make sure we don't leak from the downloader eventloop
from miro import eventloop
def beginLoop(loop):
loop.pool = Foundation.NSAutoreleasePool.alloc().init()
eventloop.connect('begin-loop', beginLoop)
eventloop.connect('thread-will-start', beginLoop)
def endLoop(loop):
del loop.pool
eventloop.connect('end-loop', endLoop)
eventloop.connect('thread-did-start', endLoop)
# And launch
from miro.dl_daemon import Democracy_Downloader
Democracy_Downloader.launch()
# Wait for the event loop thread to finish.
# Although this is theorically not necessary since the event loop thread is
# a non-daemon thread, situations where the downloader daemon exits right
# after its launch as this function returns have been seen in the wild.
eventloop.join()
def run(self):
# START LINUX: used to receive Memory Error faster in Linux
try:
import resource
resource.setrlimit(resource.RLIMIT_AS, (2 ** 30, 2 ** 30)) # 2 ** 30 == 1GB in bytes
except ImportError:
pass
# END LINUX:
while True:
exception = None
self.stop.value = False
script = self.consume.get()
if script:
help_globals = {'stop': self.stop}
try:
exec(script, help_globals)
except BaseException as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
exception = '{} {} {}'.format(exc_type, exc_obj, e.args)
if exception:
self.produce.put({'exception': exception})
else:
self.produce.put({key: value for key, value in help_globals.items()
if not callable(value) and # cannot be a function
not isinstance(value, ModuleType) and # cannot be a module
key not in ['__builtins__', 'stop']}) # cannot be built ins or synchronized objects
del help_globals
else:
break
def get_contest(self):
"""See docstring in class Loader.
"""
# Unset stack size limit
resource.setrlimit(resource.RLIMIT_STACK, (resource.RLIM_INFINITY,
resource.RLIM_INFINITY))
self.buildpath = os.path.join(self.path, "build")
# We have to avoid copying the folder contest/build
# or contest/task/build into contest/build.
# For this reason, we ignore all files and directories named "build"
# when copying recursively.
copyrecursivelyifnecessary((self.path), self.buildpath,
set(["build"]))
with chdir(self.buildpath):
rules = ".rules"
if not os.path.exists(rules):
os.mkdir(rules)
rules = os.path.abspath(rules)
self.contestconfig = ContestConfig(rules,
os.path.basename(self.path))
self.contestconfig._readconfig("contest-config.py")
tasknames = [t.name for t in self.contestconfig.tasks]
usernames = [u.username for u in self.contestconfig.users]
return self.contestconfig._makecontest(), tasknames, usernames
def init( libsyndicate, gateway_key_path ):
'''
Initialize this module.
'''
global GATEWAY_PRIVKEY_PEM
global SENDER_PUBKEY_PEM
# disable core dumps (don't want our private key to get leaked)
resource.setrlimit( resource.RLIMIT_CORE, (0, 0) )
gateway_privkey_pem = None
if gateway_key_path is not None:
fd = open( gateway_key_path, "r" )
gateway_privkey_pem = fd.read()
fd.close()
# verify
try:
gateway_privkey = CryptoKey.importKey( gateway_privkey_pem )
assert gateway_privkey.has_private(), "Not a private key"
except Exception, e:
log.exception(e)
log.error("Failed to load Gateway private key %s" % gateway_key_path )
return -1
def set_limits ():
import resource as r
if self.cpu > 0:
r.setrlimit (r.RLIMIT_CPU, [self.cpu, self.cpu])
if self.mem > 0:
mem_bytes = self.mem * 1024 * 1024
r.setrlimit (r.RLIMIT_AS, [mem_bytes, mem_bytes])
def prep_filehandles(self):
smax_open_files, hmax_open_files = resource.getrlimit(
resource.RLIMIT_NOFILE
)
if smax_open_files < REQUIRED_OPEN_FILES:
print(
' * Max open files setting is too low({0}) for running the '
'tests'.format(smax_open_files)
)
print(
' * Trying to raise the limit to {0}'.format(REQUIRED_OPEN_FILES)
)
if hmax_open_files < 4096:
hmax_open_files = 4096 # Decent default?
try:
resource.setrlimit(
resource.RLIMIT_NOFILE,
(REQUIRED_OPEN_FILES, hmax_open_files)
)
except Exception as err:
print(
'ERROR: Failed to raise the max open files setting -> '
'{0}'.format(err)
)
print('Please issue the following command on your console:')
print(' ulimit -n {0}'.format(REQUIRED_OPEN_FILES))
self.exit()
finally:
print('~' * getattr(self.options, 'output_columns', PNUM))
def _init_core_dump(self):
import resource
try:
limits = (resource.RLIM_INFINITY, resource.RLIM_INFINITY)
resource.setrlimit(resource.RLIMIT_CORE, limits)
except:
pass
def test_connection_del(self, tmpdir):
"""For issue1325."""
import os
import gc
try:
import resource
except ImportError:
pytest.skip("needs resource module")
limit = resource.getrlimit(resource.RLIMIT_NOFILE)
try:
fds = 0
while True:
fds += 1
resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1]))
try:
for p in os.pipe(): os.close(p)
except OSError:
assert fds < 100
else:
break
def open_many(cleanup):
con = []
for i in range(3):
con.append(_sqlite3.connect(str(tmpdir.join('test.db'))))
if cleanup:
con[i] = None
gc.collect(); gc.collect()
pytest.raises(_sqlite3.OperationalError, open_many, False)
gc.collect(); gc.collect()
open_many(True)
finally:
resource.setrlimit(resource.RLIMIT_NOFILE, limit)
def stdinfuzz(self):
buf = ''
while True:
try:
buf += '\x41'
resource.setrlimit(resource.RLIMIT_STACK, (-1, -1))
resource.setrlimit(resource.RLIMIT_CORE, (-1, -1))
P = Popen(self.target, stdin=PIPE)
print "[*] Sending buffer with lenght: "+str(len(buf))
P.stdin.write(buf+ "\n")
line = sys.stdin.readline()
P.poll()
ret = P.returncode
if ret is None:
continue
else:
if ret == -11:
print "\n[*] Child program crashed with SIGSEGV\n"
print "\n[*] Hit enter to continue.\n"
continue
elif ret == -4:
print "\n[+] Instruction Pointer will be at: {}\n".format(str(len(buf)))
break
else:
print "\n[*] Child program exited with code %d\n" % ret
print "\n[*] Hit enter to continue.\n"
continue
except KeyboardInterrupt:
break
def __init__(self, zmq_host, zmq_work_port, zmq_results_port, worker_id,
max_retries, profile_count=0, concurrency=256, batch_size=1):
work_endpoint = 'tcp://%s:%d' % (zmq_host, zmq_work_port)
results_endpoint = 'tcp://%s:%d' % (zmq_host, zmq_results_port)
self.worker_id = worker_id
self.max_retries = max_retries
self.profile_count = profile_count
self.batch_size = batch_size
soft_nofile, hard_nofile = resource.getrlimit(resource.RLIMIT_NOFILE)
nofile_target = 1024
if os.geteuid() == 0:
nofile_target = max(nofile_target, concurrency + 50)
hard_nofile = nofile_target
resource.setrlimit(resource.RLIMIT_NOFILE, (nofile_target,
hard_nofile))
self.concurrency = concurrency
self.conn_pools_lock = gevent.coros.Semaphore(1)
self.conn_pools = {} # hashed by storage_url
self.token_data = {}
self.token_data_lock = gevent.coros.Semaphore(1)
self.context = zmq.Context()
self.work_pull = self.context.socket(zmq.PULL)
self.work_pull.connect(work_endpoint)
self.results_push = self.context.socket(zmq.PUSH)
self.results_push.connect(results_endpoint)
self.result_queue = gevent.queue.Queue()
def limit(self):
resource.setrlimit(resource.RLIMIT_AS, (self.memorylimit.value, self.memorylimit.value + 16777216))
resource.setrlimit(resource.RLIMIT_CPU, (self.cpulimit.value, self.cpulimit.value + 1.0))
os.chroot("/tmp/pjudge/")
os.setgid(305)
os.setuid(305)
return 0
def setlimits():
cpu_limit = current_app.config.get('SKYLINES_SUBPROCESS_CPU', 120)
mem_limit = current_app.config.get('SKYLINES_SUBPROCESS_MEMORY', 256) \
* 1024 * 1024
resource.setrlimit(resource.RLIMIT_CPU, (cpu_limit, cpu_limit * 1.2))
resource.setrlimit(resource.RLIMIT_AS, (mem_limit, mem_limit * 1.2))
def __set_max_open_files(self):
# Let's check to see how our max open files(ulimit -n) setting is
mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE)
log.info(
'Current values for max open files soft/hard setting: '
'{0}/{1}'.format(
mof_s, mof_h
)
)
# Let's grab, from the configuration file, the value to raise max open
# files to
mof_c = self.opts['max_open_files']
if mof_c > mof_h:
# The configured value is higher than what's allowed
log.warning(
'The value for the \'max_open_files\' setting, {0}, is higher '
'than what the user running salt is allowed to raise to, {1}. '
'Defaulting to {1}.'.format(mof_c, mof_h)
)
mof_c = mof_h
if mof_s < mof_c:
# There's room to raise the value. Raise it!
log.warning('Raising max open files value to {0}'.format(mof_c))
resource.setrlimit(resource.RLIMIT_NOFILE, (mof_c, mof_h))
mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE)
log.warning(
'New values for max open files soft/hard values: '
'{0}/{1}'.format(mof_s, mof_h)
)
def action(self):
'''
Return the functions and the returners loaded up from the loader
module
'''
# if this is a *nix system AND modules_max_memory is set, lets enforce
# a memory limit on module imports
# this feature ONLY works on *nix like OSs (resource module doesn't work on windows)
modules_max_memory = False
if self.opts.value.get('modules_max_memory', -1) > 0 and HAS_PSUTIL and HAS_RESOURCE:
log.debug(
'modules_max_memory set, enforcing a maximum of {0}'.format(
self.opts.value['modules_max_memory'])
)
modules_max_memory = True
old_mem_limit = resource.getrlimit(resource.RLIMIT_AS)
rss, vms = psutil.Process(os.getpid()).get_memory_info()
mem_limit = rss + vms + self.opts.value['modules_max_memory']
resource.setrlimit(resource.RLIMIT_AS, (mem_limit, mem_limit))
elif self.opts.value.get('modules_max_memory', -1) > 0:
if not HAS_PSUTIL:
log.error('Unable to enforce modules_max_memory because psutil is missing')
if not HAS_RESOURCE:
log.error('Unable to enforce modules_max_memory because resource is missing')
self.opts.value['grains'] = salt.loader.grains(self.opts.value)
self.grains.value = self.opts.value['grains']
self.modules.value = salt.loader.minion_mods(self.opts.value)
self.returners.value = salt.loader.returners(self.opts.value, self.modules.value)
# we're done, reset the limits!
if modules_max_memory is True:
resource.setrlimit(resource.RLIMIT_AS, old_mem_limit)
def initialize(self):
# initialize the base class
GPP_base.initialize(self)
self._popen_lock = threading.Lock()
# Setup the TMPDIR environment variable for use in diskCapacity requests
if not os.environ.has_key("TMPDIR"):
os.environ["TMPDIR"] = tempfile.gettempdir()
nproc = resource.getrlimit(resource.RLIMIT_NPROC)
if nproc[0] < nproc[1]:
#Max the softlimit out
resource.setrlimit(resource.RLIMIT_NPROC, (nproc[1], nproc[1]))
if nproc[1] < 1024:
self._log.warning("Your system nproc hard limit [%s} is set abnormally low", nproc[1])
######################
# Set initial capacities
self.memCapacity = int(self.memTotal * self.memThresholdDecimal) # starts out as the same value
self.loadCapacity = self.loadTotal * self.loadThresholdDecimal
self.mcastnicIngressCapacity = int(self.mcastnicIngressTotal * self.mcastNicThresholdDecimal)
self.mcastnicEgressCapacity = int(self.mcastnicEgressTotal * self.mcastNicThresholdDecimal)
self.init_processor_flags()
self.next_property_event = None
self.start()
def testClushConfigSetRlimit(self):
"""test CLI.Config.ClushConfig (setrlimit)"""
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
hard2 = min(32768, hard)
f = tempfile.NamedTemporaryFile(prefix='testclushconfig')
f.write("""
[Main]
fanout: 42
connect_timeout: 14
command_timeout: 0
history_size: 100
color: auto
fd_max: %d
verbosity: 1
""" % hard2)
f.flush()
parser = OptionParser("dummy")
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options, filename=f.name)
self.assert_(config != None)
display = Display(options, config)
self.assert_(display != None)
# force a lower soft limit
resource.setrlimit(resource.RLIMIT_NOFILE, (hard2/2, hard))
# max_fdlimit should increase soft limit again
set_fdlimit(config.fd_max, display)
# verify
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
self.assertEqual(soft, hard2)
f.close()
def nukeize(ip, port, rounds):
n=0
try: # RFC793 will lacks an exception if reset is not sent
resource.setrlimit(resource.RLIMIT_NOFILE, (100000, 100000)) # modify kernel ulimit to: 100000
os.system("iptables -A OUTPUT -d %s -p tcp --dport %d --tcp-flags RST RST -j DROP"%(ip, port)) # modify IPTABLES
os.system("iptables -A OUTPUT -d %s -p tcp --dport %d --tcp-flags FIN FIN -j DROP"%(ip, port))
epoll = select.epoll()
connections = {}
for x in range (0,int(rounds)):
try:
n=n+1
s = connect(ip, port)
print "[Info] [AI] [NUKE] Firing 'nuke' ["+str(n)+"] -> [SHOCKING!]"
connections[s.fileno()] = s
epoll.register(s.fileno(), select.EPOLLOUT|select.EPOLLONESHOT)
while True:
n=n+1
events = epoll.poll(1)
for fileno, event in events:
s = connections.pop(s.fileno())
print "[Info] [AI] [NUKE] Firing 'nuke' ["+str(n)+"] -> [SHOCKING!]"
if s:
s.close()
s = connect(ip, port)
connections[s.fileno()] = s
epoll.register(s.fileno(), select.EPOLLOUT|select.EPOLLONESHOT)
except:
print "[Error] [AI] [NUKE] Failed to engage with 'nuke' ["+str(n)+"]"
os.system('iptables -D OUTPUT -d %s -p tcp --dport %d --tcp-flags FIN FIN -j DROP' %(ip, port)) # restore IPTABLES
os.system('iptables -D OUTPUT -d %s -p tcp --dport %d --tcp-flags RST RST -j DROP' %(ip, port))
except:
print("[Error] [AI] [NUKE] Failing to engage... -> Is still target online? -> [Checking!]")
def chkernal():
resource.setrlimit(resource.RLIMIT_NOFILE,(1000000,1000000))
os.system('sysctl -w net.ipv4.ip_local_port_range="1025 65535"')
os.system('sysctl -w net.ipv4.tcp_rmem="4096 4096 16777216"')
os.system('sysctl -w net.ipv4.tcp_wmem="4096 4096 16777216"')
os.system('sysctl -w net.netfilter.nf_conntrack_max=250000')
def __enter__(self):
self._old_limits = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, self._limits)
from core.settings import SESSION_ID_LENGTH
from core.settings import SESSIONS
from core.settings import TRAILS_FILE
from core.settings import UNAUTHORIZED_SLEEP_TIME
from core.settings import VERSION
try:
# Reference: https://bugs.python.org/issue7980
# Reference: http://code-trick.com/python-bug-attribute-error-_strptime/
import _strptime
except ImportError:
pass
try:
import resource
resource.setrlimit(resource.RLIMIT_NOFILE, (MAX_NOFILE, MAX_NOFILE))
except:
pass
def start_httpd(address=None, port=None, join=False, pem=None):
"""
Starts HTTP server
"""
class ThreadingServer(SocketServer.ThreadingMixIn,
BaseHTTPServer.HTTPServer):
def server_bind(self):
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
BaseHTTPServer.HTTPServer.server_bind(self)
def finish_request(self, *args, **kwargs):
def setnmaplimits():
"""Enforces limits from NMAP_LIMITS global variable."""
for limit, value in viewitems(NMAP_LIMITS):
resource.setrlimit(limit, value)
# MacOSX (a.k.a. Darwin) has a default stack size that is too small
# for deeply recursive regular expressions. We see this as crashes in
# the Python test suite when running test_re.py and test_sre.py. The
# fix is to set the stack limit to 2048.
# This approach may also be useful for other Unixy platforms that
# suffer from small default stack limits.
if sys.platform == 'darwin':
try:
import resource
except ImportError:
pass
else:
soft, hard = resource.getrlimit(resource.RLIMIT_STACK)
newsoft = min(hard, max(soft, 1024 * 2048))
resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard))
import test as _test
from test import test_support
RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', 'bsddb',
'decimal', 'compiler', 'subprocess', 'urlfetch')
def usage(code, msg=''):
print __doc__
if msg: print msg
sys.exit(code)
def main(tests=None,
from data.dataset import Dataset, TestDataset, inverse_normalize
from model import FasterRCNNVGG16
from torch.autograd import Variable
from torch.utils import data as data_
from trainer import FasterRCNNTrainer
from utils import array_tool as at
from utils.vis_tool import visdom_bbox
from utils.eval_tool import eval_detection_voc
from tensorboardX import SummaryWriter
# fix for ulimit
# https://github.com/pytorch/pytorch/issues/973#issuecomment-346405667
import resource
rlimit = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (20480, rlimit[1]))
matplotlib.use('agg')
def eval(dataloader, faster_rcnn, test_num=10000):
pred_bboxes, pred_labels, pred_scores = list(), list(), list()
gt_bboxes, gt_labels, gt_difficults = list(), list(), list()
for ii, (imgs, sizes, gt_bboxes_, gt_labels_,
gt_difficults_) in tqdm(enumerate(dataloader)):
sizes = [sizes[0][0], sizes[1][0]]
pred_bboxes_, pred_labels_, pred_scores_ = faster_rcnn.predict(
imgs, [sizes])
gt_bboxes += list(gt_bboxes_.numpy())
gt_labels += list(gt_labels_.numpy())
gt_difficults += list(gt_difficults_.numpy())
def __exit__(self, *_):
resource.setrlimit(resource.RLIMIT_CORE, self._old_limits)
def __enter__(self):
self._old_limits = resource.getrlimit(resource.RLIMIT_CORE)
resource.setrlimit(resource.RLIMIT_CORE, (-1, -1))
#!/usr/bin/python
# -*- coding: utf-8 -*-
import math
from heapq import *
import resource
resource.setrlimit(resource.RLIMIT_AS, (2**32, 2**32))
import random
def length(point1, point2):
return math.sqrt((point1[0] - point2[0])**2 + (point1[1] - point2[1])**2)
def distance(points):
dist = dict()
for i in xrange(len(points)):
for j in xrange(i + 1):
d = length(points[i], points[j])
dist[(i, j)] = d
dist[(j, i)] = d
return dist
def kruskal(points, nodeCount): #returns a list of the MST edges.
mst = [
] #initializing the priority queue, grabbing all the first items first.
seen = set([0])
heap = []
for i in xrange(1, nodeCount):
heappush(heap, (length(points[0], points[i]), '0', str(i)))
#!/usr/bin/python
import threading
import time
import resource
def test():
time.sleep(1)
threading.stack_size(32768)
resource.setrlimit(resource.RLIMIT_NPROC, (65535, 65535))
xxx = []
for x in range(50000):
xxx.append(threading.Thread(target=test))
try:
xxx[x].start()
except Exception, error:
print error
xxx.pop()
break
print 'there are %s threads running' % threading.activeCount()
for x in xxx:
x.join()
def startAdmin(adminClass, addrOfStarter, endpointPrep, transportClass,
adminAddr, capabilities, logDefs, concurrency_context):
# Unix Daemonization; skipped if not available
import os, sys
if hasattr(os, 'setsid'):
os.setsid()
try:
import resource
resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) # No core dumps
except Exception:
pass
if hasattr(os, 'fork'):
if os.fork(): sys.exit(0)
# Slight trickiness here. There may *already* be an admin bound
# to this start address. However, the external process attempting
# to start is going to wait for the EndpointConnected message
# before continuing, so ensure that message is *not* sent until
# the local admin Transport has had time to bind and listen to the
# local address, but also ensure that the message is *always* sent
# even if the local admin could not start (caller will use
# _verifyAdminRunning to ensure things are OK.
transport = transportClass(endpointPrep)
try:
admin = adminClass(transport, adminAddr, capabilities, logDefs,
concurrency_context)
except Exception:
transport.scheduleTransmit(
None,
TransmitIntent(addrOfStarter, EndpointConnected(0)).addCallback(
onFailure=actorStartupFailed))
raise
# Send of EndpointConnected is deferred until the logger is setup. See MultiProcReplicator.h_LoggerConnected below.
admin.addrOfStarter = addrOfStarter
setProcName(
adminClass.__name__.rpartition('.')[-1], admin.transport.myAddress)
# Admin does not do normal signal handling, but does want to know if children exit
for each in range(1, signal.NSIG):
# n.b. normally Python intercepts SIGINT to turn it into a
# KeyboardInterrupt exception. However, these Actors should
# be detached from the keyboard, so revert to normal SIGINT
# behavior.
if each not in uncatchable_signals:
if each in child_exit_signals:
set_signal_handler(each, admin.signalChildDied)
if hasattr(signal, 'SIGUSR1'):
set_signal_handler(signal.SIGUSR1, signal_admin_sts(admin))
_startLogger(transportClass, transport, admin, capabilities, logDefs,
concurrency_context)
#closeUnusedFiles(transport)
# Admin should never enter TX-only flow control state because this
# could deadlock or other non-progress conditions, especially if
# using admin routing.
transport.enableRXPauseFlowControl(False)
admin.run()
def memory_limit(percentage: float):
if platform.system() != "Linux":
print('Only works on linux!')
return
soft, hard = resource.getrlimit(resource.RLIMIT_AS)
resource.setrlimit(resource.RLIMIT_AS, (int(get_memory() * 1024 * percentage), hard))
def get_tf_dataset(quantize_bits,
batch_size,
dataset_name="celeb_a",
n_batches=None,
split="train",
crop=False,
label_keep_percent=1.0,
random_label_percent=0.0):
if random_label_percent > 0:
assert 0, "Random labels not implemented for tf datasets"
# https://github.com/tensorflow/datasets/issues/1441#issuecomment-581660890
import resource
low, high = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (high, high))
import tensorflow as tf
tf.config.set_visible_devices([], 'GPU')
import tensorflow_datasets as tfds
if dataset_name == "cifar10":
n_classes = 10
elif dataset_name == "cifar100":
n_classes = 100
elif "imagenet" in dataset_name:
n_classes = 1000
elif "svhn" in dataset_name:
n_classes = 10
elif "mnist" in dataset_name:
n_classes = 10
else:
n_classes = None
def central_crop(x):
x["image"] = x["image"][::2, ::2][26:-19, 12:-13]
return x
def to_float(x):
x["image"] = tf.cast(x["image"], dtype=tf.float32)
return x
def random_flip(x):
x["image"] = tf.image.random_flip_left_right(x["image"])
return x
def quantize(x):
quantize_factor = 256 / (2**quantize_bits)
x["image"] = x["image"] // quantize_factor
return x
def to_expected_input(x):
out = {"x": x["image"]}
if "label" in x:
labels = x["label"]
labels_one_hot = tf.one_hot(labels, n_classes)
out["y"] = labels_one_hot
out["y_non_one_hot"] = x["label"]
return out
rng = tf.random.Generator.from_seed(int(split == "train"))
def make_semi_supervised(x):
if "y_non_one_hot" in x:
labels = x["y_non_one_hot"]
x["y_is_labeled"] = rng.binomial(shape=labels.shape,
counts=tf.ones(labels.shape),
probs=tf.ones(labels.shape) *
label_keep_percent)
del x["y_non_one_hot"]
return x
ds = tfds.load(dataset_name, split=split)
if crop:
ds = ds.map(central_crop)
ds = ds.map(to_float)
ds = ds.map(random_flip)
ds = ds.map(quantize)
ds = ds.map(to_expected_input)
ds = ds.map(make_semi_supervised)
if split == "train":
ds = ds.shuffle(15000).repeat()
ds = ds.batch(batch_size)
if (n_batches is not None):
ds = ds.batch(n_batches, drop_remainder=True)
ds = ds.prefetch(tf.data.experimental.AUTOTUNE)
ds = ds.as_numpy_iterator()
return ds
from core.config import cfg, cfg_from_file, cfg_from_list, assert_and_infer_cfg
from datasets.roidb import combined_roidb_for_training
from roi_data.loader import RoiDataLoader, MinibatchSampler, collate_minibatch
from modeling.model_builder import Generalized_RCNN
from utils.detectron_weight_helper import load_detectron_weight
from utils.logging import setup_logging
from utils.timer import Timer
from utils.training_stats import TrainingStats
# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('roi_data.loader').setLevel(logging.INFO)
# RuntimeError: received 0 items of ancdata. Issue: pytorch/pytorch#973
rlimit = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (4096, rlimit[1]))
def parse_args():
"""Parse input arguments"""
parser = argparse.ArgumentParser(description='Train a X-RCNN network')
parser.add_argument(
'--dataset', dest='dataset', required=True,
help='Dataset to use')
parser.add_argument(
'--cfg', dest='cfg_file', required=True,
help='Config file for training (and optionally testing)')
parser.add_argument(
'--set', dest='set_cfgs',
help='Set config keys. Key value sequence seperate by whitespace.'
'e.g. [key] [value] [key] [value]',
print('Connection', connection_count, 'closed')
connection_count -= 1
async def listen(address):
global connection_count
server_sock = socket.socket()
server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_sock.bind(address)
server_sock.listen()
server_sock.setblocking(False)
while True:
client_sock, addr = await loop.sock_accept(server_sock)
connection_count += 1
print('Connection', connection_count, 'from', addr)
loop.create_task(echo(client_sock))
if __name__ == '__main__':
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
print('Setting max number of open files to', hard)
resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
print('Starting echo server')
loop.create_task(listen(('', 25000)))
try:
loop.run()
except KeyboardInterrupt:
print('Stopping echo server')
def fixLimits():
"Fix ridiculously small resource limits."
setrlimit(RLIMIT_NPROC, (8192, 8192))
setrlimit(RLIMIT_NOFILE, (16384, 16384))
def set_fsize():
# here we limit the logsize
resource.setrlimit(resource.RLIMIT_FSIZE,
(self.trace_log_limit, self.trace_log_limit))
def rlimitTestAndSet( name, limit ):
"Helper function to set rlimits"
soft, hard = getrlimit( name )
if soft < limit:
hardLimit = hard if limit < hard else limit
setrlimit( name, ( limit, hardLimit ) )
def SetRLimitToMax():
(soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
sys.stdout = AutoFlush(sys.stdout)
sys.stderr = AutoFlush(sys.stderr)
sock = socket.fromfd(int(sys.argv[1]), socket.AF_UNIX, socket.SOCK_STREAM)
sock_lock = Lock()
update_valid_fds()
# Set the highest open file limit we can.
# At least OS X seems to like claiming no limit as max without
# allowing that to be set, so let's do some retrying.
r1, r2 = resource.getrlimit(resource.RLIMIT_NOFILE)
limits = [500000, 100000, 50000, 10000, 5000, 1000, r1, r2]
limits = sorted((v for v in limits if r1 <= v <= r2), reverse=True)
for try_limit in limits:
try:
resource.setrlimit(resource.RLIMIT_NOFILE, (try_limit, r2))
break
except ValueError:
pass
r1, r2 = resource.getrlimit(resource.RLIMIT_NOFILE)
if r1 < r2:
print("WARNING: Failed to raise RLIMIT_NOFILE to %d. Set to %d." % (
r2,
r1,
))
if r1 < 5000:
print("WARNING: RLIMIT_NOFILE is %d, that's not much." % (r1, ))
while True:
op, length = struct.unpack('<cI', recvall(sock, 5, True))
data = recvall(sock, length, True)
import sys, re, os, signal
import resource, subprocess
from copy import deepcopy
from select import select
from time import time
from queue import Queue, Empty
from threading import Thread
# This is needed so that the recursive SMT2 S-expression parser
# does not run out of stack frames when parsing large expressions
smtio_reclimit = 64 * 1024
smtio_stacksize = 128 * 1024 * 1024
if sys.getrecursionlimit() < smtio_reclimit:
sys.setrecursionlimit(smtio_reclimit)
if resource.getrlimit(resource.RLIMIT_STACK)[0] < smtio_stacksize:
resource.setrlimit(resource.RLIMIT_STACK, (smtio_stacksize, -1))
# currently running solvers (so we can kill them)
running_solvers = dict()
forced_shutdown = False
solvers_index = 0
def force_shutdown(signum, frame):
global forced_shutdown
if not forced_shutdown:
forced_shutdown = True
if signum is not None:
print("<%s>" % signal.Signals(signum).name)
for p in running_solvers.values():
# os.killpg(os.getpgid(p.pid), signal.SIGTERM)
def test_max_open_files(self):
with TestsLoggingHandler() as handler:
logmsg_dbg = (
'DEBUG:This salt-master instance has accepted {0} minion keys.'
)
logmsg_chk = (
'{0}:The number of accepted minion keys({1}) should be lower '
'than 1/4 of the max open files soft setting({2}). According '
'to the system\'s hard limit, there\'s still a margin of {3} '
'to raise the salt\'s max_open_files setting. Please consider '
'raising this value.')
logmsg_crash = (
'{0}:The number of accepted minion keys({1}) should be lower '
'than 1/4 of the max open files soft setting({2}). '
'salt-master will crash pretty soon! According to the '
'system\'s hard limit, there\'s still a margin of {3} to '
'raise the salt\'s max_open_files setting. Please consider '
'raising this value.')
mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE)
tempdir = tempfile.mkdtemp(prefix='fake-keys')
keys_dir = os.path.join(tempdir, 'minions')
os.makedirs(keys_dir)
mof_test = 256
resource.setrlimit(resource.RLIMIT_NOFILE, (mof_test, mof_h))
try:
prev = 0
for newmax, level in ((24, None), (66, 'INFO'),
(127, 'WARNING'), (196, 'CRITICAL')):
for n in range(prev, newmax):
kpath = os.path.join(keys_dir, str(n))
with salt.utils.fopen(kpath, 'w') as fp_:
fp_.write(str(n))
opts = {'max_open_files': newmax, 'pki_dir': tempdir}
check_max_open_files(opts)
if level is None:
# No log message is triggered, only the DEBUG one which
# tells us how many minion keys were accepted.
self.assertEqual([logmsg_dbg.format(newmax)],
handler.messages)
else:
self.assertIn(logmsg_dbg.format(newmax),
handler.messages)
self.assertIn(
logmsg_chk.format(
level,
newmax,
mof_test,
mof_h - newmax,
), handler.messages)
handler.clear()
prev = newmax
newmax = mof_test
for n in range(prev, newmax):
kpath = os.path.join(keys_dir, str(n))
with salt.utils.fopen(kpath, 'w') as fp_:
fp_.write(str(n))
opts = {'max_open_files': newmax, 'pki_dir': tempdir}
check_max_open_files(opts)
self.assertIn(logmsg_dbg.format(newmax), handler.messages)
self.assertIn(
logmsg_crash.format(
'CRITICAL',
newmax,
mof_test,
mof_h - newmax,
), handler.messages)
handler.clear()
except IOError as err:
if err.errno == 24:
# Too many open files
self.skipTest('We\'ve hit the max open files setting')
raise
finally:
shutil.rmtree(tempdir)
resource.setrlimit(resource.RLIMIT_NOFILE, (mof_s, mof_h))
def main():
"""
For some reason (take a wild guess) Commvault has decided that
their long options will take the form of '-option' not the standard
'--option'.
Always set HOME to '/root', as the commvault environment is bare
"""
os.environ['HOME'] = '/root'
os.environ['TMPDIR'] = '/tmp'
# ensure we do not inherit commvault's LD_LIBRARY_PATH
os.environ.pop('LD_LIBRARY_PATH', None)
holland_conf = '/etc/holland/holland.conf'
if sys.platform.startswith('freebsd'):
holland_conf = '/usr/local' + holland_conf
parser = ArgumentParser()
parser.add_argument("--config-file", "-c", metavar="<file>",
help="Read configuration from the given file")
parser.add_argument('-l', '--log-level', metavar='<log-level>',
choices=['critical', 'error', 'warning', 'info', 'debug'],
help="Specify the log level. "
"One of: critical,error,warning,info,debug")
parser.add_argument("--quiet", "-q", action="store_true",
help="Don't log to console")
parser.add_argument("--verbose", "-v", action="store_true",
help="Verbose output")
parser.add_argument("--bksets", "-b", metavar="<bkset>,<bkset>...",
help="only run the specified backupset",
default=[], action=ArgList)
parser.add_argument("-bkplevel", type=int)
parser.add_argument("-attempt", type=int)
parser.add_argument("-status", type=int)
parser.add_argument("-job", type=int)
parser.add_argument("-vm")
parser.add_argument("-cn")
parser.set_defaults(
config_file=os.getenv('HOLLAND_CONFIG') or holland_conf,
verbose=False,
)
args, largs = parser.parse_known_args(sys.argv[1:])
if args.log_level:
args.log_level = format_loglevel(args.log_level)
bootstrap(args)
logging.info("Holland (commvault agent) %s started with pid %d",
HOLLAND_VERSION, os.getpid())
# Commvault usually runs with a very low default limit for nofile
# so a best effort is taken to raise that here.
try:
resource.setrlimit(resource.RLIMIT_NOFILE, (262144, 262144))
logging.debug("(Adjusted ulimit -n (RLIMIT_NOFILE) to %d)", 262144)
except (ValueError, resource.error) as exc:
logging.debug("Failed to raise RLIMIT_NOFILE: %s", exc)
args.command = 'backup'
args.dry_run = 0
args.no_lock = 0
largs = args.bksets
spool = HOLLANDCFG.lookup('holland.backup-directory')
status_file = '%s/%s/newest/job_%s' % (spool, args.bksets[0], args.job)
pid_name = 'holland_commvault_%s' % args.job
pid_location = '/var/run/%s.pid' % pid_name
try:
with PidFile(pid_name):
ret = 0
if run(args, largs):
ret = 1
status = open(status_file, 'w')
status.write(str(ret))
status.close()
return ret
except PidFileAlreadyLockedError:
ret = 1
pid_file = open(pid_location, 'r')
pid = pid_file.read()
pid_file.close()
logging.info("Holland (commvault agent) is already running, waiting for the pid %s", pid)
count = 0
while os.path.isfile(pid_location):
sleep(10)
count = count + 1
#~14 hour timeout
if count > 5040:
logging.info("Holland (commvault agent) timed out after %s seconds", count*10)
return 1
try:
status = open(status_file, 'r')
ret = int(status.read())
except IOError:
logging.info("Holland (commvault agent) failed to open/read status file")
return 1
else:
status.close()
return ret
except IOError:
logging.info("Holland (commvault agent) must have permission to write to /var/run")
return 1
def run_experiment(openmlid: int, num_pipelines: int, seed: int,
timeout: int, folder: str, prob_dp: float, prob_fp: float, config_map: typing.Dict):
# TODO: built in check whether file already exists, in that case we can skipp
# CPU
print("CPU Settings:")
for v in ["OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS",
"BLIS_NUM_THREADS"]:
print(f"\t{v}: {os.environ[v] if v in os.environ else 'n/a'}")
# memory limits
memory_limit = 14 * 1024
print("Setting memory limit to " + str(memory_limit) + "MB")
soft, hard = resource.getrlimit(resource.RLIMIT_AS)
resource.setrlimit(resource.RLIMIT_AS,
(memory_limit * 1024 * 1024, memory_limit * 1024 * 1024))
# configure lccv logger (by default set to WARN, change it to DEBUG if tests fail)
lccv_logger = logging.getLogger("lccv")
lccv_logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
lccv_logger.addHandler(ch)
exp_logger = logging.getLogger("experimenter")
exp_logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
exp_logger.addHandler(ch)
eval_logger = logging.getLogger("evalutils")
eval_logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
eval_logger.addHandler(ch)
exp_logger.info("Starting python script")
exp_logger.info(f"""Running experiment under following conditions:
OpenML id: {openmlid}
Seed: {seed}
timeout (per single evaluation): {timeout}
Num Pipelines: {num_pipelines}
Probability to draw a data-preprocessor: {prob_dp}
Probability to draw a feature-preprocessor: {prob_fp}
""")
# CPU
exp_logger.info("CPU Settings:")
for v in ["OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS",
"BLIS_NUM_THREADS"]:
exp_logger.info(f"\t{v}: {os.environ[v] if v in os.environ else 'n/a'}")
# memory limits
memory_limit = 14 * 1024
exp_logger.info("Setting memory limit to " + str(memory_limit) + "MB")
soft, hard = resource.getrlimit(resource.RLIMIT_AS)
resource.setrlimit(resource.RLIMIT_AS,
(memory_limit * 1024 * 1024, memory_limit * 1024 * 1024))
# load data
exp_logger.info("Reading dataset")
X, y = get_dataset(openmlid)
exp_logger.info(
f"ready. Dataset shape is {X.shape}, label column shape is {y.shape}. Now running the algorithm")
if X.shape[0] <= 0:
raise Exception("Dataset size invalid!")
if X.shape[0] != len(y):
raise Exception("X and y do not have the same size.")
# creating learner sequence
sampler = PipelineSampler("searchspace.json", X, y, seed, dp_proba=prob_dp,
fp_proba=prob_fp)
test_learners = [sampler.sample(do_build=False) for i in
range(num_pipelines)]
exp_logger.info(f"Evaluating portfolio of {len(test_learners)} learners.")
# run lccv
epsilon = 0.0
validators = [(lccv90flex, lambda r: r[0], config_map)]
key = "lccv90flex"
result = \
evaluate_validators(validators, test_learners, X, y, timeout, seed=seed,
repeats=100, epsilon=epsilon)[key]
model = result[0]
runtime = result[1]
if model is not None:
error_rate = np.round(result[2][0], 4)
model_name = str(model).replace("\n", " ")
exp_logger.info(f"""Run completed. Here are the details:
Model: {model}
Error Rate: {error_rate}
Runtime: {runtime}
Results in final evaluation: {np.round(result[2][1], 4)}""")
else:
exp_logger.info("No model was chosen. Assigning maximum error rate")
error_rate = 1
model_name = "None"
# write result
output = (model_name, error_rate, runtime, result[3], result[4])
with open(folder + "/results.txt", "w") as outfile:
json.dump(output, outfile)
exp_logger.info(
f"Experiment ready. Results written to {folder}/results.txt")
def main():
# find specified locustfile and make sure it exists, using a very simplified
# command line parser that is only used to parse the -f option
locustfile = parse_locustfile_option()
# import the locustfile
docstring, user_classes = load_locustfile(locustfile)
# parse all command line options
options = parse_options()
if options.slave or options.expect_slaves:
sys.stderr.write(
"[DEPRECATED] Usage of slave has been deprecated, use --worker or --expect-workers\n"
)
sys.exit(1)
# setup logging
if not options.skip_log_setup:
if options.loglevel.upper() in [
"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"
]:
setup_logging(options.loglevel, options.logfile)
else:
sys.stderr.write(
"Invalid --loglevel. Valid values are: DEBUG/INFO/WARNING/ERROR/CRITICAL\n"
)
sys.exit(1)
logger = logging.getLogger(__name__)
greenlet_exception_handler = greenlet_exception_logger(logger)
if options.list_commands:
print("Available Users:")
for name in user_classes:
print(" " + name)
sys.exit(0)
if not user_classes:
logger.error("No User class found!")
sys.exit(1)
# make sure specified User exists
if options.user_classes:
missing = set(options.user_classes) - set(user_classes.keys())
if missing:
logger.error("Unknown User(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(options.user_classes) & set(user_classes.keys())
user_classes = [user_classes[n] for n in names]
else:
# list() call is needed to consume the dict_view object in Python 3
user_classes = list(user_classes.values())
try:
import resource
if resource.getrlimit(resource.RLIMIT_NOFILE)[0] < 10000:
# Increasing the limit to 10000 within a running process should work on at least MacOS.
# It does not work on all OS:es, but we should be no worse off for trying.
resource.setrlimit(resource.RLIMIT_NOFILE,
[10000, resource.RLIM_INFINITY])
except:
logger.warning(
"System open file limit setting is not high enough for load testing, and the OS wouldnt allow locust to increase it by itself. See https://docs.locust.io/en/stable/installation.html#increasing-maximum-number-of-open-files-limit for more info."
)
# create locust Environment
environment = create_environment(user_classes,
options,
events=locust.events)
if options.show_task_ratio:
print("\n Task ratio per User class")
print("-" * 80)
print_task_ratio(user_classes)
print("\n Total task ratio")
print("-" * 80)
print_task_ratio(user_classes, total=True)
sys.exit(0)
if options.show_task_ratio_json:
from json import dumps
task_data = {
"per_class": get_task_ratio_dict(user_classes),
"total": get_task_ratio_dict(user_classes, total=True)
}
print(dumps(task_data))
sys.exit(0)
if options.step_time:
if not options.step_load:
logger.error(
"The --step-time argument can only be used together with --step-load"
)
sys.exit(1)
if options.worker:
logger.error(
"--step-time should be specified on the master node, and not on worker nodes"
)
sys.exit(1)
try:
options.step_time = parse_timespan(options.step_time)
except ValueError:
logger.error(
"Valid --step-time formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc."
)
sys.exit(1)
if options.master:
runner = environment.create_master_runner(
master_bind_host=options.master_bind_host,
master_bind_port=options.master_bind_port,
)
elif options.worker:
try:
runner = environment.create_worker_runner(options.master_host,
options.master_port)
except socket.error as e:
logger.error("Failed to connect to the Locust master: %s", e)
sys.exit(-1)
else:
runner = environment.create_local_runner()
# main_greenlet is pointing to runners.greenlet by default, it will point the web greenlet later if in web mode
main_greenlet = runner.greenlet
if options.run_time:
if not options.headless:
logger.error(
"The --run-time argument can only be used together with --headless"
)
sys.exit(1)
if options.worker:
logger.error(
"--run-time should be specified on the master node, and not on worker nodes"
)
sys.exit(1)
try:
options.run_time = parse_timespan(options.run_time)
except ValueError:
logger.error(
"Valid --run-time formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc."
)
sys.exit(1)
def spawn_run_time_limit_greenlet():
logger.info("Run time limit set to %s seconds" % options.run_time)
def timelimit_stop():
logger.info("Time limit reached. Stopping Locust.")
runner.quit()
gevent.spawn_later(
options.run_time,
timelimit_stop).link_exception(greenlet_exception_handler)
# start Web UI
if not options.headless and not options.worker:
# spawn web greenlet
protocol = "https" if options.tls_cert and options.tls_key else "http"
logger.info("Starting web interface at %s://%s:%s" %
(protocol, options.web_host, options.web_port))
try:
if options.web_host == "*":
# special check for "*" so that we're consistent with --master-bind-host
web_host = ''
else:
web_host = options.web_host
web_ui = environment.create_web_ui(
host=web_host,
port=options.web_port,
auth_credentials=options.web_auth,
tls_cert=options.tls_cert,
tls_key=options.tls_key,
)
except AuthCredentialsError:
logger.error(
"Credentials supplied with --web-auth should have the format: username:password"
)
sys.exit(1)
else:
main_greenlet = web_ui.greenlet
else:
web_ui = None
# Fire locust init event which can be used by end-users' code to run setup code that
# need access to the Environment, Runner or WebUI
environment.events.init.fire(environment=environment,
runner=runner,
web_ui=web_ui)
if options.headless:
# headless mode
if options.master:
# wait for worker nodes to connect
while len(runner.clients.ready) < options.expect_workers:
logging.info(
"Waiting for workers to be ready, %s of %s connected",
len(runner.clients.ready), options.expect_workers)
time.sleep(1)
if not options.worker:
# apply headless mode defaults
if options.num_users is None:
options.num_users = 1
if options.hatch_rate is None:
options.hatch_rate = 1
if options.step_users is None:
options.step_users = 1
# start the test
if options.step_time:
runner.start_stepload(options.num_users, options.hatch_rate,
options.step_users, options.step_time)
else:
runner.start(options.num_users, options.hatch_rate)
if options.run_time:
spawn_run_time_limit_greenlet()
stats_printer_greenlet = None
if not options.only_summary and (options.print_stats or
(options.headless
and not options.worker)):
# spawn stats printing greenlet
stats_printer_greenlet = gevent.spawn(stats_printer(runner.stats))
stats_printer_greenlet.link_exception(greenlet_exception_handler)
if options.csv_prefix:
gevent.spawn(
stats_writer,
environment,
options.csv_prefix,
full_history=options.stats_history_enabled).link_exception(
greenlet_exception_handler)
def shutdown():
"""
Shut down locust by firing quitting event, printing/writing stats and exiting
"""
logger.info("Running teardowns...")
environment.events.quitting.fire(environment=environment, reverse=True)
# determine the process exit code
if log.unhandled_greenlet_exception:
code = 2
elif environment.process_exit_code is not None:
code = environment.process_exit_code
elif len(runner.errors) or len(runner.exceptions):
code = options.exit_code_on_error
else:
code = 0
logger.info("Shutting down (exit code %s), bye." % code)
if stats_printer_greenlet is not None:
stats_printer_greenlet.kill(block=False)
logger.info("Cleaning up runner...")
if runner is not None:
runner.quit()
print_stats(runner.stats, current=False)
print_percentile_stats(runner.stats)
if options.csv_prefix:
write_csv_files(environment,
options.csv_prefix,
full_history=options.stats_history_enabled)
print_error_report(runner.stats)
sys.exit(code)
# install SIGTERM handler
def sig_term_handler():
logger.info("Got SIGTERM signal")
shutdown()
gevent.signal_handler(signal.SIGTERM, sig_term_handler)
try:
logger.info("Starting Locust %s" % version)
main_greenlet.join()
shutdown()
except KeyboardInterrupt as e:
shutdown()
def initialize(topsrcdir):
# Ensure we are running Python 3.6+. We run this check as soon as
# possible to avoid a cryptic import/usage error.
if sys.version_info < (3, 6):
print("Python 3.6+ is required to run mach.")
print("You are running Python", platform.python_version())
if sys.platform.startswith("linux"):
print(INSTALL_PYTHON_GUIDANCE_LINUX)
elif sys.platform.startswith("darwin"):
print(INSTALL_PYTHON_GUIDANCE_OSX)
elif "MOZILLABUILD" in os.environ:
print(INSTALL_PYTHON_GUIDANCE_MOZILLABUILD)
else:
print(INSTALL_PYTHON_GUIDANCE_OTHER)
sys.exit(1)
# This directory was deleted in bug 1666345, but there may be some ignored
# files here. We can safely just delete it for the user so they don't have
# to clean the repo themselves.
deleted_dir = os.path.join(topsrcdir, "third_party", "python", "psutil")
if os.path.exists(deleted_dir):
shutil.rmtree(deleted_dir, ignore_errors=True)
state_dir = _create_state_dir()
_activate_python_environment(topsrcdir, state_dir)
import mach.base
import mach.main
from mach.util import setenv
from mozboot.util import get_state_dir
# Set a reasonable limit to the number of open files.
#
# Some linux systems set `ulimit -n` to a very high number, which works
# well for systems that run servers, but this setting causes performance
# problems when programs close file descriptors before forking, like
# Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the
# default in Python 3), or Rust's stdlib. In some cases, Firefox does the
# same thing when spawning processes. We would prefer to lower this limit
# to avoid such performance problems; processes spawned by `mach` will
# inherit the limit set here.
#
# The Firefox build defaults the soft limit to 1024, except for builds that
# do LTO, where the soft limit is 8192. We're going to default to the
# latter, since people do occasionally do LTO builds on their local
# machines, and requiring them to discover another magical setting after
# setting up an LTO build in the first place doesn't seem good.
#
# This code mimics the code in taskcluster/scripts/run-task.
try:
import resource
# Keep the hard limit the same, though, allowing processes to change
# their soft limit if they need to (Firefox does, for instance).
(soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE)
# Permit people to override our default limit if necessary via
# MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses.
limit = os.environ.get("MOZ_LIMIT_NOFILE")
if limit:
limit = int(limit)
else:
# If no explicit limit is given, use our default if it's less than
# the current soft limit. For instance, the default on macOS is
# 256, so we'd pick that rather than our default.
limit = min(soft, 8192)
# Now apply the limit, if it's different from the original one.
if limit != soft:
resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
except ImportError:
# The resource module is UNIX only.
pass
def resolve_repository():
import mozversioncontrol
try:
# This API doesn't respect the vcs binary choices from configure.
# If we ever need to use the VCS binary here, consider something
# more robust.
return mozversioncontrol.get_repository_object(path=topsrcdir)
except (mozversioncontrol.InvalidRepoPath, mozversioncontrol.MissingVCSTool):
return None
def pre_dispatch_handler(context, handler, args):
# If --disable-tests flag was enabled in the mozconfig used to compile
# the build, tests will be disabled. Instead of trying to run
# nonexistent tests then reporting a failure, this will prevent mach
# from progressing beyond this point.
if handler.category == "testing" and not handler.ok_if_tests_disabled:
from mozbuild.base import BuildEnvironmentNotFoundException
try:
from mozbuild.base import MozbuildObject
# all environments should have an instance of build object.
build = MozbuildObject.from_environment()
if build is not None and not getattr(
build, "substs", {"ENABLE_TESTS": True}
).get("ENABLE_TESTS"):
print(
"Tests have been disabled with --disable-tests.\n"
+ "Remove the flag, and re-compile to enable tests."
)
sys.exit(1)
except BuildEnvironmentNotFoundException:
# likely automation environment, so do nothing.
pass
def post_dispatch_handler(
context, handler, instance, success, start_time, end_time, depth, args
):
"""Perform global operations after command dispatch.
For now, we will use this to handle build system telemetry.
"""
# Don't finalize telemetry data if this mach command was invoked as part of
# another mach command.
if depth != 1:
return
_finalize_telemetry_glean(
context.telemetry, handler.name == "bootstrap", success
)
def populate_context(key=None):
if key is None:
return
if key == "state_dir":
return state_dir
if key == "local_state_dir":
return get_state_dir(srcdir=True)
if key == "topdir":
return topsrcdir
if key == "pre_dispatch_handler":
return pre_dispatch_handler
if key == "post_dispatch_handler":
return post_dispatch_handler
if key == "repository":
return resolve_repository()
raise AttributeError(key)
# Note which process is top-level so that recursive mach invocations can avoid writing
# telemetry data.
if "MACH_MAIN_PID" not in os.environ:
setenv("MACH_MAIN_PID", str(os.getpid()))
driver = mach.main.Mach(os.getcwd())
driver.populate_context_handler = populate_context
if not driver.settings_paths:
# default global machrc location
driver.settings_paths.append(state_dir)
# always load local repository configuration
driver.settings_paths.append(topsrcdir)
for category, meta in CATEGORIES.items():
driver.define_category(category, meta["short"], meta["long"], meta["priority"])
# Sparse checkouts may not have all mach_commands.py files. Ignore
# errors from missing files. Same for spidermonkey tarballs.
repo = resolve_repository()
missing_ok = (
repo is not None and repo.sparse_checkout_present()
) or os.path.exists(os.path.join(topsrcdir, "INSTALL"))
for path in MACH_MODULES:
try:
driver.load_commands_from_file(os.path.join(topsrcdir, path))
except mach.base.MissingFileError:
if not missing_ok:
raise
return driver
import cobrascape.ensemble as ens
from cobra.io import load_json_model
from cobrascape.species import load_json_obj, save_json_obj
import pandas as pd
import numpy as np
import sys, os, argparse, resource, warnings, itertools
from os import listdir
# import seaborn as sns
# import matplotlib.pyplot as plt
from random import shuffle
from sklearn.metrics import roc_curve, auc
from collections import Counter
from tqdm import tqdm
warnings.filterwarnings("ignore") # sklearn gives hella warnings.
resource.setrlimit(resource.RLIMIT_NOFILE, (10000, -1))
# Argument parsing
parser = argparse.ArgumentParser(description='Evaluate MAC samples')
### Required parameters
parser.add_argument(
'-f',
dest='mnc_dir',
required=True,
help=
'Path to folder containing MAC samples. Should be same as -o argument of 01_sample_macs.py'
)
### Optional parameters (see Methods section of Kavvas et al 2020 for parameters utilized in study)
parser.add_argument(
'--testset',
dest='train_test',
def tearDown(self):
super(test_schema03, self).tearDown()
resource.setrlimit(resource.RLIMIT_NOFILE, self.origFileLimit)
import signal
import time
# Set up a signal handler to notify us
# when we run out of time.
def time_expired(n, stack):
print('EXPIRED :', time.ctime())
raise SystemExit('(time ran out)')
signal.signal(signal.SIGXCPU, time_expired)
# Adjust the CPU time limit
soft, hard = resource.getrlimit(resource.RLIMIT_CPU)
print('Soft limit starts as :', soft)
resource.setrlimit(resource.RLIMIT_CPU, (1, hard))
soft, hard = resource.getrlimit(resource.RLIMIT_CPU)
print('Soft limit changed to :', soft)
print()
# Consume some CPU time in a pointless exercise
print('Starting:', time.ctime())
for i in range(200000):
for i in range(200000):
v = i * i
# We should never make it this far
print('Exiting :', time.ctime())
broadcast_buffers=False,
find_unused_parameters=True)
do_train(cfg, args, model, optimizer, resume=args.resume)
return do_test(cfg, model)
if __name__ == "__main__":
import resource
# RuntimeError: received 0 items of ancdata. Issue: pytorch/pytorch#973
rlimit = resource.getrlimit(resource.RLIMIT_NOFILE)
hard_limit = rlimit[1]
soft_limit = min(500000, hard_limit)
iprint("soft limit: ", soft_limit, "hard limit: ", hard_limit)
resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit))
args = my_default_argument_parser().parse_args()
iprint("Command Line Args:", args)
if args.eval_only:
torch.multiprocessing.set_sharing_strategy("file_system")
USE_HVD = False
if args.use_hvd:
if comm.HVD_AVAILABLE:
iprint("Using horovod")
comm.init_hvd()
USE_HVD = True
main(args)
else:
def setlimits():
print "Setting resource limit in child"
for k, v in limits.iteritems():
resource.setrlimit(v[0], (v[1], v[1]))
