def setup(self): require_user('root') self._nofile = resource.getrlimit(resource.RLIMIT_NOFILE) soft, hard = self._nofile new_limit = (min(soft, RESPAWNS / 2), min(hard, RESPAWNS / 2)) resource.setrlimit(resource.RLIMIT_NOFILE, new_limit) self._socket_fd_count = count_socket_fds()
def preexec_fn(): streams = [sys.stdin] if self.close_child_stdout: streams.append(sys.stdout) if self.close_child_stderr: streams.append(sys.stderr) self._null_streams(streams) os.setsid() for limit, value in self.rlimits.items(): res = getattr(resource, 'RLIMIT_%s' % limit.upper(), None) if res is None: raise ValueError('unknown rlimit "%s"' % limit) # TODO(petef): support hard/soft limits resource.setrlimit(res, (value, value)) if self.gid: try: os.setgid(self.gid) except OverflowError: if not ctypes: raise # versions of python < 2.6.2 don't manage unsigned int for # groups like on osx or fedora os.setgid(-ctypes.c_int(-self.gid).value) if self.uid: os.setuid(self.uid)
def _suppress_core_files(self): """Try to prevent core files from being created. Returns previous ulimit if successful, else None. """ if sys.platform == 'darwin': # Check if the 'Crash Reporter' on OSX was configured # in 'Developer' mode and warn that it will get triggered # when it is. # # This assumes that this context manager is used in tests # that might trigger the next manager. value = subprocess.Popen(['/usr/bin/defaults', 'read', 'com.apple.CrashReporter', 'DialogType'], stdout=subprocess.PIPE).communicate()[0] if value.strip() == b'developer': print "this tests triggers the Crash Reporter, that is intentional" sys.stdout.flush() try: import resource old_limit = resource.getrlimit(resource.RLIMIT_CORE) resource.setrlimit(resource.RLIMIT_CORE, (0,0)) return old_limit except (ImportError, ValueError, resource.error): return None
def test_configurations_signed_data(self): # Use a limit of ~4GiB limit = 4000 * 1024 * 1024 resource.setrlimit(resource.RLIMIT_AS, (limit, limit)) cs = ParamSklearnClassifier.get_hyperparameter_search_space( dataset_properties={'signed': True}) print(cs) for i in range(10): config = cs.sample_configuration() config._populate_values() if config['classifier:passive_aggressive:n_iter'] is not None: config._values['classifier:passive_aggressive:n_iter'] = 5 if config['classifier:sgd:n_iter'] is not None: config._values['classifier:sgd:n_iter'] = 5 X_train, Y_train, X_test, Y_test = get_dataset(dataset='digits') cls = ParamSklearnClassifier(config, random_state=1) print(config) try: cls.fit(X_train, Y_train) X_test_ = X_test.copy() predictions = cls.predict(X_test) self.assertIsInstance(predictions, np.ndarray) predicted_probabiliets = cls.predict_proba(X_test_) self.assertIsInstance(predicted_probabiliets, np.ndarray) except ValueError as e: if "Floating-point under-/overflow occurred at epoch" in \ e.args[0] or \ "removed all features" in e.args[0] or \ "all features are discarded" in e.args[0]: continue else: print(config) print(traceback.format_exc()) raise e except RuntimeWarning as e: if "invalid value encountered in sqrt" in e.args[0]: continue elif "divide by zero encountered in" in e.args[0]: continue elif "invalid value encountered in divide" in e.args[0]: continue elif "invalid value encountered in true_divide" in e.args[0]: continue else: print(config) print(traceback.format_exc()) raise e except UserWarning as e: if "FastICA did not converge" in e.args[0]: continue else: print(config) print(traceback.format_exc()) raise e except MemoryError as e: continue
def init_tests(self): """ Initialize testing infrastructure - sockets, resource limits, etc. """ # Init Twisted factory. self.server_factory = network.TestServerFactory(controller = self) #self.client_factory = TestClientFactory(controller = self) ports = sorted(self.test_ports) log.notice("Binding to test ports: %s", ", ".join(map(str, ports))) # Sort to try privileged ports first, since sets have no # guaranteed ordering. for port in ports: reactor.listenTCP(port, self.server_factory) # Set RLIMIT_NOFILE to its hard limit; we want to be able to # use as many file descriptors as the system will allow. # NOTE: Your soft/hard limits are inherited from the root user! # The root user does NOT always have unlimited file descriptors. # Take this into account when editing /etc/security/limits.conf. (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE) log.verbose1("RLIMIT_NOFILE: soft = %d, hard = %d", soft, hard) if soft < hard: log.debug("Increasing RLIMIT_NOFILE soft limit to %d.", hard) resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard)) log.debug("Initializing test threads.") # TODO: Configure me! scheduler_class = getattr(scheduler, config.scheduler) self.scheduler = scheduler_class(controller = self, max_pending_factor = config.max_pending_factor, export_interval = config.export_interval) T = threading.Thread self.schedule_thread = T(target = Controller.test_schedule_thread, name = "Scheduler", args = (self,)) self.watchdog_thread = T(target = Controller.watchdog, name = "Watchdog", args = (self,))
def main(): args = parse_args() program = args.program if not os.path.isabs(program): # program uses a relative path: try to find the absolute path # to the executable if sys.version_info >= (3, 3): import shutil program_abs = shutil.which(program) else: import distutils.spawn program_abs = distutils.spawn.find_executable(program) if program_abs: program = program_abs for arg_name, rlimit in RESOURCES: value = getattr(args, arg_name) if value is None: continue try: resource.setrlimit(rlimit, (value, value)) except ValueError as exc: print("%s: failed to set the %s resource limit: %s" % (USAGE_PROGRAM, arg_name.upper(), exc), file=sys.stderr) sys.exit(1) try: os.execv(program, [program] + args.program_args) except Exception as exc: print("%s: failed to execute %s: %s" % (USAGE_PROGRAM, program, exc), file=sys.stderr) sys.exit(1)
def _restore_rlimits(self): try: import resource resource.setrlimit(resource.RLIMIT_NOFILE, (self.soft_max_open_files, self.hard_max_open_files)) except ImportError: pass
def set_limit(kind, soft_limit, hard_limit=None): hard_limit = hard_limit or soft_limit try: resource.setrlimit(kind, (soft_limit, hard_limit)) except (OSError, ValueError), err: sys.stderr.write('Resource limit for %s could not be set to %s (%s)\n' % (kind, (soft_limit, hard_limit), err))
def process_limit(self): # If our parent changed sucide if self.ppid != os.getppid(): _logger.info("Worker (%s) Parent changed", self.pid) self.alive = False # check for lifetime if self.request_count >= self.request_max: _logger.info("Worker (%d) max request (%s) reached.", self.pid, self.request_count) self.alive = False # Reset the worker if it consumes too much memory (e.g. caused by a memory leak). rss, vms = memory_info(psutil.Process(os.getpid())) if vms > config['limit_memory_soft']: _logger.info('Worker (%d) virtual memory limit (%s) reached.', self.pid, vms) self.alive = False # Commit suicide after the request. # VMS and RLIMIT_AS are the same thing: virtual memory, a.k.a. address space soft, hard = resource.getrlimit(resource.RLIMIT_AS) resource.setrlimit(resource.RLIMIT_AS, (config['limit_memory_hard'], hard)) # SIGXCPU (exceeded CPU time) signal handler will raise an exception. r = resource.getrusage(resource.RUSAGE_SELF) cpu_time = r.ru_utime + r.ru_stime def time_expired(n, stack): _logger.info('Worker (%d) CPU time limit (%s) reached.', self.pid, config['limit_time_cpu']) # We dont suicide in such case raise Exception('CPU time limit exceeded.') signal.signal(signal.SIGXCPU, time_expired) soft, hard = resource.getrlimit(resource.RLIMIT_CPU) resource.setrlimit(resource.RLIMIT_CPU, (cpu_time + config['limit_time_cpu'], hard))
def collate(self, restart=False): import resource as res # Set the stacksize to be unlimited res.setrlimit(res.RLIMIT_STACK, (res.RLIM_INFINITY, res.RLIM_INFINITY)) restart_path = os.path.join(self.run_path, 'RESTART') nc_files = [os.path.join(self.run_path, f) for f in os.listdir(self.run_path) if f.endswith('.nc.0000')] if restart: restart_files = [os.path.join(restart_path, f) for f in os.listdir(restart_path) if f.endswith('.nc.0000')] nc_files.extend(restart_files) mppnc_path = os.path.join(self.bin_path, 'mppnccombine') for f in nc_files: cmd = [mppnc_path, '-r', '-64', f] sp.Popen(cmd).wait() if self.archive_group: self.regroup()
def set_open_files_limit(desired_limit): """ On POSIX systems, set the open files limit to the desired number, unless it is already equal to or higher than that. Setting a high limit enables Flintrock to launch or interact with really large clusters. Background discussion: https://github.com/nchammas/flintrock/issues/81 """ soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) if soft_limit < desired_limit: if desired_limit > hard_limit: warnings.warn( "Flintrock cannot set the open files limit to {desired} " "because the OS hard limit is {hard}. Going with {hard}. " "You may have problems launching or interacting with " "really large clusters." .format( desired=desired_limit, hard=hard_limit), category=RuntimeWarning, stacklevel=2) resource.setrlimit( resource.RLIMIT_NOFILE, (min(desired_limit, hard_limit), hard_limit))
def setUp(self): self.origFileLimit = resource.getrlimit(resource.RLIMIT_NOFILE) newlimit = (self.OPEN_FILE_LIMIT, self.origFileLimit[1]) if newlimit[0] > newlimit[1]: self.skipTest('Require %d open files, only %d available' % newlimit) resource.setrlimit(resource.RLIMIT_NOFILE, newlimit) super(test_schema03, self).setUp()
def launch_downloader_daemon(): # Increase the maximum file descriptor count (to the max) # NOTE: the info logging is REQUIRED for some unknown reason, if it is not # done here, no further logging can be done in the daemon and it gets stuck. try: import resource logging.info('Increasing file descriptor count limit in Downloader') resource.setrlimit(resource.RLIMIT_NOFILE, (10240, -1)) except ValueError: logging.warn('setrlimit failed.') # Make sure we don't leak from the downloader eventloop from miro import eventloop def beginLoop(loop): loop.pool = Foundation.NSAutoreleasePool.alloc().init() eventloop.connect('begin-loop', beginLoop) eventloop.connect('thread-will-start', beginLoop) def endLoop(loop): del loop.pool eventloop.connect('end-loop', endLoop) eventloop.connect('thread-did-start', endLoop) # And launch from miro.dl_daemon import Democracy_Downloader Democracy_Downloader.launch() # Wait for the event loop thread to finish. # Although this is theorically not necessary since the event loop thread is # a non-daemon thread, situations where the downloader daemon exits right # after its launch as this function returns have been seen in the wild. eventloop.join()
def run(self): # START LINUX: used to receive Memory Error faster in Linux try: import resource resource.setrlimit(resource.RLIMIT_AS, (2 ** 30, 2 ** 30)) # 2 ** 30 == 1GB in bytes except ImportError: pass # END LINUX: while True: exception = None self.stop.value = False script = self.consume.get() if script: help_globals = {'stop': self.stop} try: exec(script, help_globals) except BaseException as e: exc_type, exc_obj, exc_tb = sys.exc_info() exception = '{} {} {}'.format(exc_type, exc_obj, e.args) if exception: self.produce.put({'exception': exception}) else: self.produce.put({key: value for key, value in help_globals.items() if not callable(value) and # cannot be a function not isinstance(value, ModuleType) and # cannot be a module key not in ['__builtins__', 'stop']}) # cannot be built ins or synchronized objects del help_globals else: break
def get_contest(self): """See docstring in class Loader. """ # Unset stack size limit resource.setrlimit(resource.RLIMIT_STACK, (resource.RLIM_INFINITY, resource.RLIM_INFINITY)) self.buildpath = os.path.join(self.path, "build") # We have to avoid copying the folder contest/build # or contest/task/build into contest/build. # For this reason, we ignore all files and directories named "build" # when copying recursively. copyrecursivelyifnecessary((self.path), self.buildpath, set(["build"])) with chdir(self.buildpath): rules = ".rules" if not os.path.exists(rules): os.mkdir(rules) rules = os.path.abspath(rules) self.contestconfig = ContestConfig(rules, os.path.basename(self.path)) self.contestconfig._readconfig("contest-config.py") tasknames = [t.name for t in self.contestconfig.tasks] usernames = [u.username for u in self.contestconfig.users] return self.contestconfig._makecontest(), tasknames, usernames
def init( libsyndicate, gateway_key_path ): ''' Initialize this module. ''' global GATEWAY_PRIVKEY_PEM global SENDER_PUBKEY_PEM # disable core dumps (don't want our private key to get leaked) resource.setrlimit( resource.RLIMIT_CORE, (0, 0) ) gateway_privkey_pem = None if gateway_key_path is not None: fd = open( gateway_key_path, "r" ) gateway_privkey_pem = fd.read() fd.close() # verify try: gateway_privkey = CryptoKey.importKey( gateway_privkey_pem ) assert gateway_privkey.has_private(), "Not a private key" except Exception, e: log.exception(e) log.error("Failed to load Gateway private key %s" % gateway_key_path ) return -1
def set_limits (): import resource as r if self.cpu > 0: r.setrlimit (r.RLIMIT_CPU, [self.cpu, self.cpu]) if self.mem > 0: mem_bytes = self.mem * 1024 * 1024 r.setrlimit (r.RLIMIT_AS, [mem_bytes, mem_bytes])
def prep_filehandles(self): smax_open_files, hmax_open_files = resource.getrlimit( resource.RLIMIT_NOFILE ) if smax_open_files < REQUIRED_OPEN_FILES: print( ' * Max open files setting is too low({0}) for running the ' 'tests'.format(smax_open_files) ) print( ' * Trying to raise the limit to {0}'.format(REQUIRED_OPEN_FILES) ) if hmax_open_files < 4096: hmax_open_files = 4096 # Decent default? try: resource.setrlimit( resource.RLIMIT_NOFILE, (REQUIRED_OPEN_FILES, hmax_open_files) ) except Exception as err: print( 'ERROR: Failed to raise the max open files setting -> ' '{0}'.format(err) ) print('Please issue the following command on your console:') print(' ulimit -n {0}'.format(REQUIRED_OPEN_FILES)) self.exit() finally: print('~' * getattr(self.options, 'output_columns', PNUM))
def _init_core_dump(self): import resource try: limits = (resource.RLIM_INFINITY, resource.RLIM_INFINITY) resource.setrlimit(resource.RLIMIT_CORE, limits) except: pass
def test_connection_del(self, tmpdir): """For issue1325.""" import os import gc try: import resource except ImportError: pytest.skip("needs resource module") limit = resource.getrlimit(resource.RLIMIT_NOFILE) try: fds = 0 while True: fds += 1 resource.setrlimit(resource.RLIMIT_NOFILE, (fds, limit[1])) try: for p in os.pipe(): os.close(p) except OSError: assert fds < 100 else: break def open_many(cleanup): con = [] for i in range(3): con.append(_sqlite3.connect(str(tmpdir.join('test.db')))) if cleanup: con[i] = None gc.collect(); gc.collect() pytest.raises(_sqlite3.OperationalError, open_many, False) gc.collect(); gc.collect() open_many(True) finally: resource.setrlimit(resource.RLIMIT_NOFILE, limit)
def stdinfuzz(self): buf = '' while True: try: buf += '\x41' resource.setrlimit(resource.RLIMIT_STACK, (-1, -1)) resource.setrlimit(resource.RLIMIT_CORE, (-1, -1)) P = Popen(self.target, stdin=PIPE) print "[*] Sending buffer with lenght: "+str(len(buf)) P.stdin.write(buf+ "\n") line = sys.stdin.readline() P.poll() ret = P.returncode if ret is None: continue else: if ret == -11: print "\n[*] Child program crashed with SIGSEGV\n" print "\n[*] Hit enter to continue.\n" continue elif ret == -4: print "\n[+] Instruction Pointer will be at: {}\n".format(str(len(buf))) break else: print "\n[*] Child program exited with code %d\n" % ret print "\n[*] Hit enter to continue.\n" continue except KeyboardInterrupt: break
def __init__(self, zmq_host, zmq_work_port, zmq_results_port, worker_id, max_retries, profile_count=0, concurrency=256, batch_size=1): work_endpoint = 'tcp://%s:%d' % (zmq_host, zmq_work_port) results_endpoint = 'tcp://%s:%d' % (zmq_host, zmq_results_port) self.worker_id = worker_id self.max_retries = max_retries self.profile_count = profile_count self.batch_size = batch_size soft_nofile, hard_nofile = resource.getrlimit(resource.RLIMIT_NOFILE) nofile_target = 1024 if os.geteuid() == 0: nofile_target = max(nofile_target, concurrency + 50) hard_nofile = nofile_target resource.setrlimit(resource.RLIMIT_NOFILE, (nofile_target, hard_nofile)) self.concurrency = concurrency self.conn_pools_lock = gevent.coros.Semaphore(1) self.conn_pools = {} # hashed by storage_url self.token_data = {} self.token_data_lock = gevent.coros.Semaphore(1) self.context = zmq.Context() self.work_pull = self.context.socket(zmq.PULL) self.work_pull.connect(work_endpoint) self.results_push = self.context.socket(zmq.PUSH) self.results_push.connect(results_endpoint) self.result_queue = gevent.queue.Queue()
def limit(self): resource.setrlimit(resource.RLIMIT_AS, (self.memorylimit.value, self.memorylimit.value + 16777216)) resource.setrlimit(resource.RLIMIT_CPU, (self.cpulimit.value, self.cpulimit.value + 1.0)) os.chroot("/tmp/pjudge/") os.setgid(305) os.setuid(305) return 0
def setlimits(): cpu_limit = current_app.config.get('SKYLINES_SUBPROCESS_CPU', 120) mem_limit = current_app.config.get('SKYLINES_SUBPROCESS_MEMORY', 256) \ * 1024 * 1024 resource.setrlimit(resource.RLIMIT_CPU, (cpu_limit, cpu_limit * 1.2)) resource.setrlimit(resource.RLIMIT_AS, (mem_limit, mem_limit * 1.2))
def __set_max_open_files(self): # Let's check to see how our max open files(ulimit -n) setting is mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE) log.info( 'Current values for max open files soft/hard setting: ' '{0}/{1}'.format( mof_s, mof_h ) ) # Let's grab, from the configuration file, the value to raise max open # files to mof_c = self.opts['max_open_files'] if mof_c > mof_h: # The configured value is higher than what's allowed log.warning( 'The value for the \'max_open_files\' setting, {0}, is higher ' 'than what the user running salt is allowed to raise to, {1}. ' 'Defaulting to {1}.'.format(mof_c, mof_h) ) mof_c = mof_h if mof_s < mof_c: # There's room to raise the value. Raise it! log.warning('Raising max open files value to {0}'.format(mof_c)) resource.setrlimit(resource.RLIMIT_NOFILE, (mof_c, mof_h)) mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE) log.warning( 'New values for max open files soft/hard values: ' '{0}/{1}'.format(mof_s, mof_h) )
def action(self): ''' Return the functions and the returners loaded up from the loader module ''' # if this is a *nix system AND modules_max_memory is set, lets enforce # a memory limit on module imports # this feature ONLY works on *nix like OSs (resource module doesn't work on windows) modules_max_memory = False if self.opts.value.get('modules_max_memory', -1) > 0 and HAS_PSUTIL and HAS_RESOURCE: log.debug( 'modules_max_memory set, enforcing a maximum of {0}'.format( self.opts.value['modules_max_memory']) ) modules_max_memory = True old_mem_limit = resource.getrlimit(resource.RLIMIT_AS) rss, vms = psutil.Process(os.getpid()).get_memory_info() mem_limit = rss + vms + self.opts.value['modules_max_memory'] resource.setrlimit(resource.RLIMIT_AS, (mem_limit, mem_limit)) elif self.opts.value.get('modules_max_memory', -1) > 0: if not HAS_PSUTIL: log.error('Unable to enforce modules_max_memory because psutil is missing') if not HAS_RESOURCE: log.error('Unable to enforce modules_max_memory because resource is missing') self.opts.value['grains'] = salt.loader.grains(self.opts.value) self.grains.value = self.opts.value['grains'] self.modules.value = salt.loader.minion_mods(self.opts.value) self.returners.value = salt.loader.returners(self.opts.value, self.modules.value) # we're done, reset the limits! if modules_max_memory is True: resource.setrlimit(resource.RLIMIT_AS, old_mem_limit)
def initialize(self): # initialize the base class GPP_base.initialize(self) self._popen_lock = threading.Lock() # Setup the TMPDIR environment variable for use in diskCapacity requests if not os.environ.has_key("TMPDIR"): os.environ["TMPDIR"] = tempfile.gettempdir() nproc = resource.getrlimit(resource.RLIMIT_NPROC) if nproc[0] < nproc[1]: #Max the softlimit out resource.setrlimit(resource.RLIMIT_NPROC, (nproc[1], nproc[1])) if nproc[1] < 1024: self._log.warning("Your system nproc hard limit [%s} is set abnormally low", nproc[1]) ###################### # Set initial capacities self.memCapacity = int(self.memTotal * self.memThresholdDecimal) # starts out as the same value self.loadCapacity = self.loadTotal * self.loadThresholdDecimal self.mcastnicIngressCapacity = int(self.mcastnicIngressTotal * self.mcastNicThresholdDecimal) self.mcastnicEgressCapacity = int(self.mcastnicEgressTotal * self.mcastNicThresholdDecimal) self.init_processor_flags() self.next_property_event = None self.start()
def testClushConfigSetRlimit(self): """test CLI.Config.ClushConfig (setrlimit)""" soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) hard2 = min(32768, hard) f = tempfile.NamedTemporaryFile(prefix='testclushconfig') f.write(""" [Main] fanout: 42 connect_timeout: 14 command_timeout: 0 history_size: 100 color: auto fd_max: %d verbosity: 1 """ % hard2) f.flush() parser = OptionParser("dummy") parser.install_display_options(verbose_options=True) parser.install_connector_options() options, _ = parser.parse_args([]) config = ClushConfig(options, filename=f.name) self.assert_(config != None) display = Display(options, config) self.assert_(display != None) # force a lower soft limit resource.setrlimit(resource.RLIMIT_NOFILE, (hard2/2, hard)) # max_fdlimit should increase soft limit again set_fdlimit(config.fd_max, display) # verify soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) self.assertEqual(soft, hard2) f.close()
def nukeize(ip, port, rounds): n=0 try: # RFC793 will lacks an exception if reset is not sent resource.setrlimit(resource.RLIMIT_NOFILE, (100000, 100000)) # modify kernel ulimit to: 100000 os.system("iptables -A OUTPUT -d %s -p tcp --dport %d --tcp-flags RST RST -j DROP"%(ip, port)) # modify IPTABLES os.system("iptables -A OUTPUT -d %s -p tcp --dport %d --tcp-flags FIN FIN -j DROP"%(ip, port)) epoll = select.epoll() connections = {} for x in range (0,int(rounds)): try: n=n+1 s = connect(ip, port) print "[Info] [AI] [NUKE] Firing 'nuke' ["+str(n)+"] -> [SHOCKING!]" connections[s.fileno()] = s epoll.register(s.fileno(), select.EPOLLOUT|select.EPOLLONESHOT) while True: n=n+1 events = epoll.poll(1) for fileno, event in events: s = connections.pop(s.fileno()) print "[Info] [AI] [NUKE] Firing 'nuke' ["+str(n)+"] -> [SHOCKING!]" if s: s.close() s = connect(ip, port) connections[s.fileno()] = s epoll.register(s.fileno(), select.EPOLLOUT|select.EPOLLONESHOT) except: print "[Error] [AI] [NUKE] Failed to engage with 'nuke' ["+str(n)+"]" os.system('iptables -D OUTPUT -d %s -p tcp --dport %d --tcp-flags FIN FIN -j DROP' %(ip, port)) # restore IPTABLES os.system('iptables -D OUTPUT -d %s -p tcp --dport %d --tcp-flags RST RST -j DROP' %(ip, port)) except: print("[Error] [AI] [NUKE] Failing to engage... -> Is still target online? -> [Checking!]")
def chkernal(): resource.setrlimit(resource.RLIMIT_NOFILE,(1000000,1000000)) os.system('sysctl -w net.ipv4.ip_local_port_range="1025 65535"') os.system('sysctl -w net.ipv4.tcp_rmem="4096 4096 16777216"') os.system('sysctl -w net.ipv4.tcp_wmem="4096 4096 16777216"') os.system('sysctl -w net.netfilter.nf_conntrack_max=250000')
def __enter__(self): self._old_limits = resource.getrlimit(resource.RLIMIT_NOFILE) resource.setrlimit(resource.RLIMIT_NOFILE, self._limits)
from core.settings import SESSION_ID_LENGTH from core.settings import SESSIONS from core.settings import TRAILS_FILE from core.settings import UNAUTHORIZED_SLEEP_TIME from core.settings import VERSION try: # Reference: https://bugs.python.org/issue7980 # Reference: http://code-trick.com/python-bug-attribute-error-_strptime/ import _strptime except ImportError: pass try: import resource resource.setrlimit(resource.RLIMIT_NOFILE, (MAX_NOFILE, MAX_NOFILE)) except: pass def start_httpd(address=None, port=None, join=False, pem=None): """ Starts HTTP server """ class ThreadingServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): def server_bind(self): self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) BaseHTTPServer.HTTPServer.server_bind(self) def finish_request(self, *args, **kwargs):
def setnmaplimits(): """Enforces limits from NMAP_LIMITS global variable.""" for limit, value in viewitems(NMAP_LIMITS): resource.setrlimit(limit, value)
# MacOSX (a.k.a. Darwin) has a default stack size that is too small # for deeply recursive regular expressions. We see this as crashes in # the Python test suite when running test_re.py and test_sre.py. The # fix is to set the stack limit to 2048. # This approach may also be useful for other Unixy platforms that # suffer from small default stack limits. if sys.platform == 'darwin': try: import resource except ImportError: pass else: soft, hard = resource.getrlimit(resource.RLIMIT_STACK) newsoft = min(hard, max(soft, 1024 * 2048)) resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) import test as _test from test import test_support RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', 'bsddb', 'decimal', 'compiler', 'subprocess', 'urlfetch') def usage(code, msg=''): print __doc__ if msg: print msg sys.exit(code) def main(tests=None,
from data.dataset import Dataset, TestDataset, inverse_normalize from model import FasterRCNNVGG16 from torch.autograd import Variable from torch.utils import data as data_ from trainer import FasterRCNNTrainer from utils import array_tool as at from utils.vis_tool import visdom_bbox from utils.eval_tool import eval_detection_voc from tensorboardX import SummaryWriter # fix for ulimit # https://github.com/pytorch/pytorch/issues/973#issuecomment-346405667 import resource rlimit = resource.getrlimit(resource.RLIMIT_NOFILE) resource.setrlimit(resource.RLIMIT_NOFILE, (20480, rlimit[1])) matplotlib.use('agg') def eval(dataloader, faster_rcnn, test_num=10000): pred_bboxes, pred_labels, pred_scores = list(), list(), list() gt_bboxes, gt_labels, gt_difficults = list(), list(), list() for ii, (imgs, sizes, gt_bboxes_, gt_labels_, gt_difficults_) in tqdm(enumerate(dataloader)): sizes = [sizes[0][0], sizes[1][0]] pred_bboxes_, pred_labels_, pred_scores_ = faster_rcnn.predict( imgs, [sizes]) gt_bboxes += list(gt_bboxes_.numpy()) gt_labels += list(gt_labels_.numpy()) gt_difficults += list(gt_difficults_.numpy())
def __exit__(self, *_): resource.setrlimit(resource.RLIMIT_CORE, self._old_limits)
def __enter__(self): self._old_limits = resource.getrlimit(resource.RLIMIT_CORE) resource.setrlimit(resource.RLIMIT_CORE, (-1, -1))
#!/usr/bin/python # -*- coding: utf-8 -*- import math from heapq import * import resource resource.setrlimit(resource.RLIMIT_AS, (2**32, 2**32)) import random def length(point1, point2): return math.sqrt((point1[0] - point2[0])**2 + (point1[1] - point2[1])**2) def distance(points): dist = dict() for i in xrange(len(points)): for j in xrange(i + 1): d = length(points[i], points[j]) dist[(i, j)] = d dist[(j, i)] = d return dist def kruskal(points, nodeCount): #returns a list of the MST edges. mst = [ ] #initializing the priority queue, grabbing all the first items first. seen = set([0]) heap = [] for i in xrange(1, nodeCount): heappush(heap, (length(points[0], points[i]), '0', str(i)))
#!/usr/bin/python import threading import time import resource def test(): time.sleep(1) threading.stack_size(32768) resource.setrlimit(resource.RLIMIT_NPROC, (65535, 65535)) xxx = [] for x in range(50000): xxx.append(threading.Thread(target=test)) try: xxx[x].start() except Exception, error: print error xxx.pop() break print 'there are %s threads running' % threading.activeCount() for x in xxx: x.join()
def startAdmin(adminClass, addrOfStarter, endpointPrep, transportClass, adminAddr, capabilities, logDefs, concurrency_context): # Unix Daemonization; skipped if not available import os, sys if hasattr(os, 'setsid'): os.setsid() try: import resource resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) # No core dumps except Exception: pass if hasattr(os, 'fork'): if os.fork(): sys.exit(0) # Slight trickiness here. There may *already* be an admin bound # to this start address. However, the external process attempting # to start is going to wait for the EndpointConnected message # before continuing, so ensure that message is *not* sent until # the local admin Transport has had time to bind and listen to the # local address, but also ensure that the message is *always* sent # even if the local admin could not start (caller will use # _verifyAdminRunning to ensure things are OK. transport = transportClass(endpointPrep) try: admin = adminClass(transport, adminAddr, capabilities, logDefs, concurrency_context) except Exception: transport.scheduleTransmit( None, TransmitIntent(addrOfStarter, EndpointConnected(0)).addCallback( onFailure=actorStartupFailed)) raise # Send of EndpointConnected is deferred until the logger is setup. See MultiProcReplicator.h_LoggerConnected below. admin.addrOfStarter = addrOfStarter setProcName( adminClass.__name__.rpartition('.')[-1], admin.transport.myAddress) # Admin does not do normal signal handling, but does want to know if children exit for each in range(1, signal.NSIG): # n.b. normally Python intercepts SIGINT to turn it into a # KeyboardInterrupt exception. However, these Actors should # be detached from the keyboard, so revert to normal SIGINT # behavior. if each not in uncatchable_signals: if each in child_exit_signals: set_signal_handler(each, admin.signalChildDied) if hasattr(signal, 'SIGUSR1'): set_signal_handler(signal.SIGUSR1, signal_admin_sts(admin)) _startLogger(transportClass, transport, admin, capabilities, logDefs, concurrency_context) #closeUnusedFiles(transport) # Admin should never enter TX-only flow control state because this # could deadlock or other non-progress conditions, especially if # using admin routing. transport.enableRXPauseFlowControl(False) admin.run()
def memory_limit(percentage: float): if platform.system() != "Linux": print('Only works on linux!') return soft, hard = resource.getrlimit(resource.RLIMIT_AS) resource.setrlimit(resource.RLIMIT_AS, (int(get_memory() * 1024 * percentage), hard))
def get_tf_dataset(quantize_bits, batch_size, dataset_name="celeb_a", n_batches=None, split="train", crop=False, label_keep_percent=1.0, random_label_percent=0.0): if random_label_percent > 0: assert 0, "Random labels not implemented for tf datasets" # https://github.com/tensorflow/datasets/issues/1441#issuecomment-581660890 import resource low, high = resource.getrlimit(resource.RLIMIT_NOFILE) resource.setrlimit(resource.RLIMIT_NOFILE, (high, high)) import tensorflow as tf tf.config.set_visible_devices([], 'GPU') import tensorflow_datasets as tfds if dataset_name == "cifar10": n_classes = 10 elif dataset_name == "cifar100": n_classes = 100 elif "imagenet" in dataset_name: n_classes = 1000 elif "svhn" in dataset_name: n_classes = 10 elif "mnist" in dataset_name: n_classes = 10 else: n_classes = None def central_crop(x): x["image"] = x["image"][::2, ::2][26:-19, 12:-13] return x def to_float(x): x["image"] = tf.cast(x["image"], dtype=tf.float32) return x def random_flip(x): x["image"] = tf.image.random_flip_left_right(x["image"]) return x def quantize(x): quantize_factor = 256 / (2**quantize_bits) x["image"] = x["image"] // quantize_factor return x def to_expected_input(x): out = {"x": x["image"]} if "label" in x: labels = x["label"] labels_one_hot = tf.one_hot(labels, n_classes) out["y"] = labels_one_hot out["y_non_one_hot"] = x["label"] return out rng = tf.random.Generator.from_seed(int(split == "train")) def make_semi_supervised(x): if "y_non_one_hot" in x: labels = x["y_non_one_hot"] x["y_is_labeled"] = rng.binomial(shape=labels.shape, counts=tf.ones(labels.shape), probs=tf.ones(labels.shape) * label_keep_percent) del x["y_non_one_hot"] return x ds = tfds.load(dataset_name, split=split) if crop: ds = ds.map(central_crop) ds = ds.map(to_float) ds = ds.map(random_flip) ds = ds.map(quantize) ds = ds.map(to_expected_input) ds = ds.map(make_semi_supervised) if split == "train": ds = ds.shuffle(15000).repeat() ds = ds.batch(batch_size) if (n_batches is not None): ds = ds.batch(n_batches, drop_remainder=True) ds = ds.prefetch(tf.data.experimental.AUTOTUNE) ds = ds.as_numpy_iterator() return ds
from core.config import cfg, cfg_from_file, cfg_from_list, assert_and_infer_cfg from datasets.roidb import combined_roidb_for_training from roi_data.loader import RoiDataLoader, MinibatchSampler, collate_minibatch from modeling.model_builder import Generalized_RCNN from utils.detectron_weight_helper import load_detectron_weight from utils.logging import setup_logging from utils.timer import Timer from utils.training_stats import TrainingStats # Set up logging and load config options logger = setup_logging(__name__) logging.getLogger('roi_data.loader').setLevel(logging.INFO) # RuntimeError: received 0 items of ancdata. Issue: pytorch/pytorch#973 rlimit = resource.getrlimit(resource.RLIMIT_NOFILE) resource.setrlimit(resource.RLIMIT_NOFILE, (4096, rlimit[1])) def parse_args(): """Parse input arguments""" parser = argparse.ArgumentParser(description='Train a X-RCNN network') parser.add_argument( '--dataset', dest='dataset', required=True, help='Dataset to use') parser.add_argument( '--cfg', dest='cfg_file', required=True, help='Config file for training (and optionally testing)') parser.add_argument( '--set', dest='set_cfgs', help='Set config keys. Key value sequence seperate by whitespace.' 'e.g. [key] [value] [key] [value]',
print('Connection', connection_count, 'closed') connection_count -= 1 async def listen(address): global connection_count server_sock = socket.socket() server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server_sock.bind(address) server_sock.listen() server_sock.setblocking(False) while True: client_sock, addr = await loop.sock_accept(server_sock) connection_count += 1 print('Connection', connection_count, 'from', addr) loop.create_task(echo(client_sock)) if __name__ == '__main__': soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) print('Setting max number of open files to', hard) resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard)) print('Starting echo server') loop.create_task(listen(('', 25000))) try: loop.run() except KeyboardInterrupt: print('Stopping echo server')
def fixLimits(): "Fix ridiculously small resource limits." setrlimit(RLIMIT_NPROC, (8192, 8192)) setrlimit(RLIMIT_NOFILE, (16384, 16384))
def set_fsize(): # here we limit the logsize resource.setrlimit(resource.RLIMIT_FSIZE, (self.trace_log_limit, self.trace_log_limit))
def rlimitTestAndSet( name, limit ): "Helper function to set rlimits" soft, hard = getrlimit( name ) if soft < limit: hardLimit = hard if limit < hard else limit setrlimit( name, ( limit, hardLimit ) )
def SetRLimitToMax(): (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE) resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
sys.stdout = AutoFlush(sys.stdout) sys.stderr = AutoFlush(sys.stderr) sock = socket.fromfd(int(sys.argv[1]), socket.AF_UNIX, socket.SOCK_STREAM) sock_lock = Lock() update_valid_fds() # Set the highest open file limit we can. # At least OS X seems to like claiming no limit as max without # allowing that to be set, so let's do some retrying. r1, r2 = resource.getrlimit(resource.RLIMIT_NOFILE) limits = [500000, 100000, 50000, 10000, 5000, 1000, r1, r2] limits = sorted((v for v in limits if r1 <= v <= r2), reverse=True) for try_limit in limits: try: resource.setrlimit(resource.RLIMIT_NOFILE, (try_limit, r2)) break except ValueError: pass r1, r2 = resource.getrlimit(resource.RLIMIT_NOFILE) if r1 < r2: print("WARNING: Failed to raise RLIMIT_NOFILE to %d. Set to %d." % ( r2, r1, )) if r1 < 5000: print("WARNING: RLIMIT_NOFILE is %d, that's not much." % (r1, )) while True: op, length = struct.unpack('<cI', recvall(sock, 5, True)) data = recvall(sock, length, True)
import sys, re, os, signal import resource, subprocess from copy import deepcopy from select import select from time import time from queue import Queue, Empty from threading import Thread # This is needed so that the recursive SMT2 S-expression parser # does not run out of stack frames when parsing large expressions smtio_reclimit = 64 * 1024 smtio_stacksize = 128 * 1024 * 1024 if sys.getrecursionlimit() < smtio_reclimit: sys.setrecursionlimit(smtio_reclimit) if resource.getrlimit(resource.RLIMIT_STACK)[0] < smtio_stacksize: resource.setrlimit(resource.RLIMIT_STACK, (smtio_stacksize, -1)) # currently running solvers (so we can kill them) running_solvers = dict() forced_shutdown = False solvers_index = 0 def force_shutdown(signum, frame): global forced_shutdown if not forced_shutdown: forced_shutdown = True if signum is not None: print("<%s>" % signal.Signals(signum).name) for p in running_solvers.values(): # os.killpg(os.getpgid(p.pid), signal.SIGTERM)
def test_max_open_files(self): with TestsLoggingHandler() as handler: logmsg_dbg = ( 'DEBUG:This salt-master instance has accepted {0} minion keys.' ) logmsg_chk = ( '{0}:The number of accepted minion keys({1}) should be lower ' 'than 1/4 of the max open files soft setting({2}). According ' 'to the system\'s hard limit, there\'s still a margin of {3} ' 'to raise the salt\'s max_open_files setting. Please consider ' 'raising this value.') logmsg_crash = ( '{0}:The number of accepted minion keys({1}) should be lower ' 'than 1/4 of the max open files soft setting({2}). ' 'salt-master will crash pretty soon! According to the ' 'system\'s hard limit, there\'s still a margin of {3} to ' 'raise the salt\'s max_open_files setting. Please consider ' 'raising this value.') mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE) tempdir = tempfile.mkdtemp(prefix='fake-keys') keys_dir = os.path.join(tempdir, 'minions') os.makedirs(keys_dir) mof_test = 256 resource.setrlimit(resource.RLIMIT_NOFILE, (mof_test, mof_h)) try: prev = 0 for newmax, level in ((24, None), (66, 'INFO'), (127, 'WARNING'), (196, 'CRITICAL')): for n in range(prev, newmax): kpath = os.path.join(keys_dir, str(n)) with salt.utils.fopen(kpath, 'w') as fp_: fp_.write(str(n)) opts = {'max_open_files': newmax, 'pki_dir': tempdir} check_max_open_files(opts) if level is None: # No log message is triggered, only the DEBUG one which # tells us how many minion keys were accepted. self.assertEqual([logmsg_dbg.format(newmax)], handler.messages) else: self.assertIn(logmsg_dbg.format(newmax), handler.messages) self.assertIn( logmsg_chk.format( level, newmax, mof_test, mof_h - newmax, ), handler.messages) handler.clear() prev = newmax newmax = mof_test for n in range(prev, newmax): kpath = os.path.join(keys_dir, str(n)) with salt.utils.fopen(kpath, 'w') as fp_: fp_.write(str(n)) opts = {'max_open_files': newmax, 'pki_dir': tempdir} check_max_open_files(opts) self.assertIn(logmsg_dbg.format(newmax), handler.messages) self.assertIn( logmsg_crash.format( 'CRITICAL', newmax, mof_test, mof_h - newmax, ), handler.messages) handler.clear() except IOError as err: if err.errno == 24: # Too many open files self.skipTest('We\'ve hit the max open files setting') raise finally: shutil.rmtree(tempdir) resource.setrlimit(resource.RLIMIT_NOFILE, (mof_s, mof_h))
def main(): """ For some reason (take a wild guess) Commvault has decided that their long options will take the form of '-option' not the standard '--option'. Always set HOME to '/root', as the commvault environment is bare """ os.environ['HOME'] = '/root' os.environ['TMPDIR'] = '/tmp' # ensure we do not inherit commvault's LD_LIBRARY_PATH os.environ.pop('LD_LIBRARY_PATH', None) holland_conf = '/etc/holland/holland.conf' if sys.platform.startswith('freebsd'): holland_conf = '/usr/local' + holland_conf parser = ArgumentParser() parser.add_argument("--config-file", "-c", metavar="<file>", help="Read configuration from the given file") parser.add_argument('-l', '--log-level', metavar='<log-level>', choices=['critical', 'error', 'warning', 'info', 'debug'], help="Specify the log level. " "One of: critical,error,warning,info,debug") parser.add_argument("--quiet", "-q", action="store_true", help="Don't log to console") parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output") parser.add_argument("--bksets", "-b", metavar="<bkset>,<bkset>...", help="only run the specified backupset", default=[], action=ArgList) parser.add_argument("-bkplevel", type=int) parser.add_argument("-attempt", type=int) parser.add_argument("-status", type=int) parser.add_argument("-job", type=int) parser.add_argument("-vm") parser.add_argument("-cn") parser.set_defaults( config_file=os.getenv('HOLLAND_CONFIG') or holland_conf, verbose=False, ) args, largs = parser.parse_known_args(sys.argv[1:]) if args.log_level: args.log_level = format_loglevel(args.log_level) bootstrap(args) logging.info("Holland (commvault agent) %s started with pid %d", HOLLAND_VERSION, os.getpid()) # Commvault usually runs with a very low default limit for nofile # so a best effort is taken to raise that here. try: resource.setrlimit(resource.RLIMIT_NOFILE, (262144, 262144)) logging.debug("(Adjusted ulimit -n (RLIMIT_NOFILE) to %d)", 262144) except (ValueError, resource.error) as exc: logging.debug("Failed to raise RLIMIT_NOFILE: %s", exc) args.command = 'backup' args.dry_run = 0 args.no_lock = 0 largs = args.bksets spool = HOLLANDCFG.lookup('holland.backup-directory') status_file = '%s/%s/newest/job_%s' % (spool, args.bksets[0], args.job) pid_name = 'holland_commvault_%s' % args.job pid_location = '/var/run/%s.pid' % pid_name try: with PidFile(pid_name): ret = 0 if run(args, largs): ret = 1 status = open(status_file, 'w') status.write(str(ret)) status.close() return ret except PidFileAlreadyLockedError: ret = 1 pid_file = open(pid_location, 'r') pid = pid_file.read() pid_file.close() logging.info("Holland (commvault agent) is already running, waiting for the pid %s", pid) count = 0 while os.path.isfile(pid_location): sleep(10) count = count + 1 #~14 hour timeout if count > 5040: logging.info("Holland (commvault agent) timed out after %s seconds", count*10) return 1 try: status = open(status_file, 'r') ret = int(status.read()) except IOError: logging.info("Holland (commvault agent) failed to open/read status file") return 1 else: status.close() return ret except IOError: logging.info("Holland (commvault agent) must have permission to write to /var/run") return 1
def run_experiment(openmlid: int, num_pipelines: int, seed: int, timeout: int, folder: str, prob_dp: float, prob_fp: float, config_map: typing.Dict): # TODO: built in check whether file already exists, in that case we can skipp # CPU print("CPU Settings:") for v in ["OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS", "BLIS_NUM_THREADS"]: print(f"\t{v}: {os.environ[v] if v in os.environ else 'n/a'}") # memory limits memory_limit = 14 * 1024 print("Setting memory limit to " + str(memory_limit) + "MB") soft, hard = resource.getrlimit(resource.RLIMIT_AS) resource.setrlimit(resource.RLIMIT_AS, (memory_limit * 1024 * 1024, memory_limit * 1024 * 1024)) # configure lccv logger (by default set to WARN, change it to DEBUG if tests fail) lccv_logger = logging.getLogger("lccv") lccv_logger.setLevel(logging.DEBUG) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) lccv_logger.addHandler(ch) exp_logger = logging.getLogger("experimenter") exp_logger.setLevel(logging.DEBUG) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) exp_logger.addHandler(ch) eval_logger = logging.getLogger("evalutils") eval_logger.setLevel(logging.DEBUG) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) eval_logger.addHandler(ch) exp_logger.info("Starting python script") exp_logger.info(f"""Running experiment under following conditions: OpenML id: {openmlid} Seed: {seed} timeout (per single evaluation): {timeout} Num Pipelines: {num_pipelines} Probability to draw a data-preprocessor: {prob_dp} Probability to draw a feature-preprocessor: {prob_fp} """) # CPU exp_logger.info("CPU Settings:") for v in ["OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS", "BLIS_NUM_THREADS"]: exp_logger.info(f"\t{v}: {os.environ[v] if v in os.environ else 'n/a'}") # memory limits memory_limit = 14 * 1024 exp_logger.info("Setting memory limit to " + str(memory_limit) + "MB") soft, hard = resource.getrlimit(resource.RLIMIT_AS) resource.setrlimit(resource.RLIMIT_AS, (memory_limit * 1024 * 1024, memory_limit * 1024 * 1024)) # load data exp_logger.info("Reading dataset") X, y = get_dataset(openmlid) exp_logger.info( f"ready. Dataset shape is {X.shape}, label column shape is {y.shape}. Now running the algorithm") if X.shape[0] <= 0: raise Exception("Dataset size invalid!") if X.shape[0] != len(y): raise Exception("X and y do not have the same size.") # creating learner sequence sampler = PipelineSampler("searchspace.json", X, y, seed, dp_proba=prob_dp, fp_proba=prob_fp) test_learners = [sampler.sample(do_build=False) for i in range(num_pipelines)] exp_logger.info(f"Evaluating portfolio of {len(test_learners)} learners.") # run lccv epsilon = 0.0 validators = [(lccv90flex, lambda r: r[0], config_map)] key = "lccv90flex" result = \ evaluate_validators(validators, test_learners, X, y, timeout, seed=seed, repeats=100, epsilon=epsilon)[key] model = result[0] runtime = result[1] if model is not None: error_rate = np.round(result[2][0], 4) model_name = str(model).replace("\n", " ") exp_logger.info(f"""Run completed. Here are the details: Model: {model} Error Rate: {error_rate} Runtime: {runtime} Results in final evaluation: {np.round(result[2][1], 4)}""") else: exp_logger.info("No model was chosen. Assigning maximum error rate") error_rate = 1 model_name = "None" # write result output = (model_name, error_rate, runtime, result[3], result[4]) with open(folder + "/results.txt", "w") as outfile: json.dump(output, outfile) exp_logger.info( f"Experiment ready. Results written to {folder}/results.txt")
def main(): # find specified locustfile and make sure it exists, using a very simplified # command line parser that is only used to parse the -f option locustfile = parse_locustfile_option() # import the locustfile docstring, user_classes = load_locustfile(locustfile) # parse all command line options options = parse_options() if options.slave or options.expect_slaves: sys.stderr.write( "[DEPRECATED] Usage of slave has been deprecated, use --worker or --expect-workers\n" ) sys.exit(1) # setup logging if not options.skip_log_setup: if options.loglevel.upper() in [ "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL" ]: setup_logging(options.loglevel, options.logfile) else: sys.stderr.write( "Invalid --loglevel. Valid values are: DEBUG/INFO/WARNING/ERROR/CRITICAL\n" ) sys.exit(1) logger = logging.getLogger(__name__) greenlet_exception_handler = greenlet_exception_logger(logger) if options.list_commands: print("Available Users:") for name in user_classes: print(" " + name) sys.exit(0) if not user_classes: logger.error("No User class found!") sys.exit(1) # make sure specified User exists if options.user_classes: missing = set(options.user_classes) - set(user_classes.keys()) if missing: logger.error("Unknown User(s): %s\n" % (", ".join(missing))) sys.exit(1) else: names = set(options.user_classes) & set(user_classes.keys()) user_classes = [user_classes[n] for n in names] else: # list() call is needed to consume the dict_view object in Python 3 user_classes = list(user_classes.values()) try: import resource if resource.getrlimit(resource.RLIMIT_NOFILE)[0] < 10000: # Increasing the limit to 10000 within a running process should work on at least MacOS. # It does not work on all OS:es, but we should be no worse off for trying. resource.setrlimit(resource.RLIMIT_NOFILE, [10000, resource.RLIM_INFINITY]) except: logger.warning( "System open file limit setting is not high enough for load testing, and the OS wouldnt allow locust to increase it by itself. See https://docs.locust.io/en/stable/installation.html#increasing-maximum-number-of-open-files-limit for more info." ) # create locust Environment environment = create_environment(user_classes, options, events=locust.events) if options.show_task_ratio: print("\n Task ratio per User class") print("-" * 80) print_task_ratio(user_classes) print("\n Total task ratio") print("-" * 80) print_task_ratio(user_classes, total=True) sys.exit(0) if options.show_task_ratio_json: from json import dumps task_data = { "per_class": get_task_ratio_dict(user_classes), "total": get_task_ratio_dict(user_classes, total=True) } print(dumps(task_data)) sys.exit(0) if options.step_time: if not options.step_load: logger.error( "The --step-time argument can only be used together with --step-load" ) sys.exit(1) if options.worker: logger.error( "--step-time should be specified on the master node, and not on worker nodes" ) sys.exit(1) try: options.step_time = parse_timespan(options.step_time) except ValueError: logger.error( "Valid --step-time formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc." ) sys.exit(1) if options.master: runner = environment.create_master_runner( master_bind_host=options.master_bind_host, master_bind_port=options.master_bind_port, ) elif options.worker: try: runner = environment.create_worker_runner(options.master_host, options.master_port) except socket.error as e: logger.error("Failed to connect to the Locust master: %s", e) sys.exit(-1) else: runner = environment.create_local_runner() # main_greenlet is pointing to runners.greenlet by default, it will point the web greenlet later if in web mode main_greenlet = runner.greenlet if options.run_time: if not options.headless: logger.error( "The --run-time argument can only be used together with --headless" ) sys.exit(1) if options.worker: logger.error( "--run-time should be specified on the master node, and not on worker nodes" ) sys.exit(1) try: options.run_time = parse_timespan(options.run_time) except ValueError: logger.error( "Valid --run-time formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc." ) sys.exit(1) def spawn_run_time_limit_greenlet(): logger.info("Run time limit set to %s seconds" % options.run_time) def timelimit_stop(): logger.info("Time limit reached. Stopping Locust.") runner.quit() gevent.spawn_later( options.run_time, timelimit_stop).link_exception(greenlet_exception_handler) # start Web UI if not options.headless and not options.worker: # spawn web greenlet protocol = "https" if options.tls_cert and options.tls_key else "http" logger.info("Starting web interface at %s://%s:%s" % (protocol, options.web_host, options.web_port)) try: if options.web_host == "*": # special check for "*" so that we're consistent with --master-bind-host web_host = '' else: web_host = options.web_host web_ui = environment.create_web_ui( host=web_host, port=options.web_port, auth_credentials=options.web_auth, tls_cert=options.tls_cert, tls_key=options.tls_key, ) except AuthCredentialsError: logger.error( "Credentials supplied with --web-auth should have the format: username:password" ) sys.exit(1) else: main_greenlet = web_ui.greenlet else: web_ui = None # Fire locust init event which can be used by end-users' code to run setup code that # need access to the Environment, Runner or WebUI environment.events.init.fire(environment=environment, runner=runner, web_ui=web_ui) if options.headless: # headless mode if options.master: # wait for worker nodes to connect while len(runner.clients.ready) < options.expect_workers: logging.info( "Waiting for workers to be ready, %s of %s connected", len(runner.clients.ready), options.expect_workers) time.sleep(1) if not options.worker: # apply headless mode defaults if options.num_users is None: options.num_users = 1 if options.hatch_rate is None: options.hatch_rate = 1 if options.step_users is None: options.step_users = 1 # start the test if options.step_time: runner.start_stepload(options.num_users, options.hatch_rate, options.step_users, options.step_time) else: runner.start(options.num_users, options.hatch_rate) if options.run_time: spawn_run_time_limit_greenlet() stats_printer_greenlet = None if not options.only_summary and (options.print_stats or (options.headless and not options.worker)): # spawn stats printing greenlet stats_printer_greenlet = gevent.spawn(stats_printer(runner.stats)) stats_printer_greenlet.link_exception(greenlet_exception_handler) if options.csv_prefix: gevent.spawn( stats_writer, environment, options.csv_prefix, full_history=options.stats_history_enabled).link_exception( greenlet_exception_handler) def shutdown(): """ Shut down locust by firing quitting event, printing/writing stats and exiting """ logger.info("Running teardowns...") environment.events.quitting.fire(environment=environment, reverse=True) # determine the process exit code if log.unhandled_greenlet_exception: code = 2 elif environment.process_exit_code is not None: code = environment.process_exit_code elif len(runner.errors) or len(runner.exceptions): code = options.exit_code_on_error else: code = 0 logger.info("Shutting down (exit code %s), bye." % code) if stats_printer_greenlet is not None: stats_printer_greenlet.kill(block=False) logger.info("Cleaning up runner...") if runner is not None: runner.quit() print_stats(runner.stats, current=False) print_percentile_stats(runner.stats) if options.csv_prefix: write_csv_files(environment, options.csv_prefix, full_history=options.stats_history_enabled) print_error_report(runner.stats) sys.exit(code) # install SIGTERM handler def sig_term_handler(): logger.info("Got SIGTERM signal") shutdown() gevent.signal_handler(signal.SIGTERM, sig_term_handler) try: logger.info("Starting Locust %s" % version) main_greenlet.join() shutdown() except KeyboardInterrupt as e: shutdown()
def initialize(topsrcdir): # Ensure we are running Python 3.6+. We run this check as soon as # possible to avoid a cryptic import/usage error. if sys.version_info < (3, 6): print("Python 3.6+ is required to run mach.") print("You are running Python", platform.python_version()) if sys.platform.startswith("linux"): print(INSTALL_PYTHON_GUIDANCE_LINUX) elif sys.platform.startswith("darwin"): print(INSTALL_PYTHON_GUIDANCE_OSX) elif "MOZILLABUILD" in os.environ: print(INSTALL_PYTHON_GUIDANCE_MOZILLABUILD) else: print(INSTALL_PYTHON_GUIDANCE_OTHER) sys.exit(1) # This directory was deleted in bug 1666345, but there may be some ignored # files here. We can safely just delete it for the user so they don't have # to clean the repo themselves. deleted_dir = os.path.join(topsrcdir, "third_party", "python", "psutil") if os.path.exists(deleted_dir): shutil.rmtree(deleted_dir, ignore_errors=True) state_dir = _create_state_dir() _activate_python_environment(topsrcdir, state_dir) import mach.base import mach.main from mach.util import setenv from mozboot.util import get_state_dir # Set a reasonable limit to the number of open files. # # Some linux systems set `ulimit -n` to a very high number, which works # well for systems that run servers, but this setting causes performance # problems when programs close file descriptors before forking, like # Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the # default in Python 3), or Rust's stdlib. In some cases, Firefox does the # same thing when spawning processes. We would prefer to lower this limit # to avoid such performance problems; processes spawned by `mach` will # inherit the limit set here. # # The Firefox build defaults the soft limit to 1024, except for builds that # do LTO, where the soft limit is 8192. We're going to default to the # latter, since people do occasionally do LTO builds on their local # machines, and requiring them to discover another magical setting after # setting up an LTO build in the first place doesn't seem good. # # This code mimics the code in taskcluster/scripts/run-task. try: import resource # Keep the hard limit the same, though, allowing processes to change # their soft limit if they need to (Firefox does, for instance). (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE) # Permit people to override our default limit if necessary via # MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses. limit = os.environ.get("MOZ_LIMIT_NOFILE") if limit: limit = int(limit) else: # If no explicit limit is given, use our default if it's less than # the current soft limit. For instance, the default on macOS is # 256, so we'd pick that rather than our default. limit = min(soft, 8192) # Now apply the limit, if it's different from the original one. if limit != soft: resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) except ImportError: # The resource module is UNIX only. pass def resolve_repository(): import mozversioncontrol try: # This API doesn't respect the vcs binary choices from configure. # If we ever need to use the VCS binary here, consider something # more robust. return mozversioncontrol.get_repository_object(path=topsrcdir) except (mozversioncontrol.InvalidRepoPath, mozversioncontrol.MissingVCSTool): return None def pre_dispatch_handler(context, handler, args): # If --disable-tests flag was enabled in the mozconfig used to compile # the build, tests will be disabled. Instead of trying to run # nonexistent tests then reporting a failure, this will prevent mach # from progressing beyond this point. if handler.category == "testing" and not handler.ok_if_tests_disabled: from mozbuild.base import BuildEnvironmentNotFoundException try: from mozbuild.base import MozbuildObject # all environments should have an instance of build object. build = MozbuildObject.from_environment() if build is not None and not getattr( build, "substs", {"ENABLE_TESTS": True} ).get("ENABLE_TESTS"): print( "Tests have been disabled with --disable-tests.\n" + "Remove the flag, and re-compile to enable tests." ) sys.exit(1) except BuildEnvironmentNotFoundException: # likely automation environment, so do nothing. pass def post_dispatch_handler( context, handler, instance, success, start_time, end_time, depth, args ): """Perform global operations after command dispatch. For now, we will use this to handle build system telemetry. """ # Don't finalize telemetry data if this mach command was invoked as part of # another mach command. if depth != 1: return _finalize_telemetry_glean( context.telemetry, handler.name == "bootstrap", success ) def populate_context(key=None): if key is None: return if key == "state_dir": return state_dir if key == "local_state_dir": return get_state_dir(srcdir=True) if key == "topdir": return topsrcdir if key == "pre_dispatch_handler": return pre_dispatch_handler if key == "post_dispatch_handler": return post_dispatch_handler if key == "repository": return resolve_repository() raise AttributeError(key) # Note which process is top-level so that recursive mach invocations can avoid writing # telemetry data. if "MACH_MAIN_PID" not in os.environ: setenv("MACH_MAIN_PID", str(os.getpid())) driver = mach.main.Mach(os.getcwd()) driver.populate_context_handler = populate_context if not driver.settings_paths: # default global machrc location driver.settings_paths.append(state_dir) # always load local repository configuration driver.settings_paths.append(topsrcdir) for category, meta in CATEGORIES.items(): driver.define_category(category, meta["short"], meta["long"], meta["priority"]) # Sparse checkouts may not have all mach_commands.py files. Ignore # errors from missing files. Same for spidermonkey tarballs. repo = resolve_repository() missing_ok = ( repo is not None and repo.sparse_checkout_present() ) or os.path.exists(os.path.join(topsrcdir, "INSTALL")) for path in MACH_MODULES: try: driver.load_commands_from_file(os.path.join(topsrcdir, path)) except mach.base.MissingFileError: if not missing_ok: raise return driver
import cobrascape.ensemble as ens from cobra.io import load_json_model from cobrascape.species import load_json_obj, save_json_obj import pandas as pd import numpy as np import sys, os, argparse, resource, warnings, itertools from os import listdir # import seaborn as sns # import matplotlib.pyplot as plt from random import shuffle from sklearn.metrics import roc_curve, auc from collections import Counter from tqdm import tqdm warnings.filterwarnings("ignore") # sklearn gives hella warnings. resource.setrlimit(resource.RLIMIT_NOFILE, (10000, -1)) # Argument parsing parser = argparse.ArgumentParser(description='Evaluate MAC samples') ### Required parameters parser.add_argument( '-f', dest='mnc_dir', required=True, help= 'Path to folder containing MAC samples. Should be same as -o argument of 01_sample_macs.py' ) ### Optional parameters (see Methods section of Kavvas et al 2020 for parameters utilized in study) parser.add_argument( '--testset', dest='train_test',
def tearDown(self): super(test_schema03, self).tearDown() resource.setrlimit(resource.RLIMIT_NOFILE, self.origFileLimit)
import signal import time # Set up a signal handler to notify us # when we run out of time. def time_expired(n, stack): print('EXPIRED :', time.ctime()) raise SystemExit('(time ran out)') signal.signal(signal.SIGXCPU, time_expired) # Adjust the CPU time limit soft, hard = resource.getrlimit(resource.RLIMIT_CPU) print('Soft limit starts as :', soft) resource.setrlimit(resource.RLIMIT_CPU, (1, hard)) soft, hard = resource.getrlimit(resource.RLIMIT_CPU) print('Soft limit changed to :', soft) print() # Consume some CPU time in a pointless exercise print('Starting:', time.ctime()) for i in range(200000): for i in range(200000): v = i * i # We should never make it this far print('Exiting :', time.ctime())
broadcast_buffers=False, find_unused_parameters=True) do_train(cfg, args, model, optimizer, resume=args.resume) return do_test(cfg, model) if __name__ == "__main__": import resource # RuntimeError: received 0 items of ancdata. Issue: pytorch/pytorch#973 rlimit = resource.getrlimit(resource.RLIMIT_NOFILE) hard_limit = rlimit[1] soft_limit = min(500000, hard_limit) iprint("soft limit: ", soft_limit, "hard limit: ", hard_limit) resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit)) args = my_default_argument_parser().parse_args() iprint("Command Line Args:", args) if args.eval_only: torch.multiprocessing.set_sharing_strategy("file_system") USE_HVD = False if args.use_hvd: if comm.HVD_AVAILABLE: iprint("Using horovod") comm.init_hvd() USE_HVD = True main(args) else:
def setlimits(): print "Setting resource limit in child" for k, v in limits.iteritems(): resource.setrlimit(v[0], (v[1], v[1]))