def main(): ''' Run code specifed by data received over pipe ''' global _forking_is_enabled _Django_old_layout_hack__load() assert is_forking(sys.argv) _forking_is_enabled = False handle = int(sys.argv[-1]) if sys.platform == 'win32': fd = msvcrt.open_osfhandle(handle, os.O_RDONLY) else: fd = handle from_parent = os.fdopen(fd, 'rb') process.current_process()._inheriting = True preparation_data = load(from_parent) prepare(preparation_data) # Huge hack to make logging before Process.run work. try: os.environ["MP_MAIN_FILE"] = sys.modules["__main__"].__file__ except KeyError: pass except AttributeError: pass loglevel = os.environ.get("_MP_FORK_LOGLEVEL_") logfile = os.environ.get("_MP_FORK_LOGFILE_") or None format = os.environ.get("_MP_FORK_LOGFORMAT_") if loglevel: from billiard import util import logging logger = util.get_logger() logger.setLevel(int(loglevel)) if not logger.handlers: logger._rudimentary_setup = True logfile = logfile or sys.__stderr__ if hasattr(logfile, "write"): handler = logging.StreamHandler(logfile) else: handler = logging.FileHandler(logfile) formatter = logging.Formatter( format or util.DEFAULT_LOGGING_FORMAT, ) handler.setFormatter(formatter) logger.addHandler(handler) self = load(from_parent) process.current_process()._inheriting = False from_parent.close() exitcode = self._bootstrap() exit(exitcode)
def get_preparation_data(name): ''' Return info about parent needed by child to unpickle process object ''' from billiard.util import _logger, _log_to_stderr d = dict( name=name, sys_path=sys.path, sys_argv=sys.argv, log_to_stderr=_log_to_stderr, orig_dir=process.ORIGINAL_DIR, authkey=process.current_process().authkey, ) if _logger is not None: d['log_level'] = _logger.getEffectiveLevel() if not WINEXE and not WINSERVICE: main_path = getattr(sys.modules['__main__'], '__file__', None) if not main_path and sys.argv[0] not in ('', '-c'): main_path = sys.argv[0] if main_path is not None: if (not os.path.isabs(main_path) and process.ORIGINAL_DIR is not None): main_path = os.path.join(process.ORIGINAL_DIR, main_path) d['main_path'] = os.path.normpath(main_path) return d
def test_worker_term_hard_handler_only_stop_MainProcess(self): try: import _multiprocessing # noqa except ImportError: raise SkipTest('only relevant for multiprocessing') process = current_process() name, process.name = process.name, 'OtherProcess' try: with patch('celery.apps.worker.active_thread_count') as c: c.return_value = 3 worker = self._Worker() handlers = self.psig( cd.install_worker_term_hard_handler, worker) try: handlers['SIGQUIT']('SIGQUIT', object()) self.assertTrue(state.should_terminate) finally: state.should_terminate = None with patch('celery.apps.worker.active_thread_count') as c: c.return_value = 1 worker = self._Worker() handlers = self.psig( cd.install_worker_term_hard_handler, worker) try: with self.assertRaises(WorkerTerminate): handlers['SIGQUIT']('SIGQUIT', object()) finally: state.should_terminate = None finally: process.name = name
def current_process(): try: from billiard import process except ImportError: pass else: return process.current_process()
def test_worker_int_handler_only_stop_MainProcess(self): try: import _multiprocessing # noqa except ImportError: raise SkipTest("only relevant for multiprocessing") process = current_process() name, process.name = process.name, "OtherProcess" with patch("celery.apps.worker.active_thread_count") as c: c.return_value = 3 try: worker = self._Worker() handlers = self.psig(cd.install_worker_int_handler, worker) handlers["SIGINT"]("SIGINT", object()) self.assertTrue(state.should_stop) finally: process.name = name state.should_stop = None with patch("celery.apps.worker.active_thread_count") as c: c.return_value = 1 try: worker = self._Worker() handlers = self.psig(cd.install_worker_int_handler, worker) with self.assertRaises(WorkerShutdown): handlers["SIGINT"]("SIGINT", object()) finally: process.name = name state.should_stop = None
def test_worker_term_hard_handler_only_stop_MainProcess(self): process = current_process() name, process.name = process.name, 'OtherProcess' try: with patch('celery.apps.worker.active_thread_count') as c: c.return_value = 3 worker = self._Worker() handlers = self.psig( cd.install_worker_term_hard_handler, worker) try: handlers['SIGQUIT']('SIGQUIT', object()) assert state.should_terminate finally: state.should_terminate = None with patch('celery.apps.worker.active_thread_count') as c: c.return_value = 1 worker = self._Worker() handlers = self.psig( cd.install_worker_term_hard_handler, worker) try: with pytest.raises(WorkerTerminate): handlers['SIGQUIT']('SIGQUIT', object()) finally: state.should_terminate = None finally: process.name = name
def current_process(): try: from billiard import process except ImportError: # pragma: no cover pass else: return process.current_process()
def get_command_line(): ''' Returns prefix of command line used for spawning a child process ''' if process.current_process()._identity == () and is_forking(sys.argv): raise RuntimeError(''' Attempt to start a new process before the current process has finished its bootstrapping phase. This probably means that have forgotten to use the proper idiom in the main module: if __name__ == '__main__': freeze_support() ... The "freeze_support()" line can be omitted if the program is not going to be frozen to produce a Windows executable.''') if getattr(sys, 'frozen', False): return [sys.executable, '--billiard-fork'] else: prog = 'from billiard.forking import main; main()' os.environ['PYTHONPATH'] = ':'.join(sys.path) return [_python_exe, '-c', prog, '--billiard-fork']
def get_command_line(): """ Returns prefix of command line used for spawning a child process """ if process.current_process()._identity == () and is_forking(sys.argv): raise RuntimeError( """ Attempt to start a new process before the current process has finished its bootstrapping phase. This probably means that have forgotten to use the proper idiom in the main module: if __name__ == '__main__': freeze_support() ... The "freeze_support()" line can be omitted if the program is not going to be frozen to produce a Windows executable.""" ) if getattr(sys, "frozen", False): return [sys.executable, "--billiard-fork"] else: prog = "from billiard.forking import main; main()" return [_python_exe, "-c", prog, "--billiard-fork"]
def set_mp_process_title(progname, info=None, hostname=None): # noqa """Set the :command:`ps` name from the current process name. Only works if :pypi:`setproctitle` is installed. """ if hostname: progname = '{0}: {1}'.format(progname, hostname) name = current_process().name if current_process else 'MainProcess' return set_process_title('{0}:{1}'.format(progname, name), info=info)
def set_mp_process_title(progname, info=None, hostname=None): # noqa """Set the ps name using the multiprocessing process name. Only works if :mod:`setproctitle` is installed. """ if hostname: progname = '{0}: {1}'.format(progname, hostname) name = current_process().name if current_process else 'MainProcess' return set_process_title('{0}:{1}'.format(progname, name), info=info)
def set_mp_process_title(progname, info=None, hostname=None): # noqa """Set the :command:`ps` name using the :mod:`multiprocessing` process name. Only works if :pypi:`setproctitle` is installed. """ if hostname: progname = "{0}: {1}".format(progname, hostname) name = current_process().name if current_process else "MainProcess" return set_process_title("{0}:{1}".format(progname, name), info=info)
def _handle_request(*args): with in_sighandler(): from celery.worker import state if current_process()._name == "MainProcess": if callback: callback(worker) safe_say("worker: {0} shutdown (MainProcess)".format(how)) if active_thread_count() > 1: setattr(state, {"Warm": "should_stop", "Cold": "should_terminate"}[how], exitcode) else: raise exc(exitcode)
def _handle_request(*args): with in_sighandler(): from celery.worker import state if current_process()._name == 'MainProcess': if callback: callback(worker) safe_say('worker: {0} shutdown (MainProcess)'.format(how)) if active_thread_count() > 1: setattr(state, {'Warm': 'should_stop', 'Cold': 'should_terminate'}[how], exitcode) else: raise exc(exitcode)
def _app_or_default_trace(app=None): # pragma: no cover from traceback import print_stack try: from billiard.process import current_process except ImportError: current_process = None if app is None: if getattr(_state._tls, 'current_app', None): print('-- RETURNING TO CURRENT APP --') # noqa+ print_stack() return _state._tls.current_app if not current_process or current_process()._name == 'MainProcess': raise Exception('DEFAULT APP') print('-- RETURNING TO DEFAULT APP --') # noqa+ print_stack() return _state.default_app return app
def get_preparation_data(name): ''' Return info about parent needed by child to unpickle process object ''' from billiard.util import _logger, _log_to_stderr d = dict( name=name, sys_path=sys.path, sys_argv=sys.argv, log_to_stderr=_log_to_stderr, orig_dir=process.ORIGINAL_DIR, authkey=process.current_process().authkey, ) if _logger is not None: d['log_level'] = _logger.getEffectiveLevel() if not WINEXE and not WINSERVICE: main_path = getattr(sys.modules['__main__'], '__file__', None) if not main_path and sys.argv[0] not in ('', '-c'): main_path = sys.argv[0] if main_path is not None: if (not os.path.isabs(main_path) and process.ORIGINAL_DIR is not None): main_path = os.path.join(process.ORIGINAL_DIR, main_path) d['main_path'] = os.path.normpath(main_path) return d # # Make (Pipe)Connection picklable # def reduce_connection(conn): if not Popen.thread_is_spawning(): raise RuntimeError( 'By default %s objects can only be shared between processes\n' 'using inheritance' % type(conn).__name__ ) return type(conn), (Popen.duplicate_for_child(conn.fileno()), conn.readable, conn.writable) ForkingPickler.register(Connection, reduce_connection) ForkingPickler.register(PipeConnection, reduce_connection)
def test_worker_term_handler_only_stop_MainProcess(self): process = current_process() name, process.name = process.name, 'OtherProcess' try: with patch('celery.apps.worker.active_thread_count') as c: c.return_value = 3 worker = self._Worker() handlers = self.psig(cd.install_worker_term_handler, worker) handlers['SIGTERM']('SIGTERM', object()) self.assertEqual(state.should_stop, EX_OK) with patch('celery.apps.worker.active_thread_count') as c: c.return_value = 1 worker = self._Worker() handlers = self.psig(cd.install_worker_term_handler, worker) with self.assertRaises(WorkerShutdown): handlers['SIGTERM']('SIGTERM', object()) finally: process.name = name state.should_stop = None
def get_avail_port(self, host, port, search_limit=100, skew=+0): try: _, skew = current_process().name.split('-') skew = int(skew) except ValueError: pass this_port = None for i in range(search_limit): _sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) this_port = port + skew + i try: _sock.bind((host, this_port)) except socket.error as exc: if exc.errno in [errno.EADDRINUSE, errno.EINVAL]: continue raise else: return _sock, this_port else: raise Exception(NO_AVAILABLE_PORT.format(self=self))
def test_worker_int_handler_only_stop_MainProcess(self): process = current_process() name, process.name = process.name, 'OtherProcess' with patch('celery.apps.worker.active_thread_count') as c: c.return_value = 3 try: worker = self._Worker() handlers = self.psig(cd.install_worker_int_handler, worker) handlers['SIGINT']('SIGINT', object()) assert state.should_stop finally: process.name = name state.should_stop = None with patch('celery.apps.worker.active_thread_count') as c: c.return_value = 1 try: worker = self._Worker() handlers = self.psig(cd.install_worker_int_handler, worker) with pytest.raises(WorkerShutdown): handlers['SIGINT']('SIGINT', object()) finally: process.name = name state.should_stop = None
import atexit from billiard.process import current_process from celery.five import monotonic from celery.utils.debug import memdump, sample_mem all_count = 0 bench_first = None bench_start = None bench_last = None bench_every = C_BENCH_EVERY bench_sample = [] __reserved = task_reserved __ready = task_ready if current_process()._name == "MainProcess": @atexit.register def on_shutdown(): if bench_first is not None and bench_last is not None: print("- Time spent in benchmark: {0!r}".format(bench_last - bench_first)) print("- Avg: {0}".format( sum(bench_sample) / len(bench_sample))) memdump() def task_reserved(request): # noqa """Called when a task is reserved by the worker.""" global bench_start global bench_first now = None
def prepare(data): ''' Try to get current process ready to unpickle process object ''' old_main_modules.append(sys.modules['__main__']) if 'name' in data: process.current_process().name = data['name'] if 'authkey' in data: process.current_process()._authkey = data['authkey'] if 'log_to_stderr' in data and data['log_to_stderr']: util.log_to_stderr() if 'log_level' in data: util.get_logger().setLevel(data['log_level']) if 'sys_path' in data: sys.path = data['sys_path'] if 'sys_argv' in data: sys.argv = data['sys_argv'] if 'dir' in data: os.chdir(data['dir']) if 'orig_dir' in data: process.ORIGINAL_DIR = data['orig_dir'] if 'main_path' in data: main_path = data['main_path'] main_name = os.path.splitext(os.path.basename(main_path))[0] if main_name == '__init__': main_name = os.path.basename(os.path.dirname(main_path)) if main_name == '__main__': main_module = sys.modules['__main__'] main_module.__file__ = main_path elif main_name != 'ipython': # Main modules not actually called __main__.py may # contain additional code that should still be executed import imp if main_path is None: dirs = None elif os.path.basename(main_path).startswith('__init__.py'): dirs = [os.path.dirname(os.path.dirname(main_path))] else: dirs = [os.path.dirname(main_path)] assert main_name not in sys.modules, main_name file, path_name, etc = imp.find_module(main_name, dirs) try: # We would like to do "imp.load_module('__main__', ...)" # here. However, that would cause 'if __name__ == # "__main__"' clauses to be executed. main_module = imp.load_module( '__parents_main__', file, path_name, etc ) finally: if file: file.close() sys.modules['__main__'] = main_module main_module.__name__ = '__main__' # Try to make the potentially picklable objects in # sys.modules['__main__'] realize they are in the main # module -- somewhat ugly. for obj in list(main_module.__dict__.values()): try: if obj.__module__ == '__parents_main__': obj.__module__ = '__main__' except Exception: pass
import atexit from billiard.process import current_process from celery.five import monotonic from celery.utils.debug import memdump, sample_mem all_count = 0 bench_first = None bench_start = None bench_last = None bench_every = C_BENCH_EVERY bench_sample = [] __reserved = task_reserved __ready = task_ready if current_process()._name == 'MainProcess': @atexit.register def on_shutdown(): if bench_first is not None and bench_last is not None: print('- Time spent in benchmark: {0!r}'.format( bench_last - bench_first)) print('- Avg: {0}'.format( sum(bench_sample) / len(bench_sample))) memdump() def task_reserved(request): # noqa global bench_start global bench_first now = None if bench_start is None: bench_start = now = monotonic()
def prepare(data): ''' Try to get current process ready to unpickle process object ''' old_main_modules.append(sys.modules['__main__']) if 'name' in data: process.current_process().name = data['name'] if 'authkey' in data: process.current_process()._authkey = data['authkey'] if 'log_to_stderr' in data and data['log_to_stderr']: util.log_to_stderr() if 'log_level' in data: util.get_logger().setLevel(data['log_level']) if 'sys_path' in data: sys.path = data['sys_path'] if 'sys_argv' in data: sys.argv = data['sys_argv'] if 'dir' in data: os.chdir(data['dir']) if 'orig_dir' in data: process.ORIGINAL_DIR = data['orig_dir'] if 'main_path' in data: main_path = data['main_path'] main_name = os.path.splitext(os.path.basename(main_path))[0] if main_name == '__init__': main_name = os.path.basename(os.path.dirname(main_path)) if main_name == '__main__': main_module = sys.modules['__main__'] main_module.__file__ = main_path elif main_name != 'ipython': # Main modules not actually called __main__.py may # contain additional code that should still be executed import imp if main_path is None: dirs = None elif os.path.basename(main_path).startswith('__init__.py'): dirs = [os.path.dirname(os.path.dirname(main_path))] else: dirs = [os.path.dirname(main_path)] assert main_name not in sys.modules, main_name file, path_name, etc = imp.find_module(main_name, dirs) try: # We would like to do "imp.load_module('__main__', ...)" # here. However, that would cause 'if __name__ == # "__main__"' clauses to be executed. main_module = imp.load_module('__parents_main__', file, path_name, etc) finally: if file: file.close() sys.modules['__main__'] = main_module main_module.__name__ = '__main__' # Try to make the potentially picklable objects in # sys.modules['__main__'] realize they are in the main # module -- somewhat ugly. for obj in list(main_module.__dict__.values()): try: if obj.__module__ == '__parents_main__': obj.__module__ = '__main__' except Exception: pass
import atexit from billiard.process import current_process from celery.five import monotonic from celery.utils.debug import memdump, sample_mem all_count = 0 bench_first = None bench_start = None bench_last = None bench_every = C_BENCH_EVERY bench_sample = [] __reserved = task_reserved __ready = task_ready if current_process()._name == "MainProcess": @atexit.register def on_shutdown(): if bench_first is not None and bench_last is not None: print("- Time spent in benchmark: {0!r}".format(bench_last - bench_first)) print("- Avg: {0}".format(sum(bench_sample) / len(bench_sample))) memdump() def task_reserved(request): # noqa """Called when a task is reserved by the worker.""" global bench_start global bench_first now = None if bench_start is None: bench_start = now = monotonic()
import atexit from billiard.process import current_process from celery.five import monotonic from celery.utils.debug import memdump, sample_mem all_count = 0 bench_first = None bench_start = None bench_last = None bench_every = C_BENCH_EVERY bench_sample = [] __reserved = task_reserved __ready = task_ready if current_process()._name == 'MainProcess': @atexit.register def on_shutdown(): if bench_first is not None and bench_last is not None: print('- Time spent in benchmark: {0!r}'.format(bench_last - bench_first)) print('- Avg: {0}'.format( sum(bench_sample) / len(bench_sample))) memdump() def task_reserved(request): # noqa global bench_start global bench_first now = None if bench_start is None: