def get_command_line(): ''' Returns prefix of command line used for spawning a child process ''' if getattr(process.current_process(), '_inheriting', False): raise RuntimeError(''' Attempt to start a new process before the current process has finished its bootstrapping phase. This probably means that you are on Windows and you have forgotten to use the proper idiom in the main module: if __name__ == '__main__': freeze_support() ... The "freeze_support()" line can be omitted if the program is not going to be frozen to produce a Windows executable.''') if getattr(sys, 'frozen', False): return [sys.executable, '--multiprocessing-fork'] else: prog = 'from multiprocessing.forking import main; main()' opts = util._args_from_interpreter_flags() return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork']
def get_command_line(**kwds): ''' Returns prefix of command line used for spawning a child process ''' if getattr(sys, 'frozen', False): return ([sys.executable, '--multiprocessing-fork'] + ['%s=%r' % item for item in kwds.items()]) else: from gevent.monkey import saved from gevent import config from gevent._config import ImportableSetting from geventmp.monkey import GEVENT_SAVED_MODULE_SETTINGS prog = 'from gevent import monkey; monkey.patch_all(**%r); ' + \ 'from gevent import config; [setattr(config, k, v) for k, v in %r.items()]; ' + \ 'from multiprocessing.spawn import spawn_main; ' + \ 'spawn_main(%s);' prog %= (saved[GEVENT_SAVED_MODULE_SETTINGS], { k: getattr(config, k) for k in dir(config) if not isinstance(config.settings[k], ImportableSetting) }, ', '.join('%s=%r' % item for item in kwds.items())) opts = util._args_from_interpreter_flags() return [spawn._python_exe ] + opts + ['-c', prog, '--multiprocessing-fork']
def get_command_line(): ''' Returns prefix of command line used for spawning a child process ''' if getattr(process.current_process(), '_inheriting', False): raise RuntimeError(''' Attempt to start a new process before the current process has finished its bootstrapping phase. This probably means that you are on Windows and you have forgotten to use the proper idiom in the main module: if __name__ == '__main__': freeze_support() ... The "freeze_support()" line can be omitted if the program is not going to be frozen to produce a Windows executable.''') if getattr(sys, 'frozen', False): return [sys.executable, '--multiprocessing-fork'] else: prog = 'from multiprocessing.forking import main; main()' opts = util._args_from_interpreter_flags() return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork']
def ensure_running(self): '''Make sure that a fork server is running. This can be called from any process. Note that usually a child process will just reuse the forkserver started by its parent, so ensure_running() will do nothing. ''' with self._lock: resource_tracker.ensure_running() if self._forkserver_pid is not None: # forkserver was launched before, is it still running? pid, status = os.waitpid(self._forkserver_pid, os.WNOHANG) if not pid: # still alive return # dead, launch it again os.close(self._forkserver_alive_fd) self._forkserver_address = None self._forkserver_alive_fd = None self._forkserver_pid = None # XXX only thing that changed! cmd = ('from tractor._forkserver_override import main; ' + 'main(%d, %d, %r, **%r)') if self._preload_modules: desired_keys = {'main_path', 'sys_path'} data = spawn.get_preparation_data('ignore') data = {x: y for x, y in data.items() if x in desired_keys} else: data = {} with socket.socket(socket.AF_UNIX) as listener: address = connection.arbitrary_address('AF_UNIX') listener.bind(address) if not util.is_abstract_socket_namespace(address): os.chmod(address, 0o600) listener.listen() # all client processes own the write end of the "alive" pipe; # when they all terminate the read end becomes ready. alive_r, alive_w = os.pipe() try: fds_to_pass = [listener.fileno(), alive_r] cmd %= (listener.fileno(), alive_r, self._preload_modules, data) exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() args += ['-c', cmd] pid = util.spawnv_passfds(exe, args, fds_to_pass) except: os.close(alive_w) raise finally: os.close(alive_r) self._forkserver_address = address self._forkserver_alive_fd = alive_w self._forkserver_pid = pid
def get_command_line(): if getattr(process.current_process(), '_inheriting', False): raise RuntimeError('\n Attempt to start a new process before the current process\n has finished its bootstrapping phase.\n\n This probably means that you are on Windows and you have\n forgotten to use the proper idiom in the main module:\n\n if __name__ == \'__main__\':\n freeze_support()\n ...\n\n The "freeze_support()" line can be omitted if the program\n is not going to be frozen to produce a Windows executable.') if getattr(sys, 'frozen', False): return [sys.executable, '--multiprocessing-fork'] prog = 'from multiprocessing.forking import main; main()' opts = util._args_from_interpreter_flags() return [ _python_exe] + opts + ['-c', prog, '--multiprocessing-fork']
def get_command_line(self, **kwds): """Returns prefix of command line used for spawning a child process.""" prog = get_fiber_init() prog = prog.format(**kwds) opts = util._args_from_interpreter_flags() if config.debug: return ([get_python_exe(self.backend.name)] + opts + ["-u", "-c", prog, "--multiprocessing-fork"]) return ([get_python_exe(self.backend.name)] + opts + ["-c", prog, "--multiprocessing-fork"])
def get_command_line(pipe_handle, **kwds): ''' Returns prefix of command line used for spawning a child process ''' if getattr(sys, 'frozen', False): return ([sys.executable, '--multiprocessing-fork', pipe_handle]) else: prog = 'from joblib.externals.loky.backend.popen_loky_win32 import main; main()' opts = util._args_from_interpreter_flags() return [spawn.get_executable()] + opts + [ '-c', prog, '--multiprocessing-fork', pipe_handle]
def get_command_line(pipe_handle, **kwds): ''' Returns prefix of command line used for spawning a child process ''' if getattr(sys, 'frozen', False): return ([sys.executable, '--multiprocessing-fork', pipe_handle]) else: prog = 'from joblib.externals.loky.backend.popen_loky_win32 import main; main()' opts = util._args_from_interpreter_flags() return [spawn.get_executable()] + opts + [ '-c', prog, '--multiprocessing-fork', pipe_handle]
def get_command_line(pipe_handle, **kwds): ''' Returns prefix of command line used for spawning a child process ''' if getattr(sys, 'frozen', False): return ([sys.executable, '--multiprocessing-fork', pipe_handle]) else: prog = 'from multiprocessing.forking import main; main()' opts = util._args_from_interpreter_flags() return [_python_exe] + opts + [ '-c', prog, '--multiprocessing-fork', pipe_handle]
def get_command_line(pipe_handle, **kwds): ''' Returns prefix of command line used for spawning a child process ''' if getattr(sys, 'frozen', False): return ([sys.executable, '--multiprocessing-fork']) else: prog = 'from multiprocessing.forking import main; main()' opts = util._args_from_interpreter_flags() return [_python_exe] + opts + [ '-c', prog, '--multiprocessing-fork', pipe_handle ]
def ensure_running(self): '''Make sure that semaphore tracker process is running. This can be run from any process. Usually a child process will use the semaphore created by its parent.''' with self._lock: if self._fd is not None: # semaphore tracker was launched before, is it still running? if self._check_alive(): # => still alive return # => dead, launch it again os.close(self._fd) self._fd = None self._pid = None warnings.warn('semaphore_tracker: process died unexpectedly, ' 'relaunching. Some semaphores might leak.') fds_to_pass = [] try: fds_to_pass.append(sys.stderr.fileno()) except Exception: pass cmd = 'from {} import main; main(%d)'.format(main.__module__) r, w = os.pipe() try: fds_to_pass.append(r) # process will out live us, so no need to wait on pid exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() # In python 3.3, there is a bug which put `-RRRRR..` instead of # `-R` in args. Replace it to get the correct flags. # See https://github.com/python/cpython/blob/3.3/Lib/subprocess.py#L488 if sys.version_info[:2] <= (3, 3): import re for i in range(1, len(args)): args[i] = re.sub("-R+", "-R", args[i]) args += ['-c', cmd % r] util.debug("launching Semaphore tracker: {}".format(args)) pid = spawnv_passfds(exe, args, fds_to_pass) except BaseException: os.close(w) raise else: self._fd = w self._pid = pid finally: os.close(r)
def ensure_running(self): '''Make sure that semaphore tracker process is running. This can be run from any process. Usually a child process will use the semaphore created by its parent.''' with self._lock: if self._fd is not None: # semaphore tracker was launched before, is it still running? if self._check_alive(): # => still alive return # => dead, launch it again os.close(self._fd) self._fd = None self._pid = None warnings.warn('semaphore_tracker: process died unexpectedly, ' 'relaunching. Some semaphores might leak.') fds_to_pass = [] try: fds_to_pass.append(sys.stderr.fileno()) except Exception: pass cmd = 'from {} import main; main(%d)'.format(main.__module__) r, w = os.pipe() try: fds_to_pass.append(r) # process will out live us, so no need to wait on pid exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() # In python 3.3, there is a bug which put `-RRRRR..` instead of # `-R` in args. Replace it to get the correct flags. # See https://github.com/python/cpython/blob/3.3/Lib/subprocess.py#L488 if sys.version_info[:2] <= (3, 3): import re for i in range(1, len(args)): args[i] = re.sub("-R+", "-R", args[i]) args += ['-c', cmd % r] util.debug("launching Semaphore tracker: {}".format(args)) pid = spawnv_passfds(exe, args, fds_to_pass) except BaseException: os.close(w) raise else: self._fd = w self._pid = pid finally: os.close(r)
def get_command_line(**kwds): ''' Returns prefix of command line used for spawning a child process ''' if getattr(sys, 'frozen', False): return ([sys.executable, '--multiprocessing-fork'] + ['%s=%r' % item for item in kwds.items()]) else: from gevent.monkey import saved prog = 'from gevent import monkey; monkey.patch_all(%s); ' + \ 'from multiprocessing.spawn import spawn_main; ' + \ 'spawn_main(%s);' prog %= (', '.join('%s=%r' % item for item in saved["_gevent_saved_patch_all"].items()), ', '.join('%s=%r' % item for item in kwds.items())) opts = util._args_from_interpreter_flags() return [spawn._python_exe] + opts + ['-c', prog, '--multiprocessing-fork']
def ensure_running(self): '''Make sure that semaphore tracker process is running. This can be run from any process. Usually a child process will use the semaphore created by its parent.''' with self._lock: if self._fd is not None: return fds_to_pass = [] try: fds_to_pass.append(sys.stderr.fileno()) except Exception: pass cmd = ('from loky.backend.semaphore_tracker import main;' 'main(%d)') r, w = os.pipe() try: fds_to_pass.append(r) # process will out live us, so no need to wait on pid exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() if len(args[-1]) > 100: warnings.warn( "args_from_flag returned -R+ -> why??", RuntimeWarning) import re args[-1] = re.sub("-R+", "-R", args[-1]) args += ['-c', cmd % r] util.debug("launching Semaphore tracker: {}".format(args)) spawnv_passfds(exe, args, fds_to_pass) except: os.close(w) raise else: self._fd = w finally: os.close(r)
def ensure_running(self): '''Make sure that resource tracker process is running. This can be run from any process. Usually a child process will use the resource created by its parent.''' with self._lock: if self._fd is not None: # resource tracker was launched before, is it still running? if self._check_alive(): # => still alive return # => dead, launch it again os.close(self._fd) if os.name == "posix": try: # At this point, the resource_tracker process has been # killed or crashed. Let's remove the process entry # from the process table to avoid zombie processes. os.waitpid(self._pid, 0) except OSError: # The process was terminated or is a child from an # ancestor of the current process. pass self._fd = None self._pid = None warnings.warn('resource_tracker: process died unexpectedly, ' 'relaunching. Some folders/sempahores might ' 'leak.') fds_to_pass = [] try: fds_to_pass.append(sys.stderr.fileno()) except Exception: pass r, w = os.pipe() if sys.platform == "win32": _r = duplicate(msvcrt.get_osfhandle(r), inheritable=True) os.close(r) r = _r cmd = 'from {} import main; main({}, {})'.format( main.__module__, r, VERBOSE) try: fds_to_pass.append(r) # process will out live us, so no need to wait on pid exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() # In python 3.3, there is a bug which put `-RRRRR..` instead of # `-R` in args. Replace it to get the correct flags. # See https://github.com/python/cpython/blob/3.3/Lib/subprocess.py#L488 if sys.version_info[:2] <= (3, 3): import re for i in range(1, len(args)): args[i] = re.sub("-R+", "-R", args[i]) args += ['-c', cmd] util.debug("launching resource tracker: {}".format(args)) # bpo-33613: Register a signal mask that will block the # signals. This signal mask will be inherited by the child # that is going to be spawned and will protect the child from a # race condition that can make the child die before it # registers signal handlers for SIGINT and SIGTERM. The mask is # unregistered after spawning the child. try: if _HAVE_SIGMASK: signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) pid = spawnv_passfds(exe, args, fds_to_pass) finally: if _HAVE_SIGMASK: signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS) except BaseException: os.close(w) raise else: self._fd = w self._pid = pid finally: if sys.platform == "win32": _winapi.CloseHandle(r) else: os.close(r)
def ensure_running(self): '''Make sure that resource tracker process is running. This can be run from any process. Usually a child process will use the resource created by its parent.''' with self._lock: if self._fd is not None: # resource tracker was launched before, is it still running? if self._check_alive(): # => still alive return # => dead, launch it again os.close(self._fd) # Clean-up to avoid dangling processes. try: # _pid can be None if this process is a child from another # python process, which has started the resource_tracker. if self._pid is not None: os.waitpid(self._pid, 0) except ChildProcessError: # The resource_tracker has already been terminated. pass self._fd = None self._pid = None warnings.warn('resource_tracker: process died unexpectedly, ' 'relaunching. Some resources might leak.') fds_to_pass = [] try: fds_to_pass.append(sys.stderr.fileno()) except Exception: pass cmd = 'from shared_memory.resource_tracker import main;main(%d)' r, w = os.pipe() try: fds_to_pass.append(r) # process will out live us, so no need to wait on pid exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() args += ['-c', cmd % r] # bpo-33613: Register a signal mask that will block the signals. # This signal mask will be inherited by the child that is going # to be spawned and will protect the child from a race condition # that can make the child die before it registers signal handlers # for SIGINT and SIGTERM. The mask is unregistered after spawning # the child. try: if _HAVE_SIGMASK: signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) pid = util.spawnv_passfds(exe, args, fds_to_pass) finally: if _HAVE_SIGMASK: signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS) except: os.close(w) raise else: self._fd = w self._pid = pid finally: os.close(r)