Ejemplo n.º 1
0
    def get_command_line():
        '''
        Returns prefix of command line used for spawning a child process
        '''
        if getattr(process.current_process(), '_inheriting', False):
            raise RuntimeError('''
            Attempt to start a new process before the current process
            has finished its bootstrapping phase.

            This probably means that you are on Windows and you have
            forgotten to use the proper idiom in the main module:

                if __name__ == '__main__':
                    freeze_support()
                    ...

            The "freeze_support()" line can be omitted if the program
            is not going to be frozen to produce a Windows executable.''')

        if getattr(sys, 'frozen', False):
            return [sys.executable, '--multiprocessing-fork']
        else:
            prog = 'from multiprocessing.forking import main; main()'
            opts = util._args_from_interpreter_flags()
            return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork']
Ejemplo n.º 2
0
def get_command_line(**kwds):
    '''
    Returns prefix of command line used for spawning a child process
    '''
    if getattr(sys, 'frozen', False):
        return ([sys.executable, '--multiprocessing-fork'] +
                ['%s=%r' % item for item in kwds.items()])
    else:
        from gevent.monkey import saved
        from gevent import config
        from gevent._config import ImportableSetting
        from geventmp.monkey import GEVENT_SAVED_MODULE_SETTINGS

        prog = 'from gevent import monkey; monkey.patch_all(**%r); ' + \
               'from gevent import config; [setattr(config, k, v) for k, v in %r.items()]; ' + \
               'from multiprocessing.spawn import spawn_main; ' + \
               'spawn_main(%s);'

        prog %= (saved[GEVENT_SAVED_MODULE_SETTINGS], {
            k: getattr(config, k)
            for k in dir(config)
            if not isinstance(config.settings[k], ImportableSetting)
        }, ', '.join('%s=%r' % item for item in kwds.items()))

        opts = util._args_from_interpreter_flags()
        return [spawn._python_exe
                ] + opts + ['-c', prog, '--multiprocessing-fork']
Ejemplo n.º 3
0
    def get_command_line():
        '''
        Returns prefix of command line used for spawning a child process
        '''
        if getattr(process.current_process(), '_inheriting', False):
            raise RuntimeError('''
            Attempt to start a new process before the current process
            has finished its bootstrapping phase.

            This probably means that you are on Windows and you have
            forgotten to use the proper idiom in the main module:

                if __name__ == '__main__':
                    freeze_support()
                    ...

            The "freeze_support()" line can be omitted if the program
            is not going to be frozen to produce a Windows executable.''')

        if getattr(sys, 'frozen', False):
            return [sys.executable, '--multiprocessing-fork']
        else:
            prog = 'from multiprocessing.forking import main; main()'
            opts = util._args_from_interpreter_flags()
            return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork']
Ejemplo n.º 4
0
    def ensure_running(self):
        '''Make sure that a fork server is running.

        This can be called from any process.  Note that usually a child
        process will just reuse the forkserver started by its parent, so
        ensure_running() will do nothing.
        '''
        with self._lock:
            resource_tracker.ensure_running()
            if self._forkserver_pid is not None:
                # forkserver was launched before, is it still running?
                pid, status = os.waitpid(self._forkserver_pid, os.WNOHANG)
                if not pid:
                    # still alive
                    return
                # dead, launch it again
                os.close(self._forkserver_alive_fd)
                self._forkserver_address = None
                self._forkserver_alive_fd = None
                self._forkserver_pid = None

            # XXX only thing that changed!
            cmd = ('from tractor._forkserver_override import main; ' +
                   'main(%d, %d, %r, **%r)')

            if self._preload_modules:
                desired_keys = {'main_path', 'sys_path'}
                data = spawn.get_preparation_data('ignore')
                data = {x: y for x, y in data.items() if x in desired_keys}
            else:
                data = {}

            with socket.socket(socket.AF_UNIX) as listener:
                address = connection.arbitrary_address('AF_UNIX')
                listener.bind(address)
                if not util.is_abstract_socket_namespace(address):
                    os.chmod(address, 0o600)
                listener.listen()

                # all client processes own the write end of the "alive" pipe;
                # when they all terminate the read end becomes ready.
                alive_r, alive_w = os.pipe()
                try:
                    fds_to_pass = [listener.fileno(), alive_r]
                    cmd %= (listener.fileno(), alive_r, self._preload_modules,
                            data)
                    exe = spawn.get_executable()
                    args = [exe] + util._args_from_interpreter_flags()
                    args += ['-c', cmd]
                    pid = util.spawnv_passfds(exe, args, fds_to_pass)
                except:
                    os.close(alive_w)
                    raise
                finally:
                    os.close(alive_r)
                self._forkserver_address = address
                self._forkserver_alive_fd = alive_w
                self._forkserver_pid = pid
 def get_command_line():
     if getattr(process.current_process(), '_inheriting', False):
         raise RuntimeError('\n            Attempt to start a new process before the current process\n            has finished its bootstrapping phase.\n\n            This probably means that you are on Windows and you have\n            forgotten to use the proper idiom in the main module:\n\n                if __name__ == \'__main__\':\n                    freeze_support()\n                    ...\n\n            The "freeze_support()" line can be omitted if the program\n            is not going to be frozen to produce a Windows executable.')
     if getattr(sys, 'frozen', False):
         return [sys.executable, '--multiprocessing-fork']
     prog = 'from multiprocessing.forking import main; main()'
     opts = util._args_from_interpreter_flags()
     return [
      _python_exe] + opts + ['-c', prog, '--multiprocessing-fork']
Ejemplo n.º 6
0
    def get_command_line(self, **kwds):
        """Returns prefix of command line used for spawning a child process."""
        prog = get_fiber_init()
        prog = prog.format(**kwds)
        opts = util._args_from_interpreter_flags()
        if config.debug:
            return ([get_python_exe(self.backend.name)] + opts +
                    ["-u", "-c", prog, "--multiprocessing-fork"])

        return ([get_python_exe(self.backend.name)] + opts +
                ["-c", prog, "--multiprocessing-fork"])
Ejemplo n.º 7
0
def get_command_line(pipe_handle, **kwds):
    '''
    Returns prefix of command line used for spawning a child process
    '''
    if getattr(sys, 'frozen', False):
        return ([sys.executable, '--multiprocessing-fork', pipe_handle])
    else:
        prog = 'from joblib.externals.loky.backend.popen_loky_win32 import main; main()'
        opts = util._args_from_interpreter_flags()
        return [spawn.get_executable()] + opts + [
            '-c', prog, '--multiprocessing-fork', pipe_handle]
Ejemplo n.º 8
0
 def get_command_line(pipe_handle, **kwds):
     '''
     Returns prefix of command line used for spawning a child process
     '''
     if getattr(sys, 'frozen', False):
         return ([sys.executable, '--multiprocessing-fork', pipe_handle])
     else:
         prog = 'from joblib.externals.loky.backend.popen_loky_win32 import main; main()'
         opts = util._args_from_interpreter_flags()
         return [spawn.get_executable()] + opts + [
             '-c', prog, '--multiprocessing-fork', pipe_handle]
Ejemplo n.º 9
0
 def get_command_line(pipe_handle, **kwds):
     '''
     Returns prefix of command line used for spawning a child process
     '''
     if getattr(sys, 'frozen', False):
         return ([sys.executable, '--multiprocessing-fork', pipe_handle])
     else:
         prog = 'from multiprocessing.forking import main; main()'
         opts = util._args_from_interpreter_flags()
         return [_python_exe] + opts + [
             '-c', prog, '--multiprocessing-fork', pipe_handle]
Ejemplo n.º 10
0
Archivo: spawn.py Proyecto: rth/loky
 def get_command_line(pipe_handle, **kwds):
     '''
     Returns prefix of command line used for spawning a child process
     '''
     if getattr(sys, 'frozen', False):
         return ([sys.executable, '--multiprocessing-fork'])
     else:
         prog = 'from multiprocessing.forking import main; main()'
         opts = util._args_from_interpreter_flags()
         return [_python_exe] + opts + [
             '-c', prog, '--multiprocessing-fork', pipe_handle
         ]
Ejemplo n.º 11
0
    def ensure_running(self):
        '''Make sure that semaphore tracker process is running.

        This can be run from any process.  Usually a child process will use
        the semaphore created by its parent.'''
        with self._lock:
            if self._fd is not None:
                # semaphore tracker was launched before, is it still running?
                if self._check_alive():
                    # => still alive
                    return
                # => dead, launch it again
                os.close(self._fd)
                self._fd = None
                self._pid = None

                warnings.warn('semaphore_tracker: process died unexpectedly, '
                              'relaunching.  Some semaphores might leak.')

            fds_to_pass = []
            try:
                fds_to_pass.append(sys.stderr.fileno())
            except Exception:
                pass

            cmd = 'from {} import main; main(%d)'.format(main.__module__)
            r, w = os.pipe()
            try:
                fds_to_pass.append(r)
                # process will out live us, so no need to wait on pid
                exe = spawn.get_executable()
                args = [exe] + util._args_from_interpreter_flags()
                # In python 3.3, there is a bug which put `-RRRRR..` instead of
                # `-R` in args. Replace it to get the correct flags.
                # See https://github.com/python/cpython/blob/3.3/Lib/subprocess.py#L488
                if sys.version_info[:2] <= (3, 3):
                    import re
                    for i in range(1, len(args)):
                        args[i] = re.sub("-R+", "-R", args[i])
                args += ['-c', cmd % r]
                util.debug("launching Semaphore tracker: {}".format(args))
                pid = spawnv_passfds(exe, args, fds_to_pass)
            except BaseException:
                os.close(w)
                raise
            else:
                self._fd = w
                self._pid = pid
            finally:
                os.close(r)
Ejemplo n.º 12
0
    def ensure_running(self):
        '''Make sure that semaphore tracker process is running.

        This can be run from any process.  Usually a child process will use
        the semaphore created by its parent.'''
        with self._lock:
            if self._fd is not None:
                # semaphore tracker was launched before, is it still running?
                if self._check_alive():
                    # => still alive
                    return
                # => dead, launch it again
                os.close(self._fd)
                self._fd = None
                self._pid = None

                warnings.warn('semaphore_tracker: process died unexpectedly, '
                              'relaunching.  Some semaphores might leak.')

            fds_to_pass = []
            try:
                fds_to_pass.append(sys.stderr.fileno())
            except Exception:
                pass

            cmd = 'from {} import main; main(%d)'.format(main.__module__)
            r, w = os.pipe()
            try:
                fds_to_pass.append(r)
                # process will out live us, so no need to wait on pid
                exe = spawn.get_executable()
                args = [exe] + util._args_from_interpreter_flags()
                # In python 3.3, there is a bug which put `-RRRRR..` instead of
                # `-R` in args. Replace it to get the correct flags.
                # See https://github.com/python/cpython/blob/3.3/Lib/subprocess.py#L488
                if sys.version_info[:2] <= (3, 3):
                    import re
                    for i in range(1, len(args)):
                        args[i] = re.sub("-R+", "-R", args[i])
                args += ['-c', cmd % r]
                util.debug("launching Semaphore tracker: {}".format(args))
                pid = spawnv_passfds(exe, args, fds_to_pass)
            except BaseException:
                os.close(w)
                raise
            else:
                self._fd = w
                self._pid = pid
            finally:
                os.close(r)
Ejemplo n.º 13
0
def get_command_line(**kwds):
    '''
    Returns prefix of command line used for spawning a child process
    '''
    if getattr(sys, 'frozen', False):
        return ([sys.executable, '--multiprocessing-fork'] +
                ['%s=%r' % item for item in kwds.items()])
    else:
        from gevent.monkey import saved

        prog = 'from gevent import monkey; monkey.patch_all(%s); ' + \
               'from multiprocessing.spawn import spawn_main; ' + \
               'spawn_main(%s);'

        prog %= (', '.join('%s=%r' % item for item in saved["_gevent_saved_patch_all"].items()),
                 ', '.join('%s=%r' % item for item in kwds.items()))
        opts = util._args_from_interpreter_flags()
        return [spawn._python_exe] + opts + ['-c', prog, '--multiprocessing-fork']
Ejemplo n.º 14
0
    def ensure_running(self):
        '''Make sure that semaphore tracker process is running.

        This can be run from any process.  Usually a child process will use
        the semaphore created by its parent.'''
        with self._lock:
            if self._fd is not None:
                return
            fds_to_pass = []
            try:
                fds_to_pass.append(sys.stderr.fileno())
            except Exception:
                pass
            cmd = ('from loky.backend.semaphore_tracker import main;'
                   'main(%d)')
            r, w = os.pipe()
            try:
                fds_to_pass.append(r)
                # process will out live us, so no need to wait on pid
                exe = spawn.get_executable()
                args = [exe] + util._args_from_interpreter_flags()
                if len(args[-1]) > 100:
                    warnings.warn(
                        "args_from_flag returned -R+ -> why??", RuntimeWarning)
                    import re
                    args[-1] = re.sub("-R+", "-R", args[-1])
                args += ['-c', cmd % r]
                util.debug("launching Semaphore tracker: {}".format(args))
                spawnv_passfds(exe, args, fds_to_pass)
            except:
                os.close(w)
                raise
            else:
                self._fd = w
            finally:
                os.close(r)
Ejemplo n.º 15
0
    def ensure_running(self):
        '''Make sure that resource tracker process is running.

        This can be run from any process.  Usually a child process will use
        the resource created by its parent.'''
        with self._lock:
            if self._fd is not None:
                # resource tracker was launched before, is it still running?
                if self._check_alive():
                    # => still alive
                    return
                # => dead, launch it again
                os.close(self._fd)
                if os.name == "posix":
                    try:
                        # At this point, the resource_tracker process has been
                        # killed or crashed. Let's remove the process entry
                        # from the process table to avoid zombie processes.
                        os.waitpid(self._pid, 0)
                    except OSError:
                        # The process was terminated or is a child from an
                        # ancestor of the current process.
                        pass
                self._fd = None
                self._pid = None

                warnings.warn('resource_tracker: process died unexpectedly, '
                              'relaunching.  Some folders/sempahores might '
                              'leak.')

            fds_to_pass = []
            try:
                fds_to_pass.append(sys.stderr.fileno())
            except Exception:
                pass

            r, w = os.pipe()
            if sys.platform == "win32":
                _r = duplicate(msvcrt.get_osfhandle(r), inheritable=True)
                os.close(r)
                r = _r

            cmd = 'from {} import main; main({}, {})'.format(
                main.__module__, r, VERBOSE)
            try:
                fds_to_pass.append(r)
                # process will out live us, so no need to wait on pid
                exe = spawn.get_executable()
                args = [exe] + util._args_from_interpreter_flags()
                # In python 3.3, there is a bug which put `-RRRRR..` instead of
                # `-R` in args. Replace it to get the correct flags.
                # See https://github.com/python/cpython/blob/3.3/Lib/subprocess.py#L488
                if sys.version_info[:2] <= (3, 3):
                    import re
                    for i in range(1, len(args)):
                        args[i] = re.sub("-R+", "-R", args[i])
                args += ['-c', cmd]
                util.debug("launching resource tracker: {}".format(args))
                # bpo-33613: Register a signal mask that will block the
                # signals.  This signal mask will be inherited by the child
                # that is going to be spawned and will protect the child from a
                # race condition that can make the child die before it
                # registers signal handlers for SIGINT and SIGTERM. The mask is
                # unregistered after spawning the child.
                try:
                    if _HAVE_SIGMASK:
                        signal.pthread_sigmask(signal.SIG_BLOCK,
                                               _IGNORED_SIGNALS)
                    pid = spawnv_passfds(exe, args, fds_to_pass)
                finally:
                    if _HAVE_SIGMASK:
                        signal.pthread_sigmask(signal.SIG_UNBLOCK,
                                               _IGNORED_SIGNALS)
            except BaseException:
                os.close(w)
                raise
            else:
                self._fd = w
                self._pid = pid
            finally:
                if sys.platform == "win32":
                    _winapi.CloseHandle(r)
                else:
                    os.close(r)
    def ensure_running(self):
        '''Make sure that resource tracker process is running.
        This can be run from any process.  Usually a child process will use
        the resource created by its parent.'''
        with self._lock:
            if self._fd is not None:
                # resource tracker was launched before, is it still running?
                if self._check_alive():
                    # => still alive
                    return
                # => dead, launch it again
                os.close(self._fd)

                # Clean-up to avoid dangling processes.
                try:
                    # _pid can be None if this process is a child from another
                    # python process, which has started the resource_tracker.
                    if self._pid is not None:
                        os.waitpid(self._pid, 0)
                except ChildProcessError:
                    # The resource_tracker has already been terminated.
                    pass
                self._fd = None
                self._pid = None

                warnings.warn('resource_tracker: process died unexpectedly, '
                              'relaunching.  Some resources might leak.')

            fds_to_pass = []
            try:
                fds_to_pass.append(sys.stderr.fileno())
            except Exception:
                pass
            cmd = 'from shared_memory.resource_tracker import main;main(%d)'
            r, w = os.pipe()
            try:
                fds_to_pass.append(r)
                # process will out live us, so no need to wait on pid
                exe = spawn.get_executable()
                args = [exe] + util._args_from_interpreter_flags()
                args += ['-c', cmd % r]
                # bpo-33613: Register a signal mask that will block the signals.
                # This signal mask will be inherited by the child that is going
                # to be spawned and will protect the child from a race condition
                # that can make the child die before it registers signal handlers
                # for SIGINT and SIGTERM. The mask is unregistered after spawning
                # the child.
                try:
                    if _HAVE_SIGMASK:
                        signal.pthread_sigmask(signal.SIG_BLOCK,
                                               _IGNORED_SIGNALS)
                    pid = util.spawnv_passfds(exe, args, fds_to_pass)
                finally:
                    if _HAVE_SIGMASK:
                        signal.pthread_sigmask(signal.SIG_UNBLOCK,
                                               _IGNORED_SIGNALS)
            except:
                os.close(w)
                raise
            else:
                self._fd = w
                self._pid = pid
            finally:
                os.close(r)