def _run(self): from subprocess import Popen from multiprocessing.connection import Client, arbitrary_address if sys.platform == 'win32': addr = arbitrary_address('AF_PIPE') else: addr = arbitrary_address('AF_UNIX') supp_server = os.path.join(os.path.dirname(__file__), 'server.py') args = [self.executable, supp_server, addr] env = os.environ.copy() if self.env: env.update(self.env) if self.logfile and 'SUPP_LOG_FILE' not in env: env['SUPP_LOG_FILE'] = self.logfile self.proc = Popen(args, env=env) start = time.time() while True: try: self.conn = Client(addr) except Exception as e: if time.time() - start > 5: raise Exception('Supp server launching timeout exceed: ' + str(e)) time.sleep(0.3) else: break
def create_listener(authkey, backlog=4): address = arbitrary_address('AF_PIPE') if address[1] == ':': address = address[2:] return address, Listener(address=address, authkey=authkey, backlog=backlog)
def __init__(self, notify_on_job_done=lambda x: x, pool_size=None, limit=sys.maxint, enforce_cpu_limit=True): Thread.__init__(self) self.daemon = True global _counter self.id = _counter + 1 _counter += 1 if enforce_cpu_limit: limit = min(limit, cpu_count()) self.pool_size = limit if pool_size is None else pool_size self.notify_on_job_done = notify_on_job_done self.auth_key = os.urandom(32) self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX') if iswindows and self.address[1] == ':': self.address = self.address[2:] self.listener = Listener(address=self.address, authkey=self.auth_key, backlog=4) self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue() self.kill_queue = Queue() self.waiting_jobs = [] self.workers = deque() self.launched_worker_count = 0 self._worker_launch_lock = RLock() self.start()
def __init__(self, parent): QThread.__init__(self, parent) self.keep_going = True self.current_command = None self.out_queue = Queue() self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX') self.auth_key = os.urandom(32) if iswindows and self.address[1] == ':': self.address = self.address[2:] self.listener = Listener(address=self.address, authkey=self.auth_key, backlog=4) self.env = { 'CALIBRE_SIMPLE_WORKER': 'calibre.utils.pyconsole.interpreter:main', 'CALIBRE_WORKER_ADDRESS': hexlify(cPickle.dumps(self.listener.address, -1)), 'CALIBRE_WORKER_KEY': hexlify(self.auth_key) } self.process = Process(self.env) self.output_file_buf = self.process(redirect_output=False) self.conn = self.listener.accept() self.start()
def run_test(project_dir, executable=None, match=None, files=[], env=None): from subprocess import Popen from multiprocessing.connection import Client, arbitrary_address addr = arbitrary_address('AF_UNIX') filename = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'launcher/pt.py') executable = executable or sys.executable args = [executable, filename, addr, '-q'] if match: args.append('-k %s' % match) environ = None if env: environ = os.environ.copy() environ.update(env) args.extend(files) proc = Popen(args, cwd=project_dir, env=environ) start = time.time() while not os.path.exists(addr): if time.time() - start > 5: raise Exception('py.test launching timeout exceed') time.sleep(0.01) conn = Client(addr) return proc, conn
def run_server(project_dir, executable=None, env=None): import os.path, time from subprocess import Popen from multiprocessing.connection import Client, arbitrary_address addr = arbitrary_address('AF_UNIX') executable = executable or sys.executable args = [executable, __file__, addr] environ = None if env: environ = os.environ.copy() environ.update(env) proc = Popen(args, cwd=project_dir, env=environ) start = time.time() while not os.path.exists(addr): if time.time() - start > 5: raise Exception('py.test launching timeout exceed') time.sleep(0.01) conn = Client(addr) return proc, conn
def __init__(self, notify_on_job_done=lambda x: x, pool_size=None, limit=sys.maxint, enforce_cpu_limit=True): Thread.__init__(self) self.daemon = True global _counter self.id = _counter+1 _counter += 1 if enforce_cpu_limit: limit = min(limit, cpu_count()) self.pool_size = limit if pool_size is None else pool_size self.notify_on_job_done = notify_on_job_done self.auth_key = os.urandom(32) self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX') if iswindows and self.address[1] == ':': self.address = self.address[2:] self.listener = Listener(address=self.address, authkey=self.auth_key, backlog=4) self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue() self.kill_queue = Queue() self.waiting_jobs = [] self.workers = deque() self.launched_worker_count = 0 self._worker_launch_lock = RLock() self.start()
def ensure_running(self): '''Make sure that a fork server is running. This can be called from any process. Note that usually a child process will just reuse the forkserver started by its parent, so ensure_running() will do nothing. ''' with self._lock: resource_tracker.ensure_running() if self._forkserver_pid is not None: # forkserver was launched before, is it still running? pid, status = os.waitpid(self._forkserver_pid, os.WNOHANG) if not pid: # still alive return # dead, launch it again os.close(self._forkserver_alive_fd) self._forkserver_address = None self._forkserver_alive_fd = None self._forkserver_pid = None # XXX only thing that changed! cmd = ('from tractor._forkserver_override import main; ' + 'main(%d, %d, %r, **%r)') if self._preload_modules: desired_keys = {'main_path', 'sys_path'} data = spawn.get_preparation_data('ignore') data = {x: y for x, y in data.items() if x in desired_keys} else: data = {} with socket.socket(socket.AF_UNIX) as listener: address = connection.arbitrary_address('AF_UNIX') listener.bind(address) if not util.is_abstract_socket_namespace(address): os.chmod(address, 0o600) listener.listen() # all client processes own the write end of the "alive" pipe; # when they all terminate the read end becomes ready. alive_r, alive_w = os.pipe() try: fds_to_pass = [listener.fileno(), alive_r] cmd %= (listener.fileno(), alive_r, self._preload_modules, data) exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() args += ['-c', cmd] pid = util.spawnv_passfds(exe, args, fds_to_pass) except: os.close(alive_w) raise finally: os.close(alive_r) self._forkserver_address = address self._forkserver_alive_fd = alive_w self._forkserver_pid = pid
def get_open_address(): """Return an open address to use for a multiprocessing manager.""" if sys.platform == 'win32': return arbitrary_address("AF_PIPE") else: s = socket.socket(socket.AF_INET) s.bind(('localhost', 0)) addr = s.getsockname() s.close() return addr
def __init__(self, dispatcher): Thread.__init__(self) self.daemon = True self.auth_key = os.urandom(32) self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX') if iswindows and self.address[1] == ':': self.address = self.address[2:] self.listener = Listener(address=self.address, authkey=self.auth_key, backlog=4) self.keep_going = True self.dispatcher = dispatcher
def create_worker(env, priority="normal", cwd=None, func="main"): address = arbitrary_address("AF_PIPE" if iswindows else "AF_UNIX") if iswindows and address[1] == ":": address = address[2:] auth_key = os.urandom(32) listener = Listener(address=address, authkey=auth_key) env = dict(env) env.update( { "CALIBRE_WORKER_ADDRESS": hexlify(cPickle.dumps(listener.address, -1)), "CALIBRE_WORKER_KEY": hexlify(auth_key), "CALIBRE_SIMPLE_WORKER": "calibre.utils.ipc.simple_worker:%s" % func, } ) w = Worker(env) w(cwd=cwd, priority=priority) return listener, w
def create_worker(env, priority='normal', cwd=None, func='main'): address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX') if iswindows and address[1] == ':': address = address[2:] auth_key = os.urandom(32) listener = Listener(address=address, authkey=auth_key) env = dict(env) env.update({ 'CALIBRE_WORKER_ADDRESS' : hexlify(cPickle.dumps(listener.address, -1)), 'CALIBRE_WORKER_KEY' : hexlify(auth_key), 'CALIBRE_SIMPLE_WORKER': 'calibre.utils.ipc.simple_worker:%s' % func, }) w = Worker(env) w(cwd=cwd, priority=priority) return listener, w
def create_worker(env, priority='normal', cwd=None, func='main'): address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX') if iswindows and address[1] == ':': address = address[2:] auth_key = os.urandom(32) listener = Listener(address=address, authkey=auth_key) env = dict(env) env.update({ 'CALIBRE_WORKER_ADDRESS': hexlify(cPickle.dumps(listener.address, -1)), 'CALIBRE_WORKER_KEY': hexlify(auth_key), 'CALIBRE_SIMPLE_WORKER': 'calibre.utils.ipc.simple_worker:%s' % func, }) w = Worker(env) w(cwd=cwd, priority=priority) return listener, w
def _NamedPipe(): #TODO check that created handles are properly destroyed import multiprocessing.connection as mpc import serial, ctypes, msvcrt kernel32 = ctypes.windll.kernel32 address = mpc.arbitrary_address('AF_PIPE') h1 = kernel32.CreateNamedPipeA(address, mpc.win32.PIPE_ACCESS_INBOUND | 0x40000000, # FILE_FLAG_OVERLAPPED 0, 1, 0,0, mpc.win32.NMPWAIT_WAIT_FOREVER, None) h2 = kernel32.CreateFileA(address, mpc.win32.GENERIC_WRITE, 0, None, mpc.win32.OPEN_EXISTING, 0, None) overlapped = serial.win32.OVERLAPPED() overlapped.hEvent = serial.win32.CreateEvent(None, 1, 0, None) try: err = kernel32.ConnectNamedPipe(h1, ctypes.byref(overlapped)) if err == 0 and serial.win32.GetLastError() == serial.win32.ERROR_IO_PENDING: kernel32.WaitForSingleObject(overlapped.hEvent, -1) finally: kernel32.CloseHandle(overlapped.hEvent) fdr = os.fdopen(msvcrt.open_osfhandle(h1, 0), 'rb') fdw = os.fdopen(msvcrt.open_osfhandle(h2, 0), 'wb') return fdr, fdw
def run_test(project_dir, match=None, files=[]): from subprocess import Popen from multiprocessing.connection import Client, arbitrary_address addr = arbitrary_address("AF_UNIX") filename = os.path.join(os.path.dirname(os.path.abspath(__file__)), "launcher/pt.py") args = [sys.executable, filename, addr, "-q"] if match: args.append("-k %s" % match) args.extend(files) proc = Popen(args, cwd=project_dir) start = time.time() while not os.path.exists(addr): if time.time() - start > 5: raise Exception("py.test launching timeout exceed") time.sleep(0.01) conn = Client(addr) return proc, conn
def create_master_listener(fd): addr = arbitrary_address('AF_UNIX') os.write(fd, addr) os.close(fd) return Listener(addr)
def fork_job( mod_name, func_name, args=(), kwargs={}, timeout=300, # seconds cwd=None, priority='normal', env={}, no_output=False, heartbeat=None, abort=None, module_is_source_code=False): ''' Run a job in a worker process. A job is simply a function that will be called with the supplied arguments, in the worker process. The result of the function will be returned. If an error occurs a WorkerError is raised. :param mod_name: Module to import in the worker process :param func_name: Function to call in the worker process from the imported module :param args: Positional arguments to pass to the function :param kwargs: Keyword arguments to pass to the function :param timeout: The time in seconds to wait for the worker process to complete. If it takes longer a WorkerError is raised and the process is killed. :param cwd: The working directory for the worker process. I recommend against using this, unless you are sure the path is pure ASCII. :param priority: The process priority for the worker process :param env: Extra environment variables to set for the worker process :param no_output: If True, the stdout and stderr of the worker process are discarded :param heartbeat: If not None, it is used to check if the worker has hung, instead of a simple timeout. It must be a callable that takes no arguments and returns True or False. The worker will be assumed to have hung if this function returns False. At that point, the process will be killed and a WorkerError will be raised. :param abort: If not None, it must be an Event. As soon as abort.is_set() returns True, the worker process is killed. No error is raised. :param module_is_source_code: If True, the ``mod`` is treated as python source rather than a module name to import. The source is executed as a module. Useful if you want to use fork_job from within a script to run some dynamically generated python. :return: A dictionary with the keys result and stdout_stderr. result is the return value of the function (it must be picklable). stdout_stderr is the path to a file that contains the stdout and stderr of the worker process. If you set no_output=True, then this will not be present. ''' ans = {'result': None, 'stdout_stderr': None} address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX') if iswindows and address[1] == ':': address = address[2:] auth_key = os.urandom(32) listener = Listener(address=address, authkey=auth_key) env = dict(env) env.update({ 'CALIBRE_WORKER_ADDRESS': hexlify(cPickle.dumps(listener.address, -1)), 'CALIBRE_WORKER_KEY': hexlify(auth_key), 'CALIBRE_SIMPLE_WORKER': 'calibre.utils.ipc.simple_worker:main', }) w = Worker(env) w(cwd=cwd, priority=priority) try: communicate(ans, w, listener, (mod_name, func_name, args, kwargs, module_is_source_code), timeout=timeout, heartbeat=heartbeat, abort=abort) finally: t = Thread(target=w.kill) t.daemon = True t.start() if no_output: try: os.remove(w.log_path) except: pass if not no_output: ans['stdout_stderr'] = w.log_path return ans
def create_listener(authkey, backlog=4): address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX') if iswindows and address[1] == ':': address = address[2:] listener = Listener(address=address, authkey=authkey, backlog=backlog) return address, listener
def fork_job(mod_name, func_name, args=(), kwargs={}, timeout=300, # seconds cwd=None, priority='normal', env={}, no_output=False, heartbeat=None, abort=None, module_is_source_code=False): ''' Run a job in a worker process. A job is simply a function that will be called with the supplied arguments, in the worker process. The result of the function will be returned. If an error occurs a WorkerError is raised. :param mod_name: Module to import in the worker process :param func_name: Function to call in the worker process from the imported module :param args: Positional arguments to pass to the function :param kwargs: Keyword arguments to pass to the function :param timeout: The time in seconds to wait for the worker process to complete. If it takes longer a WorkerError is raised and the process is killed. :param cwd: The working directory for the worker process. I recommend against using this, unless you are sure the path is pure ASCII. :param priority: The process priority for the worker process :param env: Extra environment variables to set for the worker process :param no_output: If True, the stdout and stderr of the worker process are discarded :param heartbeat: If not None, it is used to check if the worker has hung, instead of a simple timeout. It must be a callable that takes no arguments and returns True or False. The worker will be assumed to have hung if this function returns False. At that point, the process will be killed and a WorkerError will be raised. :param abort: If not None, it must be an Event. As soon as abort.is_set() returns True, the worker process is killed. No error is raised. :param module_is_source_code: If True, the ``mod`` is treated as python source rather than a module name to import. The source is executed as a module. Useful if you want to use fork_job from within a script to run some dynamically generated python. :return: A dictionary with the keys result and stdout_stderr. result is the return value of the function (it must be picklable). stdout_stderr is the path to a file that contains the stdout and stderr of the worker process. If you set no_output=True, then this will not be present. ''' ans = {'result':None, 'stdout_stderr':None} address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX') if iswindows and address[1] == ':': address = address[2:] auth_key = os.urandom(32) listener = Listener(address=address, authkey=auth_key) env = dict(env) env.update({ 'CALIBRE_WORKER_ADDRESS' : hexlify(cPickle.dumps(listener.address, -1)), 'CALIBRE_WORKER_KEY' : hexlify(auth_key), 'CALIBRE_SIMPLE_WORKER': 'calibre.utils.ipc.simple_worker:main', }) w = Worker(env) w(cwd=cwd, priority=priority) try: communicate(ans, w, listener, (mod_name, func_name, args, kwargs, module_is_source_code), timeout=timeout, heartbeat=heartbeat, abort=abort) finally: t = Thread(target=w.kill) t.daemon=True t.start() if no_output: try: os.remove(w.log_path) except: pass if not no_output: ans['stdout_stderr'] = w.log_path return ans