def _run(self):
from subprocess import Popen
from multiprocessing.connection import Client, arbitrary_address
if sys.platform == 'win32':
addr = arbitrary_address('AF_PIPE')
else:
addr = arbitrary_address('AF_UNIX')
supp_server = os.path.join(os.path.dirname(__file__), 'server.py')
args = [self.executable, supp_server, addr]
env = os.environ.copy()
if self.env:
env.update(self.env)
if self.logfile and 'SUPP_LOG_FILE' not in env:
env['SUPP_LOG_FILE'] = self.logfile
self.proc = Popen(args, env=env)
start = time.time()
while True:
try:
self.conn = Client(addr)
except Exception as e:
if time.time() - start > 5:
raise Exception('Supp server launching timeout exceed: ' + str(e))
time.sleep(0.3)
else:
break
def create_listener(authkey, backlog=4):
address = arbitrary_address('AF_PIPE')
if address[1] == ':':
address = address[2:]
return address, Listener(address=address,
authkey=authkey,
backlog=backlog)
def __init__(self,
notify_on_job_done=lambda x: x,
pool_size=None,
limit=sys.maxint,
enforce_cpu_limit=True):
Thread.__init__(self)
self.daemon = True
global _counter
self.id = _counter + 1
_counter += 1
if enforce_cpu_limit:
limit = min(limit, cpu_count())
self.pool_size = limit if pool_size is None else pool_size
self.notify_on_job_done = notify_on_job_done
self.auth_key = os.urandom(32)
self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
if iswindows and self.address[1] == ':':
self.address = self.address[2:]
self.listener = Listener(address=self.address,
authkey=self.auth_key,
backlog=4)
self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue()
self.kill_queue = Queue()
self.waiting_jobs = []
self.workers = deque()
self.launched_worker_count = 0
self._worker_launch_lock = RLock()
self.start()
def __init__(self, parent):
QThread.__init__(self, parent)
self.keep_going = True
self.current_command = None
self.out_queue = Queue()
self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
self.auth_key = os.urandom(32)
if iswindows and self.address[1] == ':':
self.address = self.address[2:]
self.listener = Listener(address=self.address,
authkey=self.auth_key,
backlog=4)
self.env = {
'CALIBRE_SIMPLE_WORKER':
'calibre.utils.pyconsole.interpreter:main',
'CALIBRE_WORKER_ADDRESS':
hexlify(cPickle.dumps(self.listener.address, -1)),
'CALIBRE_WORKER_KEY':
hexlify(self.auth_key)
}
self.process = Process(self.env)
self.output_file_buf = self.process(redirect_output=False)
self.conn = self.listener.accept()
self.start()
def run_test(project_dir, executable=None, match=None, files=[], env=None):
from subprocess import Popen
from multiprocessing.connection import Client, arbitrary_address
addr = arbitrary_address('AF_UNIX')
filename = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'launcher/pt.py')
executable = executable or sys.executable
args = [executable, filename, addr, '-q']
if match:
args.append('-k %s' % match)
environ = None
if env:
environ = os.environ.copy()
environ.update(env)
args.extend(files)
proc = Popen(args, cwd=project_dir, env=environ)
start = time.time()
while not os.path.exists(addr):
if time.time() - start > 5:
raise Exception('py.test launching timeout exceed')
time.sleep(0.01)
conn = Client(addr)
return proc, conn
def run_server(project_dir, executable=None, env=None):
import os.path, time
from subprocess import Popen
from multiprocessing.connection import Client, arbitrary_address
addr = arbitrary_address('AF_UNIX')
executable = executable or sys.executable
args = [executable, __file__, addr]
environ = None
if env:
environ = os.environ.copy()
environ.update(env)
proc = Popen(args, cwd=project_dir, env=environ)
start = time.time()
while not os.path.exists(addr):
if time.time() - start > 5:
raise Exception('py.test launching timeout exceed')
time.sleep(0.01)
conn = Client(addr)
return proc, conn
def __init__(self, notify_on_job_done=lambda x: x, pool_size=None,
limit=sys.maxint, enforce_cpu_limit=True):
Thread.__init__(self)
self.daemon = True
global _counter
self.id = _counter+1
_counter += 1
if enforce_cpu_limit:
limit = min(limit, cpu_count())
self.pool_size = limit if pool_size is None else pool_size
self.notify_on_job_done = notify_on_job_done
self.auth_key = os.urandom(32)
self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
if iswindows and self.address[1] == ':':
self.address = self.address[2:]
self.listener = Listener(address=self.address,
authkey=self.auth_key, backlog=4)
self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue()
self.kill_queue = Queue()
self.waiting_jobs = []
self.workers = deque()
self.launched_worker_count = 0
self._worker_launch_lock = RLock()
self.start()
def ensure_running(self):
'''Make sure that a fork server is running.
This can be called from any process. Note that usually a child
process will just reuse the forkserver started by its parent, so
ensure_running() will do nothing.
'''
with self._lock:
resource_tracker.ensure_running()
if self._forkserver_pid is not None:
# forkserver was launched before, is it still running?
pid, status = os.waitpid(self._forkserver_pid, os.WNOHANG)
if not pid:
# still alive
return
# dead, launch it again
os.close(self._forkserver_alive_fd)
self._forkserver_address = None
self._forkserver_alive_fd = None
self._forkserver_pid = None
# XXX only thing that changed!
cmd = ('from tractor._forkserver_override import main; ' +
'main(%d, %d, %r, **%r)')
if self._preload_modules:
desired_keys = {'main_path', 'sys_path'}
data = spawn.get_preparation_data('ignore')
data = {x: y for x, y in data.items() if x in desired_keys}
else:
data = {}
with socket.socket(socket.AF_UNIX) as listener:
address = connection.arbitrary_address('AF_UNIX')
listener.bind(address)
if not util.is_abstract_socket_namespace(address):
os.chmod(address, 0o600)
listener.listen()
# all client processes own the write end of the "alive" pipe;
# when they all terminate the read end becomes ready.
alive_r, alive_w = os.pipe()
try:
fds_to_pass = [listener.fileno(), alive_r]
cmd %= (listener.fileno(), alive_r, self._preload_modules,
data)
exe = spawn.get_executable()
args = [exe] + util._args_from_interpreter_flags()
args += ['-c', cmd]
pid = util.spawnv_passfds(exe, args, fds_to_pass)
except:
os.close(alive_w)
raise
finally:
os.close(alive_r)
self._forkserver_address = address
self._forkserver_alive_fd = alive_w
self._forkserver_pid = pid
def get_open_address():
"""Return an open address to use for a multiprocessing manager."""
if sys.platform == 'win32':
return arbitrary_address("AF_PIPE")
else:
s = socket.socket(socket.AF_INET)
s.bind(('localhost', 0))
addr = s.getsockname()
s.close()
return addr
def __init__(self, dispatcher):
Thread.__init__(self)
self.daemon = True
self.auth_key = os.urandom(32)
self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
if iswindows and self.address[1] == ':':
self.address = self.address[2:]
self.listener = Listener(address=self.address,
authkey=self.auth_key, backlog=4)
self.keep_going = True
self.dispatcher = dispatcher
def create_worker(env, priority="normal", cwd=None, func="main"):
address = arbitrary_address("AF_PIPE" if iswindows else "AF_UNIX")
if iswindows and address[1] == ":":
address = address[2:]
auth_key = os.urandom(32)
listener = Listener(address=address, authkey=auth_key)
env = dict(env)
env.update(
{
"CALIBRE_WORKER_ADDRESS": hexlify(cPickle.dumps(listener.address, -1)),
"CALIBRE_WORKER_KEY": hexlify(auth_key),
"CALIBRE_SIMPLE_WORKER": "calibre.utils.ipc.simple_worker:%s" % func,
}
)
w = Worker(env)
w(cwd=cwd, priority=priority)
return listener, w
def create_worker(env, priority='normal', cwd=None, func='main'):
address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
if iswindows and address[1] == ':':
address = address[2:]
auth_key = os.urandom(32)
listener = Listener(address=address, authkey=auth_key)
env = dict(env)
env.update({
'CALIBRE_WORKER_ADDRESS' :
hexlify(cPickle.dumps(listener.address, -1)),
'CALIBRE_WORKER_KEY' : hexlify(auth_key),
'CALIBRE_SIMPLE_WORKER':
'calibre.utils.ipc.simple_worker:%s' % func,
})
w = Worker(env)
w(cwd=cwd, priority=priority)
return listener, w
def create_worker(env, priority='normal', cwd=None, func='main'):
address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
if iswindows and address[1] == ':':
address = address[2:]
auth_key = os.urandom(32)
listener = Listener(address=address, authkey=auth_key)
env = dict(env)
env.update({
'CALIBRE_WORKER_ADDRESS':
hexlify(cPickle.dumps(listener.address, -1)),
'CALIBRE_WORKER_KEY':
hexlify(auth_key),
'CALIBRE_SIMPLE_WORKER':
'calibre.utils.ipc.simple_worker:%s' % func,
})
w = Worker(env)
w(cwd=cwd, priority=priority)
return listener, w
def _NamedPipe():
#TODO check that created handles are properly destroyed
import multiprocessing.connection as mpc
import serial, ctypes, msvcrt
kernel32 = ctypes.windll.kernel32
address = mpc.arbitrary_address('AF_PIPE')
h1 = kernel32.CreateNamedPipeA(address,
mpc.win32.PIPE_ACCESS_INBOUND | 0x40000000, # FILE_FLAG_OVERLAPPED
0, 1, 0,0, mpc.win32.NMPWAIT_WAIT_FOREVER, None)
h2 = kernel32.CreateFileA(address, mpc.win32.GENERIC_WRITE, 0,
None, mpc.win32.OPEN_EXISTING, 0, None)
overlapped = serial.win32.OVERLAPPED()
overlapped.hEvent = serial.win32.CreateEvent(None, 1, 0, None)
try:
err = kernel32.ConnectNamedPipe(h1, ctypes.byref(overlapped))
if err == 0 and serial.win32.GetLastError() == serial.win32.ERROR_IO_PENDING:
kernel32.WaitForSingleObject(overlapped.hEvent, -1)
finally:
kernel32.CloseHandle(overlapped.hEvent)
fdr = os.fdopen(msvcrt.open_osfhandle(h1, 0), 'rb')
fdw = os.fdopen(msvcrt.open_osfhandle(h2, 0), 'wb')
return fdr, fdw
def run_test(project_dir, match=None, files=[]):
from subprocess import Popen
from multiprocessing.connection import Client, arbitrary_address
addr = arbitrary_address("AF_UNIX")
filename = os.path.join(os.path.dirname(os.path.abspath(__file__)), "launcher/pt.py")
args = [sys.executable, filename, addr, "-q"]
if match:
args.append("-k %s" % match)
args.extend(files)
proc = Popen(args, cwd=project_dir)
start = time.time()
while not os.path.exists(addr):
if time.time() - start > 5:
raise Exception("py.test launching timeout exceed")
time.sleep(0.01)
conn = Client(addr)
return proc, conn
def __init__(self, parent):
QThread.__init__(self, parent)
self.keep_going = True
self.current_command = None
self.out_queue = Queue()
self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
self.auth_key = os.urandom(32)
if iswindows and self.address[1] == ':':
self.address = self.address[2:]
self.listener = Listener(address=self.address,
authkey=self.auth_key, backlog=4)
self.env = {
'CALIBRE_SIMPLE_WORKER':
'calibre.utils.pyconsole.interpreter:main',
'CALIBRE_WORKER_ADDRESS':
hexlify(cPickle.dumps(self.listener.address, -1)),
'CALIBRE_WORKER_KEY': hexlify(self.auth_key)
}
self.process = Process(self.env)
self.output_file_buf = self.process(redirect_output=False)
self.conn = self.listener.accept()
self.start()
def create_master_listener(fd):
addr = arbitrary_address('AF_UNIX')
os.write(fd, addr)
os.close(fd)
return Listener(addr)
def fork_job(
mod_name,
func_name,
args=(),
kwargs={},
timeout=300, # seconds
cwd=None,
priority='normal',
env={},
no_output=False,
heartbeat=None,
abort=None,
module_is_source_code=False):
'''
Run a job in a worker process. A job is simply a function that will be
called with the supplied arguments, in the worker process.
The result of the function will be returned.
If an error occurs a WorkerError is raised.
:param mod_name: Module to import in the worker process
:param func_name: Function to call in the worker process from the imported
module
:param args: Positional arguments to pass to the function
:param kwargs: Keyword arguments to pass to the function
:param timeout: The time in seconds to wait for the worker process to
complete. If it takes longer a WorkerError is raised and the process is
killed.
:param cwd: The working directory for the worker process. I recommend
against using this, unless you are sure the path is pure ASCII.
:param priority: The process priority for the worker process
:param env: Extra environment variables to set for the worker process
:param no_output: If True, the stdout and stderr of the worker process are
discarded
:param heartbeat: If not None, it is used to check if the worker has hung,
instead of a simple timeout. It must be a callable that takes no
arguments and returns True or False. The worker will be assumed to have
hung if this function returns False. At that point, the process will be
killed and a WorkerError will be raised.
:param abort: If not None, it must be an Event. As soon as abort.is_set()
returns True, the worker process is killed. No error is raised.
:param module_is_source_code: If True, the ``mod`` is treated as python
source rather than a module name to import. The source is executed as a
module. Useful if you want to use fork_job from within a script to run some
dynamically generated python.
:return: A dictionary with the keys result and stdout_stderr. result is the
return value of the function (it must be picklable). stdout_stderr is the
path to a file that contains the stdout and stderr of the worker process.
If you set no_output=True, then this will not be present.
'''
ans = {'result': None, 'stdout_stderr': None}
address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
if iswindows and address[1] == ':':
address = address[2:]
auth_key = os.urandom(32)
listener = Listener(address=address, authkey=auth_key)
env = dict(env)
env.update({
'CALIBRE_WORKER_ADDRESS':
hexlify(cPickle.dumps(listener.address, -1)),
'CALIBRE_WORKER_KEY':
hexlify(auth_key),
'CALIBRE_SIMPLE_WORKER':
'calibre.utils.ipc.simple_worker:main',
})
w = Worker(env)
w(cwd=cwd, priority=priority)
try:
communicate(ans,
w,
listener,
(mod_name, func_name, args, kwargs, module_is_source_code),
timeout=timeout,
heartbeat=heartbeat,
abort=abort)
finally:
t = Thread(target=w.kill)
t.daemon = True
t.start()
if no_output:
try:
os.remove(w.log_path)
except:
pass
if not no_output:
ans['stdout_stderr'] = w.log_path
return ans
def create_listener(authkey, backlog=4):
address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
if iswindows and address[1] == ':':
address = address[2:]
listener = Listener(address=address, authkey=authkey, backlog=backlog)
return address, listener
def fork_job(mod_name, func_name, args=(), kwargs={}, timeout=300, # seconds
cwd=None, priority='normal', env={}, no_output=False, heartbeat=None,
abort=None, module_is_source_code=False):
'''
Run a job in a worker process. A job is simply a function that will be
called with the supplied arguments, in the worker process.
The result of the function will be returned.
If an error occurs a WorkerError is raised.
:param mod_name: Module to import in the worker process
:param func_name: Function to call in the worker process from the imported
module
:param args: Positional arguments to pass to the function
:param kwargs: Keyword arguments to pass to the function
:param timeout: The time in seconds to wait for the worker process to
complete. If it takes longer a WorkerError is raised and the process is
killed.
:param cwd: The working directory for the worker process. I recommend
against using this, unless you are sure the path is pure ASCII.
:param priority: The process priority for the worker process
:param env: Extra environment variables to set for the worker process
:param no_output: If True, the stdout and stderr of the worker process are
discarded
:param heartbeat: If not None, it is used to check if the worker has hung,
instead of a simple timeout. It must be a callable that takes no
arguments and returns True or False. The worker will be assumed to have
hung if this function returns False. At that point, the process will be
killed and a WorkerError will be raised.
:param abort: If not None, it must be an Event. As soon as abort.is_set()
returns True, the worker process is killed. No error is raised.
:param module_is_source_code: If True, the ``mod`` is treated as python
source rather than a module name to import. The source is executed as a
module. Useful if you want to use fork_job from within a script to run some
dynamically generated python.
:return: A dictionary with the keys result and stdout_stderr. result is the
return value of the function (it must be picklable). stdout_stderr is the
path to a file that contains the stdout and stderr of the worker process.
If you set no_output=True, then this will not be present.
'''
ans = {'result':None, 'stdout_stderr':None}
address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
if iswindows and address[1] == ':':
address = address[2:]
auth_key = os.urandom(32)
listener = Listener(address=address, authkey=auth_key)
env = dict(env)
env.update({
'CALIBRE_WORKER_ADDRESS' :
hexlify(cPickle.dumps(listener.address, -1)),
'CALIBRE_WORKER_KEY' : hexlify(auth_key),
'CALIBRE_SIMPLE_WORKER':
'calibre.utils.ipc.simple_worker:main',
})
w = Worker(env)
w(cwd=cwd, priority=priority)
try:
communicate(ans, w, listener, (mod_name, func_name, args, kwargs,
module_is_source_code), timeout=timeout, heartbeat=heartbeat,
abort=abort)
finally:
t = Thread(target=w.kill)
t.daemon=True
t.start()
if no_output:
try:
os.remove(w.log_path)
except:
pass
if not no_output:
ans['stdout_stderr'] = w.log_path
return ans
def create_listener(authkey, backlog=4):
address = arbitrary_address('AF_PIPE')
if address[1] == ':':
address = address[2:]
return address, Listener(address=address, authkey=authkey, backlog=backlog)
