def do_launch(self, env, gui, redirect_output, rfile, job_name=None):
w = Worker(env, gui=gui, job_name=job_name)
try:
w(redirect_output=redirect_output)
conn = eintr_retry_call(self.listener.accept)
if conn is None:
raise Exception('Failed to launch worker process')
except BaseException:
try:
w.kill()
except:
pass
import traceback
return traceback.format_exc()
return ConnectedWorker(w, conn, rfile)
def start_pipe_worker(command, env=None, priority='normal', **process_args):
import subprocess
from functools import partial
w = Worker(env or {})
args = {'stdout': subprocess.PIPE, 'stdin': subprocess.PIPE, 'env': w.env}
args.update(process_args)
if iswindows:
import win32process
priority = {
'high': win32process.HIGH_PRIORITY_CLASS,
'normal': win32process.NORMAL_PRIORITY_CLASS,
'low': win32process.IDLE_PRIORITY_CLASS
}[priority]
args['creationflags'] = win32process.CREATE_NO_WINDOW | priority
else:
def renice(niceness):
try:
os.nice(niceness)
except:
pass
niceness = {'normal': 0, 'low': 10, 'high': 20}[priority]
args['preexec_fn'] = partial(renice, niceness)
args['close_fds'] = True
exe = w.executable
cmd = [exe] if isinstance(exe, str) else exe
p = subprocess.Popen(cmd + ['--pipe-worker', command], **args)
return p
def start_pipe_worker(command, env=None, priority='normal'):
import subprocess, atexit
from functools import partial
w = Worker(env or {})
args = {'stdout': subprocess.PIPE, 'stdin': subprocess.PIPE, 'env': w.env}
if iswindows:
import win32process
priority = {
'high': win32process.HIGH_PRIORITY_CLASS,
'normal': win32process.NORMAL_PRIORITY_CLASS,
'low': win32process.IDLE_PRIORITY_CLASS
}[priority]
args['creationflags'] = win32process.CREATE_NO_WINDOW | priority
else:
def renice(niceness):
try:
os.nice(niceness)
except:
pass
niceness = {'normal': 0, 'low': 10, 'high': 20}[priority]
args['preexec_fn'] = partial(renice, niceness)
p = subprocess.Popen([w.executable, '--pipe-worker', command], **args)
atexit.register(w.kill)
return p
def test_executables(self):
from calibre.utils.ipc.launch import Worker
from calibre.ebooks.pdf.pdftohtml import PDFTOHTML
w = Worker({})
self.assertTrue(os.path.exists(w.executable), 'calibre-parallel (%s) does not exist' % w.executable)
self.assertTrue(os.path.exists(w.gui_executable), 'calibre-parallel-gui (%s) does not exist' % w.gui_executable)
self.assertTrue(os.path.exists(PDFTOHTML), 'pdftohtml (%s) does not exist' % PDFTOHTML)
if iswindows:
from calibre.devices.usbms.device import eject_exe
self.assertTrue(os.path.exists(eject_exe()), 'calibre-eject.exe (%s) does not exist' % eject_exe())
def do_launch(self, gui, redirect_output, rfile, job_name=None):
a, b = Pipe()
with a:
env = {
'CALIBRE_WORKER_FD': str(a.fileno()),
'CALIBRE_WORKER_RESULT': environ_item(as_hex_unicode(rfile))
}
w = Worker(env, gui=gui, job_name=job_name)
try:
w(pass_fds=(a.fileno(), ), redirect_output=redirect_output)
except BaseException:
try:
w.kill()
except:
pass
b.close()
import traceback
return traceback.format_exc()
return ConnectedWorker(w, b, rfile)
def create_worker(env, priority='normal', cwd=None, func='main'):
env = dict(env)
a, b = Pipe()
with a:
env.update({
'CALIBRE_WORKER_FD':
str(a.fileno()),
'CALIBRE_SIMPLE_WORKER':
environ_item('calibre.utils.ipc.simple_worker:%s' % func),
})
w = Worker(env)
w(cwd=cwd, priority=priority, pass_fds=(a.fileno(), ))
return b, w
def create_worker(env, priority='normal', cwd=None, func='main'):
from calibre.utils.ipc.server import create_listener
auth_key = os.urandom(32)
address, listener = create_listener(auth_key)
env = dict(env)
env.update({
'CALIBRE_WORKER_ADDRESS': environ_item(as_hex_unicode(msgpack_dumps(address))),
'CALIBRE_WORKER_KEY': environ_item(as_hex_unicode(auth_key)),
'CALIBRE_SIMPLE_WORKER': environ_item('calibre.utils.ipc.simple_worker:%s' % func),
})
w = Worker(env)
w(cwd=cwd, priority=priority)
return listener, w
def create_worker(env, priority='normal', cwd=None, func='main'):
from calibre.utils.ipc.server import create_listener
auth_key = os.urandom(32)
address, listener = create_listener(auth_key)
env = dict(env)
env.update({
'CALIBRE_WORKER_ADDRESS' :
hexlify(cPickle.dumps(listener.address, -1)),
'CALIBRE_WORKER_KEY' : hexlify(auth_key),
'CALIBRE_SIMPLE_WORKER':
'calibre.utils.ipc.simple_worker:%s' % func,
})
w = Worker(env)
w(cwd=cwd, priority=priority)
return listener, w
def test_executables():
from calibre.utils.ipc.launch import Worker
if getattr(sys, 'frozen', False):
w = Worker({})
if not os.path.exists(w.executable):
raise SystemExit('calibre-parallel (%s) does not exist' %
w.executable)
if not os.path.exists(w.gui_executable):
raise SystemExit('calibre-parallel-gui (%s) does not exist' %
w.gui_executable)
if iswindows:
from calibre.devices.usbms.device import eject_exe
if not os.path.exists(eject_exe()):
raise SystemExit('calibre-eject.exe (%s) does not exist' %
eject_exe())
from calibre.ebooks.pdf.pdftohtml import PDFTOHTML
if not os.path.exists(PDFTOHTML):
raise SystemExit('pdftohtml (%s) does not exist' % PDFTOHTML)
fprint('executables OK!')
def start_pipe_worker(command, env=None, priority='normal', **process_args):
import subprocess
w = Worker(env or {})
args = {'stdout': subprocess.PIPE, 'stdin': subprocess.PIPE, 'env': w.env}
args.update(process_args)
if iswindows:
priority = {
'high': subprocess.HIGH_PRIORITY_CLASS,
'normal': subprocess.NORMAL_PRIORITY_CLASS,
'low': subprocess.IDLE_PRIORITY_CLASS
}[priority]
args['creationflags'] = subprocess.CREATE_NO_WINDOW | priority
else:
niceness = {'normal': 0, 'low': 10, 'high': 20}[priority]
args['env']['CALIBRE_WORKER_NICENESS'] = str(niceness)
exe = w.executable
cmd = [exe] if isinstance(exe, string_or_bytes) else exe
p = subprocess.Popen(cmd + ['--pipe-worker', command], **args)
return p
def create_worker(env, priority='normal', cwd=None, func='main'):
address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
if iswindows and address[1] == ':':
address = address[2:]
auth_key = os.urandom(32)
listener = Listener(address=address, authkey=auth_key)
env = dict(env)
env.update({
'CALIBRE_WORKER_ADDRESS':
hexlify(cPickle.dumps(listener.address, -1)),
'CALIBRE_WORKER_KEY':
hexlify(auth_key),
'CALIBRE_SIMPLE_WORKER':
'calibre.utils.ipc.simple_worker:%s' % func,
})
w = Worker(env)
w(cwd=cwd, priority=priority)
return listener, w
def start_pipe_worker(command, env=None, priority='normal', **process_args):
import subprocess
w = Worker(env or {})
args = {
'stdout': subprocess.PIPE,
'stdin': subprocess.PIPE,
'env': w.env,
'close_fds': True
}
args.update(process_args)
pass_fds = None
try:
if iswindows:
priority = {
'high': subprocess.HIGH_PRIORITY_CLASS,
'normal': subprocess.NORMAL_PRIORITY_CLASS,
'low': subprocess.IDLE_PRIORITY_CLASS
}[priority]
args['creationflags'] = subprocess.CREATE_NO_WINDOW | priority
pass_fds = args.pop('pass_fds', None)
if pass_fds:
for fd in pass_fds:
os.set_handle_inheritable(fd, True)
args['startupinfo'] = subprocess.STARTUPINFO(
lpAttributeList={'handle_list': pass_fds})
else:
niceness = {'normal': 0, 'low': 10, 'high': 20}[priority]
args['env']['CALIBRE_WORKER_NICENESS'] = str(niceness)
exe = w.executable
cmd = [exe] if isinstance(exe, string_or_bytes) else exe
p = subprocess.Popen(cmd + ['--pipe-worker', command], **args)
finally:
if iswindows and pass_fds:
for fd in pass_fds:
os.set_handle_inheritable(fd, False)
return p
def fork_job(
mod_name,
func_name,
args=(),
kwargs={},
timeout=300, # seconds
cwd=None,
priority='normal',
env={},
no_output=False,
heartbeat=None,
abort=None,
module_is_source_code=False):
'''
Run a job in a worker process. A job is simply a function that will be
called with the supplied arguments, in the worker process.
The result of the function will be returned.
If an error occurs a WorkerError is raised.
:param mod_name: Module to import in the worker process
:param func_name: Function to call in the worker process from the imported
module
:param args: Positional arguments to pass to the function
:param kwargs: Keyword arguments to pass to the function
:param timeout: The time in seconds to wait for the worker process to
complete. If it takes longer a WorkerError is raised and the process is
killed.
:param cwd: The working directory for the worker process. I recommend
against using this, unless you are sure the path is pure ASCII.
:param priority: The process priority for the worker process
:param env: Extra environment variables to set for the worker process
:param no_output: If True, the stdout and stderr of the worker process are
discarded
:param heartbeat: If not None, it is used to check if the worker has hung,
instead of a simple timeout. It must be a callable that takes no
arguments and returns True or False. The worker will be assumed to have
hung if this function returns False. At that point, the process will be
killed and a WorkerError will be raised.
:param abort: If not None, it must be an Event. As soon as abort.is_set()
returns True, the worker process is killed. No error is raised.
:param module_is_source_code: If True, the ``mod`` is treated as python
source rather than a module name to import. The source is executed as a
module. Useful if you want to use fork_job from within a script to run some
dynamically generated python.
:return: A dictionary with the keys result and stdout_stderr. result is the
return value of the function (it must be picklable). stdout_stderr is the
path to a file that contains the stdout and stderr of the worker process.
If you set no_output=True, then this will not be present.
'''
ans = {'result': None, 'stdout_stderr': None}
address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
if iswindows and address[1] == ':':
address = address[2:]
auth_key = os.urandom(32)
listener = Listener(address=address, authkey=auth_key)
env = dict(env)
env.update({
'CALIBRE_WORKER_ADDRESS':
hexlify(cPickle.dumps(listener.address, -1)),
'CALIBRE_WORKER_KEY':
hexlify(auth_key),
'CALIBRE_SIMPLE_WORKER':
'calibre.utils.ipc.simple_worker:main',
})
w = Worker(env)
w(cwd=cwd, priority=priority)
try:
communicate(ans,
w,
listener,
(mod_name, func_name, args, kwargs, module_is_source_code),
timeout=timeout,
heartbeat=heartbeat,
abort=abort)
finally:
t = Thread(target=w.kill)
t.daemon = True
t.start()
if no_output:
try:
os.remove(w.log_path)
except:
pass
if not no_output:
ans['stdout_stderr'] = w.log_path
return ans