def __call__(self, job):
eintr_retry_call(self.conn.send_bytes, pickle_dumps(job))
if job is not None:
self.job_id = job.id
t = Thread(target=self.recv, name='PoolWorker-' + self.name)
t.daemon = True
t.start()
def __call__(self, job):
eintr_retry_call(self.conn.send_bytes, pickle_dumps(job))
if job is not None:
self.job_id = job.id
t = Thread(target=self.recv, name='PoolWorker-'+self.name)
t.daemon = True
t.start()
def handle_event(self, event):
if isinstance(event, Job):
job = event
if not self.available_workers:
if len(self.busy_workers) >= self.max_workers:
self.pending_jobs.append(job)
return
if self.start_worker() is False:
return False
return self.run_job(job)
elif isinstance(event, WorkerResult):
worker_result = event
self.busy_workers.pop(worker_result.worker, None)
self.available_workers.append(worker_result.worker)
self.tracker.task_done()
if worker_result.is_terminal_failure:
self.terminal_failure = TerminalFailure(
'Worker process crashed while executing job',
worker_result.result.traceback, worker_result.id)
self.terminal_error()
return False
self.results.put(worker_result)
else:
self.common_data = pickle_dumps(event)
if len(self.common_data) > MAX_SIZE:
self.cd_file = PersistentTemporaryFile('pool_common_data')
with self.cd_file as f:
f.write(self.common_data)
self.common_data = pickle_dumps(File(f.name))
for worker in self.available_workers:
try:
worker.set_common_data(self.common_data)
except Exception:
import traceback
self.terminal_failure = TerminalFailure(
'Worker process crashed while sending common data',
traceback.format_exc(), None)
self.terminal_error()
return False
while self.pending_jobs and self.available_workers:
if self.run_job(self.pending_jobs.pop()) is False:
return False
def create_worker(self):
p = start_worker('from {0} import run_main, {1}; run_main({1})'.format(self.__class__.__module__, 'worker_main'))
sys.stdout.flush()
eintr_retry_call(p.stdin.write, self.worker_data)
p.stdin.flush(), p.stdin.close()
conn = eintr_retry_call(self.listener.accept)
w = Worker(p, conn, self.events, self.name)
if self.common_data != pickle_dumps(None):
w.set_common_data(self.common_data)
return w
def create_worker(self):
p = start_worker('from {0} import run_main, {1}; run_main({1})'.format(self.__class__.__module__, 'worker_main'))
sys.stdout.flush()
eintr_retry_call(p.stdin.write, self.worker_data)
p.stdin.flush(), p.stdin.close()
conn = eintr_retry_call(self.listener.accept)
w = Worker(p, conn, self.events, self.name)
if self.common_data != pickle_dumps(None):
w.set_common_data(self.common_data)
return w
def create_worker(self):
a, b = Pipe()
with a:
cmd = 'from {0} import run_main, {1}; run_main({2!r}, {1})'.format(
self.__class__.__module__, 'worker_main', a.fileno())
p = start_worker(cmd, (a.fileno(), ))
sys.stdout.flush()
p.stdin.close()
w = Worker(p, b, self.events, self.name)
if self.common_data != pickle_dumps(None):
w.set_common_data(self.common_data)
return w
def worker_main(conn):
from importlib import import_module
common_data = None
while True:
try:
job = pickle_loads(eintr_retry_call(conn.recv_bytes))
except EOFError:
break
except KeyboardInterrupt:
break
except Exception:
prints('recv() failed in worker, terminating worker',
file=sys.stderr)
import traceback
traceback.print_exc()
return 1
if job is None:
break
if not isinstance(job, Job):
if isinstance(job, File):
with lopen(job.name, 'rb') as f:
common_data = f.read()
common_data = pickle_loads(common_data)
else:
common_data = job
continue
try:
if '\n' in job.module:
import_module('calibre.customize.ui') # Load plugins
from calibre.utils.ipc.simple_worker import compile_code
mod = compile_code(job.module)
func = mod[job.func]
else:
func = getattr(import_module(job.module), job.func)
if common_data is not None:
job.kwargs['common_data'] = common_data
result = func(*job.args, **job.kwargs)
result = Result(result, None, None)
except Exception as err:
import traceback
result = Result(None, as_unicode(err), traceback.format_exc())
try:
eintr_retry_call(conn.send_bytes, pickle_dumps(result))
except EOFError:
break
except Exception:
prints('send() failed in worker, terminating worker',
file=sys.stderr)
import traceback
traceback.print_exc()
return 1
return 0
def handle_event(self, event):
if isinstance(event, Job):
job = event
if not self.available_workers:
if len(self.busy_workers) >= self.max_workers:
self.pending_jobs.append(job)
return
if self.start_worker() is False:
return False
return self.run_job(job)
elif isinstance(event, WorkerResult):
worker_result = event
self.busy_workers.pop(worker_result.worker, None)
self.available_workers.append(worker_result.worker)
self.tracker.task_done()
if worker_result.is_terminal_failure:
self.terminal_failure = TerminalFailure('Worker process crashed while executing job', worker_result.result.traceback, worker_result.id)
self.terminal_error()
return False
self.results.put(worker_result)
else:
self.common_data = pickle_dumps(event)
if len(self.common_data) > MAX_SIZE:
self.cd_file = PersistentTemporaryFile('pool_common_data')
with self.cd_file as f:
f.write(self.common_data)
self.common_data = pickle_dumps(File(f.name))
for worker in self.available_workers:
try:
worker.set_common_data(self.common_data)
except Exception:
import traceback
self.terminal_failure = TerminalFailure('Worker process crashed while sending common data', traceback.format_exc(), None)
self.terminal_error()
return False
while self.pending_jobs and self.available_workers:
if self.run_job(self.pending_jobs.pop()) is False:
return False
def __init__(self, max_workers=None, name=None):
Thread.__init__(self, name=name)
self.max_workers = max_workers or detect_ncpus()
self.available_workers = []
self.busy_workers = {}
self.pending_jobs = []
self.events = Queue()
self.results = Queue()
self.tracker = Queue()
self.terminal_failure = None
self.common_data = pickle_dumps(None)
self.shutting_down = False
self.start()
def worker_main(conn):
from importlib import import_module
common_data = None
while True:
try:
job = pickle_loads(eintr_retry_call(conn.recv_bytes))
except EOFError:
break
except KeyboardInterrupt:
break
except Exception:
prints('recv() failed in worker, terminating worker', file=sys.stderr)
import traceback
traceback.print_exc()
return 1
if job is None:
break
if not isinstance(job, Job):
if isinstance(job, File):
with lopen(job.name, 'rb') as f:
common_data = f.read()
common_data = pickle_loads(common_data)
else:
common_data = job
continue
try:
if '\n' in job.module:
import_module('calibre.customize.ui') # Load plugins
from calibre.utils.ipc.simple_worker import compile_code
mod = compile_code(job.module)
func = mod[job.func]
else:
func = getattr(import_module(job.module), job.func)
if common_data is not None:
job.kwargs['common_data'] = common_data
result = func(*job.args, **job.kwargs)
result = Result(result, None, None)
except Exception as err:
import traceback
result = Result(None, as_unicode(err), traceback.format_exc())
try:
eintr_retry_call(conn.send_bytes, pickle_dumps(result))
except EOFError:
break
except Exception:
prints('send() failed in worker, terminating worker', file=sys.stderr)
import traceback
traceback.print_exc()
return 1
return 0
def __init__(self, max_workers=None, name=None):
Thread.__init__(self, name=name)
self.max_workers = max_workers or detect_ncpus()
self.available_workers = []
self.busy_workers = {}
self.pending_jobs = []
self.events = Queue()
self.results = Queue()
self.tracker = Queue()
self.terminal_failure = None
self.common_data = pickle_dumps(None)
self.worker_data = None
self.shutting_down = False
self.start()
def main():
if iswindows:
if '--multiprocessing-fork' in sys.argv:
# We are using the multiprocessing module on windows to launch a
# worker process
from multiprocessing import freeze_support
freeze_support()
return 0
# Close open file descriptors inherited from parent
# On Unix this is done by the subprocess module
os.closerange(3, 256)
if isosx and 'CALIBRE_WORKER_ADDRESS' not in os.environ and 'CALIBRE_SIMPLE_WORKER' not in os.environ and '--pipe-worker' not in sys.argv:
# On some OS X computers launchd apparently tries to
# launch the last run process from the bundle
# so launch the gui as usual
from calibre.gui2.main import main as gui_main
return gui_main(['calibre'])
csw = os.environ.get('CALIBRE_SIMPLE_WORKER', None)
if csw:
mod, _, func = csw.partition(':')
mod = importlib.import_module(mod)
func = getattr(mod, func)
func()
return
if '--pipe-worker' in sys.argv:
try:
exec (sys.argv[-1])
except Exception:
print('Failed to run pipe worker with command:', sys.argv[-1])
raise
return
address = msgpack_loads(from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS']))
key = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY'])
resultf = from_hex_unicode(os.environ['CALIBRE_WORKER_RESULT'])
with closing(Client(address, authkey=key)) as conn:
name, args, kwargs, desc = eintr_retry_call(conn.recv)
if desc:
prints(desc)
sys.stdout.flush()
func, notification = get_func(name)
notifier = Progress(conn)
if notification:
kwargs[notification] = notifier
notifier.start()
result = func(*args, **kwargs)
if result is not None and os.path.exists(os.path.dirname(resultf)):
with lopen(resultf, 'wb') as f:
f.write(pickle_dumps(result))
notifier.queue.put(None)
try:
sys.stdout.flush()
except EnvironmentError:
pass # Happens sometimes on OS X for GUI processes (EPIPE)
try:
sys.stderr.flush()
except EnvironmentError:
pass # Happens sometimes on OS X for GUI processes (EPIPE)
return 0
def main():
if iswindows:
if '--multiprocessing-fork' in sys.argv:
# We are using the multiprocessing module on windows to launch a
# worker process
from multiprocessing import freeze_support
freeze_support()
return 0
if ismacos and 'CALIBRE_WORKER_FD' not in os.environ and 'CALIBRE_SIMPLE_WORKER' not in os.environ and '--pipe-worker' not in sys.argv:
# On some OS X computers launchd apparently tries to
# launch the last run process from the bundle
# so launch the gui as usual
from calibre.gui2.main import main as gui_main
return gui_main(['calibre'])
niceness = os.environ.pop('CALIBRE_WORKER_NICENESS', None)
if niceness:
try:
os.nice(int(niceness))
except Exception:
pass
csw = os.environ.pop('CALIBRE_SIMPLE_WORKER', None)
if csw:
mod, _, func = csw.partition(':')
mod = importlib.import_module(mod)
func = getattr(mod, func)
func()
return
if '--pipe-worker' in sys.argv:
try:
exec(sys.argv[-1])
except Exception:
print('Failed to run pipe worker with command:', sys.argv[-1])
sys.stdout.flush()
raise
return
fd = int(os.environ['CALIBRE_WORKER_FD'])
resultf = from_hex_unicode(os.environ['CALIBRE_WORKER_RESULT'])
with Connection(fd) as conn:
name, args, kwargs, desc = eintr_retry_call(conn.recv)
if desc:
prints(desc)
sys.stdout.flush()
func, notification = get_func(name)
notifier = Progress(conn)
if notification:
kwargs[notification] = notifier
notifier.start()
result = func(*args, **kwargs)
if result is not None:
os.makedirs(os.path.dirname(resultf), exist_ok=True)
with lopen(resultf, 'wb') as f:
f.write(pickle_dumps(result))
notifier.queue.put(None)
try:
sys.stdout.flush()
except OSError:
pass # Happens sometimes on OS X for GUI processes (EPIPE)
try:
sys.stderr.flush()
except OSError:
pass # Happens sometimes on OS X for GUI processes (EPIPE)
return 0
def main():
if iswindows:
if '--multiprocessing-fork' in sys.argv:
# We are using the multiprocessing module on windows to launch a
# worker process
from multiprocessing import freeze_support
freeze_support()
return 0
# Close open file descriptors inherited from parent
# On Unix this is done by the subprocess module
os.closerange(3, 256)
if isosx and 'CALIBRE_WORKER_ADDRESS' not in os.environ and 'CALIBRE_SIMPLE_WORKER' not in os.environ and '--pipe-worker' not in sys.argv:
# On some OS X computers launchd apparently tries to
# launch the last run process from the bundle
# so launch the gui as usual
from calibre.gui2.main import main as gui_main
return gui_main(['calibre'])
csw = os.environ.get('CALIBRE_SIMPLE_WORKER', None)
if csw:
mod, _, func = csw.partition(':')
mod = importlib.import_module(mod)
func = getattr(mod, func)
func()
return
if '--pipe-worker' in sys.argv:
try:
exec(sys.argv[-1])
except Exception:
print('Failed to run pipe worker with command:', sys.argv[-1])
raise
return
address = msgpack_loads(from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS']))
key = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY'])
resultf = from_hex_unicode(os.environ['CALIBRE_WORKER_RESULT'])
with closing(Client(address, authkey=key)) as conn:
name, args, kwargs, desc = eintr_retry_call(conn.recv)
if desc:
prints(desc)
sys.stdout.flush()
func, notification = get_func(name)
notifier = Progress(conn)
if notification:
kwargs[notification] = notifier
notifier.start()
result = func(*args, **kwargs)
if result is not None and os.path.exists(os.path.dirname(resultf)):
with lopen(resultf, 'wb') as f:
f.write(pickle_dumps(result))
notifier.queue.put(None)
try:
sys.stdout.flush()
except EnvironmentError:
pass # Happens sometimes on OS X for GUI processes (EPIPE)
try:
sys.stderr.flush()
except EnvironmentError:
pass # Happens sometimes on OS X for GUI processes (EPIPE)
return 0
