def __call__(self, job): eintr_retry_call(self.conn.send_bytes, pickle_dumps(job)) if job is not None: self.job_id = job.id t = Thread(target=self.recv, name='PoolWorker-' + self.name) t.daemon = True t.start()
def __call__(self, job): eintr_retry_call(self.conn.send_bytes, pickle_dumps(job)) if job is not None: self.job_id = job.id t = Thread(target=self.recv, name='PoolWorker-'+self.name) t.daemon = True t.start()
def handle_event(self, event): if isinstance(event, Job): job = event if not self.available_workers: if len(self.busy_workers) >= self.max_workers: self.pending_jobs.append(job) return if self.start_worker() is False: return False return self.run_job(job) elif isinstance(event, WorkerResult): worker_result = event self.busy_workers.pop(worker_result.worker, None) self.available_workers.append(worker_result.worker) self.tracker.task_done() if worker_result.is_terminal_failure: self.terminal_failure = TerminalFailure( 'Worker process crashed while executing job', worker_result.result.traceback, worker_result.id) self.terminal_error() return False self.results.put(worker_result) else: self.common_data = pickle_dumps(event) if len(self.common_data) > MAX_SIZE: self.cd_file = PersistentTemporaryFile('pool_common_data') with self.cd_file as f: f.write(self.common_data) self.common_data = pickle_dumps(File(f.name)) for worker in self.available_workers: try: worker.set_common_data(self.common_data) except Exception: import traceback self.terminal_failure = TerminalFailure( 'Worker process crashed while sending common data', traceback.format_exc(), None) self.terminal_error() return False while self.pending_jobs and self.available_workers: if self.run_job(self.pending_jobs.pop()) is False: return False
def create_worker(self): p = start_worker('from {0} import run_main, {1}; run_main({1})'.format(self.__class__.__module__, 'worker_main')) sys.stdout.flush() eintr_retry_call(p.stdin.write, self.worker_data) p.stdin.flush(), p.stdin.close() conn = eintr_retry_call(self.listener.accept) w = Worker(p, conn, self.events, self.name) if self.common_data != pickle_dumps(None): w.set_common_data(self.common_data) return w
def create_worker(self): a, b = Pipe() with a: cmd = 'from {0} import run_main, {1}; run_main({2!r}, {1})'.format( self.__class__.__module__, 'worker_main', a.fileno()) p = start_worker(cmd, (a.fileno(), )) sys.stdout.flush() p.stdin.close() w = Worker(p, b, self.events, self.name) if self.common_data != pickle_dumps(None): w.set_common_data(self.common_data) return w
def worker_main(conn): from importlib import import_module common_data = None while True: try: job = pickle_loads(eintr_retry_call(conn.recv_bytes)) except EOFError: break except KeyboardInterrupt: break except Exception: prints('recv() failed in worker, terminating worker', file=sys.stderr) import traceback traceback.print_exc() return 1 if job is None: break if not isinstance(job, Job): if isinstance(job, File): with lopen(job.name, 'rb') as f: common_data = f.read() common_data = pickle_loads(common_data) else: common_data = job continue try: if '\n' in job.module: import_module('calibre.customize.ui') # Load plugins from calibre.utils.ipc.simple_worker import compile_code mod = compile_code(job.module) func = mod[job.func] else: func = getattr(import_module(job.module), job.func) if common_data is not None: job.kwargs['common_data'] = common_data result = func(*job.args, **job.kwargs) result = Result(result, None, None) except Exception as err: import traceback result = Result(None, as_unicode(err), traceback.format_exc()) try: eintr_retry_call(conn.send_bytes, pickle_dumps(result)) except EOFError: break except Exception: prints('send() failed in worker, terminating worker', file=sys.stderr) import traceback traceback.print_exc() return 1 return 0
def handle_event(self, event): if isinstance(event, Job): job = event if not self.available_workers: if len(self.busy_workers) >= self.max_workers: self.pending_jobs.append(job) return if self.start_worker() is False: return False return self.run_job(job) elif isinstance(event, WorkerResult): worker_result = event self.busy_workers.pop(worker_result.worker, None) self.available_workers.append(worker_result.worker) self.tracker.task_done() if worker_result.is_terminal_failure: self.terminal_failure = TerminalFailure('Worker process crashed while executing job', worker_result.result.traceback, worker_result.id) self.terminal_error() return False self.results.put(worker_result) else: self.common_data = pickle_dumps(event) if len(self.common_data) > MAX_SIZE: self.cd_file = PersistentTemporaryFile('pool_common_data') with self.cd_file as f: f.write(self.common_data) self.common_data = pickle_dumps(File(f.name)) for worker in self.available_workers: try: worker.set_common_data(self.common_data) except Exception: import traceback self.terminal_failure = TerminalFailure('Worker process crashed while sending common data', traceback.format_exc(), None) self.terminal_error() return False while self.pending_jobs and self.available_workers: if self.run_job(self.pending_jobs.pop()) is False: return False
def __init__(self, max_workers=None, name=None): Thread.__init__(self, name=name) self.max_workers = max_workers or detect_ncpus() self.available_workers = [] self.busy_workers = {} self.pending_jobs = [] self.events = Queue() self.results = Queue() self.tracker = Queue() self.terminal_failure = None self.common_data = pickle_dumps(None) self.shutting_down = False self.start()
def __init__(self, max_workers=None, name=None): Thread.__init__(self, name=name) self.max_workers = max_workers or detect_ncpus() self.available_workers = [] self.busy_workers = {} self.pending_jobs = [] self.events = Queue() self.results = Queue() self.tracker = Queue() self.terminal_failure = None self.common_data = pickle_dumps(None) self.worker_data = None self.shutting_down = False self.start()
def main(): if iswindows: if '--multiprocessing-fork' in sys.argv: # We are using the multiprocessing module on windows to launch a # worker process from multiprocessing import freeze_support freeze_support() return 0 # Close open file descriptors inherited from parent # On Unix this is done by the subprocess module os.closerange(3, 256) if isosx and 'CALIBRE_WORKER_ADDRESS' not in os.environ and 'CALIBRE_SIMPLE_WORKER' not in os.environ and '--pipe-worker' not in sys.argv: # On some OS X computers launchd apparently tries to # launch the last run process from the bundle # so launch the gui as usual from calibre.gui2.main import main as gui_main return gui_main(['calibre']) csw = os.environ.get('CALIBRE_SIMPLE_WORKER', None) if csw: mod, _, func = csw.partition(':') mod = importlib.import_module(mod) func = getattr(mod, func) func() return if '--pipe-worker' in sys.argv: try: exec (sys.argv[-1]) except Exception: print('Failed to run pipe worker with command:', sys.argv[-1]) raise return address = msgpack_loads(from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS'])) key = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY']) resultf = from_hex_unicode(os.environ['CALIBRE_WORKER_RESULT']) with closing(Client(address, authkey=key)) as conn: name, args, kwargs, desc = eintr_retry_call(conn.recv) if desc: prints(desc) sys.stdout.flush() func, notification = get_func(name) notifier = Progress(conn) if notification: kwargs[notification] = notifier notifier.start() result = func(*args, **kwargs) if result is not None and os.path.exists(os.path.dirname(resultf)): with lopen(resultf, 'wb') as f: f.write(pickle_dumps(result)) notifier.queue.put(None) try: sys.stdout.flush() except EnvironmentError: pass # Happens sometimes on OS X for GUI processes (EPIPE) try: sys.stderr.flush() except EnvironmentError: pass # Happens sometimes on OS X for GUI processes (EPIPE) return 0
def main(): if iswindows: if '--multiprocessing-fork' in sys.argv: # We are using the multiprocessing module on windows to launch a # worker process from multiprocessing import freeze_support freeze_support() return 0 if ismacos and 'CALIBRE_WORKER_FD' not in os.environ and 'CALIBRE_SIMPLE_WORKER' not in os.environ and '--pipe-worker' not in sys.argv: # On some OS X computers launchd apparently tries to # launch the last run process from the bundle # so launch the gui as usual from calibre.gui2.main import main as gui_main return gui_main(['calibre']) niceness = os.environ.pop('CALIBRE_WORKER_NICENESS', None) if niceness: try: os.nice(int(niceness)) except Exception: pass csw = os.environ.pop('CALIBRE_SIMPLE_WORKER', None) if csw: mod, _, func = csw.partition(':') mod = importlib.import_module(mod) func = getattr(mod, func) func() return if '--pipe-worker' in sys.argv: try: exec(sys.argv[-1]) except Exception: print('Failed to run pipe worker with command:', sys.argv[-1]) sys.stdout.flush() raise return fd = int(os.environ['CALIBRE_WORKER_FD']) resultf = from_hex_unicode(os.environ['CALIBRE_WORKER_RESULT']) with Connection(fd) as conn: name, args, kwargs, desc = eintr_retry_call(conn.recv) if desc: prints(desc) sys.stdout.flush() func, notification = get_func(name) notifier = Progress(conn) if notification: kwargs[notification] = notifier notifier.start() result = func(*args, **kwargs) if result is not None: os.makedirs(os.path.dirname(resultf), exist_ok=True) with lopen(resultf, 'wb') as f: f.write(pickle_dumps(result)) notifier.queue.put(None) try: sys.stdout.flush() except OSError: pass # Happens sometimes on OS X for GUI processes (EPIPE) try: sys.stderr.flush() except OSError: pass # Happens sometimes on OS X for GUI processes (EPIPE) return 0
def main(): if iswindows: if '--multiprocessing-fork' in sys.argv: # We are using the multiprocessing module on windows to launch a # worker process from multiprocessing import freeze_support freeze_support() return 0 # Close open file descriptors inherited from parent # On Unix this is done by the subprocess module os.closerange(3, 256) if isosx and 'CALIBRE_WORKER_ADDRESS' not in os.environ and 'CALIBRE_SIMPLE_WORKER' not in os.environ and '--pipe-worker' not in sys.argv: # On some OS X computers launchd apparently tries to # launch the last run process from the bundle # so launch the gui as usual from calibre.gui2.main import main as gui_main return gui_main(['calibre']) csw = os.environ.get('CALIBRE_SIMPLE_WORKER', None) if csw: mod, _, func = csw.partition(':') mod = importlib.import_module(mod) func = getattr(mod, func) func() return if '--pipe-worker' in sys.argv: try: exec(sys.argv[-1]) except Exception: print('Failed to run pipe worker with command:', sys.argv[-1]) raise return address = msgpack_loads(from_hex_bytes(os.environ['CALIBRE_WORKER_ADDRESS'])) key = from_hex_bytes(os.environ['CALIBRE_WORKER_KEY']) resultf = from_hex_unicode(os.environ['CALIBRE_WORKER_RESULT']) with closing(Client(address, authkey=key)) as conn: name, args, kwargs, desc = eintr_retry_call(conn.recv) if desc: prints(desc) sys.stdout.flush() func, notification = get_func(name) notifier = Progress(conn) if notification: kwargs[notification] = notifier notifier.start() result = func(*args, **kwargs) if result is not None and os.path.exists(os.path.dirname(resultf)): with lopen(resultf, 'wb') as f: f.write(pickle_dumps(result)) notifier.queue.put(None) try: sys.stdout.flush() except EnvironmentError: pass # Happens sometimes on OS X for GUI processes (EPIPE) try: sys.stderr.flush() except EnvironmentError: pass # Happens sometimes on OS X for GUI processes (EPIPE) return 0