def single_covers(title, authors, identifiers, caches, tdir): patch_plugins() load_caches(caches) log = GUILog() results = Queue() worker = Thread(target=run_download, args=(log, results, Event()), kwargs=dict(title=title, authors=authors, identifiers=identifiers)) worker.daemon = True worker.start() c = Counter() while worker.is_alive(): try: plugin, width, height, fmt, data = results.get(True, 1) except Empty: continue else: name = plugin.name if plugin.can_get_multiple_covers: name += '{%d}'%c[plugin.name] c[plugin.name] += 1 name = '%s,,%s,,%s,,%s.cover'%(name, width, height, fmt) with open(os.path.join(tdir, name), 'wb') as f: f.write(data) os.mkdir(os.path.join(tdir, name+'.done')) return log.dump()
class DBThread(Thread): CLOSE = '-------close---------' def __init__(self, path, row_factory): Thread.__init__(self) self.setDaemon(True) self.path = path self.unhandled_error = (None, '') self.row_factory = row_factory self.requests = Queue(1) self.results = Queue(1) self.conn = None def connect(self): self.conn = do_connect(self.path, self.row_factory) def run(self): try: self.connect() while True: func, args, kwargs = self.requests.get() if func == self.CLOSE: self.conn.close() break if func == 'dump': try: ok, res = True, tuple(self.conn.iterdump()) except Exception as err: ok, res = False, (err, traceback.format_exc()) elif func == 'create_dynamic_filter': try: f = DynamicFilter(args[0]) self.conn.create_function(args[0], 1, f) ok, res = True, f except Exception as err: ok, res = False, (err, traceback.format_exc()) else: bfunc = getattr(self.conn, func) try: for i in range(3): try: ok, res = True, bfunc(*args, **kwargs) break except OperationalError as err: # Retry if unable to open db file e = str(err) if 'unable to open' not in e or i == 2: if 'unable to open' in e: prints( 'Unable to open database for func', func, reprlib.repr(args), reprlib.repr(kwargs)) raise time.sleep(0.5) except Exception as err: ok, res = False, (err, traceback.format_exc()) self.results.put((ok, res)) except Exception as err: self.unhandled_error = (err, traceback.format_exc())
class Watcher(WatcherBase): def __init__(self, root_dirs, worker, log): WatcherBase.__init__(self, worker, log) self.stream = Stream(self.notify, *(x.encode('utf-8') for x in root_dirs), file_events=True) self.wait_queue = Queue() def wakeup(self): self.wait_queue.put(True) def loop(self): observer = Observer() observer.schedule(self.stream) observer.daemon = True observer.start() try: while True: try: # Cannot use blocking get() as it is not interrupted by # Ctrl-C if self.wait_queue.get(10000) is True: self.force_restart() except Empty: pass finally: observer.unschedule(self.stream) observer.stop() def notify(self, ev): name = ev.name if isinstance(name, bytes): name = name.decode('utf-8') if self.file_is_watched(name): self.handle_modified({name})
class DBThread(Thread): CLOSE = '-------close---------' def __init__(self, path, row_factory): Thread.__init__(self) self.setDaemon(True) self.path = path self.unhandled_error = (None, '') self.row_factory = row_factory self.requests = Queue(1) self.results = Queue(1) self.conn = None def connect(self): self.conn = do_connect(self.path, self.row_factory) def run(self): try: self.connect() while True: func, args, kwargs = self.requests.get() if func == self.CLOSE: self.conn.close() break if func == 'dump': try: ok, res = True, tuple(self.conn.iterdump()) except Exception as err: ok, res = False, (err, traceback.format_exc()) elif func == 'create_dynamic_filter': try: f = DynamicFilter(args[0]) self.conn.create_function(args[0], 1, f) ok, res = True, f except Exception as err: ok, res = False, (err, traceback.format_exc()) else: bfunc = getattr(self.conn, func) try: for i in range(3): try: ok, res = True, bfunc(*args, **kwargs) break except OperationalError as err: # Retry if unable to open db file e = str(err) if 'unable to open' not in e or i == 2: if 'unable to open' in e: prints('Unable to open database for func', func, reprlib.repr(args), reprlib.repr(kwargs)) raise time.sleep(0.5) except Exception as err: ok, res = False, (err, traceback.format_exc()) self.results.put((ok, res)) except Exception as err: self.unhandled_error = (err, traceback.format_exc())
class AnnotationsSaveWorker(Thread): def __init__(self): Thread.__init__(self, name='AnnotSaveWorker') self.daemon = True self.queue = Queue() def shutdown(self): if self.is_alive(): self.queue.put(None) self.join() def run(self): while True: x = self.queue.get() if x is None: return annotations_list = x['annotations_list'] annotations_path_key = x['annotations_path_key'] bld = x['book_library_details'] pathtoebook = x['pathtoebook'] in_book_file = x['in_book_file'] sync_annots_user = x['sync_annots_user'] try: save_annotations(annotations_list, annotations_path_key, bld, pathtoebook, in_book_file, sync_annots_user) except Exception: import traceback traceback.print_exc() def save_annotations(self, current_book_data, in_book_file=True, sync_annots_user=''): alist = tuple( annotations_as_copied_list(current_book_data['annotations_map'])) ebp = current_book_data['pathtoebook'] can_save_in_book_file = ebp.lower().endswith('.epub') self.queue.put({ 'annotations_list': alist, 'annotations_path_key': current_book_data['annotations_path_key'], 'book_library_details': current_book_data['book_library_details'], 'pathtoebook': current_book_data['pathtoebook'], 'in_book_file': in_book_file and can_save_in_book_file, 'sync_annots_user': sync_annots_user, })
class Progress(Thread): def __init__(self, conn): Thread.__init__(self) self.daemon = True self.conn = conn self.queue = Queue() def __call__(self, percent, msg=''): self.queue.put((percent, msg)) def run(self): while True: x = self.queue.get() if x is None: break try: eintr_retry_call(self.conn.send, x) except: break
class Watcher(WatcherBase): def __init__(self, root_dirs, worker, log): WatcherBase.__init__(self, worker, log) self.watchers = [] self.modified_queue = Queue() for d in frozenset(root_dirs): self.watchers.append(TreeWatcher(d, self.modified_queue)) def wakeup(self): self.modified_queue.put(True) def loop(self): for w in self.watchers: w.start() with HandleInterrupt(lambda: self.modified_queue.put(None)): while True: path = self.modified_queue.get() if path is None: break if path is True: self.force_restart() else: self.handle_modified({path})
class Watcher(WatcherBase): def __init__(self, root_dirs, worker, log): WatcherBase.__init__(self, worker, log) self.watchers = [] self.modified_queue = Queue() for d in frozenset(root_dirs): self.watchers.append(TreeWatcher(d, self.modified_queue)) def wakeup(self): self.modified_queue.put(True) def loop(self): for w in self.watchers: w.start() with HandleInterrupt(lambda : self.modified_queue.put(None)): while True: path = self.modified_queue.get() if path is None: break if path is True: self.force_restart() else: self.handle_modified({path})
class Server(Thread): def __init__(self, notify_on_job_done=lambda x: x, pool_size=None, limit=sys.maxsize, enforce_cpu_limit=True): Thread.__init__(self) self.daemon = True self.id = next(server_counter) + 1 if enforce_cpu_limit: limit = min(limit, cpu_count()) self.pool_size = limit if pool_size is None else pool_size self.notify_on_job_done = notify_on_job_done self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue() self.kill_queue = Queue() self.waiting_jobs = [] self.workers = deque() self.launched_worker_counter = count() next(self.launched_worker_counter) self.start() def launch_worker(self, gui=False, redirect_output=None, job_name=None): start = time.monotonic() id = next(self.launched_worker_counter) fd, rfile = tempfile.mkstemp(prefix='ipc_result_%d_%d_' % (self.id, id), dir=base_dir(), suffix='.pickle') os.close(fd) if redirect_output is None: redirect_output = not gui cw = self.do_launch(gui, redirect_output, rfile, job_name=job_name) if isinstance(cw, string_or_bytes): raise CriticalError('Failed to launch worker process:\n' + force_unicode(cw)) if DEBUG: print( f'Worker Launch took: {time.monotonic() - start:.2f} seconds') return cw def do_launch(self, gui, redirect_output, rfile, job_name=None): a, b = Pipe() with a: env = { 'CALIBRE_WORKER_FD': str(a.fileno()), 'CALIBRE_WORKER_RESULT': environ_item(as_hex_unicode(rfile)) } w = Worker(env, gui=gui, job_name=job_name) try: w(pass_fds=(a.fileno(), ), redirect_output=redirect_output) except BaseException: try: w.kill() except: pass b.close() import traceback return traceback.format_exc() return ConnectedWorker(w, b, rfile) def add_job(self, job): job.done2 = self.notify_on_job_done self.add_jobs_queue.put(job) def run_job(self, job, gui=True, redirect_output=False): w = self.launch_worker(gui=gui, redirect_output=redirect_output, job_name=getattr(job, 'name', None)) w.start_job(job) def run(self): while True: try: job = self.add_jobs_queue.get(True, 0.2) if job is None: break self.waiting_jobs.insert(0, job) except Empty: pass # Get notifications from worker process for worker in self.workers: while True: try: n = worker.notifications.get_nowait() worker.job.notifications.put(n) self.changed_jobs_queue.put(worker.job) except Empty: break # Remove finished jobs for worker in [w for w in self.workers if not w.is_alive]: try: worker.close_log_file() except: pass self.workers.remove(worker) job = worker.job if worker.returncode != 0: job.failed = True job.returncode = worker.returncode elif os.path.exists(worker.rfile): try: with lopen(worker.rfile, 'rb') as f: job.result = pickle_loads(f.read()) os.remove(worker.rfile) except: pass job.duration = time.time() - job.start_time self.changed_jobs_queue.put(job) # Start waiting jobs sj = self.suitable_waiting_job() if sj is not None: job = self.waiting_jobs.pop(sj) job.start_time = time.time() if job.kill_on_start: job.duration = 0.0 job.returncode = 1 job.killed = job.failed = True job.result = None else: worker = self.launch_worker() worker.start_job(job) self.workers.append(worker) job.log_path = worker.log_path self.changed_jobs_queue.put(job) while True: try: j = self.kill_queue.get_nowait() self._kill_job(j) except Empty: break def suitable_waiting_job(self): available_workers = self.pool_size - len(self.workers) for worker in self.workers: job = worker.job if job.core_usage == -1: available_workers = 0 elif job.core_usage > 1: available_workers -= job.core_usage - 1 if available_workers < 1: return None for i, job in enumerate(self.waiting_jobs): if job.core_usage == -1: if available_workers >= self.pool_size: return i elif job.core_usage <= available_workers: return i def kill_job(self, job): self.kill_queue.put(job) def killall(self): for worker in self.workers: self.kill_queue.put(worker.job) def _kill_job(self, job): if job.start_time is None: job.kill_on_start = True return for worker in self.workers: if job is worker.job: worker.kill() job.killed = True break def split(self, tasks): ''' Split a list into a list of sub lists, with the number of sub lists being no more than the number of workers this server supports. Each sublist contains 2-tuples of the form (i, x) where x is an element from the original list and i is the index of the element x in the original list. ''' ans, count, pos = [], 0, 0 delta = int(ceil(len(tasks) / float(self.pool_size))) while count < len(tasks): section = [] for t in tasks[pos:pos + delta]: section.append((count, t)) count += 1 ans.append(section) pos += delta return ans def close(self): try: self.add_jobs_queue.put(None) except: pass try: self.listener.close() except: pass time.sleep(0.2) for worker in list(self.workers): try: worker.kill() except: pass def __enter__(self): return self def __exit__(self, *args): self.close()
class JobsManager(object): def __init__(self, opts, log): mj = opts.max_jobs if mj < 1: mj = detect_ncpus() self.log = log self.max_jobs = max(1, mj) self.max_job_time = max(0, opts.max_job_time * 60) self.lock = RLock() self.jobs = {} self.finished_jobs = {} self.events = Queue() self.job_id = count() self.waiting_job_ids = set() self.waiting_jobs = deque() self.max_block = None self.shutting_down = False self.event_loop = None def start_job(self, name, module, func, args=(), kwargs=None, job_done_callback=None, job_data=None): with self.lock: if self.shutting_down: return None if self.event_loop is None: self.event_loop = t = Thread(name='JobsEventLoop', target=self.run) t.daemon = True t.start() job_id = next(self.job_id) self.events.put( StartEvent(job_id, name, module, func, args, kwargs or {}, job_done_callback, job_data)) self.waiting_job_ids.add(job_id) return job_id def job_status(self, job_id): with self.lock: if not self.shutting_down: if job_id in self.finished_jobs: job = self.finished_jobs[job_id] return 'finished', job.result, job.traceback, job.was_aborted if job_id in self.jobs: return 'running', None, None, None if job_id in self.waiting_job_ids: return 'waiting', None, None, None return None, None, None, None def abort_job(self, job_id): job = self.jobs.get(job_id) if job is not None: job.abort_event.set() def wait_for_running_job(self, job_id, timeout=None): job = self.jobs.get(job_id) if job is not None: job.wait_for_end.wait(timeout) if not job.done: return False while job_id not in self.finished_jobs: time.sleep(0.001) return True def shutdown(self, timeout=5.0): with self.lock: self.shutting_down = True for job in itervalues(self.jobs): job.abort_event.set() self.events.put(False) def wait_for_shutdown(self, wait_till): for job in itervalues(self.jobs): delta = wait_till - monotonic() if delta > 0: job.join(delta) if self.event_loop is not None: delta = wait_till - monotonic() if delta > 0: self.event_loop.join(delta) # Internal API {{{ def run(self): while not self.shutting_down: if self.max_block is None: ev = self.events.get() else: try: ev = self.events.get(block=True, timeout=self.max_block) except Empty: ev = None if self.shutting_down: break if ev is None: self.abort_hanging_jobs() elif isinstance(ev, StartEvent): self.waiting_jobs.append(ev) self.start_waiting_jobs() elif isinstance(ev, DoneEvent): self.job_finished(ev.job_id) elif ev is False: break def start_waiting_jobs(self): with self.lock: while self.waiting_jobs and len(self.jobs) < self.max_jobs: ev = self.waiting_jobs.popleft() self.jobs[ev.job_id] = Job(ev, self.events) self.waiting_job_ids.discard(ev.job_id) self.update_max_block() def update_max_block(self): with self.lock: mb = None now = monotonic() for job in itervalues(self.jobs): if not job.done and not job.abort_event.is_set(): delta = self.max_job_time - (now - job.start_time) if delta <= 0: self.max_block = 0 return if mb is None: mb = delta else: mb = min(mb, delta) self.max_block = mb def abort_hanging_jobs(self): now = monotonic() found = False for job in itervalues(self.jobs): if not job.done and not job.abort_event.is_set(): delta = self.max_job_time - (now - job.start_time) if delta <= 0: job.abort_event.set() found = True if found: self.update_max_block() def job_finished(self, job_id): with self.lock: self.finished_jobs[job_id] = job = self.jobs.pop(job_id) if job.callback is not None: try: job.callback(job) except Exception: import traceback self.log.error('Error running callback for job: %s:\n%s' % (job.name, traceback.format_exc())) self.prune_finished_jobs() if job.traceback and not job.was_aborted: logdata = job.read_log() self.log.error('The job: %s failed:\n%s\n%s' % (job.job_name, logdata, job.traceback)) job.remove_log() self.start_waiting_jobs() def prune_finished_jobs(self): with self.lock: remove = [] now = monotonic() for job_id, job in iteritems(self.finished_jobs): if now - job.end_time > 3600: remove.append(job_id) for job_id in remove: del self.finished_jobs[job_id]
class JobsManager(object): def __init__(self, opts, log): mj = opts.max_jobs if mj < 1: mj = detect_ncpus() self.log = log self.max_jobs = max(1, mj) self.max_job_time = max(0, opts.max_job_time * 60) self.lock = RLock() self.jobs = {} self.finished_jobs = {} self.events = Queue() self.job_id = count() self.waiting_job_ids = set() self.waiting_jobs = deque() self.max_block = None self.shutting_down = False self.event_loop = None def start_job(self, name, module, func, args=(), kwargs=None, job_done_callback=None, job_data=None): with self.lock: if self.shutting_down: return None if self.event_loop is None: self.event_loop = t = Thread(name='JobsEventLoop', target=self.run) t.daemon = True t.start() job_id = next(self.job_id) self.events.put(StartEvent(job_id, name, module, func, args, kwargs or {}, job_done_callback, job_data)) self.waiting_job_ids.add(job_id) return job_id def job_status(self, job_id): with self.lock: if not self.shutting_down: if job_id in self.finished_jobs: job = self.finished_jobs[job_id] return 'finished', job.result, job.traceback, job.was_aborted if job_id in self.jobs: return 'running', None, None, None if job_id in self.waiting_job_ids: return 'waiting', None, None, None return None, None, None, None def abort_job(self, job_id): job = self.jobs.get(job_id) if job is not None: job.abort_event.set() def wait_for_running_job(self, job_id, timeout=None): job = self.jobs.get(job_id) if job is not None: job.wait_for_end.wait(timeout) if not job.done: return False while job_id not in self.finished_jobs: time.sleep(0.001) return True def shutdown(self, timeout=5.0): with self.lock: self.shutting_down = True for job in itervalues(self.jobs): job.abort_event.set() self.events.put(False) def wait_for_shutdown(self, wait_till): for job in itervalues(self.jobs): delta = wait_till - monotonic() if delta > 0: job.join(delta) if self.event_loop is not None: delta = wait_till - monotonic() if delta > 0: self.event_loop.join(delta) # Internal API {{{ def run(self): while not self.shutting_down: if self.max_block is None: ev = self.events.get() else: try: ev = self.events.get(block=True, timeout=self.max_block) except Empty: ev = None if self.shutting_down: break if ev is None: self.abort_hanging_jobs() elif isinstance(ev, StartEvent): self.waiting_jobs.append(ev) self.start_waiting_jobs() elif isinstance(ev, DoneEvent): self.job_finished(ev.job_id) elif ev is False: break def start_waiting_jobs(self): with self.lock: while self.waiting_jobs and len(self.jobs) < self.max_jobs: ev = self.waiting_jobs.popleft() self.jobs[ev.job_id] = Job(ev, self.events) self.waiting_job_ids.discard(ev.job_id) self.update_max_block() def update_max_block(self): with self.lock: mb = None now = monotonic() for job in itervalues(self.jobs): if not job.done and not job.abort_event.is_set(): delta = self.max_job_time - (now - job.start_time) if delta <= 0: self.max_block = 0 return if mb is None: mb = delta else: mb = min(mb, delta) self.max_block = mb def abort_hanging_jobs(self): now = monotonic() found = False for job in itervalues(self.jobs): if not job.done and not job.abort_event.is_set(): delta = self.max_job_time - (now - job.start_time) if delta <= 0: job.abort_event.set() found = True if found: self.update_max_block() def job_finished(self, job_id): with self.lock: self.finished_jobs[job_id] = job = self.jobs.pop(job_id) if job.callback is not None: try: job.callback(job) except Exception: import traceback self.log.error('Error running callback for job: %s:\n%s' % (job.name, traceback.format_exc())) self.prune_finished_jobs() if job.traceback and not job.was_aborted: logdata = job.read_log() self.log.error('The job: %s failed:\n%s\n%s' % (job.job_name, logdata, job.traceback)) job.remove_log() self.start_waiting_jobs() def prune_finished_jobs(self): with self.lock: remove = [] now = monotonic() for job_id, job in iteritems(self.finished_jobs): if now - job.end_time > 3600: remove.append(job_id) for job_id in remove: del self.finished_jobs[job_id]
class ParseWorker(Thread): daemon = True SLEEP_TIME = 1 def __init__(self): Thread.__init__(self) self.requests = Queue() self.request_count = 0 self.parse_items = {} self.launch_error = None def run(self): mod, func = 'calibre.gui2.tweak_book.preview', 'parse_html' try: # Connect to the worker and send a dummy job to initialize it self.worker = offload_worker(priority='low') self.worker(mod, func, '<p></p>') except: import traceback traceback.print_exc() self.launch_error = traceback.format_exc() return while True: time.sleep(self.SLEEP_TIME) x = self.requests.get() requests = [x] while True: try: requests.append(self.requests.get_nowait()) except Empty: break if shutdown in requests: self.worker.shutdown() break request = sorted(requests, reverse=True)[0] del requests pi, data = request[1:] try: res = self.worker(mod, func, data) except: import traceback traceback.print_exc() else: pi.parsing_done = True parsed_data = res['result'] if res['tb']: prints("Parser error:") prints(res['tb']) else: pi.parsed_data = parsed_data def add_request(self, name): data = get_data(name) ldata, hdata = len(data), hash(data) pi = self.parse_items.get(name, None) if pi is None: self.parse_items[name] = pi = ParseItem(name) else: if pi.parsing_done and pi.length == ldata and pi.fingerprint == hdata: return pi.parsed_data = None pi.parsing_done = False pi.length, pi.fingerprint = ldata, hdata self.requests.put((self.request_count, pi, data)) self.request_count += 1 def shutdown(self): self.requests.put(shutdown) def get_data(self, name): return getattr(self.parse_items.get(name, None), 'parsed_data', None) def clear(self): self.parse_items.clear() def is_alive(self): return Thread.is_alive(self) or (hasattr(self, 'worker') and self.worker.is_alive())
class SearchPanel(QWidget): # {{{ search_requested = pyqtSignal(object) results_found = pyqtSignal(object) show_search_result = pyqtSignal(object) def __init__(self, parent=None): QWidget.__init__(self, parent) self.last_hidden_text_warning = None self.current_search = None self.l = l = QVBoxLayout(self) l.setContentsMargins(0, 0, 0, 0) self.search_input = si = SearchInput(self) self.searcher = None self.search_tasks = Queue() self.results_found.connect(self.on_result_found, type=Qt.QueuedConnection) si.do_search.connect(self.search_requested) l.addWidget(si) self.results = r = Results(self) r.show_search_result.connect(self.do_show_search_result, type=Qt.QueuedConnection) r.currentRowChanged.connect(self.update_hidden_message) l.addWidget(r, 100) self.spinner = s = BusySpinner(self) s.setVisible(False) l.addWidget(s) self.hidden_message = la = QLabel(_('This text is hidden in the book and cannot be displayed')) la.setStyleSheet('QLabel { margin-left: 1ex }') la.setWordWrap(True) la.setVisible(False) l.addWidget(la) def update_hidden_message(self): self.hidden_message.setVisible(self.results.current_result_is_hidden) def focus_input(self, text=None): self.search_input.focus_input(text) def start_search(self, search_query, current_name): if self.current_search is not None and search_query == self.current_search: self.find_next_requested(search_query.backwards) return if self.searcher is None: self.searcher = Thread(name='Searcher', target=self.run_searches) self.searcher.daemon = True self.searcher.start() self.results.clear() self.hidden_message.setVisible(False) self.spinner.start() self.current_search = search_query self.last_hidden_text_warning = None self.search_tasks.put((search_query, current_name)) def run_searches(self): while True: x = self.search_tasks.get() if x is None: break search_query, current_name = x try: manifest = get_manifest() or {} spine = manifest.get('spine', ()) idx_map = {name: i for i, name in enumerate(spine)} spine_idx = idx_map.get(current_name, -1) except Exception: import traceback traceback.print_exc() spine_idx = -1 if spine_idx < 0: self.results_found.emit(SearchFinished(search_query)) continue for name in spine: counter = Counter() spine_idx = idx_map[name] try: for i, result in enumerate(search_in_name(name, search_query)): before, text, after = result q = (before or '')[-5:] + text + (after or '')[:5] self.results_found.emit(SearchResult(search_query, before, text, after, q, name, spine_idx, counter[q])) counter[q] += 1 except Exception: import traceback traceback.print_exc() self.results_found.emit(SearchFinished(search_query)) def on_result_found(self, result): if self.current_search is None or result.search_query != self.current_search: return if isinstance(result, SearchFinished): self.spinner.stop() if not self.results.count(): self.show_no_results_found() return if self.results.add_result(result) == 1: # first result self.results.setCurrentRow(0) self.results.item_activated() self.update_hidden_message() def visibility_changed(self, visible): if visible: self.focus_input() def clear_searches(self): self.current_search = None self.last_hidden_text_warning = None searchable_text_for_name.cache_clear() self.spinner.stop() self.results.clear() def shutdown(self): self.search_tasks.put(None) self.spinner.stop() self.current_search = None self.last_hidden_text_warning = None self.searcher = None def find_next_requested(self, previous): self.results.find_next(previous) def do_show_search_result(self, sr): self.show_search_result.emit(sr.for_js) def search_result_not_found(self, sr): self.results.search_result_not_found(sr) self.update_hidden_message() def show_no_results_found(self): msg = _('No matches were found for:') warning_dialog(self, _('No matches found'), msg + ' <b>{}</b>'.format(self.current_search.text), show=True)
class GenericDownloadThreadPool(object): ''' add_task must be implemented in a subclass and must GenericDownloadThreadPool.add_task must be called at the end of the function. ''' def __init__(self, thread_type, thread_count=1): self.thread_type = thread_type self.thread_count = thread_count self.tasks = Queue() self.results = Queue() self.threads = [] def set_thread_count(self, thread_count): self.thread_count = thread_count def add_task(self): ''' This must be implemented in a sub class and this function must be called at the end of the add_task function in the sub class. The implementation of this function (in this base class) starts any threads necessary to fill the pool if it is not already full. ''' for i in range(self.thread_count - self.running_threads_count()): t = self.thread_type(self.tasks, self.results) self.threads.append(t) t.start() def abort(self): self.tasks = Queue() self.results = Queue() for t in self.threads: t.abort() self.threads = [] def has_tasks(self): return not self.tasks.empty() def get_result(self): return self.results.get() def get_result_no_wait(self): return self.results.get_nowait() def result_count(self): return len(self.results) def has_results(self): return not self.results.empty() def threads_running(self): return self.running_threads_count() > 0 def running_threads_count(self): count = 0 for t in self.threads: if t.is_alive(): count += 1 return count
class Server(Thread): def __init__(self, notify_on_job_done=lambda x: x, pool_size=None, limit=sys.maxsize, enforce_cpu_limit=True): Thread.__init__(self) self.daemon = True global _counter self.id = _counter+1 _counter += 1 if enforce_cpu_limit: limit = min(limit, cpu_count()) self.pool_size = limit if pool_size is None else pool_size self.notify_on_job_done = notify_on_job_done self.auth_key = os.urandom(32) self.address, self.listener = create_listener(self.auth_key, backlog=4) self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue() self.kill_queue = Queue() self.waiting_jobs = [] self.workers = deque() self.launched_worker_count = 0 self._worker_launch_lock = RLock() self.start() def launch_worker(self, gui=False, redirect_output=None, job_name=None): start = time.time() with self._worker_launch_lock: self.launched_worker_count += 1 id = self.launched_worker_count fd, rfile = tempfile.mkstemp(prefix=u'ipc_result_%d_%d_'%(self.id, id), dir=base_dir(), suffix=u'.pickle') os.close(fd) if redirect_output is None: redirect_output = not gui env = { 'CALIBRE_WORKER_ADDRESS' : environ_item(as_hex_unicode(msgpack_dumps(self.address))), 'CALIBRE_WORKER_KEY' : environ_item(as_hex_unicode(self.auth_key)), 'CALIBRE_WORKER_RESULT' : environ_item(as_hex_unicode(rfile)), } cw = self.do_launch(env, gui, redirect_output, rfile, job_name=job_name) if isinstance(cw, string_or_bytes): raise CriticalError('Failed to launch worker process:\n'+cw) if DEBUG: print('Worker Launch took:', time.time() - start) return cw def do_launch(self, env, gui, redirect_output, rfile, job_name=None): w = Worker(env, gui=gui, job_name=job_name) try: w(redirect_output=redirect_output) conn = eintr_retry_call(self.listener.accept) if conn is None: raise Exception('Failed to launch worker process') except BaseException: try: w.kill() except: pass import traceback return traceback.format_exc() return ConnectedWorker(w, conn, rfile) def add_job(self, job): job.done2 = self.notify_on_job_done self.add_jobs_queue.put(job) def run_job(self, job, gui=True, redirect_output=False): w = self.launch_worker(gui=gui, redirect_output=redirect_output, job_name=getattr(job, 'name', None)) w.start_job(job) def run(self): while True: try: job = self.add_jobs_queue.get(True, 0.2) if job is None: break self.waiting_jobs.insert(0, job) except Empty: pass # Get notifications from worker process for worker in self.workers: while True: try: n = worker.notifications.get_nowait() worker.job.notifications.put(n) self.changed_jobs_queue.put(worker.job) except Empty: break # Remove finished jobs for worker in [w for w in self.workers if not w.is_alive]: try: worker.close_log_file() except: pass self.workers.remove(worker) job = worker.job if worker.returncode != 0: job.failed = True job.returncode = worker.returncode elif os.path.exists(worker.rfile): try: with lopen(worker.rfile, 'rb') as f: job.result = pickle_loads(f.read()) os.remove(worker.rfile) except: pass job.duration = time.time() - job.start_time self.changed_jobs_queue.put(job) # Start waiting jobs sj = self.suitable_waiting_job() if sj is not None: job = self.waiting_jobs.pop(sj) job.start_time = time.time() if job.kill_on_start: job.duration = 0.0 job.returncode = 1 job.killed = job.failed = True job.result = None else: worker = self.launch_worker() worker.start_job(job) self.workers.append(worker) job.log_path = worker.log_path self.changed_jobs_queue.put(job) while True: try: j = self.kill_queue.get_nowait() self._kill_job(j) except Empty: break def suitable_waiting_job(self): available_workers = self.pool_size - len(self.workers) for worker in self.workers: job = worker.job if job.core_usage == -1: available_workers = 0 elif job.core_usage > 1: available_workers -= job.core_usage - 1 if available_workers < 1: return None for i, job in enumerate(self.waiting_jobs): if job.core_usage == -1: if available_workers >= self.pool_size: return i elif job.core_usage <= available_workers: return i def kill_job(self, job): self.kill_queue.put(job) def killall(self): for worker in self.workers: self.kill_queue.put(worker.job) def _kill_job(self, job): if job.start_time is None: job.kill_on_start = True return for worker in self.workers: if job is worker.job: worker.kill() job.killed = True break def split(self, tasks): ''' Split a list into a list of sub lists, with the number of sub lists being no more than the number of workers this server supports. Each sublist contains 2-tuples of the form (i, x) where x is an element from the original list and i is the index of the element x in the original list. ''' ans, count, pos = [], 0, 0 delta = int(ceil(len(tasks)/float(self.pool_size))) while count < len(tasks): section = [] for t in tasks[pos:pos+delta]: section.append((count, t)) count += 1 ans.append(section) pos += delta return ans def close(self): try: self.add_jobs_queue.put(None) except: pass try: self.listener.close() except: pass time.sleep(0.2) for worker in list(self.workers): try: worker.kill() except: pass def __enter__(self): return self def __exit__(self, *args): self.close()
class Pool(Thread): daemon = True def __init__(self, max_workers=None, name=None): Thread.__init__(self, name=name) self.max_workers = max_workers or detect_ncpus() self.available_workers = [] self.busy_workers = {} self.pending_jobs = [] self.events = Queue() self.results = Queue() self.tracker = Queue() self.terminal_failure = None self.common_data = pickle_dumps(None) self.worker_data = None self.shutting_down = False self.start() def set_common_data(self, data=None): ''' Set some data that will be passed to all subsequent jobs without needing to be transmitted every time. You must call this method before queueing any jobs, otherwise the behavior is undefined. You can call it after all jobs are done, then it will be used for the new round of jobs. Can raise the :class:`Failure` exception is data could not be sent to workers.''' if self.failed: raise Failure(self.terminal_failure) self.events.put(data) def __call__(self, job_id, module, func, *args, **kwargs): ''' Schedule a job. The job will be run in a worker process, with the result placed in self.results. If a terminal failure has occurred previously, this method will raise the :class:`Failure` exception. :param job_id: A unique id for the job. The result will have this id. :param module: Either a fully qualified python module name or python source code which will be executed as a module. Source code is detected by the presence of newlines in module. :param func: Name of the function from ``module`` that will be executed. ``args`` and ``kwargs`` will be passed to the function. ''' if self.failed: raise Failure(self.terminal_failure) job = Job(job_id, module, func, args, kwargs) self.tracker.put(None) self.events.put(job) def wait_for_tasks(self, timeout=None): ''' Wait for all queued jobs to be completed, if timeout is not None, will raise a RuntimeError if jobs are not completed in the specified time. Will raise a :class:`Failure` exception if a terminal failure has occurred previously. ''' if self.failed: raise Failure(self.terminal_failure) if timeout is None: self.tracker.join() else: join_with_timeout(self.tracker, timeout) def shutdown(self, wait_time=0.1): ''' Shutdown this pool, terminating all worker process. The pool cannot be used after a shutdown. ''' self.shutting_down = True self.events.put(None) self.shutdown_workers(wait_time=wait_time) def create_worker(self): p = start_worker('from {0} import run_main, {1}; run_main({1})'.format(self.__class__.__module__, 'worker_main')) sys.stdout.flush() eintr_retry_call(p.stdin.write, self.worker_data) p.stdin.flush(), p.stdin.close() conn = eintr_retry_call(self.listener.accept) w = Worker(p, conn, self.events, self.name) if self.common_data != pickle_dumps(None): w.set_common_data(self.common_data) return w def start_worker(self): try: w = self.create_worker() if not self.shutting_down: self.available_workers.append(w) except Exception: import traceback self.terminal_failure = TerminalFailure('Failed to start worker process', traceback.format_exc(), None) self.terminal_error() return False def run(self): from calibre.utils.ipc.server import create_listener self.auth_key = os.urandom(32) self.address, self.listener = create_listener(self.auth_key) self.worker_data = msgpack_dumps((self.address, self.auth_key)) if self.start_worker() is False: return while True: event = self.events.get() if event is None or self.shutting_down: break if self.handle_event(event) is False: break def handle_event(self, event): if isinstance(event, Job): job = event if not self.available_workers: if len(self.busy_workers) >= self.max_workers: self.pending_jobs.append(job) return if self.start_worker() is False: return False return self.run_job(job) elif isinstance(event, WorkerResult): worker_result = event self.busy_workers.pop(worker_result.worker, None) self.available_workers.append(worker_result.worker) self.tracker.task_done() if worker_result.is_terminal_failure: self.terminal_failure = TerminalFailure('Worker process crashed while executing job', worker_result.result.traceback, worker_result.id) self.terminal_error() return False self.results.put(worker_result) else: self.common_data = pickle_dumps(event) if len(self.common_data) > MAX_SIZE: self.cd_file = PersistentTemporaryFile('pool_common_data') with self.cd_file as f: f.write(self.common_data) self.common_data = pickle_dumps(File(f.name)) for worker in self.available_workers: try: worker.set_common_data(self.common_data) except Exception: import traceback self.terminal_failure = TerminalFailure('Worker process crashed while sending common data', traceback.format_exc(), None) self.terminal_error() return False while self.pending_jobs and self.available_workers: if self.run_job(self.pending_jobs.pop()) is False: return False def run_job(self, job): worker = self.available_workers.pop() try: worker(job) except Exception: import traceback self.terminal_failure = TerminalFailure('Worker process crashed while sending job', traceback.format_exc(), job.id) self.terminal_error() return False self.busy_workers[worker] = job @property def failed(self): return self.terminal_failure is not None def terminal_error(self): if self.shutting_down: return for worker, job in iteritems(self.busy_workers): self.results.put(WorkerResult(job.id, Result(None, None, None), True, worker)) self.tracker.task_done() while self.pending_jobs: job = self.pending_jobs.pop() self.results.put(WorkerResult(job.id, Result(None, None, None), True, None)) self.tracker.task_done() self.shutdown() def shutdown_workers(self, wait_time=0.1): self.worker_data = self.common_data = None for worker in self.busy_workers: if worker.process.poll() is None: try: worker.process.terminate() except EnvironmentError: pass # If the process has already been killed workers = [w.process for w in self.available_workers + list(self.busy_workers)] aw = list(self.available_workers) def join(): for w in aw: try: w(None) except Exception: pass for w in workers: try: w.wait() except Exception: pass reaper = Thread(target=join, name='ReapPoolWorkers') reaper.daemon = True reaper.start() reaper.join(wait_time) for w in self.available_workers + list(self.busy_workers): try: w.conn.close() except Exception: pass for w in workers: if w.poll() is None: try: w.kill() except EnvironmentError: pass del self.available_workers[:] self.busy_workers.clear() if hasattr(self, 'cd_file'): try: os.remove(self.cd_file.name) except EnvironmentError: pass
class CompletionWorker(Thread): daemon = True def __init__(self, result_callback=lambda x: x, worker_entry_point='main'): Thread.__init__(self) self.worker_entry_point = worker_entry_point self.start() self.main_queue = Queue() self.result_callback = result_callback self.reap_thread = None self.shutting_down = False self.connected = Event() self.current_completion_request = None self.latest_completion_request_id = None self.request_count = 0 self.lock = RLock() def launch_worker_process(self): from calibre.utils.ipc.server import create_listener from calibre.utils.ipc.pool import start_worker self.worker_process = p = start_worker( 'from {0} import run_main, {1}; run_main({1})'.format( self.__class__.__module__, self.worker_entry_point)) auth_key = os.urandom(32) address, self.listener = create_listener(auth_key) eintr_retry_call(p.stdin.write, msgpack_dumps((address, auth_key))) p.stdin.flush(), p.stdin.close() self.control_conn = eintr_retry_call(self.listener.accept) self.data_conn = eintr_retry_call(self.listener.accept) self.data_thread = t = Thread(name='CWData', target=self.handle_data_requests) t.daemon = True t.start() self.connected.set() def send(self, data, conn=None): conn = conn or self.control_conn try: eintr_retry_call(conn.send, data) except: if not self.shutting_down: raise def recv(self, conn=None): conn = conn or self.control_conn try: return eintr_retry_call(conn.recv) except: if not self.shutting_down: raise def wait_for_connection(self, timeout=None): self.connected.wait(timeout) def handle_data_requests(self): from calibre.gui2.tweak_book.completion.basic import handle_data_request while True: try: req = self.recv(self.data_conn) except EOFError: break except Exception: import traceback traceback.print_exc() break if req is None or self.shutting_down: break result, tb = handle_data_request(req) try: self.send((result, tb), self.data_conn) except EOFError: break except Exception: import traceback traceback.print_exc() break def run(self): self.launch_worker_process() while True: obj = self.main_queue.get() if obj is None: break req_type, req_data = obj try: if req_type is COMPLETION_REQUEST: with self.lock: if self.current_completion_request is not None: ccr, self.current_completion_request = self.current_completion_request, None self.send_completion_request(ccr) elif req_type is CLEAR_REQUEST: self.send(req_data) except EOFError: break except Exception: import traceback traceback.print_exc() def send_completion_request(self, request): self.send(request) result = self.recv() if result.request_id == self.latest_completion_request_id: try: self.result_callback(result) except Exception: import traceback traceback.print_exc() def clear_caches(self, cache_type=None): self.main_queue.put( (CLEAR_REQUEST, Request(None, 'clear_caches', cache_type, None))) def queue_completion(self, request_id, completion_type, completion_data, query=None): with self.lock: self.current_completion_request = Request(request_id, completion_type, completion_data, query) self.latest_completion_request_id = self.current_completion_request.id self.main_queue.put((COMPLETION_REQUEST, None)) def shutdown(self): self.shutting_down = True self.main_queue.put(None) for conn in (getattr(self, 'control_conn', None), getattr(self, 'data_conn', None)): try: conn.close() except Exception: pass p = self.worker_process if p.poll() is None: self.worker_process.terminate() t = self.reap_thread = Thread(target=p.wait) t.daemon = True t.start() def join(self, timeout=0.2): if self.reap_thread is not None: self.reap_thread.join(timeout) if not iswindows and self.worker_process.returncode is None: self.worker_process.kill() return self.worker_process.returncode
class SearchPanel(QWidget): # {{{ search_requested = pyqtSignal(object) results_found = pyqtSignal(object) show_search_result = pyqtSignal(object) def __init__(self, parent=None): QWidget.__init__(self, parent) self.last_hidden_text_warning = None self.current_search = None self.l = l = QVBoxLayout(self) l.setContentsMargins(0, 0, 0, 0) self.search_input = si = SearchInput(self) self.searcher = None self.search_tasks = Queue() self.results_found.connect(self.on_result_found, type=Qt.QueuedConnection) si.do_search.connect(self.search_requested) l.addWidget(si) self.results = r = Results(self) r.show_search_result.connect(self.do_show_search_result, type=Qt.QueuedConnection) l.addWidget(r, 100) self.spinner = s = BusySpinner(self) s.setVisible(False) l.addWidget(s) def focus_input(self): self.search_input.focus_input() def start_search(self, search_query, current_name): if self.current_search is not None and search_query == self.current_search: self.find_next_requested(search_query.backwards) return if self.searcher is None: self.searcher = Thread(name='Searcher', target=self.run_searches) self.searcher.daemon = True self.searcher.start() self.results.clear() self.spinner.start() self.current_search = search_query self.last_hidden_text_warning = None self.search_tasks.put((search_query, current_name)) def run_searches(self): while True: x = self.search_tasks.get() if x is None: break search_query, current_name = x try: manifest = get_manifest() or {} spine = manifest.get('spine', ()) idx_map = {name: i for i, name in enumerate(spine)} spine_idx = idx_map.get(current_name, -1) except Exception: import traceback traceback.print_exc() spine_idx = -1 if spine_idx < 0: self.results_found.emit(SearchFinished(search_query)) continue for name in spine: counter = Counter() spine_idx = idx_map[name] try: for i, result in enumerate( search_in_name(name, search_query)): before, text, after = result self.results_found.emit( SearchResult(search_query, before, text, after, name, spine_idx, counter[text])) counter[text] += 1 except Exception: import traceback traceback.print_exc() self.results_found.emit(SearchFinished(search_query)) def on_result_found(self, result): if self.current_search is None or result.search_query != self.current_search: return if isinstance(result, SearchFinished): self.spinner.stop() if not self.results.count(): self.show_no_results_found() return if self.results.add_result(result) == 1: # first result self.results.setCurrentRow(0) self.results.item_activated() def visibility_changed(self, visible): if visible: self.focus_input() def clear_searches(self): self.current_search = None self.last_hidden_text_warning = None searchable_text_for_name.cache_clear() self.spinner.stop() self.results.clear() def shutdown(self): self.search_tasks.put(None) self.spinner.stop() self.current_search = None self.last_hidden_text_warning = None self.searcher = None def find_next_requested(self, previous): self.results.find_next(previous) def do_show_search_result(self, sr): self.show_search_result.emit(sr.for_js) def search_result_not_found(self, sr): self.results.search_result_not_found(sr) if self.results.count(): now = monotonic() if self.last_hidden_text_warning is None or self.current_search != self.last_hidden_text_warning[ 1] or now - self.last_hidden_text_warning[0] > 5: self.last_hidden_text_warning = now, self.current_search warning_dialog( self, _('Hidden text'), _('Some search results were for hidden or non-reflowable text, they will be removed.' ), show=True) elif self.last_hidden_text_warning is not None: self.last_hidden_text_warning = now, self.last_hidden_text_warning[ 1] if not self.results.count() and not self.spinner.is_running: self.show_no_results_found() def show_no_results_found(self): has_hidden_text = self.last_hidden_text_warning is not None and self.last_hidden_text_warning[ 1] == self.current_search if self.current_search: if has_hidden_text: msg = _('No displayable matches were found for:') else: msg = _('No matches were found for:') warning_dialog(self, _('No matches found'), msg + ' <b>{}</b>'.format(self.current_search.text), show=True)
class CompletionWorker(Thread): daemon = True def __init__(self, result_callback=lambda x:x, worker_entry_point='main'): Thread.__init__(self) self.worker_entry_point = worker_entry_point self.start() self.main_queue = Queue() self.result_callback = result_callback self.reap_thread = None self.shutting_down = False self.connected = Event() self.current_completion_request = None self.latest_completion_request_id = None self.request_count = 0 self.lock = RLock() def launch_worker_process(self): from calibre.utils.ipc.server import create_listener from calibre.utils.ipc.pool import start_worker self.worker_process = p = start_worker( 'from {0} import run_main, {1}; run_main({1})'.format(self.__class__.__module__, self.worker_entry_point)) auth_key = os.urandom(32) address, self.listener = create_listener(auth_key) eintr_retry_call(p.stdin.write, msgpack_dumps((address, auth_key))) p.stdin.flush(), p.stdin.close() self.control_conn = eintr_retry_call(self.listener.accept) self.data_conn = eintr_retry_call(self.listener.accept) self.data_thread = t = Thread(name='CWData', target=self.handle_data_requests) t.daemon = True t.start() self.connected.set() def send(self, data, conn=None): conn = conn or self.control_conn try: eintr_retry_call(conn.send, data) except: if not self.shutting_down: raise def recv(self, conn=None): conn = conn or self.control_conn try: return eintr_retry_call(conn.recv) except: if not self.shutting_down: raise def wait_for_connection(self, timeout=None): self.connected.wait(timeout) def handle_data_requests(self): from calibre.gui2.tweak_book.completion.basic import handle_data_request while True: try: req = self.recv(self.data_conn) except EOFError: break except Exception: import traceback traceback.print_exc() break if req is None or self.shutting_down: break result, tb = handle_data_request(req) try: self.send((result, tb), self.data_conn) except EOFError: break except Exception: import traceback traceback.print_exc() break def run(self): self.launch_worker_process() while True: obj = self.main_queue.get() if obj is None: break req_type, req_data = obj try: if req_type is COMPLETION_REQUEST: with self.lock: if self.current_completion_request is not None: ccr, self.current_completion_request = self.current_completion_request, None self.send_completion_request(ccr) elif req_type is CLEAR_REQUEST: self.send(req_data) except EOFError: break except Exception: import traceback traceback.print_exc() def send_completion_request(self, request): self.send(request) result = self.recv() if result.request_id == self.latest_completion_request_id: try: self.result_callback(result) except Exception: import traceback traceback.print_exc() def clear_caches(self, cache_type=None): self.main_queue.put((CLEAR_REQUEST, Request(None, 'clear_caches', cache_type, None))) def queue_completion(self, request_id, completion_type, completion_data, query=None): with self.lock: self.current_completion_request = Request(request_id, completion_type, completion_data, query) self.latest_completion_request_id = self.current_completion_request.id self.main_queue.put((COMPLETION_REQUEST, None)) def shutdown(self): self.shutting_down = True self.main_queue.put(None) for conn in (getattr(self, 'control_conn', None), getattr(self, 'data_conn', None)): try: conn.close() except Exception: pass p = self.worker_process if p.poll() is None: self.worker_process.terminate() t = self.reap_thread = Thread(target=p.wait) t.daemon = True t.start() def join(self, timeout=0.2): if self.reap_thread is not None: self.reap_thread.join(timeout) if not iswindows and self.worker_process.returncode is None: self.worker_process.kill() return self.worker_process.returncode
class SearchPanel(QWidget): # {{{ search_requested = pyqtSignal(object) results_found = pyqtSignal(object) show_search_result = pyqtSignal(object) count_changed = pyqtSignal(object) hide_search_panel = pyqtSignal() goto_cfi = pyqtSignal(object) def __init__(self, parent=None): QWidget.__init__(self, parent) self.discovery_counter = 0 self.last_hidden_text_warning = None self.current_search = None self.anchor_cfi = None self.l = l = QVBoxLayout(self) l.setContentsMargins(0, 0, 0, 0) self.search_input = si = SearchInput(self) self.searcher = None self.search_tasks = Queue() self.results_found.connect(self.on_result_found, type=Qt.ConnectionType.QueuedConnection) si.do_search.connect(self.search_requested) si.cleared.connect(self.search_cleared) si.go_back.connect(self.go_back) l.addWidget(si) self.results = r = Results(self) r.count_changed.connect(self.count_changed) r.show_search_result.connect(self.do_show_search_result, type=Qt.ConnectionType.QueuedConnection) r.current_result_changed.connect(self.update_hidden_message) l.addWidget(r, 100) self.spinner = s = BusySpinner(self) s.setVisible(False) l.addWidget(s) self.hidden_message = la = QLabel( _('This text is hidden in the book and cannot be displayed')) la.setStyleSheet('QLabel { margin-left: 1ex }') la.setWordWrap(True) la.setVisible(False) l.addWidget(la) def go_back(self): if self.anchor_cfi: self.goto_cfi.emit(self.anchor_cfi) def update_hidden_message(self): self.hidden_message.setVisible(self.results.current_result_is_hidden) def focus_input(self, text=None): self.search_input.focus_input(text) def search_cleared(self): self.results.clear_all_results() self.current_search = None def start_search(self, search_query, current_name): if self.current_search is not None and search_query == self.current_search: self.find_next_requested(search_query.backwards) return if self.searcher is None: self.searcher = Thread(name='Searcher', target=self.run_searches) self.searcher.daemon = True self.searcher.start() self.results.clear_all_results() self.hidden_message.setVisible(False) self.spinner.start() self.current_search = search_query self.last_hidden_text_warning = None self.search_tasks.put((search_query, current_name)) self.discovery_counter += 1 def set_anchor_cfi(self, pos_data): self.anchor_cfi = pos_data['cfi'] def run_searches(self): while True: x = self.search_tasks.get() if x is None: break search_query, current_name = x try: manifest = get_manifest() or {} spine = manifest.get('spine', ()) idx_map = {name: i for i, name in enumerate(spine)} spine_idx = idx_map.get(current_name, -1) except Exception: import traceback traceback.print_exc() spine_idx = -1 if spine_idx < 0: self.results_found.emit(SearchFinished(search_query)) continue num_in_spine = len(spine) result_num = 0 for n in range(num_in_spine): idx = (spine_idx + n) % num_in_spine name = spine[idx] counter = Counter() try: for i, result in enumerate( search_in_name(name, search_query)): before, text, after, offset = result q = (before or '')[-15:] + text + (after or '')[:15] result_num += 1 self.results_found.emit( SearchResult(search_query, before, text, after, q, name, idx, counter[q], offset, result_num)) counter[q] += 1 except Exception: import traceback traceback.print_exc() self.results_found.emit(SearchFinished(search_query)) def on_result_found(self, result): if self.current_search is None or result.search_query != self.current_search: return if isinstance(result, SearchFinished): self.spinner.stop() if self.results.number_of_results: self.results.ensure_current_result_visible() else: self.show_no_results_found() return self.results.add_result(result) obj = result.for_js obj['on_discovery'] = self.discovery_counter self.show_search_result.emit(obj) self.update_hidden_message() def visibility_changed(self, visible): if visible: self.focus_input() def clear_searches(self): self.current_search = None self.last_hidden_text_warning = None searchable_text_for_name.cache_clear() toc_offset_map_for_name.cache_clear() get_toc_data.cache_clear() self.spinner.stop() self.results.clear_all_results() def shutdown(self): self.search_tasks.put(None) self.spinner.stop() self.current_search = None self.last_hidden_text_warning = None self.searcher = None def find_next_requested(self, previous): self.results.find_next(previous) def trigger(self): self.search_input.find_next() def do_show_search_result(self, sr): self.show_search_result.emit(sr.for_js) def search_result_not_found(self, sr): self.results.search_result_not_found(sr) self.update_hidden_message() def search_result_discovered(self, sr): self.results.search_result_discovered(sr) def show_no_results_found(self): msg = _('No matches were found for:') warning_dialog(self, _('No matches found'), msg + f' <b>{self.current_search.text}</b>', show=True) def keyPressEvent(self, ev): if ev.key() == Qt.Key.Key_Escape: self.hide_search_panel.emit() ev.accept() return return QWidget.keyPressEvent(self, ev)
class DeleteService(Thread): ''' Provide a blocking file delete implementation with support for the recycle bin. On windows, deleting files to the recycle bin spins the event loop, which can cause locking errors in the main thread. We get around this by only moving the files/folders to be deleted out of the library in the main thread, they are deleted to recycle bin in a separate worker thread. This has the added advantage that doing a restore from the recycle bin wont cause metadata.db and the file system to get out of sync. Also, deleting becomes much faster, since in the common case, the move is done by a simple os.rename(). The downside is that if the user quits calibre while a long move to recycle bin is happening, the files may not all be deleted.''' daemon = True def __init__(self): Thread.__init__(self) self.requests = Queue() if isosx: plugins['cocoa'][0].enable_cocoa_multithreading() def shutdown(self, timeout=20): self.requests.put(None) self.join(timeout) def create_staging(self, library_path): base_path = os.path.dirname(library_path) base = os.path.basename(library_path) try: ans = tempfile.mkdtemp(prefix=base + ' deleted ', dir=base_path) except OSError: ans = tempfile.mkdtemp(prefix=base + ' deleted ') atexit.register(remove_dir, ans) return ans def remove_dir_if_empty(self, path): try: os.rmdir(path) except OSError as e: if e.errno == errno.ENOTEMPTY or len(os.listdir(path)) > 0: # Some linux systems appear to raise an EPERM instead of an # ENOTEMPTY, see https://bugs.launchpad.net/bugs/1240797 return raise def delete_books(self, paths, library_path): tdir = self.create_staging(library_path) self.queue_paths(tdir, paths, delete_empty_parent=True) def queue_paths(self, tdir, paths, delete_empty_parent=True): try: self._queue_paths(tdir, paths, delete_empty_parent=delete_empty_parent) except: if os.path.exists(tdir): shutil.rmtree(tdir, ignore_errors=True) raise def _queue_paths(self, tdir, paths, delete_empty_parent=True): requests = [] for path in paths: if os.path.exists(path): basename = os.path.basename(path) c = 0 while True: dest = os.path.join(tdir, basename) if not os.path.exists(dest): break c += 1 basename = '%d - %s' % (c, os.path.basename(path)) try: shutil.move(path, dest) except EnvironmentError: if os.path.isdir(path): # shutil.move may have partially copied the directory, # so the subsequent call to move() will fail as the # destination directory already exists raise # Wait a little in case something has locked a file time.sleep(1) shutil.move(path, dest) if delete_empty_parent: remove_dir_if_empty(os.path.dirname(path), ignore_metadata_caches=True) requests.append(dest) if not requests: remove_dir_if_empty(tdir) else: self.requests.put(tdir) def delete_files(self, paths, library_path): tdir = self.create_staging(library_path) self.queue_paths(tdir, paths, delete_empty_parent=False) def run(self): while True: x = self.requests.get() try: if x is None: break try: self.do_delete(x) except: import traceback traceback.print_exc() finally: self.requests.task_done() def wait(self): 'Blocks until all pending deletes have completed' self.requests.join() def do_delete(self, tdir): if os.path.exists(tdir): try: for x in os.listdir(tdir): x = os.path.join(tdir, x) if os.path.isdir(x): delete_tree(x) else: delete_file(x) finally: shutil.rmtree(tdir)
class Repl(Thread): LINE_CONTINUATION_CHARS = r'\:' daemon = True def __init__(self, ps1='>>> ', ps2='... ', show_js=False, libdir=None): Thread.__init__(self, name='RapydScriptREPL') self.to_python = to_python self.JSError = JSError self.enc = getattr(sys.stdin, 'encoding', None) or 'utf-8' try: import readline self.readline = readline except ImportError: pass self.output = ANSIStream(sys.stdout) self.to_repl = Queue() self.from_repl = Queue() self.ps1, self.ps2 = ps1, ps2 self.show_js, self.libdir = show_js, libdir self.prompt = '' self.completions = None self.start() def init_ctx(self): self.prompt = self.ps1 self.ctx = compiler() self.ctx.g.Duktape.write = self.output.write self.ctx.eval( r'''console = { log: function() { Duktape.write(Array.prototype.slice.call(arguments).join(' ') + '\n');}}; console['error'] = console['log'];''') self.ctx.g.repl_options = { 'show_js': self.show_js, 'histfile': False, 'input': True, 'output': True, 'ps1': self.ps1, 'ps2': self.ps2, 'terminal': self.output.isatty, 'enum_global': 'Object.keys(this)', 'lib_path': self.libdir or os.path.dirname( P(COMPILER_PATH )) # TODO: Change this to load pyj files from the src code } def get_from_repl(self): while True: try: return self.from_repl.get(True, 1) except Empty: if not self.is_alive(): raise SystemExit(1) def run(self): self.init_ctx() rl = None def set_prompt(p): self.prompt = p def prompt(lw): self.from_repl.put(to_python(lw)) self.ctx.g.set_prompt = set_prompt self.ctx.g.prompt = prompt self.ctx.eval(''' listeners = {}; rl = { setPrompt:set_prompt, write:Duktape.write, clearLine: function() {}, on: function(ev, cb) { listeners[ev] = cb; return rl; }, prompt: prompt, sync_prompt: true, send_line: function(line) { listeners['line'](line); }, send_interrupt: function() { listeners['SIGINT'](); }, close: function() {listeners['close'](); }, }; repl_options.readline = { createInterface: function(options) { rl.completer = options.completer; return rl; }}; exports.init_repl(repl_options) ''', fname='<init repl>') rl = self.ctx.g.rl completer = to_python(rl.completer) send_interrupt = to_python(rl.send_interrupt) send_line = to_python(rl.send_line) while True: ev, line = self.to_repl.get() try: if ev == 'SIGINT': self.output.write('\n') send_interrupt() elif ev == 'line': send_line(line) else: val = completer(line) val = to_python(val) self.from_repl.put(val[0]) except Exception as e: if isinstance(e, JSError): print(e.stack or error_message(e), file=sys.stderr) else: import traceback traceback.print_exc() for i in range(100): # Do this many times to ensure we dont deadlock self.from_repl.put(None) def __call__(self): if hasattr(self, 'readline'): history = os.path.join(cache_dir(), 'pyj-repl-history.txt') self.readline.parse_and_bind("tab: complete") try: self.readline.read_history_file(history) except EnvironmentError as e: if e.errno != errno.ENOENT: raise atexit.register(partial(self.readline.write_history_file, history)) def completer(text, num): if self.completions is None: self.to_repl.put(('complete', text)) self.completions = list(filter(None, self.get_from_repl())) if not self.completions: return None try: return self.completions[num] except (IndexError, TypeError, AttributeError, KeyError): self.completions = None if hasattr(self, 'readline'): self.readline.set_completer(completer) while True: lw = self.get_from_repl() if lw is None: raise SystemExit(1) q = self.prompt if hasattr(self, 'readline'): self.readline.set_pre_input_hook(lambda: ( self.readline.insert_text(lw), self.readline.redisplay())) else: q += lw try: line = raw_input(q) self.to_repl.put(('line', line)) except EOFError: return except KeyboardInterrupt: self.to_repl.put(('SIGINT', None))
class Repl(Thread): LINE_CONTINUATION_CHARS = r'\:' daemon = True def __init__(self, ps1='>>> ', ps2='... ', show_js=False, libdir=None): Thread.__init__(self, name='RapydScriptREPL') self.to_python = to_python self.JSError = JSError self.enc = getattr(sys.stdin, 'encoding', None) or 'utf-8' try: import readline self.readline = readline except ImportError: pass self.output = ANSIStream(sys.stdout) self.to_repl = Queue() self.from_repl = Queue() self.ps1, self.ps2 = ps1, ps2 self.show_js, self.libdir = show_js, libdir self.prompt = '' self.completions = None self.start() def init_ctx(self): self.prompt = self.ps1 self.ctx = compiler() self.ctx.g.Duktape.write = self.output.write self.ctx.eval(r'''console = { log: function() { Duktape.write(Array.prototype.slice.call(arguments).join(' ') + '\n');}}; console['error'] = console['log'];''') self.ctx.g.repl_options = { 'show_js': self.show_js, 'histfile':False, 'input':True, 'output':True, 'ps1':self.ps1, 'ps2':self.ps2, 'terminal':self.output.isatty, 'enum_global': 'Object.keys(this)', 'lib_path': self.libdir or os.path.dirname(P(COMPILER_PATH)) # TODO: Change this to load pyj files from the src code } def get_from_repl(self): while True: try: return self.from_repl.get(True, 1) except Empty: if not self.is_alive(): raise SystemExit(1) def run(self): self.init_ctx() rl = None def set_prompt(p): self.prompt = p def prompt(lw): self.from_repl.put(to_python(lw)) self.ctx.g.set_prompt = set_prompt self.ctx.g.prompt = prompt self.ctx.eval(''' listeners = {}; rl = { setPrompt:set_prompt, write:Duktape.write, clearLine: function() {}, on: function(ev, cb) { listeners[ev] = cb; return rl; }, prompt: prompt, sync_prompt: true, send_line: function(line) { listeners['line'](line); }, send_interrupt: function() { listeners['SIGINT'](); }, close: function() {listeners['close'](); }, }; repl_options.readline = { createInterface: function(options) { rl.completer = options.completer; return rl; }}; exports.init_repl(repl_options) ''', fname='<init repl>') rl = self.ctx.g.rl completer = to_python(rl.completer) send_interrupt = to_python(rl.send_interrupt) send_line = to_python(rl.send_line) while True: ev, line = self.to_repl.get() try: if ev == 'SIGINT': self.output.write('\n') send_interrupt() elif ev == 'line': send_line(line) else: val = completer(line) val = to_python(val) self.from_repl.put(val[0]) except Exception as e: if isinstance(e, JSError): print(e.stack or error_message(e), file=sys.stderr) else: import traceback traceback.print_exc() for i in range(100): # Do this many times to ensure we dont deadlock self.from_repl.put(None) def __call__(self): if hasattr(self, 'readline'): history = os.path.join(cache_dir(), 'pyj-repl-history.txt') self.readline.parse_and_bind("tab: complete") try: self.readline.read_history_file(history) except EnvironmentError as e: if e.errno != errno.ENOENT: raise atexit.register(partial(self.readline.write_history_file, history)) def completer(text, num): if self.completions is None: self.to_repl.put(('complete', text)) self.completions = list(filter(None, self.get_from_repl())) if not self.completions: return None try: return self.completions[num] except (IndexError, TypeError, AttributeError, KeyError): self.completions = None if hasattr(self, 'readline'): self.readline.set_completer(completer) while True: lw = self.get_from_repl() if lw is None: raise SystemExit(1) q = self.prompt if hasattr(self, 'readline'): self.readline.set_pre_input_hook(lambda:(self.readline.insert_text(lw), self.readline.redisplay())) else: q += lw try: line = raw_input(q) self.to_repl.put(('line', line)) except EOFError: return except KeyboardInterrupt: self.to_repl.put(('SIGINT', None))
class DeleteService(Thread): ''' Provide a blocking file delete implementation with support for the recycle bin. On windows, deleting files to the recycle bin spins the event loop, which can cause locking errors in the main thread. We get around this by only moving the files/folders to be deleted out of the library in the main thread, they are deleted to recycle bin in a separate worker thread. This has the added advantage that doing a restore from the recycle bin wont cause metadata.db and the file system to get out of sync. Also, deleting becomes much faster, since in the common case, the move is done by a simple os.rename(). The downside is that if the user quits calibre while a long move to recycle bin is happening, the files may not all be deleted.''' daemon = True def __init__(self): Thread.__init__(self) self.requests = Queue() def shutdown(self, timeout=20): self.requests.put(None) self.join(timeout) def create_staging(self, library_path): base_path = os.path.dirname(library_path) base = os.path.basename(library_path) try: ans = tempfile.mkdtemp(prefix=base+' deleted ', dir=base_path) except OSError: ans = tempfile.mkdtemp(prefix=base+' deleted ') atexit.register(remove_dir, ans) return ans def remove_dir_if_empty(self, path): try: os.rmdir(path) except OSError as e: if e.errno == errno.ENOTEMPTY or len(os.listdir(path)) > 0: # Some linux systems appear to raise an EPERM instead of an # ENOTEMPTY, see https://bugs.launchpad.net/bugs/1240797 return raise def delete_books(self, paths, library_path): tdir = self.create_staging(library_path) self.queue_paths(tdir, paths, delete_empty_parent=True) def queue_paths(self, tdir, paths, delete_empty_parent=True): try: self._queue_paths(tdir, paths, delete_empty_parent=delete_empty_parent) except: if os.path.exists(tdir): shutil.rmtree(tdir, ignore_errors=True) raise def _queue_paths(self, tdir, paths, delete_empty_parent=True): requests = [] for path in paths: if os.path.exists(path): basename = os.path.basename(path) c = 0 while True: dest = os.path.join(tdir, basename) if not os.path.exists(dest): break c += 1 basename = '%d - %s' % (c, os.path.basename(path)) try: shutil.move(path, dest) except EnvironmentError: if os.path.isdir(path): # shutil.move may have partially copied the directory, # so the subsequent call to move() will fail as the # destination directory already exists raise # Wait a little in case something has locked a file time.sleep(1) shutil.move(path, dest) if delete_empty_parent: remove_dir_if_empty(os.path.dirname(path), ignore_metadata_caches=True) requests.append(dest) if not requests: remove_dir_if_empty(tdir) else: self.requests.put(tdir) def delete_files(self, paths, library_path): tdir = self.create_staging(library_path) self.queue_paths(tdir, paths, delete_empty_parent=False) def run(self): while True: x = self.requests.get() try: if x is None: break try: self.do_delete(x) except: import traceback traceback.print_exc() finally: self.requests.task_done() def wait(self): 'Blocks until all pending deletes have completed' self.requests.join() def do_delete(self, tdir): if os.path.exists(tdir): try: for x in os.listdir(tdir): x = os.path.join(tdir, x) if os.path.isdir(x): delete_tree(x) else: delete_file(x) finally: shutil.rmtree(tdir)
class Pool(Thread): daemon = True def __init__(self, max_workers=None, name=None): Thread.__init__(self, name=name) self.max_workers = max_workers or detect_ncpus() self.available_workers = [] self.busy_workers = {} self.pending_jobs = [] self.events = Queue() self.results = Queue() self.tracker = Queue() self.terminal_failure = None self.common_data = pickle_dumps(None) self.shutting_down = False self.start() def set_common_data(self, data=None): ''' Set some data that will be passed to all subsequent jobs without needing to be transmitted every time. You must call this method before queueing any jobs, otherwise the behavior is undefined. You can call it after all jobs are done, then it will be used for the new round of jobs. Can raise the :class:`Failure` exception is data could not be sent to workers.''' if self.failed: raise Failure(self.terminal_failure) self.events.put(data) def __call__(self, job_id, module, func, *args, **kwargs): ''' Schedule a job. The job will be run in a worker process, with the result placed in self.results. If a terminal failure has occurred previously, this method will raise the :class:`Failure` exception. :param job_id: A unique id for the job. The result will have this id. :param module: Either a fully qualified python module name or python source code which will be executed as a module. Source code is detected by the presence of newlines in module. :param func: Name of the function from ``module`` that will be executed. ``args`` and ``kwargs`` will be passed to the function. ''' if self.failed: raise Failure(self.terminal_failure) job = Job(job_id, module, func, args, kwargs) self.tracker.put(None) self.events.put(job) def wait_for_tasks(self, timeout=None): ''' Wait for all queued jobs to be completed, if timeout is not None, will raise a RuntimeError if jobs are not completed in the specified time. Will raise a :class:`Failure` exception if a terminal failure has occurred previously. ''' if self.failed: raise Failure(self.terminal_failure) if timeout is None: self.tracker.join() else: join_with_timeout(self.tracker, timeout) def shutdown(self, wait_time=0.1): ''' Shutdown this pool, terminating all worker process. The pool cannot be used after a shutdown. ''' self.shutting_down = True self.events.put(None) self.shutdown_workers(wait_time=wait_time) def create_worker(self): a, b = Pipe() with a: cmd = 'from {0} import run_main, {1}; run_main({2!r}, {1})'.format( self.__class__.__module__, 'worker_main', a.fileno()) p = start_worker(cmd, (a.fileno(), )) sys.stdout.flush() p.stdin.close() w = Worker(p, b, self.events, self.name) if self.common_data != pickle_dumps(None): w.set_common_data(self.common_data) return w def start_worker(self): try: w = self.create_worker() if not self.shutting_down: self.available_workers.append(w) except Exception: import traceback self.terminal_failure = TerminalFailure( 'Failed to start worker process', traceback.format_exc(), None) self.terminal_error() return False def run(self): if self.start_worker() is False: return while True: event = self.events.get() if event is None or self.shutting_down: break if self.handle_event(event) is False: break def handle_event(self, event): if isinstance(event, Job): job = event if not self.available_workers: if len(self.busy_workers) >= self.max_workers: self.pending_jobs.append(job) return if self.start_worker() is False: return False return self.run_job(job) elif isinstance(event, WorkerResult): worker_result = event self.busy_workers.pop(worker_result.worker, None) self.available_workers.append(worker_result.worker) self.tracker.task_done() if worker_result.is_terminal_failure: self.terminal_failure = TerminalFailure( 'Worker process crashed while executing job', worker_result.result.traceback, worker_result.id) self.terminal_error() return False self.results.put(worker_result) else: self.common_data = pickle_dumps(event) if len(self.common_data) > MAX_SIZE: self.cd_file = PersistentTemporaryFile('pool_common_data') with self.cd_file as f: f.write(self.common_data) self.common_data = pickle_dumps(File(f.name)) for worker in self.available_workers: try: worker.set_common_data(self.common_data) except Exception: import traceback self.terminal_failure = TerminalFailure( 'Worker process crashed while sending common data', traceback.format_exc(), None) self.terminal_error() return False while self.pending_jobs and self.available_workers: if self.run_job(self.pending_jobs.pop()) is False: return False def run_job(self, job): worker = self.available_workers.pop() try: worker(job) except Exception: import traceback self.terminal_failure = TerminalFailure( 'Worker process crashed while sending job', traceback.format_exc(), job.id) self.terminal_error() return False self.busy_workers[worker] = job @property def failed(self): return self.terminal_failure is not None def terminal_error(self): if self.shutting_down: return for worker, job in iteritems(self.busy_workers): self.results.put( WorkerResult(job.id, Result(None, None, None), True, worker)) self.tracker.task_done() while self.pending_jobs: job = self.pending_jobs.pop() self.results.put( WorkerResult(job.id, Result(None, None, None), True, None)) self.tracker.task_done() self.shutdown() def shutdown_workers(self, wait_time=0.1): self.worker_data = self.common_data = None for worker in self.busy_workers: if worker.process.poll() is None: try: worker.process.terminate() except OSError: pass # If the process has already been killed workers = [ w.process for w in self.available_workers + list(self.busy_workers) ] aw = list(self.available_workers) def join(): for w in aw: try: w(None) except Exception: pass for w in workers: try: w.wait() except Exception: pass reaper = Thread(target=join, name='ReapPoolWorkers') reaper.daemon = True reaper.start() reaper.join(wait_time) for w in self.available_workers + list(self.busy_workers): try: w.conn.close() except Exception: pass for w in workers: if w.poll() is None: try: w.kill() except OSError: pass del self.available_workers[:] self.busy_workers.clear() if hasattr(self, 'cd_file'): try: os.remove(self.cd_file.name) except OSError: pass