def download_cover(log, title=None, authors=None, identifiers={}, timeout=30): ''' Synchronous cover download. Returns the "best" cover as per user prefs/cover resolution. Returned cover is a tuple: (plugin, width, height, fmt, data) Returns None if no cover is found. ''' rq = Queue() abort = Event() run_download(log, rq, abort, title=title, authors=authors, identifiers=identifiers, timeout=timeout, get_best_cover=True) results = [] while True: try: results.append(rq.get_nowait()) except Empty: break cp = msprefs['cover_priorities'] def keygen(result): plugin, width, height, fmt, data = result return (cp.get(plugin.name, 1), 1/(width*height)) results.sort(key=keygen) return results[0] if results else None
class ParseWorker(Thread): daemon = True SLEEP_TIME = 1 def __init__(self): Thread.__init__(self) self.requests = Queue() self.request_count = 0 self.parse_items = {} self.launch_error = None def run(self): mod, func = 'calibre.gui2.tweak_book.preview', 'parse_html' try: # Connect to the worker and send a dummy job to initialize it self.worker = offload_worker(priority='low') self.worker(mod, func, '<p></p>') except: import traceback traceback.print_exc() self.launch_error = traceback.format_exc() return while True: time.sleep(self.SLEEP_TIME) x = self.requests.get() requests = [x] while True: try: requests.append(self.requests.get_nowait()) except Empty: break if shutdown in requests: self.worker.shutdown() break request = sorted(requests, reverse=True)[0] del requests pi, data = request[1:] try: res = self.worker(mod, func, data) except: import traceback traceback.print_exc() else: pi.parsing_done = True parsed_data = res['result'] if res['tb']: prints("Parser error:") prints(res['tb']) else: pi.parsed_data = parsed_data def add_request(self, name): data = get_data(name) ldata, hdata = len(data), hash(data) pi = self.parse_items.get(name, None) if pi is None: self.parse_items[name] = pi = ParseItem(name) else: if pi.parsing_done and pi.length == ldata and pi.fingerprint == hdata: return pi.parsed_data = None pi.parsing_done = False pi.length, pi.fingerprint = ldata, hdata self.requests.put((self.request_count, pi, data)) self.request_count += 1 def shutdown(self): self.requests.put(shutdown) def get_data(self, name): return getattr(self.parse_items.get(name, None), 'parsed_data', None) def clear(self): self.parse_items.clear() def is_alive(self): return Thread.is_alive(self) or (hasattr(self, 'worker') and self.worker.is_alive())
def test_identify_plugin( name, tests, modify_plugin=lambda plugin: None, # {{{ fail_missing_meta=True): ''' :param name: Plugin name :param tests: List of 2-tuples. Each two tuple is of the form (args, test_funcs). args is a dict of keyword arguments to pass to the identify method. test_funcs are callables that accept a Metadata object and return True iff the object passes the test. ''' plugin = None for x in all_metadata_plugins(): if x.name == name and 'identify' in x.capabilities: plugin = x break modify_plugin(plugin) prints('Testing the identify function of', plugin.name) prints('Using extra headers:', plugin.browser.addheaders) tdir, lf, log, abort = init_test(plugin.name) prints('Log saved to', lf) times = [] for kwargs, test_funcs in tests: log('') log('#' * 80) log('### Running test with:', kwargs) log('#' * 80) prints('Running test with:', kwargs) rq = Queue() args = (log, rq, abort) start_time = time.time() plugin.running_a_test = True try: err = plugin.identify(*args, **kwargs) finally: plugin.running_a_test = False total_time = time.time() - start_time times.append(total_time) if err is not None: prints('identify returned an error for args', args) prints(err) break results = [] while True: try: results.append(rq.get_nowait()) except Empty: break prints('Found', len(results), 'matches:', end=' ') prints('Smaller relevance means better match') results.sort(key=plugin.identify_results_keygen( title=kwargs.get('title', None), authors=kwargs.get('authors', None), identifiers=kwargs.get('identifiers', {}))) for i, mi in enumerate(results): prints('*' * 30, 'Relevance:', i, '*' * 30) if mi.rating: mi.rating *= 2 prints(mi) prints('\nCached cover URL :', plugin.get_cached_cover_url(mi.identifiers)) prints('*' * 75, '\n\n') possibles = [] for mi in results: test_failed = False for tfunc in test_funcs: if not tfunc(mi): test_failed = True break if not test_failed: possibles.append(mi) if not possibles: prints('ERROR: No results that passed all tests were found') prints('Log saved to', lf) log.close() dump_log(lf) raise SystemExit(1) good = [x for x in possibles if plugin.test_fields(x) is None] if not good: prints('Failed to find', plugin.test_fields(possibles[0])) if fail_missing_meta: raise SystemExit(1) if results[0] is not possibles[0]: prints('Most relevant result failed the tests') raise SystemExit(1) if 'cover' in plugin.capabilities: rq = Queue() mi = results[0] plugin.download_cover(log, rq, abort, title=mi.title, authors=mi.authors, identifiers=mi.identifiers) results = [] while True: try: results.append(rq.get_nowait()) except Empty: break if not results and fail_missing_meta: prints('Cover download failed') raise SystemExit(1) elif results: cdata = results[0] cover = os.path.join( tdir, plugin.name.replace(' ', '') + '-%s-cover.jpg' % sanitize_file_name(mi.title.replace(' ', '_'))) with open(cover, 'wb') as f: f.write(cdata[-1]) prints('Cover downloaded to:', cover) if len(cdata[-1]) < 10240: prints('Downloaded cover too small') raise SystemExit(1) prints('Average time per query', sum(times) / len(times)) if os.stat(lf).st_size > 10: prints('There were some errors/warnings, see log', lf)
class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ TagBrowserMixin, CoverFlowMixin, LibraryViewMixin, SearchBoxMixin, SavedSearchBoxMixin, SearchRestrictionMixin, LayoutMixin, UpdateMixin, EbookDownloadMixin ): 'The main GUI' proceed_requested = pyqtSignal(object, object) book_converted = pyqtSignal(object, object) shutting_down = False def __init__(self, opts, parent=None, gui_debug=None): MainWindow.__init__(self, opts, parent=parent, disable_automatic_gc=True) self.setWindowIcon(QApplication.instance().windowIcon()) self.jobs_pointer = Pointer(self) self.proceed_requested.connect(self.do_proceed, type=Qt.QueuedConnection) self.proceed_question = ProceedQuestion(self) self.job_error_dialog = JobError(self) self.keyboard = Manager(self) get_gui.ans = self self.opts = opts self.device_connected = None self.gui_debug = gui_debug self.iactions = OrderedDict() # Actions for action in interface_actions(): if opts.ignore_plugins and action.plugin_path is not None: continue try: ac = self.init_iaction(action) except: # Ignore errors in loading user supplied plugins import traceback traceback.print_exc() if action.plugin_path is None: raise continue ac.plugin_path = action.plugin_path ac.interface_action_base_plugin = action self.add_iaction(ac) self.load_store_plugins() def init_iaction(self, action): ac = action.load_actual_plugin(self) ac.plugin_path = action.plugin_path ac.interface_action_base_plugin = action action.actual_iaction_plugin_loaded = True return ac def add_iaction(self, ac): acmap = self.iactions if ac.name in acmap: if ac.priority >= acmap[ac.name].priority: acmap[ac.name] = ac else: acmap[ac.name] = ac def load_store_plugins(self): from calibre.gui2.store.loader import Stores self.istores = Stores() for store in available_store_plugins(): if self.opts.ignore_plugins and store.plugin_path is not None: continue try: st = self.init_istore(store) self.add_istore(st) except: # Ignore errors in loading user supplied plugins import traceback traceback.print_exc() if store.plugin_path is None: raise continue self.istores.builtins_loaded() def init_istore(self, store): st = store.load_actual_plugin(self) st.plugin_path = store.plugin_path st.base_plugin = store store.actual_istore_plugin_loaded = True return st def add_istore(self, st): stmap = self.istores if st.name in stmap: if st.priority >= stmap[st.name].priority: stmap[st.name] = st else: stmap[st.name] = st def initialize(self, library_path, db, listener, actions, show_gui=True): opts = self.opts self.preferences_action, self.quit_action = actions self.library_path = library_path self.library_broker = GuiLibraryBroker(db) self.content_server = None self.server_change_notification_timer = t = QTimer(self) self.server_changes = Queue() t.setInterval(1000), t.timeout.connect(self.handle_changes_from_server_debounced), t.setSingleShot(True) self._spare_pool = None self.must_restart_before_config = False self.listener = Listener(listener) self.check_messages_timer = QTimer() self.check_messages_timer.timeout.connect(self.another_instance_wants_to_talk) self.check_messages_timer.start(1000) for ac in self.iactions.values(): try: ac.do_genesis() except Exception: # Ignore errors in third party plugins import traceback traceback.print_exc() if getattr(ac, 'plugin_path', None) is None: raise self.donate_action = QAction(QIcon(I('donate.png')), _('&Donate to support calibre'), self) for st in self.istores.values(): st.do_genesis() MainWindowMixin.init_main_window_mixin(self, db) # Jobs Button {{{ self.job_manager = JobManager() self.jobs_dialog = JobsDialog(self, self.job_manager) self.jobs_button = JobsButton(parent=self) self.jobs_button.initialize(self.jobs_dialog, self.job_manager) # }}} LayoutMixin.init_layout_mixin(self) DeviceMixin.init_device_mixin(self) self.progress_indicator = ProgressIndicator(self) self.progress_indicator.pos = (0, 20) self.verbose = opts.verbose self.get_metadata = GetMetadata() self.upload_memory = {} self.metadata_dialogs = [] self.default_thumbnail = None self.tb_wrapper = textwrap.TextWrapper(width=40) self.viewers = collections.deque() self.system_tray_icon = None do_systray = config['systray_icon'] or opts.start_in_tray if do_systray: self.system_tray_icon = factory(app_id='com.calibre-ebook.gui').create_system_tray_icon(parent=self, title='calibre') if self.system_tray_icon is not None: self.system_tray_icon.setIcon(QIcon(I('lt.png', allow_user_override=False))) if not (iswindows or isosx): self.system_tray_icon.setIcon(QIcon.fromTheme('calibre-tray', self.system_tray_icon.icon())) self.system_tray_icon.setToolTip(self.jobs_button.tray_tooltip()) self.system_tray_icon.setVisible(True) self.jobs_button.tray_tooltip_updated.connect(self.system_tray_icon.setToolTip) elif do_systray: prints('Failed to create system tray icon, your desktop environment probably' ' does not support the StatusNotifier spec https://www.freedesktop.org/wiki/Specifications/StatusNotifierItem/') self.system_tray_menu = QMenu(self) self.toggle_to_tray_action = self.system_tray_menu.addAction(QIcon(I('page.png')), '') self.toggle_to_tray_action.triggered.connect(self.system_tray_icon_activated) self.system_tray_menu.addAction(self.donate_action) self.eject_action = self.system_tray_menu.addAction( QIcon(I('eject.png')), _('&Eject connected device')) self.eject_action.setEnabled(False) self.addAction(self.quit_action) self.system_tray_menu.addAction(self.quit_action) self.keyboard.register_shortcut('quit calibre', _('Quit calibre'), default_keys=('Ctrl+Q',), action=self.quit_action) if self.system_tray_icon is not None: self.system_tray_icon.setContextMenu(self.system_tray_menu) self.system_tray_icon.activated.connect(self.system_tray_icon_activated) self.quit_action.triggered[bool].connect(self.quit) self.donate_action.triggered[bool].connect(self.donate) self.minimize_action = QAction(_('Minimize the calibre window'), self) self.addAction(self.minimize_action) self.keyboard.register_shortcut('minimize calibre', self.minimize_action.text(), default_keys=(), action=self.minimize_action) self.minimize_action.triggered.connect(self.showMinimized) self.esc_action = QAction(self) self.addAction(self.esc_action) self.keyboard.register_shortcut('clear current search', _('Clear the current search'), default_keys=('Esc',), action=self.esc_action) self.esc_action.triggered.connect(self.esc) self.shift_esc_action = QAction(self) self.addAction(self.shift_esc_action) self.keyboard.register_shortcut('focus book list', _('Focus the book list'), default_keys=('Shift+Esc',), action=self.shift_esc_action) self.shift_esc_action.triggered.connect(self.shift_esc) self.ctrl_esc_action = QAction(self) self.addAction(self.ctrl_esc_action) self.keyboard.register_shortcut('clear virtual library', _('Clear the virtual library'), default_keys=('Ctrl+Esc',), action=self.ctrl_esc_action) self.ctrl_esc_action.triggered.connect(self.ctrl_esc) self.alt_esc_action = QAction(self) self.addAction(self.alt_esc_action) self.keyboard.register_shortcut('clear additional restriction', _('Clear the additional restriction'), default_keys=('Alt+Esc',), action=self.alt_esc_action) self.alt_esc_action.triggered.connect(self.clear_additional_restriction) # ###################### Start spare job server ######################## QTimer.singleShot(1000, self.create_spare_pool) # ###################### Location Manager ######################## self.location_manager.location_selected.connect(self.location_selected) self.location_manager.unmount_device.connect(self.device_manager.umount_device) self.location_manager.configure_device.connect(self.configure_connected_device) self.location_manager.update_device_metadata.connect(self.update_metadata_on_device) self.eject_action.triggered.connect(self.device_manager.umount_device) # ################### Update notification ################### UpdateMixin.init_update_mixin(self, opts) # ###################### Search boxes ######################## SearchRestrictionMixin.init_search_restriction_mixin(self) SavedSearchBoxMixin.init_saved_seach_box_mixin(self) # ###################### Library view ######################## LibraryViewMixin.init_library_view_mixin(self, db) SearchBoxMixin.init_search_box_mixin(self) # Requires current_db self.library_view.model().count_changed_signal.connect( self.iactions['Choose Library'].count_changed) if not gprefs.get('quick_start_guide_added', False): try: add_quick_start_guide(self.library_view) except: import traceback traceback.print_exc() for view in ('library', 'memory', 'card_a', 'card_b'): v = getattr(self, '%s_view' % view) v.selectionModel().selectionChanged.connect(self.update_status_bar) v.model().count_changed_signal.connect(self.update_status_bar) self.library_view.model().count_changed() self.bars_manager.database_changed(self.library_view.model().db) self.library_view.model().database_changed.connect(self.bars_manager.database_changed, type=Qt.QueuedConnection) # ########################## Tags Browser ############################## TagBrowserMixin.init_tag_browser_mixin(self, db) self.library_view.model().database_changed.connect(self.populate_tb_manage_menu, type=Qt.QueuedConnection) # ######################## Search Restriction ########################## if db.prefs['virtual_lib_on_startup']: self.apply_virtual_library(db.prefs['virtual_lib_on_startup']) self.rebuild_vl_tabs() # ########################## Cover Flow ################################ CoverFlowMixin.init_cover_flow_mixin(self) self._calculated_available_height = min(max_available_height()-15, self.height()) self.resize(self.width(), self._calculated_available_height) self.build_context_menus() for ac in self.iactions.values(): try: ac.gui_layout_complete() except: import traceback traceback.print_exc() if ac.plugin_path is None: raise if config['autolaunch_server']: self.start_content_server() self.read_settings() self.finalize_layout() self.bars_manager.start_animation() self.set_window_title() for ac in self.iactions.values(): try: ac.initialization_complete() except: import traceback traceback.print_exc() if ac.plugin_path is None: raise self.set_current_library_information(current_library_name(), db.library_id, db.field_metadata) register_keyboard_shortcuts() self.keyboard.finalize() if show_gui: # Note this has to come after restoreGeometry() because of # https://bugreports.qt.io/browse/QTBUG-56831 self.show() if self.system_tray_icon is not None and self.system_tray_icon.isVisible() and opts.start_in_tray: self.hide_windows() self.auto_adder = AutoAdder(gprefs['auto_add_path'], self) # Now that the gui is initialized we can restore the quickview state # The same thing will be true for any action-based operation with a # layout button from calibre.gui2.actions.show_quickview import get_quickview_action_plugin qv = get_quickview_action_plugin() if qv: qv.qv_button.restore_state() self.save_layout_state() # Collect cycles now gc.collect() QApplication.instance().shutdown_signal_received.connect(self.quit) if show_gui and self.gui_debug is not None: QTimer.singleShot(10, self.show_gui_debug_msg) self.iactions['Connect Share'].check_smartdevice_menus() QTimer.singleShot(1, self.start_smartdevice) QTimer.singleShot(100, self.update_toggle_to_tray_action) def show_gui_debug_msg(self): info_dialog(self, _('Debug mode'), '<p>' + _('You have started calibre in debug mode. After you ' 'quit calibre, the debug log will be available in ' 'the file: %s<p>The ' 'log will be displayed automatically.')%self.gui_debug, show=True) def esc(self, *args): self.search.clear() def shift_esc(self): self.current_view().setFocus(Qt.OtherFocusReason) def ctrl_esc(self): self.apply_virtual_library() self.current_view().setFocus(Qt.OtherFocusReason) def start_smartdevice(self): message = None if self.device_manager.get_option('smartdevice', 'autostart'): try: message = self.device_manager.start_plugin('smartdevice') except: message = 'start smartdevice unknown exception' prints(message) import traceback traceback.print_exc() if message: if not self.device_manager.is_running('Wireless Devices'): error_dialog(self, _('Problem starting the wireless device'), _('The wireless device driver had problems starting. ' 'It said "%s"')%message, show=True) self.iactions['Connect Share'].set_smartdevice_action_state() def start_content_server(self, check_started=True): from calibre.srv.embedded import Server if not gprefs.get('server3_warning_done', False): gprefs.set('server3_warning_done', True) if os.path.exists(os.path.join(config_dir, 'server.py')): try: os.remove(os.path.join(config_dir, 'server.py')) except EnvironmentError: pass warning_dialog(self, _('Content server changed!'), _( 'calibre 3 comes with a completely re-written content server.' ' As such any custom configuration you have for the content' ' server no longer applies. You should check and refresh your' ' settings in Preferences->Sharing->Sharing over the net'), show=True) self.content_server = Server(self.library_broker, Dispatcher(self.handle_changes_from_server)) self.content_server.state_callback = Dispatcher( self.iactions['Connect Share'].content_server_state_changed) if check_started: self.content_server.start_failure_callback = \ Dispatcher(self.content_server_start_failed) self.content_server.start() def handle_changes_from_server(self, library_path, change_event): if DEBUG: prints('Received server change event: {} for {}'.format(change_event, library_path)) if self.library_broker.is_gui_library(library_path): self.server_changes.put((library_path, change_event)) self.server_change_notification_timer.start() def handle_changes_from_server_debounced(self): if self.shutting_down: return changes = [] while True: try: library_path, change_event = self.server_changes.get_nowait() except Empty: break if self.library_broker.is_gui_library(library_path): changes.append(change_event) if changes: handle_changes(changes, self) def content_server_start_failed(self, msg): self.content_server = None error_dialog(self, _('Failed to start Content server'), _('Could not start the Content server. Error:\n\n%s')%msg, show=True) def resizeEvent(self, ev): MainWindow.resizeEvent(self, ev) self.search.setMaximumWidth(self.width()-150) def create_spare_pool(self, *args): if self._spare_pool is None: num = min(detect_ncpus(), config['worker_limit']//2) self._spare_pool = Pool(max_workers=num, name='GUIPool') def spare_pool(self): ans, self._spare_pool = self._spare_pool, None QTimer.singleShot(1000, self.create_spare_pool) return ans def do_proceed(self, func, payload): if callable(func): func(payload) def no_op(self, *args): pass def system_tray_icon_activated(self, r=False): if r in (QSystemTrayIcon.Trigger, QSystemTrayIcon.MiddleClick, False): if self.isVisible(): if self.isMinimized(): self.showNormal() else: self.hide_windows() else: self.show_windows() if self.isMinimized(): self.showNormal() @property def is_minimized_to_tray(self): return getattr(self, '__systray_minimized', False) def ask_a_yes_no_question(self, title, msg, det_msg='', show_copy_button=False, ans_when_user_unavailable=True, skip_dialog_name=None, skipped_value=True): if self.is_minimized_to_tray: return ans_when_user_unavailable return question_dialog(self, title, msg, det_msg=det_msg, show_copy_button=show_copy_button, skip_dialog_name=skip_dialog_name, skip_dialog_skipped_value=skipped_value) def update_toggle_to_tray_action(self, *args): if hasattr(self, 'toggle_to_tray_action'): self.toggle_to_tray_action.setText( _('Hide main window') if self.isVisible() else _('Show main window')) def hide_windows(self): for window in QApplication.topLevelWidgets(): if isinstance(window, (MainWindow, QDialog)) and \ window.isVisible(): window.hide() setattr(window, '__systray_minimized', True) self.update_toggle_to_tray_action() def show_windows(self, *args): for window in QApplication.topLevelWidgets(): if getattr(window, '__systray_minimized', False): window.show() setattr(window, '__systray_minimized', False) self.update_toggle_to_tray_action() def test_server(self, *args): if self.content_server is not None and \ self.content_server.exception is not None: error_dialog(self, _('Failed to start Content server'), unicode_type(self.content_server.exception)).exec_() @property def current_db(self): return self.library_view.model().db def refresh_all(self): m = self.library_view.model() m.db.data.refresh(clear_caches=False, do_search=False) self.saved_searches_changed(recount=False) m.resort() m.research() self.tags_view.recount() def handle_cli_args(self, args): if isinstance(args, string_or_bytes): args = [args] files = [os.path.abspath(p) for p in args if not os.path.isdir(p) and os.access(p, os.R_OK)] if files: self.iactions['Add Books'].add_filesystem_book(files) def another_instance_wants_to_talk(self): try: msg = self.listener.queue.get_nowait() except Empty: return if isinstance(msg, bytes): msg = msg.decode('utf-8', 'replace') if msg.startswith('launched:'): import json try: argv = json.loads(msg[len('launched:'):]) except ValueError: prints('Failed to decode message from other instance: %r' % msg) if DEBUG: error_dialog(self, 'Invalid message', 'Received an invalid message from other calibre instance.' ' Do you have multiple versions of calibre installed?', det_msg='Invalid msg: %r' % msg, show=True) argv = () if isinstance(argv, (list, tuple)) and len(argv) > 1: self.handle_cli_args(argv[1:]) self.setWindowState(self.windowState() & ~Qt.WindowMinimized|Qt.WindowActive) self.show_windows() self.raise_() self.activateWindow() elif msg.startswith('refreshdb:'): m = self.library_view.model() m.db.new_api.reload_from_db() self.refresh_all() elif msg.startswith('shutdown:'): self.quit(confirm_quit=False) elif msg.startswith('bookedited:'): parts = msg.split(':')[1:] try: book_id, fmt, library_id = parts[:3] book_id = int(book_id) m = self.library_view.model() db = m.db.new_api if m.db.library_id == library_id and db.has_id(book_id): db.format_metadata(book_id, fmt, allow_cache=False, update_db=True) db.update_last_modified((book_id,)) m.refresh_ids((book_id,)) except Exception: import traceback traceback.print_exc() elif msg.startswith('web-store:'): import json try: data = json.loads(msg[len('web-store:'):]) except ValueError: prints('Failed to decode message from other instance: %r' % msg) path = data['path'] if data['tags']: before = self.current_db.new_api.all_book_ids() self.iactions['Add Books'].add_filesystem_book([path], allow_device=False) if data['tags']: db = self.current_db.new_api after = self.current_db.new_api.all_book_ids() for book_id in after - before: tags = list(db.field_for('tags', book_id)) tags += list(data['tags']) self.current_db.new_api.set_field('tags', {book_id: tags}) else: prints('Ignoring unknown message from other instance: %r' % msg[:20]) def current_view(self): '''Convenience method that returns the currently visible view ''' idx = self.stack.currentIndex() if idx == 0: return self.library_view if idx == 1: return self.memory_view if idx == 2: return self.card_a_view if idx == 3: return self.card_b_view def booklists(self): return self.memory_view.model().db, self.card_a_view.model().db, self.card_b_view.model().db def library_moved(self, newloc, copy_structure=False, allow_rebuild=False): if newloc is None: return with self.library_broker: default_prefs = None try: olddb = self.library_view.model().db if copy_structure: default_prefs = olddb.prefs except: olddb = None if copy_structure and olddb is not None and default_prefs is not None: default_prefs['field_metadata'] = olddb.new_api.field_metadata.all_metadata() db = self.library_broker.prepare_for_gui_library_change(newloc) if db is None: try: db = LibraryDatabase(newloc, default_prefs=default_prefs) except apsw.Error: if not allow_rebuild: raise import traceback repair = question_dialog(self, _('Corrupted database'), _('The library database at %s appears to be corrupted. Do ' 'you want calibre to try and rebuild it automatically? ' 'The rebuild may not be completely successful.') % force_unicode(newloc, filesystem_encoding), det_msg=traceback.format_exc() ) if repair: from calibre.gui2.dialogs.restore_library import repair_library_at if repair_library_at(newloc, parent=self): db = LibraryDatabase(newloc, default_prefs=default_prefs) else: return else: return self.library_path = newloc prefs['library_path'] = self.library_path self.book_on_device(None, reset=True) db.set_book_on_device_func(self.book_on_device) self.library_view.set_database(db) self.tags_view.set_database(db, self.alter_tb) self.library_view.model().set_book_on_device_func(self.book_on_device) self.status_bar.clear_message() self.search.clear() self.saved_search.clear() self.book_details.reset_info() # self.library_view.model().count_changed() db = self.library_view.model().db self.iactions['Choose Library'].count_changed(db.count()) self.set_window_title() self.apply_named_search_restriction('') # reset restriction to null self.saved_searches_changed(recount=False) # reload the search restrictions combo box if db.prefs['virtual_lib_on_startup']: self.apply_virtual_library(db.prefs['virtual_lib_on_startup']) self.rebuild_vl_tabs() for action in self.iactions.values(): action.library_changed(db) self.library_broker.gui_library_changed(db, olddb) if self.device_connected: self.set_books_in_library(self.booklists(), reset=True) self.refresh_ondevice() self.memory_view.reset() self.card_a_view.reset() self.card_b_view.reset() self.set_current_library_information(current_library_name(), db.library_id, db.field_metadata) self.library_view.set_current_row(0) # Run a garbage collection now so that it does not freeze the # interface later gc.collect() def set_window_title(self): db = self.current_db restrictions = [x for x in (db.data.get_base_restriction_name(), db.data.get_search_restriction_name()) if x] restrictions = ' :: '.join(restrictions) font = QFont() if restrictions: restrictions = ' :: ' + restrictions font.setBold(True) font.setItalic(True) self.virtual_library.setFont(font) title = '{0} - || {1}{2} ||'.format( __appname__, self.iactions['Choose Library'].library_name(), restrictions) self.setWindowTitle(title) def location_selected(self, location): ''' Called when a location icon is clicked (e.g. Library) ''' page = 0 if location == 'library' else 1 if location == 'main' else 2 if location == 'carda' else 3 self.stack.setCurrentIndex(page) self.book_details.reset_info() for x in ('tb', 'cb'): splitter = getattr(self, x+'_splitter') splitter.button.setEnabled(location == 'library') for action in self.iactions.values(): action.location_selected(location) if location == 'library': self.virtual_library_menu.setEnabled(True) self.highlight_only_button.setEnabled(True) self.vl_tabs.setEnabled(True) else: self.virtual_library_menu.setEnabled(False) self.highlight_only_button.setEnabled(False) self.vl_tabs.setEnabled(False) # Reset the view in case something changed while it was invisible self.current_view().reset() self.set_number_of_books_shown() self.update_status_bar() def job_exception(self, job, dialog_title=_('Conversion error'), retry_func=None): if not hasattr(self, '_modeless_dialogs'): self._modeless_dialogs = [] minz = self.is_minimized_to_tray if self.isVisible(): for x in list(self._modeless_dialogs): if not x.isVisible(): self._modeless_dialogs.remove(x) try: if 'calibre.ebooks.DRMError' in job.details: if not minz: from calibre.gui2.dialogs.drm_error import DRMErrorMessage d = DRMErrorMessage(self, _('Cannot convert') + ' ' + job.description.split(':')[-1].partition('(')[-1][:-1]) d.setModal(False) d.show() self._modeless_dialogs.append(d) return if 'calibre.ebooks.oeb.transforms.split.SplitError' in job.details: title = job.description.split(':')[-1].partition('(')[-1][:-1] msg = _('<p><b>Failed to convert: %s')%title msg += '<p>'+_(''' Many older e-book reader devices are incapable of displaying EPUB files that have internal components over a certain size. Therefore, when converting to EPUB, calibre automatically tries to split up the EPUB into smaller sized pieces. For some files that are large undifferentiated blocks of text, this splitting fails. <p>You can <b>work around the problem</b> by either increasing the maximum split size under <i>EPUB output</i> in the conversion dialog, or by turning on Heuristic Processing, also in the conversion dialog. Note that if you make the maximum split size too large, your e-book reader may have trouble with the EPUB. ''') if not minz: d = error_dialog(self, _('Conversion Failed'), msg, det_msg=job.details) d.setModal(False) d.show() self._modeless_dialogs.append(d) return if 'calibre.ebooks.mobi.reader.mobi6.KFXError:' in job.details: if not minz: title = job.description.split(':')[-1].partition('(')[-1][:-1] msg = _('<p><b>Failed to convert: %s') % title idx = job.details.index('calibre.ebooks.mobi.reader.mobi6.KFXError:') msg += '<p>' + re.sub(r'(https:\S+)', r'<a href="\1">{}</a>'.format(_('here')), job.details[idx:].partition(':')[2].strip()) d = error_dialog(self, _('Conversion failed'), msg, det_msg=job.details) d.setModal(False) d.show() self._modeless_dialogs.append(d) return if 'calibre.web.feeds.input.RecipeDisabled' in job.details: if not minz: msg = job.details msg = msg[msg.find('calibre.web.feeds.input.RecipeDisabled:'):] msg = msg.partition(':')[-1] d = error_dialog(self, _('Recipe Disabled'), '<p>%s</p>'%msg) d.setModal(False) d.show() self._modeless_dialogs.append(d) return if 'calibre.ebooks.conversion.ConversionUserFeedBack:' in job.details: if not minz: import json payload = job.details.rpartition( 'calibre.ebooks.conversion.ConversionUserFeedBack:')[-1] payload = json.loads('{' + payload.partition('{')[-1]) d = {'info':info_dialog, 'warn':warning_dialog, 'error':error_dialog}.get(payload['level'], error_dialog) d = d(self, payload['title'], '<p>%s</p>'%payload['msg'], det_msg=payload['det_msg']) d.setModal(False) d.show() self._modeless_dialogs.append(d) return except: pass if job.killed: return try: prints(job.details, file=sys.stderr) except: pass if not minz: self.job_error_dialog.show_error(dialog_title, _('<b>Failed</b>')+': '+unicode_type(job.description), det_msg=job.details, retry_func=retry_func) def read_settings(self): geometry = config['main_window_geometry'] if geometry is not None: self.restoreGeometry(geometry) self.read_layout_settings() def write_settings(self): with gprefs: # Only write to gprefs once config.set('main_window_geometry', self.saveGeometry()) dynamic.set('sort_history', self.library_view.model().sort_history) self.save_layout_state() self.stack.tb_widget.save_state() def quit(self, checked=True, restart=False, debug_on_restart=False, confirm_quit=True): if self.shutting_down: return if confirm_quit and not self.confirm_quit(): return try: self.shutdown() except: pass self.restart_after_quit = restart self.debug_on_restart = debug_on_restart if self.system_tray_icon is not None and self.restart_after_quit: # Needed on windows to prevent multiple systray icons self.system_tray_icon.setVisible(False) QApplication.instance().quit() def donate(self, *args): from calibre.utils.localization import localize_website_link open_url(QUrl(localize_website_link('https://calibre-ebook.com/donate'))) def confirm_quit(self): if self.job_manager.has_jobs(): msg = _('There are active jobs. Are you sure you want to quit?') if self.job_manager.has_device_jobs(): msg = '<p>'+__appname__ + \ _(''' is communicating with the device!<br> Quitting may cause corruption on the device.<br> Are you sure you want to quit?''')+'</p>' if not question_dialog(self, _('Active jobs'), msg): return False if self.proceed_question.questions: msg = _('There are library updates waiting. Are you sure you want to quit?') if not question_dialog(self, _('Library updates waiting'), msg): return False from calibre.db.delete_service import has_jobs if has_jobs(): msg = _('Some deleted books are still being moved to the Recycle ' 'Bin, if you quit now, they will be left behind. Are you ' 'sure you want to quit?') if not question_dialog(self, _('Active jobs'), msg): return False return True def shutdown(self, write_settings=True): self.shutting_down = True self.show_shutdown_message() self.server_change_notification_timer.stop() from calibre.customize.ui import has_library_closed_plugins if has_library_closed_plugins(): self.show_shutdown_message( _('Running database shutdown plugins. This could take a few seconds...')) self.grid_view.shutdown() db = None try: db = self.library_view.model().db cf = db.clean except: pass else: cf() # Save the current field_metadata for applications like calibre2opds # Goes here, because if cf is valid, db is valid. db.new_api.set_pref('field_metadata', db.field_metadata.all_metadata()) db.commit_dirty_cache() db.prefs.write_serialized(prefs['library_path']) for action in self.iactions.values(): if not action.shutting_down(): return if write_settings: self.write_settings() self.check_messages_timer.stop() if getattr(self, 'update_checker', None): self.update_checker.shutdown() self.listener.close() self.job_manager.server.close() self.job_manager.threaded_server.close() self.device_manager.keep_going = False self.auto_adder.stop() # Do not report any errors that happen after the shutdown # We cannot restore the original excepthook as that causes PyQt to # call abort() on unhandled exceptions import traceback def eh(t, v, tb): try: traceback.print_exception(t, v, tb, file=sys.stderr) except: pass sys.excepthook = eh mb = self.library_view.model().metadata_backup if mb is not None: mb.stop() self.library_view.model().close() try: try: if self.content_server is not None: # If the Content server has any sockets being closed then # this can take quite a long time (minutes). Tell the user that it is # happening. self.show_shutdown_message( _('Shutting down the Content server. This could take a while...')) s = self.content_server self.content_server = None s.exit() except: pass except KeyboardInterrupt: pass self.hide_windows() if self._spare_pool is not None: self._spare_pool.shutdown() from calibre.db.delete_service import shutdown shutdown() time.sleep(2) self.istores.join() return True def run_wizard(self, *args): if self.confirm_quit(): self.run_wizard_b4_shutdown = True self.restart_after_quit = True try: self.shutdown(write_settings=False) except: pass QApplication.instance().quit() def closeEvent(self, e): if self.shutting_down: return self.write_settings() if self.system_tray_icon is not None and self.system_tray_icon.isVisible(): if not dynamic['systray_msg'] and not isosx: info_dialog(self, 'calibre', 'calibre '+ _('will keep running in the system tray. To close it, ' 'choose <b>Quit</b> in the context menu of the ' 'system tray.'), show_copy_button=False).exec_() dynamic['systray_msg'] = True self.hide_windows() e.ignore() else: if self.confirm_quit(): try: self.shutdown(write_settings=False) except: import traceback traceback.print_exc() e.accept() else: e.ignore()
class BaseJob: WAITING = 0 RUNNING = 1 FINISHED = 2 def __init__(self, description, done=lambda x: x): self.id = next(job_counter) self.description = description self.done = done self.done2 = None self.killed = False self.failed = False self.kill_on_start = False self.start_time = None self.result = None self.duration = None self.log_path = None self.notifications = Queue() self._run_state = self.WAITING self.percent = 0 self._message = None self._status_text = _('Waiting...') self._done_called = False self.core_usage = 1 self.timed_out = False def update(self, consume_notifications=True): if self.duration is not None: self._run_state = self.FINISHED self.percent = 100 if self.killed: if self.timed_out: self._status_text = _('Aborted, taking too long') else: self._status_text = _('Stopped') else: self._status_text = _('Error') if self.failed else _( 'Finished') if DEBUG: try: prints('Job:', self.id, self.description, 'finished') prints('\t'.join(self.details.splitlines(True))) except: pass if not self._done_called: self._done_called = True try: self.done(self) except: pass try: if callable(self.done2): self.done2(self) except: pass elif self.start_time is not None: self._run_state = self.RUNNING self._status_text = _('Working...') if consume_notifications: return self.consume_notifications() return False def consume_notifications(self): got_notification = False while self.notifications is not None: try: self.percent, self._message = self.notifications.get_nowait() self.percent *= 100. got_notification = True except Empty: break return got_notification @property def status_text(self): if self._run_state == self.FINISHED or not self._message: return self._status_text return self._message @property def run_state(self): return self._run_state @property def running_time(self): if self.duration is not None: return self.duration if self.start_time is not None: return time.time() - self.start_time return None @property def is_finished(self): return self._run_state == self.FINISHED @property def is_started(self): return self._run_state != self.WAITING @property def is_running(self): return self.is_started and not self.is_finished def __hash__(self): return id(self) def __eq__(self, other): return self is other def __ne__(self, other): return self is not other def __lt__(self, other): return self.compare_to_other(other) < 0 def __le__(self, other): return self.compare_to_other(other) <= 0 def __gt__(self, other): return self.compare_to_other(other) > 0 def __ge__(self, other): return self.compare_to_other(other) >= 0 def compare_to_other(self, other): if self.is_finished != other.is_finished: return 1 if self.is_finished else -1 if self.start_time is None: if other.start_time is None: # Both waiting return cmp(other.id, self.id) return 1 if other.start_time is None: return -1 # Both running return cmp((other.start_time, id(other)), (self.start_time, id(self))) @property def log_file(self): if self.log_path: return open(self.log_path, 'rb') return io.BytesIO( _('No details available.').encode('utf-8', 'replace')) @property def details(self): return self.log_file.read().decode('utf-8', 'replace')
def run_download(log, results, abort, title=None, authors=None, identifiers={}, timeout=30, get_best_cover=False): ''' Run the cover download, putting results into the queue :param:`results`. Each result is a tuple of the form: (plugin, width, height, fmt, bytes) ''' if title == _('Unknown'): title = None if authors == [_('Unknown')]: authors = None plugins = [p for p in metadata_plugins(['cover']) if p.is_configured()] rq = Queue() workers = [ Worker(p, abort, title, authors, identifiers, timeout, rq, get_best_cover=get_best_cover) for p in plugins ] for w in workers: w.start() first_result_at = None wait_time = msprefs['wait_after_first_cover_result'] found_results = {} start_time = time.time( ) # Use a global timeout to workaround misbehaving plugins that hang while time.time() - start_time < 301: time.sleep(0.1) try: x = rq.get_nowait() result = process_result(log, x) if result is not None: results.put(result) found_results[result[0]] = result if first_result_at is not None: first_result_at = time.time() except Empty: pass if not is_worker_alive(workers): break if first_result_at is not None and time.time( ) - first_result_at > wait_time: log('Not waiting for any more results') abort.set() if abort.is_set(): break while True: try: x = rq.get_nowait() result = process_result(log, x) if result is not None: results.put(result) found_results[result[0]] = result except Empty: break for w in workers: wlog = w.buf.getvalue().strip() log('\n' + '*' * 30, w.plugin.name, 'Covers', '*' * 30) log('Request extra headers:', w.plugin.browser.addheaders) if w.plugin in found_results: result = found_results[w.plugin] log('Downloaded cover:', '%dx%d' % (result[1], result[2])) else: log('Failed to download valid cover') if w.time_spent is None: log('Download aborted') else: log('Took', w.time_spent, 'seconds') if wlog: log(wlog) log('\n' + '*' * 80)
class JobManager(QAbstractTableModel, AdaptSQP): # {{{ job_added = pyqtSignal(int) job_done = pyqtSignal(int) def __init__(self): QAbstractTableModel.__init__(self) SearchQueryParser.__init__(self, ['all']) self.wait_icon = (QIcon(I('jobs.png'))) self.running_icon = (QIcon(I('exec.png'))) self.error_icon = (QIcon(I('dialog_error.png'))) self.done_icon = (QIcon(I('ok.png'))) self.jobs = [] self.add_job = Dispatcher(self._add_job) self.server = Server(limit=config['worker_limit'] // 2, enforce_cpu_limit=config['enforce_cpu_limit']) self.threaded_server = ThreadedJobServer() self.changed_queue = Queue() self.timer = QTimer(self) self.timer.timeout.connect(self.update, type=Qt.ConnectionType.QueuedConnection) self.timer.start(1000) def columnCount(self, parent=QModelIndex()): return 5 def rowCount(self, parent=QModelIndex()): return len(self.jobs) def headerData(self, section, orientation, role): if role != Qt.ItemDataRole.DisplayRole: return None if orientation == Qt.Orientation.Horizontal: return ({ 0: _('Job'), 1: _('Status'), 2: _('Progress'), 3: _('Running time'), 4: _('Start time'), }.get(section, '')) else: return (section + 1) def show_tooltip(self, arg): widget, pos = arg QToolTip.showText(pos, self.get_tooltip()) def get_tooltip(self): running_jobs = [j for j in self.jobs if j.run_state == j.RUNNING] waiting_jobs = [j for j in self.jobs if j.run_state == j.WAITING] lines = [ ngettext('There is a running job:', 'There are {} running jobs:', len(running_jobs)).format(len(running_jobs)) ] for job in running_jobs: desc = job.description if not desc: desc = _('Unknown job') p = 100. if job.is_finished else job.percent lines.append('%s: %.0f%% done' % (desc, p)) l = ngettext('There is a waiting job', 'There are {} waiting jobs', len(waiting_jobs)).format(len(waiting_jobs)) lines.extend(['', l]) for job in waiting_jobs: desc = job.description if not desc: desc = _('Unknown job') lines.append(desc) return '\n'.join(['calibre', ''] + lines) def data(self, index, role): try: if role not in (Qt.ItemDataRole.DisplayRole, Qt.ItemDataRole.DecorationRole): return None row, col = index.row(), index.column() job = self.jobs[row] if role == Qt.ItemDataRole.DisplayRole: if col == 0: desc = job.description if not desc: desc = _('Unknown job') return (desc) if col == 1: return (job.status_text) if col == 2: p = 100. if job.is_finished else job.percent return (p) if col == 3: rtime = job.running_time if rtime is None: return None return human_readable_interval(rtime) if col == 4 and job.start_time is not None: return (strftime('%H:%M -- %d %b', time.localtime(job.start_time))) if role == Qt.ItemDataRole.DecorationRole and col == 0: state = job.run_state if state == job.WAITING: return self.wait_icon if state == job.RUNNING: return self.running_icon if job.killed or job.failed: return self.error_icon return self.done_icon except: import traceback traceback.print_exc() return None def update(self): try: self._update() except BaseException: import traceback traceback.print_exc() def _update(self): # Update running time for i, j in enumerate(self.jobs): if j.run_state == j.RUNNING: idx = self.index(i, 3) self.dataChanged.emit(idx, idx) # Update parallel jobs jobs = set() while True: try: jobs.add(self.server.changed_jobs_queue.get_nowait()) except Empty: break # Update device jobs while True: try: jobs.add(self.changed_queue.get_nowait()) except Empty: break # Update threaded jobs while True: try: jobs.add(self.threaded_server.changed_jobs.get_nowait()) except Empty: break if jobs: needs_reset = False for job in jobs: orig_state = job.run_state job.update() if orig_state != job.run_state: needs_reset = True if job.is_finished: self.job_done.emit(len(self.unfinished_jobs())) if needs_reset: self.modelAboutToBeReset.emit() self.jobs.sort() self.modelReset.emit() else: for job in jobs: idx = self.jobs.index(job) self.dataChanged.emit(self.index(idx, 0), self.index(idx, 3)) # Kill parallel jobs that have gone on too long try: wmax_time = gprefs['worker_max_time'] * 60 except: wmax_time = 0 if wmax_time > 0: for job in self.jobs: if isinstance(job, ParallelJob): rtime = job.running_time if (rtime is not None and rtime > wmax_time and job.duration is None): job.timed_out = True self.server.kill_job(job) def _add_job(self, job): self.modelAboutToBeReset.emit() self.jobs.append(job) self.jobs.sort() self.job_added.emit(len(self.unfinished_jobs())) self.modelReset.emit() def done_jobs(self): return [j for j in self.jobs if j.is_finished] def unfinished_jobs(self): return [j for j in self.jobs if not j.is_finished] def row_to_job(self, row): return self.jobs[row] def rows_to_jobs(self, rows): return [self.jobs[row] for row in rows] def has_device_jobs(self, queued_also=False): for job in self.jobs: if isinstance(job, DeviceJob): if job.duration is None: # Running or waiting if (job.is_running or queued_also): return True return False def has_jobs(self): for job in self.jobs: if job.is_running: return True return False def run_job(self, done, name, args=[], kwargs={}, description='', core_usage=1): job = ParallelJob(name, description, done, args=args, kwargs=kwargs) job.core_usage = core_usage self.add_job(job) self.server.add_job(job) return job def run_threaded_job(self, job): self.add_job(job) self.threaded_server.add_job(job) def launch_gui_app(self, name, args=(), kwargs=None, description=''): job = ParallelJob(name, description, lambda x: x, args=list(args), kwargs=kwargs or {}) self.server.run_job(job, gui=True, redirect_output=False) def _kill_job(self, job): if isinstance(job, ParallelJob): self.server.kill_job(job) elif isinstance(job, ThreadedJob): self.threaded_server.kill_job(job) else: job.kill_on_start = True def hide_jobs(self, rows): for r in rows: self.jobs[r].hidden_in_gui = True for r in rows: self.dataChanged.emit(self.index(r, 0), self.index(r, 0)) def show_hidden_jobs(self): for j in self.jobs: j.hidden_in_gui = False for r in range(len(self.jobs)): self.dataChanged.emit(self.index(r, 0), self.index(r, 0)) def kill_job(self, job, view): if isinstance(job, DeviceJob): return error_dialog( view, _('Cannot kill job'), _('Cannot kill jobs that communicate with the device')).exec_( ) if job.duration is not None: return error_dialog(view, _('Cannot kill job'), _('Job has already run')).exec_() if not getattr(job, 'killable', True): return error_dialog(view, _('Cannot kill job'), _('This job cannot be stopped'), show=True) self._kill_job(job) def kill_multiple_jobs(self, jobs, view): devjobs = [j for j in jobs if isinstance(j, DeviceJob)] if devjobs: error_dialog( view, _('Cannot kill job'), _('Cannot kill jobs that communicate with the device')).exec_( ) jobs = [j for j in jobs if not isinstance(j, DeviceJob)] jobs = [j for j in jobs if j.duration is None] unkillable = [j for j in jobs if not getattr(j, 'killable', True)] if unkillable: names = '\n'.join(as_unicode(j.description) for j in unkillable) error_dialog( view, _('Cannot kill job'), _('Some of the jobs cannot be stopped. Click "Show details"' ' to see the list of unstoppable jobs.'), det_msg=names, show=True) jobs = [j for j in jobs if getattr(j, 'killable', True)] jobs = [j for j in jobs if j.duration is None] for j in jobs: self._kill_job(j) def kill_all_jobs(self): for job in self.jobs: if (isinstance(job, DeviceJob) or job.duration is not None or not getattr(job, 'killable', True)): continue self._kill_job(job) def terminate_all_jobs(self): self.server.killall() for job in self.jobs: if (isinstance(job, DeviceJob) or job.duration is not None or not getattr(job, 'killable', True)): continue if not isinstance(job, ParallelJob): self._kill_job(job) def universal_set(self): return { i for i, j in enumerate(self.jobs) if not getattr(j, 'hidden_in_gui', False) } def get_matches(self, location, query, candidates=None): if candidates is None: candidates = self.universal_set() ans = set() if not query: return ans query = lower(query) for j in candidates: job = self.jobs[j] if job.description and query in lower(job.description): ans.add(j) return ans def find(self, query): query = query.strip() rows = self.parse(query) return rows
class JobManager(QAbstractTableModel, AdaptSQP): # {{{ job_added = pyqtSignal(int) job_done = pyqtSignal(int) def __init__(self): QAbstractTableModel.__init__(self) SearchQueryParser.__init__(self, ['all']) self.wait_icon = (QIcon(I('jobs.png'))) self.running_icon = (QIcon(I('exec.png'))) self.error_icon = (QIcon(I('dialog_error.png'))) self.done_icon = (QIcon(I('ok.png'))) self.jobs = [] self.add_job = Dispatcher(self._add_job) self.server = Server(limit=int(config['worker_limit']/2.0), enforce_cpu_limit=config['enforce_cpu_limit']) self.threaded_server = ThreadedJobServer() self.changed_queue = Queue() self.timer = QTimer(self) self.timer.timeout.connect(self.update, type=Qt.QueuedConnection) self.timer.start(1000) def columnCount(self, parent=QModelIndex()): return 5 def rowCount(self, parent=QModelIndex()): return len(self.jobs) def headerData(self, section, orientation, role): if role != Qt.DisplayRole: return None if orientation == Qt.Horizontal: return ({ 0: _('Job'), 1: _('Status'), 2: _('Progress'), 3: _('Running time'), 4: _('Start time'), }.get(section, '')) else: return (section+1) def show_tooltip(self, arg): widget, pos = arg QToolTip.showText(pos, self.get_tooltip()) def get_tooltip(self): running_jobs = [j for j in self.jobs if j.run_state == j.RUNNING] waiting_jobs = [j for j in self.jobs if j.run_state == j.WAITING] lines = [ngettext('There is a running job:', 'There are {} running jobs:', len(running_jobs)).format(len(running_jobs))] for job in running_jobs: desc = job.description if not desc: desc = _('Unknown job') p = 100. if job.is_finished else job.percent lines.append('%s: %.0f%% done'%(desc, p)) l = ngettext('There is a waiting job', 'There are {} waiting jobs', len(waiting_jobs)).format(len(waiting_jobs)) lines.extend(['', l]) for job in waiting_jobs: desc = job.description if not desc: desc = _('Unknown job') lines.append(desc) return '\n'.join(['calibre', '']+ lines) def data(self, index, role): try: if role not in (Qt.DisplayRole, Qt.DecorationRole): return None row, col = index.row(), index.column() job = self.jobs[row] if role == Qt.DisplayRole: if col == 0: desc = job.description if not desc: desc = _('Unknown job') return (desc) if col == 1: return (job.status_text) if col == 2: p = 100. if job.is_finished else job.percent return (p) if col == 3: rtime = job.running_time if rtime is None: return None return human_readable_interval(rtime) if col == 4 and job.start_time is not None: return (strftime(u'%H:%M -- %d %b', time.localtime(job.start_time))) if role == Qt.DecorationRole and col == 0: state = job.run_state if state == job.WAITING: return self.wait_icon if state == job.RUNNING: return self.running_icon if job.killed or job.failed: return self.error_icon return self.done_icon except: import traceback traceback.print_exc() return None def update(self): try: self._update() except BaseException: import traceback traceback.print_exc() def _update(self): # Update running time for i, j in enumerate(self.jobs): if j.run_state == j.RUNNING: idx = self.index(i, 3) self.dataChanged.emit(idx, idx) # Update parallel jobs jobs = set([]) while True: try: jobs.add(self.server.changed_jobs_queue.get_nowait()) except Empty: break # Update device jobs while True: try: jobs.add(self.changed_queue.get_nowait()) except Empty: break # Update threaded jobs while True: try: jobs.add(self.threaded_server.changed_jobs.get_nowait()) except Empty: break if jobs: needs_reset = False for job in jobs: orig_state = job.run_state job.update() if orig_state != job.run_state: needs_reset = True if job.is_finished: self.job_done.emit(len(self.unfinished_jobs())) if needs_reset: self.modelAboutToBeReset.emit() self.jobs.sort() self.modelReset.emit() else: for job in jobs: idx = self.jobs.index(job) self.dataChanged.emit( self.index(idx, 0), self.index(idx, 3)) # Kill parallel jobs that have gone on too long try: wmax_time = gprefs['worker_max_time'] * 60 except: wmax_time = 0 if wmax_time > 0: for job in self.jobs: if isinstance(job, ParallelJob): rtime = job.running_time if (rtime is not None and rtime > wmax_time and job.duration is None): job.timed_out = True self.server.kill_job(job) def _add_job(self, job): self.modelAboutToBeReset.emit() self.jobs.append(job) self.jobs.sort() self.job_added.emit(len(self.unfinished_jobs())) self.modelReset.emit() def done_jobs(self): return [j for j in self.jobs if j.is_finished] def unfinished_jobs(self): return [j for j in self.jobs if not j.is_finished] def row_to_job(self, row): return self.jobs[row] def rows_to_jobs(self, rows): return [self.jobs[row] for row in rows] def has_device_jobs(self, queued_also=False): for job in self.jobs: if isinstance(job, DeviceJob): if job.duration is None: # Running or waiting if (job.is_running or queued_also): return True return False def has_jobs(self): for job in self.jobs: if job.is_running: return True return False def run_job(self, done, name, args=[], kwargs={}, description='', core_usage=1): job = ParallelJob(name, description, done, args=args, kwargs=kwargs) job.core_usage = core_usage self.add_job(job) self.server.add_job(job) return job def run_threaded_job(self, job): self.add_job(job) self.threaded_server.add_job(job) def launch_gui_app(self, name, args=[], kwargs={}, description=''): job = ParallelJob(name, description, lambda x: x, args=args, kwargs=kwargs) self.server.run_job(job, gui=True, redirect_output=False) def _kill_job(self, job): if isinstance(job, ParallelJob): self.server.kill_job(job) elif isinstance(job, ThreadedJob): self.threaded_server.kill_job(job) else: job.kill_on_start = True def hide_jobs(self, rows): for r in rows: self.jobs[r].hidden_in_gui = True for r in rows: self.dataChanged.emit(self.index(r, 0), self.index(r, 0)) def show_hidden_jobs(self): for j in self.jobs: j.hidden_in_gui = False for r in range(len(self.jobs)): self.dataChanged.emit(self.index(r, 0), self.index(r, 0)) def kill_job(self, job, view): if isinstance(job, DeviceJob): return error_dialog(view, _('Cannot kill job'), _('Cannot kill jobs that communicate with the device')).exec_() if job.duration is not None: return error_dialog(view, _('Cannot kill job'), _('Job has already run')).exec_() if not getattr(job, 'killable', True): return error_dialog(view, _('Cannot kill job'), _('This job cannot be stopped'), show=True) self._kill_job(job) def kill_multiple_jobs(self, jobs, view): devjobs = [j for j in jobs if isinstance(j, DeviceJob)] if devjobs: error_dialog(view, _('Cannot kill job'), _('Cannot kill jobs that communicate with the device')).exec_() jobs = [j for j in jobs if not isinstance(j, DeviceJob)] jobs = [j for j in jobs if j.duration is None] unkillable = [j for j in jobs if not getattr(j, 'killable', True)] if unkillable: names = u'\n'.join(as_unicode(j.description) for j in unkillable) error_dialog(view, _('Cannot kill job'), _('Some of the jobs cannot be stopped. Click Show details' ' to see the list of unstoppable jobs.'), det_msg=names, show=True) jobs = [j for j in jobs if getattr(j, 'killable', True)] jobs = [j for j in jobs if j.duration is None] for j in jobs: self._kill_job(j) def kill_all_jobs(self): for job in self.jobs: if (isinstance(job, DeviceJob) or job.duration is not None or not getattr(job, 'killable', True)): continue self._kill_job(job) def terminate_all_jobs(self): self.server.killall() for job in self.jobs: if (isinstance(job, DeviceJob) or job.duration is not None or not getattr(job, 'killable', True)): continue if not isinstance(job, ParallelJob): self._kill_job(job) def universal_set(self): return {i for i, j in enumerate(self.jobs) if not getattr(j, 'hidden_in_gui', False)} def get_matches(self, location, query, candidates=None): if candidates is None: candidates = self.universal_set() ans = set() if not query: return ans query = lower(query) for j in candidates: job = self.jobs[j] if job.description and query in lower(job.description): ans.add(j) return ans def find(self, query): query = query.strip() rows = self.parse(query) return rows
class WebSocketConnection(HTTPConnection): # Internal API {{{ in_websocket_mode = False websocket_handler = None def __init__(self, *args, **kwargs): global conn_id HTTPConnection.__init__(self, *args, **kwargs) self.sendq = Queue() self.control_frames = deque() self.cf_lock = Lock() self.sending = None self.send_buf = None self.frag_decoder = UTF8Decoder() self.ws_close_received = self.ws_close_sent = False conn_id += 1 self.websocket_connection_id = conn_id self.stop_reading = False def finalize_headers(self, inheaders): upgrade = inheaders.get('Upgrade', '') key = inheaders.get('Sec-WebSocket-Key', None) conn = { x.strip().lower() for x in inheaders.get('Connection', '').split(',') } if key is None or upgrade.lower( ) != 'websocket' or 'upgrade' not in conn: return HTTPConnection.finalize_headers(self, inheaders) ver = inheaders.get('Sec-WebSocket-Version', 'Unknown') try: ver_ok = int(ver) >= 13 except Exception: ver_ok = False if not ver_ok: return self.simple_response( http_client.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver) if self.method != 'GET': return self.simple_response( http_client.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method) response = HANDSHAKE_STR % as_base64_unicode( sha1((key + GUID_STR).encode('utf-8')).digest()) self.optimize_for_sending_packet() self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self.set_state(WRITE, self.upgrade_connection_to_ws, ReadOnlyFileBuffer(response.encode('ascii')), inheaders) def upgrade_connection_to_ws(self, buf, inheaders, event): if self.write(buf): if self.websocket_handler is None: self.websocket_handler = DummyHandler() self.read_frame, self.current_recv_opcode = ReadFrame(), None self.in_websocket_mode = True try: self.websocket_handler.handle_websocket_upgrade( self.websocket_connection_id, weakref.ref(self), inheaders) except Exception as err: self.log.exception('Error in WebSockets upgrade handler:') self.websocket_close( UNEXPECTED_ERROR, 'Unexpected error in handler: %r' % as_unicode(err)) self.handle_event = self.ws_duplex self.set_ws_state() self.end_send_optimization() def set_ws_state(self): if self.ws_close_sent or self.ws_close_received: if self.ws_close_sent: self.ready = False else: self.wait_for = WRITE return if self.send_buf is not None or self.sending is not None: self.wait_for = RDWR else: try: self.sending = self.sendq.get_nowait() except Empty: with self.cf_lock: if self.control_frames: self.wait_for = RDWR else: self.wait_for = READ else: self.wait_for = RDWR if self.stop_reading: if self.wait_for is READ: self.ready = False elif self.wait_for is RDWR: self.wait_for = WRITE def ws_duplex(self, event): if event is READ: self.ws_read() elif event is WRITE: self.ws_write() self.set_ws_state() def ws_read(self): if not self.stop_reading: self.read_frame(self) def ws_data_received(self, data, opcode, frame_starting, frame_finished, is_final_frame_of_message): if opcode in CONTROL_CODES: return self.ws_control_frame(opcode, data) message_starting = self.current_recv_opcode is None if message_starting: if opcode == CONTINUATION: self.log.error( 'Client sent continuation frame with no message to continue' ) self.websocket_close( PROTOCOL_ERROR, 'Continuation frame without any message to continue') return self.current_recv_opcode = opcode elif frame_starting and opcode != CONTINUATION: self.log.error( 'Client sent continuation frame with non-zero opcode') self.websocket_close(PROTOCOL_ERROR, 'Continuation frame with non-zero opcode') return message_finished = frame_finished and is_final_frame_of_message if self.current_recv_opcode == TEXT: if message_starting: self.frag_decoder.reset() empty_data = len(data) == 0 try: data = self.frag_decoder(data) except ValueError: self.frag_decoder.reset() self.log.error('Client sent undecodeable UTF-8') return self.websocket_close(INCONSISTENT_DATA, 'Not valid UTF-8') if message_finished: if (not data and not empty_data) or self.frag_decoder.state: self.frag_decoder.reset() self.log.error('Client sent undecodeable UTF-8') return self.websocket_close(INCONSISTENT_DATA, 'Not valid UTF-8') if message_finished: self.current_recv_opcode = None self.frag_decoder.reset() try: self.handle_websocket_data(data, message_starting, message_finished) except Exception as err: self.log.exception('Error in WebSockets data handler:') self.websocket_close( UNEXPECTED_ERROR, 'Unexpected error in handler: %r' % as_unicode(err)) def ws_control_frame(self, opcode, data): if opcode in (PING, CLOSE): rcode = PONG if opcode == PING else CLOSE if opcode == CLOSE: self.ws_close_received = True self.stop_reading = True if data: try: close_code = unpack_from(b'!H', data)[0] except struct_error: data = pack( b'!H', PROTOCOL_ERROR ) + b'close frame data must be atleast two bytes' else: try: utf8_decode(data[2:]) except ValueError: data = pack( b'!H', PROTOCOL_ERROR ) + b'close frame data must be valid UTF-8' else: if close_code < 1000 or close_code in RESERVED_CLOSE_CODES or ( 1011 < close_code < 3000): data = pack( b'!H', PROTOCOL_ERROR) + b'close code reserved' else: close_code = NORMAL_CLOSE data = pack(b'!H', close_code) f = ReadOnlyFileBuffer(create_frame(1, rcode, data)) f.is_close_frame = opcode == CLOSE with self.cf_lock: self.control_frames.append(f) elif opcode == PONG: try: self.websocket_handler.handle_websocket_pong( self.websocket_connection_id, data) except Exception: self.log.exception('Error in PONG handler:') self.set_ws_state() def websocket_close(self, code=NORMAL_CLOSE, reason=b''): if isinstance(reason, unicode_type): reason = reason.encode('utf-8') self.stop_reading = True reason = reason[:123] if code is None and not reason: f = ReadOnlyFileBuffer(create_frame(1, CLOSE, b'')) else: f = ReadOnlyFileBuffer( create_frame(1, CLOSE, pack(b'!H', code) + reason)) f.is_close_frame = True with self.cf_lock: self.control_frames.append(f) self.set_ws_state() def ws_write(self): if self.ws_close_sent: return if self.send_buf is not None: if self.write(self.send_buf): self.end_send_optimization() if getattr(self.send_buf, 'is_close_frame', False): self.ws_close_sent = True self.send_buf = None else: with self.cf_lock: try: self.send_buf = self.control_frames.popleft() except IndexError: if self.sending is not None: self.send_buf = self.sending.create_frame() if self.send_buf is None: self.sending = None if self.send_buf is not None: self.optimize_for_sending_packet() def close(self): if self.in_websocket_mode: try: self.websocket_handler.handle_websocket_close( self.websocket_connection_id) except Exception: self.log.exception('Error in WebSocket close handler') # Try to write a close frame, just once try: if self.send_buf is None and not self.ws_close_sent: self.websocket_close(SHUTTING_DOWN, 'Shutting down') with self.cf_lock: self.write(self.control_frames.pop()) except Exception: pass Connection.close(self) else: HTTPConnection.close(self) # }}} def send_websocket_message(self, buf, wakeup=True): ''' Send a complete message. This class will take care of splitting it into appropriate frames automatically. `buf` must be a file like object. ''' self.sendq.put(MessageWriter(buf)) self.wait_for = RDWR if wakeup: self.wakeup() def send_websocket_frame(self, data, is_first=True, is_last=True): ''' Useful for streaming handlers that want to break up messages into frames themselves. Note that these frames will be interleaved with control frames, so they should not be too large. ''' opcode = (TEXT if isinstance(data, unicode_type) else BINARY) if is_first else CONTINUATION fin = 1 if is_last else 0 frame = create_frame(fin, opcode, data) with self.cf_lock: self.control_frames.append(ReadOnlyFileBuffer(frame)) def send_websocket_ping(self, data=b''): ''' Send a PING to the remote client, it should reply with a PONG which will be sent to the handle_websocket_pong callback in your handler. ''' if isinstance(data, unicode_type): data = data.encode('utf-8') frame = create_frame(True, PING, data) with self.cf_lock: self.control_frames.append(ReadOnlyFileBuffer(frame)) def handle_websocket_data(self, data, message_starting, message_finished): ''' Called when some data is received from the remote client. In general the data may not constitute a complete "message", use the message_starting and message_finished flags to re-assemble it into a complete message in the handler. Note that for binary data, data is a mutable object. If you intend to keep it around after this method returns, create a bytestring from it, using tobytes(). ''' self.websocket_handler.handle_websocket_data( self.websocket_connection_id, data, message_starting, message_finished)
class ParseWorker(Thread): daemon = True SLEEP_TIME = 1 def __init__(self): Thread.__init__(self) self.requests = Queue() self.request_count = 0 self.parse_items = {} self.launch_error = None def run(self): mod, func = 'calibre.gui2.tweak_book.preview', 'parse_html' try: # Connect to the worker and send a dummy job to initialize it self.worker = offload_worker(priority='low') self.worker(mod, func, '<p></p>') except: import traceback traceback.print_exc() self.launch_error = traceback.format_exc() return while True: time.sleep(self.SLEEP_TIME) x = self.requests.get() requests = [x] while True: try: requests.append(self.requests.get_nowait()) except Empty: break if shutdown in requests: self.worker.shutdown() break request = sorted(requests, reverse=True)[0] del requests pi, data = request[1:] try: res = self.worker(mod, func, data) except: import traceback traceback.print_exc() else: pi.parsing_done = True parsed_data = res['result'] if res['tb']: prints("Parser error:") prints(res['tb']) else: pi.parsed_data = parsed_data def add_request(self, name): data = get_data(name) ldata, hdata = len(data), hash(data) pi = self.parse_items.get(name, None) if pi is None: self.parse_items[name] = pi = ParseItem(name) else: if pi.parsing_done and pi.length == ldata and pi.fingerprint == hdata: return pi.parsed_data = None pi.parsing_done = False pi.length, pi.fingerprint = ldata, hdata self.requests.put((self.request_count, pi, data)) self.request_count += 1 def shutdown(self): self.requests.put(shutdown) def get_data(self, name): return getattr(self.parse_items.get(name, None), 'parsed_data', None) def clear(self): self.parse_items.clear() def is_alive(self): return Thread.is_alive(self) or (hasattr(self, 'worker') and self.worker.is_alive())
def test_identify_plugin(name, tests, modify_plugin=lambda plugin:None, # {{{ fail_missing_meta=True): ''' :param name: Plugin name :param tests: List of 2-tuples. Each two tuple is of the form (args, test_funcs). args is a dict of keyword arguments to pass to the identify method. test_funcs are callables that accept a Metadata object and return True iff the object passes the test. ''' plugin = None for x in all_metadata_plugins(): if x.name == name and 'identify' in x.capabilities: plugin = x break modify_plugin(plugin) prints('Testing the identify function of', plugin.name) prints('Using extra headers:', plugin.browser.addheaders) tdir, lf, log, abort = init_test(plugin.name) prints('Log saved to', lf) times = [] for kwargs, test_funcs in tests: log('') log('#'*80) log('### Running test with:', kwargs) log('#'*80) prints('Running test with:', kwargs) rq = Queue() args = (log, rq, abort) start_time = time.time() plugin.running_a_test = True try: err = plugin.identify(*args, **kwargs) finally: plugin.running_a_test = False total_time = time.time() - start_time times.append(total_time) if err is not None: prints('identify returned an error for args', args) prints(err) break results = [] while True: try: results.append(rq.get_nowait()) except Empty: break prints('Found', len(results), 'matches:', end=' ') prints('Smaller relevance means better match') results.sort(key=plugin.identify_results_keygen( title=kwargs.get('title', None), authors=kwargs.get('authors', None), identifiers=kwargs.get('identifiers', {}))) for i, mi in enumerate(results): prints('*'*30, 'Relevance:', i, '*'*30) if mi.rating: mi.rating *= 2 prints(mi) prints('\nCached cover URL :', plugin.get_cached_cover_url(mi.identifiers)) prints('*'*75, '\n\n') possibles = [] for mi in results: test_failed = False for tfunc in test_funcs: if not tfunc(mi): test_failed = True break if not test_failed: possibles.append(mi) if not possibles: prints('ERROR: No results that passed all tests were found') prints('Log saved to', lf) log.close() dump_log(lf) raise SystemExit(1) good = [x for x in possibles if plugin.test_fields(x) is None] if not good: prints('Failed to find', plugin.test_fields(possibles[0])) if fail_missing_meta: raise SystemExit(1) if results[0] is not possibles[0]: prints('Most relevant result failed the tests') raise SystemExit(1) if 'cover' in plugin.capabilities: rq = Queue() mi = results[0] plugin.download_cover(log, rq, abort, title=mi.title, authors=mi.authors, identifiers=mi.identifiers) results = [] while True: try: results.append(rq.get_nowait()) except Empty: break if not results and fail_missing_meta: prints('Cover download failed') raise SystemExit(1) elif results: cdata = results[0] cover = os.path.join(tdir, plugin.name.replace(' ', '')+'-%s-cover.jpg'%sanitize_file_name(mi.title.replace(' ', '_'))) with open(cover, 'wb') as f: f.write(cdata[-1]) prints('Cover downloaded to:', cover) if len(cdata[-1]) < 10240: prints('Downloaded cover too small') raise SystemExit(1) prints('Average time per query', sum(times)/len(times)) if os.stat(lf).st_size > 10: prints('There were some errors/warnings, see log', lf)
class WebSocketConnection(HTTPConnection): # Internal API {{{ in_websocket_mode = False websocket_handler = None def __init__(self, *args, **kwargs): global conn_id HTTPConnection.__init__(self, *args, **kwargs) self.sendq = Queue() self.control_frames = deque() self.cf_lock = Lock() self.sending = None self.send_buf = None self.frag_decoder = UTF8Decoder() self.ws_close_received = self.ws_close_sent = False conn_id += 1 self.websocket_connection_id = conn_id self.stop_reading = False def finalize_headers(self, inheaders): upgrade = inheaders.get('Upgrade', '') key = inheaders.get('Sec-WebSocket-Key', None) conn = {x.strip().lower() for x in inheaders.get('Connection', '').split(',')} if key is None or upgrade.lower() != 'websocket' or 'upgrade' not in conn: return HTTPConnection.finalize_headers(self, inheaders) ver = inheaders.get('Sec-WebSocket-Version', 'Unknown') try: ver_ok = int(ver) >= 13 except Exception: ver_ok = False if not ver_ok: return self.simple_response(http_client.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver) if self.method != 'GET': return self.simple_response(http_client.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method) response = HANDSHAKE_STR % as_base64_unicode(sha1((key + GUID_STR).encode('utf-8')).digest()) self.optimize_for_sending_packet() self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self.set_state(WRITE, self.upgrade_connection_to_ws, ReadOnlyFileBuffer(response.encode('ascii')), inheaders) def upgrade_connection_to_ws(self, buf, inheaders, event): if self.write(buf): if self.websocket_handler is None: self.websocket_handler = DummyHandler() self.read_frame, self.current_recv_opcode = ReadFrame(), None self.in_websocket_mode = True try: self.websocket_handler.handle_websocket_upgrade(self.websocket_connection_id, weakref.ref(self), inheaders) except Exception as err: self.log.exception('Error in WebSockets upgrade handler:') self.websocket_close(UNEXPECTED_ERROR, 'Unexpected error in handler: %r' % as_unicode(err)) self.handle_event = self.ws_duplex self.set_ws_state() self.end_send_optimization() def set_ws_state(self): if self.ws_close_sent or self.ws_close_received: if self.ws_close_sent: self.ready = False else: self.wait_for = WRITE return if self.send_buf is not None or self.sending is not None: self.wait_for = RDWR else: try: self.sending = self.sendq.get_nowait() except Empty: with self.cf_lock: if self.control_frames: self.wait_for = RDWR else: self.wait_for = READ else: self.wait_for = RDWR if self.stop_reading: if self.wait_for is READ: self.ready = False elif self.wait_for is RDWR: self.wait_for = WRITE def ws_duplex(self, event): if event is READ: self.ws_read() elif event is WRITE: self.ws_write() self.set_ws_state() def ws_read(self): if not self.stop_reading: self.read_frame(self) def ws_data_received(self, data, opcode, frame_starting, frame_finished, is_final_frame_of_message): if opcode in CONTROL_CODES: return self.ws_control_frame(opcode, data) message_starting = self.current_recv_opcode is None if message_starting: if opcode == CONTINUATION: self.log.error('Client sent continuation frame with no message to continue') self.websocket_close(PROTOCOL_ERROR, 'Continuation frame without any message to continue') return self.current_recv_opcode = opcode elif frame_starting and opcode != CONTINUATION: self.log.error('Client sent continuation frame with non-zero opcode') self.websocket_close(PROTOCOL_ERROR, 'Continuation frame with non-zero opcode') return message_finished = frame_finished and is_final_frame_of_message if self.current_recv_opcode == TEXT: if message_starting: self.frag_decoder.reset() empty_data = len(data) == 0 try: data = self.frag_decoder(data) except ValueError: self.frag_decoder.reset() self.log.error('Client sent undecodeable UTF-8') return self.websocket_close(INCONSISTENT_DATA, 'Not valid UTF-8') if message_finished: if (not data and not empty_data) or self.frag_decoder.state: self.frag_decoder.reset() self.log.error('Client sent undecodeable UTF-8') return self.websocket_close(INCONSISTENT_DATA, 'Not valid UTF-8') if message_finished: self.current_recv_opcode = None self.frag_decoder.reset() try: self.handle_websocket_data(data, message_starting, message_finished) except Exception as err: self.log.exception('Error in WebSockets data handler:') self.websocket_close(UNEXPECTED_ERROR, 'Unexpected error in handler: %r' % as_unicode(err)) def ws_control_frame(self, opcode, data): if opcode in (PING, CLOSE): rcode = PONG if opcode == PING else CLOSE if opcode == CLOSE: self.ws_close_received = True self.stop_reading = True if data: try: close_code = unpack_from(b'!H', data)[0] except struct_error: data = pack(b'!H', PROTOCOL_ERROR) + b'close frame data must be atleast two bytes' else: try: utf8_decode(data[2:]) except ValueError: data = pack(b'!H', PROTOCOL_ERROR) + b'close frame data must be valid UTF-8' else: if close_code < 1000 or close_code in RESERVED_CLOSE_CODES or (1011 < close_code < 3000): data = pack(b'!H', PROTOCOL_ERROR) + b'close code reserved' else: close_code = NORMAL_CLOSE data = pack(b'!H', close_code) f = ReadOnlyFileBuffer(create_frame(1, rcode, data)) f.is_close_frame = opcode == CLOSE with self.cf_lock: self.control_frames.append(f) elif opcode == PONG: try: self.websocket_handler.handle_websocket_pong(self.websocket_connection_id, data) except Exception: self.log.exception('Error in PONG handler:') self.set_ws_state() def websocket_close(self, code=NORMAL_CLOSE, reason=b''): if isinstance(reason, type('')): reason = reason.encode('utf-8') self.stop_reading = True reason = reason[:123] if code is None and not reason: f = ReadOnlyFileBuffer(create_frame(1, CLOSE, b'')) else: f = ReadOnlyFileBuffer(create_frame(1, CLOSE, pack(b'!H', code) + reason)) f.is_close_frame = True with self.cf_lock: self.control_frames.append(f) self.set_ws_state() def ws_write(self): if self.ws_close_sent: return if self.send_buf is not None: if self.write(self.send_buf): self.end_send_optimization() if getattr(self.send_buf, 'is_close_frame', False): self.ws_close_sent = True self.send_buf = None else: with self.cf_lock: try: self.send_buf = self.control_frames.popleft() except IndexError: if self.sending is not None: self.send_buf = self.sending.create_frame() if self.send_buf is None: self.sending = None if self.send_buf is not None: self.optimize_for_sending_packet() def close(self): if self.in_websocket_mode: try: self.websocket_handler.handle_websocket_close(self.websocket_connection_id) except Exception: self.log.exception('Error in WebSocket close handler') # Try to write a close frame, just once try: if self.send_buf is None and not self.ws_close_sent: self.websocket_close(SHUTTING_DOWN, 'Shutting down') with self.cf_lock: self.write(self.control_frames.pop()) except Exception: pass Connection.close(self) else: HTTPConnection.close(self) # }}} def send_websocket_message(self, buf, wakeup=True): ''' Send a complete message. This class will take care of splitting it into appropriate frames automatically. `buf` must be a file like object. ''' self.sendq.put(MessageWriter(buf)) self.wait_for = RDWR if wakeup: self.wakeup() def send_websocket_frame(self, data, is_first=True, is_last=True): ''' Useful for streaming handlers that want to break up messages into frames themselves. Note that these frames will be interleaved with control frames, so they should not be too large. ''' opcode = (TEXT if isinstance(data, type('')) else BINARY) if is_first else CONTINUATION fin = 1 if is_last else 0 frame = create_frame(fin, opcode, data) with self.cf_lock: self.control_frames.append(ReadOnlyFileBuffer(frame)) def send_websocket_ping(self, data=b''): ''' Send a PING to the remote client, it should reply with a PONG which will be sent to the handle_websocket_pong callback in your handler. ''' if isinstance(data, type('')): data = data.encode('utf-8') frame = create_frame(True, PING, data) with self.cf_lock: self.control_frames.append(ReadOnlyFileBuffer(frame)) def handle_websocket_data(self, data, message_starting, message_finished): ''' Called when some data is received from the remote client. In general the data may not constitute a complete "message", use the message_starting and message_finished flags to re-assemble it into a complete message in the handler. Note that for binary data, data is a mutable object. If you intend to keep it around after this method returns, create a bytestring from it, using tobytes(). ''' self.websocket_handler.handle_websocket_data(self.websocket_connection_id, data, message_starting, message_finished)
class Server(Thread): def __init__(self, notify_on_job_done=lambda x: x, pool_size=None, limit=sys.maxsize, enforce_cpu_limit=True): Thread.__init__(self) self.daemon = True global _counter self.id = _counter+1 _counter += 1 if enforce_cpu_limit: limit = min(limit, cpu_count()) self.pool_size = limit if pool_size is None else pool_size self.notify_on_job_done = notify_on_job_done self.auth_key = os.urandom(32) self.address, self.listener = create_listener(self.auth_key, backlog=4) self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue() self.kill_queue = Queue() self.waiting_jobs = [] self.workers = deque() self.launched_worker_count = 0 self._worker_launch_lock = RLock() self.start() def launch_worker(self, gui=False, redirect_output=None, job_name=None): start = time.time() with self._worker_launch_lock: self.launched_worker_count += 1 id = self.launched_worker_count fd, rfile = tempfile.mkstemp(prefix=u'ipc_result_%d_%d_'%(self.id, id), dir=base_dir(), suffix=u'.pickle') os.close(fd) if redirect_output is None: redirect_output = not gui env = { 'CALIBRE_WORKER_ADDRESS' : environ_item(as_hex_unicode(msgpack_dumps(self.address))), 'CALIBRE_WORKER_KEY' : environ_item(as_hex_unicode(self.auth_key)), 'CALIBRE_WORKER_RESULT' : environ_item(as_hex_unicode(rfile)), } cw = self.do_launch(env, gui, redirect_output, rfile, job_name=job_name) if isinstance(cw, string_or_bytes): raise CriticalError('Failed to launch worker process:\n'+cw) if DEBUG: print('Worker Launch took:', time.time() - start) return cw def do_launch(self, env, gui, redirect_output, rfile, job_name=None): w = Worker(env, gui=gui, job_name=job_name) try: w(redirect_output=redirect_output) conn = eintr_retry_call(self.listener.accept) if conn is None: raise Exception('Failed to launch worker process') except BaseException: try: w.kill() except: pass import traceback return traceback.format_exc() return ConnectedWorker(w, conn, rfile) def add_job(self, job): job.done2 = self.notify_on_job_done self.add_jobs_queue.put(job) def run_job(self, job, gui=True, redirect_output=False): w = self.launch_worker(gui=gui, redirect_output=redirect_output, job_name=getattr(job, 'name', None)) w.start_job(job) def run(self): while True: try: job = self.add_jobs_queue.get(True, 0.2) if job is None: break self.waiting_jobs.insert(0, job) except Empty: pass # Get notifications from worker process for worker in self.workers: while True: try: n = worker.notifications.get_nowait() worker.job.notifications.put(n) self.changed_jobs_queue.put(worker.job) except Empty: break # Remove finished jobs for worker in [w for w in self.workers if not w.is_alive]: try: worker.close_log_file() except: pass self.workers.remove(worker) job = worker.job if worker.returncode != 0: job.failed = True job.returncode = worker.returncode elif os.path.exists(worker.rfile): try: with lopen(worker.rfile, 'rb') as f: job.result = pickle_loads(f.read()) os.remove(worker.rfile) except: pass job.duration = time.time() - job.start_time self.changed_jobs_queue.put(job) # Start waiting jobs sj = self.suitable_waiting_job() if sj is not None: job = self.waiting_jobs.pop(sj) job.start_time = time.time() if job.kill_on_start: job.duration = 0.0 job.returncode = 1 job.killed = job.failed = True job.result = None else: worker = self.launch_worker() worker.start_job(job) self.workers.append(worker) job.log_path = worker.log_path self.changed_jobs_queue.put(job) while True: try: j = self.kill_queue.get_nowait() self._kill_job(j) except Empty: break def suitable_waiting_job(self): available_workers = self.pool_size - len(self.workers) for worker in self.workers: job = worker.job if job.core_usage == -1: available_workers = 0 elif job.core_usage > 1: available_workers -= job.core_usage - 1 if available_workers < 1: return None for i, job in enumerate(self.waiting_jobs): if job.core_usage == -1: if available_workers >= self.pool_size: return i elif job.core_usage <= available_workers: return i def kill_job(self, job): self.kill_queue.put(job) def killall(self): for worker in self.workers: self.kill_queue.put(worker.job) def _kill_job(self, job): if job.start_time is None: job.kill_on_start = True return for worker in self.workers: if job is worker.job: worker.kill() job.killed = True break def split(self, tasks): ''' Split a list into a list of sub lists, with the number of sub lists being no more than the number of workers this server supports. Each sublist contains 2-tuples of the form (i, x) where x is an element from the original list and i is the index of the element x in the original list. ''' ans, count, pos = [], 0, 0 delta = int(ceil(len(tasks)/float(self.pool_size))) while count < len(tasks): section = [] for t in tasks[pos:pos+delta]: section.append((count, t)) count += 1 ans.append(section) pos += delta return ans def close(self): try: self.add_jobs_queue.put(None) except: pass try: self.listener.close() except: pass time.sleep(0.2) for worker in list(self.workers): try: worker.kill() except: pass def __enter__(self): return self def __exit__(self, *args): self.close()
class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ TagBrowserMixin, CoverFlowMixin, LibraryViewMixin, SearchBoxMixin, SavedSearchBoxMixin, SearchRestrictionMixin, LayoutMixin, UpdateMixin, EbookDownloadMixin ): 'The main GUI' proceed_requested = pyqtSignal(object, object) book_converted = pyqtSignal(object, object) shutting_down = False def __init__(self, opts, parent=None, gui_debug=None): MainWindow.__init__(self, opts, parent=parent, disable_automatic_gc=True) self.setWindowIcon(QApplication.instance().windowIcon()) self.jobs_pointer = Pointer(self) self.proceed_requested.connect(self.do_proceed, type=Qt.QueuedConnection) self.proceed_question = ProceedQuestion(self) self.job_error_dialog = JobError(self) self.keyboard = Manager(self) get_gui.ans = self self.opts = opts self.device_connected = None self.gui_debug = gui_debug self.iactions = OrderedDict() # Actions for action in interface_actions(): if opts.ignore_plugins and action.plugin_path is not None: continue try: ac = self.init_iaction(action) except: # Ignore errors in loading user supplied plugins import traceback traceback.print_exc() if action.plugin_path is None: raise continue ac.plugin_path = action.plugin_path ac.interface_action_base_plugin = action self.add_iaction(ac) self.load_store_plugins() def init_iaction(self, action): ac = action.load_actual_plugin(self) ac.plugin_path = action.plugin_path ac.interface_action_base_plugin = action action.actual_iaction_plugin_loaded = True return ac def add_iaction(self, ac): acmap = self.iactions if ac.name in acmap: if ac.priority >= acmap[ac.name].priority: acmap[ac.name] = ac else: acmap[ac.name] = ac def load_store_plugins(self): from calibre.gui2.store.loader import Stores self.istores = Stores() for store in available_store_plugins(): if self.opts.ignore_plugins and store.plugin_path is not None: continue try: st = self.init_istore(store) self.add_istore(st) except: # Ignore errors in loading user supplied plugins import traceback traceback.print_exc() if store.plugin_path is None: raise continue self.istores.builtins_loaded() def init_istore(self, store): st = store.load_actual_plugin(self) st.plugin_path = store.plugin_path st.base_plugin = store store.actual_istore_plugin_loaded = True return st def add_istore(self, st): stmap = self.istores if st.name in stmap: if st.priority >= stmap[st.name].priority: stmap[st.name] = st else: stmap[st.name] = st def initialize(self, library_path, db, listener, actions, show_gui=True): opts = self.opts self.preferences_action, self.quit_action = actions self.library_path = library_path self.library_broker = GuiLibraryBroker(db) self.content_server = None self.server_change_notification_timer = t = QTimer(self) self.server_changes = Queue() t.setInterval(1000), t.timeout.connect(self.handle_changes_from_server_debounced), t.setSingleShot(True) self._spare_pool = None self.must_restart_before_config = False self.listener = Listener(listener) self.check_messages_timer = QTimer() self.check_messages_timer.timeout.connect(self.another_instance_wants_to_talk) self.check_messages_timer.start(1000) for ac in self.iactions.values(): try: ac.do_genesis() except Exception: # Ignore errors in third party plugins import traceback traceback.print_exc() if getattr(ac, 'plugin_path', None) is None: raise self.donate_action = QAction(QIcon(I('donate.png')), _('&Donate to support calibre'), self) for st in self.istores.values(): st.do_genesis() MainWindowMixin.init_main_window_mixin(self, db) # Jobs Button {{{ self.job_manager = JobManager() self.jobs_dialog = JobsDialog(self, self.job_manager) self.jobs_button = JobsButton(parent=self) self.jobs_button.initialize(self.jobs_dialog, self.job_manager) # }}} LayoutMixin.init_layout_mixin(self) DeviceMixin.init_device_mixin(self) self.progress_indicator = ProgressIndicator(self) self.progress_indicator.pos = (0, 20) self.verbose = opts.verbose self.get_metadata = GetMetadata() self.upload_memory = {} self.metadata_dialogs = [] self.default_thumbnail = None self.tb_wrapper = textwrap.TextWrapper(width=40) self.viewers = collections.deque() self.system_tray_icon = None do_systray = config['systray_icon'] or opts.start_in_tray if do_systray: self.system_tray_icon = factory(app_id='com.calibre-ebook.gui').create_system_tray_icon(parent=self, title='calibre') if self.system_tray_icon is not None: self.system_tray_icon.setIcon(QIcon(I('lt.png', allow_user_override=False))) if not (iswindows or isosx): self.system_tray_icon.setIcon(QIcon.fromTheme('calibre-tray', self.system_tray_icon.icon())) self.system_tray_icon.setToolTip(self.jobs_button.tray_tooltip()) self.system_tray_icon.setVisible(True) self.jobs_button.tray_tooltip_updated.connect(self.system_tray_icon.setToolTip) elif do_systray: prints('Failed to create system tray icon, your desktop environment probably' ' does not support the StatusNotifier spec https://www.freedesktop.org/wiki/Specifications/StatusNotifierItem/') self.system_tray_menu = QMenu(self) self.toggle_to_tray_action = self.system_tray_menu.addAction(QIcon(I('page.png')), '') self.toggle_to_tray_action.triggered.connect(self.system_tray_icon_activated) self.system_tray_menu.addAction(self.donate_action) self.eject_action = self.system_tray_menu.addAction( QIcon(I('eject.png')), _('&Eject connected device')) self.eject_action.setEnabled(False) self.addAction(self.quit_action) self.system_tray_menu.addAction(self.quit_action) self.keyboard.register_shortcut('quit calibre', _('Quit calibre'), default_keys=('Ctrl+Q',), action=self.quit_action) if self.system_tray_icon is not None: self.system_tray_icon.setContextMenu(self.system_tray_menu) self.system_tray_icon.activated.connect(self.system_tray_icon_activated) self.quit_action.triggered[bool].connect(self.quit) self.donate_action.triggered[bool].connect(self.donate) self.minimize_action = QAction(_('Minimize the calibre window'), self) self.addAction(self.minimize_action) self.keyboard.register_shortcut('minimize calibre', self.minimize_action.text(), default_keys=(), action=self.minimize_action) self.minimize_action.triggered.connect(self.showMinimized) self.esc_action = QAction(self) self.addAction(self.esc_action) self.keyboard.register_shortcut('clear current search', _('Clear the current search'), default_keys=('Esc',), action=self.esc_action) self.esc_action.triggered.connect(self.esc) self.shift_esc_action = QAction(self) self.addAction(self.shift_esc_action) self.keyboard.register_shortcut('focus book list', _('Focus the book list'), default_keys=('Shift+Esc',), action=self.shift_esc_action) self.shift_esc_action.triggered.connect(self.shift_esc) self.ctrl_esc_action = QAction(self) self.addAction(self.ctrl_esc_action) self.keyboard.register_shortcut('clear virtual library', _('Clear the virtual library'), default_keys=('Ctrl+Esc',), action=self.ctrl_esc_action) self.ctrl_esc_action.triggered.connect(self.ctrl_esc) self.alt_esc_action = QAction(self) self.addAction(self.alt_esc_action) self.keyboard.register_shortcut('clear additional restriction', _('Clear the additional restriction'), default_keys=('Alt+Esc',), action=self.alt_esc_action) self.alt_esc_action.triggered.connect(self.clear_additional_restriction) # ###################### Start spare job server ######################## QTimer.singleShot(1000, self.create_spare_pool) # ###################### Location Manager ######################## self.location_manager.location_selected.connect(self.location_selected) self.location_manager.unmount_device.connect(self.device_manager.umount_device) self.location_manager.configure_device.connect(self.configure_connected_device) self.location_manager.update_device_metadata.connect(self.update_metadata_on_device) self.eject_action.triggered.connect(self.device_manager.umount_device) # ################### Update notification ################### UpdateMixin.init_update_mixin(self, opts) # ###################### Search boxes ######################## SearchRestrictionMixin.init_search_restriction_mixin(self) SavedSearchBoxMixin.init_saved_seach_box_mixin(self) # ###################### Library view ######################## LibraryViewMixin.init_library_view_mixin(self, db) SearchBoxMixin.init_search_box_mixin(self) # Requires current_db self.library_view.model().count_changed_signal.connect( self.iactions['Choose Library'].count_changed) if not gprefs.get('quick_start_guide_added', False): try: add_quick_start_guide(self.library_view) except: import traceback traceback.print_exc() for view in ('library', 'memory', 'card_a', 'card_b'): v = getattr(self, '%s_view' % view) v.selectionModel().selectionChanged.connect(self.update_status_bar) v.model().count_changed_signal.connect(self.update_status_bar) self.library_view.model().count_changed() self.bars_manager.database_changed(self.library_view.model().db) self.library_view.model().database_changed.connect(self.bars_manager.database_changed, type=Qt.QueuedConnection) # ########################## Tags Browser ############################## TagBrowserMixin.init_tag_browser_mixin(self, db) self.library_view.model().database_changed.connect(self.populate_tb_manage_menu, type=Qt.QueuedConnection) # ######################## Search Restriction ########################## if db.prefs['virtual_lib_on_startup']: self.apply_virtual_library(db.prefs['virtual_lib_on_startup']) self.rebuild_vl_tabs() # ########################## Cover Flow ################################ CoverFlowMixin.init_cover_flow_mixin(self) self._calculated_available_height = min(max_available_height()-15, self.height()) self.resize(self.width(), self._calculated_available_height) self.build_context_menus() for ac in self.iactions.values(): try: ac.gui_layout_complete() except: import traceback traceback.print_exc() if ac.plugin_path is None: raise if config['autolaunch_server']: self.start_content_server() self.read_settings() self.finalize_layout() self.bars_manager.start_animation() self.set_window_title() for ac in self.iactions.values(): try: ac.initialization_complete() except: import traceback traceback.print_exc() if ac.plugin_path is None: raise self.set_current_library_information(current_library_name(), db.library_id, db.field_metadata) register_keyboard_shortcuts() self.keyboard.finalize() if show_gui: # Note this has to come after restoreGeometry() because of # https://bugreports.qt.io/browse/QTBUG-56831 self.show() if self.system_tray_icon is not None and self.system_tray_icon.isVisible() and opts.start_in_tray: self.hide_windows() self.auto_adder = AutoAdder(gprefs['auto_add_path'], self) # Now that the gui is initialized we can restore the quickview state # The same thing will be true for any action-based operation with a # layout button from calibre.gui2.actions.show_quickview import get_quickview_action_plugin qv = get_quickview_action_plugin() if qv: qv.qv_button.restore_state() self.save_layout_state() # Collect cycles now gc.collect() QApplication.instance().shutdown_signal_received.connect(self.quit) if show_gui and self.gui_debug is not None: QTimer.singleShot(10, self.show_gui_debug_msg) self.iactions['Connect Share'].check_smartdevice_menus() QTimer.singleShot(1, self.start_smartdevice) QTimer.singleShot(100, self.update_toggle_to_tray_action) def show_gui_debug_msg(self): info_dialog(self, _('Debug mode'), '<p>' + _('You have started calibre in debug mode. After you ' 'quit calibre, the debug log will be available in ' 'the file: %s<p>The ' 'log will be displayed automatically.')%self.gui_debug, show=True) def esc(self, *args): self.search.clear() def shift_esc(self): self.current_view().setFocus(Qt.OtherFocusReason) def ctrl_esc(self): self.apply_virtual_library() self.current_view().setFocus(Qt.OtherFocusReason) def start_smartdevice(self): message = None if self.device_manager.get_option('smartdevice', 'autostart'): try: message = self.device_manager.start_plugin('smartdevice') except: message = 'start smartdevice unknown exception' prints(message) import traceback traceback.print_exc() if message: if not self.device_manager.is_running('Wireless Devices'): error_dialog(self, _('Problem starting the wireless device'), _('The wireless device driver had problems starting. ' 'It said "%s"')%message, show=True) self.iactions['Connect Share'].set_smartdevice_action_state() def start_content_server(self, check_started=True): from calibre.srv.embedded import Server if not gprefs.get('server3_warning_done', False): gprefs.set('server3_warning_done', True) if os.path.exists(os.path.join(config_dir, 'server.py')): try: os.remove(os.path.join(config_dir, 'server.py')) except EnvironmentError: pass warning_dialog(self, _('Content server changed!'), _( 'calibre 3 comes with a completely re-written content server.' ' As such any custom configuration you have for the content' ' server no longer applies. You should check and refresh your' ' settings in Preferences->Sharing->Sharing over the net'), show=True) self.content_server = Server(self.library_broker, Dispatcher(self.handle_changes_from_server)) self.content_server.state_callback = Dispatcher( self.iactions['Connect Share'].content_server_state_changed) if check_started: self.content_server.start_failure_callback = \ Dispatcher(self.content_server_start_failed) self.content_server.start() def handle_changes_from_server(self, library_path, change_event): if DEBUG: prints('Received server change event: {} for {}'.format(change_event, library_path)) if self.library_broker.is_gui_library(library_path): self.server_changes.put((library_path, change_event)) self.server_change_notification_timer.start() def handle_changes_from_server_debounced(self): if self.shutting_down: return changes = [] while True: try: library_path, change_event = self.server_changes.get_nowait() except Empty: break if self.library_broker.is_gui_library(library_path): changes.append(change_event) if changes: handle_changes(changes, self) def content_server_start_failed(self, msg): self.content_server = None error_dialog(self, _('Failed to start Content server'), _('Could not start the Content server. Error:\n\n%s')%msg, show=True) def resizeEvent(self, ev): MainWindow.resizeEvent(self, ev) self.search.setMaximumWidth(self.width()-150) def create_spare_pool(self, *args): if self._spare_pool is None: num = min(detect_ncpus(), int(config['worker_limit']/2.0)) self._spare_pool = Pool(max_workers=num, name='GUIPool') def spare_pool(self): ans, self._spare_pool = self._spare_pool, None QTimer.singleShot(1000, self.create_spare_pool) return ans def do_proceed(self, func, payload): if callable(func): func(payload) def no_op(self, *args): pass def system_tray_icon_activated(self, r=False): if r in (QSystemTrayIcon.Trigger, QSystemTrayIcon.MiddleClick, False): if self.isVisible(): if self.isMinimized(): self.showNormal() else: self.hide_windows() else: self.show_windows() if self.isMinimized(): self.showNormal() @property def is_minimized_to_tray(self): return getattr(self, '__systray_minimized', False) def ask_a_yes_no_question(self, title, msg, det_msg='', show_copy_button=False, ans_when_user_unavailable=True, skip_dialog_name=None, skipped_value=True): if self.is_minimized_to_tray: return ans_when_user_unavailable return question_dialog(self, title, msg, det_msg=det_msg, show_copy_button=show_copy_button, skip_dialog_name=skip_dialog_name, skip_dialog_skipped_value=skipped_value) def update_toggle_to_tray_action(self, *args): if hasattr(self, 'toggle_to_tray_action'): self.toggle_to_tray_action.setText( _('Hide main window') if self.isVisible() else _('Show main window')) def hide_windows(self): for window in QApplication.topLevelWidgets(): if isinstance(window, (MainWindow, QDialog)) and \ window.isVisible(): window.hide() setattr(window, '__systray_minimized', True) self.update_toggle_to_tray_action() def show_windows(self, *args): for window in QApplication.topLevelWidgets(): if getattr(window, '__systray_minimized', False): window.show() setattr(window, '__systray_minimized', False) self.update_toggle_to_tray_action() def test_server(self, *args): if self.content_server is not None and \ self.content_server.exception is not None: error_dialog(self, _('Failed to start Content server'), unicode_type(self.content_server.exception)).exec_() @property def current_db(self): return self.library_view.model().db def refresh_all(self): m = self.library_view.model() m.db.data.refresh(clear_caches=False, do_search=False) self.saved_searches_changed(recount=False) m.resort() m.research() self.tags_view.recount() def handle_cli_args(self, args): if isinstance(args, string_or_bytes): args = [args] files = [os.path.abspath(p) for p in args if not os.path.isdir(p) and os.access(p, os.R_OK)] if files: self.iactions['Add Books'].add_filesystem_book(files) def another_instance_wants_to_talk(self): try: msg = self.listener.queue.get_nowait() except Empty: return if msg.startswith('launched:'): import json try: argv = json.loads(msg[len('launched:'):]) except ValueError: prints('Failed to decode message from other instance: %r' % msg) if DEBUG: error_dialog(self, 'Invalid message', 'Received an invalid message from other calibre instance.' ' Do you have multiple versions of calibre installed?', det_msg='Invalid msg: %r' % msg, show=True) argv = () if isinstance(argv, (list, tuple)) and len(argv) > 1: self.handle_cli_args(argv[1:]) self.setWindowState(self.windowState() & ~Qt.WindowMinimized|Qt.WindowActive) self.show_windows() self.raise_() self.activateWindow() elif msg.startswith('refreshdb:'): m = self.library_view.model() m.db.new_api.reload_from_db() self.refresh_all() elif msg.startswith('shutdown:'): self.quit(confirm_quit=False) elif msg.startswith('bookedited:'): parts = msg.split(':')[1:] try: book_id, fmt, library_id = parts[:3] book_id = int(book_id) m = self.library_view.model() db = m.db.new_api if m.db.library_id == library_id and db.has_id(book_id): db.format_metadata(book_id, fmt, allow_cache=False, update_db=True) db.update_last_modified((book_id,)) m.refresh_ids((book_id,)) except Exception: import traceback traceback.print_exc() else: print(msg) def current_view(self): '''Convenience method that returns the currently visible view ''' idx = self.stack.currentIndex() if idx == 0: return self.library_view if idx == 1: return self.memory_view if idx == 2: return self.card_a_view if idx == 3: return self.card_b_view def booklists(self): return self.memory_view.model().db, self.card_a_view.model().db, self.card_b_view.model().db def library_moved(self, newloc, copy_structure=False, allow_rebuild=False): if newloc is None: return with self.library_broker: default_prefs = None try: olddb = self.library_view.model().db if copy_structure: default_prefs = olddb.prefs except: olddb = None if copy_structure and olddb is not None and default_prefs is not None: default_prefs['field_metadata'] = olddb.new_api.field_metadata.all_metadata() db = self.library_broker.prepare_for_gui_library_change(newloc) if db is None: try: db = LibraryDatabase(newloc, default_prefs=default_prefs) except apsw.Error: if not allow_rebuild: raise import traceback repair = question_dialog(self, _('Corrupted database'), _('The library database at %s appears to be corrupted. Do ' 'you want calibre to try and rebuild it automatically? ' 'The rebuild may not be completely successful.') % force_unicode(newloc, filesystem_encoding), det_msg=traceback.format_exc() ) if repair: from calibre.gui2.dialogs.restore_library import repair_library_at if repair_library_at(newloc, parent=self): db = LibraryDatabase(newloc, default_prefs=default_prefs) else: return else: return self.library_path = newloc prefs['library_path'] = self.library_path self.book_on_device(None, reset=True) db.set_book_on_device_func(self.book_on_device) self.library_view.set_database(db) self.tags_view.set_database(db, self.alter_tb) self.library_view.model().set_book_on_device_func(self.book_on_device) self.status_bar.clear_message() self.search.clear() self.saved_search.clear() self.book_details.reset_info() # self.library_view.model().count_changed() db = self.library_view.model().db self.iactions['Choose Library'].count_changed(db.count()) self.set_window_title() self.apply_named_search_restriction('') # reset restriction to null self.saved_searches_changed(recount=False) # reload the search restrictions combo box if db.prefs['virtual_lib_on_startup']: self.apply_virtual_library(db.prefs['virtual_lib_on_startup']) self.rebuild_vl_tabs() for action in self.iactions.values(): action.library_changed(db) self.library_broker.gui_library_changed(db, olddb) if self.device_connected: self.set_books_in_library(self.booklists(), reset=True) self.refresh_ondevice() self.memory_view.reset() self.card_a_view.reset() self.card_b_view.reset() self.set_current_library_information(current_library_name(), db.library_id, db.field_metadata) self.library_view.set_current_row(0) # Run a garbage collection now so that it does not freeze the # interface later gc.collect() def set_window_title(self): db = self.current_db restrictions = [x for x in (db.data.get_base_restriction_name(), db.data.get_search_restriction_name()) if x] restrictions = ' :: '.join(restrictions) font = QFont() if restrictions: restrictions = ' :: ' + restrictions font.setBold(True) font.setItalic(True) self.virtual_library.setFont(font) title = u'{0} - || {1}{2} ||'.format( __appname__, self.iactions['Choose Library'].library_name(), restrictions) self.setWindowTitle(title) def location_selected(self, location): ''' Called when a location icon is clicked (e.g. Library) ''' page = 0 if location == 'library' else 1 if location == 'main' else 2 if location == 'carda' else 3 self.stack.setCurrentIndex(page) self.book_details.reset_info() for x in ('tb', 'cb'): splitter = getattr(self, x+'_splitter') splitter.button.setEnabled(location == 'library') for action in self.iactions.values(): action.location_selected(location) if location == 'library': self.virtual_library_menu.setEnabled(True) self.highlight_only_button.setEnabled(True) self.vl_tabs.setEnabled(True) else: self.virtual_library_menu.setEnabled(False) self.highlight_only_button.setEnabled(False) self.vl_tabs.setEnabled(False) # Reset the view in case something changed while it was invisible self.current_view().reset() self.set_number_of_books_shown() self.update_status_bar() def job_exception(self, job, dialog_title=_('Conversion error'), retry_func=None): if not hasattr(self, '_modeless_dialogs'): self._modeless_dialogs = [] minz = self.is_minimized_to_tray if self.isVisible(): for x in list(self._modeless_dialogs): if not x.isVisible(): self._modeless_dialogs.remove(x) try: if 'calibre.ebooks.DRMError' in job.details: if not minz: from calibre.gui2.dialogs.drm_error import DRMErrorMessage d = DRMErrorMessage(self, _('Cannot convert') + ' ' + job.description.split(':')[-1].partition('(')[-1][:-1]) d.setModal(False) d.show() self._modeless_dialogs.append(d) return if 'calibre.ebooks.oeb.transforms.split.SplitError' in job.details: title = job.description.split(':')[-1].partition('(')[-1][:-1] msg = _('<p><b>Failed to convert: %s')%title msg += '<p>'+_(''' Many older e-book reader devices are incapable of displaying EPUB files that have internal components over a certain size. Therefore, when converting to EPUB, calibre automatically tries to split up the EPUB into smaller sized pieces. For some files that are large undifferentiated blocks of text, this splitting fails. <p>You can <b>work around the problem</b> by either increasing the maximum split size under <i>EPUB output</i> in the conversion dialog, or by turning on Heuristic Processing, also in the conversion dialog. Note that if you make the maximum split size too large, your e-book reader may have trouble with the EPUB. ''') if not minz: d = error_dialog(self, _('Conversion Failed'), msg, det_msg=job.details) d.setModal(False) d.show() self._modeless_dialogs.append(d) return if 'calibre.ebooks.mobi.reader.mobi6.KFXError:' in job.details: if not minz: title = job.description.split(':')[-1].partition('(')[-1][:-1] msg = _('<p><b>Failed to convert: %s') % title idx = job.details.index('calibre.ebooks.mobi.reader.mobi6.KFXError:') msg += '<p>' + re.sub(r'(https:\S+)', r'<a href="\1">{}</a>'.format(_('here')), job.details[idx:].partition(':')[2].strip()) d = error_dialog(self, _('Conversion failed'), msg, det_msg=job.details) d.setModal(False) d.show() self._modeless_dialogs.append(d) return if 'calibre.web.feeds.input.RecipeDisabled' in job.details: if not minz: msg = job.details msg = msg[msg.find('calibre.web.feeds.input.RecipeDisabled:'):] msg = msg.partition(':')[-1] d = error_dialog(self, _('Recipe Disabled'), '<p>%s</p>'%msg) d.setModal(False) d.show() self._modeless_dialogs.append(d) return if 'calibre.ebooks.conversion.ConversionUserFeedBack:' in job.details: if not minz: import json payload = job.details.rpartition( 'calibre.ebooks.conversion.ConversionUserFeedBack:')[-1] payload = json.loads('{' + payload.partition('{')[-1]) d = {'info':info_dialog, 'warn':warning_dialog, 'error':error_dialog}.get(payload['level'], error_dialog) d = d(self, payload['title'], '<p>%s</p>'%payload['msg'], det_msg=payload['det_msg']) d.setModal(False) d.show() self._modeless_dialogs.append(d) return except: pass if job.killed: return try: prints(job.details, file=sys.stderr) except: pass if not minz: self.job_error_dialog.show_error(dialog_title, _('<b>Failed</b>')+': '+unicode_type(job.description), det_msg=job.details, retry_func=retry_func) def read_settings(self): geometry = config['main_window_geometry'] if geometry is not None: self.restoreGeometry(geometry) self.read_layout_settings() def write_settings(self): with gprefs: # Only write to gprefs once config.set('main_window_geometry', self.saveGeometry()) dynamic.set('sort_history', self.library_view.model().sort_history) self.save_layout_state() self.stack.tb_widget.save_state() def quit(self, checked=True, restart=False, debug_on_restart=False, confirm_quit=True): if self.shutting_down: return if confirm_quit and not self.confirm_quit(): return try: self.shutdown() except: pass self.restart_after_quit = restart self.debug_on_restart = debug_on_restart if self.system_tray_icon is not None and self.restart_after_quit: # Needed on windows to prevent multiple systray icons self.system_tray_icon.setVisible(False) QApplication.instance().quit() def donate(self, *args): from calibre.utils.localization import localize_website_link open_url(QUrl(localize_website_link('https://calibre-ebook.com/donate'))) def confirm_quit(self): if self.job_manager.has_jobs(): msg = _('There are active jobs. Are you sure you want to quit?') if self.job_manager.has_device_jobs(): msg = '<p>'+__appname__ + \ _(''' is communicating with the device!<br> Quitting may cause corruption on the device.<br> Are you sure you want to quit?''')+'</p>' if not question_dialog(self, _('Active jobs'), msg): return False if self.proceed_question.questions: msg = _('There are library updates waiting. Are you sure you want to quit?') if not question_dialog(self, _('Library updates waiting'), msg): return False from calibre.db.delete_service import has_jobs if has_jobs(): msg = _('Some deleted books are still being moved to the Recycle ' 'Bin, if you quit now, they will be left behind. Are you ' 'sure you want to quit?') if not question_dialog(self, _('Active jobs'), msg): return False return True def shutdown(self, write_settings=True): self.shutting_down = True self.show_shutdown_message() self.server_change_notification_timer.stop() from calibre.customize.ui import has_library_closed_plugins if has_library_closed_plugins(): self.show_shutdown_message( _('Running database shutdown plugins. This could take a few seconds...')) self.grid_view.shutdown() db = None try: db = self.library_view.model().db cf = db.clean except: pass else: cf() # Save the current field_metadata for applications like calibre2opds # Goes here, because if cf is valid, db is valid. db.new_api.set_pref('field_metadata', db.field_metadata.all_metadata()) db.commit_dirty_cache() db.prefs.write_serialized(prefs['library_path']) for action in self.iactions.values(): if not action.shutting_down(): return if write_settings: self.write_settings() self.check_messages_timer.stop() if getattr(self, 'update_checker', None): self.update_checker.shutdown() self.listener.close() self.job_manager.server.close() self.job_manager.threaded_server.close() self.device_manager.keep_going = False self.auto_adder.stop() # Do not report any errors that happen after the shutdown # We cannot restore the original excepthook as that causes PyQt to # call abort() on unhandled exceptions import traceback def eh(t, v, tb): try: traceback.print_exception(t, v, tb, file=sys.stderr) except: pass sys.excepthook = eh mb = self.library_view.model().metadata_backup if mb is not None: mb.stop() self.library_view.model().close() try: try: if self.content_server is not None: # If the Content server has any sockets being closed then # this can take quite a long time (minutes). Tell the user that it is # happening. self.show_shutdown_message( _('Shutting down the Content server. This could take a while...')) s = self.content_server self.content_server = None s.exit() except: pass except KeyboardInterrupt: pass self.hide_windows() if self._spare_pool is not None: self._spare_pool.shutdown() from calibre.db.delete_service import shutdown shutdown() time.sleep(2) self.istores.join() return True def run_wizard(self, *args): if self.confirm_quit(): self.run_wizard_b4_shutdown = True self.restart_after_quit = True try: self.shutdown(write_settings=False) except: pass QApplication.instance().quit() def closeEvent(self, e): if self.shutting_down: return self.write_settings() if self.system_tray_icon is not None and self.system_tray_icon.isVisible(): if not dynamic['systray_msg'] and not isosx: info_dialog(self, 'calibre', 'calibre '+ _('will keep running in the system tray. To close it, ' 'choose <b>Quit</b> in the context menu of the ' 'system tray.'), show_copy_button=False).exec_() dynamic['systray_msg'] = True self.hide_windows() e.ignore() else: if self.confirm_quit(): try: self.shutdown(write_settings=False) except: import traceback traceback.print_exc() e.accept() else: e.ignore()
class Server(Thread): def __init__(self, notify_on_job_done=lambda x: x, pool_size=None, limit=sys.maxsize, enforce_cpu_limit=True): Thread.__init__(self) self.daemon = True self.id = next(server_counter) + 1 if enforce_cpu_limit: limit = min(limit, cpu_count()) self.pool_size = limit if pool_size is None else pool_size self.notify_on_job_done = notify_on_job_done self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue() self.kill_queue = Queue() self.waiting_jobs = [] self.workers = deque() self.launched_worker_counter = count() next(self.launched_worker_counter) self.start() def launch_worker(self, gui=False, redirect_output=None, job_name=None): start = time.monotonic() id = next(self.launched_worker_counter) fd, rfile = tempfile.mkstemp(prefix='ipc_result_%d_%d_' % (self.id, id), dir=base_dir(), suffix='.pickle') os.close(fd) if redirect_output is None: redirect_output = not gui cw = self.do_launch(gui, redirect_output, rfile, job_name=job_name) if isinstance(cw, string_or_bytes): raise CriticalError('Failed to launch worker process:\n' + force_unicode(cw)) if DEBUG: print( f'Worker Launch took: {time.monotonic() - start:.2f} seconds') return cw def do_launch(self, gui, redirect_output, rfile, job_name=None): a, b = Pipe() with a: env = { 'CALIBRE_WORKER_FD': str(a.fileno()), 'CALIBRE_WORKER_RESULT': environ_item(as_hex_unicode(rfile)) } w = Worker(env, gui=gui, job_name=job_name) try: w(pass_fds=(a.fileno(), ), redirect_output=redirect_output) except BaseException: try: w.kill() except: pass b.close() import traceback return traceback.format_exc() return ConnectedWorker(w, b, rfile) def add_job(self, job): job.done2 = self.notify_on_job_done self.add_jobs_queue.put(job) def run_job(self, job, gui=True, redirect_output=False): w = self.launch_worker(gui=gui, redirect_output=redirect_output, job_name=getattr(job, 'name', None)) w.start_job(job) def run(self): while True: try: job = self.add_jobs_queue.get(True, 0.2) if job is None: break self.waiting_jobs.insert(0, job) except Empty: pass # Get notifications from worker process for worker in self.workers: while True: try: n = worker.notifications.get_nowait() worker.job.notifications.put(n) self.changed_jobs_queue.put(worker.job) except Empty: break # Remove finished jobs for worker in [w for w in self.workers if not w.is_alive]: try: worker.close_log_file() except: pass self.workers.remove(worker) job = worker.job if worker.returncode != 0: job.failed = True job.returncode = worker.returncode elif os.path.exists(worker.rfile): try: with lopen(worker.rfile, 'rb') as f: job.result = pickle_loads(f.read()) os.remove(worker.rfile) except: pass job.duration = time.time() - job.start_time self.changed_jobs_queue.put(job) # Start waiting jobs sj = self.suitable_waiting_job() if sj is not None: job = self.waiting_jobs.pop(sj) job.start_time = time.time() if job.kill_on_start: job.duration = 0.0 job.returncode = 1 job.killed = job.failed = True job.result = None else: worker = self.launch_worker() worker.start_job(job) self.workers.append(worker) job.log_path = worker.log_path self.changed_jobs_queue.put(job) while True: try: j = self.kill_queue.get_nowait() self._kill_job(j) except Empty: break def suitable_waiting_job(self): available_workers = self.pool_size - len(self.workers) for worker in self.workers: job = worker.job if job.core_usage == -1: available_workers = 0 elif job.core_usage > 1: available_workers -= job.core_usage - 1 if available_workers < 1: return None for i, job in enumerate(self.waiting_jobs): if job.core_usage == -1: if available_workers >= self.pool_size: return i elif job.core_usage <= available_workers: return i def kill_job(self, job): self.kill_queue.put(job) def killall(self): for worker in self.workers: self.kill_queue.put(worker.job) def _kill_job(self, job): if job.start_time is None: job.kill_on_start = True return for worker in self.workers: if job is worker.job: worker.kill() job.killed = True break def split(self, tasks): ''' Split a list into a list of sub lists, with the number of sub lists being no more than the number of workers this server supports. Each sublist contains 2-tuples of the form (i, x) where x is an element from the original list and i is the index of the element x in the original list. ''' ans, count, pos = [], 0, 0 delta = int(ceil(len(tasks) / float(self.pool_size))) while count < len(tasks): section = [] for t in tasks[pos:pos + delta]: section.append((count, t)) count += 1 ans.append(section) pos += delta return ans def close(self): try: self.add_jobs_queue.put(None) except: pass try: self.listener.close() except: pass time.sleep(0.2) for worker in list(self.workers): try: worker.kill() except: pass def __enter__(self): return self def __exit__(self, *args): self.close()
class GenericDownloadThreadPool(object): ''' add_task must be implemented in a subclass and must GenericDownloadThreadPool.add_task must be called at the end of the function. ''' def __init__(self, thread_type, thread_count=1): self.thread_type = thread_type self.thread_count = thread_count self.tasks = Queue() self.results = Queue() self.threads = [] def set_thread_count(self, thread_count): self.thread_count = thread_count def add_task(self): ''' This must be implemented in a sub class and this function must be called at the end of the add_task function in the sub class. The implementation of this function (in this base class) starts any threads necessary to fill the pool if it is not already full. ''' for i in range(self.thread_count - self.running_threads_count()): t = self.thread_type(self.tasks, self.results) self.threads.append(t) t.start() def abort(self): self.tasks = Queue() self.results = Queue() for t in self.threads: t.abort() self.threads = [] def has_tasks(self): return not self.tasks.empty() def get_result(self): return self.results.get() def get_result_no_wait(self): return self.results.get_nowait() def result_count(self): return len(self.results) def has_results(self): return not self.results.empty() def threads_running(self): return self.running_threads_count() > 0 def running_threads_count(self): count = 0 for t in self.threads: if t.is_alive(): count += 1 return count
class GenericDownloadThreadPool(object): ''' add_task must be implemented in a subclass and must GenericDownloadThreadPool.add_task must be called at the end of the function. ''' def __init__(self, thread_type, thread_count=1): self.thread_type = thread_type self.thread_count = thread_count self.tasks = Queue() self.results = Queue() self.threads = [] def set_thread_count(self, thread_count): self.thread_count = thread_count def add_task(self): ''' This must be implemented in a sub class and this function must be called at the end of the add_task function in the sub class. The implementation of this function (in this base class) starts any threads necessary to fill the pool if it is not already full. ''' for i in range(self.thread_count - self.running_threads_count()): t = self.thread_type(self.tasks, self.results) self.threads.append(t) t.start() def abort(self): self.tasks = Queue() self.results = Queue() for t in self.threads: t.abort() self.threads = [] def has_tasks(self): return not self.tasks.empty() def get_result(self): return self.results.get() def get_result_no_wait(self): return self.results.get_nowait() def result_count(self): return len(self.results) def has_results(self): return not self.results.empty() def threads_running(self): return self.running_threads_count() > 0 def running_threads_count(self): count = 0 for t in self.threads: if t.is_alive(): count += 1 return count
class BaseJob(object): WAITING = 0 RUNNING = 1 FINISHED = 2 def __init__(self, description, done=lambda x: x): self.id = next(job_counter) self.description = description self.done = done self.done2 = None self.killed = False self.failed = False self.kill_on_start = False self.start_time = None self.result = None self.duration = None self.log_path = None self.notifications = Queue() self._run_state = self.WAITING self.percent = 0 self._message = None self._status_text = _('Waiting...') self._done_called = False self.core_usage = 1 self.timed_out = False def update(self, consume_notifications=True): if self.duration is not None: self._run_state = self.FINISHED self.percent = 100 if self.killed: if self.timed_out: self._status_text = _('Aborted, taking too long') else: self._status_text = _('Stopped') else: self._status_text = _('Error') if self.failed else _('Finished') if DEBUG: try: prints('Job:', self.id, self.description, 'finished', safe_encode=True) prints('\t'.join(self.details.splitlines(True)), safe_encode=True) except: pass if not self._done_called: self._done_called = True try: self.done(self) except: pass try: if callable(self.done2): self.done2(self) except: pass elif self.start_time is not None: self._run_state = self.RUNNING self._status_text = _('Working...') if consume_notifications: return self.consume_notifications() return False def consume_notifications(self): got_notification = False while self.notifications is not None: try: self.percent, self._message = self.notifications.get_nowait() self.percent *= 100. got_notification = True except Empty: break return got_notification @property def status_text(self): if self._run_state == self.FINISHED or not self._message: return self._status_text return self._message @property def run_state(self): return self._run_state @property def running_time(self): if self.duration is not None: return self.duration if self.start_time is not None: return time.time() - self.start_time return None @property def is_finished(self): return self._run_state == self.FINISHED @property def is_started(self): return self._run_state != self.WAITING @property def is_running(self): return self.is_started and not self.is_finished def __hash__(self): return id(self) def __eq__(self, other): return self is other def __ne__(self, other): return self is not other def __lt__(self, other): return self.compare_to_other(other) < 0 def __le__(self, other): return self.compare_to_other(other) <= 0 def __gt__(self, other): return self.compare_to_other(other) > 0 def __ge__(self, other): return self.compare_to_other(other) >= 0 def compare_to_other(self, other): if self.is_finished != other.is_finished: return 1 if self.is_finished else -1 if self.start_time is None: if other.start_time is None: # Both waiting return cmp(other.id, self.id) return 1 if other.start_time is None: return -1 # Both running return cmp((other.start_time, id(other)), (self.start_time, id(self))) @property def log_file(self): if self.log_path: return open(self.log_path, 'rb') return io.BytesIO(_('No details available.').encode('utf-8', 'replace')) @property def details(self): return self.log_file.read().decode('utf-8', 'replace')