class Watcher(WatcherBase):
def __init__(self, root_dirs, worker, log):
WatcherBase.__init__(self, worker, log)
self.stream = Stream(self.notify, *(x.encode('utf-8') for x in root_dirs), file_events=True)
self.wait_queue = Queue()
def wakeup(self):
self.wait_queue.put(True)
def loop(self):
observer = Observer()
observer.schedule(self.stream)
observer.daemon = True
observer.start()
try:
while True:
try:
# Cannot use blocking get() as it is not interrupted by
# Ctrl-C
if self.wait_queue.get(10000) is True:
self.force_restart()
except Empty:
pass
finally:
observer.unschedule(self.stream)
observer.stop()
def notify(self, ev):
name = ev.name
if isinstance(name, bytes):
name = name.decode('utf-8')
if self.file_is_watched(name):
self.handle_modified({name})
class DBThread(Thread):
CLOSE = '-------close---------'
def __init__(self, path, row_factory):
Thread.__init__(self)
self.setDaemon(True)
self.path = path
self.unhandled_error = (None, '')
self.row_factory = row_factory
self.requests = Queue(1)
self.results = Queue(1)
self.conn = None
def connect(self):
self.conn = do_connect(self.path, self.row_factory)
def run(self):
try:
self.connect()
while True:
func, args, kwargs = self.requests.get()
if func == self.CLOSE:
self.conn.close()
break
if func == 'dump':
try:
ok, res = True, tuple(self.conn.iterdump())
except Exception as err:
ok, res = False, (err, traceback.format_exc())
elif func == 'create_dynamic_filter':
try:
f = DynamicFilter(args[0])
self.conn.create_function(args[0], 1, f)
ok, res = True, f
except Exception as err:
ok, res = False, (err, traceback.format_exc())
else:
bfunc = getattr(self.conn, func)
try:
for i in range(3):
try:
ok, res = True, bfunc(*args, **kwargs)
break
except OperationalError as err:
# Retry if unable to open db file
e = str(err)
if 'unable to open' not in e or i == 2:
if 'unable to open' in e:
prints('Unable to open database for func',
func, reprlib.repr(args),
reprlib.repr(kwargs))
raise
time.sleep(0.5)
except Exception as err:
ok, res = False, (err, traceback.format_exc())
self.results.put((ok, res))
except Exception as err:
self.unhandled_error = (err, traceback.format_exc())
class ConnectedWorker(Thread):
def __init__(self, worker, conn, rfile):
Thread.__init__(self)
self.daemon = True
self.conn = conn
self.worker = worker
self.notifications = Queue()
self._returncode = 'dummy'
self.killed = False
self.log_path = worker.log_path
self.rfile = rfile
self.close_log_file = getattr(worker, 'close_log_file', None)
def start_job(self, job):
notification = PARALLEL_FUNCS[job.name][-1] is not None
eintr_retry_call(self.conn.send, (job.name, job.args, job.kwargs, job.description))
if notification:
self.start()
else:
self.conn.close()
self.job = job
def run(self):
while True:
try:
x = eintr_retry_call(self.conn.recv)
self.notifications.put(x)
except BaseException:
break
try:
self.conn.close()
except BaseException:
pass
def kill(self):
self.killed = True
try:
self.worker.kill()
except BaseException:
pass
@property
def is_alive(self):
return not self.killed and self.worker.is_alive
@property
def returncode(self):
if self._returncode != 'dummy':
return self._returncode
r = self.worker.returncode
if self.killed and r is None:
self._returncode = 1
return 1
if r is not None:
self._returncode = r
return r
def compress_images(container, report=None, names=None, jpeg_quality=None, progress_callback=lambda n, t, name:True):
images = get_compressible_images(container)
if names is not None:
images &= set(names)
results = {}
queue = Queue()
abort = Event()
for name in images:
queue.put(name)
def pc(name):
keep_going = progress_callback(len(results), len(images), name)
if not keep_going:
abort.set()
progress_callback(0, len(images), '')
[Worker(abort, 'CompressImage%d' % i, queue, results, container, jpeg_quality, pc) for i in range(min(detect_ncpus(), len(images)))]
queue.join()
before_total = after_total = 0
changed = False
for name, (ok, res) in iteritems(results):
name = force_unicode(name, filesystem_encoding)
if ok:
before, after = res
if before != after:
changed = True
before_total += before
after_total += after
if report:
if before != after:
report(_('{0} compressed from {1} to {2} bytes [{3:.1%} reduction]').format(
name, human_readable(before), human_readable(after), (before - after)/before))
else:
report(_('{0} could not be further compressed').format(name))
else:
report(_('Failed to process {0} with error:').format(name))
report(res)
if report:
if changed:
report('')
report(_('Total image filesize reduced from {0} to {1} [{2:.1%} reduction]').format(
human_readable(before_total), human_readable(after_total), (before_total - after_total)/before_total))
else:
report(_('Images are already fully optimized'))
return changed, results
class Progress(Thread):
def __init__(self, conn):
Thread.__init__(self)
self.daemon = True
self.conn = conn
self.queue = Queue()
def __call__(self, percent, msg=''):
self.queue.put((percent, msg))
def run(self):
while True:
x = self.queue.get()
if x is None:
break
try:
eintr_retry_call(self.conn.send, x)
except:
break
class Progress(Thread):
def __init__(self, conn):
Thread.__init__(self)
self.daemon = True
self.conn = conn
self.queue = Queue()
def __call__(self, percent, msg=''):
self.queue.put((percent, msg))
def run(self):
while True:
x = self.queue.get()
if x is None:
break
try:
eintr_retry_call(self.conn.send, x)
except:
break
class Watcher(WatcherBase):
def __init__(self, root_dirs, worker, log):
WatcherBase.__init__(self, worker, log)
self.watchers = []
self.modified_queue = Queue()
for d in frozenset(root_dirs):
self.watchers.append(TreeWatcher(d, self.modified_queue))
def wakeup(self):
self.modified_queue.put(True)
def loop(self):
for w in self.watchers:
w.start()
with HandleInterrupt(lambda: self.modified_queue.put(None)):
while True:
path = self.modified_queue.get()
if path is None:
break
if path is True:
self.force_restart()
else:
self.handle_modified({path})
class Watcher(WatcherBase):
def __init__(self, root_dirs, worker, log):
WatcherBase.__init__(self, worker, log)
self.watchers = []
self.modified_queue = Queue()
for d in frozenset(root_dirs):
self.watchers.append(TreeWatcher(d, self.modified_queue))
def wakeup(self):
self.modified_queue.put(True)
def loop(self):
for w in self.watchers:
w.start()
with HandleInterrupt(lambda : self.modified_queue.put(None)):
while True:
path = self.modified_queue.get()
if path is None:
break
if path is True:
self.force_restart()
else:
self.handle_modified({path})
class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
TagBrowserMixin, CoverFlowMixin, LibraryViewMixin, SearchBoxMixin,
SavedSearchBoxMixin, SearchRestrictionMixin, LayoutMixin, UpdateMixin,
EbookDownloadMixin
):
'The main GUI'
proceed_requested = pyqtSignal(object, object)
book_converted = pyqtSignal(object, object)
shutting_down = False
def __init__(self, opts, parent=None, gui_debug=None):
MainWindow.__init__(self, opts, parent=parent, disable_automatic_gc=True)
self.setWindowIcon(QApplication.instance().windowIcon())
self.jobs_pointer = Pointer(self)
self.proceed_requested.connect(self.do_proceed,
type=Qt.QueuedConnection)
self.proceed_question = ProceedQuestion(self)
self.job_error_dialog = JobError(self)
self.keyboard = Manager(self)
get_gui.ans = self
self.opts = opts
self.device_connected = None
self.gui_debug = gui_debug
self.iactions = OrderedDict()
# Actions
for action in interface_actions():
if opts.ignore_plugins and action.plugin_path is not None:
continue
try:
ac = self.init_iaction(action)
except:
# Ignore errors in loading user supplied plugins
import traceback
traceback.print_exc()
if action.plugin_path is None:
raise
continue
ac.plugin_path = action.plugin_path
ac.interface_action_base_plugin = action
self.add_iaction(ac)
self.load_store_plugins()
def init_iaction(self, action):
ac = action.load_actual_plugin(self)
ac.plugin_path = action.plugin_path
ac.interface_action_base_plugin = action
action.actual_iaction_plugin_loaded = True
return ac
def add_iaction(self, ac):
acmap = self.iactions
if ac.name in acmap:
if ac.priority >= acmap[ac.name].priority:
acmap[ac.name] = ac
else:
acmap[ac.name] = ac
def load_store_plugins(self):
from calibre.gui2.store.loader import Stores
self.istores = Stores()
for store in available_store_plugins():
if self.opts.ignore_plugins and store.plugin_path is not None:
continue
try:
st = self.init_istore(store)
self.add_istore(st)
except:
# Ignore errors in loading user supplied plugins
import traceback
traceback.print_exc()
if store.plugin_path is None:
raise
continue
self.istores.builtins_loaded()
def init_istore(self, store):
st = store.load_actual_plugin(self)
st.plugin_path = store.plugin_path
st.base_plugin = store
store.actual_istore_plugin_loaded = True
return st
def add_istore(self, st):
stmap = self.istores
if st.name in stmap:
if st.priority >= stmap[st.name].priority:
stmap[st.name] = st
else:
stmap[st.name] = st
def initialize(self, library_path, db, listener, actions, show_gui=True):
opts = self.opts
self.preferences_action, self.quit_action = actions
self.library_path = library_path
self.library_broker = GuiLibraryBroker(db)
self.content_server = None
self.server_change_notification_timer = t = QTimer(self)
self.server_changes = Queue()
t.setInterval(1000), t.timeout.connect(self.handle_changes_from_server_debounced), t.setSingleShot(True)
self._spare_pool = None
self.must_restart_before_config = False
self.listener = Listener(listener)
self.check_messages_timer = QTimer()
self.check_messages_timer.timeout.connect(self.another_instance_wants_to_talk)
self.check_messages_timer.start(1000)
for ac in self.iactions.values():
try:
ac.do_genesis()
except Exception:
# Ignore errors in third party plugins
import traceback
traceback.print_exc()
if getattr(ac, 'plugin_path', None) is None:
raise
self.donate_action = QAction(QIcon(I('donate.png')),
_('&Donate to support calibre'), self)
for st in self.istores.values():
st.do_genesis()
MainWindowMixin.init_main_window_mixin(self, db)
# Jobs Button {{{
self.job_manager = JobManager()
self.jobs_dialog = JobsDialog(self, self.job_manager)
self.jobs_button = JobsButton(parent=self)
self.jobs_button.initialize(self.jobs_dialog, self.job_manager)
# }}}
LayoutMixin.init_layout_mixin(self)
DeviceMixin.init_device_mixin(self)
self.progress_indicator = ProgressIndicator(self)
self.progress_indicator.pos = (0, 20)
self.verbose = opts.verbose
self.get_metadata = GetMetadata()
self.upload_memory = {}
self.metadata_dialogs = []
self.default_thumbnail = None
self.tb_wrapper = textwrap.TextWrapper(width=40)
self.viewers = collections.deque()
self.system_tray_icon = None
do_systray = config['systray_icon'] or opts.start_in_tray
if do_systray:
self.system_tray_icon = factory(app_id='com.calibre-ebook.gui').create_system_tray_icon(parent=self, title='calibre')
if self.system_tray_icon is not None:
self.system_tray_icon.setIcon(QIcon(I('lt.png', allow_user_override=False)))
if not (iswindows or isosx):
self.system_tray_icon.setIcon(QIcon.fromTheme('calibre-tray', self.system_tray_icon.icon()))
self.system_tray_icon.setToolTip(self.jobs_button.tray_tooltip())
self.system_tray_icon.setVisible(True)
self.jobs_button.tray_tooltip_updated.connect(self.system_tray_icon.setToolTip)
elif do_systray:
prints('Failed to create system tray icon, your desktop environment probably'
' does not support the StatusNotifier spec https://www.freedesktop.org/wiki/Specifications/StatusNotifierItem/')
self.system_tray_menu = QMenu(self)
self.toggle_to_tray_action = self.system_tray_menu.addAction(QIcon(I('page.png')), '')
self.toggle_to_tray_action.triggered.connect(self.system_tray_icon_activated)
self.system_tray_menu.addAction(self.donate_action)
self.eject_action = self.system_tray_menu.addAction(
QIcon(I('eject.png')), _('&Eject connected device'))
self.eject_action.setEnabled(False)
self.addAction(self.quit_action)
self.system_tray_menu.addAction(self.quit_action)
self.keyboard.register_shortcut('quit calibre', _('Quit calibre'),
default_keys=('Ctrl+Q',), action=self.quit_action)
if self.system_tray_icon is not None:
self.system_tray_icon.setContextMenu(self.system_tray_menu)
self.system_tray_icon.activated.connect(self.system_tray_icon_activated)
self.quit_action.triggered[bool].connect(self.quit)
self.donate_action.triggered[bool].connect(self.donate)
self.minimize_action = QAction(_('Minimize the calibre window'), self)
self.addAction(self.minimize_action)
self.keyboard.register_shortcut('minimize calibre', self.minimize_action.text(),
default_keys=(), action=self.minimize_action)
self.minimize_action.triggered.connect(self.showMinimized)
self.esc_action = QAction(self)
self.addAction(self.esc_action)
self.keyboard.register_shortcut('clear current search',
_('Clear the current search'), default_keys=('Esc',),
action=self.esc_action)
self.esc_action.triggered.connect(self.esc)
self.shift_esc_action = QAction(self)
self.addAction(self.shift_esc_action)
self.keyboard.register_shortcut('focus book list',
_('Focus the book list'), default_keys=('Shift+Esc',),
action=self.shift_esc_action)
self.shift_esc_action.triggered.connect(self.shift_esc)
self.ctrl_esc_action = QAction(self)
self.addAction(self.ctrl_esc_action)
self.keyboard.register_shortcut('clear virtual library',
_('Clear the virtual library'), default_keys=('Ctrl+Esc',),
action=self.ctrl_esc_action)
self.ctrl_esc_action.triggered.connect(self.ctrl_esc)
self.alt_esc_action = QAction(self)
self.addAction(self.alt_esc_action)
self.keyboard.register_shortcut('clear additional restriction',
_('Clear the additional restriction'), default_keys=('Alt+Esc',),
action=self.alt_esc_action)
self.alt_esc_action.triggered.connect(self.clear_additional_restriction)
# ###################### Start spare job server ########################
QTimer.singleShot(1000, self.create_spare_pool)
# ###################### Location Manager ########################
self.location_manager.location_selected.connect(self.location_selected)
self.location_manager.unmount_device.connect(self.device_manager.umount_device)
self.location_manager.configure_device.connect(self.configure_connected_device)
self.location_manager.update_device_metadata.connect(self.update_metadata_on_device)
self.eject_action.triggered.connect(self.device_manager.umount_device)
# ################### Update notification ###################
UpdateMixin.init_update_mixin(self, opts)
# ###################### Search boxes ########################
SearchRestrictionMixin.init_search_restriction_mixin(self)
SavedSearchBoxMixin.init_saved_seach_box_mixin(self)
# ###################### Library view ########################
LibraryViewMixin.init_library_view_mixin(self, db)
SearchBoxMixin.init_search_box_mixin(self) # Requires current_db
self.library_view.model().count_changed_signal.connect(
self.iactions['Choose Library'].count_changed)
if not gprefs.get('quick_start_guide_added', False):
try:
add_quick_start_guide(self.library_view)
except:
import traceback
traceback.print_exc()
for view in ('library', 'memory', 'card_a', 'card_b'):
v = getattr(self, '%s_view' % view)
v.selectionModel().selectionChanged.connect(self.update_status_bar)
v.model().count_changed_signal.connect(self.update_status_bar)
self.library_view.model().count_changed()
self.bars_manager.database_changed(self.library_view.model().db)
self.library_view.model().database_changed.connect(self.bars_manager.database_changed,
type=Qt.QueuedConnection)
# ########################## Tags Browser ##############################
TagBrowserMixin.init_tag_browser_mixin(self, db)
self.library_view.model().database_changed.connect(self.populate_tb_manage_menu, type=Qt.QueuedConnection)
# ######################## Search Restriction ##########################
if db.prefs['virtual_lib_on_startup']:
self.apply_virtual_library(db.prefs['virtual_lib_on_startup'])
self.rebuild_vl_tabs()
# ########################## Cover Flow ################################
CoverFlowMixin.init_cover_flow_mixin(self)
self._calculated_available_height = min(max_available_height()-15,
self.height())
self.resize(self.width(), self._calculated_available_height)
self.build_context_menus()
for ac in self.iactions.values():
try:
ac.gui_layout_complete()
except:
import traceback
traceback.print_exc()
if ac.plugin_path is None:
raise
if config['autolaunch_server']:
self.start_content_server()
self.read_settings()
self.finalize_layout()
self.bars_manager.start_animation()
self.set_window_title()
for ac in self.iactions.values():
try:
ac.initialization_complete()
except:
import traceback
traceback.print_exc()
if ac.plugin_path is None:
raise
self.set_current_library_information(current_library_name(), db.library_id,
db.field_metadata)
register_keyboard_shortcuts()
self.keyboard.finalize()
if show_gui:
# Note this has to come after restoreGeometry() because of
# https://bugreports.qt.io/browse/QTBUG-56831
self.show()
if self.system_tray_icon is not None and self.system_tray_icon.isVisible() and opts.start_in_tray:
self.hide_windows()
self.auto_adder = AutoAdder(gprefs['auto_add_path'], self)
# Now that the gui is initialized we can restore the quickview state
# The same thing will be true for any action-based operation with a
# layout button
from calibre.gui2.actions.show_quickview import get_quickview_action_plugin
qv = get_quickview_action_plugin()
if qv:
qv.qv_button.restore_state()
self.save_layout_state()
# Collect cycles now
gc.collect()
QApplication.instance().shutdown_signal_received.connect(self.quit)
if show_gui and self.gui_debug is not None:
QTimer.singleShot(10, self.show_gui_debug_msg)
self.iactions['Connect Share'].check_smartdevice_menus()
QTimer.singleShot(1, self.start_smartdevice)
QTimer.singleShot(100, self.update_toggle_to_tray_action)
def show_gui_debug_msg(self):
info_dialog(self, _('Debug mode'), '<p>' +
_('You have started calibre in debug mode. After you '
'quit calibre, the debug log will be available in '
'the file: %s<p>The '
'log will be displayed automatically.')%self.gui_debug, show=True)
def esc(self, *args):
self.search.clear()
def shift_esc(self):
self.current_view().setFocus(Qt.OtherFocusReason)
def ctrl_esc(self):
self.apply_virtual_library()
self.current_view().setFocus(Qt.OtherFocusReason)
def start_smartdevice(self):
message = None
if self.device_manager.get_option('smartdevice', 'autostart'):
try:
message = self.device_manager.start_plugin('smartdevice')
except:
message = 'start smartdevice unknown exception'
prints(message)
import traceback
traceback.print_exc()
if message:
if not self.device_manager.is_running('Wireless Devices'):
error_dialog(self, _('Problem starting the wireless device'),
_('The wireless device driver had problems starting. '
'It said "%s"')%message, show=True)
self.iactions['Connect Share'].set_smartdevice_action_state()
def start_content_server(self, check_started=True):
from calibre.srv.embedded import Server
if not gprefs.get('server3_warning_done', False):
gprefs.set('server3_warning_done', True)
if os.path.exists(os.path.join(config_dir, 'server.py')):
try:
os.remove(os.path.join(config_dir, 'server.py'))
except EnvironmentError:
pass
warning_dialog(self, _('Content server changed!'), _(
'calibre 3 comes with a completely re-written content server.'
' As such any custom configuration you have for the content'
' server no longer applies. You should check and refresh your'
' settings in Preferences->Sharing->Sharing over the net'), show=True)
self.content_server = Server(self.library_broker, Dispatcher(self.handle_changes_from_server))
self.content_server.state_callback = Dispatcher(
self.iactions['Connect Share'].content_server_state_changed)
if check_started:
self.content_server.start_failure_callback = \
Dispatcher(self.content_server_start_failed)
self.content_server.start()
def handle_changes_from_server(self, library_path, change_event):
if DEBUG:
prints('Received server change event: {} for {}'.format(change_event, library_path))
if self.library_broker.is_gui_library(library_path):
self.server_changes.put((library_path, change_event))
self.server_change_notification_timer.start()
def handle_changes_from_server_debounced(self):
if self.shutting_down:
return
changes = []
while True:
try:
library_path, change_event = self.server_changes.get_nowait()
except Empty:
break
if self.library_broker.is_gui_library(library_path):
changes.append(change_event)
if changes:
handle_changes(changes, self)
def content_server_start_failed(self, msg):
self.content_server = None
error_dialog(self, _('Failed to start Content server'),
_('Could not start the Content server. Error:\n\n%s')%msg,
show=True)
def resizeEvent(self, ev):
MainWindow.resizeEvent(self, ev)
self.search.setMaximumWidth(self.width()-150)
def create_spare_pool(self, *args):
if self._spare_pool is None:
num = min(detect_ncpus(), int(config['worker_limit']/2.0))
self._spare_pool = Pool(max_workers=num, name='GUIPool')
def spare_pool(self):
ans, self._spare_pool = self._spare_pool, None
QTimer.singleShot(1000, self.create_spare_pool)
return ans
def do_proceed(self, func, payload):
if callable(func):
func(payload)
def no_op(self, *args):
pass
def system_tray_icon_activated(self, r=False):
if r in (QSystemTrayIcon.Trigger, QSystemTrayIcon.MiddleClick, False):
if self.isVisible():
if self.isMinimized():
self.showNormal()
else:
self.hide_windows()
else:
self.show_windows()
if self.isMinimized():
self.showNormal()
@property
def is_minimized_to_tray(self):
return getattr(self, '__systray_minimized', False)
def ask_a_yes_no_question(self, title, msg, det_msg='',
show_copy_button=False, ans_when_user_unavailable=True,
skip_dialog_name=None, skipped_value=True):
if self.is_minimized_to_tray:
return ans_when_user_unavailable
return question_dialog(self, title, msg, det_msg=det_msg,
show_copy_button=show_copy_button,
skip_dialog_name=skip_dialog_name,
skip_dialog_skipped_value=skipped_value)
def update_toggle_to_tray_action(self, *args):
if hasattr(self, 'toggle_to_tray_action'):
self.toggle_to_tray_action.setText(
_('Hide main window') if self.isVisible() else _('Show main window'))
def hide_windows(self):
for window in QApplication.topLevelWidgets():
if isinstance(window, (MainWindow, QDialog)) and \
window.isVisible():
window.hide()
setattr(window, '__systray_minimized', True)
self.update_toggle_to_tray_action()
def show_windows(self, *args):
for window in QApplication.topLevelWidgets():
if getattr(window, '__systray_minimized', False):
window.show()
setattr(window, '__systray_minimized', False)
self.update_toggle_to_tray_action()
def test_server(self, *args):
if self.content_server is not None and \
self.content_server.exception is not None:
error_dialog(self, _('Failed to start Content server'),
unicode_type(self.content_server.exception)).exec_()
@property
def current_db(self):
return self.library_view.model().db
def refresh_all(self):
m = self.library_view.model()
m.db.data.refresh(clear_caches=False, do_search=False)
self.saved_searches_changed(recount=False)
m.resort()
m.research()
self.tags_view.recount()
def handle_cli_args(self, args):
if isinstance(args, string_or_bytes):
args = [args]
files = [os.path.abspath(p) for p in args if not os.path.isdir(p) and os.access(p, os.R_OK)]
if files:
self.iactions['Add Books'].add_filesystem_book(files)
def another_instance_wants_to_talk(self):
try:
msg = self.listener.queue.get_nowait()
except Empty:
return
if msg.startswith('launched:'):
import json
try:
argv = json.loads(msg[len('launched:'):])
except ValueError:
prints('Failed to decode message from other instance: %r' % msg)
if DEBUG:
error_dialog(self, 'Invalid message',
'Received an invalid message from other calibre instance.'
' Do you have multiple versions of calibre installed?',
det_msg='Invalid msg: %r' % msg, show=True)
argv = ()
if isinstance(argv, (list, tuple)) and len(argv) > 1:
self.handle_cli_args(argv[1:])
self.setWindowState(self.windowState() & ~Qt.WindowMinimized|Qt.WindowActive)
self.show_windows()
self.raise_()
self.activateWindow()
elif msg.startswith('refreshdb:'):
m = self.library_view.model()
m.db.new_api.reload_from_db()
self.refresh_all()
elif msg.startswith('shutdown:'):
self.quit(confirm_quit=False)
elif msg.startswith('bookedited:'):
parts = msg.split(':')[1:]
try:
book_id, fmt, library_id = parts[:3]
book_id = int(book_id)
m = self.library_view.model()
db = m.db.new_api
if m.db.library_id == library_id and db.has_id(book_id):
db.format_metadata(book_id, fmt, allow_cache=False, update_db=True)
db.update_last_modified((book_id,))
m.refresh_ids((book_id,))
except Exception:
import traceback
traceback.print_exc()
else:
print(msg)
def current_view(self):
'''Convenience method that returns the currently visible view '''
idx = self.stack.currentIndex()
if idx == 0:
return self.library_view
if idx == 1:
return self.memory_view
if idx == 2:
return self.card_a_view
if idx == 3:
return self.card_b_view
def booklists(self):
return self.memory_view.model().db, self.card_a_view.model().db, self.card_b_view.model().db
def library_moved(self, newloc, copy_structure=False, allow_rebuild=False):
if newloc is None:
return
with self.library_broker:
default_prefs = None
try:
olddb = self.library_view.model().db
if copy_structure:
default_prefs = olddb.prefs
except:
olddb = None
if copy_structure and olddb is not None and default_prefs is not None:
default_prefs['field_metadata'] = olddb.new_api.field_metadata.all_metadata()
db = self.library_broker.prepare_for_gui_library_change(newloc)
if db is None:
try:
db = LibraryDatabase(newloc, default_prefs=default_prefs)
except apsw.Error:
if not allow_rebuild:
raise
import traceback
repair = question_dialog(self, _('Corrupted database'),
_('The library database at %s appears to be corrupted. Do '
'you want calibre to try and rebuild it automatically? '
'The rebuild may not be completely successful.')
% force_unicode(newloc, filesystem_encoding),
det_msg=traceback.format_exc()
)
if repair:
from calibre.gui2.dialogs.restore_library import repair_library_at
if repair_library_at(newloc, parent=self):
db = LibraryDatabase(newloc, default_prefs=default_prefs)
else:
return
else:
return
self.library_path = newloc
prefs['library_path'] = self.library_path
self.book_on_device(None, reset=True)
db.set_book_on_device_func(self.book_on_device)
self.library_view.set_database(db)
self.tags_view.set_database(db, self.alter_tb)
self.library_view.model().set_book_on_device_func(self.book_on_device)
self.status_bar.clear_message()
self.search.clear()
self.saved_search.clear()
self.book_details.reset_info()
# self.library_view.model().count_changed()
db = self.library_view.model().db
self.iactions['Choose Library'].count_changed(db.count())
self.set_window_title()
self.apply_named_search_restriction('') # reset restriction to null
self.saved_searches_changed(recount=False) # reload the search restrictions combo box
if db.prefs['virtual_lib_on_startup']:
self.apply_virtual_library(db.prefs['virtual_lib_on_startup'])
self.rebuild_vl_tabs()
for action in self.iactions.values():
action.library_changed(db)
self.library_broker.gui_library_changed(db, olddb)
if self.device_connected:
self.set_books_in_library(self.booklists(), reset=True)
self.refresh_ondevice()
self.memory_view.reset()
self.card_a_view.reset()
self.card_b_view.reset()
self.set_current_library_information(current_library_name(), db.library_id,
db.field_metadata)
self.library_view.set_current_row(0)
# Run a garbage collection now so that it does not freeze the
# interface later
gc.collect()
def set_window_title(self):
db = self.current_db
restrictions = [x for x in (db.data.get_base_restriction_name(),
db.data.get_search_restriction_name()) if x]
restrictions = ' :: '.join(restrictions)
font = QFont()
if restrictions:
restrictions = ' :: ' + restrictions
font.setBold(True)
font.setItalic(True)
self.virtual_library.setFont(font)
title = u'{0} - || {1}{2} ||'.format(
__appname__, self.iactions['Choose Library'].library_name(), restrictions)
self.setWindowTitle(title)
def location_selected(self, location):
'''
Called when a location icon is clicked (e.g. Library)
'''
page = 0 if location == 'library' else 1 if location == 'main' else 2 if location == 'carda' else 3
self.stack.setCurrentIndex(page)
self.book_details.reset_info()
for x in ('tb', 'cb'):
splitter = getattr(self, x+'_splitter')
splitter.button.setEnabled(location == 'library')
for action in self.iactions.values():
action.location_selected(location)
if location == 'library':
self.virtual_library_menu.setEnabled(True)
self.highlight_only_button.setEnabled(True)
self.vl_tabs.setEnabled(True)
else:
self.virtual_library_menu.setEnabled(False)
self.highlight_only_button.setEnabled(False)
self.vl_tabs.setEnabled(False)
# Reset the view in case something changed while it was invisible
self.current_view().reset()
self.set_number_of_books_shown()
self.update_status_bar()
def job_exception(self, job, dialog_title=_('Conversion error'), retry_func=None):
if not hasattr(self, '_modeless_dialogs'):
self._modeless_dialogs = []
minz = self.is_minimized_to_tray
if self.isVisible():
for x in list(self._modeless_dialogs):
if not x.isVisible():
self._modeless_dialogs.remove(x)
try:
if 'calibre.ebooks.DRMError' in job.details:
if not minz:
from calibre.gui2.dialogs.drm_error import DRMErrorMessage
d = DRMErrorMessage(self, _('Cannot convert') + ' ' +
job.description.split(':')[-1].partition('(')[-1][:-1])
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.ebooks.oeb.transforms.split.SplitError' in job.details:
title = job.description.split(':')[-1].partition('(')[-1][:-1]
msg = _('<p><b>Failed to convert: %s')%title
msg += '<p>'+_('''
Many older e-book reader devices are incapable of displaying
EPUB files that have internal components over a certain size.
Therefore, when converting to EPUB, calibre automatically tries
to split up the EPUB into smaller sized pieces. For some
files that are large undifferentiated blocks of text, this
splitting fails.
<p>You can <b>work around the problem</b> by either increasing the
maximum split size under <i>EPUB output</i> in the conversion dialog,
or by turning on Heuristic Processing, also in the conversion
dialog. Note that if you make the maximum split size too large,
your e-book reader may have trouble with the EPUB.
''')
if not minz:
d = error_dialog(self, _('Conversion Failed'), msg,
det_msg=job.details)
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.ebooks.mobi.reader.mobi6.KFXError:' in job.details:
if not minz:
title = job.description.split(':')[-1].partition('(')[-1][:-1]
msg = _('<p><b>Failed to convert: %s') % title
idx = job.details.index('calibre.ebooks.mobi.reader.mobi6.KFXError:')
msg += '<p>' + re.sub(r'(https:\S+)', r'<a href="\1">{}</a>'.format(_('here')),
job.details[idx:].partition(':')[2].strip())
d = error_dialog(self, _('Conversion failed'), msg, det_msg=job.details)
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.web.feeds.input.RecipeDisabled' in job.details:
if not minz:
msg = job.details
msg = msg[msg.find('calibre.web.feeds.input.RecipeDisabled:'):]
msg = msg.partition(':')[-1]
d = error_dialog(self, _('Recipe Disabled'),
'<p>%s</p>'%msg)
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.ebooks.conversion.ConversionUserFeedBack:' in job.details:
if not minz:
import json
payload = job.details.rpartition(
'calibre.ebooks.conversion.ConversionUserFeedBack:')[-1]
payload = json.loads('{' + payload.partition('{')[-1])
d = {'info':info_dialog, 'warn':warning_dialog,
'error':error_dialog}.get(payload['level'],
error_dialog)
d = d(self, payload['title'],
'<p>%s</p>'%payload['msg'],
det_msg=payload['det_msg'])
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
except:
pass
if job.killed:
return
try:
prints(job.details, file=sys.stderr)
except:
pass
if not minz:
self.job_error_dialog.show_error(dialog_title,
_('<b>Failed</b>')+': '+unicode_type(job.description),
det_msg=job.details, retry_func=retry_func)
def read_settings(self):
geometry = config['main_window_geometry']
if geometry is not None:
self.restoreGeometry(geometry)
self.read_layout_settings()
def write_settings(self):
with gprefs: # Only write to gprefs once
config.set('main_window_geometry', self.saveGeometry())
dynamic.set('sort_history', self.library_view.model().sort_history)
self.save_layout_state()
self.stack.tb_widget.save_state()
def quit(self, checked=True, restart=False, debug_on_restart=False,
confirm_quit=True):
if self.shutting_down:
return
if confirm_quit and not self.confirm_quit():
return
try:
self.shutdown()
except:
pass
self.restart_after_quit = restart
self.debug_on_restart = debug_on_restart
if self.system_tray_icon is not None and self.restart_after_quit:
# Needed on windows to prevent multiple systray icons
self.system_tray_icon.setVisible(False)
QApplication.instance().quit()
def donate(self, *args):
from calibre.utils.localization import localize_website_link
open_url(QUrl(localize_website_link('https://calibre-ebook.com/donate')))
def confirm_quit(self):
if self.job_manager.has_jobs():
msg = _('There are active jobs. Are you sure you want to quit?')
if self.job_manager.has_device_jobs():
msg = '<p>'+__appname__ + \
_(''' is communicating with the device!<br>
Quitting may cause corruption on the device.<br>
Are you sure you want to quit?''')+'</p>'
if not question_dialog(self, _('Active jobs'), msg):
return False
if self.proceed_question.questions:
msg = _('There are library updates waiting. Are you sure you want to quit?')
if not question_dialog(self, _('Library updates waiting'), msg):
return False
from calibre.db.delete_service import has_jobs
if has_jobs():
msg = _('Some deleted books are still being moved to the Recycle '
'Bin, if you quit now, they will be left behind. Are you '
'sure you want to quit?')
if not question_dialog(self, _('Active jobs'), msg):
return False
return True
def shutdown(self, write_settings=True):
self.shutting_down = True
self.show_shutdown_message()
self.server_change_notification_timer.stop()
from calibre.customize.ui import has_library_closed_plugins
if has_library_closed_plugins():
self.show_shutdown_message(
_('Running database shutdown plugins. This could take a few seconds...'))
self.grid_view.shutdown()
db = None
try:
db = self.library_view.model().db
cf = db.clean
except:
pass
else:
cf()
# Save the current field_metadata for applications like calibre2opds
# Goes here, because if cf is valid, db is valid.
db.new_api.set_pref('field_metadata', db.field_metadata.all_metadata())
db.commit_dirty_cache()
db.prefs.write_serialized(prefs['library_path'])
for action in self.iactions.values():
if not action.shutting_down():
return
if write_settings:
self.write_settings()
self.check_messages_timer.stop()
if getattr(self, 'update_checker', None):
self.update_checker.shutdown()
self.listener.close()
self.job_manager.server.close()
self.job_manager.threaded_server.close()
self.device_manager.keep_going = False
self.auto_adder.stop()
# Do not report any errors that happen after the shutdown
# We cannot restore the original excepthook as that causes PyQt to
# call abort() on unhandled exceptions
import traceback
def eh(t, v, tb):
try:
traceback.print_exception(t, v, tb, file=sys.stderr)
except:
pass
sys.excepthook = eh
mb = self.library_view.model().metadata_backup
if mb is not None:
mb.stop()
self.library_view.model().close()
try:
try:
if self.content_server is not None:
# If the Content server has any sockets being closed then
# this can take quite a long time (minutes). Tell the user that it is
# happening.
self.show_shutdown_message(
_('Shutting down the Content server. This could take a while...'))
s = self.content_server
self.content_server = None
s.exit()
except:
pass
except KeyboardInterrupt:
pass
self.hide_windows()
if self._spare_pool is not None:
self._spare_pool.shutdown()
from calibre.db.delete_service import shutdown
shutdown()
time.sleep(2)
self.istores.join()
return True
def run_wizard(self, *args):
if self.confirm_quit():
self.run_wizard_b4_shutdown = True
self.restart_after_quit = True
try:
self.shutdown(write_settings=False)
except:
pass
QApplication.instance().quit()
def closeEvent(self, e):
if self.shutting_down:
return
self.write_settings()
if self.system_tray_icon is not None and self.system_tray_icon.isVisible():
if not dynamic['systray_msg'] and not isosx:
info_dialog(self, 'calibre', 'calibre '+
_('will keep running in the system tray. To close it, '
'choose <b>Quit</b> in the context menu of the '
'system tray.'), show_copy_button=False).exec_()
dynamic['systray_msg'] = True
self.hide_windows()
e.ignore()
else:
if self.confirm_quit():
try:
self.shutdown(write_settings=False)
except:
import traceback
traceback.print_exc()
e.accept()
else:
e.ignore()
class CoverWorker(Thread): # {{{
def __init__(self, log, abort, title, authors, identifiers, caches):
Thread.__init__(self, name='CoverWorker')
self.daemon = True
self.log, self.abort = log, abort
self.title, self.authors, self.identifiers = (title, authors,
identifiers)
self.caches = caches
self.rq = Queue()
self.error = None
def fake_run(self):
images = [
'donate.png',
'config.png',
'column.png',
'eject.png',
]
time.sleep(2)
for pl, im in zip(metadata_plugins(['cover']), images):
self.rq.put((pl.name, 1, 1, 'png', I(im, data=True)))
def run(self):
try:
if DEBUG_DIALOG:
self.fake_run()
else:
self.run_fork()
except WorkerError as e:
self.error = force_unicode(e.orig_tb)
except:
import traceback
self.error = force_unicode(traceback.format_exc())
def run_fork(self):
with TemporaryDirectory('_single_metadata_download') as tdir:
self.keep_going = True
t = Thread(target=self.monitor_tdir, args=(tdir, ))
t.daemon = True
t.start()
try:
res = fork_job('calibre.ebooks.metadata.sources.worker',
'single_covers',
(self.title, self.authors, self.identifiers,
self.caches, tdir),
no_output=True,
abort=self.abort)
self.log.append_dump(res['result'])
finally:
self.keep_going = False
t.join()
def scan_once(self, tdir, seen):
for x in list(os.listdir(tdir)):
if x in seen:
continue
if x.endswith('.cover') and os.path.exists(
os.path.join(tdir, x + '.done')):
name = x.rpartition('.')[0]
try:
plugin_name, width, height, fmt = name.split(',,')
width, height = int(width), int(height)
with open(os.path.join(tdir, x), 'rb') as f:
data = f.read()
except:
import traceback
traceback.print_exc()
else:
seen.add(x)
self.rq.put((plugin_name, width, height, fmt, data))
def monitor_tdir(self, tdir):
seen = set()
while self.keep_going:
time.sleep(1)
self.scan_once(tdir, seen)
# One last scan after the download process has ended
self.scan_once(tdir, seen)
class CoverWorker(Thread): # {{{
def __init__(self, log, abort, title, authors, identifiers, caches):
Thread.__init__(self)
self.daemon = True
self.log, self.abort = log, abort
self.title, self.authors, self.identifiers = (title, authors,
identifiers)
self.caches = caches
self.rq = Queue()
self.error = None
def fake_run(self):
images = ['donate.png', 'config.png', 'column.png', 'eject.png', ]
time.sleep(2)
for pl, im in zip(metadata_plugins(['cover']), images):
self.rq.put((pl.name, 1, 1, 'png', I(im, data=True)))
def run(self):
try:
if DEBUG_DIALOG:
self.fake_run()
else:
self.run_fork()
except WorkerError as e:
self.error = force_unicode(e.orig_tb)
except:
import traceback
self.error = force_unicode(traceback.format_exc())
def run_fork(self):
with TemporaryDirectory('_single_metadata_download') as tdir:
self.keep_going = True
t = Thread(target=self.monitor_tdir, args=(tdir,))
t.daemon = True
t.start()
try:
res = fork_job('calibre.ebooks.metadata.sources.worker',
'single_covers',
(self.title, self.authors, self.identifiers, self.caches,
tdir),
no_output=True, abort=self.abort)
self.log.append_dump(res['result'])
finally:
self.keep_going = False
t.join()
def scan_once(self, tdir, seen):
for x in list(os.listdir(tdir)):
if x in seen:
continue
if x.endswith('.cover') and os.path.exists(os.path.join(tdir,
x+'.done')):
name = x.rpartition('.')[0]
try:
plugin_name, width, height, fmt = name.split(',,')
width, height = int(width), int(height)
with open(os.path.join(tdir, x), 'rb') as f:
data = f.read()
except:
import traceback
traceback.print_exc()
else:
seen.add(x)
self.rq.put((plugin_name, width, height, fmt, data))
def monitor_tdir(self, tdir):
seen = set()
while self.keep_going:
time.sleep(1)
self.scan_once(tdir, seen)
# One last scan after the download process has ended
self.scan_once(tdir, seen)
class Pool(Thread):
daemon = True
def __init__(self, max_workers=None, name=None):
Thread.__init__(self, name=name)
self.max_workers = max_workers or detect_ncpus()
self.available_workers = []
self.busy_workers = {}
self.pending_jobs = []
self.events = Queue()
self.results = Queue()
self.tracker = Queue()
self.terminal_failure = None
self.common_data = pickle_dumps(None)
self.worker_data = None
self.shutting_down = False
self.start()
def set_common_data(self, data=None):
''' Set some data that will be passed to all subsequent jobs without
needing to be transmitted every time. You must call this method before
queueing any jobs, otherwise the behavior is undefined. You can call it
after all jobs are done, then it will be used for the new round of
jobs. Can raise the :class:`Failure` exception is data could not be
sent to workers.'''
if self.failed:
raise Failure(self.terminal_failure)
self.events.put(data)
def __call__(self, job_id, module, func, *args, **kwargs):
'''
Schedule a job. The job will be run in a worker process, with the
result placed in self.results. If a terminal failure has occurred
previously, this method will raise the :class:`Failure` exception.
:param job_id: A unique id for the job. The result will have this id.
:param module: Either a fully qualified python module name or python
source code which will be executed as a module.
Source code is detected by the presence of newlines in module.
:param func: Name of the function from ``module`` that will be
executed. ``args`` and ``kwargs`` will be passed to the function.
'''
if self.failed:
raise Failure(self.terminal_failure)
job = Job(job_id, module, func, args, kwargs)
self.tracker.put(None)
self.events.put(job)
def wait_for_tasks(self, timeout=None):
''' Wait for all queued jobs to be completed, if timeout is not None,
will raise a RuntimeError if jobs are not completed in the specified
time. Will raise a :class:`Failure` exception if a terminal failure has
occurred previously. '''
if self.failed:
raise Failure(self.terminal_failure)
if timeout is None:
self.tracker.join()
else:
join_with_timeout(self.tracker, timeout)
def shutdown(self, wait_time=0.1):
''' Shutdown this pool, terminating all worker process. The pool cannot
be used after a shutdown. '''
self.shutting_down = True
self.events.put(None)
self.shutdown_workers(wait_time=wait_time)
def create_worker(self):
p = start_worker('from {0} import run_main, {1}; run_main({1})'.format(self.__class__.__module__, 'worker_main'))
sys.stdout.flush()
eintr_retry_call(p.stdin.write, self.worker_data)
p.stdin.flush(), p.stdin.close()
conn = eintr_retry_call(self.listener.accept)
w = Worker(p, conn, self.events, self.name)
if self.common_data != pickle_dumps(None):
w.set_common_data(self.common_data)
return w
def start_worker(self):
try:
w = self.create_worker()
if not self.shutting_down:
self.available_workers.append(w)
except Exception:
import traceback
self.terminal_failure = TerminalFailure('Failed to start worker process', traceback.format_exc(), None)
self.terminal_error()
return False
def run(self):
from calibre.utils.ipc.server import create_listener
self.auth_key = os.urandom(32)
self.address, self.listener = create_listener(self.auth_key)
self.worker_data = msgpack_dumps((self.address, self.auth_key))
if self.start_worker() is False:
return
while True:
event = self.events.get()
if event is None or self.shutting_down:
break
if self.handle_event(event) is False:
break
def handle_event(self, event):
if isinstance(event, Job):
job = event
if not self.available_workers:
if len(self.busy_workers) >= self.max_workers:
self.pending_jobs.append(job)
return
if self.start_worker() is False:
return False
return self.run_job(job)
elif isinstance(event, WorkerResult):
worker_result = event
self.busy_workers.pop(worker_result.worker, None)
self.available_workers.append(worker_result.worker)
self.tracker.task_done()
if worker_result.is_terminal_failure:
self.terminal_failure = TerminalFailure('Worker process crashed while executing job', worker_result.result.traceback, worker_result.id)
self.terminal_error()
return False
self.results.put(worker_result)
else:
self.common_data = pickle_dumps(event)
if len(self.common_data) > MAX_SIZE:
self.cd_file = PersistentTemporaryFile('pool_common_data')
with self.cd_file as f:
f.write(self.common_data)
self.common_data = pickle_dumps(File(f.name))
for worker in self.available_workers:
try:
worker.set_common_data(self.common_data)
except Exception:
import traceback
self.terminal_failure = TerminalFailure('Worker process crashed while sending common data', traceback.format_exc(), None)
self.terminal_error()
return False
while self.pending_jobs and self.available_workers:
if self.run_job(self.pending_jobs.pop()) is False:
return False
def run_job(self, job):
worker = self.available_workers.pop()
try:
worker(job)
except Exception:
import traceback
self.terminal_failure = TerminalFailure('Worker process crashed while sending job', traceback.format_exc(), job.id)
self.terminal_error()
return False
self.busy_workers[worker] = job
@property
def failed(self):
return self.terminal_failure is not None
def terminal_error(self):
if self.shutting_down:
return
for worker, job in iteritems(self.busy_workers):
self.results.put(WorkerResult(job.id, Result(None, None, None), True, worker))
self.tracker.task_done()
while self.pending_jobs:
job = self.pending_jobs.pop()
self.results.put(WorkerResult(job.id, Result(None, None, None), True, None))
self.tracker.task_done()
self.shutdown()
def shutdown_workers(self, wait_time=0.1):
self.worker_data = self.common_data = None
for worker in self.busy_workers:
if worker.process.poll() is None:
try:
worker.process.terminate()
except EnvironmentError:
pass # If the process has already been killed
workers = [w.process for w in self.available_workers + list(self.busy_workers)]
aw = list(self.available_workers)
def join():
for w in aw:
try:
w(None)
except Exception:
pass
for w in workers:
try:
w.wait()
except Exception:
pass
reaper = Thread(target=join, name='ReapPoolWorkers')
reaper.daemon = True
reaper.start()
reaper.join(wait_time)
for w in self.available_workers + list(self.busy_workers):
try:
w.conn.close()
except Exception:
pass
for w in workers:
if w.poll() is None:
try:
w.kill()
except EnvironmentError:
pass
del self.available_workers[:]
self.busy_workers.clear()
if hasattr(self, 'cd_file'):
try:
os.remove(self.cd_file.name)
except EnvironmentError:
pass
class Server(Thread):
def __init__(self, notify_on_job_done=lambda x: x, pool_size=None,
limit=sys.maxsize, enforce_cpu_limit=True):
Thread.__init__(self)
self.daemon = True
global _counter
self.id = _counter+1
_counter += 1
if enforce_cpu_limit:
limit = min(limit, cpu_count())
self.pool_size = limit if pool_size is None else pool_size
self.notify_on_job_done = notify_on_job_done
self.auth_key = os.urandom(32)
self.address, self.listener = create_listener(self.auth_key, backlog=4)
self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue()
self.kill_queue = Queue()
self.waiting_jobs = []
self.workers = deque()
self.launched_worker_count = 0
self._worker_launch_lock = RLock()
self.start()
def launch_worker(self, gui=False, redirect_output=None, job_name=None):
start = time.time()
with self._worker_launch_lock:
self.launched_worker_count += 1
id = self.launched_worker_count
fd, rfile = tempfile.mkstemp(prefix=u'ipc_result_%d_%d_'%(self.id, id),
dir=base_dir(), suffix=u'.pickle')
os.close(fd)
if redirect_output is None:
redirect_output = not gui
env = {
'CALIBRE_WORKER_ADDRESS' : environ_item(as_hex_unicode(msgpack_dumps(self.address))),
'CALIBRE_WORKER_KEY' : environ_item(as_hex_unicode(self.auth_key)),
'CALIBRE_WORKER_RESULT' : environ_item(as_hex_unicode(rfile)),
}
cw = self.do_launch(env, gui, redirect_output, rfile, job_name=job_name)
if isinstance(cw, string_or_bytes):
raise CriticalError('Failed to launch worker process:\n'+cw)
if DEBUG:
print('Worker Launch took:', time.time() - start)
return cw
def do_launch(self, env, gui, redirect_output, rfile, job_name=None):
w = Worker(env, gui=gui, job_name=job_name)
try:
w(redirect_output=redirect_output)
conn = eintr_retry_call(self.listener.accept)
if conn is None:
raise Exception('Failed to launch worker process')
except BaseException:
try:
w.kill()
except:
pass
import traceback
return traceback.format_exc()
return ConnectedWorker(w, conn, rfile)
def add_job(self, job):
job.done2 = self.notify_on_job_done
self.add_jobs_queue.put(job)
def run_job(self, job, gui=True, redirect_output=False):
w = self.launch_worker(gui=gui, redirect_output=redirect_output, job_name=getattr(job, 'name', None))
w.start_job(job)
def run(self):
while True:
try:
job = self.add_jobs_queue.get(True, 0.2)
if job is None:
break
self.waiting_jobs.insert(0, job)
except Empty:
pass
# Get notifications from worker process
for worker in self.workers:
while True:
try:
n = worker.notifications.get_nowait()
worker.job.notifications.put(n)
self.changed_jobs_queue.put(worker.job)
except Empty:
break
# Remove finished jobs
for worker in [w for w in self.workers if not w.is_alive]:
try:
worker.close_log_file()
except:
pass
self.workers.remove(worker)
job = worker.job
if worker.returncode != 0:
job.failed = True
job.returncode = worker.returncode
elif os.path.exists(worker.rfile):
try:
with lopen(worker.rfile, 'rb') as f:
job.result = pickle_loads(f.read())
os.remove(worker.rfile)
except:
pass
job.duration = time.time() - job.start_time
self.changed_jobs_queue.put(job)
# Start waiting jobs
sj = self.suitable_waiting_job()
if sj is not None:
job = self.waiting_jobs.pop(sj)
job.start_time = time.time()
if job.kill_on_start:
job.duration = 0.0
job.returncode = 1
job.killed = job.failed = True
job.result = None
else:
worker = self.launch_worker()
worker.start_job(job)
self.workers.append(worker)
job.log_path = worker.log_path
self.changed_jobs_queue.put(job)
while True:
try:
j = self.kill_queue.get_nowait()
self._kill_job(j)
except Empty:
break
def suitable_waiting_job(self):
available_workers = self.pool_size - len(self.workers)
for worker in self.workers:
job = worker.job
if job.core_usage == -1:
available_workers = 0
elif job.core_usage > 1:
available_workers -= job.core_usage - 1
if available_workers < 1:
return None
for i, job in enumerate(self.waiting_jobs):
if job.core_usage == -1:
if available_workers >= self.pool_size:
return i
elif job.core_usage <= available_workers:
return i
def kill_job(self, job):
self.kill_queue.put(job)
def killall(self):
for worker in self.workers:
self.kill_queue.put(worker.job)
def _kill_job(self, job):
if job.start_time is None:
job.kill_on_start = True
return
for worker in self.workers:
if job is worker.job:
worker.kill()
job.killed = True
break
def split(self, tasks):
'''
Split a list into a list of sub lists, with the number of sub lists being
no more than the number of workers this server supports. Each sublist contains
2-tuples of the form (i, x) where x is an element from the original list
and i is the index of the element x in the original list.
'''
ans, count, pos = [], 0, 0
delta = int(ceil(len(tasks)/float(self.pool_size)))
while count < len(tasks):
section = []
for t in tasks[pos:pos+delta]:
section.append((count, t))
count += 1
ans.append(section)
pos += delta
return ans
def close(self):
try:
self.add_jobs_queue.put(None)
except:
pass
try:
self.listener.close()
except:
pass
time.sleep(0.2)
for worker in list(self.workers):
try:
worker.kill()
except:
pass
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
class WebSocketConnection(HTTPConnection):
# Internal API {{{
in_websocket_mode = False
websocket_handler = None
def __init__(self, *args, **kwargs):
global conn_id
HTTPConnection.__init__(self, *args, **kwargs)
self.sendq = Queue()
self.control_frames = deque()
self.cf_lock = Lock()
self.sending = None
self.send_buf = None
self.frag_decoder = UTF8Decoder()
self.ws_close_received = self.ws_close_sent = False
conn_id += 1
self.websocket_connection_id = conn_id
self.stop_reading = False
def finalize_headers(self, inheaders):
upgrade = inheaders.get('Upgrade', '')
key = inheaders.get('Sec-WebSocket-Key', None)
conn = {x.strip().lower() for x in inheaders.get('Connection', '').split(',')}
if key is None or upgrade.lower() != 'websocket' or 'upgrade' not in conn:
return HTTPConnection.finalize_headers(self, inheaders)
ver = inheaders.get('Sec-WebSocket-Version', 'Unknown')
try:
ver_ok = int(ver) >= 13
except Exception:
ver_ok = False
if not ver_ok:
return self.simple_response(http_client.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver)
if self.method != 'GET':
return self.simple_response(http_client.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method)
response = HANDSHAKE_STR % as_base64_unicode(sha1((key + GUID_STR).encode('utf-8')).digest())
self.optimize_for_sending_packet()
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.set_state(WRITE, self.upgrade_connection_to_ws, ReadOnlyFileBuffer(response.encode('ascii')), inheaders)
def upgrade_connection_to_ws(self, buf, inheaders, event):
if self.write(buf):
if self.websocket_handler is None:
self.websocket_handler = DummyHandler()
self.read_frame, self.current_recv_opcode = ReadFrame(), None
self.in_websocket_mode = True
try:
self.websocket_handler.handle_websocket_upgrade(self.websocket_connection_id, weakref.ref(self), inheaders)
except Exception as err:
self.log.exception('Error in WebSockets upgrade handler:')
self.websocket_close(UNEXPECTED_ERROR, 'Unexpected error in handler: %r' % as_unicode(err))
self.handle_event = self.ws_duplex
self.set_ws_state()
self.end_send_optimization()
def set_ws_state(self):
if self.ws_close_sent or self.ws_close_received:
if self.ws_close_sent:
self.ready = False
else:
self.wait_for = WRITE
return
if self.send_buf is not None or self.sending is not None:
self.wait_for = RDWR
else:
try:
self.sending = self.sendq.get_nowait()
except Empty:
with self.cf_lock:
if self.control_frames:
self.wait_for = RDWR
else:
self.wait_for = READ
else:
self.wait_for = RDWR
if self.stop_reading:
if self.wait_for is READ:
self.ready = False
elif self.wait_for is RDWR:
self.wait_for = WRITE
def ws_duplex(self, event):
if event is READ:
self.ws_read()
elif event is WRITE:
self.ws_write()
self.set_ws_state()
def ws_read(self):
if not self.stop_reading:
self.read_frame(self)
def ws_data_received(self, data, opcode, frame_starting, frame_finished, is_final_frame_of_message):
if opcode in CONTROL_CODES:
return self.ws_control_frame(opcode, data)
message_starting = self.current_recv_opcode is None
if message_starting:
if opcode == CONTINUATION:
self.log.error('Client sent continuation frame with no message to continue')
self.websocket_close(PROTOCOL_ERROR, 'Continuation frame without any message to continue')
return
self.current_recv_opcode = opcode
elif frame_starting and opcode != CONTINUATION:
self.log.error('Client sent continuation frame with non-zero opcode')
self.websocket_close(PROTOCOL_ERROR, 'Continuation frame with non-zero opcode')
return
message_finished = frame_finished and is_final_frame_of_message
if self.current_recv_opcode == TEXT:
if message_starting:
self.frag_decoder.reset()
empty_data = len(data) == 0
try:
data = self.frag_decoder(data)
except ValueError:
self.frag_decoder.reset()
self.log.error('Client sent undecodeable UTF-8')
return self.websocket_close(INCONSISTENT_DATA, 'Not valid UTF-8')
if message_finished:
if (not data and not empty_data) or self.frag_decoder.state:
self.frag_decoder.reset()
self.log.error('Client sent undecodeable UTF-8')
return self.websocket_close(INCONSISTENT_DATA, 'Not valid UTF-8')
if message_finished:
self.current_recv_opcode = None
self.frag_decoder.reset()
try:
self.handle_websocket_data(data, message_starting, message_finished)
except Exception as err:
self.log.exception('Error in WebSockets data handler:')
self.websocket_close(UNEXPECTED_ERROR, 'Unexpected error in handler: %r' % as_unicode(err))
def ws_control_frame(self, opcode, data):
if opcode in (PING, CLOSE):
rcode = PONG if opcode == PING else CLOSE
if opcode == CLOSE:
self.ws_close_received = True
self.stop_reading = True
if data:
try:
close_code = unpack_from(b'!H', data)[0]
except struct_error:
data = pack(b'!H', PROTOCOL_ERROR) + b'close frame data must be atleast two bytes'
else:
try:
utf8_decode(data[2:])
except ValueError:
data = pack(b'!H', PROTOCOL_ERROR) + b'close frame data must be valid UTF-8'
else:
if close_code < 1000 or close_code in RESERVED_CLOSE_CODES or (1011 < close_code < 3000):
data = pack(b'!H', PROTOCOL_ERROR) + b'close code reserved'
else:
close_code = NORMAL_CLOSE
data = pack(b'!H', close_code)
f = ReadOnlyFileBuffer(create_frame(1, rcode, data))
f.is_close_frame = opcode == CLOSE
with self.cf_lock:
self.control_frames.append(f)
elif opcode == PONG:
try:
self.websocket_handler.handle_websocket_pong(self.websocket_connection_id, data)
except Exception:
self.log.exception('Error in PONG handler:')
self.set_ws_state()
def websocket_close(self, code=NORMAL_CLOSE, reason=b''):
if isinstance(reason, type('')):
reason = reason.encode('utf-8')
self.stop_reading = True
reason = reason[:123]
if code is None and not reason:
f = ReadOnlyFileBuffer(create_frame(1, CLOSE, b''))
else:
f = ReadOnlyFileBuffer(create_frame(1, CLOSE, pack(b'!H', code) + reason))
f.is_close_frame = True
with self.cf_lock:
self.control_frames.append(f)
self.set_ws_state()
def ws_write(self):
if self.ws_close_sent:
return
if self.send_buf is not None:
if self.write(self.send_buf):
self.end_send_optimization()
if getattr(self.send_buf, 'is_close_frame', False):
self.ws_close_sent = True
self.send_buf = None
else:
with self.cf_lock:
try:
self.send_buf = self.control_frames.popleft()
except IndexError:
if self.sending is not None:
self.send_buf = self.sending.create_frame()
if self.send_buf is None:
self.sending = None
if self.send_buf is not None:
self.optimize_for_sending_packet()
def close(self):
if self.in_websocket_mode:
try:
self.websocket_handler.handle_websocket_close(self.websocket_connection_id)
except Exception:
self.log.exception('Error in WebSocket close handler')
# Try to write a close frame, just once
try:
if self.send_buf is None and not self.ws_close_sent:
self.websocket_close(SHUTTING_DOWN, 'Shutting down')
with self.cf_lock:
self.write(self.control_frames.pop())
except Exception:
pass
Connection.close(self)
else:
HTTPConnection.close(self)
# }}}
def send_websocket_message(self, buf, wakeup=True):
''' Send a complete message. This class will take care of splitting it
into appropriate frames automatically. `buf` must be a file like object. '''
self.sendq.put(MessageWriter(buf))
self.wait_for = RDWR
if wakeup:
self.wakeup()
def send_websocket_frame(self, data, is_first=True, is_last=True):
''' Useful for streaming handlers that want to break up messages into
frames themselves. Note that these frames will be interleaved with
control frames, so they should not be too large. '''
opcode = (TEXT if isinstance(data, type('')) else BINARY) if is_first else CONTINUATION
fin = 1 if is_last else 0
frame = create_frame(fin, opcode, data)
with self.cf_lock:
self.control_frames.append(ReadOnlyFileBuffer(frame))
def send_websocket_ping(self, data=b''):
''' Send a PING to the remote client, it should reply with a PONG which
will be sent to the handle_websocket_pong callback in your handler. '''
if isinstance(data, type('')):
data = data.encode('utf-8')
frame = create_frame(True, PING, data)
with self.cf_lock:
self.control_frames.append(ReadOnlyFileBuffer(frame))
def handle_websocket_data(self, data, message_starting, message_finished):
''' Called when some data is received from the remote client. In
general the data may not constitute a complete "message", use the
message_starting and message_finished flags to re-assemble it into a
complete message in the handler. Note that for binary data, data is a
mutable object. If you intend to keep it around after this method
returns, create a bytestring from it, using tobytes(). '''
self.websocket_handler.handle_websocket_data(self.websocket_connection_id, data, message_starting, message_finished)
class CompletionWorker(Thread):
daemon = True
def __init__(self, result_callback=lambda x:x, worker_entry_point='main'):
Thread.__init__(self)
self.worker_entry_point = worker_entry_point
self.start()
self.main_queue = Queue()
self.result_callback = result_callback
self.reap_thread = None
self.shutting_down = False
self.connected = Event()
self.current_completion_request = None
self.latest_completion_request_id = None
self.request_count = 0
self.lock = RLock()
def launch_worker_process(self):
from calibre.utils.ipc.server import create_listener
from calibre.utils.ipc.pool import start_worker
self.worker_process = p = start_worker(
'from {0} import run_main, {1}; run_main({1})'.format(self.__class__.__module__, self.worker_entry_point))
auth_key = os.urandom(32)
address, self.listener = create_listener(auth_key)
eintr_retry_call(p.stdin.write, msgpack_dumps((address, auth_key)))
p.stdin.flush(), p.stdin.close()
self.control_conn = eintr_retry_call(self.listener.accept)
self.data_conn = eintr_retry_call(self.listener.accept)
self.data_thread = t = Thread(name='CWData', target=self.handle_data_requests)
t.daemon = True
t.start()
self.connected.set()
def send(self, data, conn=None):
conn = conn or self.control_conn
try:
eintr_retry_call(conn.send, data)
except:
if not self.shutting_down:
raise
def recv(self, conn=None):
conn = conn or self.control_conn
try:
return eintr_retry_call(conn.recv)
except:
if not self.shutting_down:
raise
def wait_for_connection(self, timeout=None):
self.connected.wait(timeout)
def handle_data_requests(self):
from calibre.gui2.tweak_book.completion.basic import handle_data_request
while True:
try:
req = self.recv(self.data_conn)
except EOFError:
break
except Exception:
import traceback
traceback.print_exc()
break
if req is None or self.shutting_down:
break
result, tb = handle_data_request(req)
try:
self.send((result, tb), self.data_conn)
except EOFError:
break
except Exception:
import traceback
traceback.print_exc()
break
def run(self):
self.launch_worker_process()
while True:
obj = self.main_queue.get()
if obj is None:
break
req_type, req_data = obj
try:
if req_type is COMPLETION_REQUEST:
with self.lock:
if self.current_completion_request is not None:
ccr, self.current_completion_request = self.current_completion_request, None
self.send_completion_request(ccr)
elif req_type is CLEAR_REQUEST:
self.send(req_data)
except EOFError:
break
except Exception:
import traceback
traceback.print_exc()
def send_completion_request(self, request):
self.send(request)
result = self.recv()
if result.request_id == self.latest_completion_request_id:
try:
self.result_callback(result)
except Exception:
import traceback
traceback.print_exc()
def clear_caches(self, cache_type=None):
self.main_queue.put((CLEAR_REQUEST, Request(None, 'clear_caches', cache_type, None)))
def queue_completion(self, request_id, completion_type, completion_data, query=None):
with self.lock:
self.current_completion_request = Request(request_id, completion_type, completion_data, query)
self.latest_completion_request_id = self.current_completion_request.id
self.main_queue.put((COMPLETION_REQUEST, None))
def shutdown(self):
self.shutting_down = True
self.main_queue.put(None)
for conn in (getattr(self, 'control_conn', None), getattr(self, 'data_conn', None)):
try:
conn.close()
except Exception:
pass
p = self.worker_process
if p.poll() is None:
self.worker_process.terminate()
t = self.reap_thread = Thread(target=p.wait)
t.daemon = True
t.start()
def join(self, timeout=0.2):
if self.reap_thread is not None:
self.reap_thread.join(timeout)
if not iswindows and self.worker_process.returncode is None:
self.worker_process.kill()
return self.worker_process.returncode
class Server(Thread):
def __init__(self,
notify_on_job_done=lambda x: x,
pool_size=None,
limit=sys.maxsize,
enforce_cpu_limit=True):
Thread.__init__(self)
self.daemon = True
self.id = next(server_counter) + 1
if enforce_cpu_limit:
limit = min(limit, cpu_count())
self.pool_size = limit if pool_size is None else pool_size
self.notify_on_job_done = notify_on_job_done
self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue()
self.kill_queue = Queue()
self.waiting_jobs = []
self.workers = deque()
self.launched_worker_counter = count()
next(self.launched_worker_counter)
self.start()
def launch_worker(self, gui=False, redirect_output=None, job_name=None):
start = time.monotonic()
id = next(self.launched_worker_counter)
fd, rfile = tempfile.mkstemp(prefix='ipc_result_%d_%d_' %
(self.id, id),
dir=base_dir(),
suffix='.pickle')
os.close(fd)
if redirect_output is None:
redirect_output = not gui
cw = self.do_launch(gui, redirect_output, rfile, job_name=job_name)
if isinstance(cw, string_or_bytes):
raise CriticalError('Failed to launch worker process:\n' +
force_unicode(cw))
if DEBUG:
print(
f'Worker Launch took: {time.monotonic() - start:.2f} seconds')
return cw
def do_launch(self, gui, redirect_output, rfile, job_name=None):
a, b = Pipe()
with a:
env = {
'CALIBRE_WORKER_FD': str(a.fileno()),
'CALIBRE_WORKER_RESULT': environ_item(as_hex_unicode(rfile))
}
w = Worker(env, gui=gui, job_name=job_name)
try:
w(pass_fds=(a.fileno(), ), redirect_output=redirect_output)
except BaseException:
try:
w.kill()
except:
pass
b.close()
import traceback
return traceback.format_exc()
return ConnectedWorker(w, b, rfile)
def add_job(self, job):
job.done2 = self.notify_on_job_done
self.add_jobs_queue.put(job)
def run_job(self, job, gui=True, redirect_output=False):
w = self.launch_worker(gui=gui,
redirect_output=redirect_output,
job_name=getattr(job, 'name', None))
w.start_job(job)
def run(self):
while True:
try:
job = self.add_jobs_queue.get(True, 0.2)
if job is None:
break
self.waiting_jobs.insert(0, job)
except Empty:
pass
# Get notifications from worker process
for worker in self.workers:
while True:
try:
n = worker.notifications.get_nowait()
worker.job.notifications.put(n)
self.changed_jobs_queue.put(worker.job)
except Empty:
break
# Remove finished jobs
for worker in [w for w in self.workers if not w.is_alive]:
try:
worker.close_log_file()
except:
pass
self.workers.remove(worker)
job = worker.job
if worker.returncode != 0:
job.failed = True
job.returncode = worker.returncode
elif os.path.exists(worker.rfile):
try:
with lopen(worker.rfile, 'rb') as f:
job.result = pickle_loads(f.read())
os.remove(worker.rfile)
except:
pass
job.duration = time.time() - job.start_time
self.changed_jobs_queue.put(job)
# Start waiting jobs
sj = self.suitable_waiting_job()
if sj is not None:
job = self.waiting_jobs.pop(sj)
job.start_time = time.time()
if job.kill_on_start:
job.duration = 0.0
job.returncode = 1
job.killed = job.failed = True
job.result = None
else:
worker = self.launch_worker()
worker.start_job(job)
self.workers.append(worker)
job.log_path = worker.log_path
self.changed_jobs_queue.put(job)
while True:
try:
j = self.kill_queue.get_nowait()
self._kill_job(j)
except Empty:
break
def suitable_waiting_job(self):
available_workers = self.pool_size - len(self.workers)
for worker in self.workers:
job = worker.job
if job.core_usage == -1:
available_workers = 0
elif job.core_usage > 1:
available_workers -= job.core_usage - 1
if available_workers < 1:
return None
for i, job in enumerate(self.waiting_jobs):
if job.core_usage == -1:
if available_workers >= self.pool_size:
return i
elif job.core_usage <= available_workers:
return i
def kill_job(self, job):
self.kill_queue.put(job)
def killall(self):
for worker in self.workers:
self.kill_queue.put(worker.job)
def _kill_job(self, job):
if job.start_time is None:
job.kill_on_start = True
return
for worker in self.workers:
if job is worker.job:
worker.kill()
job.killed = True
break
def split(self, tasks):
'''
Split a list into a list of sub lists, with the number of sub lists being
no more than the number of workers this server supports. Each sublist contains
2-tuples of the form (i, x) where x is an element from the original list
and i is the index of the element x in the original list.
'''
ans, count, pos = [], 0, 0
delta = int(ceil(len(tasks) / float(self.pool_size)))
while count < len(tasks):
section = []
for t in tasks[pos:pos + delta]:
section.append((count, t))
count += 1
ans.append(section)
pos += delta
return ans
def close(self):
try:
self.add_jobs_queue.put(None)
except:
pass
try:
self.listener.close()
except:
pass
time.sleep(0.2)
for worker in list(self.workers):
try:
worker.kill()
except:
pass
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
class JobsManager(object):
def __init__(self, opts, log):
mj = opts.max_jobs
if mj < 1:
mj = detect_ncpus()
self.log = log
self.max_jobs = max(1, mj)
self.max_job_time = max(0, opts.max_job_time * 60)
self.lock = RLock()
self.jobs = {}
self.finished_jobs = {}
self.events = Queue()
self.job_id = count()
self.waiting_job_ids = set()
self.waiting_jobs = deque()
self.max_block = None
self.shutting_down = False
self.event_loop = None
def start_job(self, name, module, func, args=(), kwargs=None, job_done_callback=None, job_data=None):
with self.lock:
if self.shutting_down:
return None
if self.event_loop is None:
self.event_loop = t = Thread(name='JobsEventLoop', target=self.run)
t.daemon = True
t.start()
job_id = next(self.job_id)
self.events.put(StartEvent(job_id, name, module, func, args, kwargs or {}, job_done_callback, job_data))
self.waiting_job_ids.add(job_id)
return job_id
def job_status(self, job_id):
with self.lock:
if not self.shutting_down:
if job_id in self.finished_jobs:
job = self.finished_jobs[job_id]
return 'finished', job.result, job.traceback, job.was_aborted
if job_id in self.jobs:
return 'running', None, None, None
if job_id in self.waiting_job_ids:
return 'waiting', None, None, None
return None, None, None, None
def abort_job(self, job_id):
job = self.jobs.get(job_id)
if job is not None:
job.abort_event.set()
def wait_for_running_job(self, job_id, timeout=None):
job = self.jobs.get(job_id)
if job is not None:
job.wait_for_end.wait(timeout)
if not job.done:
return False
while job_id not in self.finished_jobs:
time.sleep(0.001)
return True
def shutdown(self, timeout=5.0):
with self.lock:
self.shutting_down = True
for job in itervalues(self.jobs):
job.abort_event.set()
self.events.put(False)
def wait_for_shutdown(self, wait_till):
for job in itervalues(self.jobs):
delta = wait_till - monotonic()
if delta > 0:
job.join(delta)
if self.event_loop is not None:
delta = wait_till - monotonic()
if delta > 0:
self.event_loop.join(delta)
# Internal API {{{
def run(self):
while not self.shutting_down:
if self.max_block is None:
ev = self.events.get()
else:
try:
ev = self.events.get(block=True, timeout=self.max_block)
except Empty:
ev = None
if self.shutting_down:
break
if ev is None:
self.abort_hanging_jobs()
elif isinstance(ev, StartEvent):
self.waiting_jobs.append(ev)
self.start_waiting_jobs()
elif isinstance(ev, DoneEvent):
self.job_finished(ev.job_id)
elif ev is False:
break
def start_waiting_jobs(self):
with self.lock:
while self.waiting_jobs and len(self.jobs) < self.max_jobs:
ev = self.waiting_jobs.popleft()
self.jobs[ev.job_id] = Job(ev, self.events)
self.waiting_job_ids.discard(ev.job_id)
self.update_max_block()
def update_max_block(self):
with self.lock:
mb = None
now = monotonic()
for job in itervalues(self.jobs):
if not job.done and not job.abort_event.is_set():
delta = self.max_job_time - (now - job.start_time)
if delta <= 0:
self.max_block = 0
return
if mb is None:
mb = delta
else:
mb = min(mb, delta)
self.max_block = mb
def abort_hanging_jobs(self):
now = monotonic()
found = False
for job in itervalues(self.jobs):
if not job.done and not job.abort_event.is_set():
delta = self.max_job_time - (now - job.start_time)
if delta <= 0:
job.abort_event.set()
found = True
if found:
self.update_max_block()
def job_finished(self, job_id):
with self.lock:
self.finished_jobs[job_id] = job = self.jobs.pop(job_id)
if job.callback is not None:
try:
job.callback(job)
except Exception:
import traceback
self.log.error('Error running callback for job: %s:\n%s' % (job.name, traceback.format_exc()))
self.prune_finished_jobs()
if job.traceback and not job.was_aborted:
logdata = job.read_log()
self.log.error('The job: %s failed:\n%s\n%s' % (job.job_name, logdata, job.traceback))
job.remove_log()
self.start_waiting_jobs()
def prune_finished_jobs(self):
with self.lock:
remove = []
now = monotonic()
for job_id, job in iteritems(self.finished_jobs):
if now - job.end_time > 3600:
remove.append(job_id)
for job_id in remove:
del self.finished_jobs[job_id]
class SearchPanel(QWidget): # {{{
search_requested = pyqtSignal(object)
results_found = pyqtSignal(object)
show_search_result = pyqtSignal(object)
def __init__(self, parent=None):
QWidget.__init__(self, parent)
self.last_hidden_text_warning = None
self.current_search = None
self.l = l = QVBoxLayout(self)
l.setContentsMargins(0, 0, 0, 0)
self.search_input = si = SearchInput(self)
self.searcher = None
self.search_tasks = Queue()
self.results_found.connect(self.on_result_found, type=Qt.QueuedConnection)
si.do_search.connect(self.search_requested)
l.addWidget(si)
self.results = r = Results(self)
r.show_search_result.connect(self.do_show_search_result, type=Qt.QueuedConnection)
r.currentRowChanged.connect(self.update_hidden_message)
l.addWidget(r, 100)
self.spinner = s = BusySpinner(self)
s.setVisible(False)
l.addWidget(s)
self.hidden_message = la = QLabel(_('This text is hidden in the book and cannot be displayed'))
la.setStyleSheet('QLabel { margin-left: 1ex }')
la.setWordWrap(True)
la.setVisible(False)
l.addWidget(la)
def update_hidden_message(self):
self.hidden_message.setVisible(self.results.current_result_is_hidden)
def focus_input(self, text=None):
self.search_input.focus_input(text)
def start_search(self, search_query, current_name):
if self.current_search is not None and search_query == self.current_search:
self.find_next_requested(search_query.backwards)
return
if self.searcher is None:
self.searcher = Thread(name='Searcher', target=self.run_searches)
self.searcher.daemon = True
self.searcher.start()
self.results.clear()
self.hidden_message.setVisible(False)
self.spinner.start()
self.current_search = search_query
self.last_hidden_text_warning = None
self.search_tasks.put((search_query, current_name))
def run_searches(self):
while True:
x = self.search_tasks.get()
if x is None:
break
search_query, current_name = x
try:
manifest = get_manifest() or {}
spine = manifest.get('spine', ())
idx_map = {name: i for i, name in enumerate(spine)}
spine_idx = idx_map.get(current_name, -1)
except Exception:
import traceback
traceback.print_exc()
spine_idx = -1
if spine_idx < 0:
self.results_found.emit(SearchFinished(search_query))
continue
for name in spine:
counter = Counter()
spine_idx = idx_map[name]
try:
for i, result in enumerate(search_in_name(name, search_query)):
before, text, after = result
q = (before or '')[-5:] + text + (after or '')[:5]
self.results_found.emit(SearchResult(search_query, before, text, after, q, name, spine_idx, counter[q]))
counter[q] += 1
except Exception:
import traceback
traceback.print_exc()
self.results_found.emit(SearchFinished(search_query))
def on_result_found(self, result):
if self.current_search is None or result.search_query != self.current_search:
return
if isinstance(result, SearchFinished):
self.spinner.stop()
if not self.results.count():
self.show_no_results_found()
return
if self.results.add_result(result) == 1:
# first result
self.results.setCurrentRow(0)
self.results.item_activated()
self.update_hidden_message()
def visibility_changed(self, visible):
if visible:
self.focus_input()
def clear_searches(self):
self.current_search = None
self.last_hidden_text_warning = None
searchable_text_for_name.cache_clear()
self.spinner.stop()
self.results.clear()
def shutdown(self):
self.search_tasks.put(None)
self.spinner.stop()
self.current_search = None
self.last_hidden_text_warning = None
self.searcher = None
def find_next_requested(self, previous):
self.results.find_next(previous)
def do_show_search_result(self, sr):
self.show_search_result.emit(sr.for_js)
def search_result_not_found(self, sr):
self.results.search_result_not_found(sr)
self.update_hidden_message()
def show_no_results_found(self):
msg = _('No matches were found for:')
warning_dialog(self, _('No matches found'), msg + ' <b>{}</b>'.format(self.current_search.text), show=True)
class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
TagBrowserMixin, CoverFlowMixin, LibraryViewMixin, SearchBoxMixin,
SavedSearchBoxMixin, SearchRestrictionMixin, LayoutMixin, UpdateMixin,
EbookDownloadMixin
):
'The main GUI'
proceed_requested = pyqtSignal(object, object)
book_converted = pyqtSignal(object, object)
shutting_down = False
def __init__(self, opts, parent=None, gui_debug=None):
MainWindow.__init__(self, opts, parent=parent, disable_automatic_gc=True)
self.setWindowIcon(QApplication.instance().windowIcon())
self.jobs_pointer = Pointer(self)
self.proceed_requested.connect(self.do_proceed,
type=Qt.QueuedConnection)
self.proceed_question = ProceedQuestion(self)
self.job_error_dialog = JobError(self)
self.keyboard = Manager(self)
get_gui.ans = self
self.opts = opts
self.device_connected = None
self.gui_debug = gui_debug
self.iactions = OrderedDict()
# Actions
for action in interface_actions():
if opts.ignore_plugins and action.plugin_path is not None:
continue
try:
ac = self.init_iaction(action)
except:
# Ignore errors in loading user supplied plugins
import traceback
traceback.print_exc()
if action.plugin_path is None:
raise
continue
ac.plugin_path = action.plugin_path
ac.interface_action_base_plugin = action
self.add_iaction(ac)
self.load_store_plugins()
def init_iaction(self, action):
ac = action.load_actual_plugin(self)
ac.plugin_path = action.plugin_path
ac.interface_action_base_plugin = action
action.actual_iaction_plugin_loaded = True
return ac
def add_iaction(self, ac):
acmap = self.iactions
if ac.name in acmap:
if ac.priority >= acmap[ac.name].priority:
acmap[ac.name] = ac
else:
acmap[ac.name] = ac
def load_store_plugins(self):
from calibre.gui2.store.loader import Stores
self.istores = Stores()
for store in available_store_plugins():
if self.opts.ignore_plugins and store.plugin_path is not None:
continue
try:
st = self.init_istore(store)
self.add_istore(st)
except:
# Ignore errors in loading user supplied plugins
import traceback
traceback.print_exc()
if store.plugin_path is None:
raise
continue
self.istores.builtins_loaded()
def init_istore(self, store):
st = store.load_actual_plugin(self)
st.plugin_path = store.plugin_path
st.base_plugin = store
store.actual_istore_plugin_loaded = True
return st
def add_istore(self, st):
stmap = self.istores
if st.name in stmap:
if st.priority >= stmap[st.name].priority:
stmap[st.name] = st
else:
stmap[st.name] = st
def initialize(self, library_path, db, listener, actions, show_gui=True):
opts = self.opts
self.preferences_action, self.quit_action = actions
self.library_path = library_path
self.library_broker = GuiLibraryBroker(db)
self.content_server = None
self.server_change_notification_timer = t = QTimer(self)
self.server_changes = Queue()
t.setInterval(1000), t.timeout.connect(self.handle_changes_from_server_debounced), t.setSingleShot(True)
self._spare_pool = None
self.must_restart_before_config = False
self.listener = Listener(listener)
self.check_messages_timer = QTimer()
self.check_messages_timer.timeout.connect(self.another_instance_wants_to_talk)
self.check_messages_timer.start(1000)
for ac in self.iactions.values():
try:
ac.do_genesis()
except Exception:
# Ignore errors in third party plugins
import traceback
traceback.print_exc()
if getattr(ac, 'plugin_path', None) is None:
raise
self.donate_action = QAction(QIcon(I('donate.png')),
_('&Donate to support calibre'), self)
for st in self.istores.values():
st.do_genesis()
MainWindowMixin.init_main_window_mixin(self, db)
# Jobs Button {{{
self.job_manager = JobManager()
self.jobs_dialog = JobsDialog(self, self.job_manager)
self.jobs_button = JobsButton(parent=self)
self.jobs_button.initialize(self.jobs_dialog, self.job_manager)
# }}}
LayoutMixin.init_layout_mixin(self)
DeviceMixin.init_device_mixin(self)
self.progress_indicator = ProgressIndicator(self)
self.progress_indicator.pos = (0, 20)
self.verbose = opts.verbose
self.get_metadata = GetMetadata()
self.upload_memory = {}
self.metadata_dialogs = []
self.default_thumbnail = None
self.tb_wrapper = textwrap.TextWrapper(width=40)
self.viewers = collections.deque()
self.system_tray_icon = None
do_systray = config['systray_icon'] or opts.start_in_tray
if do_systray:
self.system_tray_icon = factory(app_id='com.calibre-ebook.gui').create_system_tray_icon(parent=self, title='calibre')
if self.system_tray_icon is not None:
self.system_tray_icon.setIcon(QIcon(I('lt.png', allow_user_override=False)))
if not (iswindows or isosx):
self.system_tray_icon.setIcon(QIcon.fromTheme('calibre-tray', self.system_tray_icon.icon()))
self.system_tray_icon.setToolTip(self.jobs_button.tray_tooltip())
self.system_tray_icon.setVisible(True)
self.jobs_button.tray_tooltip_updated.connect(self.system_tray_icon.setToolTip)
elif do_systray:
prints('Failed to create system tray icon, your desktop environment probably'
' does not support the StatusNotifier spec https://www.freedesktop.org/wiki/Specifications/StatusNotifierItem/')
self.system_tray_menu = QMenu(self)
self.toggle_to_tray_action = self.system_tray_menu.addAction(QIcon(I('page.png')), '')
self.toggle_to_tray_action.triggered.connect(self.system_tray_icon_activated)
self.system_tray_menu.addAction(self.donate_action)
self.eject_action = self.system_tray_menu.addAction(
QIcon(I('eject.png')), _('&Eject connected device'))
self.eject_action.setEnabled(False)
self.addAction(self.quit_action)
self.system_tray_menu.addAction(self.quit_action)
self.keyboard.register_shortcut('quit calibre', _('Quit calibre'),
default_keys=('Ctrl+Q',), action=self.quit_action)
if self.system_tray_icon is not None:
self.system_tray_icon.setContextMenu(self.system_tray_menu)
self.system_tray_icon.activated.connect(self.system_tray_icon_activated)
self.quit_action.triggered[bool].connect(self.quit)
self.donate_action.triggered[bool].connect(self.donate)
self.minimize_action = QAction(_('Minimize the calibre window'), self)
self.addAction(self.minimize_action)
self.keyboard.register_shortcut('minimize calibre', self.minimize_action.text(),
default_keys=(), action=self.minimize_action)
self.minimize_action.triggered.connect(self.showMinimized)
self.esc_action = QAction(self)
self.addAction(self.esc_action)
self.keyboard.register_shortcut('clear current search',
_('Clear the current search'), default_keys=('Esc',),
action=self.esc_action)
self.esc_action.triggered.connect(self.esc)
self.shift_esc_action = QAction(self)
self.addAction(self.shift_esc_action)
self.keyboard.register_shortcut('focus book list',
_('Focus the book list'), default_keys=('Shift+Esc',),
action=self.shift_esc_action)
self.shift_esc_action.triggered.connect(self.shift_esc)
self.ctrl_esc_action = QAction(self)
self.addAction(self.ctrl_esc_action)
self.keyboard.register_shortcut('clear virtual library',
_('Clear the virtual library'), default_keys=('Ctrl+Esc',),
action=self.ctrl_esc_action)
self.ctrl_esc_action.triggered.connect(self.ctrl_esc)
self.alt_esc_action = QAction(self)
self.addAction(self.alt_esc_action)
self.keyboard.register_shortcut('clear additional restriction',
_('Clear the additional restriction'), default_keys=('Alt+Esc',),
action=self.alt_esc_action)
self.alt_esc_action.triggered.connect(self.clear_additional_restriction)
# ###################### Start spare job server ########################
QTimer.singleShot(1000, self.create_spare_pool)
# ###################### Location Manager ########################
self.location_manager.location_selected.connect(self.location_selected)
self.location_manager.unmount_device.connect(self.device_manager.umount_device)
self.location_manager.configure_device.connect(self.configure_connected_device)
self.location_manager.update_device_metadata.connect(self.update_metadata_on_device)
self.eject_action.triggered.connect(self.device_manager.umount_device)
# ################### Update notification ###################
UpdateMixin.init_update_mixin(self, opts)
# ###################### Search boxes ########################
SearchRestrictionMixin.init_search_restriction_mixin(self)
SavedSearchBoxMixin.init_saved_seach_box_mixin(self)
# ###################### Library view ########################
LibraryViewMixin.init_library_view_mixin(self, db)
SearchBoxMixin.init_search_box_mixin(self) # Requires current_db
self.library_view.model().count_changed_signal.connect(
self.iactions['Choose Library'].count_changed)
if not gprefs.get('quick_start_guide_added', False):
try:
add_quick_start_guide(self.library_view)
except:
import traceback
traceback.print_exc()
for view in ('library', 'memory', 'card_a', 'card_b'):
v = getattr(self, '%s_view' % view)
v.selectionModel().selectionChanged.connect(self.update_status_bar)
v.model().count_changed_signal.connect(self.update_status_bar)
self.library_view.model().count_changed()
self.bars_manager.database_changed(self.library_view.model().db)
self.library_view.model().database_changed.connect(self.bars_manager.database_changed,
type=Qt.QueuedConnection)
# ########################## Tags Browser ##############################
TagBrowserMixin.init_tag_browser_mixin(self, db)
self.library_view.model().database_changed.connect(self.populate_tb_manage_menu, type=Qt.QueuedConnection)
# ######################## Search Restriction ##########################
if db.prefs['virtual_lib_on_startup']:
self.apply_virtual_library(db.prefs['virtual_lib_on_startup'])
self.rebuild_vl_tabs()
# ########################## Cover Flow ################################
CoverFlowMixin.init_cover_flow_mixin(self)
self._calculated_available_height = min(max_available_height()-15,
self.height())
self.resize(self.width(), self._calculated_available_height)
self.build_context_menus()
for ac in self.iactions.values():
try:
ac.gui_layout_complete()
except:
import traceback
traceback.print_exc()
if ac.plugin_path is None:
raise
if config['autolaunch_server']:
self.start_content_server()
self.read_settings()
self.finalize_layout()
self.bars_manager.start_animation()
self.set_window_title()
for ac in self.iactions.values():
try:
ac.initialization_complete()
except:
import traceback
traceback.print_exc()
if ac.plugin_path is None:
raise
self.set_current_library_information(current_library_name(), db.library_id,
db.field_metadata)
register_keyboard_shortcuts()
self.keyboard.finalize()
if show_gui:
# Note this has to come after restoreGeometry() because of
# https://bugreports.qt.io/browse/QTBUG-56831
self.show()
if self.system_tray_icon is not None and self.system_tray_icon.isVisible() and opts.start_in_tray:
self.hide_windows()
self.auto_adder = AutoAdder(gprefs['auto_add_path'], self)
# Now that the gui is initialized we can restore the quickview state
# The same thing will be true for any action-based operation with a
# layout button
from calibre.gui2.actions.show_quickview import get_quickview_action_plugin
qv = get_quickview_action_plugin()
if qv:
qv.qv_button.restore_state()
self.save_layout_state()
# Collect cycles now
gc.collect()
QApplication.instance().shutdown_signal_received.connect(self.quit)
if show_gui and self.gui_debug is not None:
QTimer.singleShot(10, self.show_gui_debug_msg)
self.iactions['Connect Share'].check_smartdevice_menus()
QTimer.singleShot(1, self.start_smartdevice)
QTimer.singleShot(100, self.update_toggle_to_tray_action)
def show_gui_debug_msg(self):
info_dialog(self, _('Debug mode'), '<p>' +
_('You have started calibre in debug mode. After you '
'quit calibre, the debug log will be available in '
'the file: %s<p>The '
'log will be displayed automatically.')%self.gui_debug, show=True)
def esc(self, *args):
self.search.clear()
def shift_esc(self):
self.current_view().setFocus(Qt.OtherFocusReason)
def ctrl_esc(self):
self.apply_virtual_library()
self.current_view().setFocus(Qt.OtherFocusReason)
def start_smartdevice(self):
message = None
if self.device_manager.get_option('smartdevice', 'autostart'):
try:
message = self.device_manager.start_plugin('smartdevice')
except:
message = 'start smartdevice unknown exception'
prints(message)
import traceback
traceback.print_exc()
if message:
if not self.device_manager.is_running('Wireless Devices'):
error_dialog(self, _('Problem starting the wireless device'),
_('The wireless device driver had problems starting. '
'It said "%s"')%message, show=True)
self.iactions['Connect Share'].set_smartdevice_action_state()
def start_content_server(self, check_started=True):
from calibre.srv.embedded import Server
if not gprefs.get('server3_warning_done', False):
gprefs.set('server3_warning_done', True)
if os.path.exists(os.path.join(config_dir, 'server.py')):
try:
os.remove(os.path.join(config_dir, 'server.py'))
except EnvironmentError:
pass
warning_dialog(self, _('Content server changed!'), _(
'calibre 3 comes with a completely re-written content server.'
' As such any custom configuration you have for the content'
' server no longer applies. You should check and refresh your'
' settings in Preferences->Sharing->Sharing over the net'), show=True)
self.content_server = Server(self.library_broker, Dispatcher(self.handle_changes_from_server))
self.content_server.state_callback = Dispatcher(
self.iactions['Connect Share'].content_server_state_changed)
if check_started:
self.content_server.start_failure_callback = \
Dispatcher(self.content_server_start_failed)
self.content_server.start()
def handle_changes_from_server(self, library_path, change_event):
if DEBUG:
prints('Received server change event: {} for {}'.format(change_event, library_path))
if self.library_broker.is_gui_library(library_path):
self.server_changes.put((library_path, change_event))
self.server_change_notification_timer.start()
def handle_changes_from_server_debounced(self):
if self.shutting_down:
return
changes = []
while True:
try:
library_path, change_event = self.server_changes.get_nowait()
except Empty:
break
if self.library_broker.is_gui_library(library_path):
changes.append(change_event)
if changes:
handle_changes(changes, self)
def content_server_start_failed(self, msg):
self.content_server = None
error_dialog(self, _('Failed to start Content server'),
_('Could not start the Content server. Error:\n\n%s')%msg,
show=True)
def resizeEvent(self, ev):
MainWindow.resizeEvent(self, ev)
self.search.setMaximumWidth(self.width()-150)
def create_spare_pool(self, *args):
if self._spare_pool is None:
num = min(detect_ncpus(), config['worker_limit']//2)
self._spare_pool = Pool(max_workers=num, name='GUIPool')
def spare_pool(self):
ans, self._spare_pool = self._spare_pool, None
QTimer.singleShot(1000, self.create_spare_pool)
return ans
def do_proceed(self, func, payload):
if callable(func):
func(payload)
def no_op(self, *args):
pass
def system_tray_icon_activated(self, r=False):
if r in (QSystemTrayIcon.Trigger, QSystemTrayIcon.MiddleClick, False):
if self.isVisible():
if self.isMinimized():
self.showNormal()
else:
self.hide_windows()
else:
self.show_windows()
if self.isMinimized():
self.showNormal()
@property
def is_minimized_to_tray(self):
return getattr(self, '__systray_minimized', False)
def ask_a_yes_no_question(self, title, msg, det_msg='',
show_copy_button=False, ans_when_user_unavailable=True,
skip_dialog_name=None, skipped_value=True):
if self.is_minimized_to_tray:
return ans_when_user_unavailable
return question_dialog(self, title, msg, det_msg=det_msg,
show_copy_button=show_copy_button,
skip_dialog_name=skip_dialog_name,
skip_dialog_skipped_value=skipped_value)
def update_toggle_to_tray_action(self, *args):
if hasattr(self, 'toggle_to_tray_action'):
self.toggle_to_tray_action.setText(
_('Hide main window') if self.isVisible() else _('Show main window'))
def hide_windows(self):
for window in QApplication.topLevelWidgets():
if isinstance(window, (MainWindow, QDialog)) and \
window.isVisible():
window.hide()
setattr(window, '__systray_minimized', True)
self.update_toggle_to_tray_action()
def show_windows(self, *args):
for window in QApplication.topLevelWidgets():
if getattr(window, '__systray_minimized', False):
window.show()
setattr(window, '__systray_minimized', False)
self.update_toggle_to_tray_action()
def test_server(self, *args):
if self.content_server is not None and \
self.content_server.exception is not None:
error_dialog(self, _('Failed to start Content server'),
unicode_type(self.content_server.exception)).exec_()
@property
def current_db(self):
return self.library_view.model().db
def refresh_all(self):
m = self.library_view.model()
m.db.data.refresh(clear_caches=False, do_search=False)
self.saved_searches_changed(recount=False)
m.resort()
m.research()
self.tags_view.recount()
def handle_cli_args(self, args):
if isinstance(args, string_or_bytes):
args = [args]
files = [os.path.abspath(p) for p in args if not os.path.isdir(p) and os.access(p, os.R_OK)]
if files:
self.iactions['Add Books'].add_filesystem_book(files)
def another_instance_wants_to_talk(self):
try:
msg = self.listener.queue.get_nowait()
except Empty:
return
if isinstance(msg, bytes):
msg = msg.decode('utf-8', 'replace')
if msg.startswith('launched:'):
import json
try:
argv = json.loads(msg[len('launched:'):])
except ValueError:
prints('Failed to decode message from other instance: %r' % msg)
if DEBUG:
error_dialog(self, 'Invalid message',
'Received an invalid message from other calibre instance.'
' Do you have multiple versions of calibre installed?',
det_msg='Invalid msg: %r' % msg, show=True)
argv = ()
if isinstance(argv, (list, tuple)) and len(argv) > 1:
self.handle_cli_args(argv[1:])
self.setWindowState(self.windowState() & ~Qt.WindowMinimized|Qt.WindowActive)
self.show_windows()
self.raise_()
self.activateWindow()
elif msg.startswith('refreshdb:'):
m = self.library_view.model()
m.db.new_api.reload_from_db()
self.refresh_all()
elif msg.startswith('shutdown:'):
self.quit(confirm_quit=False)
elif msg.startswith('bookedited:'):
parts = msg.split(':')[1:]
try:
book_id, fmt, library_id = parts[:3]
book_id = int(book_id)
m = self.library_view.model()
db = m.db.new_api
if m.db.library_id == library_id and db.has_id(book_id):
db.format_metadata(book_id, fmt, allow_cache=False, update_db=True)
db.update_last_modified((book_id,))
m.refresh_ids((book_id,))
except Exception:
import traceback
traceback.print_exc()
elif msg.startswith('web-store:'):
import json
try:
data = json.loads(msg[len('web-store:'):])
except ValueError:
prints('Failed to decode message from other instance: %r' % msg)
path = data['path']
if data['tags']:
before = self.current_db.new_api.all_book_ids()
self.iactions['Add Books'].add_filesystem_book([path], allow_device=False)
if data['tags']:
db = self.current_db.new_api
after = self.current_db.new_api.all_book_ids()
for book_id in after - before:
tags = list(db.field_for('tags', book_id))
tags += list(data['tags'])
self.current_db.new_api.set_field('tags', {book_id: tags})
else:
prints('Ignoring unknown message from other instance: %r' % msg[:20])
def current_view(self):
'''Convenience method that returns the currently visible view '''
idx = self.stack.currentIndex()
if idx == 0:
return self.library_view
if idx == 1:
return self.memory_view
if idx == 2:
return self.card_a_view
if idx == 3:
return self.card_b_view
def booklists(self):
return self.memory_view.model().db, self.card_a_view.model().db, self.card_b_view.model().db
def library_moved(self, newloc, copy_structure=False, allow_rebuild=False):
if newloc is None:
return
with self.library_broker:
default_prefs = None
try:
olddb = self.library_view.model().db
if copy_structure:
default_prefs = olddb.prefs
except:
olddb = None
if copy_structure and olddb is not None and default_prefs is not None:
default_prefs['field_metadata'] = olddb.new_api.field_metadata.all_metadata()
db = self.library_broker.prepare_for_gui_library_change(newloc)
if db is None:
try:
db = LibraryDatabase(newloc, default_prefs=default_prefs)
except apsw.Error:
if not allow_rebuild:
raise
import traceback
repair = question_dialog(self, _('Corrupted database'),
_('The library database at %s appears to be corrupted. Do '
'you want calibre to try and rebuild it automatically? '
'The rebuild may not be completely successful.')
% force_unicode(newloc, filesystem_encoding),
det_msg=traceback.format_exc()
)
if repair:
from calibre.gui2.dialogs.restore_library import repair_library_at
if repair_library_at(newloc, parent=self):
db = LibraryDatabase(newloc, default_prefs=default_prefs)
else:
return
else:
return
self.library_path = newloc
prefs['library_path'] = self.library_path
self.book_on_device(None, reset=True)
db.set_book_on_device_func(self.book_on_device)
self.library_view.set_database(db)
self.tags_view.set_database(db, self.alter_tb)
self.library_view.model().set_book_on_device_func(self.book_on_device)
self.status_bar.clear_message()
self.search.clear()
self.saved_search.clear()
self.book_details.reset_info()
# self.library_view.model().count_changed()
db = self.library_view.model().db
self.iactions['Choose Library'].count_changed(db.count())
self.set_window_title()
self.apply_named_search_restriction('') # reset restriction to null
self.saved_searches_changed(recount=False) # reload the search restrictions combo box
if db.prefs['virtual_lib_on_startup']:
self.apply_virtual_library(db.prefs['virtual_lib_on_startup'])
self.rebuild_vl_tabs()
for action in self.iactions.values():
action.library_changed(db)
self.library_broker.gui_library_changed(db, olddb)
if self.device_connected:
self.set_books_in_library(self.booklists(), reset=True)
self.refresh_ondevice()
self.memory_view.reset()
self.card_a_view.reset()
self.card_b_view.reset()
self.set_current_library_information(current_library_name(), db.library_id,
db.field_metadata)
self.library_view.set_current_row(0)
# Run a garbage collection now so that it does not freeze the
# interface later
gc.collect()
def set_window_title(self):
db = self.current_db
restrictions = [x for x in (db.data.get_base_restriction_name(),
db.data.get_search_restriction_name()) if x]
restrictions = ' :: '.join(restrictions)
font = QFont()
if restrictions:
restrictions = ' :: ' + restrictions
font.setBold(True)
font.setItalic(True)
self.virtual_library.setFont(font)
title = '{0} - || {1}{2} ||'.format(
__appname__, self.iactions['Choose Library'].library_name(), restrictions)
self.setWindowTitle(title)
def location_selected(self, location):
'''
Called when a location icon is clicked (e.g. Library)
'''
page = 0 if location == 'library' else 1 if location == 'main' else 2 if location == 'carda' else 3
self.stack.setCurrentIndex(page)
self.book_details.reset_info()
for x in ('tb', 'cb'):
splitter = getattr(self, x+'_splitter')
splitter.button.setEnabled(location == 'library')
for action in self.iactions.values():
action.location_selected(location)
if location == 'library':
self.virtual_library_menu.setEnabled(True)
self.highlight_only_button.setEnabled(True)
self.vl_tabs.setEnabled(True)
else:
self.virtual_library_menu.setEnabled(False)
self.highlight_only_button.setEnabled(False)
self.vl_tabs.setEnabled(False)
# Reset the view in case something changed while it was invisible
self.current_view().reset()
self.set_number_of_books_shown()
self.update_status_bar()
def job_exception(self, job, dialog_title=_('Conversion error'), retry_func=None):
if not hasattr(self, '_modeless_dialogs'):
self._modeless_dialogs = []
minz = self.is_minimized_to_tray
if self.isVisible():
for x in list(self._modeless_dialogs):
if not x.isVisible():
self._modeless_dialogs.remove(x)
try:
if 'calibre.ebooks.DRMError' in job.details:
if not minz:
from calibre.gui2.dialogs.drm_error import DRMErrorMessage
d = DRMErrorMessage(self, _('Cannot convert') + ' ' +
job.description.split(':')[-1].partition('(')[-1][:-1])
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.ebooks.oeb.transforms.split.SplitError' in job.details:
title = job.description.split(':')[-1].partition('(')[-1][:-1]
msg = _('<p><b>Failed to convert: %s')%title
msg += '<p>'+_('''
Many older e-book reader devices are incapable of displaying
EPUB files that have internal components over a certain size.
Therefore, when converting to EPUB, calibre automatically tries
to split up the EPUB into smaller sized pieces. For some
files that are large undifferentiated blocks of text, this
splitting fails.
<p>You can <b>work around the problem</b> by either increasing the
maximum split size under <i>EPUB output</i> in the conversion dialog,
or by turning on Heuristic Processing, also in the conversion
dialog. Note that if you make the maximum split size too large,
your e-book reader may have trouble with the EPUB.
''')
if not minz:
d = error_dialog(self, _('Conversion Failed'), msg,
det_msg=job.details)
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.ebooks.mobi.reader.mobi6.KFXError:' in job.details:
if not minz:
title = job.description.split(':')[-1].partition('(')[-1][:-1]
msg = _('<p><b>Failed to convert: %s') % title
idx = job.details.index('calibre.ebooks.mobi.reader.mobi6.KFXError:')
msg += '<p>' + re.sub(r'(https:\S+)', r'<a href="\1">{}</a>'.format(_('here')),
job.details[idx:].partition(':')[2].strip())
d = error_dialog(self, _('Conversion failed'), msg, det_msg=job.details)
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.web.feeds.input.RecipeDisabled' in job.details:
if not minz:
msg = job.details
msg = msg[msg.find('calibre.web.feeds.input.RecipeDisabled:'):]
msg = msg.partition(':')[-1]
d = error_dialog(self, _('Recipe Disabled'),
'<p>%s</p>'%msg)
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.ebooks.conversion.ConversionUserFeedBack:' in job.details:
if not minz:
import json
payload = job.details.rpartition(
'calibre.ebooks.conversion.ConversionUserFeedBack:')[-1]
payload = json.loads('{' + payload.partition('{')[-1])
d = {'info':info_dialog, 'warn':warning_dialog,
'error':error_dialog}.get(payload['level'],
error_dialog)
d = d(self, payload['title'],
'<p>%s</p>'%payload['msg'],
det_msg=payload['det_msg'])
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
except:
pass
if job.killed:
return
try:
prints(job.details, file=sys.stderr)
except:
pass
if not minz:
self.job_error_dialog.show_error(dialog_title,
_('<b>Failed</b>')+': '+unicode_type(job.description),
det_msg=job.details, retry_func=retry_func)
def read_settings(self):
geometry = config['main_window_geometry']
if geometry is not None:
self.restoreGeometry(geometry)
self.read_layout_settings()
def write_settings(self):
with gprefs: # Only write to gprefs once
config.set('main_window_geometry', self.saveGeometry())
dynamic.set('sort_history', self.library_view.model().sort_history)
self.save_layout_state()
self.stack.tb_widget.save_state()
def quit(self, checked=True, restart=False, debug_on_restart=False,
confirm_quit=True):
if self.shutting_down:
return
if confirm_quit and not self.confirm_quit():
return
try:
self.shutdown()
except:
pass
self.restart_after_quit = restart
self.debug_on_restart = debug_on_restart
if self.system_tray_icon is not None and self.restart_after_quit:
# Needed on windows to prevent multiple systray icons
self.system_tray_icon.setVisible(False)
QApplication.instance().quit()
def donate(self, *args):
from calibre.utils.localization import localize_website_link
open_url(QUrl(localize_website_link('https://calibre-ebook.com/donate')))
def confirm_quit(self):
if self.job_manager.has_jobs():
msg = _('There are active jobs. Are you sure you want to quit?')
if self.job_manager.has_device_jobs():
msg = '<p>'+__appname__ + \
_(''' is communicating with the device!<br>
Quitting may cause corruption on the device.<br>
Are you sure you want to quit?''')+'</p>'
if not question_dialog(self, _('Active jobs'), msg):
return False
if self.proceed_question.questions:
msg = _('There are library updates waiting. Are you sure you want to quit?')
if not question_dialog(self, _('Library updates waiting'), msg):
return False
from calibre.db.delete_service import has_jobs
if has_jobs():
msg = _('Some deleted books are still being moved to the Recycle '
'Bin, if you quit now, they will be left behind. Are you '
'sure you want to quit?')
if not question_dialog(self, _('Active jobs'), msg):
return False
return True
def shutdown(self, write_settings=True):
self.shutting_down = True
self.show_shutdown_message()
self.server_change_notification_timer.stop()
from calibre.customize.ui import has_library_closed_plugins
if has_library_closed_plugins():
self.show_shutdown_message(
_('Running database shutdown plugins. This could take a few seconds...'))
self.grid_view.shutdown()
db = None
try:
db = self.library_view.model().db
cf = db.clean
except:
pass
else:
cf()
# Save the current field_metadata for applications like calibre2opds
# Goes here, because if cf is valid, db is valid.
db.new_api.set_pref('field_metadata', db.field_metadata.all_metadata())
db.commit_dirty_cache()
db.prefs.write_serialized(prefs['library_path'])
for action in self.iactions.values():
if not action.shutting_down():
return
if write_settings:
self.write_settings()
self.check_messages_timer.stop()
if getattr(self, 'update_checker', None):
self.update_checker.shutdown()
self.listener.close()
self.job_manager.server.close()
self.job_manager.threaded_server.close()
self.device_manager.keep_going = False
self.auto_adder.stop()
# Do not report any errors that happen after the shutdown
# We cannot restore the original excepthook as that causes PyQt to
# call abort() on unhandled exceptions
import traceback
def eh(t, v, tb):
try:
traceback.print_exception(t, v, tb, file=sys.stderr)
except:
pass
sys.excepthook = eh
mb = self.library_view.model().metadata_backup
if mb is not None:
mb.stop()
self.library_view.model().close()
try:
try:
if self.content_server is not None:
# If the Content server has any sockets being closed then
# this can take quite a long time (minutes). Tell the user that it is
# happening.
self.show_shutdown_message(
_('Shutting down the Content server. This could take a while...'))
s = self.content_server
self.content_server = None
s.exit()
except:
pass
except KeyboardInterrupt:
pass
self.hide_windows()
if self._spare_pool is not None:
self._spare_pool.shutdown()
from calibre.db.delete_service import shutdown
shutdown()
time.sleep(2)
self.istores.join()
return True
def run_wizard(self, *args):
if self.confirm_quit():
self.run_wizard_b4_shutdown = True
self.restart_after_quit = True
try:
self.shutdown(write_settings=False)
except:
pass
QApplication.instance().quit()
def closeEvent(self, e):
if self.shutting_down:
return
self.write_settings()
if self.system_tray_icon is not None and self.system_tray_icon.isVisible():
if not dynamic['systray_msg'] and not isosx:
info_dialog(self, 'calibre', 'calibre '+
_('will keep running in the system tray. To close it, '
'choose <b>Quit</b> in the context menu of the '
'system tray.'), show_copy_button=False).exec_()
dynamic['systray_msg'] = True
self.hide_windows()
e.ignore()
else:
if self.confirm_quit():
try:
self.shutdown(write_settings=False)
except:
import traceback
traceback.print_exc()
e.accept()
else:
e.ignore()
class DeleteService(Thread):
''' Provide a blocking file delete implementation with support for the
recycle bin. On windows, deleting files to the recycle bin spins the event
loop, which can cause locking errors in the main thread. We get around this
by only moving the files/folders to be deleted out of the library in the
main thread, they are deleted to recycle bin in a separate worker thread.
This has the added advantage that doing a restore from the recycle bin wont
cause metadata.db and the file system to get out of sync. Also, deleting
becomes much faster, since in the common case, the move is done by a simple
os.rename(). The downside is that if the user quits calibre while a long
move to recycle bin is happening, the files may not all be deleted.'''
daemon = True
def __init__(self):
Thread.__init__(self)
self.requests = Queue()
if isosx:
plugins['cocoa'][0].enable_cocoa_multithreading()
def shutdown(self, timeout=20):
self.requests.put(None)
self.join(timeout)
def create_staging(self, library_path):
base_path = os.path.dirname(library_path)
base = os.path.basename(library_path)
try:
ans = tempfile.mkdtemp(prefix=base + ' deleted ', dir=base_path)
except OSError:
ans = tempfile.mkdtemp(prefix=base + ' deleted ')
atexit.register(remove_dir, ans)
return ans
def remove_dir_if_empty(self, path):
try:
os.rmdir(path)
except OSError as e:
if e.errno == errno.ENOTEMPTY or len(os.listdir(path)) > 0:
# Some linux systems appear to raise an EPERM instead of an
# ENOTEMPTY, see https://bugs.launchpad.net/bugs/1240797
return
raise
def delete_books(self, paths, library_path):
tdir = self.create_staging(library_path)
self.queue_paths(tdir, paths, delete_empty_parent=True)
def queue_paths(self, tdir, paths, delete_empty_parent=True):
try:
self._queue_paths(tdir,
paths,
delete_empty_parent=delete_empty_parent)
except:
if os.path.exists(tdir):
shutil.rmtree(tdir, ignore_errors=True)
raise
def _queue_paths(self, tdir, paths, delete_empty_parent=True):
requests = []
for path in paths:
if os.path.exists(path):
basename = os.path.basename(path)
c = 0
while True:
dest = os.path.join(tdir, basename)
if not os.path.exists(dest):
break
c += 1
basename = '%d - %s' % (c, os.path.basename(path))
try:
shutil.move(path, dest)
except EnvironmentError:
if os.path.isdir(path):
# shutil.move may have partially copied the directory,
# so the subsequent call to move() will fail as the
# destination directory already exists
raise
# Wait a little in case something has locked a file
time.sleep(1)
shutil.move(path, dest)
if delete_empty_parent:
remove_dir_if_empty(os.path.dirname(path),
ignore_metadata_caches=True)
requests.append(dest)
if not requests:
remove_dir_if_empty(tdir)
else:
self.requests.put(tdir)
def delete_files(self, paths, library_path):
tdir = self.create_staging(library_path)
self.queue_paths(tdir, paths, delete_empty_parent=False)
def run(self):
while True:
x = self.requests.get()
try:
if x is None:
break
try:
self.do_delete(x)
except:
import traceback
traceback.print_exc()
finally:
self.requests.task_done()
def wait(self):
'Blocks until all pending deletes have completed'
self.requests.join()
def do_delete(self, tdir):
if os.path.exists(tdir):
try:
for x in os.listdir(tdir):
x = os.path.join(tdir, x)
if os.path.isdir(x):
delete_tree(x)
else:
delete_file(x)
finally:
shutil.rmtree(tdir)
def compress_images(container,
report=None,
names=None,
jpeg_quality=None,
progress_callback=lambda n, t, name: True):
images = get_compressible_images(container)
if names is not None:
images &= set(names)
results = {}
queue = Queue()
abort = Event()
seen = set()
num_to_process = 0
for name in sorted(images):
path = os.path.abspath(container.get_file_path_for_processing(name))
path_key = os.path.normcase(path)
if path_key not in seen:
num_to_process += 1
queue.put((name, path, container.mime_map[name]))
seen.add(path_key)
def pc(name):
keep_going = progress_callback(len(results), num_to_process, name)
if not keep_going:
abort.set()
progress_callback(0, num_to_process, '')
[
Worker(abort, 'CompressImage%d' % i, queue, results, jpeg_quality, pc)
for i in range(min(detect_ncpus(), num_to_process))
]
queue.join()
before_total = after_total = 0
processed_num = 0
changed = False
for name, (ok, res) in iteritems(results):
name = force_unicode(name, filesystem_encoding)
if ok:
before, after = res
if before != after:
changed = True
processed_num += 1
before_total += before
after_total += after
if report:
if before != after:
report(
_('{0} compressed from {1} to {2} bytes [{3:.1%} reduction]'
).format(name, human_readable(before),
human_readable(after),
(before - after) / before))
else:
report(
_('{0} could not be further compressed').format(name))
else:
report(_('Failed to process {0} with error:').format(name))
report(res)
if report:
if changed:
report('')
report(
_('Total image filesize reduced from {0} to {1} [{2:.1%} reduction, {3} images changed]'
).format(human_readable(before_total),
human_readable(after_total),
(before_total - after_total) / before_total,
processed_num))
else:
report(_('Images are already fully optimized'))
return changed, results
def check_external_links(container,
progress_callback=(lambda num, total: None),
check_anchors=True):
progress_callback(0, 0)
external_links = defaultdict(list)
for name, mt in iteritems(container.mime_map):
if mt in OEB_DOCS or mt in OEB_STYLES:
for href, lnum, col in container.iterlinks(name):
purl = urlparse(href)
if purl.scheme in ('http', 'https'):
external_links[href].append((name, href, lnum, col))
if not external_links:
return []
items = Queue()
ans = []
for el in iteritems(external_links):
items.put(el)
progress_callback(0, len(external_links))
done = []
downloaded_html_ids = {}
def check_links():
br = browser(honor_time=False, verify_ssl_certificates=False)
while True:
try:
full_href, locations = items.get_nowait()
except Empty:
return
href, frag = full_href.partition('#')[::2]
try:
res = br.open(href, timeout=10)
except Exception as e:
ans.append((locations, e, full_href))
else:
if frag and check_anchors:
ct = res.info().get('Content-Type')
if ct and ct.split(';')[0].lower() in {
'text/html', XHTML_MIME
}:
ids = downloaded_html_ids.get(href)
if ids is None:
try:
ids = downloaded_html_ids[href] = get_html_ids(
res.read())
except Exception:
ids = downloaded_html_ids[href] = frozenset()
if frag not in ids:
ans.append(
(locations,
ValueError(
'HTML anchor {} not found on the page'.
format(frag)), full_href))
res.close()
finally:
done.append(None)
progress_callback(len(done), len(external_links))
workers = [
Thread(name="CheckLinks", target=check_links)
for i in range(min(10, len(external_links)))
]
for w in workers:
w.daemon = True
w.start()
for w in workers:
w.join()
return ans
class Pool(Thread):
daemon = True
def __init__(self, max_workers=None, name=None):
Thread.__init__(self, name=name)
self.max_workers = max_workers or detect_ncpus()
self.available_workers = []
self.busy_workers = {}
self.pending_jobs = []
self.events = Queue()
self.results = Queue()
self.tracker = Queue()
self.terminal_failure = None
self.common_data = pickle_dumps(None)
self.shutting_down = False
self.start()
def set_common_data(self, data=None):
''' Set some data that will be passed to all subsequent jobs without
needing to be transmitted every time. You must call this method before
queueing any jobs, otherwise the behavior is undefined. You can call it
after all jobs are done, then it will be used for the new round of
jobs. Can raise the :class:`Failure` exception is data could not be
sent to workers.'''
if self.failed:
raise Failure(self.terminal_failure)
self.events.put(data)
def __call__(self, job_id, module, func, *args, **kwargs):
'''
Schedule a job. The job will be run in a worker process, with the
result placed in self.results. If a terminal failure has occurred
previously, this method will raise the :class:`Failure` exception.
:param job_id: A unique id for the job. The result will have this id.
:param module: Either a fully qualified python module name or python
source code which will be executed as a module.
Source code is detected by the presence of newlines in module.
:param func: Name of the function from ``module`` that will be
executed. ``args`` and ``kwargs`` will be passed to the function.
'''
if self.failed:
raise Failure(self.terminal_failure)
job = Job(job_id, module, func, args, kwargs)
self.tracker.put(None)
self.events.put(job)
def wait_for_tasks(self, timeout=None):
''' Wait for all queued jobs to be completed, if timeout is not None,
will raise a RuntimeError if jobs are not completed in the specified
time. Will raise a :class:`Failure` exception if a terminal failure has
occurred previously. '''
if self.failed:
raise Failure(self.terminal_failure)
if timeout is None:
self.tracker.join()
else:
join_with_timeout(self.tracker, timeout)
def shutdown(self, wait_time=0.1):
''' Shutdown this pool, terminating all worker process. The pool cannot
be used after a shutdown. '''
self.shutting_down = True
self.events.put(None)
self.shutdown_workers(wait_time=wait_time)
def create_worker(self):
a, b = Pipe()
with a:
cmd = 'from {0} import run_main, {1}; run_main({2!r}, {1})'.format(
self.__class__.__module__, 'worker_main', a.fileno())
p = start_worker(cmd, (a.fileno(), ))
sys.stdout.flush()
p.stdin.close()
w = Worker(p, b, self.events, self.name)
if self.common_data != pickle_dumps(None):
w.set_common_data(self.common_data)
return w
def start_worker(self):
try:
w = self.create_worker()
if not self.shutting_down:
self.available_workers.append(w)
except Exception:
import traceback
self.terminal_failure = TerminalFailure(
'Failed to start worker process', traceback.format_exc(), None)
self.terminal_error()
return False
def run(self):
if self.start_worker() is False:
return
while True:
event = self.events.get()
if event is None or self.shutting_down:
break
if self.handle_event(event) is False:
break
def handle_event(self, event):
if isinstance(event, Job):
job = event
if not self.available_workers:
if len(self.busy_workers) >= self.max_workers:
self.pending_jobs.append(job)
return
if self.start_worker() is False:
return False
return self.run_job(job)
elif isinstance(event, WorkerResult):
worker_result = event
self.busy_workers.pop(worker_result.worker, None)
self.available_workers.append(worker_result.worker)
self.tracker.task_done()
if worker_result.is_terminal_failure:
self.terminal_failure = TerminalFailure(
'Worker process crashed while executing job',
worker_result.result.traceback, worker_result.id)
self.terminal_error()
return False
self.results.put(worker_result)
else:
self.common_data = pickle_dumps(event)
if len(self.common_data) > MAX_SIZE:
self.cd_file = PersistentTemporaryFile('pool_common_data')
with self.cd_file as f:
f.write(self.common_data)
self.common_data = pickle_dumps(File(f.name))
for worker in self.available_workers:
try:
worker.set_common_data(self.common_data)
except Exception:
import traceback
self.terminal_failure = TerminalFailure(
'Worker process crashed while sending common data',
traceback.format_exc(), None)
self.terminal_error()
return False
while self.pending_jobs and self.available_workers:
if self.run_job(self.pending_jobs.pop()) is False:
return False
def run_job(self, job):
worker = self.available_workers.pop()
try:
worker(job)
except Exception:
import traceback
self.terminal_failure = TerminalFailure(
'Worker process crashed while sending job',
traceback.format_exc(), job.id)
self.terminal_error()
return False
self.busy_workers[worker] = job
@property
def failed(self):
return self.terminal_failure is not None
def terminal_error(self):
if self.shutting_down:
return
for worker, job in iteritems(self.busy_workers):
self.results.put(
WorkerResult(job.id, Result(None, None, None), True, worker))
self.tracker.task_done()
while self.pending_jobs:
job = self.pending_jobs.pop()
self.results.put(
WorkerResult(job.id, Result(None, None, None), True, None))
self.tracker.task_done()
self.shutdown()
def shutdown_workers(self, wait_time=0.1):
self.worker_data = self.common_data = None
for worker in self.busy_workers:
if worker.process.poll() is None:
try:
worker.process.terminate()
except OSError:
pass # If the process has already been killed
workers = [
w.process for w in self.available_workers + list(self.busy_workers)
]
aw = list(self.available_workers)
def join():
for w in aw:
try:
w(None)
except Exception:
pass
for w in workers:
try:
w.wait()
except Exception:
pass
reaper = Thread(target=join, name='ReapPoolWorkers')
reaper.daemon = True
reaper.start()
reaper.join(wait_time)
for w in self.available_workers + list(self.busy_workers):
try:
w.conn.close()
except Exception:
pass
for w in workers:
if w.poll() is None:
try:
w.kill()
except OSError:
pass
del self.available_workers[:]
self.busy_workers.clear()
if hasattr(self, 'cd_file'):
try:
os.remove(self.cd_file.name)
except OSError:
pass
class ThreadedJobServer(Thread):
def __init__(self):
Thread.__init__(self)
self.daemon = True
self.lock = RLock()
self.queued_jobs = []
self.running_jobs = set()
self.changed_jobs = Queue()
self.keep_going = True
def close(self):
self.keep_going = False
def add_job(self, job):
with self.lock:
self.queued_jobs.append(job)
if not self.is_alive():
self.start()
def run(self):
while self.keep_going:
try:
self.run_once()
except:
import traceback
traceback.print_exc()
time.sleep(0.1)
def run_once(self):
with self.lock:
remove = set()
for worker in self.running_jobs:
if worker.is_alive():
# Get progress notifications
if worker.job.consume_notifications():
self.changed_jobs.put(worker.job)
else:
remove.add(worker)
self.changed_jobs.put(worker.job)
for worker in remove:
self.running_jobs.remove(worker)
jobs = self.get_startable_jobs()
for job in jobs:
w = ThreadedJobWorker(job)
w.start()
self.running_jobs.add(w)
self.changed_jobs.put(job)
self.queued_jobs.remove(job)
def kill_job(self, job):
with self.lock:
if job in self.queued_jobs:
self.queued_jobs.remove(job)
elif job in self.running_jobs:
self.running_jobs.remove(job)
job.kill()
self.changed_jobs.put(job)
def running_jobs_of_type(self, type_):
return len([w for w in self.running_jobs if w.job.type == type_])
def get_startable_jobs(self):
queued_types = []
ans = []
for job in self.queued_jobs:
num = self.running_jobs_of_type(job.type)
num += queued_types.count(job.type)
if num < job.max_concurrent_count:
queued_types.append(job.type)
ans.append(job)
return ans
class Repl(Thread):
LINE_CONTINUATION_CHARS = r'\:'
daemon = True
def __init__(self, ps1='>>> ', ps2='... ', show_js=False, libdir=None):
Thread.__init__(self, name='RapydScriptREPL')
self.to_python = to_python
self.JSError = JSError
self.enc = getattr(sys.stdin, 'encoding', None) or 'utf-8'
try:
import readline
self.readline = readline
except ImportError:
pass
self.output = ANSIStream(sys.stdout)
self.to_repl = Queue()
self.from_repl = Queue()
self.ps1, self.ps2 = ps1, ps2
self.show_js, self.libdir = show_js, libdir
self.prompt = ''
self.completions = None
self.start()
def init_ctx(self):
self.prompt = self.ps1
self.ctx = compiler()
self.ctx.g.Duktape.write = self.output.write
self.ctx.eval(
r'''console = { log: function() { Duktape.write(Array.prototype.slice.call(arguments).join(' ') + '\n');}};
console['error'] = console['log'];''')
self.ctx.g.repl_options = {
'show_js': self.show_js,
'histfile': False,
'input': True,
'output': True,
'ps1': self.ps1,
'ps2': self.ps2,
'terminal': self.output.isatty,
'enum_global': 'Object.keys(this)',
'lib_path': self.libdir or os.path.dirname(
P(COMPILER_PATH
)) # TODO: Change this to load pyj files from the src code
}
def get_from_repl(self):
while True:
try:
return self.from_repl.get(True, 1)
except Empty:
if not self.is_alive():
raise SystemExit(1)
def run(self):
self.init_ctx()
rl = None
def set_prompt(p):
self.prompt = p
def prompt(lw):
self.from_repl.put(to_python(lw))
self.ctx.g.set_prompt = set_prompt
self.ctx.g.prompt = prompt
self.ctx.eval('''
listeners = {};
rl = {
setPrompt:set_prompt,
write:Duktape.write,
clearLine: function() {},
on: function(ev, cb) { listeners[ev] = cb; return rl; },
prompt: prompt,
sync_prompt: true,
send_line: function(line) { listeners['line'](line); },
send_interrupt: function() { listeners['SIGINT'](); },
close: function() {listeners['close'](); },
};
repl_options.readline = { createInterface: function(options) { rl.completer = options.completer; return rl; }};
exports.init_repl(repl_options)
''',
fname='<init repl>')
rl = self.ctx.g.rl
completer = to_python(rl.completer)
send_interrupt = to_python(rl.send_interrupt)
send_line = to_python(rl.send_line)
while True:
ev, line = self.to_repl.get()
try:
if ev == 'SIGINT':
self.output.write('\n')
send_interrupt()
elif ev == 'line':
send_line(line)
else:
val = completer(line)
val = to_python(val)
self.from_repl.put(val[0])
except Exception as e:
if isinstance(e, JSError):
print(e.stack or error_message(e), file=sys.stderr)
else:
import traceback
traceback.print_exc()
for i in range(100):
# Do this many times to ensure we dont deadlock
self.from_repl.put(None)
def __call__(self):
if hasattr(self, 'readline'):
history = os.path.join(cache_dir(), 'pyj-repl-history.txt')
self.readline.parse_and_bind("tab: complete")
try:
self.readline.read_history_file(history)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
atexit.register(partial(self.readline.write_history_file, history))
def completer(text, num):
if self.completions is None:
self.to_repl.put(('complete', text))
self.completions = list(filter(None, self.get_from_repl()))
if not self.completions:
return None
try:
return self.completions[num]
except (IndexError, TypeError, AttributeError, KeyError):
self.completions = None
if hasattr(self, 'readline'):
self.readline.set_completer(completer)
while True:
lw = self.get_from_repl()
if lw is None:
raise SystemExit(1)
q = self.prompt
if hasattr(self, 'readline'):
self.readline.set_pre_input_hook(lambda: (
self.readline.insert_text(lw), self.readline.redisplay()))
else:
q += lw
try:
line = raw_input(q)
self.to_repl.put(('line', line))
except EOFError:
return
except KeyboardInterrupt:
self.to_repl.put(('SIGINT', None))
class SearchPanel(QWidget): # {{{
search_requested = pyqtSignal(object)
results_found = pyqtSignal(object)
show_search_result = pyqtSignal(object)
count_changed = pyqtSignal(object)
hide_search_panel = pyqtSignal()
goto_cfi = pyqtSignal(object)
def __init__(self, parent=None):
QWidget.__init__(self, parent)
self.discovery_counter = 0
self.last_hidden_text_warning = None
self.current_search = None
self.anchor_cfi = None
self.l = l = QVBoxLayout(self)
l.setContentsMargins(0, 0, 0, 0)
self.search_input = si = SearchInput(self)
self.searcher = None
self.search_tasks = Queue()
self.results_found.connect(self.on_result_found,
type=Qt.ConnectionType.QueuedConnection)
si.do_search.connect(self.search_requested)
si.cleared.connect(self.search_cleared)
si.go_back.connect(self.go_back)
l.addWidget(si)
self.results = r = Results(self)
r.count_changed.connect(self.count_changed)
r.show_search_result.connect(self.do_show_search_result,
type=Qt.ConnectionType.QueuedConnection)
r.current_result_changed.connect(self.update_hidden_message)
l.addWidget(r, 100)
self.spinner = s = BusySpinner(self)
s.setVisible(False)
l.addWidget(s)
self.hidden_message = la = QLabel(
_('This text is hidden in the book and cannot be displayed'))
la.setStyleSheet('QLabel { margin-left: 1ex }')
la.setWordWrap(True)
la.setVisible(False)
l.addWidget(la)
def go_back(self):
if self.anchor_cfi:
self.goto_cfi.emit(self.anchor_cfi)
def update_hidden_message(self):
self.hidden_message.setVisible(self.results.current_result_is_hidden)
def focus_input(self, text=None):
self.search_input.focus_input(text)
def search_cleared(self):
self.results.clear_all_results()
self.current_search = None
def start_search(self, search_query, current_name):
if self.current_search is not None and search_query == self.current_search:
self.find_next_requested(search_query.backwards)
return
if self.searcher is None:
self.searcher = Thread(name='Searcher', target=self.run_searches)
self.searcher.daemon = True
self.searcher.start()
self.results.clear_all_results()
self.hidden_message.setVisible(False)
self.spinner.start()
self.current_search = search_query
self.last_hidden_text_warning = None
self.search_tasks.put((search_query, current_name))
self.discovery_counter += 1
def set_anchor_cfi(self, pos_data):
self.anchor_cfi = pos_data['cfi']
def run_searches(self):
while True:
x = self.search_tasks.get()
if x is None:
break
search_query, current_name = x
try:
manifest = get_manifest() or {}
spine = manifest.get('spine', ())
idx_map = {name: i for i, name in enumerate(spine)}
spine_idx = idx_map.get(current_name, -1)
except Exception:
import traceback
traceback.print_exc()
spine_idx = -1
if spine_idx < 0:
self.results_found.emit(SearchFinished(search_query))
continue
num_in_spine = len(spine)
result_num = 0
for n in range(num_in_spine):
idx = (spine_idx + n) % num_in_spine
name = spine[idx]
counter = Counter()
try:
for i, result in enumerate(
search_in_name(name, search_query)):
before, text, after, offset = result
q = (before or '')[-15:] + text + (after or '')[:15]
result_num += 1
self.results_found.emit(
SearchResult(search_query, before, text, after, q,
name, idx, counter[q], offset,
result_num))
counter[q] += 1
except Exception:
import traceback
traceback.print_exc()
self.results_found.emit(SearchFinished(search_query))
def on_result_found(self, result):
if self.current_search is None or result.search_query != self.current_search:
return
if isinstance(result, SearchFinished):
self.spinner.stop()
if self.results.number_of_results:
self.results.ensure_current_result_visible()
else:
self.show_no_results_found()
return
self.results.add_result(result)
obj = result.for_js
obj['on_discovery'] = self.discovery_counter
self.show_search_result.emit(obj)
self.update_hidden_message()
def visibility_changed(self, visible):
if visible:
self.focus_input()
def clear_searches(self):
self.current_search = None
self.last_hidden_text_warning = None
searchable_text_for_name.cache_clear()
toc_offset_map_for_name.cache_clear()
get_toc_data.cache_clear()
self.spinner.stop()
self.results.clear_all_results()
def shutdown(self):
self.search_tasks.put(None)
self.spinner.stop()
self.current_search = None
self.last_hidden_text_warning = None
self.searcher = None
def find_next_requested(self, previous):
self.results.find_next(previous)
def trigger(self):
self.search_input.find_next()
def do_show_search_result(self, sr):
self.show_search_result.emit(sr.for_js)
def search_result_not_found(self, sr):
self.results.search_result_not_found(sr)
self.update_hidden_message()
def search_result_discovered(self, sr):
self.results.search_result_discovered(sr)
def show_no_results_found(self):
msg = _('No matches were found for:')
warning_dialog(self,
_('No matches found'),
msg + f' <b>{self.current_search.text}</b>',
show=True)
def keyPressEvent(self, ev):
if ev.key() == Qt.Key.Key_Escape:
self.hide_search_panel.emit()
ev.accept()
return
return QWidget.keyPressEvent(self, ev)
class ParseWorker(Thread):
daemon = True
SLEEP_TIME = 1
def __init__(self):
Thread.__init__(self)
self.requests = Queue()
self.request_count = 0
self.parse_items = {}
self.launch_error = None
def run(self):
mod, func = 'calibre.gui2.tweak_book.preview', 'parse_html'
try:
# Connect to the worker and send a dummy job to initialize it
self.worker = offload_worker(priority='low')
self.worker(mod, func, '<p></p>')
except:
import traceback
traceback.print_exc()
self.launch_error = traceback.format_exc()
return
while True:
time.sleep(self.SLEEP_TIME)
x = self.requests.get()
requests = [x]
while True:
try:
requests.append(self.requests.get_nowait())
except Empty:
break
if shutdown in requests:
self.worker.shutdown()
break
request = sorted(requests, reverse=True)[0]
del requests
pi, data = request[1:]
try:
res = self.worker(mod, func, data)
except:
import traceback
traceback.print_exc()
else:
pi.parsing_done = True
parsed_data = res['result']
if res['tb']:
prints("Parser error:")
prints(res['tb'])
else:
pi.parsed_data = parsed_data
def add_request(self, name):
data = get_data(name)
ldata, hdata = len(data), hash(data)
pi = self.parse_items.get(name, None)
if pi is None:
self.parse_items[name] = pi = ParseItem(name)
else:
if pi.parsing_done and pi.length == ldata and pi.fingerprint == hdata:
return
pi.parsed_data = None
pi.parsing_done = False
pi.length, pi.fingerprint = ldata, hdata
self.requests.put((self.request_count, pi, data))
self.request_count += 1
def shutdown(self):
self.requests.put(shutdown)
def get_data(self, name):
return getattr(self.parse_items.get(name, None), 'parsed_data', None)
def clear(self):
self.parse_items.clear()
def is_alive(self):
return Thread.is_alive(self) or (hasattr(self, 'worker') and self.worker.is_alive())
class ParseWorker(Thread):
daemon = True
SLEEP_TIME = 1
def __init__(self):
Thread.__init__(self)
self.requests = Queue()
self.request_count = 0
self.parse_items = {}
self.launch_error = None
def run(self):
mod, func = 'calibre.gui2.tweak_book.preview', 'parse_html'
try:
# Connect to the worker and send a dummy job to initialize it
self.worker = offload_worker(priority='low')
self.worker(mod, func, '<p></p>')
except:
import traceback
traceback.print_exc()
self.launch_error = traceback.format_exc()
return
while True:
time.sleep(self.SLEEP_TIME)
x = self.requests.get()
requests = [x]
while True:
try:
requests.append(self.requests.get_nowait())
except Empty:
break
if shutdown in requests:
self.worker.shutdown()
break
request = sorted(requests, reverse=True)[0]
del requests
pi, data = request[1:]
try:
res = self.worker(mod, func, data)
except:
import traceback
traceback.print_exc()
else:
pi.parsing_done = True
parsed_data = res['result']
if res['tb']:
prints("Parser error:")
prints(res['tb'])
else:
pi.parsed_data = parsed_data
def add_request(self, name):
data = get_data(name)
ldata, hdata = len(data), hash(data)
pi = self.parse_items.get(name, None)
if pi is None:
self.parse_items[name] = pi = ParseItem(name)
else:
if pi.parsing_done and pi.length == ldata and pi.fingerprint == hdata:
return
pi.parsed_data = None
pi.parsing_done = False
pi.length, pi.fingerprint = ldata, hdata
self.requests.put((self.request_count, pi, data))
self.request_count += 1
def shutdown(self):
self.requests.put(shutdown)
def get_data(self, name):
return getattr(self.parse_items.get(name, None), 'parsed_data', None)
def clear(self):
self.parse_items.clear()
def is_alive(self):
return Thread.is_alive(self) or (hasattr(self, 'worker')
and self.worker.is_alive())
class WebSocketConnection(HTTPConnection):
# Internal API {{{
in_websocket_mode = False
websocket_handler = None
def __init__(self, *args, **kwargs):
global conn_id
HTTPConnection.__init__(self, *args, **kwargs)
self.sendq = Queue()
self.control_frames = deque()
self.cf_lock = Lock()
self.sending = None
self.send_buf = None
self.frag_decoder = UTF8Decoder()
self.ws_close_received = self.ws_close_sent = False
conn_id += 1
self.websocket_connection_id = conn_id
self.stop_reading = False
def finalize_headers(self, inheaders):
upgrade = inheaders.get('Upgrade', '')
key = inheaders.get('Sec-WebSocket-Key', None)
conn = {
x.strip().lower()
for x in inheaders.get('Connection', '').split(',')
}
if key is None or upgrade.lower(
) != 'websocket' or 'upgrade' not in conn:
return HTTPConnection.finalize_headers(self, inheaders)
ver = inheaders.get('Sec-WebSocket-Version', 'Unknown')
try:
ver_ok = int(ver) >= 13
except Exception:
ver_ok = False
if not ver_ok:
return self.simple_response(
http_client.BAD_REQUEST,
'Unsupported WebSocket protocol version: %s' % ver)
if self.method != 'GET':
return self.simple_response(
http_client.BAD_REQUEST,
'Invalid WebSocket method: %s' % self.method)
response = HANDSHAKE_STR % as_base64_unicode(
sha1((key + GUID_STR).encode('utf-8')).digest())
self.optimize_for_sending_packet()
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.set_state(WRITE, self.upgrade_connection_to_ws,
ReadOnlyFileBuffer(response.encode('ascii')), inheaders)
def upgrade_connection_to_ws(self, buf, inheaders, event):
if self.write(buf):
if self.websocket_handler is None:
self.websocket_handler = DummyHandler()
self.read_frame, self.current_recv_opcode = ReadFrame(), None
self.in_websocket_mode = True
try:
self.websocket_handler.handle_websocket_upgrade(
self.websocket_connection_id, weakref.ref(self), inheaders)
except Exception as err:
self.log.exception('Error in WebSockets upgrade handler:')
self.websocket_close(
UNEXPECTED_ERROR,
'Unexpected error in handler: %r' % as_unicode(err))
self.handle_event = self.ws_duplex
self.set_ws_state()
self.end_send_optimization()
def set_ws_state(self):
if self.ws_close_sent or self.ws_close_received:
if self.ws_close_sent:
self.ready = False
else:
self.wait_for = WRITE
return
if self.send_buf is not None or self.sending is not None:
self.wait_for = RDWR
else:
try:
self.sending = self.sendq.get_nowait()
except Empty:
with self.cf_lock:
if self.control_frames:
self.wait_for = RDWR
else:
self.wait_for = READ
else:
self.wait_for = RDWR
if self.stop_reading:
if self.wait_for is READ:
self.ready = False
elif self.wait_for is RDWR:
self.wait_for = WRITE
def ws_duplex(self, event):
if event is READ:
self.ws_read()
elif event is WRITE:
self.ws_write()
self.set_ws_state()
def ws_read(self):
if not self.stop_reading:
self.read_frame(self)
def ws_data_received(self, data, opcode, frame_starting, frame_finished,
is_final_frame_of_message):
if opcode in CONTROL_CODES:
return self.ws_control_frame(opcode, data)
message_starting = self.current_recv_opcode is None
if message_starting:
if opcode == CONTINUATION:
self.log.error(
'Client sent continuation frame with no message to continue'
)
self.websocket_close(
PROTOCOL_ERROR,
'Continuation frame without any message to continue')
return
self.current_recv_opcode = opcode
elif frame_starting and opcode != CONTINUATION:
self.log.error(
'Client sent continuation frame with non-zero opcode')
self.websocket_close(PROTOCOL_ERROR,
'Continuation frame with non-zero opcode')
return
message_finished = frame_finished and is_final_frame_of_message
if self.current_recv_opcode == TEXT:
if message_starting:
self.frag_decoder.reset()
empty_data = len(data) == 0
try:
data = self.frag_decoder(data)
except ValueError:
self.frag_decoder.reset()
self.log.error('Client sent undecodeable UTF-8')
return self.websocket_close(INCONSISTENT_DATA,
'Not valid UTF-8')
if message_finished:
if (not data and not empty_data) or self.frag_decoder.state:
self.frag_decoder.reset()
self.log.error('Client sent undecodeable UTF-8')
return self.websocket_close(INCONSISTENT_DATA,
'Not valid UTF-8')
if message_finished:
self.current_recv_opcode = None
self.frag_decoder.reset()
try:
self.handle_websocket_data(data, message_starting,
message_finished)
except Exception as err:
self.log.exception('Error in WebSockets data handler:')
self.websocket_close(
UNEXPECTED_ERROR,
'Unexpected error in handler: %r' % as_unicode(err))
def ws_control_frame(self, opcode, data):
if opcode in (PING, CLOSE):
rcode = PONG if opcode == PING else CLOSE
if opcode == CLOSE:
self.ws_close_received = True
self.stop_reading = True
if data:
try:
close_code = unpack_from(b'!H', data)[0]
except struct_error:
data = pack(
b'!H', PROTOCOL_ERROR
) + b'close frame data must be atleast two bytes'
else:
try:
utf8_decode(data[2:])
except ValueError:
data = pack(
b'!H', PROTOCOL_ERROR
) + b'close frame data must be valid UTF-8'
else:
if close_code < 1000 or close_code in RESERVED_CLOSE_CODES or (
1011 < close_code < 3000):
data = pack(
b'!H',
PROTOCOL_ERROR) + b'close code reserved'
else:
close_code = NORMAL_CLOSE
data = pack(b'!H', close_code)
f = ReadOnlyFileBuffer(create_frame(1, rcode, data))
f.is_close_frame = opcode == CLOSE
with self.cf_lock:
self.control_frames.append(f)
elif opcode == PONG:
try:
self.websocket_handler.handle_websocket_pong(
self.websocket_connection_id, data)
except Exception:
self.log.exception('Error in PONG handler:')
self.set_ws_state()
def websocket_close(self, code=NORMAL_CLOSE, reason=b''):
if isinstance(reason, unicode_type):
reason = reason.encode('utf-8')
self.stop_reading = True
reason = reason[:123]
if code is None and not reason:
f = ReadOnlyFileBuffer(create_frame(1, CLOSE, b''))
else:
f = ReadOnlyFileBuffer(
create_frame(1, CLOSE,
pack(b'!H', code) + reason))
f.is_close_frame = True
with self.cf_lock:
self.control_frames.append(f)
self.set_ws_state()
def ws_write(self):
if self.ws_close_sent:
return
if self.send_buf is not None:
if self.write(self.send_buf):
self.end_send_optimization()
if getattr(self.send_buf, 'is_close_frame', False):
self.ws_close_sent = True
self.send_buf = None
else:
with self.cf_lock:
try:
self.send_buf = self.control_frames.popleft()
except IndexError:
if self.sending is not None:
self.send_buf = self.sending.create_frame()
if self.send_buf is None:
self.sending = None
if self.send_buf is not None:
self.optimize_for_sending_packet()
def close(self):
if self.in_websocket_mode:
try:
self.websocket_handler.handle_websocket_close(
self.websocket_connection_id)
except Exception:
self.log.exception('Error in WebSocket close handler')
# Try to write a close frame, just once
try:
if self.send_buf is None and not self.ws_close_sent:
self.websocket_close(SHUTTING_DOWN, 'Shutting down')
with self.cf_lock:
self.write(self.control_frames.pop())
except Exception:
pass
Connection.close(self)
else:
HTTPConnection.close(self)
# }}}
def send_websocket_message(self, buf, wakeup=True):
''' Send a complete message. This class will take care of splitting it
into appropriate frames automatically. `buf` must be a file like object. '''
self.sendq.put(MessageWriter(buf))
self.wait_for = RDWR
if wakeup:
self.wakeup()
def send_websocket_frame(self, data, is_first=True, is_last=True):
''' Useful for streaming handlers that want to break up messages into
frames themselves. Note that these frames will be interleaved with
control frames, so they should not be too large. '''
opcode = (TEXT if isinstance(data, unicode_type) else
BINARY) if is_first else CONTINUATION
fin = 1 if is_last else 0
frame = create_frame(fin, opcode, data)
with self.cf_lock:
self.control_frames.append(ReadOnlyFileBuffer(frame))
def send_websocket_ping(self, data=b''):
''' Send a PING to the remote client, it should reply with a PONG which
will be sent to the handle_websocket_pong callback in your handler. '''
if isinstance(data, unicode_type):
data = data.encode('utf-8')
frame = create_frame(True, PING, data)
with self.cf_lock:
self.control_frames.append(ReadOnlyFileBuffer(frame))
def handle_websocket_data(self, data, message_starting, message_finished):
''' Called when some data is received from the remote client. In
general the data may not constitute a complete "message", use the
message_starting and message_finished flags to re-assemble it into a
complete message in the handler. Note that for binary data, data is a
mutable object. If you intend to keep it around after this method
returns, create a bytestring from it, using tobytes(). '''
self.websocket_handler.handle_websocket_data(
self.websocket_connection_id, data, message_starting,
message_finished)
class JobsManager(object):
def __init__(self, opts, log):
mj = opts.max_jobs
if mj < 1:
mj = detect_ncpus()
self.log = log
self.max_jobs = max(1, mj)
self.max_job_time = max(0, opts.max_job_time * 60)
self.lock = RLock()
self.jobs = {}
self.finished_jobs = {}
self.events = Queue()
self.job_id = count()
self.waiting_job_ids = set()
self.waiting_jobs = deque()
self.max_block = None
self.shutting_down = False
self.event_loop = None
def start_job(self,
name,
module,
func,
args=(),
kwargs=None,
job_done_callback=None,
job_data=None):
with self.lock:
if self.shutting_down:
return None
if self.event_loop is None:
self.event_loop = t = Thread(name='JobsEventLoop',
target=self.run)
t.daemon = True
t.start()
job_id = next(self.job_id)
self.events.put(
StartEvent(job_id, name, module, func, args, kwargs or {},
job_done_callback, job_data))
self.waiting_job_ids.add(job_id)
return job_id
def job_status(self, job_id):
with self.lock:
if not self.shutting_down:
if job_id in self.finished_jobs:
job = self.finished_jobs[job_id]
return 'finished', job.result, job.traceback, job.was_aborted
if job_id in self.jobs:
return 'running', None, None, None
if job_id in self.waiting_job_ids:
return 'waiting', None, None, None
return None, None, None, None
def abort_job(self, job_id):
job = self.jobs.get(job_id)
if job is not None:
job.abort_event.set()
def wait_for_running_job(self, job_id, timeout=None):
job = self.jobs.get(job_id)
if job is not None:
job.wait_for_end.wait(timeout)
if not job.done:
return False
while job_id not in self.finished_jobs:
time.sleep(0.001)
return True
def shutdown(self, timeout=5.0):
with self.lock:
self.shutting_down = True
for job in itervalues(self.jobs):
job.abort_event.set()
self.events.put(False)
def wait_for_shutdown(self, wait_till):
for job in itervalues(self.jobs):
delta = wait_till - monotonic()
if delta > 0:
job.join(delta)
if self.event_loop is not None:
delta = wait_till - monotonic()
if delta > 0:
self.event_loop.join(delta)
# Internal API {{{
def run(self):
while not self.shutting_down:
if self.max_block is None:
ev = self.events.get()
else:
try:
ev = self.events.get(block=True, timeout=self.max_block)
except Empty:
ev = None
if self.shutting_down:
break
if ev is None:
self.abort_hanging_jobs()
elif isinstance(ev, StartEvent):
self.waiting_jobs.append(ev)
self.start_waiting_jobs()
elif isinstance(ev, DoneEvent):
self.job_finished(ev.job_id)
elif ev is False:
break
def start_waiting_jobs(self):
with self.lock:
while self.waiting_jobs and len(self.jobs) < self.max_jobs:
ev = self.waiting_jobs.popleft()
self.jobs[ev.job_id] = Job(ev, self.events)
self.waiting_job_ids.discard(ev.job_id)
self.update_max_block()
def update_max_block(self):
with self.lock:
mb = None
now = monotonic()
for job in itervalues(self.jobs):
if not job.done and not job.abort_event.is_set():
delta = self.max_job_time - (now - job.start_time)
if delta <= 0:
self.max_block = 0
return
if mb is None:
mb = delta
else:
mb = min(mb, delta)
self.max_block = mb
def abort_hanging_jobs(self):
now = monotonic()
found = False
for job in itervalues(self.jobs):
if not job.done and not job.abort_event.is_set():
delta = self.max_job_time - (now - job.start_time)
if delta <= 0:
job.abort_event.set()
found = True
if found:
self.update_max_block()
def job_finished(self, job_id):
with self.lock:
self.finished_jobs[job_id] = job = self.jobs.pop(job_id)
if job.callback is not None:
try:
job.callback(job)
except Exception:
import traceback
self.log.error('Error running callback for job: %s:\n%s' %
(job.name, traceback.format_exc()))
self.prune_finished_jobs()
if job.traceback and not job.was_aborted:
logdata = job.read_log()
self.log.error('The job: %s failed:\n%s\n%s' %
(job.job_name, logdata, job.traceback))
job.remove_log()
self.start_waiting_jobs()
def prune_finished_jobs(self):
with self.lock:
remove = []
now = monotonic()
for job_id, job in iteritems(self.finished_jobs):
if now - job.end_time > 3600:
remove.append(job_id)
for job_id in remove:
del self.finished_jobs[job_id]
class CompletionWorker(Thread):
daemon = True
def __init__(self, result_callback=lambda x: x, worker_entry_point='main'):
Thread.__init__(self)
self.worker_entry_point = worker_entry_point
self.start()
self.main_queue = Queue()
self.result_callback = result_callback
self.reap_thread = None
self.shutting_down = False
self.connected = Event()
self.current_completion_request = None
self.latest_completion_request_id = None
self.request_count = 0
self.lock = RLock()
def launch_worker_process(self):
from calibre.utils.ipc.server import create_listener
from calibre.utils.ipc.pool import start_worker
self.worker_process = p = start_worker(
'from {0} import run_main, {1}; run_main({1})'.format(
self.__class__.__module__, self.worker_entry_point))
auth_key = os.urandom(32)
address, self.listener = create_listener(auth_key)
eintr_retry_call(p.stdin.write, msgpack_dumps((address, auth_key)))
p.stdin.flush(), p.stdin.close()
self.control_conn = eintr_retry_call(self.listener.accept)
self.data_conn = eintr_retry_call(self.listener.accept)
self.data_thread = t = Thread(name='CWData',
target=self.handle_data_requests)
t.daemon = True
t.start()
self.connected.set()
def send(self, data, conn=None):
conn = conn or self.control_conn
try:
eintr_retry_call(conn.send, data)
except:
if not self.shutting_down:
raise
def recv(self, conn=None):
conn = conn or self.control_conn
try:
return eintr_retry_call(conn.recv)
except:
if not self.shutting_down:
raise
def wait_for_connection(self, timeout=None):
self.connected.wait(timeout)
def handle_data_requests(self):
from calibre.gui2.tweak_book.completion.basic import handle_data_request
while True:
try:
req = self.recv(self.data_conn)
except EOFError:
break
except Exception:
import traceback
traceback.print_exc()
break
if req is None or self.shutting_down:
break
result, tb = handle_data_request(req)
try:
self.send((result, tb), self.data_conn)
except EOFError:
break
except Exception:
import traceback
traceback.print_exc()
break
def run(self):
self.launch_worker_process()
while True:
obj = self.main_queue.get()
if obj is None:
break
req_type, req_data = obj
try:
if req_type is COMPLETION_REQUEST:
with self.lock:
if self.current_completion_request is not None:
ccr, self.current_completion_request = self.current_completion_request, None
self.send_completion_request(ccr)
elif req_type is CLEAR_REQUEST:
self.send(req_data)
except EOFError:
break
except Exception:
import traceback
traceback.print_exc()
def send_completion_request(self, request):
self.send(request)
result = self.recv()
if result.request_id == self.latest_completion_request_id:
try:
self.result_callback(result)
except Exception:
import traceback
traceback.print_exc()
def clear_caches(self, cache_type=None):
self.main_queue.put(
(CLEAR_REQUEST, Request(None, 'clear_caches', cache_type, None)))
def queue_completion(self,
request_id,
completion_type,
completion_data,
query=None):
with self.lock:
self.current_completion_request = Request(request_id,
completion_type,
completion_data, query)
self.latest_completion_request_id = self.current_completion_request.id
self.main_queue.put((COMPLETION_REQUEST, None))
def shutdown(self):
self.shutting_down = True
self.main_queue.put(None)
for conn in (getattr(self, 'control_conn',
None), getattr(self, 'data_conn', None)):
try:
conn.close()
except Exception:
pass
p = self.worker_process
if p.poll() is None:
self.worker_process.terminate()
t = self.reap_thread = Thread(target=p.wait)
t.daemon = True
t.start()
def join(self, timeout=0.2):
if self.reap_thread is not None:
self.reap_thread.join(timeout)
if not iswindows and self.worker_process.returncode is None:
self.worker_process.kill()
return self.worker_process.returncode
class Repl(Thread):
LINE_CONTINUATION_CHARS = r'\:'
daemon = True
def __init__(self, ps1='>>> ', ps2='... ', show_js=False, libdir=None):
Thread.__init__(self, name='RapydScriptREPL')
self.to_python = to_python
self.JSError = JSError
self.enc = getattr(sys.stdin, 'encoding', None) or 'utf-8'
try:
import readline
self.readline = readline
except ImportError:
pass
self.output = ANSIStream(sys.stdout)
self.to_repl = Queue()
self.from_repl = Queue()
self.ps1, self.ps2 = ps1, ps2
self.show_js, self.libdir = show_js, libdir
self.prompt = ''
self.completions = None
self.start()
def init_ctx(self):
self.prompt = self.ps1
self.ctx = compiler()
self.ctx.g.Duktape.write = self.output.write
self.ctx.eval(r'''console = { log: function() { Duktape.write(Array.prototype.slice.call(arguments).join(' ') + '\n');}};
console['error'] = console['log'];''')
self.ctx.g.repl_options = {
'show_js': self.show_js,
'histfile':False,
'input':True, 'output':True, 'ps1':self.ps1, 'ps2':self.ps2,
'terminal':self.output.isatty,
'enum_global': 'Object.keys(this)',
'lib_path': self.libdir or os.path.dirname(P(COMPILER_PATH)) # TODO: Change this to load pyj files from the src code
}
def get_from_repl(self):
while True:
try:
return self.from_repl.get(True, 1)
except Empty:
if not self.is_alive():
raise SystemExit(1)
def run(self):
self.init_ctx()
rl = None
def set_prompt(p):
self.prompt = p
def prompt(lw):
self.from_repl.put(to_python(lw))
self.ctx.g.set_prompt = set_prompt
self.ctx.g.prompt = prompt
self.ctx.eval('''
listeners = {};
rl = {
setPrompt:set_prompt,
write:Duktape.write,
clearLine: function() {},
on: function(ev, cb) { listeners[ev] = cb; return rl; },
prompt: prompt,
sync_prompt: true,
send_line: function(line) { listeners['line'](line); },
send_interrupt: function() { listeners['SIGINT'](); },
close: function() {listeners['close'](); },
};
repl_options.readline = { createInterface: function(options) { rl.completer = options.completer; return rl; }};
exports.init_repl(repl_options)
''', fname='<init repl>')
rl = self.ctx.g.rl
completer = to_python(rl.completer)
send_interrupt = to_python(rl.send_interrupt)
send_line = to_python(rl.send_line)
while True:
ev, line = self.to_repl.get()
try:
if ev == 'SIGINT':
self.output.write('\n')
send_interrupt()
elif ev == 'line':
send_line(line)
else:
val = completer(line)
val = to_python(val)
self.from_repl.put(val[0])
except Exception as e:
if isinstance(e, JSError):
print(e.stack or error_message(e), file=sys.stderr)
else:
import traceback
traceback.print_exc()
for i in range(100):
# Do this many times to ensure we dont deadlock
self.from_repl.put(None)
def __call__(self):
if hasattr(self, 'readline'):
history = os.path.join(cache_dir(), 'pyj-repl-history.txt')
self.readline.parse_and_bind("tab: complete")
try:
self.readline.read_history_file(history)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
atexit.register(partial(self.readline.write_history_file, history))
def completer(text, num):
if self.completions is None:
self.to_repl.put(('complete', text))
self.completions = list(filter(None, self.get_from_repl()))
if not self.completions:
return None
try:
return self.completions[num]
except (IndexError, TypeError, AttributeError, KeyError):
self.completions = None
if hasattr(self, 'readline'):
self.readline.set_completer(completer)
while True:
lw = self.get_from_repl()
if lw is None:
raise SystemExit(1)
q = self.prompt
if hasattr(self, 'readline'):
self.readline.set_pre_input_hook(lambda:(self.readline.insert_text(lw), self.readline.redisplay()))
else:
q += lw
try:
line = raw_input(q)
self.to_repl.put(('line', line))
except EOFError:
return
except KeyboardInterrupt:
self.to_repl.put(('SIGINT', None))
class SearchPanel(QWidget): # {{{
search_requested = pyqtSignal(object)
results_found = pyqtSignal(object)
show_search_result = pyqtSignal(object)
def __init__(self, parent=None):
QWidget.__init__(self, parent)
self.last_hidden_text_warning = None
self.current_search = None
self.l = l = QVBoxLayout(self)
l.setContentsMargins(0, 0, 0, 0)
self.search_input = si = SearchInput(self)
self.searcher = None
self.search_tasks = Queue()
self.results_found.connect(self.on_result_found,
type=Qt.QueuedConnection)
si.do_search.connect(self.search_requested)
l.addWidget(si)
self.results = r = Results(self)
r.show_search_result.connect(self.do_show_search_result,
type=Qt.QueuedConnection)
l.addWidget(r, 100)
self.spinner = s = BusySpinner(self)
s.setVisible(False)
l.addWidget(s)
def focus_input(self):
self.search_input.focus_input()
def start_search(self, search_query, current_name):
if self.current_search is not None and search_query == self.current_search:
self.find_next_requested(search_query.backwards)
return
if self.searcher is None:
self.searcher = Thread(name='Searcher', target=self.run_searches)
self.searcher.daemon = True
self.searcher.start()
self.results.clear()
self.spinner.start()
self.current_search = search_query
self.last_hidden_text_warning = None
self.search_tasks.put((search_query, current_name))
def run_searches(self):
while True:
x = self.search_tasks.get()
if x is None:
break
search_query, current_name = x
try:
manifest = get_manifest() or {}
spine = manifest.get('spine', ())
idx_map = {name: i for i, name in enumerate(spine)}
spine_idx = idx_map.get(current_name, -1)
except Exception:
import traceback
traceback.print_exc()
spine_idx = -1
if spine_idx < 0:
self.results_found.emit(SearchFinished(search_query))
continue
for name in spine:
counter = Counter()
spine_idx = idx_map[name]
try:
for i, result in enumerate(
search_in_name(name, search_query)):
before, text, after = result
self.results_found.emit(
SearchResult(search_query, before, text, after,
name, spine_idx, counter[text]))
counter[text] += 1
except Exception:
import traceback
traceback.print_exc()
self.results_found.emit(SearchFinished(search_query))
def on_result_found(self, result):
if self.current_search is None or result.search_query != self.current_search:
return
if isinstance(result, SearchFinished):
self.spinner.stop()
if not self.results.count():
self.show_no_results_found()
return
if self.results.add_result(result) == 1:
# first result
self.results.setCurrentRow(0)
self.results.item_activated()
def visibility_changed(self, visible):
if visible:
self.focus_input()
def clear_searches(self):
self.current_search = None
self.last_hidden_text_warning = None
searchable_text_for_name.cache_clear()
self.spinner.stop()
self.results.clear()
def shutdown(self):
self.search_tasks.put(None)
self.spinner.stop()
self.current_search = None
self.last_hidden_text_warning = None
self.searcher = None
def find_next_requested(self, previous):
self.results.find_next(previous)
def do_show_search_result(self, sr):
self.show_search_result.emit(sr.for_js)
def search_result_not_found(self, sr):
self.results.search_result_not_found(sr)
if self.results.count():
now = monotonic()
if self.last_hidden_text_warning is None or self.current_search != self.last_hidden_text_warning[
1] or now - self.last_hidden_text_warning[0] > 5:
self.last_hidden_text_warning = now, self.current_search
warning_dialog(
self,
_('Hidden text'),
_('Some search results were for hidden or non-reflowable text, they will be removed.'
),
show=True)
elif self.last_hidden_text_warning is not None:
self.last_hidden_text_warning = now, self.last_hidden_text_warning[
1]
if not self.results.count() and not self.spinner.is_running:
self.show_no_results_found()
def show_no_results_found(self):
has_hidden_text = self.last_hidden_text_warning is not None and self.last_hidden_text_warning[
1] == self.current_search
if self.current_search:
if has_hidden_text:
msg = _('No displayable matches were found for:')
else:
msg = _('No matches were found for:')
warning_dialog(self,
_('No matches found'),
msg +
' <b>{}</b>'.format(self.current_search.text),
show=True)
class DeleteService(Thread):
''' Provide a blocking file delete implementation with support for the
recycle bin. On windows, deleting files to the recycle bin spins the event
loop, which can cause locking errors in the main thread. We get around this
by only moving the files/folders to be deleted out of the library in the
main thread, they are deleted to recycle bin in a separate worker thread.
This has the added advantage that doing a restore from the recycle bin wont
cause metadata.db and the file system to get out of sync. Also, deleting
becomes much faster, since in the common case, the move is done by a simple
os.rename(). The downside is that if the user quits calibre while a long
move to recycle bin is happening, the files may not all be deleted.'''
daemon = True
def __init__(self):
Thread.__init__(self)
self.requests = Queue()
def shutdown(self, timeout=20):
self.requests.put(None)
self.join(timeout)
def create_staging(self, library_path):
base_path = os.path.dirname(library_path)
base = os.path.basename(library_path)
try:
ans = tempfile.mkdtemp(prefix=base+' deleted ', dir=base_path)
except OSError:
ans = tempfile.mkdtemp(prefix=base+' deleted ')
atexit.register(remove_dir, ans)
return ans
def remove_dir_if_empty(self, path):
try:
os.rmdir(path)
except OSError as e:
if e.errno == errno.ENOTEMPTY or len(os.listdir(path)) > 0:
# Some linux systems appear to raise an EPERM instead of an
# ENOTEMPTY, see https://bugs.launchpad.net/bugs/1240797
return
raise
def delete_books(self, paths, library_path):
tdir = self.create_staging(library_path)
self.queue_paths(tdir, paths, delete_empty_parent=True)
def queue_paths(self, tdir, paths, delete_empty_parent=True):
try:
self._queue_paths(tdir, paths, delete_empty_parent=delete_empty_parent)
except:
if os.path.exists(tdir):
shutil.rmtree(tdir, ignore_errors=True)
raise
def _queue_paths(self, tdir, paths, delete_empty_parent=True):
requests = []
for path in paths:
if os.path.exists(path):
basename = os.path.basename(path)
c = 0
while True:
dest = os.path.join(tdir, basename)
if not os.path.exists(dest):
break
c += 1
basename = '%d - %s' % (c, os.path.basename(path))
try:
shutil.move(path, dest)
except EnvironmentError:
if os.path.isdir(path):
# shutil.move may have partially copied the directory,
# so the subsequent call to move() will fail as the
# destination directory already exists
raise
# Wait a little in case something has locked a file
time.sleep(1)
shutil.move(path, dest)
if delete_empty_parent:
remove_dir_if_empty(os.path.dirname(path), ignore_metadata_caches=True)
requests.append(dest)
if not requests:
remove_dir_if_empty(tdir)
else:
self.requests.put(tdir)
def delete_files(self, paths, library_path):
tdir = self.create_staging(library_path)
self.queue_paths(tdir, paths, delete_empty_parent=False)
def run(self):
while True:
x = self.requests.get()
try:
if x is None:
break
try:
self.do_delete(x)
except:
import traceback
traceback.print_exc()
finally:
self.requests.task_done()
def wait(self):
'Blocks until all pending deletes have completed'
self.requests.join()
def do_delete(self, tdir):
if os.path.exists(tdir):
try:
for x in os.listdir(tdir):
x = os.path.join(tdir, x)
if os.path.isdir(x):
delete_tree(x)
else:
delete_file(x)
finally:
shutil.rmtree(tdir)
class ThreadedJobServer(Thread):
def __init__(self):
Thread.__init__(self)
self.daemon = True
self.lock = RLock()
self.queued_jobs = []
self.running_jobs = set()
self.changed_jobs = Queue()
self.keep_going = True
def close(self):
self.keep_going = False
def add_job(self, job):
with self.lock:
self.queued_jobs.append(job)
if not self.is_alive():
self.start()
def run(self):
while self.keep_going:
try:
self.run_once()
except:
import traceback
traceback.print_exc()
time.sleep(0.1)
def run_once(self):
with self.lock:
remove = set()
for worker in self.running_jobs:
if worker.is_alive():
# Get progress notifications
if worker.job.consume_notifications():
self.changed_jobs.put(worker.job)
else:
remove.add(worker)
self.changed_jobs.put(worker.job)
for worker in remove:
self.running_jobs.remove(worker)
jobs = self.get_startable_jobs()
for job in jobs:
w = ThreadedJobWorker(job)
w.start()
self.running_jobs.add(w)
self.changed_jobs.put(job)
self.queued_jobs.remove(job)
def kill_job(self, job):
with self.lock:
if job in self.queued_jobs:
self.queued_jobs.remove(job)
elif job in self.running_jobs:
self.running_jobs.remove(job)
job.kill()
self.changed_jobs.put(job)
def running_jobs_of_type(self, type_):
return len([w for w in self.running_jobs if w.job.type == type_])
def get_startable_jobs(self):
queued_types = []
ans = []
for job in self.queued_jobs:
num = self.running_jobs_of_type(job.type)
num += queued_types.count(job.type)
if num < job.max_concurrent_count:
queued_types.append(job.type)
ans.append(job)
return ans
