def download_cover(log,
title=None, authors=None, identifiers={}, timeout=30):
'''
Synchronous cover download. Returns the "best" cover as per user
prefs/cover resolution.
Returned cover is a tuple: (plugin, width, height, fmt, data)
Returns None if no cover is found.
'''
rq = Queue()
abort = Event()
run_download(log, rq, abort, title=title, authors=authors,
identifiers=identifiers, timeout=timeout, get_best_cover=True)
results = []
while True:
try:
results.append(rq.get_nowait())
except Empty:
break
cp = msprefs['cover_priorities']
def keygen(result):
plugin, width, height, fmt, data = result
return (cp.get(plugin.name, 1), 1/(width*height))
results.sort(key=keygen)
return results[0] if results else None
class ParseWorker(Thread):
daemon = True
SLEEP_TIME = 1
def __init__(self):
Thread.__init__(self)
self.requests = Queue()
self.request_count = 0
self.parse_items = {}
self.launch_error = None
def run(self):
mod, func = 'calibre.gui2.tweak_book.preview', 'parse_html'
try:
# Connect to the worker and send a dummy job to initialize it
self.worker = offload_worker(priority='low')
self.worker(mod, func, '<p></p>')
except:
import traceback
traceback.print_exc()
self.launch_error = traceback.format_exc()
return
while True:
time.sleep(self.SLEEP_TIME)
x = self.requests.get()
requests = [x]
while True:
try:
requests.append(self.requests.get_nowait())
except Empty:
break
if shutdown in requests:
self.worker.shutdown()
break
request = sorted(requests, reverse=True)[0]
del requests
pi, data = request[1:]
try:
res = self.worker(mod, func, data)
except:
import traceback
traceback.print_exc()
else:
pi.parsing_done = True
parsed_data = res['result']
if res['tb']:
prints("Parser error:")
prints(res['tb'])
else:
pi.parsed_data = parsed_data
def add_request(self, name):
data = get_data(name)
ldata, hdata = len(data), hash(data)
pi = self.parse_items.get(name, None)
if pi is None:
self.parse_items[name] = pi = ParseItem(name)
else:
if pi.parsing_done and pi.length == ldata and pi.fingerprint == hdata:
return
pi.parsed_data = None
pi.parsing_done = False
pi.length, pi.fingerprint = ldata, hdata
self.requests.put((self.request_count, pi, data))
self.request_count += 1
def shutdown(self):
self.requests.put(shutdown)
def get_data(self, name):
return getattr(self.parse_items.get(name, None), 'parsed_data', None)
def clear(self):
self.parse_items.clear()
def is_alive(self):
return Thread.is_alive(self) or (hasattr(self, 'worker')
and self.worker.is_alive())
def test_identify_plugin(
name,
tests,
modify_plugin=lambda plugin: None, # {{{
fail_missing_meta=True):
'''
:param name: Plugin name
:param tests: List of 2-tuples. Each two tuple is of the form (args,
test_funcs). args is a dict of keyword arguments to pass to
the identify method. test_funcs are callables that accept a
Metadata object and return True iff the object passes the
test.
'''
plugin = None
for x in all_metadata_plugins():
if x.name == name and 'identify' in x.capabilities:
plugin = x
break
modify_plugin(plugin)
prints('Testing the identify function of', plugin.name)
prints('Using extra headers:', plugin.browser.addheaders)
tdir, lf, log, abort = init_test(plugin.name)
prints('Log saved to', lf)
times = []
for kwargs, test_funcs in tests:
log('')
log('#' * 80)
log('### Running test with:', kwargs)
log('#' * 80)
prints('Running test with:', kwargs)
rq = Queue()
args = (log, rq, abort)
start_time = time.time()
plugin.running_a_test = True
try:
err = plugin.identify(*args, **kwargs)
finally:
plugin.running_a_test = False
total_time = time.time() - start_time
times.append(total_time)
if err is not None:
prints('identify returned an error for args', args)
prints(err)
break
results = []
while True:
try:
results.append(rq.get_nowait())
except Empty:
break
prints('Found', len(results), 'matches:', end=' ')
prints('Smaller relevance means better match')
results.sort(key=plugin.identify_results_keygen(
title=kwargs.get('title', None),
authors=kwargs.get('authors', None),
identifiers=kwargs.get('identifiers', {})))
for i, mi in enumerate(results):
prints('*' * 30, 'Relevance:', i, '*' * 30)
if mi.rating:
mi.rating *= 2
prints(mi)
prints('\nCached cover URL :',
plugin.get_cached_cover_url(mi.identifiers))
prints('*' * 75, '\n\n')
possibles = []
for mi in results:
test_failed = False
for tfunc in test_funcs:
if not tfunc(mi):
test_failed = True
break
if not test_failed:
possibles.append(mi)
if not possibles:
prints('ERROR: No results that passed all tests were found')
prints('Log saved to', lf)
log.close()
dump_log(lf)
raise SystemExit(1)
good = [x for x in possibles if plugin.test_fields(x) is None]
if not good:
prints('Failed to find', plugin.test_fields(possibles[0]))
if fail_missing_meta:
raise SystemExit(1)
if results[0] is not possibles[0]:
prints('Most relevant result failed the tests')
raise SystemExit(1)
if 'cover' in plugin.capabilities:
rq = Queue()
mi = results[0]
plugin.download_cover(log,
rq,
abort,
title=mi.title,
authors=mi.authors,
identifiers=mi.identifiers)
results = []
while True:
try:
results.append(rq.get_nowait())
except Empty:
break
if not results and fail_missing_meta:
prints('Cover download failed')
raise SystemExit(1)
elif results:
cdata = results[0]
cover = os.path.join(
tdir,
plugin.name.replace(' ', '') + '-%s-cover.jpg' %
sanitize_file_name(mi.title.replace(' ', '_')))
with open(cover, 'wb') as f:
f.write(cdata[-1])
prints('Cover downloaded to:', cover)
if len(cdata[-1]) < 10240:
prints('Downloaded cover too small')
raise SystemExit(1)
prints('Average time per query', sum(times) / len(times))
if os.stat(lf).st_size > 10:
prints('There were some errors/warnings, see log', lf)
class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
TagBrowserMixin, CoverFlowMixin, LibraryViewMixin, SearchBoxMixin,
SavedSearchBoxMixin, SearchRestrictionMixin, LayoutMixin, UpdateMixin,
EbookDownloadMixin
):
'The main GUI'
proceed_requested = pyqtSignal(object, object)
book_converted = pyqtSignal(object, object)
shutting_down = False
def __init__(self, opts, parent=None, gui_debug=None):
MainWindow.__init__(self, opts, parent=parent, disable_automatic_gc=True)
self.setWindowIcon(QApplication.instance().windowIcon())
self.jobs_pointer = Pointer(self)
self.proceed_requested.connect(self.do_proceed,
type=Qt.QueuedConnection)
self.proceed_question = ProceedQuestion(self)
self.job_error_dialog = JobError(self)
self.keyboard = Manager(self)
get_gui.ans = self
self.opts = opts
self.device_connected = None
self.gui_debug = gui_debug
self.iactions = OrderedDict()
# Actions
for action in interface_actions():
if opts.ignore_plugins and action.plugin_path is not None:
continue
try:
ac = self.init_iaction(action)
except:
# Ignore errors in loading user supplied plugins
import traceback
traceback.print_exc()
if action.plugin_path is None:
raise
continue
ac.plugin_path = action.plugin_path
ac.interface_action_base_plugin = action
self.add_iaction(ac)
self.load_store_plugins()
def init_iaction(self, action):
ac = action.load_actual_plugin(self)
ac.plugin_path = action.plugin_path
ac.interface_action_base_plugin = action
action.actual_iaction_plugin_loaded = True
return ac
def add_iaction(self, ac):
acmap = self.iactions
if ac.name in acmap:
if ac.priority >= acmap[ac.name].priority:
acmap[ac.name] = ac
else:
acmap[ac.name] = ac
def load_store_plugins(self):
from calibre.gui2.store.loader import Stores
self.istores = Stores()
for store in available_store_plugins():
if self.opts.ignore_plugins and store.plugin_path is not None:
continue
try:
st = self.init_istore(store)
self.add_istore(st)
except:
# Ignore errors in loading user supplied plugins
import traceback
traceback.print_exc()
if store.plugin_path is None:
raise
continue
self.istores.builtins_loaded()
def init_istore(self, store):
st = store.load_actual_plugin(self)
st.plugin_path = store.plugin_path
st.base_plugin = store
store.actual_istore_plugin_loaded = True
return st
def add_istore(self, st):
stmap = self.istores
if st.name in stmap:
if st.priority >= stmap[st.name].priority:
stmap[st.name] = st
else:
stmap[st.name] = st
def initialize(self, library_path, db, listener, actions, show_gui=True):
opts = self.opts
self.preferences_action, self.quit_action = actions
self.library_path = library_path
self.library_broker = GuiLibraryBroker(db)
self.content_server = None
self.server_change_notification_timer = t = QTimer(self)
self.server_changes = Queue()
t.setInterval(1000), t.timeout.connect(self.handle_changes_from_server_debounced), t.setSingleShot(True)
self._spare_pool = None
self.must_restart_before_config = False
self.listener = Listener(listener)
self.check_messages_timer = QTimer()
self.check_messages_timer.timeout.connect(self.another_instance_wants_to_talk)
self.check_messages_timer.start(1000)
for ac in self.iactions.values():
try:
ac.do_genesis()
except Exception:
# Ignore errors in third party plugins
import traceback
traceback.print_exc()
if getattr(ac, 'plugin_path', None) is None:
raise
self.donate_action = QAction(QIcon(I('donate.png')),
_('&Donate to support calibre'), self)
for st in self.istores.values():
st.do_genesis()
MainWindowMixin.init_main_window_mixin(self, db)
# Jobs Button {{{
self.job_manager = JobManager()
self.jobs_dialog = JobsDialog(self, self.job_manager)
self.jobs_button = JobsButton(parent=self)
self.jobs_button.initialize(self.jobs_dialog, self.job_manager)
# }}}
LayoutMixin.init_layout_mixin(self)
DeviceMixin.init_device_mixin(self)
self.progress_indicator = ProgressIndicator(self)
self.progress_indicator.pos = (0, 20)
self.verbose = opts.verbose
self.get_metadata = GetMetadata()
self.upload_memory = {}
self.metadata_dialogs = []
self.default_thumbnail = None
self.tb_wrapper = textwrap.TextWrapper(width=40)
self.viewers = collections.deque()
self.system_tray_icon = None
do_systray = config['systray_icon'] or opts.start_in_tray
if do_systray:
self.system_tray_icon = factory(app_id='com.calibre-ebook.gui').create_system_tray_icon(parent=self, title='calibre')
if self.system_tray_icon is not None:
self.system_tray_icon.setIcon(QIcon(I('lt.png', allow_user_override=False)))
if not (iswindows or isosx):
self.system_tray_icon.setIcon(QIcon.fromTheme('calibre-tray', self.system_tray_icon.icon()))
self.system_tray_icon.setToolTip(self.jobs_button.tray_tooltip())
self.system_tray_icon.setVisible(True)
self.jobs_button.tray_tooltip_updated.connect(self.system_tray_icon.setToolTip)
elif do_systray:
prints('Failed to create system tray icon, your desktop environment probably'
' does not support the StatusNotifier spec https://www.freedesktop.org/wiki/Specifications/StatusNotifierItem/')
self.system_tray_menu = QMenu(self)
self.toggle_to_tray_action = self.system_tray_menu.addAction(QIcon(I('page.png')), '')
self.toggle_to_tray_action.triggered.connect(self.system_tray_icon_activated)
self.system_tray_menu.addAction(self.donate_action)
self.eject_action = self.system_tray_menu.addAction(
QIcon(I('eject.png')), _('&Eject connected device'))
self.eject_action.setEnabled(False)
self.addAction(self.quit_action)
self.system_tray_menu.addAction(self.quit_action)
self.keyboard.register_shortcut('quit calibre', _('Quit calibre'),
default_keys=('Ctrl+Q',), action=self.quit_action)
if self.system_tray_icon is not None:
self.system_tray_icon.setContextMenu(self.system_tray_menu)
self.system_tray_icon.activated.connect(self.system_tray_icon_activated)
self.quit_action.triggered[bool].connect(self.quit)
self.donate_action.triggered[bool].connect(self.donate)
self.minimize_action = QAction(_('Minimize the calibre window'), self)
self.addAction(self.minimize_action)
self.keyboard.register_shortcut('minimize calibre', self.minimize_action.text(),
default_keys=(), action=self.minimize_action)
self.minimize_action.triggered.connect(self.showMinimized)
self.esc_action = QAction(self)
self.addAction(self.esc_action)
self.keyboard.register_shortcut('clear current search',
_('Clear the current search'), default_keys=('Esc',),
action=self.esc_action)
self.esc_action.triggered.connect(self.esc)
self.shift_esc_action = QAction(self)
self.addAction(self.shift_esc_action)
self.keyboard.register_shortcut('focus book list',
_('Focus the book list'), default_keys=('Shift+Esc',),
action=self.shift_esc_action)
self.shift_esc_action.triggered.connect(self.shift_esc)
self.ctrl_esc_action = QAction(self)
self.addAction(self.ctrl_esc_action)
self.keyboard.register_shortcut('clear virtual library',
_('Clear the virtual library'), default_keys=('Ctrl+Esc',),
action=self.ctrl_esc_action)
self.ctrl_esc_action.triggered.connect(self.ctrl_esc)
self.alt_esc_action = QAction(self)
self.addAction(self.alt_esc_action)
self.keyboard.register_shortcut('clear additional restriction',
_('Clear the additional restriction'), default_keys=('Alt+Esc',),
action=self.alt_esc_action)
self.alt_esc_action.triggered.connect(self.clear_additional_restriction)
# ###################### Start spare job server ########################
QTimer.singleShot(1000, self.create_spare_pool)
# ###################### Location Manager ########################
self.location_manager.location_selected.connect(self.location_selected)
self.location_manager.unmount_device.connect(self.device_manager.umount_device)
self.location_manager.configure_device.connect(self.configure_connected_device)
self.location_manager.update_device_metadata.connect(self.update_metadata_on_device)
self.eject_action.triggered.connect(self.device_manager.umount_device)
# ################### Update notification ###################
UpdateMixin.init_update_mixin(self, opts)
# ###################### Search boxes ########################
SearchRestrictionMixin.init_search_restriction_mixin(self)
SavedSearchBoxMixin.init_saved_seach_box_mixin(self)
# ###################### Library view ########################
LibraryViewMixin.init_library_view_mixin(self, db)
SearchBoxMixin.init_search_box_mixin(self) # Requires current_db
self.library_view.model().count_changed_signal.connect(
self.iactions['Choose Library'].count_changed)
if not gprefs.get('quick_start_guide_added', False):
try:
add_quick_start_guide(self.library_view)
except:
import traceback
traceback.print_exc()
for view in ('library', 'memory', 'card_a', 'card_b'):
v = getattr(self, '%s_view' % view)
v.selectionModel().selectionChanged.connect(self.update_status_bar)
v.model().count_changed_signal.connect(self.update_status_bar)
self.library_view.model().count_changed()
self.bars_manager.database_changed(self.library_view.model().db)
self.library_view.model().database_changed.connect(self.bars_manager.database_changed,
type=Qt.QueuedConnection)
# ########################## Tags Browser ##############################
TagBrowserMixin.init_tag_browser_mixin(self, db)
self.library_view.model().database_changed.connect(self.populate_tb_manage_menu, type=Qt.QueuedConnection)
# ######################## Search Restriction ##########################
if db.prefs['virtual_lib_on_startup']:
self.apply_virtual_library(db.prefs['virtual_lib_on_startup'])
self.rebuild_vl_tabs()
# ########################## Cover Flow ################################
CoverFlowMixin.init_cover_flow_mixin(self)
self._calculated_available_height = min(max_available_height()-15,
self.height())
self.resize(self.width(), self._calculated_available_height)
self.build_context_menus()
for ac in self.iactions.values():
try:
ac.gui_layout_complete()
except:
import traceback
traceback.print_exc()
if ac.plugin_path is None:
raise
if config['autolaunch_server']:
self.start_content_server()
self.read_settings()
self.finalize_layout()
self.bars_manager.start_animation()
self.set_window_title()
for ac in self.iactions.values():
try:
ac.initialization_complete()
except:
import traceback
traceback.print_exc()
if ac.plugin_path is None:
raise
self.set_current_library_information(current_library_name(), db.library_id,
db.field_metadata)
register_keyboard_shortcuts()
self.keyboard.finalize()
if show_gui:
# Note this has to come after restoreGeometry() because of
# https://bugreports.qt.io/browse/QTBUG-56831
self.show()
if self.system_tray_icon is not None and self.system_tray_icon.isVisible() and opts.start_in_tray:
self.hide_windows()
self.auto_adder = AutoAdder(gprefs['auto_add_path'], self)
# Now that the gui is initialized we can restore the quickview state
# The same thing will be true for any action-based operation with a
# layout button
from calibre.gui2.actions.show_quickview import get_quickview_action_plugin
qv = get_quickview_action_plugin()
if qv:
qv.qv_button.restore_state()
self.save_layout_state()
# Collect cycles now
gc.collect()
QApplication.instance().shutdown_signal_received.connect(self.quit)
if show_gui and self.gui_debug is not None:
QTimer.singleShot(10, self.show_gui_debug_msg)
self.iactions['Connect Share'].check_smartdevice_menus()
QTimer.singleShot(1, self.start_smartdevice)
QTimer.singleShot(100, self.update_toggle_to_tray_action)
def show_gui_debug_msg(self):
info_dialog(self, _('Debug mode'), '<p>' +
_('You have started calibre in debug mode. After you '
'quit calibre, the debug log will be available in '
'the file: %s<p>The '
'log will be displayed automatically.')%self.gui_debug, show=True)
def esc(self, *args):
self.search.clear()
def shift_esc(self):
self.current_view().setFocus(Qt.OtherFocusReason)
def ctrl_esc(self):
self.apply_virtual_library()
self.current_view().setFocus(Qt.OtherFocusReason)
def start_smartdevice(self):
message = None
if self.device_manager.get_option('smartdevice', 'autostart'):
try:
message = self.device_manager.start_plugin('smartdevice')
except:
message = 'start smartdevice unknown exception'
prints(message)
import traceback
traceback.print_exc()
if message:
if not self.device_manager.is_running('Wireless Devices'):
error_dialog(self, _('Problem starting the wireless device'),
_('The wireless device driver had problems starting. '
'It said "%s"')%message, show=True)
self.iactions['Connect Share'].set_smartdevice_action_state()
def start_content_server(self, check_started=True):
from calibre.srv.embedded import Server
if not gprefs.get('server3_warning_done', False):
gprefs.set('server3_warning_done', True)
if os.path.exists(os.path.join(config_dir, 'server.py')):
try:
os.remove(os.path.join(config_dir, 'server.py'))
except EnvironmentError:
pass
warning_dialog(self, _('Content server changed!'), _(
'calibre 3 comes with a completely re-written content server.'
' As such any custom configuration you have for the content'
' server no longer applies. You should check and refresh your'
' settings in Preferences->Sharing->Sharing over the net'), show=True)
self.content_server = Server(self.library_broker, Dispatcher(self.handle_changes_from_server))
self.content_server.state_callback = Dispatcher(
self.iactions['Connect Share'].content_server_state_changed)
if check_started:
self.content_server.start_failure_callback = \
Dispatcher(self.content_server_start_failed)
self.content_server.start()
def handle_changes_from_server(self, library_path, change_event):
if DEBUG:
prints('Received server change event: {} for {}'.format(change_event, library_path))
if self.library_broker.is_gui_library(library_path):
self.server_changes.put((library_path, change_event))
self.server_change_notification_timer.start()
def handle_changes_from_server_debounced(self):
if self.shutting_down:
return
changes = []
while True:
try:
library_path, change_event = self.server_changes.get_nowait()
except Empty:
break
if self.library_broker.is_gui_library(library_path):
changes.append(change_event)
if changes:
handle_changes(changes, self)
def content_server_start_failed(self, msg):
self.content_server = None
error_dialog(self, _('Failed to start Content server'),
_('Could not start the Content server. Error:\n\n%s')%msg,
show=True)
def resizeEvent(self, ev):
MainWindow.resizeEvent(self, ev)
self.search.setMaximumWidth(self.width()-150)
def create_spare_pool(self, *args):
if self._spare_pool is None:
num = min(detect_ncpus(), config['worker_limit']//2)
self._spare_pool = Pool(max_workers=num, name='GUIPool')
def spare_pool(self):
ans, self._spare_pool = self._spare_pool, None
QTimer.singleShot(1000, self.create_spare_pool)
return ans
def do_proceed(self, func, payload):
if callable(func):
func(payload)
def no_op(self, *args):
pass
def system_tray_icon_activated(self, r=False):
if r in (QSystemTrayIcon.Trigger, QSystemTrayIcon.MiddleClick, False):
if self.isVisible():
if self.isMinimized():
self.showNormal()
else:
self.hide_windows()
else:
self.show_windows()
if self.isMinimized():
self.showNormal()
@property
def is_minimized_to_tray(self):
return getattr(self, '__systray_minimized', False)
def ask_a_yes_no_question(self, title, msg, det_msg='',
show_copy_button=False, ans_when_user_unavailable=True,
skip_dialog_name=None, skipped_value=True):
if self.is_minimized_to_tray:
return ans_when_user_unavailable
return question_dialog(self, title, msg, det_msg=det_msg,
show_copy_button=show_copy_button,
skip_dialog_name=skip_dialog_name,
skip_dialog_skipped_value=skipped_value)
def update_toggle_to_tray_action(self, *args):
if hasattr(self, 'toggle_to_tray_action'):
self.toggle_to_tray_action.setText(
_('Hide main window') if self.isVisible() else _('Show main window'))
def hide_windows(self):
for window in QApplication.topLevelWidgets():
if isinstance(window, (MainWindow, QDialog)) and \
window.isVisible():
window.hide()
setattr(window, '__systray_minimized', True)
self.update_toggle_to_tray_action()
def show_windows(self, *args):
for window in QApplication.topLevelWidgets():
if getattr(window, '__systray_minimized', False):
window.show()
setattr(window, '__systray_minimized', False)
self.update_toggle_to_tray_action()
def test_server(self, *args):
if self.content_server is not None and \
self.content_server.exception is not None:
error_dialog(self, _('Failed to start Content server'),
unicode_type(self.content_server.exception)).exec_()
@property
def current_db(self):
return self.library_view.model().db
def refresh_all(self):
m = self.library_view.model()
m.db.data.refresh(clear_caches=False, do_search=False)
self.saved_searches_changed(recount=False)
m.resort()
m.research()
self.tags_view.recount()
def handle_cli_args(self, args):
if isinstance(args, string_or_bytes):
args = [args]
files = [os.path.abspath(p) for p in args if not os.path.isdir(p) and os.access(p, os.R_OK)]
if files:
self.iactions['Add Books'].add_filesystem_book(files)
def another_instance_wants_to_talk(self):
try:
msg = self.listener.queue.get_nowait()
except Empty:
return
if isinstance(msg, bytes):
msg = msg.decode('utf-8', 'replace')
if msg.startswith('launched:'):
import json
try:
argv = json.loads(msg[len('launched:'):])
except ValueError:
prints('Failed to decode message from other instance: %r' % msg)
if DEBUG:
error_dialog(self, 'Invalid message',
'Received an invalid message from other calibre instance.'
' Do you have multiple versions of calibre installed?',
det_msg='Invalid msg: %r' % msg, show=True)
argv = ()
if isinstance(argv, (list, tuple)) and len(argv) > 1:
self.handle_cli_args(argv[1:])
self.setWindowState(self.windowState() & ~Qt.WindowMinimized|Qt.WindowActive)
self.show_windows()
self.raise_()
self.activateWindow()
elif msg.startswith('refreshdb:'):
m = self.library_view.model()
m.db.new_api.reload_from_db()
self.refresh_all()
elif msg.startswith('shutdown:'):
self.quit(confirm_quit=False)
elif msg.startswith('bookedited:'):
parts = msg.split(':')[1:]
try:
book_id, fmt, library_id = parts[:3]
book_id = int(book_id)
m = self.library_view.model()
db = m.db.new_api
if m.db.library_id == library_id and db.has_id(book_id):
db.format_metadata(book_id, fmt, allow_cache=False, update_db=True)
db.update_last_modified((book_id,))
m.refresh_ids((book_id,))
except Exception:
import traceback
traceback.print_exc()
elif msg.startswith('web-store:'):
import json
try:
data = json.loads(msg[len('web-store:'):])
except ValueError:
prints('Failed to decode message from other instance: %r' % msg)
path = data['path']
if data['tags']:
before = self.current_db.new_api.all_book_ids()
self.iactions['Add Books'].add_filesystem_book([path], allow_device=False)
if data['tags']:
db = self.current_db.new_api
after = self.current_db.new_api.all_book_ids()
for book_id in after - before:
tags = list(db.field_for('tags', book_id))
tags += list(data['tags'])
self.current_db.new_api.set_field('tags', {book_id: tags})
else:
prints('Ignoring unknown message from other instance: %r' % msg[:20])
def current_view(self):
'''Convenience method that returns the currently visible view '''
idx = self.stack.currentIndex()
if idx == 0:
return self.library_view
if idx == 1:
return self.memory_view
if idx == 2:
return self.card_a_view
if idx == 3:
return self.card_b_view
def booklists(self):
return self.memory_view.model().db, self.card_a_view.model().db, self.card_b_view.model().db
def library_moved(self, newloc, copy_structure=False, allow_rebuild=False):
if newloc is None:
return
with self.library_broker:
default_prefs = None
try:
olddb = self.library_view.model().db
if copy_structure:
default_prefs = olddb.prefs
except:
olddb = None
if copy_structure and olddb is not None and default_prefs is not None:
default_prefs['field_metadata'] = olddb.new_api.field_metadata.all_metadata()
db = self.library_broker.prepare_for_gui_library_change(newloc)
if db is None:
try:
db = LibraryDatabase(newloc, default_prefs=default_prefs)
except apsw.Error:
if not allow_rebuild:
raise
import traceback
repair = question_dialog(self, _('Corrupted database'),
_('The library database at %s appears to be corrupted. Do '
'you want calibre to try and rebuild it automatically? '
'The rebuild may not be completely successful.')
% force_unicode(newloc, filesystem_encoding),
det_msg=traceback.format_exc()
)
if repair:
from calibre.gui2.dialogs.restore_library import repair_library_at
if repair_library_at(newloc, parent=self):
db = LibraryDatabase(newloc, default_prefs=default_prefs)
else:
return
else:
return
self.library_path = newloc
prefs['library_path'] = self.library_path
self.book_on_device(None, reset=True)
db.set_book_on_device_func(self.book_on_device)
self.library_view.set_database(db)
self.tags_view.set_database(db, self.alter_tb)
self.library_view.model().set_book_on_device_func(self.book_on_device)
self.status_bar.clear_message()
self.search.clear()
self.saved_search.clear()
self.book_details.reset_info()
# self.library_view.model().count_changed()
db = self.library_view.model().db
self.iactions['Choose Library'].count_changed(db.count())
self.set_window_title()
self.apply_named_search_restriction('') # reset restriction to null
self.saved_searches_changed(recount=False) # reload the search restrictions combo box
if db.prefs['virtual_lib_on_startup']:
self.apply_virtual_library(db.prefs['virtual_lib_on_startup'])
self.rebuild_vl_tabs()
for action in self.iactions.values():
action.library_changed(db)
self.library_broker.gui_library_changed(db, olddb)
if self.device_connected:
self.set_books_in_library(self.booklists(), reset=True)
self.refresh_ondevice()
self.memory_view.reset()
self.card_a_view.reset()
self.card_b_view.reset()
self.set_current_library_information(current_library_name(), db.library_id,
db.field_metadata)
self.library_view.set_current_row(0)
# Run a garbage collection now so that it does not freeze the
# interface later
gc.collect()
def set_window_title(self):
db = self.current_db
restrictions = [x for x in (db.data.get_base_restriction_name(),
db.data.get_search_restriction_name()) if x]
restrictions = ' :: '.join(restrictions)
font = QFont()
if restrictions:
restrictions = ' :: ' + restrictions
font.setBold(True)
font.setItalic(True)
self.virtual_library.setFont(font)
title = '{0} - || {1}{2} ||'.format(
__appname__, self.iactions['Choose Library'].library_name(), restrictions)
self.setWindowTitle(title)
def location_selected(self, location):
'''
Called when a location icon is clicked (e.g. Library)
'''
page = 0 if location == 'library' else 1 if location == 'main' else 2 if location == 'carda' else 3
self.stack.setCurrentIndex(page)
self.book_details.reset_info()
for x in ('tb', 'cb'):
splitter = getattr(self, x+'_splitter')
splitter.button.setEnabled(location == 'library')
for action in self.iactions.values():
action.location_selected(location)
if location == 'library':
self.virtual_library_menu.setEnabled(True)
self.highlight_only_button.setEnabled(True)
self.vl_tabs.setEnabled(True)
else:
self.virtual_library_menu.setEnabled(False)
self.highlight_only_button.setEnabled(False)
self.vl_tabs.setEnabled(False)
# Reset the view in case something changed while it was invisible
self.current_view().reset()
self.set_number_of_books_shown()
self.update_status_bar()
def job_exception(self, job, dialog_title=_('Conversion error'), retry_func=None):
if not hasattr(self, '_modeless_dialogs'):
self._modeless_dialogs = []
minz = self.is_minimized_to_tray
if self.isVisible():
for x in list(self._modeless_dialogs):
if not x.isVisible():
self._modeless_dialogs.remove(x)
try:
if 'calibre.ebooks.DRMError' in job.details:
if not minz:
from calibre.gui2.dialogs.drm_error import DRMErrorMessage
d = DRMErrorMessage(self, _('Cannot convert') + ' ' +
job.description.split(':')[-1].partition('(')[-1][:-1])
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.ebooks.oeb.transforms.split.SplitError' in job.details:
title = job.description.split(':')[-1].partition('(')[-1][:-1]
msg = _('<p><b>Failed to convert: %s')%title
msg += '<p>'+_('''
Many older e-book reader devices are incapable of displaying
EPUB files that have internal components over a certain size.
Therefore, when converting to EPUB, calibre automatically tries
to split up the EPUB into smaller sized pieces. For some
files that are large undifferentiated blocks of text, this
splitting fails.
<p>You can <b>work around the problem</b> by either increasing the
maximum split size under <i>EPUB output</i> in the conversion dialog,
or by turning on Heuristic Processing, also in the conversion
dialog. Note that if you make the maximum split size too large,
your e-book reader may have trouble with the EPUB.
''')
if not minz:
d = error_dialog(self, _('Conversion Failed'), msg,
det_msg=job.details)
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.ebooks.mobi.reader.mobi6.KFXError:' in job.details:
if not minz:
title = job.description.split(':')[-1].partition('(')[-1][:-1]
msg = _('<p><b>Failed to convert: %s') % title
idx = job.details.index('calibre.ebooks.mobi.reader.mobi6.KFXError:')
msg += '<p>' + re.sub(r'(https:\S+)', r'<a href="\1">{}</a>'.format(_('here')),
job.details[idx:].partition(':')[2].strip())
d = error_dialog(self, _('Conversion failed'), msg, det_msg=job.details)
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.web.feeds.input.RecipeDisabled' in job.details:
if not minz:
msg = job.details
msg = msg[msg.find('calibre.web.feeds.input.RecipeDisabled:'):]
msg = msg.partition(':')[-1]
d = error_dialog(self, _('Recipe Disabled'),
'<p>%s</p>'%msg)
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.ebooks.conversion.ConversionUserFeedBack:' in job.details:
if not minz:
import json
payload = job.details.rpartition(
'calibre.ebooks.conversion.ConversionUserFeedBack:')[-1]
payload = json.loads('{' + payload.partition('{')[-1])
d = {'info':info_dialog, 'warn':warning_dialog,
'error':error_dialog}.get(payload['level'],
error_dialog)
d = d(self, payload['title'],
'<p>%s</p>'%payload['msg'],
det_msg=payload['det_msg'])
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
except:
pass
if job.killed:
return
try:
prints(job.details, file=sys.stderr)
except:
pass
if not minz:
self.job_error_dialog.show_error(dialog_title,
_('<b>Failed</b>')+': '+unicode_type(job.description),
det_msg=job.details, retry_func=retry_func)
def read_settings(self):
geometry = config['main_window_geometry']
if geometry is not None:
self.restoreGeometry(geometry)
self.read_layout_settings()
def write_settings(self):
with gprefs: # Only write to gprefs once
config.set('main_window_geometry', self.saveGeometry())
dynamic.set('sort_history', self.library_view.model().sort_history)
self.save_layout_state()
self.stack.tb_widget.save_state()
def quit(self, checked=True, restart=False, debug_on_restart=False,
confirm_quit=True):
if self.shutting_down:
return
if confirm_quit and not self.confirm_quit():
return
try:
self.shutdown()
except:
pass
self.restart_after_quit = restart
self.debug_on_restart = debug_on_restart
if self.system_tray_icon is not None and self.restart_after_quit:
# Needed on windows to prevent multiple systray icons
self.system_tray_icon.setVisible(False)
QApplication.instance().quit()
def donate(self, *args):
from calibre.utils.localization import localize_website_link
open_url(QUrl(localize_website_link('https://calibre-ebook.com/donate')))
def confirm_quit(self):
if self.job_manager.has_jobs():
msg = _('There are active jobs. Are you sure you want to quit?')
if self.job_manager.has_device_jobs():
msg = '<p>'+__appname__ + \
_(''' is communicating with the device!<br>
Quitting may cause corruption on the device.<br>
Are you sure you want to quit?''')+'</p>'
if not question_dialog(self, _('Active jobs'), msg):
return False
if self.proceed_question.questions:
msg = _('There are library updates waiting. Are you sure you want to quit?')
if not question_dialog(self, _('Library updates waiting'), msg):
return False
from calibre.db.delete_service import has_jobs
if has_jobs():
msg = _('Some deleted books are still being moved to the Recycle '
'Bin, if you quit now, they will be left behind. Are you '
'sure you want to quit?')
if not question_dialog(self, _('Active jobs'), msg):
return False
return True
def shutdown(self, write_settings=True):
self.shutting_down = True
self.show_shutdown_message()
self.server_change_notification_timer.stop()
from calibre.customize.ui import has_library_closed_plugins
if has_library_closed_plugins():
self.show_shutdown_message(
_('Running database shutdown plugins. This could take a few seconds...'))
self.grid_view.shutdown()
db = None
try:
db = self.library_view.model().db
cf = db.clean
except:
pass
else:
cf()
# Save the current field_metadata for applications like calibre2opds
# Goes here, because if cf is valid, db is valid.
db.new_api.set_pref('field_metadata', db.field_metadata.all_metadata())
db.commit_dirty_cache()
db.prefs.write_serialized(prefs['library_path'])
for action in self.iactions.values():
if not action.shutting_down():
return
if write_settings:
self.write_settings()
self.check_messages_timer.stop()
if getattr(self, 'update_checker', None):
self.update_checker.shutdown()
self.listener.close()
self.job_manager.server.close()
self.job_manager.threaded_server.close()
self.device_manager.keep_going = False
self.auto_adder.stop()
# Do not report any errors that happen after the shutdown
# We cannot restore the original excepthook as that causes PyQt to
# call abort() on unhandled exceptions
import traceback
def eh(t, v, tb):
try:
traceback.print_exception(t, v, tb, file=sys.stderr)
except:
pass
sys.excepthook = eh
mb = self.library_view.model().metadata_backup
if mb is not None:
mb.stop()
self.library_view.model().close()
try:
try:
if self.content_server is not None:
# If the Content server has any sockets being closed then
# this can take quite a long time (minutes). Tell the user that it is
# happening.
self.show_shutdown_message(
_('Shutting down the Content server. This could take a while...'))
s = self.content_server
self.content_server = None
s.exit()
except:
pass
except KeyboardInterrupt:
pass
self.hide_windows()
if self._spare_pool is not None:
self._spare_pool.shutdown()
from calibre.db.delete_service import shutdown
shutdown()
time.sleep(2)
self.istores.join()
return True
def run_wizard(self, *args):
if self.confirm_quit():
self.run_wizard_b4_shutdown = True
self.restart_after_quit = True
try:
self.shutdown(write_settings=False)
except:
pass
QApplication.instance().quit()
def closeEvent(self, e):
if self.shutting_down:
return
self.write_settings()
if self.system_tray_icon is not None and self.system_tray_icon.isVisible():
if not dynamic['systray_msg'] and not isosx:
info_dialog(self, 'calibre', 'calibre '+
_('will keep running in the system tray. To close it, '
'choose <b>Quit</b> in the context menu of the '
'system tray.'), show_copy_button=False).exec_()
dynamic['systray_msg'] = True
self.hide_windows()
e.ignore()
else:
if self.confirm_quit():
try:
self.shutdown(write_settings=False)
except:
import traceback
traceback.print_exc()
e.accept()
else:
e.ignore()
class BaseJob:
WAITING = 0
RUNNING = 1
FINISHED = 2
def __init__(self, description, done=lambda x: x):
self.id = next(job_counter)
self.description = description
self.done = done
self.done2 = None
self.killed = False
self.failed = False
self.kill_on_start = False
self.start_time = None
self.result = None
self.duration = None
self.log_path = None
self.notifications = Queue()
self._run_state = self.WAITING
self.percent = 0
self._message = None
self._status_text = _('Waiting...')
self._done_called = False
self.core_usage = 1
self.timed_out = False
def update(self, consume_notifications=True):
if self.duration is not None:
self._run_state = self.FINISHED
self.percent = 100
if self.killed:
if self.timed_out:
self._status_text = _('Aborted, taking too long')
else:
self._status_text = _('Stopped')
else:
self._status_text = _('Error') if self.failed else _(
'Finished')
if DEBUG:
try:
prints('Job:', self.id, self.description, 'finished')
prints('\t'.join(self.details.splitlines(True)))
except:
pass
if not self._done_called:
self._done_called = True
try:
self.done(self)
except:
pass
try:
if callable(self.done2):
self.done2(self)
except:
pass
elif self.start_time is not None:
self._run_state = self.RUNNING
self._status_text = _('Working...')
if consume_notifications:
return self.consume_notifications()
return False
def consume_notifications(self):
got_notification = False
while self.notifications is not None:
try:
self.percent, self._message = self.notifications.get_nowait()
self.percent *= 100.
got_notification = True
except Empty:
break
return got_notification
@property
def status_text(self):
if self._run_state == self.FINISHED or not self._message:
return self._status_text
return self._message
@property
def run_state(self):
return self._run_state
@property
def running_time(self):
if self.duration is not None:
return self.duration
if self.start_time is not None:
return time.time() - self.start_time
return None
@property
def is_finished(self):
return self._run_state == self.FINISHED
@property
def is_started(self):
return self._run_state != self.WAITING
@property
def is_running(self):
return self.is_started and not self.is_finished
def __hash__(self):
return id(self)
def __eq__(self, other):
return self is other
def __ne__(self, other):
return self is not other
def __lt__(self, other):
return self.compare_to_other(other) < 0
def __le__(self, other):
return self.compare_to_other(other) <= 0
def __gt__(self, other):
return self.compare_to_other(other) > 0
def __ge__(self, other):
return self.compare_to_other(other) >= 0
def compare_to_other(self, other):
if self.is_finished != other.is_finished:
return 1 if self.is_finished else -1
if self.start_time is None:
if other.start_time is None: # Both waiting
return cmp(other.id, self.id)
return 1
if other.start_time is None:
return -1
# Both running
return cmp((other.start_time, id(other)), (self.start_time, id(self)))
@property
def log_file(self):
if self.log_path:
return open(self.log_path, 'rb')
return io.BytesIO(
_('No details available.').encode('utf-8', 'replace'))
@property
def details(self):
return self.log_file.read().decode('utf-8', 'replace')
def run_download(log,
results,
abort,
title=None,
authors=None,
identifiers={},
timeout=30,
get_best_cover=False):
'''
Run the cover download, putting results into the queue :param:`results`.
Each result is a tuple of the form:
(plugin, width, height, fmt, bytes)
'''
if title == _('Unknown'):
title = None
if authors == [_('Unknown')]:
authors = None
plugins = [p for p in metadata_plugins(['cover']) if p.is_configured()]
rq = Queue()
workers = [
Worker(p,
abort,
title,
authors,
identifiers,
timeout,
rq,
get_best_cover=get_best_cover) for p in plugins
]
for w in workers:
w.start()
first_result_at = None
wait_time = msprefs['wait_after_first_cover_result']
found_results = {}
start_time = time.time(
) # Use a global timeout to workaround misbehaving plugins that hang
while time.time() - start_time < 301:
time.sleep(0.1)
try:
x = rq.get_nowait()
result = process_result(log, x)
if result is not None:
results.put(result)
found_results[result[0]] = result
if first_result_at is not None:
first_result_at = time.time()
except Empty:
pass
if not is_worker_alive(workers):
break
if first_result_at is not None and time.time(
) - first_result_at > wait_time:
log('Not waiting for any more results')
abort.set()
if abort.is_set():
break
while True:
try:
x = rq.get_nowait()
result = process_result(log, x)
if result is not None:
results.put(result)
found_results[result[0]] = result
except Empty:
break
for w in workers:
wlog = w.buf.getvalue().strip()
log('\n' + '*' * 30, w.plugin.name, 'Covers', '*' * 30)
log('Request extra headers:', w.plugin.browser.addheaders)
if w.plugin in found_results:
result = found_results[w.plugin]
log('Downloaded cover:', '%dx%d' % (result[1], result[2]))
else:
log('Failed to download valid cover')
if w.time_spent is None:
log('Download aborted')
else:
log('Took', w.time_spent, 'seconds')
if wlog:
log(wlog)
log('\n' + '*' * 80)
class JobManager(QAbstractTableModel, AdaptSQP): # {{{
job_added = pyqtSignal(int)
job_done = pyqtSignal(int)
def __init__(self):
QAbstractTableModel.__init__(self)
SearchQueryParser.__init__(self, ['all'])
self.wait_icon = (QIcon(I('jobs.png')))
self.running_icon = (QIcon(I('exec.png')))
self.error_icon = (QIcon(I('dialog_error.png')))
self.done_icon = (QIcon(I('ok.png')))
self.jobs = []
self.add_job = Dispatcher(self._add_job)
self.server = Server(limit=config['worker_limit'] // 2,
enforce_cpu_limit=config['enforce_cpu_limit'])
self.threaded_server = ThreadedJobServer()
self.changed_queue = Queue()
self.timer = QTimer(self)
self.timer.timeout.connect(self.update,
type=Qt.ConnectionType.QueuedConnection)
self.timer.start(1000)
def columnCount(self, parent=QModelIndex()):
return 5
def rowCount(self, parent=QModelIndex()):
return len(self.jobs)
def headerData(self, section, orientation, role):
if role != Qt.ItemDataRole.DisplayRole:
return None
if orientation == Qt.Orientation.Horizontal:
return ({
0: _('Job'),
1: _('Status'),
2: _('Progress'),
3: _('Running time'),
4: _('Start time'),
}.get(section, ''))
else:
return (section + 1)
def show_tooltip(self, arg):
widget, pos = arg
QToolTip.showText(pos, self.get_tooltip())
def get_tooltip(self):
running_jobs = [j for j in self.jobs if j.run_state == j.RUNNING]
waiting_jobs = [j for j in self.jobs if j.run_state == j.WAITING]
lines = [
ngettext('There is a running job:', 'There are {} running jobs:',
len(running_jobs)).format(len(running_jobs))
]
for job in running_jobs:
desc = job.description
if not desc:
desc = _('Unknown job')
p = 100. if job.is_finished else job.percent
lines.append('%s: %.0f%% done' % (desc, p))
l = ngettext('There is a waiting job', 'There are {} waiting jobs',
len(waiting_jobs)).format(len(waiting_jobs))
lines.extend(['', l])
for job in waiting_jobs:
desc = job.description
if not desc:
desc = _('Unknown job')
lines.append(desc)
return '\n'.join(['calibre', ''] + lines)
def data(self, index, role):
try:
if role not in (Qt.ItemDataRole.DisplayRole,
Qt.ItemDataRole.DecorationRole):
return None
row, col = index.row(), index.column()
job = self.jobs[row]
if role == Qt.ItemDataRole.DisplayRole:
if col == 0:
desc = job.description
if not desc:
desc = _('Unknown job')
return (desc)
if col == 1:
return (job.status_text)
if col == 2:
p = 100. if job.is_finished else job.percent
return (p)
if col == 3:
rtime = job.running_time
if rtime is None:
return None
return human_readable_interval(rtime)
if col == 4 and job.start_time is not None:
return (strftime('%H:%M -- %d %b',
time.localtime(job.start_time)))
if role == Qt.ItemDataRole.DecorationRole and col == 0:
state = job.run_state
if state == job.WAITING:
return self.wait_icon
if state == job.RUNNING:
return self.running_icon
if job.killed or job.failed:
return self.error_icon
return self.done_icon
except:
import traceback
traceback.print_exc()
return None
def update(self):
try:
self._update()
except BaseException:
import traceback
traceback.print_exc()
def _update(self):
# Update running time
for i, j in enumerate(self.jobs):
if j.run_state == j.RUNNING:
idx = self.index(i, 3)
self.dataChanged.emit(idx, idx)
# Update parallel jobs
jobs = set()
while True:
try:
jobs.add(self.server.changed_jobs_queue.get_nowait())
except Empty:
break
# Update device jobs
while True:
try:
jobs.add(self.changed_queue.get_nowait())
except Empty:
break
# Update threaded jobs
while True:
try:
jobs.add(self.threaded_server.changed_jobs.get_nowait())
except Empty:
break
if jobs:
needs_reset = False
for job in jobs:
orig_state = job.run_state
job.update()
if orig_state != job.run_state:
needs_reset = True
if job.is_finished:
self.job_done.emit(len(self.unfinished_jobs()))
if needs_reset:
self.modelAboutToBeReset.emit()
self.jobs.sort()
self.modelReset.emit()
else:
for job in jobs:
idx = self.jobs.index(job)
self.dataChanged.emit(self.index(idx, 0),
self.index(idx, 3))
# Kill parallel jobs that have gone on too long
try:
wmax_time = gprefs['worker_max_time'] * 60
except:
wmax_time = 0
if wmax_time > 0:
for job in self.jobs:
if isinstance(job, ParallelJob):
rtime = job.running_time
if (rtime is not None and rtime > wmax_time
and job.duration is None):
job.timed_out = True
self.server.kill_job(job)
def _add_job(self, job):
self.modelAboutToBeReset.emit()
self.jobs.append(job)
self.jobs.sort()
self.job_added.emit(len(self.unfinished_jobs()))
self.modelReset.emit()
def done_jobs(self):
return [j for j in self.jobs if j.is_finished]
def unfinished_jobs(self):
return [j for j in self.jobs if not j.is_finished]
def row_to_job(self, row):
return self.jobs[row]
def rows_to_jobs(self, rows):
return [self.jobs[row] for row in rows]
def has_device_jobs(self, queued_also=False):
for job in self.jobs:
if isinstance(job, DeviceJob):
if job.duration is None: # Running or waiting
if (job.is_running or queued_also):
return True
return False
def has_jobs(self):
for job in self.jobs:
if job.is_running:
return True
return False
def run_job(self,
done,
name,
args=[],
kwargs={},
description='',
core_usage=1):
job = ParallelJob(name, description, done, args=args, kwargs=kwargs)
job.core_usage = core_usage
self.add_job(job)
self.server.add_job(job)
return job
def run_threaded_job(self, job):
self.add_job(job)
self.threaded_server.add_job(job)
def launch_gui_app(self, name, args=(), kwargs=None, description=''):
job = ParallelJob(name,
description,
lambda x: x,
args=list(args),
kwargs=kwargs or {})
self.server.run_job(job, gui=True, redirect_output=False)
def _kill_job(self, job):
if isinstance(job, ParallelJob):
self.server.kill_job(job)
elif isinstance(job, ThreadedJob):
self.threaded_server.kill_job(job)
else:
job.kill_on_start = True
def hide_jobs(self, rows):
for r in rows:
self.jobs[r].hidden_in_gui = True
for r in rows:
self.dataChanged.emit(self.index(r, 0), self.index(r, 0))
def show_hidden_jobs(self):
for j in self.jobs:
j.hidden_in_gui = False
for r in range(len(self.jobs)):
self.dataChanged.emit(self.index(r, 0), self.index(r, 0))
def kill_job(self, job, view):
if isinstance(job, DeviceJob):
return error_dialog(
view, _('Cannot kill job'),
_('Cannot kill jobs that communicate with the device')).exec_(
)
if job.duration is not None:
return error_dialog(view, _('Cannot kill job'),
_('Job has already run')).exec_()
if not getattr(job, 'killable', True):
return error_dialog(view,
_('Cannot kill job'),
_('This job cannot be stopped'),
show=True)
self._kill_job(job)
def kill_multiple_jobs(self, jobs, view):
devjobs = [j for j in jobs if isinstance(j, DeviceJob)]
if devjobs:
error_dialog(
view, _('Cannot kill job'),
_('Cannot kill jobs that communicate with the device')).exec_(
)
jobs = [j for j in jobs if not isinstance(j, DeviceJob)]
jobs = [j for j in jobs if j.duration is None]
unkillable = [j for j in jobs if not getattr(j, 'killable', True)]
if unkillable:
names = '\n'.join(as_unicode(j.description) for j in unkillable)
error_dialog(
view,
_('Cannot kill job'),
_('Some of the jobs cannot be stopped. Click "Show details"'
' to see the list of unstoppable jobs.'),
det_msg=names,
show=True)
jobs = [j for j in jobs if getattr(j, 'killable', True)]
jobs = [j for j in jobs if j.duration is None]
for j in jobs:
self._kill_job(j)
def kill_all_jobs(self):
for job in self.jobs:
if (isinstance(job, DeviceJob) or job.duration is not None
or not getattr(job, 'killable', True)):
continue
self._kill_job(job)
def terminate_all_jobs(self):
self.server.killall()
for job in self.jobs:
if (isinstance(job, DeviceJob) or job.duration is not None
or not getattr(job, 'killable', True)):
continue
if not isinstance(job, ParallelJob):
self._kill_job(job)
def universal_set(self):
return {
i
for i, j in enumerate(self.jobs)
if not getattr(j, 'hidden_in_gui', False)
}
def get_matches(self, location, query, candidates=None):
if candidates is None:
candidates = self.universal_set()
ans = set()
if not query:
return ans
query = lower(query)
for j in candidates:
job = self.jobs[j]
if job.description and query in lower(job.description):
ans.add(j)
return ans
def find(self, query):
query = query.strip()
rows = self.parse(query)
return rows
class JobManager(QAbstractTableModel, AdaptSQP): # {{{
job_added = pyqtSignal(int)
job_done = pyqtSignal(int)
def __init__(self):
QAbstractTableModel.__init__(self)
SearchQueryParser.__init__(self, ['all'])
self.wait_icon = (QIcon(I('jobs.png')))
self.running_icon = (QIcon(I('exec.png')))
self.error_icon = (QIcon(I('dialog_error.png')))
self.done_icon = (QIcon(I('ok.png')))
self.jobs = []
self.add_job = Dispatcher(self._add_job)
self.server = Server(limit=int(config['worker_limit']/2.0),
enforce_cpu_limit=config['enforce_cpu_limit'])
self.threaded_server = ThreadedJobServer()
self.changed_queue = Queue()
self.timer = QTimer(self)
self.timer.timeout.connect(self.update, type=Qt.QueuedConnection)
self.timer.start(1000)
def columnCount(self, parent=QModelIndex()):
return 5
def rowCount(self, parent=QModelIndex()):
return len(self.jobs)
def headerData(self, section, orientation, role):
if role != Qt.DisplayRole:
return None
if orientation == Qt.Horizontal:
return ({
0: _('Job'),
1: _('Status'),
2: _('Progress'),
3: _('Running time'),
4: _('Start time'),
}.get(section, ''))
else:
return (section+1)
def show_tooltip(self, arg):
widget, pos = arg
QToolTip.showText(pos, self.get_tooltip())
def get_tooltip(self):
running_jobs = [j for j in self.jobs if j.run_state == j.RUNNING]
waiting_jobs = [j for j in self.jobs if j.run_state == j.WAITING]
lines = [ngettext('There is a running job:', 'There are {} running jobs:', len(running_jobs)).format(len(running_jobs))]
for job in running_jobs:
desc = job.description
if not desc:
desc = _('Unknown job')
p = 100. if job.is_finished else job.percent
lines.append('%s: %.0f%% done'%(desc, p))
l = ngettext('There is a waiting job', 'There are {} waiting jobs', len(waiting_jobs)).format(len(waiting_jobs))
lines.extend(['', l])
for job in waiting_jobs:
desc = job.description
if not desc:
desc = _('Unknown job')
lines.append(desc)
return '\n'.join(['calibre', '']+ lines)
def data(self, index, role):
try:
if role not in (Qt.DisplayRole, Qt.DecorationRole):
return None
row, col = index.row(), index.column()
job = self.jobs[row]
if role == Qt.DisplayRole:
if col == 0:
desc = job.description
if not desc:
desc = _('Unknown job')
return (desc)
if col == 1:
return (job.status_text)
if col == 2:
p = 100. if job.is_finished else job.percent
return (p)
if col == 3:
rtime = job.running_time
if rtime is None:
return None
return human_readable_interval(rtime)
if col == 4 and job.start_time is not None:
return (strftime(u'%H:%M -- %d %b', time.localtime(job.start_time)))
if role == Qt.DecorationRole and col == 0:
state = job.run_state
if state == job.WAITING:
return self.wait_icon
if state == job.RUNNING:
return self.running_icon
if job.killed or job.failed:
return self.error_icon
return self.done_icon
except:
import traceback
traceback.print_exc()
return None
def update(self):
try:
self._update()
except BaseException:
import traceback
traceback.print_exc()
def _update(self):
# Update running time
for i, j in enumerate(self.jobs):
if j.run_state == j.RUNNING:
idx = self.index(i, 3)
self.dataChanged.emit(idx, idx)
# Update parallel jobs
jobs = set([])
while True:
try:
jobs.add(self.server.changed_jobs_queue.get_nowait())
except Empty:
break
# Update device jobs
while True:
try:
jobs.add(self.changed_queue.get_nowait())
except Empty:
break
# Update threaded jobs
while True:
try:
jobs.add(self.threaded_server.changed_jobs.get_nowait())
except Empty:
break
if jobs:
needs_reset = False
for job in jobs:
orig_state = job.run_state
job.update()
if orig_state != job.run_state:
needs_reset = True
if job.is_finished:
self.job_done.emit(len(self.unfinished_jobs()))
if needs_reset:
self.modelAboutToBeReset.emit()
self.jobs.sort()
self.modelReset.emit()
else:
for job in jobs:
idx = self.jobs.index(job)
self.dataChanged.emit(
self.index(idx, 0), self.index(idx, 3))
# Kill parallel jobs that have gone on too long
try:
wmax_time = gprefs['worker_max_time'] * 60
except:
wmax_time = 0
if wmax_time > 0:
for job in self.jobs:
if isinstance(job, ParallelJob):
rtime = job.running_time
if (rtime is not None and rtime > wmax_time and
job.duration is None):
job.timed_out = True
self.server.kill_job(job)
def _add_job(self, job):
self.modelAboutToBeReset.emit()
self.jobs.append(job)
self.jobs.sort()
self.job_added.emit(len(self.unfinished_jobs()))
self.modelReset.emit()
def done_jobs(self):
return [j for j in self.jobs if j.is_finished]
def unfinished_jobs(self):
return [j for j in self.jobs if not j.is_finished]
def row_to_job(self, row):
return self.jobs[row]
def rows_to_jobs(self, rows):
return [self.jobs[row] for row in rows]
def has_device_jobs(self, queued_also=False):
for job in self.jobs:
if isinstance(job, DeviceJob):
if job.duration is None: # Running or waiting
if (job.is_running or queued_also):
return True
return False
def has_jobs(self):
for job in self.jobs:
if job.is_running:
return True
return False
def run_job(self, done, name, args=[], kwargs={},
description='', core_usage=1):
job = ParallelJob(name, description, done, args=args, kwargs=kwargs)
job.core_usage = core_usage
self.add_job(job)
self.server.add_job(job)
return job
def run_threaded_job(self, job):
self.add_job(job)
self.threaded_server.add_job(job)
def launch_gui_app(self, name, args=[], kwargs={}, description=''):
job = ParallelJob(name, description, lambda x: x,
args=args, kwargs=kwargs)
self.server.run_job(job, gui=True, redirect_output=False)
def _kill_job(self, job):
if isinstance(job, ParallelJob):
self.server.kill_job(job)
elif isinstance(job, ThreadedJob):
self.threaded_server.kill_job(job)
else:
job.kill_on_start = True
def hide_jobs(self, rows):
for r in rows:
self.jobs[r].hidden_in_gui = True
for r in rows:
self.dataChanged.emit(self.index(r, 0), self.index(r, 0))
def show_hidden_jobs(self):
for j in self.jobs:
j.hidden_in_gui = False
for r in range(len(self.jobs)):
self.dataChanged.emit(self.index(r, 0), self.index(r, 0))
def kill_job(self, job, view):
if isinstance(job, DeviceJob):
return error_dialog(view, _('Cannot kill job'),
_('Cannot kill jobs that communicate with the device')).exec_()
if job.duration is not None:
return error_dialog(view, _('Cannot kill job'),
_('Job has already run')).exec_()
if not getattr(job, 'killable', True):
return error_dialog(view, _('Cannot kill job'),
_('This job cannot be stopped'), show=True)
self._kill_job(job)
def kill_multiple_jobs(self, jobs, view):
devjobs = [j for j in jobs if isinstance(j, DeviceJob)]
if devjobs:
error_dialog(view, _('Cannot kill job'),
_('Cannot kill jobs that communicate with the device')).exec_()
jobs = [j for j in jobs if not isinstance(j, DeviceJob)]
jobs = [j for j in jobs if j.duration is None]
unkillable = [j for j in jobs if not getattr(j, 'killable', True)]
if unkillable:
names = u'\n'.join(as_unicode(j.description) for j in unkillable)
error_dialog(view, _('Cannot kill job'),
_('Some of the jobs cannot be stopped. Click Show details'
' to see the list of unstoppable jobs.'), det_msg=names,
show=True)
jobs = [j for j in jobs if getattr(j, 'killable', True)]
jobs = [j for j in jobs if j.duration is None]
for j in jobs:
self._kill_job(j)
def kill_all_jobs(self):
for job in self.jobs:
if (isinstance(job, DeviceJob) or job.duration is not None or
not getattr(job, 'killable', True)):
continue
self._kill_job(job)
def terminate_all_jobs(self):
self.server.killall()
for job in self.jobs:
if (isinstance(job, DeviceJob) or job.duration is not None or
not getattr(job, 'killable', True)):
continue
if not isinstance(job, ParallelJob):
self._kill_job(job)
def universal_set(self):
return {i for i, j in enumerate(self.jobs) if not getattr(j,
'hidden_in_gui', False)}
def get_matches(self, location, query, candidates=None):
if candidates is None:
candidates = self.universal_set()
ans = set()
if not query:
return ans
query = lower(query)
for j in candidates:
job = self.jobs[j]
if job.description and query in lower(job.description):
ans.add(j)
return ans
def find(self, query):
query = query.strip()
rows = self.parse(query)
return rows
class WebSocketConnection(HTTPConnection):
# Internal API {{{
in_websocket_mode = False
websocket_handler = None
def __init__(self, *args, **kwargs):
global conn_id
HTTPConnection.__init__(self, *args, **kwargs)
self.sendq = Queue()
self.control_frames = deque()
self.cf_lock = Lock()
self.sending = None
self.send_buf = None
self.frag_decoder = UTF8Decoder()
self.ws_close_received = self.ws_close_sent = False
conn_id += 1
self.websocket_connection_id = conn_id
self.stop_reading = False
def finalize_headers(self, inheaders):
upgrade = inheaders.get('Upgrade', '')
key = inheaders.get('Sec-WebSocket-Key', None)
conn = {
x.strip().lower()
for x in inheaders.get('Connection', '').split(',')
}
if key is None or upgrade.lower(
) != 'websocket' or 'upgrade' not in conn:
return HTTPConnection.finalize_headers(self, inheaders)
ver = inheaders.get('Sec-WebSocket-Version', 'Unknown')
try:
ver_ok = int(ver) >= 13
except Exception:
ver_ok = False
if not ver_ok:
return self.simple_response(
http_client.BAD_REQUEST,
'Unsupported WebSocket protocol version: %s' % ver)
if self.method != 'GET':
return self.simple_response(
http_client.BAD_REQUEST,
'Invalid WebSocket method: %s' % self.method)
response = HANDSHAKE_STR % as_base64_unicode(
sha1((key + GUID_STR).encode('utf-8')).digest())
self.optimize_for_sending_packet()
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.set_state(WRITE, self.upgrade_connection_to_ws,
ReadOnlyFileBuffer(response.encode('ascii')), inheaders)
def upgrade_connection_to_ws(self, buf, inheaders, event):
if self.write(buf):
if self.websocket_handler is None:
self.websocket_handler = DummyHandler()
self.read_frame, self.current_recv_opcode = ReadFrame(), None
self.in_websocket_mode = True
try:
self.websocket_handler.handle_websocket_upgrade(
self.websocket_connection_id, weakref.ref(self), inheaders)
except Exception as err:
self.log.exception('Error in WebSockets upgrade handler:')
self.websocket_close(
UNEXPECTED_ERROR,
'Unexpected error in handler: %r' % as_unicode(err))
self.handle_event = self.ws_duplex
self.set_ws_state()
self.end_send_optimization()
def set_ws_state(self):
if self.ws_close_sent or self.ws_close_received:
if self.ws_close_sent:
self.ready = False
else:
self.wait_for = WRITE
return
if self.send_buf is not None or self.sending is not None:
self.wait_for = RDWR
else:
try:
self.sending = self.sendq.get_nowait()
except Empty:
with self.cf_lock:
if self.control_frames:
self.wait_for = RDWR
else:
self.wait_for = READ
else:
self.wait_for = RDWR
if self.stop_reading:
if self.wait_for is READ:
self.ready = False
elif self.wait_for is RDWR:
self.wait_for = WRITE
def ws_duplex(self, event):
if event is READ:
self.ws_read()
elif event is WRITE:
self.ws_write()
self.set_ws_state()
def ws_read(self):
if not self.stop_reading:
self.read_frame(self)
def ws_data_received(self, data, opcode, frame_starting, frame_finished,
is_final_frame_of_message):
if opcode in CONTROL_CODES:
return self.ws_control_frame(opcode, data)
message_starting = self.current_recv_opcode is None
if message_starting:
if opcode == CONTINUATION:
self.log.error(
'Client sent continuation frame with no message to continue'
)
self.websocket_close(
PROTOCOL_ERROR,
'Continuation frame without any message to continue')
return
self.current_recv_opcode = opcode
elif frame_starting and opcode != CONTINUATION:
self.log.error(
'Client sent continuation frame with non-zero opcode')
self.websocket_close(PROTOCOL_ERROR,
'Continuation frame with non-zero opcode')
return
message_finished = frame_finished and is_final_frame_of_message
if self.current_recv_opcode == TEXT:
if message_starting:
self.frag_decoder.reset()
empty_data = len(data) == 0
try:
data = self.frag_decoder(data)
except ValueError:
self.frag_decoder.reset()
self.log.error('Client sent undecodeable UTF-8')
return self.websocket_close(INCONSISTENT_DATA,
'Not valid UTF-8')
if message_finished:
if (not data and not empty_data) or self.frag_decoder.state:
self.frag_decoder.reset()
self.log.error('Client sent undecodeable UTF-8')
return self.websocket_close(INCONSISTENT_DATA,
'Not valid UTF-8')
if message_finished:
self.current_recv_opcode = None
self.frag_decoder.reset()
try:
self.handle_websocket_data(data, message_starting,
message_finished)
except Exception as err:
self.log.exception('Error in WebSockets data handler:')
self.websocket_close(
UNEXPECTED_ERROR,
'Unexpected error in handler: %r' % as_unicode(err))
def ws_control_frame(self, opcode, data):
if opcode in (PING, CLOSE):
rcode = PONG if opcode == PING else CLOSE
if opcode == CLOSE:
self.ws_close_received = True
self.stop_reading = True
if data:
try:
close_code = unpack_from(b'!H', data)[0]
except struct_error:
data = pack(
b'!H', PROTOCOL_ERROR
) + b'close frame data must be atleast two bytes'
else:
try:
utf8_decode(data[2:])
except ValueError:
data = pack(
b'!H', PROTOCOL_ERROR
) + b'close frame data must be valid UTF-8'
else:
if close_code < 1000 or close_code in RESERVED_CLOSE_CODES or (
1011 < close_code < 3000):
data = pack(
b'!H',
PROTOCOL_ERROR) + b'close code reserved'
else:
close_code = NORMAL_CLOSE
data = pack(b'!H', close_code)
f = ReadOnlyFileBuffer(create_frame(1, rcode, data))
f.is_close_frame = opcode == CLOSE
with self.cf_lock:
self.control_frames.append(f)
elif opcode == PONG:
try:
self.websocket_handler.handle_websocket_pong(
self.websocket_connection_id, data)
except Exception:
self.log.exception('Error in PONG handler:')
self.set_ws_state()
def websocket_close(self, code=NORMAL_CLOSE, reason=b''):
if isinstance(reason, unicode_type):
reason = reason.encode('utf-8')
self.stop_reading = True
reason = reason[:123]
if code is None and not reason:
f = ReadOnlyFileBuffer(create_frame(1, CLOSE, b''))
else:
f = ReadOnlyFileBuffer(
create_frame(1, CLOSE,
pack(b'!H', code) + reason))
f.is_close_frame = True
with self.cf_lock:
self.control_frames.append(f)
self.set_ws_state()
def ws_write(self):
if self.ws_close_sent:
return
if self.send_buf is not None:
if self.write(self.send_buf):
self.end_send_optimization()
if getattr(self.send_buf, 'is_close_frame', False):
self.ws_close_sent = True
self.send_buf = None
else:
with self.cf_lock:
try:
self.send_buf = self.control_frames.popleft()
except IndexError:
if self.sending is not None:
self.send_buf = self.sending.create_frame()
if self.send_buf is None:
self.sending = None
if self.send_buf is not None:
self.optimize_for_sending_packet()
def close(self):
if self.in_websocket_mode:
try:
self.websocket_handler.handle_websocket_close(
self.websocket_connection_id)
except Exception:
self.log.exception('Error in WebSocket close handler')
# Try to write a close frame, just once
try:
if self.send_buf is None and not self.ws_close_sent:
self.websocket_close(SHUTTING_DOWN, 'Shutting down')
with self.cf_lock:
self.write(self.control_frames.pop())
except Exception:
pass
Connection.close(self)
else:
HTTPConnection.close(self)
# }}}
def send_websocket_message(self, buf, wakeup=True):
''' Send a complete message. This class will take care of splitting it
into appropriate frames automatically. `buf` must be a file like object. '''
self.sendq.put(MessageWriter(buf))
self.wait_for = RDWR
if wakeup:
self.wakeup()
def send_websocket_frame(self, data, is_first=True, is_last=True):
''' Useful for streaming handlers that want to break up messages into
frames themselves. Note that these frames will be interleaved with
control frames, so they should not be too large. '''
opcode = (TEXT if isinstance(data, unicode_type) else
BINARY) if is_first else CONTINUATION
fin = 1 if is_last else 0
frame = create_frame(fin, opcode, data)
with self.cf_lock:
self.control_frames.append(ReadOnlyFileBuffer(frame))
def send_websocket_ping(self, data=b''):
''' Send a PING to the remote client, it should reply with a PONG which
will be sent to the handle_websocket_pong callback in your handler. '''
if isinstance(data, unicode_type):
data = data.encode('utf-8')
frame = create_frame(True, PING, data)
with self.cf_lock:
self.control_frames.append(ReadOnlyFileBuffer(frame))
def handle_websocket_data(self, data, message_starting, message_finished):
''' Called when some data is received from the remote client. In
general the data may not constitute a complete "message", use the
message_starting and message_finished flags to re-assemble it into a
complete message in the handler. Note that for binary data, data is a
mutable object. If you intend to keep it around after this method
returns, create a bytestring from it, using tobytes(). '''
self.websocket_handler.handle_websocket_data(
self.websocket_connection_id, data, message_starting,
message_finished)
class ParseWorker(Thread):
daemon = True
SLEEP_TIME = 1
def __init__(self):
Thread.__init__(self)
self.requests = Queue()
self.request_count = 0
self.parse_items = {}
self.launch_error = None
def run(self):
mod, func = 'calibre.gui2.tweak_book.preview', 'parse_html'
try:
# Connect to the worker and send a dummy job to initialize it
self.worker = offload_worker(priority='low')
self.worker(mod, func, '<p></p>')
except:
import traceback
traceback.print_exc()
self.launch_error = traceback.format_exc()
return
while True:
time.sleep(self.SLEEP_TIME)
x = self.requests.get()
requests = [x]
while True:
try:
requests.append(self.requests.get_nowait())
except Empty:
break
if shutdown in requests:
self.worker.shutdown()
break
request = sorted(requests, reverse=True)[0]
del requests
pi, data = request[1:]
try:
res = self.worker(mod, func, data)
except:
import traceback
traceback.print_exc()
else:
pi.parsing_done = True
parsed_data = res['result']
if res['tb']:
prints("Parser error:")
prints(res['tb'])
else:
pi.parsed_data = parsed_data
def add_request(self, name):
data = get_data(name)
ldata, hdata = len(data), hash(data)
pi = self.parse_items.get(name, None)
if pi is None:
self.parse_items[name] = pi = ParseItem(name)
else:
if pi.parsing_done and pi.length == ldata and pi.fingerprint == hdata:
return
pi.parsed_data = None
pi.parsing_done = False
pi.length, pi.fingerprint = ldata, hdata
self.requests.put((self.request_count, pi, data))
self.request_count += 1
def shutdown(self):
self.requests.put(shutdown)
def get_data(self, name):
return getattr(self.parse_items.get(name, None), 'parsed_data', None)
def clear(self):
self.parse_items.clear()
def is_alive(self):
return Thread.is_alive(self) or (hasattr(self, 'worker') and self.worker.is_alive())
def test_identify_plugin(name, tests, modify_plugin=lambda plugin:None, # {{{
fail_missing_meta=True):
'''
:param name: Plugin name
:param tests: List of 2-tuples. Each two tuple is of the form (args,
test_funcs). args is a dict of keyword arguments to pass to
the identify method. test_funcs are callables that accept a
Metadata object and return True iff the object passes the
test.
'''
plugin = None
for x in all_metadata_plugins():
if x.name == name and 'identify' in x.capabilities:
plugin = x
break
modify_plugin(plugin)
prints('Testing the identify function of', plugin.name)
prints('Using extra headers:', plugin.browser.addheaders)
tdir, lf, log, abort = init_test(plugin.name)
prints('Log saved to', lf)
times = []
for kwargs, test_funcs in tests:
log('')
log('#'*80)
log('### Running test with:', kwargs)
log('#'*80)
prints('Running test with:', kwargs)
rq = Queue()
args = (log, rq, abort)
start_time = time.time()
plugin.running_a_test = True
try:
err = plugin.identify(*args, **kwargs)
finally:
plugin.running_a_test = False
total_time = time.time() - start_time
times.append(total_time)
if err is not None:
prints('identify returned an error for args', args)
prints(err)
break
results = []
while True:
try:
results.append(rq.get_nowait())
except Empty:
break
prints('Found', len(results), 'matches:', end=' ')
prints('Smaller relevance means better match')
results.sort(key=plugin.identify_results_keygen(
title=kwargs.get('title', None), authors=kwargs.get('authors',
None), identifiers=kwargs.get('identifiers', {})))
for i, mi in enumerate(results):
prints('*'*30, 'Relevance:', i, '*'*30)
if mi.rating:
mi.rating *= 2
prints(mi)
prints('\nCached cover URL :',
plugin.get_cached_cover_url(mi.identifiers))
prints('*'*75, '\n\n')
possibles = []
for mi in results:
test_failed = False
for tfunc in test_funcs:
if not tfunc(mi):
test_failed = True
break
if not test_failed:
possibles.append(mi)
if not possibles:
prints('ERROR: No results that passed all tests were found')
prints('Log saved to', lf)
log.close()
dump_log(lf)
raise SystemExit(1)
good = [x for x in possibles if plugin.test_fields(x) is
None]
if not good:
prints('Failed to find', plugin.test_fields(possibles[0]))
if fail_missing_meta:
raise SystemExit(1)
if results[0] is not possibles[0]:
prints('Most relevant result failed the tests')
raise SystemExit(1)
if 'cover' in plugin.capabilities:
rq = Queue()
mi = results[0]
plugin.download_cover(log, rq, abort, title=mi.title,
authors=mi.authors, identifiers=mi.identifiers)
results = []
while True:
try:
results.append(rq.get_nowait())
except Empty:
break
if not results and fail_missing_meta:
prints('Cover download failed')
raise SystemExit(1)
elif results:
cdata = results[0]
cover = os.path.join(tdir, plugin.name.replace(' ',
'')+'-%s-cover.jpg'%sanitize_file_name(mi.title.replace(' ',
'_')))
with open(cover, 'wb') as f:
f.write(cdata[-1])
prints('Cover downloaded to:', cover)
if len(cdata[-1]) < 10240:
prints('Downloaded cover too small')
raise SystemExit(1)
prints('Average time per query', sum(times)/len(times))
if os.stat(lf).st_size > 10:
prints('There were some errors/warnings, see log', lf)
class WebSocketConnection(HTTPConnection):
# Internal API {{{
in_websocket_mode = False
websocket_handler = None
def __init__(self, *args, **kwargs):
global conn_id
HTTPConnection.__init__(self, *args, **kwargs)
self.sendq = Queue()
self.control_frames = deque()
self.cf_lock = Lock()
self.sending = None
self.send_buf = None
self.frag_decoder = UTF8Decoder()
self.ws_close_received = self.ws_close_sent = False
conn_id += 1
self.websocket_connection_id = conn_id
self.stop_reading = False
def finalize_headers(self, inheaders):
upgrade = inheaders.get('Upgrade', '')
key = inheaders.get('Sec-WebSocket-Key', None)
conn = {x.strip().lower() for x in inheaders.get('Connection', '').split(',')}
if key is None or upgrade.lower() != 'websocket' or 'upgrade' not in conn:
return HTTPConnection.finalize_headers(self, inheaders)
ver = inheaders.get('Sec-WebSocket-Version', 'Unknown')
try:
ver_ok = int(ver) >= 13
except Exception:
ver_ok = False
if not ver_ok:
return self.simple_response(http_client.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver)
if self.method != 'GET':
return self.simple_response(http_client.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method)
response = HANDSHAKE_STR % as_base64_unicode(sha1((key + GUID_STR).encode('utf-8')).digest())
self.optimize_for_sending_packet()
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.set_state(WRITE, self.upgrade_connection_to_ws, ReadOnlyFileBuffer(response.encode('ascii')), inheaders)
def upgrade_connection_to_ws(self, buf, inheaders, event):
if self.write(buf):
if self.websocket_handler is None:
self.websocket_handler = DummyHandler()
self.read_frame, self.current_recv_opcode = ReadFrame(), None
self.in_websocket_mode = True
try:
self.websocket_handler.handle_websocket_upgrade(self.websocket_connection_id, weakref.ref(self), inheaders)
except Exception as err:
self.log.exception('Error in WebSockets upgrade handler:')
self.websocket_close(UNEXPECTED_ERROR, 'Unexpected error in handler: %r' % as_unicode(err))
self.handle_event = self.ws_duplex
self.set_ws_state()
self.end_send_optimization()
def set_ws_state(self):
if self.ws_close_sent or self.ws_close_received:
if self.ws_close_sent:
self.ready = False
else:
self.wait_for = WRITE
return
if self.send_buf is not None or self.sending is not None:
self.wait_for = RDWR
else:
try:
self.sending = self.sendq.get_nowait()
except Empty:
with self.cf_lock:
if self.control_frames:
self.wait_for = RDWR
else:
self.wait_for = READ
else:
self.wait_for = RDWR
if self.stop_reading:
if self.wait_for is READ:
self.ready = False
elif self.wait_for is RDWR:
self.wait_for = WRITE
def ws_duplex(self, event):
if event is READ:
self.ws_read()
elif event is WRITE:
self.ws_write()
self.set_ws_state()
def ws_read(self):
if not self.stop_reading:
self.read_frame(self)
def ws_data_received(self, data, opcode, frame_starting, frame_finished, is_final_frame_of_message):
if opcode in CONTROL_CODES:
return self.ws_control_frame(opcode, data)
message_starting = self.current_recv_opcode is None
if message_starting:
if opcode == CONTINUATION:
self.log.error('Client sent continuation frame with no message to continue')
self.websocket_close(PROTOCOL_ERROR, 'Continuation frame without any message to continue')
return
self.current_recv_opcode = opcode
elif frame_starting and opcode != CONTINUATION:
self.log.error('Client sent continuation frame with non-zero opcode')
self.websocket_close(PROTOCOL_ERROR, 'Continuation frame with non-zero opcode')
return
message_finished = frame_finished and is_final_frame_of_message
if self.current_recv_opcode == TEXT:
if message_starting:
self.frag_decoder.reset()
empty_data = len(data) == 0
try:
data = self.frag_decoder(data)
except ValueError:
self.frag_decoder.reset()
self.log.error('Client sent undecodeable UTF-8')
return self.websocket_close(INCONSISTENT_DATA, 'Not valid UTF-8')
if message_finished:
if (not data and not empty_data) or self.frag_decoder.state:
self.frag_decoder.reset()
self.log.error('Client sent undecodeable UTF-8')
return self.websocket_close(INCONSISTENT_DATA, 'Not valid UTF-8')
if message_finished:
self.current_recv_opcode = None
self.frag_decoder.reset()
try:
self.handle_websocket_data(data, message_starting, message_finished)
except Exception as err:
self.log.exception('Error in WebSockets data handler:')
self.websocket_close(UNEXPECTED_ERROR, 'Unexpected error in handler: %r' % as_unicode(err))
def ws_control_frame(self, opcode, data):
if opcode in (PING, CLOSE):
rcode = PONG if opcode == PING else CLOSE
if opcode == CLOSE:
self.ws_close_received = True
self.stop_reading = True
if data:
try:
close_code = unpack_from(b'!H', data)[0]
except struct_error:
data = pack(b'!H', PROTOCOL_ERROR) + b'close frame data must be atleast two bytes'
else:
try:
utf8_decode(data[2:])
except ValueError:
data = pack(b'!H', PROTOCOL_ERROR) + b'close frame data must be valid UTF-8'
else:
if close_code < 1000 or close_code in RESERVED_CLOSE_CODES or (1011 < close_code < 3000):
data = pack(b'!H', PROTOCOL_ERROR) + b'close code reserved'
else:
close_code = NORMAL_CLOSE
data = pack(b'!H', close_code)
f = ReadOnlyFileBuffer(create_frame(1, rcode, data))
f.is_close_frame = opcode == CLOSE
with self.cf_lock:
self.control_frames.append(f)
elif opcode == PONG:
try:
self.websocket_handler.handle_websocket_pong(self.websocket_connection_id, data)
except Exception:
self.log.exception('Error in PONG handler:')
self.set_ws_state()
def websocket_close(self, code=NORMAL_CLOSE, reason=b''):
if isinstance(reason, type('')):
reason = reason.encode('utf-8')
self.stop_reading = True
reason = reason[:123]
if code is None and not reason:
f = ReadOnlyFileBuffer(create_frame(1, CLOSE, b''))
else:
f = ReadOnlyFileBuffer(create_frame(1, CLOSE, pack(b'!H', code) + reason))
f.is_close_frame = True
with self.cf_lock:
self.control_frames.append(f)
self.set_ws_state()
def ws_write(self):
if self.ws_close_sent:
return
if self.send_buf is not None:
if self.write(self.send_buf):
self.end_send_optimization()
if getattr(self.send_buf, 'is_close_frame', False):
self.ws_close_sent = True
self.send_buf = None
else:
with self.cf_lock:
try:
self.send_buf = self.control_frames.popleft()
except IndexError:
if self.sending is not None:
self.send_buf = self.sending.create_frame()
if self.send_buf is None:
self.sending = None
if self.send_buf is not None:
self.optimize_for_sending_packet()
def close(self):
if self.in_websocket_mode:
try:
self.websocket_handler.handle_websocket_close(self.websocket_connection_id)
except Exception:
self.log.exception('Error in WebSocket close handler')
# Try to write a close frame, just once
try:
if self.send_buf is None and not self.ws_close_sent:
self.websocket_close(SHUTTING_DOWN, 'Shutting down')
with self.cf_lock:
self.write(self.control_frames.pop())
except Exception:
pass
Connection.close(self)
else:
HTTPConnection.close(self)
# }}}
def send_websocket_message(self, buf, wakeup=True):
''' Send a complete message. This class will take care of splitting it
into appropriate frames automatically. `buf` must be a file like object. '''
self.sendq.put(MessageWriter(buf))
self.wait_for = RDWR
if wakeup:
self.wakeup()
def send_websocket_frame(self, data, is_first=True, is_last=True):
''' Useful for streaming handlers that want to break up messages into
frames themselves. Note that these frames will be interleaved with
control frames, so they should not be too large. '''
opcode = (TEXT if isinstance(data, type('')) else BINARY) if is_first else CONTINUATION
fin = 1 if is_last else 0
frame = create_frame(fin, opcode, data)
with self.cf_lock:
self.control_frames.append(ReadOnlyFileBuffer(frame))
def send_websocket_ping(self, data=b''):
''' Send a PING to the remote client, it should reply with a PONG which
will be sent to the handle_websocket_pong callback in your handler. '''
if isinstance(data, type('')):
data = data.encode('utf-8')
frame = create_frame(True, PING, data)
with self.cf_lock:
self.control_frames.append(ReadOnlyFileBuffer(frame))
def handle_websocket_data(self, data, message_starting, message_finished):
''' Called when some data is received from the remote client. In
general the data may not constitute a complete "message", use the
message_starting and message_finished flags to re-assemble it into a
complete message in the handler. Note that for binary data, data is a
mutable object. If you intend to keep it around after this method
returns, create a bytestring from it, using tobytes(). '''
self.websocket_handler.handle_websocket_data(self.websocket_connection_id, data, message_starting, message_finished)
class Server(Thread):
def __init__(self, notify_on_job_done=lambda x: x, pool_size=None,
limit=sys.maxsize, enforce_cpu_limit=True):
Thread.__init__(self)
self.daemon = True
global _counter
self.id = _counter+1
_counter += 1
if enforce_cpu_limit:
limit = min(limit, cpu_count())
self.pool_size = limit if pool_size is None else pool_size
self.notify_on_job_done = notify_on_job_done
self.auth_key = os.urandom(32)
self.address, self.listener = create_listener(self.auth_key, backlog=4)
self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue()
self.kill_queue = Queue()
self.waiting_jobs = []
self.workers = deque()
self.launched_worker_count = 0
self._worker_launch_lock = RLock()
self.start()
def launch_worker(self, gui=False, redirect_output=None, job_name=None):
start = time.time()
with self._worker_launch_lock:
self.launched_worker_count += 1
id = self.launched_worker_count
fd, rfile = tempfile.mkstemp(prefix=u'ipc_result_%d_%d_'%(self.id, id),
dir=base_dir(), suffix=u'.pickle')
os.close(fd)
if redirect_output is None:
redirect_output = not gui
env = {
'CALIBRE_WORKER_ADDRESS' : environ_item(as_hex_unicode(msgpack_dumps(self.address))),
'CALIBRE_WORKER_KEY' : environ_item(as_hex_unicode(self.auth_key)),
'CALIBRE_WORKER_RESULT' : environ_item(as_hex_unicode(rfile)),
}
cw = self.do_launch(env, gui, redirect_output, rfile, job_name=job_name)
if isinstance(cw, string_or_bytes):
raise CriticalError('Failed to launch worker process:\n'+cw)
if DEBUG:
print('Worker Launch took:', time.time() - start)
return cw
def do_launch(self, env, gui, redirect_output, rfile, job_name=None):
w = Worker(env, gui=gui, job_name=job_name)
try:
w(redirect_output=redirect_output)
conn = eintr_retry_call(self.listener.accept)
if conn is None:
raise Exception('Failed to launch worker process')
except BaseException:
try:
w.kill()
except:
pass
import traceback
return traceback.format_exc()
return ConnectedWorker(w, conn, rfile)
def add_job(self, job):
job.done2 = self.notify_on_job_done
self.add_jobs_queue.put(job)
def run_job(self, job, gui=True, redirect_output=False):
w = self.launch_worker(gui=gui, redirect_output=redirect_output, job_name=getattr(job, 'name', None))
w.start_job(job)
def run(self):
while True:
try:
job = self.add_jobs_queue.get(True, 0.2)
if job is None:
break
self.waiting_jobs.insert(0, job)
except Empty:
pass
# Get notifications from worker process
for worker in self.workers:
while True:
try:
n = worker.notifications.get_nowait()
worker.job.notifications.put(n)
self.changed_jobs_queue.put(worker.job)
except Empty:
break
# Remove finished jobs
for worker in [w for w in self.workers if not w.is_alive]:
try:
worker.close_log_file()
except:
pass
self.workers.remove(worker)
job = worker.job
if worker.returncode != 0:
job.failed = True
job.returncode = worker.returncode
elif os.path.exists(worker.rfile):
try:
with lopen(worker.rfile, 'rb') as f:
job.result = pickle_loads(f.read())
os.remove(worker.rfile)
except:
pass
job.duration = time.time() - job.start_time
self.changed_jobs_queue.put(job)
# Start waiting jobs
sj = self.suitable_waiting_job()
if sj is not None:
job = self.waiting_jobs.pop(sj)
job.start_time = time.time()
if job.kill_on_start:
job.duration = 0.0
job.returncode = 1
job.killed = job.failed = True
job.result = None
else:
worker = self.launch_worker()
worker.start_job(job)
self.workers.append(worker)
job.log_path = worker.log_path
self.changed_jobs_queue.put(job)
while True:
try:
j = self.kill_queue.get_nowait()
self._kill_job(j)
except Empty:
break
def suitable_waiting_job(self):
available_workers = self.pool_size - len(self.workers)
for worker in self.workers:
job = worker.job
if job.core_usage == -1:
available_workers = 0
elif job.core_usage > 1:
available_workers -= job.core_usage - 1
if available_workers < 1:
return None
for i, job in enumerate(self.waiting_jobs):
if job.core_usage == -1:
if available_workers >= self.pool_size:
return i
elif job.core_usage <= available_workers:
return i
def kill_job(self, job):
self.kill_queue.put(job)
def killall(self):
for worker in self.workers:
self.kill_queue.put(worker.job)
def _kill_job(self, job):
if job.start_time is None:
job.kill_on_start = True
return
for worker in self.workers:
if job is worker.job:
worker.kill()
job.killed = True
break
def split(self, tasks):
'''
Split a list into a list of sub lists, with the number of sub lists being
no more than the number of workers this server supports. Each sublist contains
2-tuples of the form (i, x) where x is an element from the original list
and i is the index of the element x in the original list.
'''
ans, count, pos = [], 0, 0
delta = int(ceil(len(tasks)/float(self.pool_size)))
while count < len(tasks):
section = []
for t in tasks[pos:pos+delta]:
section.append((count, t))
count += 1
ans.append(section)
pos += delta
return ans
def close(self):
try:
self.add_jobs_queue.put(None)
except:
pass
try:
self.listener.close()
except:
pass
time.sleep(0.2)
for worker in list(self.workers):
try:
worker.kill()
except:
pass
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
TagBrowserMixin, CoverFlowMixin, LibraryViewMixin, SearchBoxMixin,
SavedSearchBoxMixin, SearchRestrictionMixin, LayoutMixin, UpdateMixin,
EbookDownloadMixin
):
'The main GUI'
proceed_requested = pyqtSignal(object, object)
book_converted = pyqtSignal(object, object)
shutting_down = False
def __init__(self, opts, parent=None, gui_debug=None):
MainWindow.__init__(self, opts, parent=parent, disable_automatic_gc=True)
self.setWindowIcon(QApplication.instance().windowIcon())
self.jobs_pointer = Pointer(self)
self.proceed_requested.connect(self.do_proceed,
type=Qt.QueuedConnection)
self.proceed_question = ProceedQuestion(self)
self.job_error_dialog = JobError(self)
self.keyboard = Manager(self)
get_gui.ans = self
self.opts = opts
self.device_connected = None
self.gui_debug = gui_debug
self.iactions = OrderedDict()
# Actions
for action in interface_actions():
if opts.ignore_plugins and action.plugin_path is not None:
continue
try:
ac = self.init_iaction(action)
except:
# Ignore errors in loading user supplied plugins
import traceback
traceback.print_exc()
if action.plugin_path is None:
raise
continue
ac.plugin_path = action.plugin_path
ac.interface_action_base_plugin = action
self.add_iaction(ac)
self.load_store_plugins()
def init_iaction(self, action):
ac = action.load_actual_plugin(self)
ac.plugin_path = action.plugin_path
ac.interface_action_base_plugin = action
action.actual_iaction_plugin_loaded = True
return ac
def add_iaction(self, ac):
acmap = self.iactions
if ac.name in acmap:
if ac.priority >= acmap[ac.name].priority:
acmap[ac.name] = ac
else:
acmap[ac.name] = ac
def load_store_plugins(self):
from calibre.gui2.store.loader import Stores
self.istores = Stores()
for store in available_store_plugins():
if self.opts.ignore_plugins and store.plugin_path is not None:
continue
try:
st = self.init_istore(store)
self.add_istore(st)
except:
# Ignore errors in loading user supplied plugins
import traceback
traceback.print_exc()
if store.plugin_path is None:
raise
continue
self.istores.builtins_loaded()
def init_istore(self, store):
st = store.load_actual_plugin(self)
st.plugin_path = store.plugin_path
st.base_plugin = store
store.actual_istore_plugin_loaded = True
return st
def add_istore(self, st):
stmap = self.istores
if st.name in stmap:
if st.priority >= stmap[st.name].priority:
stmap[st.name] = st
else:
stmap[st.name] = st
def initialize(self, library_path, db, listener, actions, show_gui=True):
opts = self.opts
self.preferences_action, self.quit_action = actions
self.library_path = library_path
self.library_broker = GuiLibraryBroker(db)
self.content_server = None
self.server_change_notification_timer = t = QTimer(self)
self.server_changes = Queue()
t.setInterval(1000), t.timeout.connect(self.handle_changes_from_server_debounced), t.setSingleShot(True)
self._spare_pool = None
self.must_restart_before_config = False
self.listener = Listener(listener)
self.check_messages_timer = QTimer()
self.check_messages_timer.timeout.connect(self.another_instance_wants_to_talk)
self.check_messages_timer.start(1000)
for ac in self.iactions.values():
try:
ac.do_genesis()
except Exception:
# Ignore errors in third party plugins
import traceback
traceback.print_exc()
if getattr(ac, 'plugin_path', None) is None:
raise
self.donate_action = QAction(QIcon(I('donate.png')),
_('&Donate to support calibre'), self)
for st in self.istores.values():
st.do_genesis()
MainWindowMixin.init_main_window_mixin(self, db)
# Jobs Button {{{
self.job_manager = JobManager()
self.jobs_dialog = JobsDialog(self, self.job_manager)
self.jobs_button = JobsButton(parent=self)
self.jobs_button.initialize(self.jobs_dialog, self.job_manager)
# }}}
LayoutMixin.init_layout_mixin(self)
DeviceMixin.init_device_mixin(self)
self.progress_indicator = ProgressIndicator(self)
self.progress_indicator.pos = (0, 20)
self.verbose = opts.verbose
self.get_metadata = GetMetadata()
self.upload_memory = {}
self.metadata_dialogs = []
self.default_thumbnail = None
self.tb_wrapper = textwrap.TextWrapper(width=40)
self.viewers = collections.deque()
self.system_tray_icon = None
do_systray = config['systray_icon'] or opts.start_in_tray
if do_systray:
self.system_tray_icon = factory(app_id='com.calibre-ebook.gui').create_system_tray_icon(parent=self, title='calibre')
if self.system_tray_icon is not None:
self.system_tray_icon.setIcon(QIcon(I('lt.png', allow_user_override=False)))
if not (iswindows or isosx):
self.system_tray_icon.setIcon(QIcon.fromTheme('calibre-tray', self.system_tray_icon.icon()))
self.system_tray_icon.setToolTip(self.jobs_button.tray_tooltip())
self.system_tray_icon.setVisible(True)
self.jobs_button.tray_tooltip_updated.connect(self.system_tray_icon.setToolTip)
elif do_systray:
prints('Failed to create system tray icon, your desktop environment probably'
' does not support the StatusNotifier spec https://www.freedesktop.org/wiki/Specifications/StatusNotifierItem/')
self.system_tray_menu = QMenu(self)
self.toggle_to_tray_action = self.system_tray_menu.addAction(QIcon(I('page.png')), '')
self.toggle_to_tray_action.triggered.connect(self.system_tray_icon_activated)
self.system_tray_menu.addAction(self.donate_action)
self.eject_action = self.system_tray_menu.addAction(
QIcon(I('eject.png')), _('&Eject connected device'))
self.eject_action.setEnabled(False)
self.addAction(self.quit_action)
self.system_tray_menu.addAction(self.quit_action)
self.keyboard.register_shortcut('quit calibre', _('Quit calibre'),
default_keys=('Ctrl+Q',), action=self.quit_action)
if self.system_tray_icon is not None:
self.system_tray_icon.setContextMenu(self.system_tray_menu)
self.system_tray_icon.activated.connect(self.system_tray_icon_activated)
self.quit_action.triggered[bool].connect(self.quit)
self.donate_action.triggered[bool].connect(self.donate)
self.minimize_action = QAction(_('Minimize the calibre window'), self)
self.addAction(self.minimize_action)
self.keyboard.register_shortcut('minimize calibre', self.minimize_action.text(),
default_keys=(), action=self.minimize_action)
self.minimize_action.triggered.connect(self.showMinimized)
self.esc_action = QAction(self)
self.addAction(self.esc_action)
self.keyboard.register_shortcut('clear current search',
_('Clear the current search'), default_keys=('Esc',),
action=self.esc_action)
self.esc_action.triggered.connect(self.esc)
self.shift_esc_action = QAction(self)
self.addAction(self.shift_esc_action)
self.keyboard.register_shortcut('focus book list',
_('Focus the book list'), default_keys=('Shift+Esc',),
action=self.shift_esc_action)
self.shift_esc_action.triggered.connect(self.shift_esc)
self.ctrl_esc_action = QAction(self)
self.addAction(self.ctrl_esc_action)
self.keyboard.register_shortcut('clear virtual library',
_('Clear the virtual library'), default_keys=('Ctrl+Esc',),
action=self.ctrl_esc_action)
self.ctrl_esc_action.triggered.connect(self.ctrl_esc)
self.alt_esc_action = QAction(self)
self.addAction(self.alt_esc_action)
self.keyboard.register_shortcut('clear additional restriction',
_('Clear the additional restriction'), default_keys=('Alt+Esc',),
action=self.alt_esc_action)
self.alt_esc_action.triggered.connect(self.clear_additional_restriction)
# ###################### Start spare job server ########################
QTimer.singleShot(1000, self.create_spare_pool)
# ###################### Location Manager ########################
self.location_manager.location_selected.connect(self.location_selected)
self.location_manager.unmount_device.connect(self.device_manager.umount_device)
self.location_manager.configure_device.connect(self.configure_connected_device)
self.location_manager.update_device_metadata.connect(self.update_metadata_on_device)
self.eject_action.triggered.connect(self.device_manager.umount_device)
# ################### Update notification ###################
UpdateMixin.init_update_mixin(self, opts)
# ###################### Search boxes ########################
SearchRestrictionMixin.init_search_restriction_mixin(self)
SavedSearchBoxMixin.init_saved_seach_box_mixin(self)
# ###################### Library view ########################
LibraryViewMixin.init_library_view_mixin(self, db)
SearchBoxMixin.init_search_box_mixin(self) # Requires current_db
self.library_view.model().count_changed_signal.connect(
self.iactions['Choose Library'].count_changed)
if not gprefs.get('quick_start_guide_added', False):
try:
add_quick_start_guide(self.library_view)
except:
import traceback
traceback.print_exc()
for view in ('library', 'memory', 'card_a', 'card_b'):
v = getattr(self, '%s_view' % view)
v.selectionModel().selectionChanged.connect(self.update_status_bar)
v.model().count_changed_signal.connect(self.update_status_bar)
self.library_view.model().count_changed()
self.bars_manager.database_changed(self.library_view.model().db)
self.library_view.model().database_changed.connect(self.bars_manager.database_changed,
type=Qt.QueuedConnection)
# ########################## Tags Browser ##############################
TagBrowserMixin.init_tag_browser_mixin(self, db)
self.library_view.model().database_changed.connect(self.populate_tb_manage_menu, type=Qt.QueuedConnection)
# ######################## Search Restriction ##########################
if db.prefs['virtual_lib_on_startup']:
self.apply_virtual_library(db.prefs['virtual_lib_on_startup'])
self.rebuild_vl_tabs()
# ########################## Cover Flow ################################
CoverFlowMixin.init_cover_flow_mixin(self)
self._calculated_available_height = min(max_available_height()-15,
self.height())
self.resize(self.width(), self._calculated_available_height)
self.build_context_menus()
for ac in self.iactions.values():
try:
ac.gui_layout_complete()
except:
import traceback
traceback.print_exc()
if ac.plugin_path is None:
raise
if config['autolaunch_server']:
self.start_content_server()
self.read_settings()
self.finalize_layout()
self.bars_manager.start_animation()
self.set_window_title()
for ac in self.iactions.values():
try:
ac.initialization_complete()
except:
import traceback
traceback.print_exc()
if ac.plugin_path is None:
raise
self.set_current_library_information(current_library_name(), db.library_id,
db.field_metadata)
register_keyboard_shortcuts()
self.keyboard.finalize()
if show_gui:
# Note this has to come after restoreGeometry() because of
# https://bugreports.qt.io/browse/QTBUG-56831
self.show()
if self.system_tray_icon is not None and self.system_tray_icon.isVisible() and opts.start_in_tray:
self.hide_windows()
self.auto_adder = AutoAdder(gprefs['auto_add_path'], self)
# Now that the gui is initialized we can restore the quickview state
# The same thing will be true for any action-based operation with a
# layout button
from calibre.gui2.actions.show_quickview import get_quickview_action_plugin
qv = get_quickview_action_plugin()
if qv:
qv.qv_button.restore_state()
self.save_layout_state()
# Collect cycles now
gc.collect()
QApplication.instance().shutdown_signal_received.connect(self.quit)
if show_gui and self.gui_debug is not None:
QTimer.singleShot(10, self.show_gui_debug_msg)
self.iactions['Connect Share'].check_smartdevice_menus()
QTimer.singleShot(1, self.start_smartdevice)
QTimer.singleShot(100, self.update_toggle_to_tray_action)
def show_gui_debug_msg(self):
info_dialog(self, _('Debug mode'), '<p>' +
_('You have started calibre in debug mode. After you '
'quit calibre, the debug log will be available in '
'the file: %s<p>The '
'log will be displayed automatically.')%self.gui_debug, show=True)
def esc(self, *args):
self.search.clear()
def shift_esc(self):
self.current_view().setFocus(Qt.OtherFocusReason)
def ctrl_esc(self):
self.apply_virtual_library()
self.current_view().setFocus(Qt.OtherFocusReason)
def start_smartdevice(self):
message = None
if self.device_manager.get_option('smartdevice', 'autostart'):
try:
message = self.device_manager.start_plugin('smartdevice')
except:
message = 'start smartdevice unknown exception'
prints(message)
import traceback
traceback.print_exc()
if message:
if not self.device_manager.is_running('Wireless Devices'):
error_dialog(self, _('Problem starting the wireless device'),
_('The wireless device driver had problems starting. '
'It said "%s"')%message, show=True)
self.iactions['Connect Share'].set_smartdevice_action_state()
def start_content_server(self, check_started=True):
from calibre.srv.embedded import Server
if not gprefs.get('server3_warning_done', False):
gprefs.set('server3_warning_done', True)
if os.path.exists(os.path.join(config_dir, 'server.py')):
try:
os.remove(os.path.join(config_dir, 'server.py'))
except EnvironmentError:
pass
warning_dialog(self, _('Content server changed!'), _(
'calibre 3 comes with a completely re-written content server.'
' As such any custom configuration you have for the content'
' server no longer applies. You should check and refresh your'
' settings in Preferences->Sharing->Sharing over the net'), show=True)
self.content_server = Server(self.library_broker, Dispatcher(self.handle_changes_from_server))
self.content_server.state_callback = Dispatcher(
self.iactions['Connect Share'].content_server_state_changed)
if check_started:
self.content_server.start_failure_callback = \
Dispatcher(self.content_server_start_failed)
self.content_server.start()
def handle_changes_from_server(self, library_path, change_event):
if DEBUG:
prints('Received server change event: {} for {}'.format(change_event, library_path))
if self.library_broker.is_gui_library(library_path):
self.server_changes.put((library_path, change_event))
self.server_change_notification_timer.start()
def handle_changes_from_server_debounced(self):
if self.shutting_down:
return
changes = []
while True:
try:
library_path, change_event = self.server_changes.get_nowait()
except Empty:
break
if self.library_broker.is_gui_library(library_path):
changes.append(change_event)
if changes:
handle_changes(changes, self)
def content_server_start_failed(self, msg):
self.content_server = None
error_dialog(self, _('Failed to start Content server'),
_('Could not start the Content server. Error:\n\n%s')%msg,
show=True)
def resizeEvent(self, ev):
MainWindow.resizeEvent(self, ev)
self.search.setMaximumWidth(self.width()-150)
def create_spare_pool(self, *args):
if self._spare_pool is None:
num = min(detect_ncpus(), int(config['worker_limit']/2.0))
self._spare_pool = Pool(max_workers=num, name='GUIPool')
def spare_pool(self):
ans, self._spare_pool = self._spare_pool, None
QTimer.singleShot(1000, self.create_spare_pool)
return ans
def do_proceed(self, func, payload):
if callable(func):
func(payload)
def no_op(self, *args):
pass
def system_tray_icon_activated(self, r=False):
if r in (QSystemTrayIcon.Trigger, QSystemTrayIcon.MiddleClick, False):
if self.isVisible():
if self.isMinimized():
self.showNormal()
else:
self.hide_windows()
else:
self.show_windows()
if self.isMinimized():
self.showNormal()
@property
def is_minimized_to_tray(self):
return getattr(self, '__systray_minimized', False)
def ask_a_yes_no_question(self, title, msg, det_msg='',
show_copy_button=False, ans_when_user_unavailable=True,
skip_dialog_name=None, skipped_value=True):
if self.is_minimized_to_tray:
return ans_when_user_unavailable
return question_dialog(self, title, msg, det_msg=det_msg,
show_copy_button=show_copy_button,
skip_dialog_name=skip_dialog_name,
skip_dialog_skipped_value=skipped_value)
def update_toggle_to_tray_action(self, *args):
if hasattr(self, 'toggle_to_tray_action'):
self.toggle_to_tray_action.setText(
_('Hide main window') if self.isVisible() else _('Show main window'))
def hide_windows(self):
for window in QApplication.topLevelWidgets():
if isinstance(window, (MainWindow, QDialog)) and \
window.isVisible():
window.hide()
setattr(window, '__systray_minimized', True)
self.update_toggle_to_tray_action()
def show_windows(self, *args):
for window in QApplication.topLevelWidgets():
if getattr(window, '__systray_minimized', False):
window.show()
setattr(window, '__systray_minimized', False)
self.update_toggle_to_tray_action()
def test_server(self, *args):
if self.content_server is not None and \
self.content_server.exception is not None:
error_dialog(self, _('Failed to start Content server'),
unicode_type(self.content_server.exception)).exec_()
@property
def current_db(self):
return self.library_view.model().db
def refresh_all(self):
m = self.library_view.model()
m.db.data.refresh(clear_caches=False, do_search=False)
self.saved_searches_changed(recount=False)
m.resort()
m.research()
self.tags_view.recount()
def handle_cli_args(self, args):
if isinstance(args, string_or_bytes):
args = [args]
files = [os.path.abspath(p) for p in args if not os.path.isdir(p) and os.access(p, os.R_OK)]
if files:
self.iactions['Add Books'].add_filesystem_book(files)
def another_instance_wants_to_talk(self):
try:
msg = self.listener.queue.get_nowait()
except Empty:
return
if msg.startswith('launched:'):
import json
try:
argv = json.loads(msg[len('launched:'):])
except ValueError:
prints('Failed to decode message from other instance: %r' % msg)
if DEBUG:
error_dialog(self, 'Invalid message',
'Received an invalid message from other calibre instance.'
' Do you have multiple versions of calibre installed?',
det_msg='Invalid msg: %r' % msg, show=True)
argv = ()
if isinstance(argv, (list, tuple)) and len(argv) > 1:
self.handle_cli_args(argv[1:])
self.setWindowState(self.windowState() & ~Qt.WindowMinimized|Qt.WindowActive)
self.show_windows()
self.raise_()
self.activateWindow()
elif msg.startswith('refreshdb:'):
m = self.library_view.model()
m.db.new_api.reload_from_db()
self.refresh_all()
elif msg.startswith('shutdown:'):
self.quit(confirm_quit=False)
elif msg.startswith('bookedited:'):
parts = msg.split(':')[1:]
try:
book_id, fmt, library_id = parts[:3]
book_id = int(book_id)
m = self.library_view.model()
db = m.db.new_api
if m.db.library_id == library_id and db.has_id(book_id):
db.format_metadata(book_id, fmt, allow_cache=False, update_db=True)
db.update_last_modified((book_id,))
m.refresh_ids((book_id,))
except Exception:
import traceback
traceback.print_exc()
else:
print(msg)
def current_view(self):
'''Convenience method that returns the currently visible view '''
idx = self.stack.currentIndex()
if idx == 0:
return self.library_view
if idx == 1:
return self.memory_view
if idx == 2:
return self.card_a_view
if idx == 3:
return self.card_b_view
def booklists(self):
return self.memory_view.model().db, self.card_a_view.model().db, self.card_b_view.model().db
def library_moved(self, newloc, copy_structure=False, allow_rebuild=False):
if newloc is None:
return
with self.library_broker:
default_prefs = None
try:
olddb = self.library_view.model().db
if copy_structure:
default_prefs = olddb.prefs
except:
olddb = None
if copy_structure and olddb is not None and default_prefs is not None:
default_prefs['field_metadata'] = olddb.new_api.field_metadata.all_metadata()
db = self.library_broker.prepare_for_gui_library_change(newloc)
if db is None:
try:
db = LibraryDatabase(newloc, default_prefs=default_prefs)
except apsw.Error:
if not allow_rebuild:
raise
import traceback
repair = question_dialog(self, _('Corrupted database'),
_('The library database at %s appears to be corrupted. Do '
'you want calibre to try and rebuild it automatically? '
'The rebuild may not be completely successful.')
% force_unicode(newloc, filesystem_encoding),
det_msg=traceback.format_exc()
)
if repair:
from calibre.gui2.dialogs.restore_library import repair_library_at
if repair_library_at(newloc, parent=self):
db = LibraryDatabase(newloc, default_prefs=default_prefs)
else:
return
else:
return
self.library_path = newloc
prefs['library_path'] = self.library_path
self.book_on_device(None, reset=True)
db.set_book_on_device_func(self.book_on_device)
self.library_view.set_database(db)
self.tags_view.set_database(db, self.alter_tb)
self.library_view.model().set_book_on_device_func(self.book_on_device)
self.status_bar.clear_message()
self.search.clear()
self.saved_search.clear()
self.book_details.reset_info()
# self.library_view.model().count_changed()
db = self.library_view.model().db
self.iactions['Choose Library'].count_changed(db.count())
self.set_window_title()
self.apply_named_search_restriction('') # reset restriction to null
self.saved_searches_changed(recount=False) # reload the search restrictions combo box
if db.prefs['virtual_lib_on_startup']:
self.apply_virtual_library(db.prefs['virtual_lib_on_startup'])
self.rebuild_vl_tabs()
for action in self.iactions.values():
action.library_changed(db)
self.library_broker.gui_library_changed(db, olddb)
if self.device_connected:
self.set_books_in_library(self.booklists(), reset=True)
self.refresh_ondevice()
self.memory_view.reset()
self.card_a_view.reset()
self.card_b_view.reset()
self.set_current_library_information(current_library_name(), db.library_id,
db.field_metadata)
self.library_view.set_current_row(0)
# Run a garbage collection now so that it does not freeze the
# interface later
gc.collect()
def set_window_title(self):
db = self.current_db
restrictions = [x for x in (db.data.get_base_restriction_name(),
db.data.get_search_restriction_name()) if x]
restrictions = ' :: '.join(restrictions)
font = QFont()
if restrictions:
restrictions = ' :: ' + restrictions
font.setBold(True)
font.setItalic(True)
self.virtual_library.setFont(font)
title = u'{0} - || {1}{2} ||'.format(
__appname__, self.iactions['Choose Library'].library_name(), restrictions)
self.setWindowTitle(title)
def location_selected(self, location):
'''
Called when a location icon is clicked (e.g. Library)
'''
page = 0 if location == 'library' else 1 if location == 'main' else 2 if location == 'carda' else 3
self.stack.setCurrentIndex(page)
self.book_details.reset_info()
for x in ('tb', 'cb'):
splitter = getattr(self, x+'_splitter')
splitter.button.setEnabled(location == 'library')
for action in self.iactions.values():
action.location_selected(location)
if location == 'library':
self.virtual_library_menu.setEnabled(True)
self.highlight_only_button.setEnabled(True)
self.vl_tabs.setEnabled(True)
else:
self.virtual_library_menu.setEnabled(False)
self.highlight_only_button.setEnabled(False)
self.vl_tabs.setEnabled(False)
# Reset the view in case something changed while it was invisible
self.current_view().reset()
self.set_number_of_books_shown()
self.update_status_bar()
def job_exception(self, job, dialog_title=_('Conversion error'), retry_func=None):
if not hasattr(self, '_modeless_dialogs'):
self._modeless_dialogs = []
minz = self.is_minimized_to_tray
if self.isVisible():
for x in list(self._modeless_dialogs):
if not x.isVisible():
self._modeless_dialogs.remove(x)
try:
if 'calibre.ebooks.DRMError' in job.details:
if not minz:
from calibre.gui2.dialogs.drm_error import DRMErrorMessage
d = DRMErrorMessage(self, _('Cannot convert') + ' ' +
job.description.split(':')[-1].partition('(')[-1][:-1])
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.ebooks.oeb.transforms.split.SplitError' in job.details:
title = job.description.split(':')[-1].partition('(')[-1][:-1]
msg = _('<p><b>Failed to convert: %s')%title
msg += '<p>'+_('''
Many older e-book reader devices are incapable of displaying
EPUB files that have internal components over a certain size.
Therefore, when converting to EPUB, calibre automatically tries
to split up the EPUB into smaller sized pieces. For some
files that are large undifferentiated blocks of text, this
splitting fails.
<p>You can <b>work around the problem</b> by either increasing the
maximum split size under <i>EPUB output</i> in the conversion dialog,
or by turning on Heuristic Processing, also in the conversion
dialog. Note that if you make the maximum split size too large,
your e-book reader may have trouble with the EPUB.
''')
if not minz:
d = error_dialog(self, _('Conversion Failed'), msg,
det_msg=job.details)
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.ebooks.mobi.reader.mobi6.KFXError:' in job.details:
if not minz:
title = job.description.split(':')[-1].partition('(')[-1][:-1]
msg = _('<p><b>Failed to convert: %s') % title
idx = job.details.index('calibre.ebooks.mobi.reader.mobi6.KFXError:')
msg += '<p>' + re.sub(r'(https:\S+)', r'<a href="\1">{}</a>'.format(_('here')),
job.details[idx:].partition(':')[2].strip())
d = error_dialog(self, _('Conversion failed'), msg, det_msg=job.details)
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.web.feeds.input.RecipeDisabled' in job.details:
if not minz:
msg = job.details
msg = msg[msg.find('calibre.web.feeds.input.RecipeDisabled:'):]
msg = msg.partition(':')[-1]
d = error_dialog(self, _('Recipe Disabled'),
'<p>%s</p>'%msg)
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
if 'calibre.ebooks.conversion.ConversionUserFeedBack:' in job.details:
if not minz:
import json
payload = job.details.rpartition(
'calibre.ebooks.conversion.ConversionUserFeedBack:')[-1]
payload = json.loads('{' + payload.partition('{')[-1])
d = {'info':info_dialog, 'warn':warning_dialog,
'error':error_dialog}.get(payload['level'],
error_dialog)
d = d(self, payload['title'],
'<p>%s</p>'%payload['msg'],
det_msg=payload['det_msg'])
d.setModal(False)
d.show()
self._modeless_dialogs.append(d)
return
except:
pass
if job.killed:
return
try:
prints(job.details, file=sys.stderr)
except:
pass
if not minz:
self.job_error_dialog.show_error(dialog_title,
_('<b>Failed</b>')+': '+unicode_type(job.description),
det_msg=job.details, retry_func=retry_func)
def read_settings(self):
geometry = config['main_window_geometry']
if geometry is not None:
self.restoreGeometry(geometry)
self.read_layout_settings()
def write_settings(self):
with gprefs: # Only write to gprefs once
config.set('main_window_geometry', self.saveGeometry())
dynamic.set('sort_history', self.library_view.model().sort_history)
self.save_layout_state()
self.stack.tb_widget.save_state()
def quit(self, checked=True, restart=False, debug_on_restart=False,
confirm_quit=True):
if self.shutting_down:
return
if confirm_quit and not self.confirm_quit():
return
try:
self.shutdown()
except:
pass
self.restart_after_quit = restart
self.debug_on_restart = debug_on_restart
if self.system_tray_icon is not None and self.restart_after_quit:
# Needed on windows to prevent multiple systray icons
self.system_tray_icon.setVisible(False)
QApplication.instance().quit()
def donate(self, *args):
from calibre.utils.localization import localize_website_link
open_url(QUrl(localize_website_link('https://calibre-ebook.com/donate')))
def confirm_quit(self):
if self.job_manager.has_jobs():
msg = _('There are active jobs. Are you sure you want to quit?')
if self.job_manager.has_device_jobs():
msg = '<p>'+__appname__ + \
_(''' is communicating with the device!<br>
Quitting may cause corruption on the device.<br>
Are you sure you want to quit?''')+'</p>'
if not question_dialog(self, _('Active jobs'), msg):
return False
if self.proceed_question.questions:
msg = _('There are library updates waiting. Are you sure you want to quit?')
if not question_dialog(self, _('Library updates waiting'), msg):
return False
from calibre.db.delete_service import has_jobs
if has_jobs():
msg = _('Some deleted books are still being moved to the Recycle '
'Bin, if you quit now, they will be left behind. Are you '
'sure you want to quit?')
if not question_dialog(self, _('Active jobs'), msg):
return False
return True
def shutdown(self, write_settings=True):
self.shutting_down = True
self.show_shutdown_message()
self.server_change_notification_timer.stop()
from calibre.customize.ui import has_library_closed_plugins
if has_library_closed_plugins():
self.show_shutdown_message(
_('Running database shutdown plugins. This could take a few seconds...'))
self.grid_view.shutdown()
db = None
try:
db = self.library_view.model().db
cf = db.clean
except:
pass
else:
cf()
# Save the current field_metadata for applications like calibre2opds
# Goes here, because if cf is valid, db is valid.
db.new_api.set_pref('field_metadata', db.field_metadata.all_metadata())
db.commit_dirty_cache()
db.prefs.write_serialized(prefs['library_path'])
for action in self.iactions.values():
if not action.shutting_down():
return
if write_settings:
self.write_settings()
self.check_messages_timer.stop()
if getattr(self, 'update_checker', None):
self.update_checker.shutdown()
self.listener.close()
self.job_manager.server.close()
self.job_manager.threaded_server.close()
self.device_manager.keep_going = False
self.auto_adder.stop()
# Do not report any errors that happen after the shutdown
# We cannot restore the original excepthook as that causes PyQt to
# call abort() on unhandled exceptions
import traceback
def eh(t, v, tb):
try:
traceback.print_exception(t, v, tb, file=sys.stderr)
except:
pass
sys.excepthook = eh
mb = self.library_view.model().metadata_backup
if mb is not None:
mb.stop()
self.library_view.model().close()
try:
try:
if self.content_server is not None:
# If the Content server has any sockets being closed then
# this can take quite a long time (minutes). Tell the user that it is
# happening.
self.show_shutdown_message(
_('Shutting down the Content server. This could take a while...'))
s = self.content_server
self.content_server = None
s.exit()
except:
pass
except KeyboardInterrupt:
pass
self.hide_windows()
if self._spare_pool is not None:
self._spare_pool.shutdown()
from calibre.db.delete_service import shutdown
shutdown()
time.sleep(2)
self.istores.join()
return True
def run_wizard(self, *args):
if self.confirm_quit():
self.run_wizard_b4_shutdown = True
self.restart_after_quit = True
try:
self.shutdown(write_settings=False)
except:
pass
QApplication.instance().quit()
def closeEvent(self, e):
if self.shutting_down:
return
self.write_settings()
if self.system_tray_icon is not None and self.system_tray_icon.isVisible():
if not dynamic['systray_msg'] and not isosx:
info_dialog(self, 'calibre', 'calibre '+
_('will keep running in the system tray. To close it, '
'choose <b>Quit</b> in the context menu of the '
'system tray.'), show_copy_button=False).exec_()
dynamic['systray_msg'] = True
self.hide_windows()
e.ignore()
else:
if self.confirm_quit():
try:
self.shutdown(write_settings=False)
except:
import traceback
traceback.print_exc()
e.accept()
else:
e.ignore()
class Server(Thread):
def __init__(self,
notify_on_job_done=lambda x: x,
pool_size=None,
limit=sys.maxsize,
enforce_cpu_limit=True):
Thread.__init__(self)
self.daemon = True
self.id = next(server_counter) + 1
if enforce_cpu_limit:
limit = min(limit, cpu_count())
self.pool_size = limit if pool_size is None else pool_size
self.notify_on_job_done = notify_on_job_done
self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue()
self.kill_queue = Queue()
self.waiting_jobs = []
self.workers = deque()
self.launched_worker_counter = count()
next(self.launched_worker_counter)
self.start()
def launch_worker(self, gui=False, redirect_output=None, job_name=None):
start = time.monotonic()
id = next(self.launched_worker_counter)
fd, rfile = tempfile.mkstemp(prefix='ipc_result_%d_%d_' %
(self.id, id),
dir=base_dir(),
suffix='.pickle')
os.close(fd)
if redirect_output is None:
redirect_output = not gui
cw = self.do_launch(gui, redirect_output, rfile, job_name=job_name)
if isinstance(cw, string_or_bytes):
raise CriticalError('Failed to launch worker process:\n' +
force_unicode(cw))
if DEBUG:
print(
f'Worker Launch took: {time.monotonic() - start:.2f} seconds')
return cw
def do_launch(self, gui, redirect_output, rfile, job_name=None):
a, b = Pipe()
with a:
env = {
'CALIBRE_WORKER_FD': str(a.fileno()),
'CALIBRE_WORKER_RESULT': environ_item(as_hex_unicode(rfile))
}
w = Worker(env, gui=gui, job_name=job_name)
try:
w(pass_fds=(a.fileno(), ), redirect_output=redirect_output)
except BaseException:
try:
w.kill()
except:
pass
b.close()
import traceback
return traceback.format_exc()
return ConnectedWorker(w, b, rfile)
def add_job(self, job):
job.done2 = self.notify_on_job_done
self.add_jobs_queue.put(job)
def run_job(self, job, gui=True, redirect_output=False):
w = self.launch_worker(gui=gui,
redirect_output=redirect_output,
job_name=getattr(job, 'name', None))
w.start_job(job)
def run(self):
while True:
try:
job = self.add_jobs_queue.get(True, 0.2)
if job is None:
break
self.waiting_jobs.insert(0, job)
except Empty:
pass
# Get notifications from worker process
for worker in self.workers:
while True:
try:
n = worker.notifications.get_nowait()
worker.job.notifications.put(n)
self.changed_jobs_queue.put(worker.job)
except Empty:
break
# Remove finished jobs
for worker in [w for w in self.workers if not w.is_alive]:
try:
worker.close_log_file()
except:
pass
self.workers.remove(worker)
job = worker.job
if worker.returncode != 0:
job.failed = True
job.returncode = worker.returncode
elif os.path.exists(worker.rfile):
try:
with lopen(worker.rfile, 'rb') as f:
job.result = pickle_loads(f.read())
os.remove(worker.rfile)
except:
pass
job.duration = time.time() - job.start_time
self.changed_jobs_queue.put(job)
# Start waiting jobs
sj = self.suitable_waiting_job()
if sj is not None:
job = self.waiting_jobs.pop(sj)
job.start_time = time.time()
if job.kill_on_start:
job.duration = 0.0
job.returncode = 1
job.killed = job.failed = True
job.result = None
else:
worker = self.launch_worker()
worker.start_job(job)
self.workers.append(worker)
job.log_path = worker.log_path
self.changed_jobs_queue.put(job)
while True:
try:
j = self.kill_queue.get_nowait()
self._kill_job(j)
except Empty:
break
def suitable_waiting_job(self):
available_workers = self.pool_size - len(self.workers)
for worker in self.workers:
job = worker.job
if job.core_usage == -1:
available_workers = 0
elif job.core_usage > 1:
available_workers -= job.core_usage - 1
if available_workers < 1:
return None
for i, job in enumerate(self.waiting_jobs):
if job.core_usage == -1:
if available_workers >= self.pool_size:
return i
elif job.core_usage <= available_workers:
return i
def kill_job(self, job):
self.kill_queue.put(job)
def killall(self):
for worker in self.workers:
self.kill_queue.put(worker.job)
def _kill_job(self, job):
if job.start_time is None:
job.kill_on_start = True
return
for worker in self.workers:
if job is worker.job:
worker.kill()
job.killed = True
break
def split(self, tasks):
'''
Split a list into a list of sub lists, with the number of sub lists being
no more than the number of workers this server supports. Each sublist contains
2-tuples of the form (i, x) where x is an element from the original list
and i is the index of the element x in the original list.
'''
ans, count, pos = [], 0, 0
delta = int(ceil(len(tasks) / float(self.pool_size)))
while count < len(tasks):
section = []
for t in tasks[pos:pos + delta]:
section.append((count, t))
count += 1
ans.append(section)
pos += delta
return ans
def close(self):
try:
self.add_jobs_queue.put(None)
except:
pass
try:
self.listener.close()
except:
pass
time.sleep(0.2)
for worker in list(self.workers):
try:
worker.kill()
except:
pass
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
class GenericDownloadThreadPool(object):
'''
add_task must be implemented in a subclass and must
GenericDownloadThreadPool.add_task must be called
at the end of the function.
'''
def __init__(self, thread_type, thread_count=1):
self.thread_type = thread_type
self.thread_count = thread_count
self.tasks = Queue()
self.results = Queue()
self.threads = []
def set_thread_count(self, thread_count):
self.thread_count = thread_count
def add_task(self):
'''
This must be implemented in a sub class and this function
must be called at the end of the add_task function in
the sub class.
The implementation of this function (in this base class)
starts any threads necessary to fill the pool if it is
not already full.
'''
for i in range(self.thread_count - self.running_threads_count()):
t = self.thread_type(self.tasks, self.results)
self.threads.append(t)
t.start()
def abort(self):
self.tasks = Queue()
self.results = Queue()
for t in self.threads:
t.abort()
self.threads = []
def has_tasks(self):
return not self.tasks.empty()
def get_result(self):
return self.results.get()
def get_result_no_wait(self):
return self.results.get_nowait()
def result_count(self):
return len(self.results)
def has_results(self):
return not self.results.empty()
def threads_running(self):
return self.running_threads_count() > 0
def running_threads_count(self):
count = 0
for t in self.threads:
if t.is_alive():
count += 1
return count
class GenericDownloadThreadPool(object):
'''
add_task must be implemented in a subclass and must
GenericDownloadThreadPool.add_task must be called
at the end of the function.
'''
def __init__(self, thread_type, thread_count=1):
self.thread_type = thread_type
self.thread_count = thread_count
self.tasks = Queue()
self.results = Queue()
self.threads = []
def set_thread_count(self, thread_count):
self.thread_count = thread_count
def add_task(self):
'''
This must be implemented in a sub class and this function
must be called at the end of the add_task function in
the sub class.
The implementation of this function (in this base class)
starts any threads necessary to fill the pool if it is
not already full.
'''
for i in range(self.thread_count - self.running_threads_count()):
t = self.thread_type(self.tasks, self.results)
self.threads.append(t)
t.start()
def abort(self):
self.tasks = Queue()
self.results = Queue()
for t in self.threads:
t.abort()
self.threads = []
def has_tasks(self):
return not self.tasks.empty()
def get_result(self):
return self.results.get()
def get_result_no_wait(self):
return self.results.get_nowait()
def result_count(self):
return len(self.results)
def has_results(self):
return not self.results.empty()
def threads_running(self):
return self.running_threads_count() > 0
def running_threads_count(self):
count = 0
for t in self.threads:
if t.is_alive():
count += 1
return count
class BaseJob(object):
WAITING = 0
RUNNING = 1
FINISHED = 2
def __init__(self, description, done=lambda x: x):
self.id = next(job_counter)
self.description = description
self.done = done
self.done2 = None
self.killed = False
self.failed = False
self.kill_on_start = False
self.start_time = None
self.result = None
self.duration = None
self.log_path = None
self.notifications = Queue()
self._run_state = self.WAITING
self.percent = 0
self._message = None
self._status_text = _('Waiting...')
self._done_called = False
self.core_usage = 1
self.timed_out = False
def update(self, consume_notifications=True):
if self.duration is not None:
self._run_state = self.FINISHED
self.percent = 100
if self.killed:
if self.timed_out:
self._status_text = _('Aborted, taking too long')
else:
self._status_text = _('Stopped')
else:
self._status_text = _('Error') if self.failed else _('Finished')
if DEBUG:
try:
prints('Job:', self.id, self.description, 'finished',
safe_encode=True)
prints('\t'.join(self.details.splitlines(True)),
safe_encode=True)
except:
pass
if not self._done_called:
self._done_called = True
try:
self.done(self)
except:
pass
try:
if callable(self.done2):
self.done2(self)
except:
pass
elif self.start_time is not None:
self._run_state = self.RUNNING
self._status_text = _('Working...')
if consume_notifications:
return self.consume_notifications()
return False
def consume_notifications(self):
got_notification = False
while self.notifications is not None:
try:
self.percent, self._message = self.notifications.get_nowait()
self.percent *= 100.
got_notification = True
except Empty:
break
return got_notification
@property
def status_text(self):
if self._run_state == self.FINISHED or not self._message:
return self._status_text
return self._message
@property
def run_state(self):
return self._run_state
@property
def running_time(self):
if self.duration is not None:
return self.duration
if self.start_time is not None:
return time.time() - self.start_time
return None
@property
def is_finished(self):
return self._run_state == self.FINISHED
@property
def is_started(self):
return self._run_state != self.WAITING
@property
def is_running(self):
return self.is_started and not self.is_finished
def __hash__(self):
return id(self)
def __eq__(self, other):
return self is other
def __ne__(self, other):
return self is not other
def __lt__(self, other):
return self.compare_to_other(other) < 0
def __le__(self, other):
return self.compare_to_other(other) <= 0
def __gt__(self, other):
return self.compare_to_other(other) > 0
def __ge__(self, other):
return self.compare_to_other(other) >= 0
def compare_to_other(self, other):
if self.is_finished != other.is_finished:
return 1 if self.is_finished else -1
if self.start_time is None:
if other.start_time is None: # Both waiting
return cmp(other.id, self.id)
return 1
if other.start_time is None:
return -1
# Both running
return cmp((other.start_time, id(other)), (self.start_time, id(self)))
@property
def log_file(self):
if self.log_path:
return open(self.log_path, 'rb')
return io.BytesIO(_('No details available.').encode('utf-8',
'replace'))
@property
def details(self):
return self.log_file.read().decode('utf-8', 'replace')
