def create_locks(): ''' Return a pair of locks: (read_lock, write_lock) The read_lock can be acquired by multiple threads simultaneously, it can also be acquired multiple times by the same thread. Only one thread can hold write_lock at a time, and only if there are no current read_locks. While the write_lock is held no other threads can acquire read locks. The write_lock can also be acquired multiple times by the same thread. Both read_lock and write_lock are meant to be used in with statements (they operate on a single underlying lock. WARNING: Be very careful to not try to acquire a read lock while the same thread holds a write lock and vice versa. That is, a given thread should always release *all* locks of type A before trying to acquire a lock of type B. Bad things will happen if you violate this rule, the most benign of which is the raising of a LockingError (I haven't been able to eliminate the possibility of deadlocking in this scenario). ''' l = SHLock() wrapper = DebugRWLockWrapper if tweaks.get('newdb_debug_locking', False) else RWLockWrapper return wrapper(l), wrapper(l, is_shared=False)
def __init__(self, parent, library_path): QDialog.__init__(self, parent) self.l = QVBoxLayout() self.setLayout(self.l) self.l1 = QLabel('<b>'+_('Restoring database from backups, do not' ' interrupt, this will happen in three stages')+'...') self.setWindowTitle(_('Restoring database')) self.l.addWidget(self.l1) self.pb = QProgressBar(self) self.l.addWidget(self.pb) self.pb.setMaximum(0) self.pb.setMinimum(0) self.msg = QLabel('') self.l.addWidget(self.msg) self.msg.setWordWrap(True) self.bb = QDialogButtonBox(QDialogButtonBox.Cancel) self.l.addWidget(self.bb) self.bb.rejected.connect(self.reject) self.resize(self.sizeHint() + QSize(100, 50)) self.error = None self.rejected = False self.library_path = library_path self.update_signal.connect(self.do_update, type=Qt.QueuedConnection) if tweaks.get('use_new_db', False): from calibre.db.restore import Restore else: from calibre.library.restore import Restore self.restorer = Restore(library_path, self) self.restorer.daemon = True # Give the metadata backup thread time to stop QTimer.singleShot(2000, self.start)
def locate_program(self): program_path = tweaks.get("kfx_output_previewer_path") if program_path: return program_path if IS_WINDOWS: program_path = os.path.join(windows_user_dir(local_appdata=True), "Amazon", "Kindle Previewer 3") if not os.path.isdir(program_path): try: try: import winreg except ImportError: import _winreg as winreg key_handle = winreg.OpenKey( winreg.HKEY_CURRENT_USER, "Software\\Amazon\\Kindle Previewer 3") value, vtype = winreg.QueryValueEx(key_handle, None) if vtype != winreg.REG_SZ: raise Exception("Registry value is wrong type: %d" % vtype) program_path = value except Exception: self.log.warning( "Failed to obtain the Kindle Previewer path from the registry" ) return program_path if IS_MACOS: return "/Applications/Kindle Previewer 3.app/Contents/MacOS" raise Exception("Kindle Previewer 3 is not supported under Linux")
def get_db_loader(): from calibre.utils.config_base import tweaks if tweaks.get('use_new_db', False): from calibre.db.legacy import LibraryDatabase as cls import apsw errs = (apsw.Error,) else: from calibre.library.database2 import LibraryDatabase2 as cls from calibre.library.sqlite import sqlite, DatabaseException errs = (sqlite.Error, DatabaseException) return cls, errs
def locate_program(self): program_path = tweaks.get("kfx_output_previewer_path") if program_path: return program_path if IS_WINDOWS: program_path = os.path.join(windows_user_dir(local_appdata=True), "Amazon", "Kindle Previewer 3") if not os.path.isdir(program_path): try: try: import winreg except ImportError: import _winreg as winreg key_handle = winreg.OpenKey( winreg.HKEY_CURRENT_USER, "Software\\Amazon\\Kindle Previewer 3") value, vtype = winreg.QueryValueEx(key_handle, None) if vtype != winreg.REG_SZ: raise Exception("Registry value is wrong type: %d" % vtype) program_path = value except Exception: log.warning( "Failed to obtain the Kindle Previewer path from the registry" ) return program_path if IS_MACOS: return "/Applications/Kindle Previewer 3.app/Contents/MacOS" if IS_LINUX: userreg = os.path.join(wineprefix(), "user.reg") if not os.path.isfile(userreg): raise Exception( "Wine registry file %s not found. Ensure that Wine is correctly installed." % userreg) with io.open(userreg, "r") as file: for line in file: if line.startswith( "[Software\\\\Amazon\\\\Kindle Previewer 3]"): for line in file: match = re.search("@=\"([^\"]*)\"", line) if match: return winepath(match.group(1)) if line.startswith("["): break raise Exception("Kindle Previewer 3 not found in %s." % userreg)
class Tag(object): if tweaks.get('use_new_db', False): __slots__ = ('name', 'original_name', 'id', 'count', 'state', 'is_hierarchical', 'is_editable', 'is_searchable', 'id_set', 'avg_rating', 'sort', 'use_sort_as_name', 'tooltip', 'icon', 'category') def __init__(self, name, id=None, count=0, state=0, avg=0, sort=None, tooltip=None, icon=None, category=None, id_set=None, is_editable=True, is_searchable=True, use_sort_as_name=False): self.name = self.original_name = name self.id = id self.count = count self.state = state self.is_hierarchical = '' self.is_editable = is_editable self.is_searchable = is_searchable self.id_set = id_set if id_set is not None else set([]) self.avg_rating = avg / 2.0 if avg is not None else 0 self.sort = sort self.use_sort_as_name = use_sort_as_name if tooltip is None: tooltip = '(%s:%s)' % (category, name) if self.avg_rating > 0: if tooltip: tooltip = tooltip + ': ' tooltip = _('%(tt)sAverage rating is %(rating)3.1f') % dict( tt=tooltip, rating=self.avg_rating) self.tooltip = tooltip self.icon = icon self.category = category def __unicode__(self): return u'%s:%s:%s:%s:%s:%s' % (self.name, self.count, self.id, self.state, self.category, self.tooltip) def __str__(self): return unicode(self).encode('utf-8') def __repr__(self): return str(self)
def reinit_db(dbpath, callback=None, sql_dump=None): if not os.path.exists(dbpath): raise ValueError(dbpath + ' does not exist') from calibre.utils.config_base import tweaks if tweaks.get('use_new_db', False): return reinit_db_new(dbpath, callback, sql_dump) from calibre.library.sqlite import connect from contextlib import closing import shutil conn = connect(dbpath, False) uv = conn.get('PRAGMA user_version;', all=False) conn.execute('PRAGMA writable_schema=ON') conn.commit() if sql_dump is None: sql_lines = conn.dump() else: sql_lines = open(sql_dump, 'rb').read() conn.close() dest = dbpath + '.tmp' try: with closing(connect(dest, False)) as nconn: nconn.execute( 'create temporary table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)' ) nconn.commit() if sql_dump is None: if callable(callback): callback(len(sql_lines), True) for i, line in enumerate(sql_lines): try: nconn.execute(line) except: import traceback prints('SQL line %r failed with error:' % line) prints(traceback.format_exc()) continue finally: if callable(callback): callback(i, False) else: nconn.executescript(sql_lines) nconn.execute('pragma user_version=%d' % int(uv)) nconn.commit() os.remove(dbpath) shutil.copyfile(dest, dbpath) finally: if os.path.exists(dest): os.remove(dest) prints('Database successfully re-initialized')
def reinit_db(dbpath, callback=None, sql_dump=None): if not os.path.exists(dbpath): raise ValueError(dbpath + ' does not exist') from calibre.utils.config_base import tweaks if tweaks.get('use_new_db', False): return reinit_db_new(dbpath, callback, sql_dump) from calibre.library.sqlite import connect from contextlib import closing import shutil conn = connect(dbpath, False) uv = conn.get('PRAGMA user_version;', all=False) conn.execute('PRAGMA writable_schema=ON') conn.commit() if sql_dump is None: sql_lines = conn.dump() else: sql_lines = open(sql_dump, 'rb').read() conn.close() dest = dbpath + '.tmp' try: with closing(connect(dest, False)) as nconn: nconn.execute('create temporary table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)') nconn.commit() if sql_dump is None: if callable(callback): callback(len(sql_lines), True) for i, line in enumerate(sql_lines): try: nconn.execute(line) except: import traceback prints('SQL line %r failed with error:'%line) prints(traceback.format_exc()) continue finally: if callable(callback): callback(i, False) else: nconn.executescript(sql_lines) nconn.execute('pragma user_version=%d'%int(uv)) nconn.commit() os.remove(dbpath) shutil.copyfile(dest, dbpath) finally: if os.path.exists(dest): os.remove(dest) prints('Database successfully re-initialized')
def do_tweak(self, book_id): from calibre.utils.config_base import tweaks if not tweaks.get('test_tweak_book', False): return self.gui.iactions['Unpack Book'].do_tweak(book_id) from calibre.ebooks.oeb.polish.main import SUPPORTED db = self.gui.library_view.model().db fmts = db.formats(book_id, index_is_id=True) or '' fmts = [x.upper().strip() for x in fmts.split(',')] tweakable_fmts = set(fmts).intersection(SUPPORTED) if not tweakable_fmts: return error_dialog(self.gui, _('Cannot Tweak Book'), _('The book must be in the %s formats to tweak.' '\n\nFirst convert the book to one of these formats.') % (_(' or '.join(SUPPORTED))), show=True) from calibre.gui2.tweak_book import tprefs if len(tweakable_fmts) > 1: if tprefs['choose_tweak_fmt']: d = Choose(sorted(tweakable_fmts, key=tprefs.defaults['tweak_fmt_order'].index), self.gui) if d.exec_() != d.Accepted: return tweakable_fmts = {d.fmt} else: fmts = [f for f in tprefs['tweak_fmt_order'] if f in tweakable_fmts] if not fmts: fmts = [f for f in tprefs.defaults['tweak_fmt_order'] if f in tweakable_fmts] tweakable_fmts = {fmts[0]} fmt = tuple(tweakable_fmts)[0] path = db.new_api.format_abspath(book_id, fmt) if path is None: return error_dialog(self.gui, _('File missing'), _( 'The %s format is missing from the calibre library. You should run' ' library maintenance.') % fmt, show=True) tweak = 'ebook-tweak' self.gui.setCursor(Qt.BusyCursor) if tprefs['update_metadata_from_calibre']: from calibre.ebooks.metadata.opf2 import pretty_print from calibre.ebooks.metadata.meta import set_metadata mi = db.new_api.get_metadata(book_id) with pretty_print, open(path, 'r+b') as f: set_metadata(f, mi, stream_type=fmt.lower()) try: self.gui.job_manager.launch_gui_app(tweak, kwargs=dict(args=[tweak, path])) time.sleep(2) finally: self.gui.unsetCursor()
def set_metadata(self, stream, mi, type_): from calibre_plugins.kfx_output.kfxlib import (set_logger, YJ_Book, YJ_Metadata) from calibre.ebooks import normalize as normalize_unicode from calibre.ebooks.metadata import author_to_author_sort from calibre.utils.config_base import tweaks from calibre.utils.date import (is_date_undefined, isoformat) from calibre.utils.logging import Log from calibre.utils.localization import (canonicalize_lang, lang_as_iso639_1) def mapped_author_to_author_sort(author): if hasattr(mi, "author_sort_map"): author_sort = mi.author_sort_map.get(author) # use mapping if provided if author_sort: return author_sort return author_to_author_sort(author) def normalize(s): if not isinstance(s, type("")): s = s.decode("utf8", "ignore") return normalize_unicode(s) log = set_logger(Log()) filename = stream.name if hasattr(stream, "name") else "stream" log.info("KFX metadata writer activated for %s" % filename) try: from calibre.ebooks.conversion.config import load_defaults prefs = load_defaults('kfx_output') except Exception: prefs = {} log.info("Failed to read default KFX Output preferences") md = YJ_Metadata(author_sort_fn=mapped_author_to_author_sort) md.title = normalize(mi.title) md.authors = [normalize(author) for author in mi.authors] if mi.publisher: md.publisher = normalize(mi.publisher) if mi.pubdate and not is_date_undefined(mi.pubdate): md.issue_date = str(isoformat(mi.pubdate)[:10]) if mi.comments: # Strip user annotations a_offset = mi.comments.find('<div class="user_annotations">') ad_offset = mi.comments.find('<hr class="annotations_divider" />') if a_offset >= 0: mi.comments = mi.comments[:a_offset] if ad_offset >= 0: mi.comments = mi.comments[:ad_offset] md.description = normalize(mi.comments) if not mi.is_null('language'): lang = canonicalize_lang(mi.language) lang = lang_as_iso639_1(lang) or lang if lang: md.language = normalize(lang) if mi.cover_data[1]: md.cover_image_data = mi.cover_data elif mi.cover: md.cover_image_data = ("jpg", open(mi.cover, 'rb').read()) if not tweaks.get("kfx_output_ignore_asin_metadata", False): value = mi.identifiers.get("mobi-asin") if value is not None and re.match(ASIN_RE, value): md.asin = value else: for ident, value in mi.identifiers.items(): if ident.startswith("amazon") and re.match(ASIN_RE, value): md.asin = value break else: value = mi.identifiers.get("asin") if value is not None and re.match(ASIN_RE, value): md.asin = value if md.asin: md.cde_content_type = "EBOK" if prefs.get("approximate_pages", False): page_count = 0 number_of_pages_field = prefs.get("number_of_pages_field", AUTO_PAGES) if number_of_pages_field and number_of_pages_field != AUTO_PAGES: number_of_pages = mi.get(number_of_pages_field, "") try: page_count = int(number_of_pages) except Exception: pass else: page_count = -1 book = YJ_Book(stream, log) book.decode_book(set_metadata=md, set_approximate_pages=page_count) new_data = book.convert_to_single_kfx() set_logger() stream.seek(0) stream.truncate() stream.write(new_data) stream.seek(0)
def convert(self, oeb_book, output, input_plugin, opts, log): self.report_version(log) #for mivals in oeb_book.metadata.items.values(): # for mival in mivals: # log.info("metadata: %s" % repr(mival)) try: book_name = str(oeb_book.metadata.title[0]) except Exception: book_name = "" asin = None if not tweaks.get("kfx_output_ignore_asin_metadata", False): for idre in ["^mobi-asin$", "^amazon.*$", "^asin$"]: for ident in oeb_book.metadata["identifier"]: idtype = ident.get(OPFNS("scheme"), "").lower() if re.match(idre, idtype) and re.match(ASIN_RE, ident.value): asin = ident.value log.info("Found ASIN metadata %s: %s" % (idtype, asin)) break if asin: break #with open(opts.read_metadata_from_opf, "rb") as opff: # log.info("opf: %s" % opff.read()) if opts.approximate_pages: page_count = 0 if opts.number_of_pages_field and opts.number_of_pages_field != AUTO_PAGES and opts.read_metadata_from_opf: # This OPF contains custom column metadata not present in the oeb_book OPF opf = OPF(opts.read_metadata_from_opf, populate_spine=False, try_to_guess_cover=False, read_toc=False) mi = opf.to_book_metadata() page_count_str = mi.get(opts.number_of_pages_field, None) if page_count_str is not None: try: page_count = int(page_count_str) except Exception: pass log.info("Page count value from field %s: %d ('%s')" % (opts.number_of_pages_field, page_count, page_count_str)) else: log.warning("Book has no page count field %s" % opts.number_of_pages_field) else: page_count = -1 #log.info("oeb_book contains %d pages" % len(oeb_book.pages.pages)) #log.info("options: %s" % str(opts.__dict__)) # set default values for options expected by the EPUB Output plugin for optrec in EPUBOutput.options: setattr(opts, optrec.option.name, optrec.recommended_value) # override currently known EPUB Output plugin options opts.extract_to = None opts.dont_split_on_page_breaks = False opts.flow_size = 0 opts.no_default_epub_cover = False opts.no_svg_cover = False opts.preserve_cover_aspect_ratio = True opts.epub_flatten = False opts.epub_inline_toc = False opts.epub_toc_at_end = False opts.toc_title = None epub_filename = self.temporary_file(".epub").name self.epub_output_plugin.convert(oeb_book, epub_filename, input_plugin, opts, log) # convert input format to EPUB log.info("Successfully converted input format to EPUB") if PREPARED_FILE_SAVE_DIR: if not os.path.exists(PREPARED_FILE_SAVE_DIR): os.makedirs(PREPARED_FILE_SAVE_DIR) prepared_file_path = os.path.join(PREPARED_FILE_SAVE_DIR, os.path.basename(epub_filename)) shutil.copyfile(epub_filename, prepared_file_path) log.warning("Saved conversion input file: %s" % prepared_file_path) self.convert_using_previewer( JobLog(log), book_name, epub_filename, asin, opts.cde_type_pdoc, page_count, opts.show_kpr_logs, False, TIMEOUT if opts.enable_timeout else None, output)