def create_locks():
'''
Return a pair of locks: (read_lock, write_lock)
The read_lock can be acquired by multiple threads simultaneously, it can
also be acquired multiple times by the same thread.
Only one thread can hold write_lock at a time, and only if there are no
current read_locks. While the write_lock is held no
other threads can acquire read locks. The write_lock can also be acquired
multiple times by the same thread.
Both read_lock and write_lock are meant to be used in with statements (they
operate on a single underlying lock.
WARNING: Be very careful to not try to acquire a read lock while the same
thread holds a write lock and vice versa. That is, a given thread should
always release *all* locks of type A before trying to acquire a lock of type
B. Bad things will happen if you violate this rule, the most benign of
which is the raising of a LockingError (I haven't been able to eliminate
the possibility of deadlocking in this scenario).
'''
l = SHLock()
wrapper = DebugRWLockWrapper if tweaks.get('newdb_debug_locking',
False) else RWLockWrapper
return wrapper(l), wrapper(l, is_shared=False)
def __init__(self, parent, library_path):
QDialog.__init__(self, parent)
self.l = QVBoxLayout()
self.setLayout(self.l)
self.l1 = QLabel('<b>'+_('Restoring database from backups, do not'
' interrupt, this will happen in three stages')+'...')
self.setWindowTitle(_('Restoring database'))
self.l.addWidget(self.l1)
self.pb = QProgressBar(self)
self.l.addWidget(self.pb)
self.pb.setMaximum(0)
self.pb.setMinimum(0)
self.msg = QLabel('')
self.l.addWidget(self.msg)
self.msg.setWordWrap(True)
self.bb = QDialogButtonBox(QDialogButtonBox.Cancel)
self.l.addWidget(self.bb)
self.bb.rejected.connect(self.reject)
self.resize(self.sizeHint() + QSize(100, 50))
self.error = None
self.rejected = False
self.library_path = library_path
self.update_signal.connect(self.do_update, type=Qt.QueuedConnection)
if tweaks.get('use_new_db', False):
from calibre.db.restore import Restore
else:
from calibre.library.restore import Restore
self.restorer = Restore(library_path, self)
self.restorer.daemon = True
# Give the metadata backup thread time to stop
QTimer.singleShot(2000, self.start)
def create_locks():
'''
Return a pair of locks: (read_lock, write_lock)
The read_lock can be acquired by multiple threads simultaneously, it can
also be acquired multiple times by the same thread.
Only one thread can hold write_lock at a time, and only if there are no
current read_locks. While the write_lock is held no
other threads can acquire read locks. The write_lock can also be acquired
multiple times by the same thread.
Both read_lock and write_lock are meant to be used in with statements (they
operate on a single underlying lock.
WARNING: Be very careful to not try to acquire a read lock while the same
thread holds a write lock and vice versa. That is, a given thread should
always release *all* locks of type A before trying to acquire a lock of type
B. Bad things will happen if you violate this rule, the most benign of
which is the raising of a LockingError (I haven't been able to eliminate
the possibility of deadlocking in this scenario).
'''
l = SHLock()
wrapper = DebugRWLockWrapper if tweaks.get('newdb_debug_locking', False) else RWLockWrapper
return wrapper(l), wrapper(l, is_shared=False)
def locate_program(self):
program_path = tweaks.get("kfx_output_previewer_path")
if program_path:
return program_path
if IS_WINDOWS:
program_path = os.path.join(windows_user_dir(local_appdata=True),
"Amazon", "Kindle Previewer 3")
if not os.path.isdir(program_path):
try:
try:
import winreg
except ImportError:
import _winreg as winreg
key_handle = winreg.OpenKey(
winreg.HKEY_CURRENT_USER,
"Software\\Amazon\\Kindle Previewer 3")
value, vtype = winreg.QueryValueEx(key_handle, None)
if vtype != winreg.REG_SZ:
raise Exception("Registry value is wrong type: %d" %
vtype)
program_path = value
except Exception:
self.log.warning(
"Failed to obtain the Kindle Previewer path from the registry"
)
return program_path
if IS_MACOS:
return "/Applications/Kindle Previewer 3.app/Contents/MacOS"
raise Exception("Kindle Previewer 3 is not supported under Linux")
def get_db_loader():
from calibre.utils.config_base import tweaks
if tweaks.get('use_new_db', False):
from calibre.db.legacy import LibraryDatabase as cls
import apsw
errs = (apsw.Error,)
else:
from calibre.library.database2 import LibraryDatabase2 as cls
from calibre.library.sqlite import sqlite, DatabaseException
errs = (sqlite.Error, DatabaseException)
return cls, errs
def locate_program(self):
program_path = tweaks.get("kfx_output_previewer_path")
if program_path:
return program_path
if IS_WINDOWS:
program_path = os.path.join(windows_user_dir(local_appdata=True),
"Amazon", "Kindle Previewer 3")
if not os.path.isdir(program_path):
try:
try:
import winreg
except ImportError:
import _winreg as winreg
key_handle = winreg.OpenKey(
winreg.HKEY_CURRENT_USER,
"Software\\Amazon\\Kindle Previewer 3")
value, vtype = winreg.QueryValueEx(key_handle, None)
if vtype != winreg.REG_SZ:
raise Exception("Registry value is wrong type: %d" %
vtype)
program_path = value
except Exception:
log.warning(
"Failed to obtain the Kindle Previewer path from the registry"
)
return program_path
if IS_MACOS:
return "/Applications/Kindle Previewer 3.app/Contents/MacOS"
if IS_LINUX:
userreg = os.path.join(wineprefix(), "user.reg")
if not os.path.isfile(userreg):
raise Exception(
"Wine registry file %s not found. Ensure that Wine is correctly installed."
% userreg)
with io.open(userreg, "r") as file:
for line in file:
if line.startswith(
"[Software\\\\Amazon\\\\Kindle Previewer 3]"):
for line in file:
match = re.search("@=\"([^\"]*)\"", line)
if match:
return winepath(match.group(1))
if line.startswith("["):
break
raise Exception("Kindle Previewer 3 not found in %s." % userreg)
class Tag(object):
if tweaks.get('use_new_db', False):
__slots__ = ('name', 'original_name', 'id', 'count', 'state',
'is_hierarchical', 'is_editable', 'is_searchable',
'id_set', 'avg_rating', 'sort', 'use_sort_as_name',
'tooltip', 'icon', 'category')
def __init__(self,
name,
id=None,
count=0,
state=0,
avg=0,
sort=None,
tooltip=None,
icon=None,
category=None,
id_set=None,
is_editable=True,
is_searchable=True,
use_sort_as_name=False):
self.name = self.original_name = name
self.id = id
self.count = count
self.state = state
self.is_hierarchical = ''
self.is_editable = is_editable
self.is_searchable = is_searchable
self.id_set = id_set if id_set is not None else set([])
self.avg_rating = avg / 2.0 if avg is not None else 0
self.sort = sort
self.use_sort_as_name = use_sort_as_name
if tooltip is None:
tooltip = '(%s:%s)' % (category, name)
if self.avg_rating > 0:
if tooltip:
tooltip = tooltip + ': '
tooltip = _('%(tt)sAverage rating is %(rating)3.1f') % dict(
tt=tooltip, rating=self.avg_rating)
self.tooltip = tooltip
self.icon = icon
self.category = category
def __unicode__(self):
return u'%s:%s:%s:%s:%s:%s' % (self.name, self.count, self.id,
self.state, self.category, self.tooltip)
def __str__(self):
return unicode(self).encode('utf-8')
def __repr__(self):
return str(self)
def reinit_db(dbpath, callback=None, sql_dump=None):
if not os.path.exists(dbpath):
raise ValueError(dbpath + ' does not exist')
from calibre.utils.config_base import tweaks
if tweaks.get('use_new_db', False):
return reinit_db_new(dbpath, callback, sql_dump)
from calibre.library.sqlite import connect
from contextlib import closing
import shutil
conn = connect(dbpath, False)
uv = conn.get('PRAGMA user_version;', all=False)
conn.execute('PRAGMA writable_schema=ON')
conn.commit()
if sql_dump is None:
sql_lines = conn.dump()
else:
sql_lines = open(sql_dump, 'rb').read()
conn.close()
dest = dbpath + '.tmp'
try:
with closing(connect(dest, False)) as nconn:
nconn.execute(
'create temporary table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)'
)
nconn.commit()
if sql_dump is None:
if callable(callback):
callback(len(sql_lines), True)
for i, line in enumerate(sql_lines):
try:
nconn.execute(line)
except:
import traceback
prints('SQL line %r failed with error:' % line)
prints(traceback.format_exc())
continue
finally:
if callable(callback):
callback(i, False)
else:
nconn.executescript(sql_lines)
nconn.execute('pragma user_version=%d' % int(uv))
nconn.commit()
os.remove(dbpath)
shutil.copyfile(dest, dbpath)
finally:
if os.path.exists(dest):
os.remove(dest)
prints('Database successfully re-initialized')
def reinit_db(dbpath, callback=None, sql_dump=None):
if not os.path.exists(dbpath):
raise ValueError(dbpath + ' does not exist')
from calibre.utils.config_base import tweaks
if tweaks.get('use_new_db', False):
return reinit_db_new(dbpath, callback, sql_dump)
from calibre.library.sqlite import connect
from contextlib import closing
import shutil
conn = connect(dbpath, False)
uv = conn.get('PRAGMA user_version;', all=False)
conn.execute('PRAGMA writable_schema=ON')
conn.commit()
if sql_dump is None:
sql_lines = conn.dump()
else:
sql_lines = open(sql_dump, 'rb').read()
conn.close()
dest = dbpath + '.tmp'
try:
with closing(connect(dest, False)) as nconn:
nconn.execute('create temporary table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)')
nconn.commit()
if sql_dump is None:
if callable(callback):
callback(len(sql_lines), True)
for i, line in enumerate(sql_lines):
try:
nconn.execute(line)
except:
import traceback
prints('SQL line %r failed with error:'%line)
prints(traceback.format_exc())
continue
finally:
if callable(callback):
callback(i, False)
else:
nconn.executescript(sql_lines)
nconn.execute('pragma user_version=%d'%int(uv))
nconn.commit()
os.remove(dbpath)
shutil.copyfile(dest, dbpath)
finally:
if os.path.exists(dest):
os.remove(dest)
prints('Database successfully re-initialized')
def do_tweak(self, book_id):
from calibre.utils.config_base import tweaks
if not tweaks.get('test_tweak_book', False):
return self.gui.iactions['Unpack Book'].do_tweak(book_id)
from calibre.ebooks.oeb.polish.main import SUPPORTED
db = self.gui.library_view.model().db
fmts = db.formats(book_id, index_is_id=True) or ''
fmts = [x.upper().strip() for x in fmts.split(',')]
tweakable_fmts = set(fmts).intersection(SUPPORTED)
if not tweakable_fmts:
return error_dialog(self.gui, _('Cannot Tweak Book'),
_('The book must be in the %s formats to tweak.'
'\n\nFirst convert the book to one of these formats.') % (_(' or '.join(SUPPORTED))),
show=True)
from calibre.gui2.tweak_book import tprefs
if len(tweakable_fmts) > 1:
if tprefs['choose_tweak_fmt']:
d = Choose(sorted(tweakable_fmts, key=tprefs.defaults['tweak_fmt_order'].index), self.gui)
if d.exec_() != d.Accepted:
return
tweakable_fmts = {d.fmt}
else:
fmts = [f for f in tprefs['tweak_fmt_order'] if f in tweakable_fmts]
if not fmts:
fmts = [f for f in tprefs.defaults['tweak_fmt_order'] if f in tweakable_fmts]
tweakable_fmts = {fmts[0]}
fmt = tuple(tweakable_fmts)[0]
path = db.new_api.format_abspath(book_id, fmt)
if path is None:
return error_dialog(self.gui, _('File missing'), _(
'The %s format is missing from the calibre library. You should run'
' library maintenance.') % fmt, show=True)
tweak = 'ebook-tweak'
self.gui.setCursor(Qt.BusyCursor)
if tprefs['update_metadata_from_calibre']:
from calibre.ebooks.metadata.opf2 import pretty_print
from calibre.ebooks.metadata.meta import set_metadata
mi = db.new_api.get_metadata(book_id)
with pretty_print, open(path, 'r+b') as f:
set_metadata(f, mi, stream_type=fmt.lower())
try:
self.gui.job_manager.launch_gui_app(tweak, kwargs=dict(args=[tweak, path]))
time.sleep(2)
finally:
self.gui.unsetCursor()
def set_metadata(self, stream, mi, type_):
from calibre_plugins.kfx_output.kfxlib import (set_logger, YJ_Book, YJ_Metadata)
from calibre.ebooks import normalize as normalize_unicode
from calibre.ebooks.metadata import author_to_author_sort
from calibre.utils.config_base import tweaks
from calibre.utils.date import (is_date_undefined, isoformat)
from calibre.utils.logging import Log
from calibre.utils.localization import (canonicalize_lang, lang_as_iso639_1)
def mapped_author_to_author_sort(author):
if hasattr(mi, "author_sort_map"):
author_sort = mi.author_sort_map.get(author) # use mapping if provided
if author_sort:
return author_sort
return author_to_author_sort(author)
def normalize(s):
if not isinstance(s, type("")):
s = s.decode("utf8", "ignore")
return normalize_unicode(s)
log = set_logger(Log())
filename = stream.name if hasattr(stream, "name") else "stream"
log.info("KFX metadata writer activated for %s" % filename)
try:
from calibre.ebooks.conversion.config import load_defaults
prefs = load_defaults('kfx_output')
except Exception:
prefs = {}
log.info("Failed to read default KFX Output preferences")
md = YJ_Metadata(author_sort_fn=mapped_author_to_author_sort)
md.title = normalize(mi.title)
md.authors = [normalize(author) for author in mi.authors]
if mi.publisher:
md.publisher = normalize(mi.publisher)
if mi.pubdate and not is_date_undefined(mi.pubdate):
md.issue_date = str(isoformat(mi.pubdate)[:10])
if mi.comments:
# Strip user annotations
a_offset = mi.comments.find('<div class="user_annotations">')
ad_offset = mi.comments.find('<hr class="annotations_divider" />')
if a_offset >= 0:
mi.comments = mi.comments[:a_offset]
if ad_offset >= 0:
mi.comments = mi.comments[:ad_offset]
md.description = normalize(mi.comments)
if not mi.is_null('language'):
lang = canonicalize_lang(mi.language)
lang = lang_as_iso639_1(lang) or lang
if lang:
md.language = normalize(lang)
if mi.cover_data[1]:
md.cover_image_data = mi.cover_data
elif mi.cover:
md.cover_image_data = ("jpg", open(mi.cover, 'rb').read())
if not tweaks.get("kfx_output_ignore_asin_metadata", False):
value = mi.identifiers.get("mobi-asin")
if value is not None and re.match(ASIN_RE, value):
md.asin = value
else:
for ident, value in mi.identifiers.items():
if ident.startswith("amazon") and re.match(ASIN_RE, value):
md.asin = value
break
else:
value = mi.identifiers.get("asin")
if value is not None and re.match(ASIN_RE, value):
md.asin = value
if md.asin:
md.cde_content_type = "EBOK"
if prefs.get("approximate_pages", False):
page_count = 0
number_of_pages_field = prefs.get("number_of_pages_field", AUTO_PAGES)
if number_of_pages_field and number_of_pages_field != AUTO_PAGES:
number_of_pages = mi.get(number_of_pages_field, "")
try:
page_count = int(number_of_pages)
except Exception:
pass
else:
page_count = -1
book = YJ_Book(stream, log)
book.decode_book(set_metadata=md, set_approximate_pages=page_count)
new_data = book.convert_to_single_kfx()
set_logger()
stream.seek(0)
stream.truncate()
stream.write(new_data)
stream.seek(0)
def convert(self, oeb_book, output, input_plugin, opts, log):
self.report_version(log)
#for mivals in oeb_book.metadata.items.values():
# for mival in mivals:
# log.info("metadata: %s" % repr(mival))
try:
book_name = str(oeb_book.metadata.title[0])
except Exception:
book_name = ""
asin = None
if not tweaks.get("kfx_output_ignore_asin_metadata", False):
for idre in ["^mobi-asin$", "^amazon.*$", "^asin$"]:
for ident in oeb_book.metadata["identifier"]:
idtype = ident.get(OPFNS("scheme"), "").lower()
if re.match(idre, idtype) and re.match(ASIN_RE, ident.value):
asin = ident.value
log.info("Found ASIN metadata %s: %s" % (idtype, asin))
break
if asin:
break
#with open(opts.read_metadata_from_opf, "rb") as opff:
# log.info("opf: %s" % opff.read())
if opts.approximate_pages:
page_count = 0
if opts.number_of_pages_field and opts.number_of_pages_field != AUTO_PAGES and opts.read_metadata_from_opf:
# This OPF contains custom column metadata not present in the oeb_book OPF
opf = OPF(opts.read_metadata_from_opf, populate_spine=False, try_to_guess_cover=False, read_toc=False)
mi = opf.to_book_metadata()
page_count_str = mi.get(opts.number_of_pages_field, None)
if page_count_str is not None:
try:
page_count = int(page_count_str)
except Exception:
pass
log.info("Page count value from field %s: %d ('%s')" % (opts.number_of_pages_field, page_count, page_count_str))
else:
log.warning("Book has no page count field %s" % opts.number_of_pages_field)
else:
page_count = -1
#log.info("oeb_book contains %d pages" % len(oeb_book.pages.pages))
#log.info("options: %s" % str(opts.__dict__))
# set default values for options expected by the EPUB Output plugin
for optrec in EPUBOutput.options:
setattr(opts, optrec.option.name, optrec.recommended_value)
# override currently known EPUB Output plugin options
opts.extract_to = None
opts.dont_split_on_page_breaks = False
opts.flow_size = 0
opts.no_default_epub_cover = False
opts.no_svg_cover = False
opts.preserve_cover_aspect_ratio = True
opts.epub_flatten = False
opts.epub_inline_toc = False
opts.epub_toc_at_end = False
opts.toc_title = None
epub_filename = self.temporary_file(".epub").name
self.epub_output_plugin.convert(oeb_book, epub_filename, input_plugin, opts, log) # convert input format to EPUB
log.info("Successfully converted input format to EPUB")
if PREPARED_FILE_SAVE_DIR:
if not os.path.exists(PREPARED_FILE_SAVE_DIR):
os.makedirs(PREPARED_FILE_SAVE_DIR)
prepared_file_path = os.path.join(PREPARED_FILE_SAVE_DIR, os.path.basename(epub_filename))
shutil.copyfile(epub_filename, prepared_file_path)
log.warning("Saved conversion input file: %s" % prepared_file_path)
self.convert_using_previewer(
JobLog(log), book_name, epub_filename, asin, opts.cde_type_pdoc, page_count,
opts.show_kpr_logs, False, TIMEOUT if opts.enable_timeout else None, output)
