def delete_tags(self, item=None):
confirms, deletes = [], []
items = self.available_tags.selectedItems() if item is None else [item]
if not items:
error_dialog(self, 'No tags selected', 'You must select at least one tag from the list of Available tags.').exec_()
return
pos = self.available_tags.verticalScrollBar().value()
for item in items:
used = self.db.is_tag_used(unicode_type(item.text())) \
if self.key is None else \
self.db.is_item_used_in_multiple(unicode_type(item.text()), label=self.key)
if used:
confirms.append(item)
else:
deletes.append(item)
if confirms:
ct = ', '.join([unicode_type(item.text()) for item in confirms])
if question_dialog(self, _('Are your sure?'),
'<p>'+_('The following tags are used by one or more books. '
'Are you certain you want to delete them?')+'<br>'+ct):
deletes += confirms
for item in deletes:
if self.key is None:
self.db.delete_tag(unicode_type(item.text()))
else:
bks = self.db.delete_item_from_multiple(unicode_type(item.text()),
label=self.key)
self.db.refresh_ids(bks)
self.available_tags.takeItem(self.available_tags.row(item))
self.available_tags.verticalScrollBar().setValue(pos)
def _update_drive_info(self, storage, location_code, name=None):
from calibre.utils.date import isoformat, now
from calibre.utils.config import from_json, to_json
import uuid
f = storage.find_path(self.calibre_file_paths['driveinfo'].split('/'))
dinfo = {}
if f is not None:
try:
stream = self.get_mtp_file(f)
dinfo = json.load(stream, object_hook=from_json)
except:
prints('Failed to load existing driveinfo.calibre file, with error:')
traceback.print_exc()
dinfo = {}
if dinfo.get('device_store_uuid', None) is None:
dinfo['device_store_uuid'] = unicode_type(uuid.uuid4())
if dinfo.get('device_name', None) is None:
dinfo['device_name'] = self.current_friendly_name
if name is not None:
dinfo['device_name'] = name
dinfo['location_code'] = location_code
dinfo['last_library_uuid'] = getattr(self, 'current_library_uuid', None)
dinfo['calibre_version'] = '.'.join([unicode_type(i) for i in numeric_version])
dinfo['date_last_connected'] = isoformat(now())
dinfo['mtp_prefix'] = storage.storage_prefix
raw = json.dumps(dinfo, default=to_json)
self.put_calibre_file(storage, 'driveinfo', BytesIO(raw), len(raw))
self.driveinfo[location_code] = dinfo
def chapter_break(self, match):
chap = match.group('section')
styles = match.group('styles')
self.html_preprocess_sections = self.html_preprocess_sections + 1
self.log.debug("marked " + unicode_type(self.html_preprocess_sections) +
" section markers based on punctuation. - " + unicode_type(chap))
return '<'+styles+' style="page-break-before:always">'+chap
def migrate_old_conf(self, old_conf_path):
from calibre.utils.config import DynamicConfig
c = DynamicConfig('scheduler')
for r in c.get('scheduled_recipes', []):
try:
self.add_old_recipe(r)
except:
continue
for k in c.keys():
if k.startswith('recipe_account_info'):
try:
urn = k.replace('recipe_account_info_', '')
if urn.startswith('recipe_'):
urn = 'builtin:'+urn[7:]
else:
urn = 'custom:%d'%int(urn)
try:
username, password = c[k]
except:
username = password = ''
self.set_account_info(urn, unicode_type(username),
unicode_type(password))
except:
continue
del c
self.write_scheduler_file()
try:
os.remove(old_conf_path)
except:
pass
def test_mem_leaks(self):
import gc
from calibre.utils.mem import get_memory as memory
m = Matcher(['a'], scorer=CScorer)
m('a')
def doit(c):
m = Matcher([
c + 'im/one.gif',
c + 'im/two.gif',
c + 'text/one.html',
],
scorer=CScorer)
m('one')
start = memory()
for i in range(10):
doit(unicode_type(i))
gc.collect()
used10 = memory() - start
start = memory()
for i in range(100):
doit(unicode_type(i))
gc.collect()
used100 = memory() - start
if used100 > 0 and used10 > 0:
self.assertLessEqual(used100, 2 * used10)
def navpoint(parent, np):
text = np.text
if not text:
text = ''
c[1] += 1
item_id = 'num_%d'%c[1]
text = clean_xml_chars(text)
elem = E.navPoint(
E.navLabel(E.text(re.sub(r'\s+', ' ', text))),
E.content(src=unicode_type(np.href)+(('#' + unicode_type(np.fragment))
if np.fragment else '')),
id=item_id,
playOrder=str(np.play_order)
)
au = getattr(np, 'author', None)
if au:
au = re.sub(r'\s+', ' ', au)
elem.append(C.meta(au, name='author'))
desc = getattr(np, 'description', None)
if desc:
desc = re.sub(r'\s+', ' ', desc)
try:
elem.append(C.meta(desc, name='description'))
except ValueError:
elem.append(C.meta(clean_xml_chars(desc), name='description'))
idx = getattr(np, 'toc_thumbnail', None)
if idx:
elem.append(C.meta(idx, name='toc_thumbnail'))
parent.append(elem)
for np2 in np:
navpoint(elem, np2)
def commit(self, urn=None):
urn = self.current_urn if urn is None else urn
if not self.detail_box.isVisible() or urn is None:
return True
if self.account.isVisible():
un, pw = map(unicode_type, (self.username.text(), self.password.text()))
un, pw = un.strip(), pw.strip()
if not un and not pw and self.schedule.isChecked():
if not getattr(self, 'subscription_optional', False):
error_dialog(self, _('Need username and password'),
_('You must provide a username and/or password to '
'use this news source.'), show=True)
return False
if un or pw:
self.recipe_model.set_account_info(urn, un, pw)
else:
self.recipe_model.clear_account_info(urn)
if self.schedule.isChecked():
schedule_type, schedule = \
self.schedule_stack.currentWidget().schedule
self.recipe_model.schedule_recipe(urn, schedule_type, schedule)
else:
self.recipe_model.un_schedule_recipe(urn)
add_title_tag = self.add_title_tag.isChecked()
keep_issues = u'0'
if self.keep_issues.isEnabled():
keep_issues = unicode_type(self.keep_issues.value())
custom_tags = unicode_type(self.custom_tags.text()).strip()
custom_tags = [x.strip() for x in custom_tags.split(',')]
self.recipe_model.customize_recipe(urn, add_title_tag, custom_tags, keep_issues)
return True
def test_winutil(self):
from calibre.constants import plugins
from calibre import strftime
winutil = plugins['winutil'][0]
def au(x, name):
self.assertTrue(isinstance(x, unicode_type), name + '() did not return a unicode string')
for x in winutil.argv():
au(x, 'argv')
for x in 'username temp_path locale_name'.split():
au(getattr(winutil, x)(), x)
d = winutil.localeconv()
au(d['thousands_sep'], 'localeconv')
au(d['decimal_point'], 'localeconv')
for k, v in iteritems(d):
au(v, k)
for k in os.environ.keys():
au(winutil.getenv(unicode_type(k)), 'getenv-' + k)
os.environ['XXXTEST'] = 'YYY'
self.assertEqual(winutil.getenv(u'XXXTEST'), u'YYY')
del os.environ['XXXTEST']
self.assertIsNone(winutil.getenv(u'XXXTEST'))
t = time.localtime()
fmt = u'%Y%a%b%e%H%M'
for fmt in (fmt, fmt.encode('ascii')):
x = strftime(fmt, t)
au(x, 'strftime')
self.assertEqual(unicode_type(time.strftime(fmt.replace('%e', '%#d'), t)), x)
def set_email_settings(self, to_set):
from_ = unicode_type(self.email_from.text()).strip()
if to_set and not from_:
error_dialog(self, _('Bad configuration'),
_('You must set the From email address')).exec_()
return False
username = unicode_type(self.relay_username.text()).strip()
password = unicode_type(self.relay_password.text()).strip()
host = unicode_type(self.relay_host.text()).strip()
enc_method = ('TLS' if self.relay_tls.isChecked() else 'SSL'
if self.relay_ssl.isChecked() else 'NONE')
if host:
# Validate input
if ((username and not password) or (not username and password)):
error_dialog(self, _('Bad configuration'),
_('You must either set both the username <b>and</b> password for '
'the mail server or no username and no password at all.')).exec_()
return False
if not (username and password) and not question_dialog(
self, _('Are you sure?'),
_('No username and password set for mailserver. Most '
' mailservers need a username and password. Are you sure?')):
return False
conf = smtp_prefs()
conf.set('from_', from_)
conf.set('relay_host', host if host else None)
conf.set('relay_port', self.relay_port.value())
conf.set('relay_username', username if username else None)
conf.set('relay_password', as_hex_unicode(password))
conf.set('encryption', enc_method)
return True
def to_html(self):
'''
A HTML representation of this object.
'''
from calibre.ebooks.metadata import authors_to_string
from calibre.utils.date import isoformat
ans = [(_('Title'), unicode_type(self.title))]
ans += [(_('Author(s)'), (authors_to_string(self.authors) if self.authors else _('Unknown')))]
ans += [(_('Publisher'), unicode_type(self.publisher))]
ans += [(_('Producer'), unicode_type(self.book_producer))]
ans += [(_('Comments'), unicode_type(self.comments))]
ans += [('ISBN', unicode_type(self.isbn))]
ans += [(_('Tags'), u', '.join([unicode_type(t) for t in self.tags]))]
if self.series:
ans += [(_('Series'), unicode_type(self.series) + ' #%s'%self.format_series_index())]
ans += [(_('Languages'), u', '.join(self.languages))]
if self.timestamp is not None:
ans += [(_('Timestamp'), unicode_type(isoformat(self.timestamp, as_utc=False, sep=' ')))]
if self.pubdate is not None:
ans += [(_('Published'), unicode_type(isoformat(self.pubdate, as_utc=False, sep=' ')))]
if self.rights is not None:
ans += [(_('Rights'), unicode_type(self.rights))]
for key in self.custom_field_keys():
val = self.get(key, None)
if val:
(name, val) = self.format_field(key)
ans += [(name, val)]
for i, x in enumerate(ans):
ans[i] = u'<tr><td><b>%s</b></td><td>%s</td></tr>'%x
return u'<table>%s</table>'%u'\n'.join(ans)
def test_workers(self):
' Test worker semantics '
with TestServer(lambda data:(data.path[0] + data.read()), worker_count=3) as server:
self.ae(3, sum(int(w.is_alive()) for w in server.loop.pool.workers))
server.loop.stop()
server.join()
self.ae(0, sum(int(w.is_alive()) for w in server.loop.pool.workers))
# Test shutdown with hung worker
block = Event()
with TestServer(lambda data:block.wait(), worker_count=3, shutdown_timeout=0.1, timeout=0.01) as server:
pool = server.loop.pool
self.ae(3, sum(int(w.is_alive()) for w in pool.workers))
conn = server.connect()
conn.request('GET', '/')
with self.assertRaises(socket.timeout):
res = conn.getresponse()
if unicode_type(res.status) == unicode_type(http_client.REQUEST_TIMEOUT):
raise socket.timeout('Timeout')
raise Exception('Got unexpected response: code: %s %s headers: %r data: %r' % (
res.status, res.reason, res.getheaders(), res.read()))
self.ae(pool.busy, 1)
server.loop.log.filter_level = server.loop.log.ERROR
server.loop.stop()
server.join()
self.ae(1, sum(int(w.is_alive()) for w in pool.workers))
def gst_save_clicked(self):
idx = self.gst_names.currentIndex()
name = icu_lower(unicode_type(self.gst_names.currentText()))
if not name:
return error_dialog(self.gui, _('Grouped search terms'),
_('The search term cannot be blank'),
show=True)
if idx != 0:
orig_name = unicode_type(self.gst_names.itemData(idx) or '')
else:
orig_name = ''
if name != orig_name:
if name in self.db.field_metadata.get_search_terms() and \
name not in self.orig_gst_keys:
return error_dialog(self.gui, _('Grouped search terms'),
_('That name is already used for a column or grouped search term'),
show=True)
if name in [icu_lower(p) for p in self.db.prefs.get('user_categories', {})]:
return error_dialog(self.gui, _('Grouped search terms'),
_('That name is already used for User category'),
show=True)
val = [v.strip() for v in unicode_type(self.gst_value.text()).split(',') if v.strip()]
if not val:
return error_dialog(self.gui, _('Grouped search terms'),
_('The value box cannot be empty'), show=True)
if orig_name and name != orig_name:
del self.gst[orig_name]
self.gst_changed = True
self.gst[name] = val
self.fill_gst_box(select=name)
self.set_similar_fields(initial=False)
self.changed_signal.emit()
def box_search_string(self):
mk = self.matchkind.currentIndex()
if mk == CONTAINS_MATCH:
self.mc = ''
elif mk == EQUALS_MATCH:
self.mc = '='
else:
self.mc = '~'
ans = []
self.box_last_values = {}
title = unicode_type(self.title_box.text()).strip()
if title:
ans.append('title:"' + self.mc + title + '"')
author = unicode_type(self.author_box.text()).strip()
if author:
ans.append('author:"' + self.mc + author + '"')
price = unicode_type(self.price_box.text()).strip()
if price:
ans.append('price:"' + self.mc + price + '"')
format = unicode_type(self.format_box.text()).strip()
if format:
ans.append('format:"' + self.mc + format + '"')
drm = '' if self.drm_combo.currentIndex() == 0 else 'true' if self.drm_combo.currentIndex() == 1 else 'false'
if drm:
ans.append('drm:' + drm)
download = '' if self.download_combo.currentIndex() == 0 else 'true' if self.download_combo.currentIndex() == 1 else 'false'
if download:
ans.append('download:' + download)
affiliate = '' if self.affiliate_combo.currentIndex() == 0 else 'true' if self.affiliate_combo.currentIndex() == 1 else 'false'
if affiliate:
ans.append('affiliate:' + affiliate)
if ans:
return ' and '.join(ans)
return ''
def initialize(self, shortcut, all_shortcuts):
self.header.setText('<b>%s: %s</b>'%(_('Customize'), shortcut['name']))
self.all_shortcuts = all_shortcuts
self.shortcut = shortcut
self.default_keys = [QKeySequence(k, QKeySequence.PortableText) for k
in shortcut['default_keys']]
self.current_keys = list(shortcut['keys'])
default = ', '.join([unicode_type(k.toString(k.NativeText)) for k in
self.default_keys])
if not default:
default = _('None')
current = ', '.join([unicode_type(k.toString(k.NativeText)) for k in
self.current_keys])
if not current:
current = _('None')
self.use_default.setText(_('&Default: %(deflt)s [Currently not conflicting: %(curr)s]')%
dict(deflt=default, curr=current))
if shortcut['set_to_default']:
self.use_default.setChecked(True)
else:
self.use_custom.setChecked(True)
for key, which in zip(self.current_keys, [1,2]):
button = getattr(self, 'button%d'%which)
button.setText(key.toString(key.NativeText))
def add_recipient(self):
to = unicode_type(self.address.text()).strip()
if not to:
return error_dialog(
self, _('Need address'), _('You must specify an address'), show=True)
formats = ','.join([x.strip().upper() for x in unicode_type(self.formats.text()).strip().split(',') if x.strip()])
if not formats:
return error_dialog(
self, _('Need formats'), _('You must specify at least one format to send'), show=True)
opts = email_config().parse()
if to in opts.accounts:
return error_dialog(
self, _('Already exists'), _('The recipient %s already exists') % to, show=True)
acc = opts.accounts
acc[to] = [formats, False, False]
c = email_config()
c.set('accounts', acc)
alias = unicode_type(self.alias.text()).strip()
if alias:
opts.aliases[to] = alias
c.set('aliases', opts.aliases)
subject = unicode_type(self.subject.text()).strip()
if subject:
opts.subjects[to] = subject
c.set('subjects', opts.subjects)
self.create_item(alias or to, to, checked=True)
def request_split(self, loc, totals):
actions['split-in-preview'].setChecked(False)
loc, totals = json.loads(unicode_type(loc)), json.loads(unicode_type(totals))
if not loc or not totals:
return error_dialog(self.view(), _('Invalid location'),
_('Cannot split on the body tag'), show=True)
self.split_requested.emit(loc, totals)
def default_cover(self):
'''
Create a generic cover for books that dont have a cover
'''
if self.no_default_cover:
return None
self.log('Generating default cover')
m = self.oeb.metadata
title = unicode_type(m.title[0])
authors = [unicode_type(x) for x in m.creator if x.role == 'aut']
try:
from calibre.ebooks.covers import create_cover
series = series_index = None
if m.series:
try:
series, series_index = unicode_type(m.series[0]), m.series_index[0]
except IndexError:
pass
img_data = create_cover(title, authors, series, series_index)
id, href = self.oeb.manifest.generate('cover',
u'cover_image.jpg')
item = self.oeb.manifest.add(id, href, guess_type('t.jpg')[0],
data=img_data)
m.clear('cover')
m.add('cover', item.id)
return item.href
except:
self.log.exception('Failed to generate default cover')
return None
def parse_ncx(container, ncx_name):
root = container.parsed(ncx_name)
toc_root = TOC()
navmaps = root.xpath('//*[calibre:lower-case(local-name()) = "navmap"]')
if navmaps:
process_ncx_node(container, navmaps[0], toc_root, ncx_name)
toc_root.lang = toc_root.uid = None
for attr, val in iteritems(root.attrib):
if attr.endswith('lang'):
toc_root.lang = unicode_type(val)
break
for uid in root.xpath('//*[calibre:lower-case(local-name()) = "meta" and @name="dtb:uid"]/@content'):
if uid:
toc_root.uid = unicode_type(uid)
break
for pl in root.xpath('//*[calibre:lower-case(local-name()) = "pagelist"]'):
for pt in pl.xpath('descendant::*[calibre:lower-case(local-name()) = "pagetarget"]'):
pagenum = pt.get('value')
if pagenum:
href = pt.xpath('descendant::*[calibre:lower-case(local-name()) = "content"]/@src')
if href:
dest = container.href_to_name(href[0], base=ncx_name)
frag = urlparse(href[0]).fragment or None
toc_root.page_list.append({'dest': dest, 'pagenum': pagenum, 'frag': frag})
return toc_root
def get_cover(self):
from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES
cover_href = None
# Get the raster cover if it's available.
if self.oeb_book.metadata.cover and unicode_type(self.oeb_book.metadata.cover[0]) in self.oeb_book.manifest.ids:
id = unicode_type(self.oeb_book.metadata.cover[0])
cover_item = self.oeb_book.manifest.ids[id]
if cover_item.media_type in OEB_RASTER_IMAGES:
cover_href = cover_item.href
else:
# Figure out if we have a title page or a cover page
page_name = ''
if 'titlepage' in self.oeb_book.guide:
page_name = 'titlepage'
elif 'cover' in self.oeb_book.guide:
page_name = 'cover'
if page_name:
cover_item = self.oeb_book.manifest.hrefs[self.oeb_book.guide[page_name].href]
# Get the first image in the page
for img in cover_item.xpath('//img'):
cover_href = cover_item.abshref(img.get('src'))
break
if cover_href:
# Only write the image tag if it is in the manifest.
if cover_href in self.oeb_book.manifest.hrefs.keys():
if cover_href not in self.image_hrefs.keys():
self.image_hrefs[cover_href] = '_%s.jpg' % len(self.image_hrefs.keys())
return u'<coverpage><image xlink:href="#%s" /></coverpage>' % self.image_hrefs[cover_href]
return u''
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
if len(args) < 2:
parser.print_help()
prints(_('No file specified'), file=sys.stderr)
return 1
path = args[1]
stream_type = os.path.splitext(path)[1].replace('.', '').lower()
trying_to_set = False
for pref in config().option_set.preferences:
if pref.name in ('to_opf', 'get_cover'):
continue
if getattr(opts, pref.name) is not None:
trying_to_set = True
break
with open(path, 'rb') as stream:
mi = get_metadata(stream, stream_type, force_read_metadata=True)
if trying_to_set:
prints(_('Original metadata')+'::')
metadata = unicode_type(mi)
if trying_to_set:
metadata = '\t'+'\n\t'.join(metadata.split('\n'))
prints(metadata, safe_encode=True)
if trying_to_set:
with open(path, 'r+b') as stream:
do_set_metadata(opts, mi, stream, stream_type)
stream.seek(0)
stream.flush()
lrf = None
if stream_type == 'lrf':
if opts.lrf_bookid is not None:
lrf = LRFMetaFile(stream)
lrf.book_id = opts.lrf_bookid
mi = get_metadata(stream, stream_type, force_read_metadata=True)
prints('\n' + _('Changed metadata') + '::')
metadata = unicode_type(mi)
metadata = '\t'+'\n\t'.join(metadata.split('\n'))
prints(metadata, safe_encode=True)
if lrf is not None:
prints('\tBookID:', lrf.book_id)
if opts.to_opf is not None:
from calibre.ebooks.metadata.opf2 import OPFCreator
opf = OPFCreator(getcwd(), mi)
with open(opts.to_opf, 'wb') as f:
opf.render(f)
prints(_('OPF created in'), opts.to_opf)
if opts.get_cover is not None:
if mi.cover_data and mi.cover_data[1]:
with open(opts.get_cover, 'wb') as f:
f.write(mi.cover_data[1])
prints(_('Cover saved to'), f.name)
else:
prints(_('No cover found'), file=sys.stderr)
return 0
def save_settings(cls, config_widget):
proxy = cls._configProxy()
proxy['format_map'] = config_widget.format_map()
if cls.SUPPORTS_SUB_DIRS:
proxy['use_subdirs'] = config_widget.use_subdirs()
if not cls.MUST_READ_METADATA:
proxy['read_metadata'] = config_widget.read_metadata()
if cls.SUPPORTS_USE_AUTHOR_SORT:
proxy['use_author_sort'] = config_widget.use_author_sort()
if cls.EXTRA_CUSTOMIZATION_MESSAGE:
if isinstance(cls.EXTRA_CUSTOMIZATION_MESSAGE, list):
ec = []
for i in range(0, len(cls.EXTRA_CUSTOMIZATION_MESSAGE)):
if config_widget.opt_extra_customization[i] is None:
ec.append(None)
continue
if hasattr(config_widget.opt_extra_customization[i], 'isChecked'):
ec.append(config_widget.opt_extra_customization[i].isChecked())
elif hasattr(config_widget.opt_extra_customization[i], 'currentText'):
ec.append(unicode_type(config_widget.opt_extra_customization[i].currentText()).strip())
else:
ec.append(unicode_type(config_widget.opt_extra_customization[i].text()).strip())
else:
ec = unicode_type(config_widget.opt_extra_customization.text()).strip()
if not ec:
ec = None
proxy['extra_customization'] = ec
st = unicode_type(config_widget.opt_save_template.text())
proxy['save_template'] = st
def options(self):
# Save the currently activated fields
fields = []
for x in range(self.db_fields.count()):
item = self.db_fields.item(x)
if item.isSelected():
fields.append(unicode_type(item.text()))
gprefs.set(self.name+'_db_fields', fields)
# Dictionary currently activated fields
if len(self.db_fields.selectedItems()):
opts_dict = {'fields':[unicode_type(i.text()) for i in self.db_fields.selectedItems()]}
else:
opts_dict = {'fields':['all']}
# Save/return the current options
# bib_cit stores as text
# 'bibfile_enc','bibfile_enctag' stores as int (Indexes)
for opt in self.OPTION_FIELDS:
if opt[0] in ['bibfile_enc', 'bibfile_enctag', 'bib_entry']:
opt_value = getattr(self,opt[0]).currentIndex()
elif opt[0] in ['impcit', 'addfiles'] :
opt_value = getattr(self, opt[0]).isChecked()
else :
opt_value = unicode_type(getattr(self, opt[0]).text())
gprefs.set(self.name + '_' + opt[0], opt_value)
opts_dict[opt[0]] = opt_value
return opts_dict
def all_in_marked(self, pat, template=None):
if self.current_search_mark is None:
return 0
c = self.current_search_mark.cursor
raw = unicode_type(c.selectedText()).replace(PARAGRAPH_SEPARATOR, '\n').rstrip('\0')
if template is None:
count = len(pat.findall(raw))
else:
from calibre.gui2.tweak_book.function_replace import Function
repl_is_func = isinstance(template, Function)
if repl_is_func:
template.init_env()
raw, count = pat.subn(template, raw)
if repl_is_func:
from calibre.gui2.tweak_book.search import show_function_debug_output
if getattr(template.func, 'append_final_output_to_marked', False):
retval = template.end()
if retval:
raw += unicode_type(retval)
else:
template.end()
show_function_debug_output(template)
if count > 0:
start_pos = min(c.anchor(), c.position())
c.insertText(raw)
end_pos = max(c.anchor(), c.position())
c.setPosition(start_pos), c.setPosition(end_pos, c.KeepAnchor)
self.update_extra_selections()
return count
def init_search_box_mixin(self):
self.search.initialize('main_search_history', colorize=True,
help_text=_('Search (For advanced search click the gear icon to the left)'))
self.search.cleared.connect(self.search_box_cleared)
# Queued so that search.current_text will be correct
self.search.changed.connect(self.search_box_changed,
type=Qt.QueuedConnection)
self.search.focus_to_library.connect(self.focus_to_library)
self.advanced_search_toggle_action.triggered.connect(self.do_advanced_search)
self.search.clear()
self.search.setMaximumWidth(self.width()-150)
self.action_focus_search = QAction(self)
shortcuts = list(
map(lambda x:unicode_type(x.toString(QKeySequence.PortableText)),
QKeySequence.keyBindings(QKeySequence.Find)))
shortcuts += ['/', 'Alt+S']
self.keyboard.register_shortcut('start search', _('Start search'),
default_keys=shortcuts, action=self.action_focus_search)
self.action_focus_search.triggered.connect(self.focus_search_box)
self.addAction(self.action_focus_search)
self.search.setStatusTip(re.sub(r'<\w+>', ' ',
unicode_type(self.search.toolTip())))
self.set_highlight_only_button_icon()
self.highlight_only_button.clicked.connect(self.highlight_only_clicked)
tt = _('Enable or disable search highlighting.') + '<br><br>'
tt += config.help('highlight_search_matches')
self.highlight_only_button.setToolTip(tt)
self.highlight_only_action = ac = QAction(self)
self.addAction(ac), ac.triggered.connect(self.highlight_only_clicked)
self.keyboard.register_shortcut('highlight search results', _('Highlight search results'), action=self.highlight_only_action)
def create_cover_markup(self, img, preserve_aspect_ratio, width, height):
self.count += 1
makeelement, namespaces = self.document_relationships.namespace.makeelement, self.document_relationships.namespace.namespaces
if preserve_aspect_ratio:
if img.width >= img.height:
ar = img.height / img.width
height = ar * width
else:
ar = img.width / img.height
width = ar * height
root = etree.Element('root', nsmap=namespaces)
ans = makeelement(root, 'w:drawing', append=False)
parent = makeelement(ans, 'wp:anchor', **{'dist'+edge:'0' for edge in 'LRTB'})
parent.set('simplePos', '0'), parent.set('relativeHeight', '1'), parent.set('behindDoc',"0"), parent.set('locked', "0")
parent.set('layoutInCell', "1"), parent.set('allowOverlap', '1')
makeelement(parent, 'wp:simplePos', x='0', y='0')
makeelement(makeelement(parent, 'wp:positionH', relativeFrom='page'), 'wp:align').text = 'center'
makeelement(makeelement(parent, 'wp:positionV', relativeFrom='page'), 'wp:align').text = 'center'
width, height = map(pt_to_emu, (width, height))
makeelement(parent, 'wp:extent', cx=unicode_type(width), cy=unicode_type(height))
makeelement(parent, 'wp:effectExtent', l='0', r='0', t='0', b='0')
makeelement(parent, 'wp:wrapTopAndBottom')
self.create_docx_image_markup(parent, 'cover.jpg', _('Cover'), img.rid, width, height)
return ans
def save_search_button_clicked(self):
from calibre.gui2.ui import get_gui
db = get_gui().current_db
name = unicode_type(self.currentText())
if not name.strip():
name = unicode_type(self.search_box.text()).replace('"', '')
name = name.replace('\\', '')
if not name:
error_dialog(self, _('Create saved search'),
_('Invalid saved search name. '
'It must contain at least one letter or number'), show=True)
return
if not self.search_box.text():
error_dialog(self, _('Create saved search'),
_('There is no search to save'), show=True)
return
db.saved_search_delete(name)
db.saved_search_add(name, unicode_type(self.search_box.text()))
# now go through an initialization cycle to ensure that the combobox has
# the new search in it, that it is selected, and that the search box
# references the new search instead of the text in the search.
self.clear()
self.setCurrentIndex(self.findText(name))
self.saved_search_selected(name)
self.changed.emit()
def get_cookies(self):
'''
Writes QNetworkCookies to Mozilla cookie .txt file.
:return: The file path to the cookie file.
'''
cf = PersistentTemporaryFile(suffix='.txt')
cf.write('# Netscape HTTP Cookie File\n\n')
for c in self.page().networkAccessManager().cookieJar().allCookies():
cookie = []
domain = unicode_type(c.domain())
cookie.append(domain)
cookie.append('TRUE' if domain.startswith('.') else 'FALSE')
cookie.append(unicode_type(c.path()))
cookie.append('TRUE' if c.isSecure() else 'FALSE')
cookie.append(unicode_type(c.expirationDate().toTime_t()))
cookie.append(unicode_type(c.name()))
cookie.append(unicode_type(c.value()))
cf.write('\t'.join(cookie))
cf.write('\n')
cf.close()
return cf.name
def copy_to_clipboard(self, *args):
QApplication.clipboard().setText(
'calibre, version %s\n%s: %s\n\n%s' %
(__version__, unicode_type(self.windowTitle()),
unicode_type(self.msg_label.text()),
unicode_type(self.det_msg.toPlainText())))
self.copy_button.setText(_('Copied'))
def rule(self):
folder = unicode_type(self.folder.text()).strip()
if folder:
return (
unicode_type(self.fmt.itemData(self.fmt.currentIndex()) or ''),
folder
)
return None
def emit_navigate(self, *args):
item = self.view.currentItem()
if item is not None:
dest = unicode_type(item.data(0, DEST_ROLE) or '')
frag = unicode_type(item.data(0, FRAG_ROLE) or '')
if not frag:
frag = TOP
self.navigate_requested.emit(dest, frag)
def category(ctx, rd, encoded_name, library_id):
'''
Return a dictionary describing the category specified by name. The
Optional: ?num=100&offset=0&sort=name&sort_order=asc
The dictionary looks like::
{
'category_name': Category display name,
'base_url': Base URL for this category,
'total_num': Total numberof items in this category,
'offset': The offset for the items returned in this result,
'num': The number of items returned in this result,
'sort': How the returned items are sorted,
'sort_order': asc or desc
'subcategories': List of sub categories of this category.
'items': List of items in this category,
}
Each subcategory is a dictionary of the same form as those returned by
/ajax/categories
Each item is a dictionary of the form::
{
'name': Display name,
'average_rating': Average rating for books in this item,
'count': Number of books in this item,
'url': URL to get list of books in this item,
'has_children': If True this item contains sub categories, look
for an entry corresponding to this item in subcategories int he
main dictionary,
}
:param sort: How to sort the returned items. Choices are: name, rating,
popularity
:param sort_order: asc or desc
To learn how to create subcategories see
https://manual.calibre-ebook.com/sub_groups.html
'''
db = get_db(ctx, rd, library_id)
with db.safe_read_lock:
num, offset = get_pagination(rd.query)
sort, sort_order = rd.query.get('sort'), rd.query.get('sort_order')
sort = ensure_val(sort, 'name', 'rating', 'popularity')
sort_order = ensure_val(sort_order, 'asc', 'desc')
try:
dname = decode_name(encoded_name)
except:
raise HTTPNotFound('Invalid encoding of category name %r' %
encoded_name)
base_url = ctx.url_for(globals()['category'],
encoded_name=encoded_name,
library_id=db.server_library_id)
if dname in ('newest', 'allbooks'):
sort, sort_order = 'timestamp', 'desc'
rd.query['sort'], rd.query['sort_order'] = sort, sort_order
return books_in(ctx, rd, encoded_name, encode_name('0'),
library_id)
fm = db.field_metadata
categories = ctx.get_categories(rd, db)
hierarchical_categories = db.pref('categories_using_hierarchy', ())
subcategory = dname
toplevel = subcategory.partition('.')[0]
if toplevel == subcategory:
subcategory = None
if toplevel not in categories or toplevel not in fm:
raise HTTPNotFound('Category %r not found' % toplevel)
# Find items and sub categories
subcategories = []
meta = fm[toplevel]
item_names = {}
children = set()
if meta['kind'] == 'user':
fullname = ((toplevel + '.' +
subcategory) if subcategory is not None else toplevel)
try:
# User categories cannot be applied to books, so this is the
# complete set of items, no need to consider sub categories
items = categories[fullname]
except:
raise HTTPNotFound('User category %r not found' % fullname)
parts = fullname.split('.')
for candidate in categories:
cparts = candidate.split('.')
if len(cparts) == len(parts) + 1 and cparts[:-1] == parts:
subcategories.append({
'name': cparts[-1],
'url': candidate,
'icon': category_icon(toplevel, meta)
})
category_name = toplevel[1:].split('.')
# When browsing by user categories we ignore hierarchical normal
# columns, so children can be empty
elif toplevel in hierarchical_categories:
items = []
category_names = [
x.original_name.split('.') for x in categories[toplevel]
if '.' in x.original_name
]
if subcategory is None:
children = set(x[0] for x in category_names)
category_name = [meta['name']]
items = [
x for x in categories[toplevel]
if '.' not in x.original_name
]
else:
subcategory_parts = subcategory.split('.')[1:]
category_name = [meta['name']] + subcategory_parts
lsp = len(subcategory_parts)
children = set(
'.'.join(x) for x in category_names
if len(x) == lsp + 1 and x[:lsp] == subcategory_parts)
items = [
x for x in categories[toplevel]
if x.original_name in children
]
item_names = {
x: x.original_name.rpartition('.')[-1]
for x in items
}
# Only mark the subcategories that have children themselves as
# subcategories
children = set(
'.'.join(x[:lsp + 1]) for x in category_names
if len(x) > lsp + 1 and x[:lsp] == subcategory_parts)
subcategories = [{
'name': x.rpartition('.')[-1],
'url': toplevel + '.' + x,
'icon': category_icon(toplevel, meta)
} for x in children]
else:
items = categories[toplevel]
category_name = meta['name']
for x in subcategories:
x['url'] = ctx.url_for(globals()['category'],
encoded_name=encode_name(x['url']),
library_id=db.server_library_id)
x['icon'] = ctx.url_for(get_icon, which=x['icon'])
x['is_category'] = True
sort_keygen = {
'name': lambda x: sort_key(x.sort if x.sort else x.original_name),
'popularity': lambda x: x.count,
'rating': lambda x: x.avg_rating
}
items.sort(key=sort_keygen[sort], reverse=sort_order == 'desc')
total_num = len(items)
items = items[offset:offset + num]
items = [{
'name':
item_names.get(x, x.original_name),
'average_rating':
x.avg_rating,
'count':
x.count,
'url':
ctx.url_for(
books_in,
encoded_category=encode_name(
x.category if x.category else toplevel),
encoded_item=encode_name(
x.original_name if x.id is None else unicode_type(x.id)),
library_id=db.server_library_id),
'has_children':
x.original_name in children,
} for x in items]
return {
'category_name': category_name,
'base_url': base_url,
'total_num': total_num,
'offset': offset,
'num': len(items),
'sort': sort,
'sort_order': sort_order,
'subcategories': subcategories,
'items': items,
}
def color_to_clipboard(self):
app = QApplication.instance()
c = app.clipboard()
c.setText(unicode_type(self.color_name.color))
def setEditorData(self, editor, index):
name = unicode_type(index.data(Qt.DisplayRole) or '')
editor.setText(name)
editor.lineEdit().selectAll()
def value(self):
ans = [x.strip() for x in unicode_type(self.t.text()).strip().split(',')]
return [x for x in ans if x]
def format_map(self):
return [unicode_type(self.f.item(i).data(Qt.ItemDataRole.UserRole) or '') for i in
range(self.f.count()) if self.f.item(i).checkState()==Qt.CheckState.Checked]
def __init__(self, parent, dbspec, ids, db):
import re
from calibre import prints as info
from PyQt5.uic import compileUi
QDialog.__init__(self, parent)
self.setupUi(self)
self.dbspec, self.ids = dbspec, ids
# Display the number of books we've been passed
self.count.setText(unicode_type(self.count.text()).format(len(ids)))
# Display the last-used title
self.title.setText(
dynamic.get('catalog_last_used_title', _('My books')))
self.fmts, self.widgets = [], []
for plugin in catalog_plugins():
if plugin.name in config['disabled_plugins']:
continue
name = plugin.name.lower().replace(' ', '_')
if getattr(plugin, 'plugin_path', None) is None:
try:
catalog_widget = importlib.import_module(
'calibre.gui2.catalog.' + name)
pw = catalog_widget.PluginWidget()
pw.parent_ref = weakref.ref(self)
pw.initialize(name, db)
pw.ICON = I('forward.png')
self.widgets.append(pw)
[
self.fmts.append(
[file_type.upper(), pw.sync_enabled, pw])
for file_type in plugin.file_types
]
except ImportError:
info("ImportError initializing %s" % name)
continue
else:
# Load dynamic tab
form = os.path.join(plugin.resources_path, '%s.ui' % name)
klass = os.path.join(plugin.resources_path, '%s.py' % name)
compiled_form = os.path.join(plugin.resources_path,
'%s_ui.py' % name)
if os.path.exists(form) and os.path.exists(klass):
# info("Adding widget for user-installed Catalog plugin %s" % plugin.name)
# Compile the .ui form provided in plugin.zip
if not os.path.exists(compiled_form):
from polyglot.io import PolyglotStringIO
# info('\tCompiling form', form)
buf = PolyglotStringIO()
compileUi(form, buf)
dat = buf.getvalue()
dat = re.compile(
r'QtGui.QApplication.translate\(.+?,\s+"(.+?)(?<!\\)",.+?\)',
re.DOTALL).sub(r'_("\1")', dat)
open(compiled_form, 'wb').write(dat.encode('utf-8'))
# Import the dynamic PluginWidget() from .py file provided in plugin.zip
try:
sys.path.insert(0, plugin.resources_path)
catalog_widget = importlib.import_module(name)
pw = catalog_widget.PluginWidget()
pw.initialize(name)
pw.ICON = I('forward.png')
self.widgets.append(pw)
[
self.fmts.append(
[file_type.upper(), pw.sync_enabled, pw])
for file_type in plugin.file_types
]
except ImportError:
info("ImportError with %s" % name)
continue
finally:
sys.path.remove(plugin.resources_path)
else:
info("No dynamic tab resources found for %s" % name)
self.widgets = sorted(self.widgets, key=lambda x: x.TITLE)
# Generate a sorted list of installed catalog formats/sync_enabled pairs
fmts = sorted((x[0] for x in self.fmts))
self.sync_enabled_formats = []
for fmt in self.fmts:
if fmt[1]:
self.sync_enabled_formats.append(fmt[0])
# Callbacks when format, title changes
self.format.currentIndexChanged.connect(self.format_changed)
self.format.currentIndexChanged.connect(self.settings_changed)
self.title.editingFinished.connect(self.settings_changed)
# Add the installed catalog format list to the format QComboBox
self.format.blockSignals(True)
self.format.addItems(fmts)
pref = dynamic.get('catalog_preferred_format', 'CSV')
idx = self.format.findText(pref)
if idx > -1:
self.format.setCurrentIndex(idx)
self.format.blockSignals(False)
if self.sync.isEnabled():
self.sync.setChecked(dynamic.get('catalog_sync_to_device', True))
self.add_to_library.setChecked(
dynamic.get('catalog_add_to_library', True))
self.format.currentIndexChanged.connect(self.show_plugin_tab)
self.buttonBox.button(self.buttonBox.Apply).clicked.connect(self.apply)
self.buttonBox.button(self.buttonBox.Help).clicked.connect(self.help)
self.show_plugin_tab(None)
geom = dynamic.get('catalog_window_geom', None)
if geom is not None:
self.restoreGeometry(bytes(geom))
else:
self.resize(self.sizeHint())
d = QCoreApplication.instance().desktop()
g = d.availableGeometry(d.screenNumber(self))
self.setMaximumWidth(g.width() - 50)
self.setMaximumHeight(g.height() - 50)
def completion_selected(self, text):
before_text, after_text = self.get_completed_text(unicode_type(text))
self.setText(before_text + after_text)
self.setCursorPosition(len(before_text))
self.item_selected.emit(text)
def get_value_handler(self, g):
if g is not self.opt_markdown_extensions:
return Widget.get_value_handler(self, g)
return ', '.join(
unicode_type(i.data(Qt.UserRole) or '')
for i in itervalues(self.md_map) if i.checkState())
def write_book(self, book_id, mi, components, fmts):
base_path = os.path.join(self.root, *components)
base_dir = os.path.dirname(base_path)
if self.opts.formats and self.opts.formats != 'all':
asked_formats = {
x.lower().strip()
for x in self.opts.formats.split(',')
}
fmts = asked_formats.intersection(fmts)
if not fmts:
self.errors[book_id].append(
('critical', _('Requested formats not available')))
return
if not fmts and not self.opts.write_opf and not self.opts.save_cover:
return
# On windows python incorrectly raises an access denied exception
# when trying to create the root of a drive, like C:\
if os.path.dirname(base_dir) != base_dir:
try:
os.makedirs(base_dir)
except EnvironmentError as err:
if err.errno != errno.EEXIST:
raise
if self.opts.update_metadata:
d = {}
d['last_modified'] = mi.last_modified.isoformat()
cdata = self.db.cover(book_id)
mi.cover, mi.cover_data = None, (None, None)
if cdata:
fname = None
if self.opts.save_cover:
fname = base_path + os.extsep + 'jpg'
mi.cover = os.path.basename(fname)
elif self.opts.update_metadata:
fname = os.path.join(self.tdir, '%d.jpg' % book_id)
if fname:
with lopen(fname, 'wb') as f:
f.write(cdata)
if self.opts.update_metadata:
d['cover'] = fname
fname = None
if self.opts.write_opf:
fname = base_path + os.extsep + 'opf'
elif self.opts.update_metadata:
fname = os.path.join(self.tdir, '%d.opf' % book_id)
if fname:
opf = metadata_to_opf(mi)
with lopen(fname, 'wb') as f:
f.write(opf)
if self.opts.update_metadata:
d['opf'] = fname
mi.cover, mi.cover_data = None, (None, None)
if self.opts.update_metadata:
d['fmts'] = []
for fmt in fmts:
try:
fmtpath = self.write_fmt(book_id, fmt, base_path)
if fmtpath and self.opts.update_metadata and can_set_metadata(
fmt):
d['fmts'].append(fmtpath)
except Exception:
self.errors[book_id].append(
('fmt', (fmt, traceback.format_exc())))
if self.opts.update_metadata:
if d['fmts']:
try:
self.pool(book_id, 'calibre.library.save_to_disk',
'update_serialized_metadata', d)
except Failure as err:
error_dialog(
self.pd,
_('Critical failure'),
_('Could not save books to disk, click "Show details" for more information'
),
det_msg=unicode_type(err.failure_message) + '\n' +
unicode_type(err.details),
show=True)
self.pd.canceled = True
else:
self.pd.value += 1
self.pd.msg = self.book_id_data(book_id).title
def write(self, *args):
for a in args:
self.__output.append(unicode_type(a))
def create_menubar(self):
if isosx:
p, q = self.create_application_menubar()
q.triggered.connect(self.action_quit.trigger)
p.triggered.connect(self.action_preferences.trigger)
f = factory(app_id='com.calibre-ebook.EditBook-%d' % os.getpid())
b = f.create_window_menubar(self)
f = b.addMenu(_('&File'))
f.addAction(self.action_new_file)
f.addAction(self.action_import_files)
f.addSeparator()
f.addAction(self.action_open_book)
f.addAction(self.action_new_book)
f.addAction(self.action_import_book)
f.addAction(self.action_open_book_folder)
self.recent_books_menu = f.addMenu(_('&Recently opened books'))
self.update_recent_books()
f.addSeparator()
f.addAction(self.action_save)
f.addAction(self.action_save_copy)
f.addSeparator()
f.addAction(self.action_compare_book)
f.addAction(self.action_quit)
e = b.addMenu(_('&Edit'))
e.addAction(self.action_global_undo)
e.addAction(self.action_global_redo)
e.addAction(self.action_create_checkpoint)
e.addSeparator()
e.addAction(self.action_editor_undo)
e.addAction(self.action_editor_redo)
e.addSeparator()
e.addAction(self.action_editor_cut)
e.addAction(self.action_editor_copy)
e.addAction(self.action_editor_paste)
e.addAction(self.action_insert_char)
e.addSeparator()
e.addAction(self.action_quick_edit)
e.addAction(self.action_preferences)
e = b.addMenu(_('&Tools'))
tm = e.addMenu(_('Table of Contents'))
tm.addAction(self.action_toc)
tm.addAction(self.action_inline_toc)
e.addAction(self.action_manage_fonts)
e.addAction(self.action_embed_fonts)
e.addAction(self.action_subset_fonts)
e.addAction(self.action_compress_images)
e.addAction(self.action_smarten_punctuation)
e.addAction(self.action_remove_unused_css)
e.addAction(self.action_transform_styles)
e.addAction(self.action_fix_html_all)
e.addAction(self.action_pretty_all)
e.addAction(self.action_rationalize_folders)
e.addAction(self.action_add_cover)
e.addAction(self.action_set_semantics)
e.addAction(self.action_filter_css)
e.addAction(self.action_spell_check_book)
er = e.addMenu(_('External &links'))
er.addAction(self.action_check_external_links)
er.addAction(self.action_get_ext_resources)
e.addAction(self.action_check_book)
e.addAction(self.action_reports)
e.addAction(self.action_upgrade_book_internals)
e = b.addMenu(_('&View'))
t = e.addMenu(_('Tool&bars'))
e.addSeparator()
for name in sorted(actions, key=lambda x: sort_key(actions[x].text())):
ac = actions[name]
if name.endswith('-dock'):
e.addAction(ac)
elif name.endswith('-bar'):
t.addAction(ac)
e.addAction(self.action_browse_images)
e.addSeparator()
e.addAction(self.action_close_current_tab)
e.addAction(self.action_close_all_but_current_tab)
e = b.addMenu(_('&Search'))
a = e.addAction
a(self.action_find)
e.addSeparator()
a(self.action_find_next)
a(self.action_find_previous)
e.addSeparator()
a(self.action_replace)
a(self.action_replace_next)
a(self.action_replace_previous)
a(self.action_replace_all)
e.addSeparator()
a(self.action_count)
e.addSeparator()
a(self.action_mark)
e.addSeparator()
a(self.action_go_to_line)
e.addSeparator()
a(self.action_saved_searches)
e.aboutToShow.connect(self.search_menu_about_to_show)
e.addSeparator()
a(self.action_text_search)
if self.plugin_menu_actions:
e = b.addMenu(_('&Plugins'))
for ac in sorted(self.plugin_menu_actions,
key=lambda x: sort_key(unicode_type(x.text()))):
e.addAction(ac)
e = b.addMenu(_('&Help'))
a = e.addAction
a(self.action_help)
a(QIcon(I('donate.png')), _('&Donate to support calibre development'),
open_donate)
a(self.action_preferences)
def names(self):
for item in self._names.selectedItems():
yield unicode_type(item.data(Qt.DisplayRole) or '')
def authors(self):
ans = []
for i in range(self.al.count()):
ans.append(unicode_type(self.al.item(i).text()))
return ans or [_('Unknown')]
def text(self):
return unicode_type(self.lineEdit().text())
def show_initial_value(self, what):
what = unicode_type(what) if what else u''
self.setText(what)
self.lineEdit().selectAll()
def update_link_clicked(self, url):
url = unicode_type(url)
if url.startswith('update:'):
calibre_version, number_of_plugin_updates = msgpack_loads(from_hex_bytes(url[len('update:'):]))
self.update_found(calibre_version, number_of_plugin_updates, force=True)
def item_chosen(self, index):
if not self.isVisible():
return
self.hide()
text = self.model().data(index, Qt.ItemDataRole.DisplayRole)
self.item_selected.emit(unicode_type(text))
def run(self, path_to_output, opts, db, notification=DummyReporter()):
from calibre.library import current_library_name
from calibre.utils.date import isoformat
from calibre.utils.html2text import html2text
from calibre.utils.logging import default_log as log
from lxml import etree
from calibre.ebooks.metadata import authors_to_string
self.fmt = path_to_output.rpartition('.')[2]
self.notification = notification
current_library = current_library_name()
if getattr(opts, 'library_path', None):
current_library = os.path.basename(opts.library_path)
if opts.verbose:
opts_dict = vars(opts)
log("%s('%s'): Generating %s" %
(self.name, current_library, self.fmt.upper()))
if opts.connected_device['is_device_connected']:
log(" connected_device: %s" % opts.connected_device['name'])
if opts_dict['search_text']:
log(" --search='%s'" % opts_dict['search_text'])
if opts_dict['ids']:
log(" Book count: %d" % len(opts_dict['ids']))
if opts_dict['search_text']:
log(" (--search ignored when a subset of the database is specified)"
)
if opts_dict['fields']:
if opts_dict['fields'] == 'all':
log(" Fields: %s" % ', '.join(FIELDS[1:]))
else:
log(" Fields: %s" % opts_dict['fields'])
# If a list of ids are provided, don't use search_text
if opts.ids:
opts.search_text = None
data = self.search_sort_db(db, opts)
if not len(data):
log.error(
"\nNo matching database entries for search criteria '%s'" %
opts.search_text)
# raise SystemExit(1)
# Get the requested output fields as a list
fields = self.get_output_fields(db, opts)
# If connected device, add 'On Device' values to data
if opts.connected_device[
'is_device_connected'] and 'ondevice' in fields:
for entry in data:
entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[
entry['id']]['ondevice']
fm = {x: db.field_metadata.get(x, {}) for x in fields}
if self.fmt == 'csv':
outfile = codecs.open(path_to_output, 'w', 'utf8')
# Write a UTF-8 BOM
outfile.write('\ufeff')
# Output the field headers
outfile.write('%s\n' % ','.join(fields))
# Output the entry fields
for entry in data:
outstr = []
for field in fields:
if field.startswith('#'):
item = db.get_field(entry['id'],
field,
index_is_id=True)
if isinstance(item, (list, tuple)):
if fm.get(field,
{}).get('display',
{}).get('is_names', False):
item = ' & '.join(item)
else:
item = ', '.join(item)
elif field == 'library_name':
item = current_library
elif field == 'title_sort':
item = entry['sort']
else:
item = entry[field]
if item is None:
outstr.append('""')
continue
elif field == 'formats':
fmt_list = []
for format in item:
fmt_list.append(format.rpartition('.')[2].lower())
item = ', '.join(fmt_list)
elif field == 'authors':
item = authors_to_string(item)
elif field == 'tags':
item = ', '.join(item)
elif field == 'isbn':
# Could be 9, 10 or 13 digits, with hyphens, possibly ending in 'X'
item = '%s' % re.sub(r'[^\dX-]', '', item)
elif fm.get(field, {}).get('datatype') == 'datetime':
item = isoformat(item, as_utc=False)
elif field == 'comments':
item = item.replace('\r\n', ' ')
item = item.replace('\n', ' ')
elif fm.get(field, {}).get('datatype',
None) == 'rating' and item:
item = '%.2g' % (item / 2)
# Convert HTML to markdown text
if isinstance(item, unicode_type):
opening_tag = re.search(r'<(\w+)( |>)', item)
if opening_tag:
closing_tag = re.search(
r'<\/%s>$' % opening_tag.group(1), item)
if closing_tag:
item = html2text(item)
outstr.append('"%s"' %
unicode_type(item).replace('"', '""'))
outfile.write(','.join(outstr) + '\n')
outfile.close()
elif self.fmt == 'xml':
from lxml.builder import E
if getattr(opts, 'catalog_title', None):
root = E.calibredb(title=opts.catalog_title)
else:
root = E.calibredb()
for r in data:
record = E.record()
root.append(record)
for field in fields:
if field.startswith('#'):
val = db.get_field(r['id'], field, index_is_id=True)
if not isinstance(val, unicode_type):
val = unicode_type(val)
item = getattr(E, field.replace('#', '_'))(val)
record.append(item)
for field in ('id', 'uuid', 'publisher', 'rating', 'size',
'isbn', 'ondevice', 'identifiers'):
if field in fields:
val = r[field]
if not val:
continue
if not isinstance(val, (bytes, unicode_type)):
if (fm.get(field, {}).get('datatype', None)
== 'rating' and val):
val = '%.2g' % (val / 2)
val = unicode_type(val)
item = getattr(E, field)(val)
record.append(item)
if 'title' in fields:
title = E.title(r['title'], sort=r['sort'])
record.append(title)
if 'authors' in fields:
aus = E.authors(sort=r['author_sort'])
for au in r['authors']:
aus.append(E.author(au))
record.append(aus)
for field in ('timestamp', 'pubdate'):
if field in fields:
record.append(
getattr(E, field)(isoformat(r[field],
as_utc=False)))
if 'tags' in fields and r['tags']:
tags = E.tags()
for tag in r['tags']:
tags.append(E.tag(tag))
record.append(tags)
if 'comments' in fields and r['comments']:
record.append(E.comments(r['comments']))
if 'series' in fields and r['series']:
record.append(
E.series(r['series'],
index=unicode_type(r['series_index'])))
if 'languages' in fields and r['languages']:
record.append(E.languages(r['languages']))
if 'cover' in fields and r['cover']:
record.append(E.cover(r['cover'].replace(os.sep, '/')))
if 'formats' in fields and r['formats']:
fmt = E.formats()
for f in r['formats']:
fmt.append(E.format(f.replace(os.sep, '/')))
record.append(fmt)
if 'library_name' in fields:
record.append(E.library_name(current_library))
with open(path_to_output, 'wb') as f:
f.write(
etree.tostring(root,
encoding='utf-8',
xml_declaration=True,
pretty_print=True))
def template(self):
return unicode_type(self.t.text()).strip()
def javaScriptAlert(self, origin, msg):
self.log(unicode_type(msg))
def javaScriptConsoleMessage(self, level, msg, lineno, msgid):
self.log('JS:', unicode_type(msg))
def find(self, forwards=True):
text = unicode_type(self.search_text.text()).strip()
flags = QWebEnginePage.FindFlags(0) if forwards else QWebEnginePage.FindBackward
self.find_data = text, flags, forwards
self.view.findText(text, flags, self.find_callback)
def setModelData(self, editor, model, index):
authors = string_to_authors(unicode_type(editor.text()))
model.setData(index, authors[0])
self.edited.emit(index.row())
def book_to_json(ctx,
rd,
db,
book_id,
get_category_urls=True,
device_compatible=False,
device_for_template=None):
mi = db.get_metadata(book_id, get_cover=False)
codec = JsonCodec(db.field_metadata)
if not device_compatible:
try:
mi.rating = mi.rating / 2.
except Exception:
mi.rating = 0.0
data = codec.encode_book_metadata(mi)
for x in ('publication_type', 'size', 'db_id', 'lpath', 'mime', 'rights',
'book_producer'):
data.pop(x, None)
get = partial(ctx.url_for,
get_content,
book_id=book_id,
library_id=db.server_library_id)
data['cover'] = get(what='cover')
data['thumbnail'] = get(what='thumb')
if not device_compatible:
mi.format_metadata = {
k.lower(): dict(v)
for k, v in iteritems(mi.format_metadata)
}
for v in itervalues(mi.format_metadata):
mtime = v.get('mtime', None)
if mtime is not None:
v['mtime'] = isoformat(mtime, as_utc=True)
data['format_metadata'] = mi.format_metadata
fmts = set(x.lower() for x in mi.format_metadata)
pf = prefs['output_format'].lower()
other_fmts = list(fmts)
try:
fmt = pf if pf in fmts else other_fmts[0]
except:
fmt = None
if fmts and fmt:
other_fmts = [x for x in fmts if x != fmt]
data['formats'] = sorted(fmts)
if fmt:
data['main_format'] = {fmt: get(what=fmt)}
else:
data['main_format'] = None
data['other_formats'] = {fmt: get(what=fmt) for fmt in other_fmts}
if get_category_urls:
category_urls = data['category_urls'] = {}
all_cats = ctx.get_categories(rd, db)
for key in mi.all_field_keys():
fm = mi.metadata_for_field(key)
if (fm and fm['is_category'] and not fm['is_csp']
and key != 'formats' and fm['datatype'] != 'rating'):
categories = mi.get(key) or []
if isinstance(categories, string_or_bytes):
categories = [categories]
category_urls[key] = dbtags = {}
for category in categories:
for tag in all_cats.get(key, ()):
if tag.original_name == category:
dbtags[category] = ctx.url_for(
books_in,
encoded_category=encode_name(
tag.category if tag.category else key),
encoded_item=encode_name(
tag.original_name if tag.id is None
else unicode_type(tag.id)),
library_id=db.server_library_id)
break
else:
series = data.get('series', None) or ''
if series:
tsorder = tweaks['save_template_title_series_sorting']
series = title_sort(series, order=tsorder)
data['_series_sort_'] = series
if device_for_template:
import posixpath
from calibre.devices.utils import create_upload_path
from calibre.utils.filenames import ascii_filename as sanitize
from calibre.customize.ui import device_plugins
for device_class in device_plugins():
if device_class.__class__.__name__ == device_for_template:
template = device_class.save_template()
data['_filename_'] = create_upload_path(
mi, book_id, template, sanitize, path_type=posixpath)
break
return data, mi.last_modified
def identify(
log,
abort, # {{{
title=None,
authors=None,
identifiers={},
timeout=30,
allowed_plugins=None):
if title == _('Unknown'):
title = None
if authors == [_('Unknown')]:
authors = None
start_time = time.time()
plugins = [
p for p in metadata_plugins(['identify']) if p.is_configured() and (
allowed_plugins is None or p.name in allowed_plugins)
]
kwargs = {
'title': title,
'authors': authors,
'identifiers': identifiers,
'timeout': timeout,
}
log('Running identify query with parameters:')
log(kwargs)
log('Using plugins:',
', '.join(['%s %s' % (p.name, p.version) for p in plugins]))
log('The log from individual plugins is below')
workers = [Worker(p, kwargs, abort) for p in plugins]
for w in workers:
w.start()
first_result_at = None
results = {}
for p in plugins:
results[p] = []
logs = dict([(w.plugin, w.buf) for w in workers])
def get_results():
found = False
for w in workers:
try:
result = w.rq.get_nowait()
except Empty:
pass
else:
results[w.plugin].append(result)
found = True
return found
wait_time = msprefs['wait_after_first_identify_result']
while True:
time.sleep(0.2)
if get_results() and first_result_at is None:
first_result_at = time.time()
if not is_worker_alive(workers):
break
if (first_result_at is not None
and time.time() - first_result_at > wait_time):
log.warn('Not waiting any longer for more results. Still running'
' sources:')
for worker in workers:
if worker.is_alive():
log.debug('\t' + worker.name)
abort.set()
break
while not abort.is_set() and get_results():
pass
sort_kwargs = dict(kwargs)
for k in list(sort_kwargs):
if k not in ('title', 'authors', 'identifiers'):
sort_kwargs.pop(k)
longest, lp = -1, ''
for plugin, presults in iteritems(results):
presults.sort(key=plugin.identify_results_keygen(**sort_kwargs))
# Throw away lower priority results from the same source that have exactly the same
# title and authors as a higher priority result
filter_results = set()
filtered_results = []
for r in presults:
key = (r.title, tuple(r.authors))
if key not in filter_results:
filtered_results.append(r)
filter_results.add(key)
results[plugin] = presults = filtered_results
plog = logs[plugin].getvalue().strip()
log('\n' + '*' * 30, plugin.name, '%s' % (plugin.version, ), '*' * 30)
log('Found %d results' % len(presults))
time_spent = getattr(plugin, 'dl_time_spent', None)
if time_spent is None:
log('Downloading was aborted')
longest, lp = -1, plugin.name
else:
log('Downloading from', plugin.name, 'took', time_spent)
if time_spent > longest:
longest, lp = time_spent, plugin.name
for r in presults:
log('\n\n---')
try:
log(unicode_type(r))
except TypeError:
log(repr(r))
if plog:
log(plog)
log('\n' + '*' * 80)
dummy = Metadata(_('Unknown'))
for i, result in enumerate(presults):
for f in plugin.prefs['ignore_fields']:
if ':' not in f:
setattr(result, f, getattr(dummy, f))
if f == 'series':
result.series_index = dummy.series_index
result.relevance_in_source = i
result.has_cached_cover_url = (plugin.cached_cover_url_is_reliable
and plugin.get_cached_cover_url(
result.identifiers) is not None)
result.identify_plugin = plugin
if msprefs['txt_comments']:
if plugin.has_html_comments and result.comments:
result.comments = html2text(result.comments)
log('The identify phase took %.2f seconds' % (time.time() - start_time))
log('The longest time (%f) was taken by:' % longest, lp)
log('Merging results from different sources')
start_time = time.time()
results = merge_identify_results(results, log)
log('We have %d merged results, merging took: %.2f seconds' %
(len(results), time.time() - start_time))
tm_rules = msprefs['tag_map_rules']
if tm_rules:
from calibre.ebooks.metadata.tag_mapper import map_tags
am_rules = msprefs['author_map_rules']
if am_rules:
from calibre.ebooks.metadata.author_mapper import map_authors, compile_rules
am_rules = compile_rules(am_rules)
max_tags = msprefs['max_tags']
for r in results:
if tm_rules:
r.tags = map_tags(r.tags, tm_rules)
r.tags = r.tags[:max_tags]
if getattr(r.pubdate, 'year', 2000) <= UNDEFINED_DATE.year:
r.pubdate = None
if msprefs['swap_author_names']:
for r in results:
def swap_to_ln_fn(a):
if ',' in a:
return a
parts = a.split(None)
if len(parts) <= 1:
return a
surname = parts[-1]
return '%s, %s' % (surname, ' '.join(parts[:-1]))
r.authors = [swap_to_ln_fn(a) for a in r.authors]
if am_rules:
for r in results:
new_authors = map_authors(r.authors, am_rules)
if new_authors != r.authors:
r.authors = new_authors
r.author_sort = authors_to_sort_string(r.authors)
return results
def icon_to_clipboard(self):
app = QApplication.instance()
c = app.clipboard()
c.setText(unicode_type(self.icon_files.currentText()))
def do_book(self):
if self.i >= len(self.book_ids):
return self.do_queue()
book_id = self.book_ids[self.i]
self.i += 1
temp_files = []
try:
input_format = get_input_format_for_book(self.db, book_id, None)[0]
input_fmt = self.db.original_fmt(book_id, input_format).lower()
same_fmt = input_fmt == self.output_format.lower()
mi, opf_file = create_opf_file(self.db, book_id)
in_file = PersistentTemporaryFile('.' + input_format)
with in_file:
self.db.copy_format_to(book_id,
input_fmt,
in_file,
index_is_id=True)
out_file = PersistentTemporaryFile('.' + self.output_format)
out_file.write(as_bytes(self.output_format))
out_file.close()
temp_files = [in_file]
combined_recs = GuiRecommendations()
default_recs = bulk_defaults_for_input_format(input_format)
for key in default_recs:
combined_recs[key] = default_recs[key]
if self.use_saved_single_settings:
specific_recs = load_specifics(self.db, book_id)
for key in specific_recs:
combined_recs[key] = specific_recs[key]
for item in self.user_recs:
combined_recs[item[0]] = item[1]
save_specifics(self.db, book_id, combined_recs)
lrecs = list(combined_recs.to_recommendations())
from calibre.customize.ui import plugin_for_output_format
op = plugin_for_output_format(self.output_format)
if op and op.recommendations:
prec = {x[0] for x in op.recommendations}
for i, r in enumerate(list(lrecs)):
if r[0] in prec:
lrecs[i] = (r[0], r[1], OptionRecommendation.HIGH)
cover_file = create_cover_file(self.db, book_id)
if opf_file is not None:
lrecs.append(('read_metadata_from_opf', opf_file.name,
OptionRecommendation.HIGH))
temp_files.append(opf_file)
if cover_file is not None:
lrecs.append(
('cover', cover_file.name, OptionRecommendation.HIGH))
temp_files.append(cover_file)
for x in list(lrecs):
if x[0] == 'debug_pipeline':
lrecs.remove(x)
try:
dtitle = unicode_type(mi.title)
except:
dtitle = repr(mi.title)
if len(dtitle) > 50:
dtitle = dtitle[:50].rpartition(' ')[0] + '...'
self.setLabelText(_('Queueing ') + dtitle)
desc = _('Convert book %(num)d of %(tot)d (%(title)s)') % dict(
num=self.i, tot=len(self.book_ids), title=dtitle)
args = [in_file.name, out_file.name, lrecs]
temp_files.append(out_file)
func = 'gui_convert_override'
if same_fmt:
func += ':same_fmt'
self.jobs.append((func, args, desc, self.output_format.upper(),
book_id, temp_files))
self.changed = True
self.setValue(self.i)
except NoSupportedInputFormats:
self.bad.append(book_id)
QTimer.singleShot(0, self.do_book)
def convert_single_ebook(
parent,
db,
book_ids,
auto_conversion=False, # {{{
out_format=None,
show_no_format_warning=True):
changed = False
jobs = []
bad = []
total = len(book_ids)
if total == 0:
return None, None, None
for i, book_id in enumerate(book_ids):
temp_files = []
try:
d = SingleConfig(parent, db, book_id, None, out_format)
if auto_conversion:
d.accept()
result = QDialog.Accepted
else:
result = d.exec_()
if result == QDialog.Accepted:
# if not convert_existing(parent, db, [book_id], d.output_format):
# continue
mi = db.get_metadata(book_id, True)
in_file = PersistentTemporaryFile('.' + d.input_format)
with in_file:
input_fmt = db.original_fmt(book_id,
d.input_format).lower()
same_fmt = input_fmt == d.output_format.lower()
db.copy_format_to(book_id,
input_fmt,
in_file,
index_is_id=True)
out_file = PersistentTemporaryFile('.' + d.output_format)
out_file.write(as_bytes(d.output_format))
out_file.close()
temp_files = [in_file]
try:
dtitle = unicode_type(mi.title)
except:
dtitle = repr(mi.title)
desc = _('Convert book %(num)d of %(total)d (%(title)s)') % \
{'num':i + 1, 'total':total, 'title':dtitle}
recs = d.recommendations
if d.opf_file is not None:
recs.append(('read_metadata_from_opf', d.opf_file.name,
OptionRecommendation.HIGH))
temp_files.append(d.opf_file)
if d.cover_file is not None:
recs.append(('cover', d.cover_file.name,
OptionRecommendation.HIGH))
temp_files.append(d.cover_file)
args = [in_file.name, out_file.name, recs]
temp_files.append(out_file)
func = 'gui_convert_override'
parts = []
if not auto_conversion and d.manually_fine_tune_toc:
parts.append('manually_fine_tune_toc')
if same_fmt:
parts.append('same_fmt')
if parts:
func += ':%s' % (';'.join(parts))
jobs.append((func, args, desc, d.output_format.upper(),
book_id, temp_files))
changed = True
d.break_cycles()
except NoSupportedInputFormats as nsif:
bad.append((book_id, nsif.available_formats))
if bad and show_no_format_warning:
if len(bad) == 1 and not bad[0][1]:
title = db.title(bad[0][0], True)
warning_dialog(
parent,
_('Could not convert'),
'<p>' +
_('Could not convert <b>%s</b> as it has no e-book files. If you '
'think it should have files, but calibre is not finding '
'them, that is most likely because you moved the book\'s '
'files around outside of calibre. You will need to find those files '
'and re-add them to calibre.') % title,
show=True)
else:
res = []
for id, available_formats in bad:
title = db.title(id, True)
if available_formats:
msg = _('No supported formats (Available formats: %s)') % (
', '.join(available_formats))
else:
msg = _('This book has no actual e-book files')
res.append('%s - %s' % (title, msg))
msg = '%s' % '\n'.join(res)
warning_dialog(parent, _('Could not convert some books'), (_(
'Could not convert the book because no supported source format was found'
) if len(res) == 1 else _(
'Could not convert {num} of {tot} books, because no supported source formats were found.'
)).format(num=len(res), tot=total), msg).exec_()
return jobs, changed, bad
def safe_ord(x):
return ord_string(unicode_type(x))[0]
def accept(self):
n = unicode_type(self.vl_name.currentText()).strip()
if not n:
error_dialog(
self.gui,
_('No name'),
_('You must provide a name for the new Virtual library'),
show=True)
return
if n.startswith('*'):
error_dialog(self.gui,
_('Invalid name'),
_('A Virtual library name cannot begin with "*"'),
show=True)
return
if n in self.existing_names and n != self.editing:
if not question_dialog(
self.gui,
_('Name already in use'),
_('That name is already in use. Do you want to replace it '
'with the new search?'),
default_yes=False):
return
v = unicode_type(self.vl_text.text()).strip()
if not v:
error_dialog(
self.gui,
_('No search string'),
_('You must provide a search to define the new Virtual library'
),
show=True)
return
try:
db = self.gui.library_view.model().db
recs = db.data.search_getting_ids('',
v,
use_virtual_library=False,
sort_results=False)
except ParseException as e:
error_dialog(self.gui,
_('Invalid search'),
_('The search in the search box is not valid'),
det_msg=e.msg,
show=True)
return
if not recs and not question_dialog(
self.gui,
_('Search found no books'),
_('The search found no books, so the Virtual library '
'will be empty. Do you really want to use that search?'),
default_yes=False):
return
self.library_name = n
self.library_search = v
QDialog.accept(self)
