def do_save_book_to_disk(id_, mi, cover, plugboards, format_map, root, opts,
length):
from calibre.ebooks.metadata.meta import set_metadata
available_formats = [x.lower().strip() for x in format_map.keys()]
if mi.pubdate:
mi.pubdate = as_local_time(mi.pubdate)
if mi.timestamp:
mi.timestamp = as_local_time(mi.timestamp)
if opts.formats == 'all':
asked_formats = available_formats
else:
asked_formats = [x.lower().strip() for x in opts.formats.split(',')]
formats = set(available_formats).intersection(set(asked_formats))
if not formats:
return True, id_, mi.title
try:
components = get_components(
opts.template,
mi,
id_,
opts.timefmt,
length,
ascii_filename if opts.asciiize else sanitize_file_name_unicode,
to_lowercase=opts.to_lowercase,
replace_whitespace=opts.replace_whitespace,
safe_format=False)
except Exception, e:
raise ValueError(
_('Failed to calculate path for '
'save to disk. Template: %(templ)s\n'
'Error: %(err)s') % dict(templ=opts.template, err=e))
def do_save_book_to_disk(id_, mi, cover, plugboards,
format_map, root, opts, length):
from calibre.ebooks.metadata.meta import set_metadata
available_formats = [x.lower().strip() for x in format_map.keys()]
if mi.pubdate:
mi.pubdate = as_local_time(mi.pubdate)
if mi.timestamp:
mi.timestamp = as_local_time(mi.timestamp)
if opts.formats == 'all':
asked_formats = available_formats
else:
asked_formats = [x.lower().strip() for x in opts.formats.split(',')]
formats = set(available_formats).intersection(set(asked_formats))
if not formats:
return True, id_, mi.title
try:
components = get_components(opts.template, mi, id_, opts.timefmt, length,
ascii_filename if opts.asciiize else sanitize_file_name_unicode,
to_lowercase=opts.to_lowercase,
replace_whitespace=opts.replace_whitespace, safe_format=False)
except Exception, e:
raise ValueError(_('Failed to calculate path for '
'save to disk. Template: %(templ)s\n'
'Error: %(err)s')%dict(templ=opts.template, err=e))
def do_save_book_to_disk(db, book_id, mi, plugboards,
formats, root, opts, length):
originals = mi.cover, mi.pubdate, mi.timestamp
formats_written = False
try:
if mi.pubdate:
mi.pubdate = as_local_time(mi.pubdate)
if mi.timestamp:
mi.timestamp = as_local_time(mi.timestamp)
components = get_path_components(opts, mi, book_id, length)
base_path = os.path.join(root, *components)
base_name = os.path.basename(base_path)
dirpath = os.path.dirname(base_path)
try:
os.makedirs(dirpath)
except EnvironmentError as err:
if err.errno != errno.EEXIST:
raise
cdata = None
if opts.save_cover:
cdata = db.cover(book_id)
if cdata:
cpath = base_path + '.jpg'
with lopen(cpath, 'wb') as f:
f.write(cdata)
mi.cover = base_name+'.jpg'
if opts.write_opf:
from calibre.ebooks.metadata.opf2 import metadata_to_opf
opf = metadata_to_opf(mi)
with lopen(base_path+'.opf', 'wb') as f:
f.write(opf)
finally:
mi.cover, mi.pubdate, mi.timestamp = originals
if not formats:
return not formats_written, book_id, mi.title
for fmt in formats:
fmt_path = base_path+'.'+unicode_type(fmt)
try:
db.copy_format_to(book_id, fmt, fmt_path)
formats_written = True
except NoSuchFormat:
continue
if opts.update_metadata:
with lopen(fmt_path, 'r+b') as stream:
update_metadata(mi, fmt, stream, plugboards, cdata)
return not formats_written, book_id, mi.title
def do_save_book_to_disk(db, book_id, mi, plugboards,
formats, root, opts, length):
originals = mi.cover, mi.pubdate, mi.timestamp
formats_written = False
try:
if mi.pubdate:
mi.pubdate = as_local_time(mi.pubdate)
if mi.timestamp:
mi.timestamp = as_local_time(mi.timestamp)
components = get_path_components(opts, mi, book_id, length)
base_path = os.path.join(root, *components)
base_name = os.path.basename(base_path)
dirpath = os.path.dirname(base_path)
try:
os.makedirs(dirpath)
except EnvironmentError as err:
if err.errno != errno.EEXIST:
raise
cdata = None
if opts.save_cover:
cdata = db.cover(book_id)
if cdata:
cpath = base_path + '.jpg'
with lopen(cpath, 'wb') as f:
f.write(cdata)
mi.cover = base_name+'.jpg'
if opts.write_opf:
from calibre.ebooks.metadata.opf2 import metadata_to_opf
opf = metadata_to_opf(mi)
with lopen(base_path+'.opf', 'wb') as f:
f.write(opf)
finally:
mi.cover, mi.pubdate, mi.timestamp = originals
if not formats:
return not formats_written, book_id, mi.title
for fmt in formats:
fmt_path = base_path+'.'+str(fmt)
try:
db.copy_format_to(book_id, fmt, fmt_path)
formats_written = True
except NoSuchFormat:
continue
if opts.update_metadata:
with lopen(fmt_path, 'r+b') as stream:
update_metadata(mi, fmt, stream, plugboards, cdata)
return not formats_written, book_id, mi.title
def get_components(template, mi, id, timefmt='%b %Y', length=250,
sanitize_func=ascii_filename, replace_whitespace=False,
to_lowercase=False, safe_format=True, last_has_extension=True,
single_dir=False):
tsorder = tweaks['save_template_title_series_sorting']
format_args = FORMAT_ARGS.copy()
format_args.update(mi.all_non_none_fields())
if mi.title:
if tsorder == 'strictly_alphabetic':
v = mi.title
else:
# title_sort might be missing or empty. Check both conditions
v = mi.get('title_sort', None)
if not v:
v = title_sort(mi.title, order=tsorder)
format_args['title'] = v
if mi.authors:
format_args['authors'] = mi.format_authors()
format_args['author'] = format_args['authors']
if mi.tags:
format_args['tags'] = mi.format_tags()
if format_args['tags'].startswith('/'):
format_args['tags'] = format_args['tags'][1:]
else:
format_args['tags'] = ''
if mi.series:
format_args['series'] = title_sort(mi.series, order=tsorder)
if mi.series_index is not None:
format_args['series_index'] = mi.format_series_index()
else:
template = re.sub(r'\{series_index[^}]*?\}', '', template)
if mi.rating is not None:
format_args['rating'] = mi.format_rating(divide_by=2.0)
if mi.identifiers:
format_args['identifiers'] = mi.format_field_extended('identifiers')[1]
else:
format_args['identifiers'] = ''
if hasattr(mi.timestamp, 'timetuple'):
format_args['timestamp'] = strftime(timefmt, mi.timestamp.timetuple())
if hasattr(mi.pubdate, 'timetuple'):
format_args['pubdate'] = strftime(timefmt, mi.pubdate.timetuple())
if hasattr(mi, 'last_modified') and hasattr(mi.last_modified, 'timetuple'):
format_args['last_modified'] = strftime(timefmt, mi.last_modified.timetuple())
format_args['id'] = unicode_type(id)
# Now format the custom fields
custom_metadata = mi.get_all_user_metadata(make_copy=False)
for key in custom_metadata:
if key in format_args:
cm = custom_metadata[key]
if cm['datatype'] == 'series':
format_args[key] = title_sort(format_args[key], order=tsorder)
if key+'_index' in format_args:
format_args[key+'_index'] = fmt_sidx(format_args[key+'_index'])
elif cm['datatype'] == 'datetime':
format_args[key] = strftime(timefmt, as_local_time(format_args[key]).timetuple())
elif cm['datatype'] == 'bool':
format_args[key] = _('yes') if format_args[key] else _('no')
elif cm['datatype'] == 'rating':
format_args[key] = mi.format_rating(format_args[key],
divide_by=2.0)
elif cm['datatype'] in ['int', 'float']:
if format_args[key] != 0:
format_args[key] = unicode_type(format_args[key])
else:
format_args[key] = ''
if safe_format:
components = Formatter().safe_format(template, format_args,
'G_C-EXCEPTION!', mi)
else:
components = Formatter().unsafe_format(template, format_args, mi)
components = [x.strip() for x in components.split('/')]
components = [sanitize_func(x) for x in components if x]
if not components:
components = [unicode_type(id)]
if to_lowercase:
components = [x.lower() for x in components]
if replace_whitespace:
components = [re.sub(r'\s', '_', x) for x in components]
if single_dir:
components = components[-1:]
return shorten_components_to(length, components, last_has_extension=last_has_extension)
def do_save_book_to_disk(id_, mi, cover, plugboards,
format_map, root, opts, length):
available_formats = [x.lower().strip() for x in format_map.keys()]
if mi.pubdate:
mi.pubdate = as_local_time(mi.pubdate)
if mi.timestamp:
mi.timestamp = as_local_time(mi.timestamp)
if opts.formats == 'all':
asked_formats = available_formats
else:
asked_formats = [x.lower().strip() for x in opts.formats.split(',')]
formats = set(available_formats).intersection(set(asked_formats))
if not formats:
return True, id_, mi.title
components = get_path_components(opts, mi, id_, length)
base_path = os.path.join(root, *components)
base_name = os.path.basename(base_path)
dirpath = os.path.dirname(base_path)
# Don't test for existence first as the test could fail but
# another worker process could create the directory before
# the call to makedirs
try:
os.makedirs(dirpath)
except BaseException:
if not os.path.exists(dirpath):
raise
ocover = mi.cover
if opts.save_cover and cover:
with open(base_path+'.jpg', 'wb') as f:
f.write(cover)
mi.cover = base_name+'.jpg'
else:
mi.cover = None
if opts.write_opf:
from calibre.ebooks.metadata.opf2 import metadata_to_opf
opf = metadata_to_opf(mi)
with open(base_path+'.opf', 'wb') as f:
f.write(opf)
mi.cover = ocover
written = False
for fmt in formats:
fp = format_map.get(fmt, None)
if fp is None:
continue
stream = SpooledTemporaryFile(20*1024*1024, '_save_to_disk.'+(fmt or
'tmp'))
with open(fp, 'rb') as f:
shutil.copyfileobj(f, stream)
stream.seek(0)
written = True
if opts.update_metadata:
update_metadata(mi, fmt, stream, plugboards, cover)
stream.seek(0)
fmt_path = base_path+'.'+str(fmt)
with open(fmt_path, 'wb') as f:
shutil.copyfileobj(stream, f)
return not written, id_, mi.title
def get_components(template, mi, id, timefmt='%b %Y', length=250,
sanitize_func=ascii_filename, replace_whitespace=False,
to_lowercase=False, safe_format=True, last_has_extension=True,
single_dir=False):
tsorder = tweaks['save_template_title_series_sorting']
format_args = FORMAT_ARGS.copy()
format_args.update(mi.all_non_none_fields())
if mi.title:
if tsorder == 'strictly_alphabetic':
v = mi.title
else:
# title_sort might be missing or empty. Check both conditions
v = mi.get('title_sort', None)
if not v:
v = title_sort(mi.title, order=tsorder)
format_args['title'] = v
if mi.authors:
format_args['authors'] = mi.format_authors()
format_args['author'] = format_args['authors']
if mi.tags:
format_args['tags'] = mi.format_tags()
if format_args['tags'].startswith('/'):
format_args['tags'] = format_args['tags'][1:]
else:
format_args['tags'] = ''
if mi.series:
format_args['series'] = title_sort(mi.series, order=tsorder)
if mi.series_index is not None:
format_args['series_index'] = mi.format_series_index()
else:
template = re.sub(r'\{series_index[^}]*?\}', '', template)
if mi.rating is not None:
format_args['rating'] = mi.format_rating(divide_by=2.0)
if mi.identifiers:
format_args['identifiers'] = mi.format_field_extended('identifiers')[1]
else:
format_args['identifiers'] = ''
if hasattr(mi.timestamp, 'timetuple'):
format_args['timestamp'] = strftime(timefmt, mi.timestamp.timetuple())
if hasattr(mi.pubdate, 'timetuple'):
format_args['pubdate'] = strftime(timefmt, mi.pubdate.timetuple())
if hasattr(mi, 'last_modified') and hasattr(mi.last_modified, 'timetuple'):
format_args['last_modified'] = strftime(timefmt, mi.last_modified.timetuple())
format_args['id'] = str(id)
# Now format the custom fields
custom_metadata = mi.get_all_user_metadata(make_copy=False)
for key in custom_metadata:
if key in format_args:
cm = custom_metadata[key]
if cm['datatype'] == 'series':
format_args[key] = title_sort(format_args[key], order=tsorder)
if key+'_index' in format_args:
format_args[key+'_index'] = fmt_sidx(format_args[key+'_index'])
elif cm['datatype'] == 'datetime':
format_args[key] = strftime(timefmt, as_local_time(format_args[key]).timetuple())
elif cm['datatype'] == 'bool':
format_args[key] = _('yes') if format_args[key] else _('no')
elif cm['datatype'] == 'rating':
format_args[key] = mi.format_rating(format_args[key],
divide_by=2.0)
elif cm['datatype'] in ['int', 'float']:
if format_args[key] != 0:
format_args[key] = unicode(format_args[key])
else:
format_args[key] = ''
if safe_format:
components = Formatter().safe_format(template, format_args,
'G_C-EXCEPTION!', mi)
else:
components = Formatter().unsafe_format(template, format_args, mi)
components = [x.strip() for x in components.split('/')]
components = [sanitize_func(x) for x in components if x]
if not components:
components = [str(id)]
if to_lowercase:
components = [x.lower() for x in components]
if replace_whitespace:
components = [re.sub(r'\s', '_', x) for x in components]
if single_dir:
components = components[-1:]
return shorten_components_to(length, components, last_has_extension=last_has_extension)
def normalize_db_val(self, val):
return as_local_time(val) if val is not None else None
def render_jacket(mi,
output_profile,
alt_title=_('Unknown'),
alt_tags=[],
alt_comments='',
alt_publisher='',
rescale_fonts=False,
alt_authors=None):
css = P('jacket/stylesheet.css', data=True).decode('utf-8')
template = P('jacket/template.xhtml', data=True).decode('utf-8')
template = re.sub(r'<!--.*?-->', '', template, flags=re.DOTALL)
css = re.sub(r'/\*.*?\*/', '', css, flags=re.DOTALL)
try:
title_str = alt_title if mi.is_null('title') else mi.title
except:
title_str = _('Unknown')
title_str = escape(title_str)
title = '<span class="title">%s</span>' % title_str
series = Series(mi.series, mi.series_index)
try:
publisher = mi.publisher if not mi.is_null(
'publisher') else alt_publisher
except:
publisher = ''
publisher = escape(publisher)
try:
if is_date_undefined(mi.pubdate):
pubdate = ''
else:
dt = as_local_time(mi.pubdate)
pubdate = strftime('%Y', dt.timetuple())
except:
pubdate = ''
rating = get_rating(mi.rating, output_profile.ratings_char,
output_profile.empty_ratings_char)
tags = Tags((mi.tags if mi.tags else alt_tags), output_profile)
comments = mi.comments if mi.comments else alt_comments
comments = comments.strip()
if comments:
comments = comments_to_html(comments)
orig = mi.authors
if mi.is_null('authors'):
mi.authors = list(alt_authors or (_('Unknown'), ))
try:
author = mi.format_authors()
except:
author = ''
mi.authors = orig
author = escape(author)
has_data = {}
def generate_html(comments):
display = Attributes()
args = dict(
xmlns=XHTML_NS,
title_str=title_str,
identifiers=Identifiers(mi.identifiers),
css=css,
title=title,
author=author,
publisher=publisher,
pubdate_label=_('Published'),
pubdate=pubdate,
series_label=ngettext('Series', 'Series', 1),
series=series,
rating_label=_('Rating'),
rating=rating,
tags_label=_('Tags'),
tags=tags,
comments=comments,
footer='',
display=display,
searchable_tags=' '.join(
escape(t) + 'ttt' for t in tags.tags_list),
)
for key in mi.custom_field_keys():
m = mi.get_user_metadata(key, False) or {}
try:
display_name, val = mi.format_field_extended(key)[:2]
dkey = key.replace('#', '_')
dt = m.get('datatype')
if dt == 'series':
args[dkey] = Series(mi.get(key), mi.get(key + '_index'))
elif dt == 'rating':
args[dkey] = rating_to_stars(
mi.get(key),
m.get('display', {}).get('allow_half_stars', False))
elif dt == 'comments':
val = val or ''
ctype = m.get('display', {}).get('interpret_as') or 'html'
if ctype == 'long-text':
val = '<pre style="white-space:pre-wrap">%s</pre>' % escape(
val)
elif ctype == 'short-text':
val = '<span>%s</span>' % escape(val)
elif ctype == 'markdown':
val = markdown(val)
else:
val = comments_to_html(val)
args[dkey] = val
else:
args[dkey] = escape(val)
args[dkey + '_label'] = escape(display_name)
setattr(display, dkey,
'none' if mi.is_null(key) else 'initial')
except Exception:
# if the val (custom column contents) is None, don't add to args
pass
if False:
print("Custom column values available in jacket template:")
for key in args.keys():
if key.startswith('_') and not key.endswith('_label'):
print(" {}: {}".format('#' + key[1:], args[key]))
# Used in the comment describing use of custom columns in templates
# Don't change this unless you also change it in template.xhtml
args['_genre_label'] = args.get('_genre_label', '{_genre_label}')
args['_genre'] = args.get('_genre', '{_genre}')
has_data['series'] = bool(series)
has_data['tags'] = bool(tags)
has_data['rating'] = bool(rating)
has_data['pubdate'] = bool(pubdate)
for k, v in has_data.items():
setattr(display, k, 'initial' if v else 'none')
display.title = 'initial'
if mi.identifiers:
display.identifiers = 'initial'
formatter = SafeFormatter()
generated_html = formatter.format(template, **args)
return strip_encoding_declarations(generated_html)
from calibre.ebooks.oeb.polish.parsing import parse
raw = generate_html(comments)
root = parse(raw, line_numbers=False, force_html5_parse=True)
if rescale_fonts:
# We ensure that the conversion pipeline will set the font sizes for
# text in the jacket to the same size as the font sizes for the rest of
# the text in the book. That means that as long as the jacket uses
# relative font sizes (em or %), the post conversion font size will be
# the same as for text in the main book. So text with size x em will
# be rescaled to the same value in both the jacket and the main content.
#
# We cannot use data-calibre-rescale 100 on the body tag as that will just
# give the body tag a font size of 1em, which is useless.
for body in root.xpath('//*[local-name()="body"]'):
fw = body.makeelement(XHTML('div'))
fw.set('data-calibre-rescale', '100')
for child in body:
fw.append(child)
body.append(fw)
postprocess_jacket(root, output_profile, has_data)
from calibre.ebooks.oeb.polish.pretty import pretty_html_tree
pretty_html_tree(None, root)
return root
def render_jacket(mi,
output_profile,
alt_title=_('Unknown'),
alt_tags=[],
alt_comments='',
alt_publisher=(''),
rescale_fonts=False):
css = P('jacket/stylesheet.css', data=True).decode('utf-8')
template = P('jacket/template.xhtml', data=True).decode('utf-8')
template = re.sub(r'<!--.*?-->', '', template, flags=re.DOTALL)
css = re.sub(r'/\*.*?\*/', '', css, flags=re.DOTALL)
try:
title_str = mi.title if mi.title else alt_title
except:
title_str = _('Unknown')
title_str = escape(title_str)
title = '<span class="title">%s</span>' % title_str
series = Series(mi.series, mi.series_index)
try:
publisher = mi.publisher if mi.publisher else alt_publisher
except:
publisher = ''
publisher = escape(publisher)
try:
if is_date_undefined(mi.pubdate):
pubdate = ''
else:
dt = as_local_time(mi.pubdate)
pubdate = strftime(u'%Y', dt.timetuple())
except:
pubdate = ''
rating = get_rating(mi.rating, output_profile.ratings_char,
output_profile.empty_ratings_char)
tags = Tags((mi.tags if mi.tags else alt_tags), output_profile)
comments = mi.comments if mi.comments else alt_comments
comments = comments.strip()
orig_comments = comments
if comments:
comments = comments_to_html(comments)
try:
author = mi.format_authors()
except:
author = ''
author = escape(author)
def generate_html(comments):
args = dict(
xmlns=XHTML_NS,
title_str=title_str,
css=css,
title=title,
author=author,
publisher=publisher,
pubdate_label=_('Published'),
pubdate=pubdate,
series_label=_('Series'),
series=series,
rating_label=_('Rating'),
rating=rating,
tags_label=_('Tags'),
tags=tags,
comments=comments,
footer='',
searchable_tags=' '.join(
escape(t) + 'ttt' for t in tags.tags_list),
)
for key in mi.custom_field_keys():
m = mi.get_user_metadata(key, False) or {}
try:
display_name, val = mi.format_field_extended(key)[:2]
dkey = key.replace('#', '_')
dt = m.get('datatype')
if dt == 'series':
args[dkey] = Series(mi.get(key), mi.get(key + '_index'))
elif dt == 'rating':
args[dkey] = rating_to_stars(
mi.get(key),
m.get('display', {}).get('allow_half_stars', False))
else:
args[dkey] = escape(val)
args[dkey + '_label'] = escape(display_name)
except Exception:
# if the val (custom column contents) is None, don't add to args
pass
if False:
print("Custom column values available in jacket template:")
for key in args.keys():
if key.startswith('_') and not key.endswith('_label'):
print(" %s: %s" % ('#' + key[1:], args[key]))
# Used in the comment describing use of custom columns in templates
# Don't change this unless you also change it in template.xhtml
args['_genre_label'] = args.get('_genre_label', '{_genre_label}')
args['_genre'] = args.get('_genre', '{_genre}')
formatter = SafeFormatter()
generated_html = formatter.format(template, **args)
# Post-process the generated html to strip out empty header items
soup = BeautifulSoup(generated_html)
if not series:
series_tag = soup.find(attrs={'class': 'cbj_series'})
if series_tag is not None:
series_tag.extract()
if not rating:
rating_tag = soup.find(attrs={'class': 'cbj_rating'})
if rating_tag is not None:
rating_tag.extract()
if not tags:
tags_tag = soup.find(attrs={'class': 'cbj_tags'})
if tags_tag is not None:
tags_tag.extract()
if not pubdate:
pubdate_tag = soup.find(attrs={'class': 'cbj_pubdata'})
if pubdate_tag is not None:
pubdate_tag.extract()
if output_profile.short_name != 'kindle':
hr_tag = soup.find('hr', attrs={'class': 'cbj_kindle_banner_hr'})
if hr_tag is not None:
hr_tag.extract()
return strip_encoding_declarations(
soup.renderContents('utf-8').decode('utf-8'))
from calibre.ebooks.oeb.base import RECOVER_PARSER
try:
root = etree.fromstring(generate_html(comments), parser=RECOVER_PARSER)
except:
try:
root = etree.fromstring(generate_html(escape(orig_comments)),
parser=RECOVER_PARSER)
except:
root = etree.fromstring(generate_html(''), parser=RECOVER_PARSER)
if rescale_fonts:
# We ensure that the conversion pipeline will set the font sizes for
# text in the jacket to the same size as the font sizes for the rest of
# the text in the book. That means that as long as the jacket uses
# relative font sizes (em or %), the post conversion font size will be
# the same as for text in the main book. So text with size x em will
# be rescaled to the same value in both the jacket and the main content.
#
# We cannot use calibre_rescale_100 on the body tag as that will just
# give the body tag a font size of 1em, which is useless.
for body in root.xpath('//*[local-name()="body"]'):
fw = body.makeelement(XHTML('div'))
fw.set('class', 'calibre_rescale_100')
for child in body:
fw.append(child)
body.append(fw)
from calibre.ebooks.oeb.polish.pretty import pretty_html_tree
pretty_html_tree(None, root)
return root
def mobile(self, start="1", num="25", sort="date", search="", _=None, order="descending"):
"""
Serves metadata from the calibre database as XML.
:param sort: Sort results by ``sort``. Can be one of `title,author,rating`.
:param search: Filter results by ``search`` query. See :class:`SearchQueryParser` for query syntax
:param start,num: Return the slice `[start:start+num]` of the sorted and filtered results
:param _: Firefox seems to sometimes send this when using XMLHttpRequest with no caching
"""
try:
start = int(start)
except ValueError:
raise cherrypy.HTTPError(400, "start: %s is not an integer" % start)
try:
num = int(num)
except ValueError:
raise cherrypy.HTTPError(400, "num: %s is not an integer" % num)
if not search:
search = ""
if isbytestring(search):
search = search.decode("UTF-8")
ids = self.search_for_books(search)
FM = self.db.FIELD_MAP
items = [r for r in iter(self.db) if r[FM["id"]] in ids]
if sort is not None:
self.sort(items, sort, (order.lower().strip() == "ascending"))
CFM = self.db.field_metadata
CKEYS = [key for key in sorted(custom_fields_to_display(self.db), key=lambda x: sort_key(CFM[x]["name"]))]
# This method uses its own book dict, not the Metadata dict. The loop
# below could be changed to use db.get_metadata instead of reading
# info directly from the record made by the view, but it doesn't seem
# worth it at the moment.
books = []
for record in items[(start - 1) : (start - 1) + num]:
book = {"formats": record[FM["formats"]], "size": record[FM["size"]]}
if not book["formats"]:
book["formats"] = ""
if not book["size"]:
book["size"] = 0
book["size"] = human_readable(book["size"])
aus = record[FM["authors"]] if record[FM["authors"]] else __builtin__._("Unknown")
aut_is = CFM["authors"]["is_multiple"]
authors = aut_is["list_to_ui"].join([i.replace("|", ",") for i in aus.split(",")])
book["authors"] = authors
book["series_index"] = fmt_sidx(float(record[FM["series_index"]]))
book["series"] = record[FM["series"]]
book["tags"] = format_tag_string(record[FM["tags"]], ",", no_tag_count=True)
book["title"] = record[FM["title"]]
for x in ("timestamp", "pubdate"):
book[x] = strftime("%d %b, %Y", as_local_time(record[FM[x]]))
book["id"] = record[FM["id"]]
books.append(book)
for key in CKEYS:
def concat(name, val):
return "%s:#:%s" % (name, unicode(val))
mi = self.db.get_metadata(record[CFM["id"]["rec_index"]], index_is_id=True)
name, val = mi.format_field(key)
if not val:
continue
datatype = CFM[key]["datatype"]
if datatype in ["comments"]:
continue
if datatype == "text" and CFM[key]["is_multiple"]:
book[key] = concat(
name,
format_tag_string(
val,
CFM[key]["is_multiple"]["ui_to_list"],
no_tag_count=True,
joinval=CFM[key]["is_multiple"]["list_to_ui"],
),
)
else:
book[key] = concat(name, val)
updated = self.db.last_modified()
cherrypy.response.headers["Content-Type"] = "text/html; charset=utf-8"
cherrypy.response.headers["Last-Modified"] = self.last_modified(updated)
q = {
b"search": search.encode("utf-8"),
b"order": order.encode("utf-8"),
b"sort": sort.encode("utf-8"),
b"num": str(num).encode("utf-8"),
}
url_base = "/mobile?" + urlencode(q)
ua = cherrypy.request.headers.get("User-Agent", "").strip()
have_kobo_browser = self.is_kobo_browser(ua)
raw = html.tostring(
build_index(
books,
num,
search,
sort,
order,
start,
len(ids),
url_base,
CKEYS,
self.opts.url_prefix,
have_kobo_browser=have_kobo_browser,
),
encoding="utf-8",
pretty_print=True,
)
# tostring's include_meta_content_type is broken
raw = raw.replace("<head>", "<head>\n" '<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">')
return raw
def render_jacket(mi, output_profile,
alt_title=_('Unknown'), alt_tags=[], alt_comments='',
alt_publisher='', rescale_fonts=False, alt_authors=None):
css = P('jacket/stylesheet.css', data=True).decode('utf-8')
template = P('jacket/template.xhtml', data=True).decode('utf-8')
template = re.sub(r'<!--.*?-->', '', template, flags=re.DOTALL)
css = re.sub(r'/\*.*?\*/', '', css, flags=re.DOTALL)
try:
title_str = alt_title if mi.is_null('title') else mi.title
except:
title_str = _('Unknown')
title_str = escape(title_str)
title = '<span class="title">%s</span>' % title_str
series = Series(mi.series, mi.series_index)
try:
publisher = mi.publisher if not mi.is_null('publisher') else alt_publisher
except:
publisher = ''
publisher = escape(publisher)
try:
if is_date_undefined(mi.pubdate):
pubdate = ''
else:
dt = as_local_time(mi.pubdate)
pubdate = strftime(u'%Y', dt.timetuple())
except:
pubdate = ''
rating = get_rating(mi.rating, output_profile.ratings_char, output_profile.empty_ratings_char)
tags = Tags((mi.tags if mi.tags else alt_tags), output_profile)
comments = mi.comments if mi.comments else alt_comments
comments = comments.strip()
orig_comments = comments
if comments:
comments = comments_to_html(comments)
orig = mi.authors
if mi.is_null('authors'):
mi.authors = list(alt_authors or (_('Unknown'),))
try:
author = mi.format_authors()
except:
author = ''
mi.authors = orig
author = escape(author)
has_data = {}
def generate_html(comments):
args = dict(xmlns=XHTML_NS,
title_str=title_str,
css=css,
title=title,
author=author,
publisher=publisher,
pubdate_label=_('Published'), pubdate=pubdate,
series_label=_('Series'), series=series,
rating_label=_('Rating'), rating=rating,
tags_label=_('Tags'), tags=tags,
comments=comments,
footer='',
searchable_tags=' '.join(escape(t)+'ttt' for t in tags.tags_list),
)
for key in mi.custom_field_keys():
m = mi.get_user_metadata(key, False) or {}
try:
display_name, val = mi.format_field_extended(key)[:2]
dkey = key.replace('#', '_')
dt = m.get('datatype')
if dt == 'series':
args[dkey] = Series(mi.get(key), mi.get(key + '_index'))
elif dt == 'rating':
args[dkey] = rating_to_stars(mi.get(key), m.get('display', {}).get('allow_half_stars', False))
elif dt == 'comments':
val = val or ''
display = m.get('display', {})
ctype = display.get('interpret_as') or 'html'
if ctype == 'long-text':
val = '<pre style="white-space:pre-wrap">%s</pre>' % escape(val)
elif ctype == 'short-text':
val = '<span>%s</span>' % escape(val)
elif ctype == 'markdown':
val = markdown(val)
else:
val = comments_to_html(val)
args[dkey] = val
else:
args[dkey] = escape(val)
args[dkey+'_label'] = escape(display_name)
except Exception:
# if the val (custom column contents) is None, don't add to args
pass
if False:
print("Custom column values available in jacket template:")
for key in args.keys():
if key.startswith('_') and not key.endswith('_label'):
print(" %s: %s" % ('#' + key[1:], args[key]))
# Used in the comment describing use of custom columns in templates
# Don't change this unless you also change it in template.xhtml
args['_genre_label'] = args.get('_genre_label', '{_genre_label}')
args['_genre'] = args.get('_genre', '{_genre}')
formatter = SafeFormatter()
generated_html = formatter.format(template, **args)
has_data['series'] = bool(series)
has_data['tags'] = bool(tags)
has_data['rating'] = bool(rating)
has_data['pubdate'] = bool(pubdate)
return strip_encoding_declarations(generated_html)
from calibre.ebooks.oeb.base import RECOVER_PARSER
try:
root = etree.fromstring(generate_html(comments), parser=RECOVER_PARSER)
except:
try:
root = etree.fromstring(generate_html(escape(orig_comments)),
parser=RECOVER_PARSER)
except:
root = etree.fromstring(generate_html(''),
parser=RECOVER_PARSER)
if rescale_fonts:
# We ensure that the conversion pipeline will set the font sizes for
# text in the jacket to the same size as the font sizes for the rest of
# the text in the book. That means that as long as the jacket uses
# relative font sizes (em or %), the post conversion font size will be
# the same as for text in the main book. So text with size x em will
# be rescaled to the same value in both the jacket and the main content.
#
# We cannot use calibre_rescale_100 on the body tag as that will just
# give the body tag a font size of 1em, which is useless.
for body in root.xpath('//*[local-name()="body"]'):
fw = body.makeelement(XHTML('div'))
fw.set('class', 'calibre_rescale_100')
for child in body:
fw.append(child)
body.append(fw)
postprocess_jacket(root, output_profile, has_data)
from calibre.ebooks.oeb.polish.pretty import pretty_html_tree
pretty_html_tree(None, root)
return root
def mobile(self,
start='1',
num='25',
sort='date',
search='',
_=None,
order='descending'):
'''
Serves metadata from the calibre database as XML.
:param sort: Sort results by ``sort``. Can be one of `title,author,rating`.
:param search: Filter results by ``search`` query. See :class:`SearchQueryParser` for query syntax
:param start,num: Return the slice `[start:start+num]` of the sorted and filtered results
:param _: Firefox seems to sometimes send this when using XMLHttpRequest with no caching
'''
try:
start = int(start)
except ValueError:
raise cherrypy.HTTPError(400,
'start: %s is not an integer' % start)
try:
num = int(num)
except ValueError:
raise cherrypy.HTTPError(400, 'num: %s is not an integer' % num)
if not search:
search = ''
if isbytestring(search):
search = search.decode('UTF-8')
ids = self.search_for_books(search)
FM = self.db.FIELD_MAP
items = [r for r in iter(self.db) if r[FM['id']] in ids]
if sort is not None:
self.sort(items, sort, (order.lower().strip() == 'ascending'))
CFM = self.db.field_metadata
CKEYS = [
key for key in sorted(custom_fields_to_display(self.db),
key=lambda x: sort_key(CFM[x]['name']))
]
# This method uses its own book dict, not the Metadata dict. The loop
# below could be changed to use db.get_metadata instead of reading
# info directly from the record made by the view, but it doesn't seem
# worth it at the moment.
books = []
for record in items[(start - 1):(start - 1) + num]:
book = {
'formats': record[FM['formats']],
'size': record[FM['size']]
}
if not book['formats']:
book['formats'] = ''
if not book['size']:
book['size'] = 0
book['size'] = human_readable(book['size'])
aus = record[FM['authors']] if record[
FM['authors']] else __builtin__._('Unknown')
aut_is = CFM['authors']['is_multiple']
authors = aut_is['list_to_ui'].join(
[i.replace('|', ',') for i in aus.split(',')])
book['authors'] = authors
book['series_index'] = fmt_sidx(float(record[FM['series_index']]))
book['series'] = record[FM['series']]
book['tags'] = format_tag_string(record[FM['tags']],
',',
no_tag_count=True)
book['title'] = record[FM['title']]
for x in ('timestamp', 'pubdate'):
book[x] = strftime('%d %b, %Y', as_local_time(record[FM[x]]))
book['id'] = record[FM['id']]
books.append(book)
for key in CKEYS:
def concat(name, val):
return '%s:#:%s' % (name, unicode(val))
mi = self.db.get_metadata(record[CFM['id']['rec_index']],
index_is_id=True)
name, val = mi.format_field(key)
if not val:
continue
datatype = CFM[key]['datatype']
if datatype in ['comments']:
continue
if datatype == 'text' and CFM[key]['is_multiple']:
book[key] = concat(
name,
format_tag_string(
val,
CFM[key]['is_multiple']['ui_to_list'],
no_tag_count=True,
joinval=CFM[key]['is_multiple']['list_to_ui']))
else:
book[key] = concat(name, val)
updated = self.db.last_modified()
cherrypy.response.headers['Content-Type'] = 'text/html; charset=utf-8'
cherrypy.response.headers['Last-Modified'] = self.last_modified(
updated)
url_base = "/mobile?search=" + search + ";order=" + order + ";sort=" + sort + ";num=" + str(
num)
ua = cherrypy.request.headers.get('User-Agent', '').strip()
have_kobo_browser = self.is_kobo_browser(ua)
raw = html.tostring(build_index(books,
num,
search,
sort,
order,
start,
len(ids),
url_base,
CKEYS,
self.opts.url_prefix,
have_kobo_browser=have_kobo_browser),
encoding='utf-8',
pretty_print=True)
# tostring's include_meta_content_type is broken
raw = raw.replace(
'<head>', '<head>\n'
'<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">'
)
return raw
def mobile(self, start='1', num='25', sort='date', search='',
_=None, order='descending'):
'''
Serves metadata from the calibre database as XML.
:param sort: Sort results by ``sort``. Can be one of `title,author,rating`.
:param search: Filter results by ``search`` query. See :class:`SearchQueryParser` for query syntax
:param start,num: Return the slice `[start:start+num]` of the sorted and filtered results
:param _: Firefox seems to sometimes send this when using XMLHttpRequest with no caching
'''
try:
start = int(start)
except ValueError:
raise cherrypy.HTTPError(400, 'start: %s is not an integer'%start)
try:
num = int(num)
except ValueError:
raise cherrypy.HTTPError(400, 'num: %s is not an integer'%num)
if not search:
search = ''
if isbytestring(search):
search = search.decode('UTF-8')
ids = self.db.search_getting_ids(search.strip(), self.search_restriction)
FM = self.db.FIELD_MAP
items = [r for r in iter(self.db) if r[FM['id']] in ids]
if sort is not None:
self.sort(items, sort, (order.lower().strip() == 'ascending'))
CFM = self.db.field_metadata
CKEYS = [key for key in sorted(custom_fields_to_display(self.db),
key=lambda x:sort_key(CFM[x]['name']))]
# This method uses its own book dict, not the Metadata dict. The loop
# below could be changed to use db.get_metadata instead of reading
# info directly from the record made by the view, but it doesn't seem
# worth it at the moment.
books = []
for record in items[(start-1):(start-1)+num]:
book = {'formats':record[FM['formats']], 'size':record[FM['size']]}
if not book['formats']:
book['formats'] = ''
if not book['size']:
book['size'] = 0
book['size'] = human_readable(book['size'])
aus = record[FM['authors']] if record[FM['authors']] else __builtin__._('Unknown')
aut_is = CFM['authors']['is_multiple']
authors = aut_is['list_to_ui'].join([i.replace('|', ',') for i in aus.split(',')])
book['authors'] = authors
book['series_index'] = fmt_sidx(float(record[FM['series_index']]))
book['series'] = record[FM['series']]
book['tags'] = format_tag_string(record[FM['tags']], ',',
no_tag_count=True)
book['title'] = record[FM['title']]
for x in ('timestamp', 'pubdate'):
book[x] = strftime('%d %b, %Y', as_local_time(record[FM[x]]))
book['id'] = record[FM['id']]
books.append(book)
for key in CKEYS:
def concat(name, val):
return '%s:#:%s'%(name, unicode(val))
mi = self.db.get_metadata(record[CFM['id']['rec_index']], index_is_id=True)
name, val = mi.format_field(key)
if not val:
continue
datatype = CFM[key]['datatype']
if datatype in ['comments']:
continue
if datatype == 'text' and CFM[key]['is_multiple']:
book[key] = concat(name,
format_tag_string(val,
CFM[key]['is_multiple']['ui_to_list'],
no_tag_count=True,
joinval=CFM[key]['is_multiple']['list_to_ui']))
else:
book[key] = concat(name, val)
updated = self.db.last_modified()
cherrypy.response.headers['Content-Type'] = 'text/html; charset=utf-8'
cherrypy.response.headers['Last-Modified'] = self.last_modified(updated)
url_base = "/mobile?search=" + search+";order="+order+";sort="+sort+";num="+str(num)
raw = html.tostring(build_index(books, num, search, sort, order,
start, len(ids), url_base, CKEYS,
self.opts.url_prefix),
encoding='utf-8',
pretty_print=True)
# tostring's include_meta_content_type is broken
raw = raw.replace('<head>', '<head>\n'
'<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">')
return raw
def __init__(self, dt, render_template):
self.dt = as_local_time(dt)
self.is_date_undefined = dt is None or is_date_undefined(dt)
self.default_render = '' if self.is_date_undefined else escape(format_date(self.dt, render_template))
def get_data_as_dict(self,
prefix=None,
authors_as_string=False,
ids=None,
convert_to_local_tz=True):
'''
Return all metadata stored in the database as a dict. Includes paths to
the cover and each format.
:param prefix: The prefix for all paths. By default, the prefix is the absolute path
to the library folder.
:param ids: Set of ids to return the data for. If None return data for
all entries in database.
'''
import os
from calibre.ebooks.metadata import authors_to_string
from calibre.utils.date import as_local_time
backend = getattr(self, 'backend',
self) # Works with both old and legacy interfaces
if prefix is None:
prefix = backend.library_path
fdata = backend.custom_column_num_map
FIELDS = {
'title', 'sort', 'authors', 'author_sort', 'publisher', 'rating',
'timestamp', 'size', 'tags', 'comments', 'series', 'series_index',
'uuid', 'pubdate', 'last_modified', 'identifiers', 'languages'
}.union(set(fdata))
for x, data in iteritems(fdata):
if data['datatype'] == 'series':
FIELDS.add('%d_index' % x)
data = []
for record in self.data:
if record is None:
continue
db_id = record[self.FIELD_MAP['id']]
if ids is not None and db_id not in ids:
continue
x = {}
for field in FIELDS:
x[field] = record[self.FIELD_MAP[field]]
if convert_to_local_tz:
for tf in ('timestamp', 'pubdate', 'last_modified'):
x[tf] = as_local_time(x[tf])
data.append(x)
x['id'] = db_id
x['formats'] = []
isbn = self.isbn(db_id, index_is_id=True)
x['isbn'] = isbn if isbn else ''
if not x['authors']:
x['authors'] = _('Unknown')
x['authors'] = [i.replace('|', ',') for i in x['authors'].split(',')]
if authors_as_string:
x['authors'] = authors_to_string(x['authors'])
x['tags'] = [
i.replace('|', ',').strip() for i in x['tags'].split(',')
] if x['tags'] else []
path = os.path.join(
prefix, self.path(record[self.FIELD_MAP['id']], index_is_id=True))
x['cover'] = os.path.join(path, 'cover.jpg')
if not record[self.FIELD_MAP['cover']]:
x['cover'] = None
formats = self.formats(record[self.FIELD_MAP['id']], index_is_id=True)
if formats:
for fmt in formats.split(','):
path = self.format_abspath(x['id'], fmt, index_is_id=True)
if path is None:
continue
if prefix != self.library_path:
path = os.path.relpath(path, self.library_path)
path = os.path.join(prefix, path)
x['formats'].append(path)
x['fmt_' + fmt.lower()] = path
x['available_formats'] = [i.upper() for i in formats.split(',')]
return data
def fset(self, val):
if val is None:
val = UNDEFINED_DATE
else:
val = as_local_time(val)
self.setDateTime(val)
def do_save_book_to_disk(id_, mi, cover, plugboards, format_map, root, opts,
length):
available_formats = [x.lower().strip() for x in format_map.keys()]
if mi.pubdate:
mi.pubdate = as_local_time(mi.pubdate)
if mi.timestamp:
mi.timestamp = as_local_time(mi.timestamp)
if opts.formats == 'all':
asked_formats = available_formats
else:
asked_formats = [x.lower().strip() for x in opts.formats.split(',')]
formats = set(available_formats).intersection(set(asked_formats))
if not formats:
return True, id_, mi.title
components = get_path_components(opts, mi, id_, length)
base_path = os.path.join(root, *components)
base_name = os.path.basename(base_path)
dirpath = os.path.dirname(base_path)
# Don't test for existence first as the test could fail but
# another worker process could create the directory before
# the call to makedirs
try:
os.makedirs(dirpath)
except BaseException:
if not os.path.exists(dirpath):
raise
ocover = mi.cover
if opts.save_cover and cover:
with open(base_path + '.jpg', 'wb') as f:
f.write(cover)
mi.cover = base_name + '.jpg'
else:
mi.cover = None
if opts.write_opf:
from calibre.ebooks.metadata.opf2 import metadata_to_opf
opf = metadata_to_opf(mi)
with open(base_path + '.opf', 'wb') as f:
f.write(opf)
mi.cover = ocover
written = False
for fmt in formats:
fp = format_map.get(fmt, None)
if fp is None:
continue
stream = SpooledTemporaryFile(20 * 1024 * 1024,
'_save_to_disk.' + (fmt or 'tmp'))
with open(fp, 'rb') as f:
shutil.copyfileobj(f, stream)
stream.seek(0)
written = True
if opts.update_metadata:
update_metadata(mi, fmt, stream, plugboards, cover)
stream.seek(0)
fmt_path = base_path + '.' + str(fmt)
with open(fmt_path, 'wb') as f:
shutil.copyfileobj(stream, f)
return not written, id_, mi.title
def get_data_as_dict(self, prefix=None, authors_as_string=False, ids=None, convert_to_local_tz=True):
'''
Return all metadata stored in the database as a dict. Includes paths to
the cover and each format.
:param prefix: The prefix for all paths. By default, the prefix is the absolute path
to the library folder.
:param ids: Set of ids to return the data for. If None return data for
all entries in database.
'''
import os
from calibre.ebooks.metadata import authors_to_string
from calibre.utils.date import as_local_time
backend = getattr(self, 'backend', self) # Works with both old and legacy interfaces
if prefix is None:
prefix = backend.library_path
fdata = backend.custom_column_num_map
FIELDS = set(['title', 'sort', 'authors', 'author_sort', 'publisher',
'rating', 'timestamp', 'size', 'tags', 'comments', 'series',
'series_index', 'uuid', 'pubdate', 'last_modified', 'identifiers',
'languages']).union(set(fdata))
for x, data in fdata.iteritems():
if data['datatype'] == 'series':
FIELDS.add('%d_index'%x)
data = []
for record in self.data:
if record is None:
continue
db_id = record[self.FIELD_MAP['id']]
if ids is not None and db_id not in ids:
continue
x = {}
for field in FIELDS:
x[field] = record[self.FIELD_MAP[field]]
if convert_to_local_tz:
for tf in ('timestamp', 'pubdate', 'last_modified'):
x[tf] = as_local_time(x[tf])
data.append(x)
x['id'] = db_id
x['formats'] = []
isbn = self.isbn(db_id, index_is_id=True)
x['isbn'] = isbn if isbn else ''
if not x['authors']:
x['authors'] = _('Unknown')
x['authors'] = [i.replace('|', ',') for i in x['authors'].split(',')]
if authors_as_string:
x['authors'] = authors_to_string(x['authors'])
x['tags'] = [i.replace('|', ',').strip() for i in x['tags'].split(',')] if x['tags'] else []
path = os.path.join(prefix, self.path(record[self.FIELD_MAP['id']], index_is_id=True))
x['cover'] = os.path.join(path, 'cover.jpg')
if not record[self.FIELD_MAP['cover']]:
x['cover'] = None
formats = self.formats(record[self.FIELD_MAP['id']], index_is_id=True)
if formats:
for fmt in formats.split(','):
path = self.format_abspath(x['id'], fmt, index_is_id=True)
if path is None:
continue
if prefix != self.library_path:
path = os.path.relpath(path, self.library_path)
path = os.path.join(prefix, path)
x['formats'].append(path)
x['fmt_'+fmt.lower()] = path
x['available_formats'] = [i.upper() for i in formats.split(',')]
return data
def fset(self, val):
if val is None:
val = UNDEFINED_DATE
else:
val = as_local_time(val)
self.setDateTime(val)
