def strftime(fmt='%Y/%m/%d %H:%M:%S', dt=None):
if not hasattr(dt, 'timetuple'):
dt = nowf()
dt = dt.timetuple()
try:
return _strftime(fmt, dt)
except:
return _strftime(fmt, nowf().timetuple())
def strftime(fmt='%Y/%m/%d %H:%M:%S', dt=None):
if not hasattr(dt, 'timetuple'):
dt = nowf()
dt = dt.timetuple()
try:
return _strftime(fmt, dt)
except:
return _strftime(fmt, nowf().timetuple())
def recipe_needs_to_be_downloaded(self, recipe):
try:
typ, sch, ld = self.un_serialize_schedule(recipe)
except:
return False
def is_time(now, hour, minute):
return now.hour > hour or \
(now.hour == hour and now.minute >= minute)
def is_weekday(day, now):
return day < 0 or day > 6 or \
day == calendar.weekday(now.year, now.month, now.day)
def was_downloaded_already_today(ld_local, now):
return ld_local.date() == now.date()
if typ == 'interval':
return utcnow() - ld > timedelta(sch)
elif typ == 'day/time':
now = nowf()
try:
ld_local = ld.astimezone(local_tz)
except Exception:
return False
day, hour, minute = sch
return is_weekday(day, now) and \
not was_downloaded_already_today(ld_local, now) and \
is_time(now, hour, minute)
elif typ == 'days_of_week':
now = nowf()
try:
ld_local = ld.astimezone(local_tz)
except Exception:
return False
days, hour, minute = sch
have_day = False
for day in days:
if is_weekday(day, now):
have_day = True
break
return have_day and \
not was_downloaded_already_today(ld_local, now) and \
is_time(now, hour, minute)
elif typ == 'days_of_month':
now = nowf()
try:
ld_local = ld.astimezone(local_tz)
except Exception:
return False
days, hour, minute = sch
have_day = now.day in days
return have_day and \
not was_downloaded_already_today(ld_local, now) and \
is_time(now, hour, minute)
return False
def recipe_needs_to_be_downloaded(self, recipe):
try:
typ, sch, ld = self.un_serialize_schedule(recipe)
except:
return False
def is_time(now, hour, minute):
return now.hour > hour or \
(now.hour == hour and now.minute >= minute)
def is_weekday(day, now):
return day < 0 or day > 6 or \
day == calendar.weekday(now.year, now.month, now.day)
def was_downloaded_already_today(ld_local, now):
return ld_local.date() == now.date()
if typ == 'interval':
return utcnow() - ld > timedelta(sch)
elif typ == 'day/time':
now = nowf()
try:
ld_local = ld.astimezone(local_tz)
except Exception:
return False
day, hour, minute = sch
return is_weekday(day, now) and \
not was_downloaded_already_today(ld_local, now) and \
is_time(now, hour, minute)
elif typ == 'days_of_week':
now = nowf()
try:
ld_local = ld.astimezone(local_tz)
except Exception:
return False
days, hour, minute = sch
have_day = False
for day in days:
if is_weekday(day, now):
have_day = True
break
return have_day and \
not was_downloaded_already_today(ld_local, now) and \
is_time(now, hour, minute)
elif typ == 'days_of_month':
now = nowf()
try:
ld_local = ld.astimezone(local_tz)
except Exception:
return False
days, hour, minute = sch
have_day = now.day in days
return have_day and \
not was_downloaded_already_today(ld_local, now) and \
is_time(now, hour, minute)
return False
def update_last_modified(self, book_ids, now=None):
if now is None:
now = nowf()
if book_ids:
f = self.fields['last_modified']
f.writer.set_books({book_id: now
for book_id in book_ids}, self.backend)
def update(self, mi):
mi.title = normalize(mi.title)
def update_exth_record(rec):
recs.append(rec)
if rec[0] in self.original_exth_records:
self.original_exth_records.pop(rec[0])
if self.type != "BOOKMOBI":
raise MobiError("Setting metadata only supported for MOBI files of type 'BOOK'.\n"
"\tThis is a %r file of type %r" % (self.type[0:4], self.type[4:8]))
recs = []
added_501 = False
try:
from calibre.ebooks.conversion.config import load_defaults
prefs = load_defaults('mobi_output')
pas = prefs.get('prefer_author_sort', False)
kindle_pdoc = prefs.get('personal_doc', None)
share_not_sync = prefs.get('share_not_sync', False)
except:
pas = False
kindle_pdoc = None
share_not_sync = False
if mi.author_sort and pas:
# We want an EXTH field per author...
authors = mi.author_sort.split(' & ')
for author in authors:
update_exth_record((100, normalize(author).encode(self.codec, 'replace')))
elif mi.authors:
authors = mi.authors
for author in authors:
update_exth_record((100, normalize(author).encode(self.codec, 'replace')))
if mi.publisher:
update_exth_record((101, normalize(mi.publisher).encode(self.codec, 'replace')))
if mi.comments:
# Strip user annotations
a_offset = mi.comments.find('<div class="user_annotations">')
ad_offset = mi.comments.find('<hr class="annotations_divider" />')
if a_offset >= 0:
mi.comments = mi.comments[:a_offset]
if ad_offset >= 0:
mi.comments = mi.comments[:ad_offset]
update_exth_record((103, normalize(mi.comments).encode(self.codec, 'replace')))
if mi.isbn:
update_exth_record((104, mi.isbn.encode(self.codec, 'replace')))
if mi.tags:
# FIXME: Keep a single subject per EXTH field?
subjects = '; '.join(mi.tags)
update_exth_record((105, normalize(subjects).encode(self.codec, 'replace')))
if kindle_pdoc and kindle_pdoc in mi.tags:
added_501 = True
update_exth_record((501, b'PDOC'))
if mi.pubdate:
update_exth_record((106, str(mi.pubdate).encode(self.codec, 'replace')))
elif mi.timestamp:
update_exth_record((106, str(mi.timestamp).encode(self.codec, 'replace')))
elif self.timestamp:
update_exth_record((106, self.timestamp))
else:
update_exth_record((106, nowf().isoformat().encode(self.codec, 'replace')))
if self.cover_record is not None:
update_exth_record((201, pack('>I', self.cover_rindex)))
update_exth_record((203, pack('>I', 0)))
if self.thumbnail_record is not None:
update_exth_record((202, pack('>I', self.thumbnail_rindex)))
# Add a 113 record if not present to allow Amazon syncing
if (113 not in self.original_exth_records and
self.original_exth_records.get(501, None) == 'EBOK' and
not added_501 and not share_not_sync):
from uuid import uuid4
update_exth_record((113, str(uuid4())))
# Add a 112 record with actual UUID
if getattr(mi, 'uuid', None):
update_exth_record((112,
(u"calibre:%s" % mi.uuid).encode(self.codec, 'replace')))
if 503 in self.original_exth_records:
update_exth_record((503, mi.title.encode(self.codec, 'replace')))
# Update book producer
if getattr(mi, 'book_producer', False):
update_exth_record((108, mi.book_producer.encode(self.codec, 'replace')))
# Set langcode in EXTH header
if not mi.is_null('language'):
lang = canonicalize_lang(mi.language)
lang = lang_as_iso639_1(lang) or lang
if lang:
update_exth_record((524, lang.encode(self.codec, 'replace')))
# Include remaining original EXTH fields
for id in sorted(self.original_exth_records):
recs.append((id, self.original_exth_records[id]))
recs = sorted(recs, key=lambda x:(x[0],x[0]))
exth = StringIO()
for code, data in recs:
exth.write(pack('>II', code, len(data) + 8))
exth.write(data)
exth = exth.getvalue()
trail = len(exth) % 4
pad = '\0' * (4 - trail) # Always pad w/ at least 1 byte
exth = ['EXTH', pack('>II', len(exth) + 12, len(recs)), exth, pad]
exth = ''.join(exth)
if getattr(self, 'exth', None) is None:
raise MobiError('No existing EXTH record. Cannot update metadata.')
if not mi.is_null('language'):
self.record0[92:96] = iana2mobi(mi.language)
self.create_exth(exth=exth, new_title=mi.title)
# Fetch updated timestamp, cover_record, thumbnail_record
self.fetchEXTHFields()
if mi.cover_data[1] or mi.cover:
try:
data = mi.cover_data[1] if mi.cover_data[1] else open(mi.cover, 'rb').read()
except:
pass
else:
if is_image(self.cover_record):
size = len(self.cover_record)
cover = rescale_image(data, size)
if len(cover) <= size:
cover += b'\0' * (size - len(cover))
self.cover_record[:] = cover
if is_image(self.thumbnail_record):
size = len(self.thumbnail_record)
thumbnail = rescale_image(data, size, dimen=MAX_THUMB_DIMEN)
if len(thumbnail) <= size:
thumbnail += b'\0' * (size - len(thumbnail))
self.thumbnail_record[:] = thumbnail
return
def update(self, mi, asin=None):
mi.title = normalize(mi.title)
def update_exth_record(rec):
recs.append(rec)
if rec[0] in self.original_exth_records:
self.original_exth_records.pop(rec[0])
if self.type != b"BOOKMOBI":
raise MobiError(
"Setting metadata only supported for MOBI files of type 'BOOK'.\n"
"\tThis is a %r file of type %r" %
(self.type[0:4], self.type[4:8]))
recs = []
added_501 = False
try:
from calibre.ebooks.conversion.config import load_defaults
prefs = load_defaults('mobi_output')
pas = prefs.get('prefer_author_sort', False)
kindle_pdoc = prefs.get('personal_doc', None)
share_not_sync = prefs.get('share_not_sync', False)
except:
pas = False
kindle_pdoc = None
share_not_sync = False
if mi.author_sort and pas:
# We want an EXTH field per author...
authors = mi.author_sort.split(' & ')
for author in authors:
update_exth_record(
(100, normalize(author).encode(self.codec, 'replace')))
elif mi.authors:
authors = mi.authors
for author in authors:
update_exth_record(
(100, normalize(author).encode(self.codec, 'replace')))
if mi.publisher:
update_exth_record(
(101, normalize(mi.publisher).encode(self.codec, 'replace')))
if mi.comments:
# Strip user annotations
a_offset = mi.comments.find('<div class="user_annotations">')
ad_offset = mi.comments.find('<hr class="annotations_divider" />')
if a_offset >= 0:
mi.comments = mi.comments[:a_offset]
if ad_offset >= 0:
mi.comments = mi.comments[:ad_offset]
update_exth_record(
(103, normalize(mi.comments).encode(self.codec, 'replace')))
if mi.isbn:
update_exth_record((104, mi.isbn.encode(self.codec, 'replace')))
if mi.tags:
# FIXME: Keep a single subject per EXTH field?
subjects = '; '.join(mi.tags)
update_exth_record(
(105, normalize(subjects).encode(self.codec, 'replace')))
if kindle_pdoc and kindle_pdoc in mi.tags:
added_501 = True
update_exth_record((501, b'PDOC'))
if mi.pubdate:
update_exth_record(
(106, unicode_type(mi.pubdate).encode(self.codec, 'replace')))
elif mi.timestamp:
update_exth_record(
(106, unicode_type(mi.timestamp).encode(self.codec,
'replace')))
elif self.timestamp:
update_exth_record((106, self.timestamp))
else:
update_exth_record(
(106, nowf().isoformat().encode(self.codec, 'replace')))
if self.cover_record is not None:
update_exth_record((201, pack('>I', self.cover_rindex)))
update_exth_record((203, pack('>I', 0)))
if self.thumbnail_record is not None:
update_exth_record((202, pack('>I', self.thumbnail_rindex)))
# Add a 113 record if not present to allow Amazon syncing
if (113 not in self.original_exth_records
and self.original_exth_records.get(501, None) == 'EBOK'
and not added_501 and not share_not_sync):
from uuid import uuid4
update_exth_record((113, unicode_type(uuid4()).encode(self.codec)))
if asin is not None:
update_exth_record((113, asin.encode(self.codec)))
update_exth_record((504, asin.encode(self.codec)))
# Add a 112 record with actual UUID
if getattr(mi, 'uuid', None):
update_exth_record(
(112, ("calibre:%s" % mi.uuid).encode(self.codec, 'replace')))
if 503 in self.original_exth_records:
update_exth_record((503, mi.title.encode(self.codec, 'replace')))
# Update book producer
if getattr(mi, 'book_producer', False):
update_exth_record(
(108, mi.book_producer.encode(self.codec, 'replace')))
# Set langcode in EXTH header
if not mi.is_null('language'):
lang = canonicalize_lang(mi.language)
lang = lang_as_iso639_1(lang) or lang
if lang:
update_exth_record((524, lang.encode(self.codec, 'replace')))
# Include remaining original EXTH fields
for id in sorted(self.original_exth_records):
recs.append((id, self.original_exth_records[id]))
recs = sorted(recs, key=lambda x: (x[0], x[0]))
exth = io.BytesIO()
for code, data in recs:
exth.write(pack('>II', code, len(data) + 8))
exth.write(data)
exth = exth.getvalue()
trail = len(exth) % 4
pad = b'\0' * (4 - trail) # Always pad w/ at least 1 byte
exth = [b'EXTH', pack('>II', len(exth) + 12, len(recs)), exth, pad]
exth = b''.join(exth)
if getattr(self, 'exth', None) is None:
raise MobiError('No existing EXTH record. Cannot update metadata.')
if not mi.is_null('language'):
self.record0[92:96] = iana2mobi(mi.language)
self.create_exth(exth=exth, new_title=mi.title)
# Fetch updated timestamp, cover_record, thumbnail_record
self.fetchEXTHFields()
if mi.cover_data[1] or mi.cover:
try:
data = mi.cover_data[1]
if not data:
with open(mi.cover, 'rb') as f:
data = f.read()
except:
pass
else:
if is_image(self.cover_record):
size = len(self.cover_record)
cover = rescale_image(data, size)
if len(cover) <= size:
cover += b'\0' * (size - len(cover))
self.cover_record[:] = cover
if is_image(self.thumbnail_record):
size = len(self.thumbnail_record)
thumbnail = rescale_image(data,
size,
dimen=MAX_THUMB_DIMEN)
if len(thumbnail) <= size:
thumbnail += b'\0' * (size - len(thumbnail))
self.thumbnail_record[:] = thumbnail
return
def run(self, path_to_output, opts, db, notification=DummyReporter()):
from calibre.utils.date import isoformat
from calibre.utils.html2text import html2text
from calibre.utils.bibtex import BibTeX
from calibre.library.save_to_disk import preprocess_template
from calibre.utils.date import now as nowf
from calibre.utils.logging import default_log as log
library_name = os.path.basename(db.library_path)
def create_bibtex_entry(entry,
fields,
mode,
template_citation,
bibtexdict,
db,
citation_bibtex=True,
calibre_files=True):
#Bibtex doesn't like UTF-8 but keep unicode until writing
#Define starting chain or if book valid strict and not book return a Fail string
bibtex_entry = []
if mode != "misc" and check_entry_book_valid(entry):
bibtex_entry.append(u'@book{')
elif mode != "book":
bibtex_entry.append(u'@misc{')
else:
#case strict book
return ''
if citation_bibtex:
# Citation tag
bibtex_entry.append(
make_bibtex_citation(entry, template_citation, bibtexdict))
bibtex_entry = [u' '.join(bibtex_entry)]
for field in fields:
if field.startswith('#'):
item = db.get_field(entry['id'], field, index_is_id=True)
if isinstance(item, (bool, float, int)):
item = repr(item)
elif field == 'title_sort':
item = entry['sort']
elif field == 'library_name':
item = library_name
else:
item = entry[field]
#check if the field should be included (none or empty)
if item is None:
continue
try:
if len(item) == 0:
continue
except TypeError:
pass
if field == 'authors':
bibtex_entry.append(u'author = "%s"' %
bibtexdict.bibtex_author_format(item))
elif field == 'id':
bibtex_entry.append(u'calibreid = "%s"' % int(item))
elif field == 'rating':
bibtex_entry.append(u'rating = "%s"' % int(item))
elif field == 'size':
bibtex_entry.append(u'%s = "%s octets"' %
(field, int(item)))
elif field == 'tags':
#A list to flatten
bibtex_entry.append(
u'tags = "%s"' %
bibtexdict.utf8ToBibtex(u', '.join(item)))
elif field == 'comments':
#\n removal
item = item.replace(u'\r\n', u' ')
item = item.replace(u'\n', u' ')
# unmatched brace removal (users should use \leftbrace or \rightbrace for single braces)
item = bibtexdict.stripUnmatchedSyntax(item, u'{', u'}')
#html to text
try:
item = html2text(item)
except:
log.warn("Failed to convert comments to text")
bibtex_entry.append(u'note = "%s"' %
bibtexdict.utf8ToBibtex(item))
elif field == 'isbn':
# Could be 9, 10 or 13 digits
bibtex_entry.append(u'isbn = "%s"' % format_isbn(item))
elif field == 'formats':
#Add file path if format is selected
formats = [
format.rpartition('.')[2].lower() for format in item
]
bibtex_entry.append(u'formats = "%s"' %
u', '.join(formats))
if calibre_files:
files = [u':%s:%s' % (format, format.rpartition('.')[2].upper())\
for format in item]
bibtex_entry.append(u'file = "%s"' % u', '.join(files))
elif field == 'series_index':
bibtex_entry.append(u'volume = "%s"' % int(item))
elif field == 'timestamp':
bibtex_entry.append(u'timestamp = "%s"' %
isoformat(item).partition('T')[0])
elif field == 'pubdate':
bibtex_entry.append(u'year = "%s"' % item.year)
bibtex_entry.append(
u'month = "%s"' %
bibtexdict.utf8ToBibtex(strftime("%b", item)))
elif field.startswith('#') and isinstance(item, basestring):
bibtex_entry.append(
u'custom_%s = "%s"' %
(field[1:], bibtexdict.utf8ToBibtex(item)))
elif isinstance(item, basestring):
# elif field in ['title', 'publisher', 'cover', 'uuid', 'ondevice',
# 'author_sort', 'series', 'title_sort'] :
bibtex_entry.append(u'%s = "%s"' %
(field, bibtexdict.utf8ToBibtex(item)))
bibtex_entry = u',\n '.join(bibtex_entry)
bibtex_entry += u' }\n\n'
return bibtex_entry
def check_entry_book_valid(entry):
#Check that the required fields are ok for a book entry
for field in ['title', 'authors', 'publisher']:
if entry[field] is None or len(entry[field]) == 0:
return False
if entry['pubdate'] is None:
return False
else:
return True
def make_bibtex_citation(entry, template_citation, bibtexclass):
#define a function to replace the template entry by its value
def tpl_replace(objtplname):
tpl_field = re.sub(u'[\{\}]', u'', objtplname.group())
if tpl_field in TEMPLATE_ALLOWED_FIELDS:
if tpl_field in ['pubdate', 'timestamp']:
tpl_field = isoformat(
entry[tpl_field]).partition('T')[0]
elif tpl_field in ['tags', 'authors']:
tpl_field = entry[tpl_field][0]
elif tpl_field in ['id', 'series_index']:
tpl_field = str(entry[tpl_field])
else:
tpl_field = entry[tpl_field]
return tpl_field
else:
return u''
if len(template_citation) > 0:
tpl_citation = bibtexclass.utf8ToBibtex(
bibtexclass.ValidateCitationKey(
re.sub(u'\{[^{}]*\}', tpl_replace, template_citation)))
if len(tpl_citation) > 0:
return tpl_citation
if len(entry["isbn"]) > 0:
template_citation = u'%s' % re.sub(u'[\D]', u'', entry["isbn"])
else:
template_citation = u'%s' % str(entry["id"])
return bibtexclass.ValidateCitationKey(template_citation)
self.fmt = path_to_output.rpartition('.')[2]
self.notification = notification
# Combobox options
bibfile_enc = ['utf8', 'cp1252', 'ascii']
bibfile_enctag = ['strict', 'replace', 'ignore', 'backslashreplace']
bib_entry = ['mixed', 'misc', 'book']
# Needed beacause CLI return str vs int by widget
try:
bibfile_enc = bibfile_enc[opts.bibfile_enc]
bibfile_enctag = bibfile_enctag[opts.bibfile_enctag]
bib_entry = bib_entry[opts.bib_entry]
except:
if opts.bibfile_enc in bibfile_enc:
bibfile_enc = opts.bibfile_enc
else:
log.warn("Incorrect --choose-encoding flag, revert to default")
bibfile_enc = bibfile_enc[0]
if opts.bibfile_enctag in bibfile_enctag:
bibfile_enctag = opts.bibfile_enctag
else:
log.warn(
"Incorrect --choose-encoding-configuration flag, revert to default"
)
bibfile_enctag = bibfile_enctag[0]
if opts.bib_entry in bib_entry:
bib_entry = opts.bib_entry
else:
log.warn("Incorrect --entry-type flag, revert to default")
bib_entry = bib_entry[0]
if opts.verbose:
opts_dict = vars(opts)
log("%s(): Generating %s" % (self.name, self.fmt))
if opts.connected_device['is_device_connected']:
log(" connected_device: %s" % opts.connected_device['name'])
if opts_dict['search_text']:
log(" --search='%s'" % opts_dict['search_text'])
if opts_dict['ids']:
log(" Book count: %d" % len(opts_dict['ids']))
if opts_dict['search_text']:
log(" (--search ignored when a subset of the database is specified)"
)
if opts_dict['fields']:
if opts_dict['fields'] == 'all':
log(" Fields: %s" % ', '.join(FIELDS[1:]))
else:
log(" Fields: %s" % opts_dict['fields'])
log(" Output file will be encoded in %s with %s flag" %
(bibfile_enc, bibfile_enctag))
log(" BibTeX entry type is %s with a citation like '%s' flag" %
(bib_entry, opts_dict['bib_cit']))
# If a list of ids are provided, don't use search_text
if opts.ids:
opts.search_text = None
data = self.search_sort_db(db, opts)
if not len(data):
log.error(
"\nNo matching database entries for search criteria '%s'" %
opts.search_text)
# Get the requested output fields as a list
fields = self.get_output_fields(db, opts)
if not len(data):
log.error(
"\nNo matching database entries for search criteria '%s'" %
opts.search_text)
#Initialize BibTeX class
bibtexc = BibTeX()
#Entries writing after Bibtex formating (or not)
if bibfile_enc != 'ascii':
bibtexc.ascii_bibtex = False
else:
bibtexc.ascii_bibtex = True
#Check citation choice and go to default in case of bad CLI
if isinstance(opts.impcit, (StringType, UnicodeType)):
if opts.impcit == 'False':
citation_bibtex = False
elif opts.impcit == 'True':
citation_bibtex = True
else:
log.warn("Incorrect --create-citation, revert to default")
citation_bibtex = True
else:
citation_bibtex = opts.impcit
#Check add file entry and go to default in case of bad CLI
if isinstance(opts.addfiles, (StringType, UnicodeType)):
if opts.addfiles == 'False':
addfiles_bibtex = False
elif opts.addfiles == 'True':
addfiles_bibtex = True
else:
log.warn("Incorrect --add-files-path, revert to default")
addfiles_bibtex = True
else:
addfiles_bibtex = opts.addfiles
#Preprocess for error and light correction
template_citation = preprocess_template(opts.bib_cit)
#Open output and write entries
with codecs.open(path_to_output, 'w', bibfile_enc, bibfile_enctag)\
as outfile:
#File header
nb_entries = len(data)
#check in book strict if all is ok else throw a warning into log
if bib_entry == 'book':
nb_books = len(filter(check_entry_book_valid, data))
if nb_books < nb_entries:
log.warn("Only %d entries in %d are book compatible" %
(nb_books, nb_entries))
nb_entries = nb_books
# If connected device, add 'On Device' values to data
if opts.connected_device[
'is_device_connected'] and 'ondevice' in fields:
for entry in data:
entry[
'ondevice'] = db.catalog_plugin_on_device_temp_mapping[
entry['id']]['ondevice']
outfile.write(
u'%%%Calibre catalog\n%%%{0} entries in catalog\n\n'.format(
nb_entries))
outfile.write(
u'@preamble{"This catalog of %d entries was generated by calibre on %s"}\n\n'
% (nb_entries, nowf().strftime("%A, %d. %B %Y %H:%M").decode(
preferred_encoding)))
for entry in data:
outfile.write(
create_bibtex_entry(entry, fields, bib_entry,
template_citation, bibtexc, db,
citation_bibtex, addfiles_bibtex))
def _get_metadata(self, book_id, get_user_categories=True): # {{{
mi = Metadata(None, template_cache=self.formatter_template_cache)
author_ids = self._field_ids_for('authors', book_id)
aut_list = [self._author_data(i) for i in author_ids]
aum = []
aus = {}
aul = {}
for rec in aut_list:
aut = rec['name']
aum.append(aut)
aus[aut] = rec['sort']
aul[aut] = rec['link']
mi.title = self._field_for('title',
book_id,
default_value=_('Unknown'))
mi.authors = aum
mi.author_sort = self._field_for('author_sort',
book_id,
default_value=_('Unknown'))
mi.author_sort_map = aus
mi.author_link_map = aul
mi.comments = self._field_for('comments', book_id)
mi.publisher = self._field_for('publisher', book_id)
n = nowf()
mi.timestamp = self._field_for('timestamp', book_id, default_value=n)
mi.pubdate = self._field_for('pubdate', book_id, default_value=n)
mi.uuid = self._field_for('uuid', book_id, default_value='dummy')
mi.title_sort = self._field_for('sort',
book_id,
default_value=_('Unknown'))
mi.book_size = self._field_for('size', book_id, default_value=0)
mi.ondevice_col = self._field_for('ondevice',
book_id,
default_value='')
mi.last_modified = self._field_for('last_modified',
book_id,
default_value=n)
formats = self._field_for('formats', book_id)
mi.format_metadata = {}
mi.languages = list(self._field_for('languages', book_id))
if not formats:
good_formats = None
else:
mi.format_metadata = FormatMetadata(self, book_id, formats)
good_formats = FormatsList(formats, mi.format_metadata)
mi.formats = good_formats
mi.has_cover = _('Yes') if self._field_for(
'cover', book_id, default_value=False) else ''
mi.tags = list(self._field_for('tags', book_id, default_value=()))
mi.series = self._field_for('series', book_id)
if mi.series:
mi.series_index = self._field_for('series_index',
book_id,
default_value=1.0)
mi.rating = self._field_for('rating', book_id)
mi.set_identifiers(
self._field_for('identifiers', book_id, default_value={}))
mi.application_id = book_id
mi.id = book_id
composites = []
for key, meta in self.field_metadata.custom_iteritems():
mi.set_user_metadata(key, meta)
if meta['datatype'] == 'composite':
composites.append(key)
else:
val = self._field_for(key, book_id)
if isinstance(val, tuple):
val = list(val)
extra = self._field_for(key + '_index', book_id)
mi.set(key, val=val, extra=extra)
for key in composites:
mi.set(key, val=self._composite_for(key, book_id, mi))
user_cat_vals = {}
if get_user_categories:
user_cats = self.backend.prefs['user_categories']
for ucat in user_cats:
res = []
for name, cat, ign in user_cats[ucat]:
v = mi.get(cat, None)
if isinstance(v, list):
if name in v:
res.append([name, cat])
elif name == v:
res.append([name, cat])
user_cat_vals[ucat] = res
mi.user_categories = user_cat_vals
return mi
def update_last_modified(self, book_ids, now=None):
if now is None:
now = nowf()
if book_ids:
f = self.fields['last_modified']
f.writer.set_books({book_id:now for book_id in book_ids}, self.backend)
def _get_metadata(self, book_id, get_user_categories=True): # {{{
mi = Metadata(None, template_cache=self.formatter_template_cache)
author_ids = self._field_ids_for('authors', book_id)
aut_list = [self._author_data(i) for i in author_ids]
aum = []
aus = {}
aul = {}
for rec in aut_list:
aut = rec['name']
aum.append(aut)
aus[aut] = rec['sort']
aul[aut] = rec['link']
mi.title = self._field_for('title', book_id,
default_value=_('Unknown'))
mi.authors = aum
mi.author_sort = self._field_for('author_sort', book_id,
default_value=_('Unknown'))
mi.author_sort_map = aus
mi.author_link_map = aul
mi.comments = self._field_for('comments', book_id)
mi.publisher = self._field_for('publisher', book_id)
n = nowf()
mi.timestamp = self._field_for('timestamp', book_id, default_value=n)
mi.pubdate = self._field_for('pubdate', book_id, default_value=n)
mi.uuid = self._field_for('uuid', book_id,
default_value='dummy')
mi.title_sort = self._field_for('sort', book_id,
default_value=_('Unknown'))
mi.book_size = self._field_for('size', book_id, default_value=0)
mi.ondevice_col = self._field_for('ondevice', book_id, default_value='')
mi.last_modified = self._field_for('last_modified', book_id,
default_value=n)
formats = self._field_for('formats', book_id)
mi.format_metadata = {}
mi.languages = list(self._field_for('languages', book_id))
if not formats:
good_formats = None
else:
mi.format_metadata = FormatMetadata(self, book_id, formats)
good_formats = FormatsList(formats, mi.format_metadata)
mi.formats = good_formats
mi.has_cover = _('Yes') if self._field_for('cover', book_id,
default_value=False) else ''
mi.tags = list(self._field_for('tags', book_id, default_value=()))
mi.series = self._field_for('series', book_id)
if mi.series:
mi.series_index = self._field_for('series_index', book_id,
default_value=1.0)
mi.rating = self._field_for('rating', book_id)
mi.set_identifiers(self._field_for('identifiers', book_id,
default_value={}))
mi.application_id = book_id
mi.id = book_id
composites = []
for key, meta in self.field_metadata.custom_iteritems():
mi.set_user_metadata(key, meta)
if meta['datatype'] == 'composite':
composites.append(key)
else:
val = self._field_for(key, book_id)
if isinstance(val, tuple):
val = list(val)
extra = self._field_for(key+'_index', book_id)
mi.set(key, val=val, extra=extra)
for key in composites:
mi.set(key, val=self._composite_for(key, book_id, mi))
user_cat_vals = {}
if get_user_categories:
user_cats = self.backend.prefs['user_categories']
for ucat in user_cats:
res = []
for name,cat,ign in user_cats[ucat]:
v = mi.get(cat, None)
if isinstance(v, list):
if name in v:
res.append([name,cat])
elif name == v:
res.append([name,cat])
user_cat_vals[ucat] = res
mi.user_categories = user_cat_vals
return mi
def run(self, path_to_output, opts, db, notification=DummyReporter()):
from calibre.utils.date import isoformat
from calibre.utils.html2text import html2text
from calibre.utils.bibtex import BibTeX
from calibre.library.save_to_disk import preprocess_template
from calibre.utils.date import now as nowf
from calibre.utils.logging import default_log as log
library_name = os.path.basename(db.library_path)
def create_bibtex_entry(entry, fields, mode, template_citation,
bibtexdict, db, citation_bibtex=True, calibre_files=True):
#Bibtex doesn't like UTF-8 but keep unicode until writing
#Define starting chain or if book valid strict and not book return a Fail string
bibtex_entry = []
if mode != "misc" and check_entry_book_valid(entry) :
bibtex_entry.append(u'@book{')
elif mode != "book" :
bibtex_entry.append(u'@misc{')
else :
#case strict book
return ''
if citation_bibtex :
# Citation tag
bibtex_entry.append(make_bibtex_citation(entry, template_citation,
bibtexdict))
bibtex_entry = [u' '.join(bibtex_entry)]
for field in fields:
if field.startswith('#'):
item = db.get_field(entry['id'],field,index_is_id=True)
if isinstance(item, (bool, float, int)):
item = repr(item)
elif field == 'title_sort':
item = entry['sort']
elif field == 'library_name':
item = library_name
else:
item = entry[field]
#check if the field should be included (none or empty)
if item is None:
continue
try:
if len(item) == 0 :
continue
except TypeError:
pass
if field == 'authors' :
bibtex_entry.append(u'author = "%s"' % bibtexdict.bibtex_author_format(item))
elif field == 'id' :
bibtex_entry.append(u'calibreid = "%s"' % int(item))
elif field == 'rating' :
bibtex_entry.append(u'rating = "%s"' % int(item))
elif field == 'size' :
bibtex_entry.append(u'%s = "%s octets"' % (field, int(item)))
elif field == 'tags' :
#A list to flatten
bibtex_entry.append(u'tags = "%s"' % bibtexdict.utf8ToBibtex(u', '.join(item)))
elif field == 'comments' :
#\n removal
item = item.replace(u'\r\n',u' ')
item = item.replace(u'\n',u' ')
# unmatched brace removal (users should use \leftbrace or \rightbrace for single braces)
item = bibtexdict.stripUnmatchedSyntax(item, u'{', u'}')
#html to text
try:
item = html2text(item)
except:
log.warn("Failed to convert comments to text")
bibtex_entry.append(u'note = "%s"' % bibtexdict.utf8ToBibtex(item))
elif field == 'isbn' :
# Could be 9, 10 or 13 digits
bibtex_entry.append(u'isbn = "%s"' % format_isbn(item))
elif field == 'formats' :
#Add file path if format is selected
formats = [format.rpartition('.')[2].lower() for format in item]
bibtex_entry.append(u'formats = "%s"' % u', '.join(formats))
if calibre_files:
files = [u':%s:%s' % (format, format.rpartition('.')[2].upper())\
for format in item]
bibtex_entry.append(u'file = "%s"' % u', '.join(files))
elif field == 'series_index' :
bibtex_entry.append(u'volume = "%s"' % int(item))
elif field == 'timestamp' :
bibtex_entry.append(u'timestamp = "%s"' % isoformat(item).partition('T')[0])
elif field == 'pubdate' :
bibtex_entry.append(u'year = "%s"' % item.year)
bibtex_entry.append(u'month = "%s"' % bibtexdict.utf8ToBibtex(strftime("%b", item)))
elif field.startswith('#') and isinstance(item, basestring):
bibtex_entry.append(u'custom_%s = "%s"' % (field[1:],
bibtexdict.utf8ToBibtex(item)))
elif isinstance(item, basestring):
# elif field in ['title', 'publisher', 'cover', 'uuid', 'ondevice',
# 'author_sort', 'series', 'title_sort'] :
bibtex_entry.append(u'%s = "%s"' % (field, bibtexdict.utf8ToBibtex(item)))
bibtex_entry = u',\n '.join(bibtex_entry)
bibtex_entry += u' }\n\n'
return bibtex_entry
def check_entry_book_valid(entry):
#Check that the required fields are ok for a book entry
for field in ['title', 'authors', 'publisher'] :
if entry[field] is None or len(entry[field]) == 0 :
return False
if entry['pubdate'] is None :
return False
else :
return True
def make_bibtex_citation(entry, template_citation, bibtexclass):
#define a function to replace the template entry by its value
def tpl_replace(objtplname) :
tpl_field = re.sub(u'[\{\}]', u'', objtplname.group())
if tpl_field in TEMPLATE_ALLOWED_FIELDS :
if tpl_field in ['pubdate', 'timestamp'] :
tpl_field = isoformat(entry[tpl_field]).partition('T')[0]
elif tpl_field in ['tags', 'authors'] :
tpl_field =entry[tpl_field][0]
elif tpl_field in ['id', 'series_index'] :
tpl_field = str(entry[tpl_field])
else :
tpl_field = entry[tpl_field]
return tpl_field
else:
return u''
if len(template_citation) >0 :
tpl_citation = bibtexclass.utf8ToBibtex(
bibtexclass.ValidateCitationKey(re.sub(u'\{[^{}]*\}',
tpl_replace, template_citation)))
if len(tpl_citation) >0 :
return tpl_citation
if len(entry["isbn"]) > 0 :
template_citation = u'%s' % re.sub(u'[\D]',u'', entry["isbn"])
else :
template_citation = u'%s' % str(entry["id"])
return bibtexclass.ValidateCitationKey(template_citation)
self.fmt = path_to_output.rpartition('.')[2]
self.notification = notification
# Combobox options
bibfile_enc = ['utf8', 'cp1252', 'ascii']
bibfile_enctag = ['strict', 'replace', 'ignore', 'backslashreplace']
bib_entry = ['mixed', 'misc', 'book']
# Needed beacause CLI return str vs int by widget
try:
bibfile_enc = bibfile_enc[opts.bibfile_enc]
bibfile_enctag = bibfile_enctag[opts.bibfile_enctag]
bib_entry = bib_entry[opts.bib_entry]
except:
if opts.bibfile_enc in bibfile_enc :
bibfile_enc = opts.bibfile_enc
else :
log.warn("Incorrect --choose-encoding flag, revert to default")
bibfile_enc = bibfile_enc[0]
if opts.bibfile_enctag in bibfile_enctag :
bibfile_enctag = opts.bibfile_enctag
else :
log.warn("Incorrect --choose-encoding-configuration flag, revert to default")
bibfile_enctag = bibfile_enctag[0]
if opts.bib_entry in bib_entry :
bib_entry = opts.bib_entry
else :
log.warn("Incorrect --entry-type flag, revert to default")
bib_entry = bib_entry[0]
if opts.verbose:
opts_dict = vars(opts)
log("%s(): Generating %s" % (self.name,self.fmt))
if opts.connected_device['is_device_connected']:
log(" connected_device: %s" % opts.connected_device['name'])
if opts_dict['search_text']:
log(" --search='%s'" % opts_dict['search_text'])
if opts_dict['ids']:
log(" Book count: %d" % len(opts_dict['ids']))
if opts_dict['search_text']:
log(" (--search ignored when a subset of the database is specified)")
if opts_dict['fields']:
if opts_dict['fields'] == 'all':
log(" Fields: %s" % ', '.join(FIELDS[1:]))
else:
log(" Fields: %s" % opts_dict['fields'])
log(" Output file will be encoded in %s with %s flag" % (bibfile_enc, bibfile_enctag))
log(" BibTeX entry type is %s with a citation like '%s' flag" % (bib_entry, opts_dict['bib_cit']))
# If a list of ids are provided, don't use search_text
if opts.ids:
opts.search_text = None
data = self.search_sort_db(db, opts)
if not len(data):
log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text)
# Get the requested output fields as a list
fields = self.get_output_fields(db, opts)
if not len(data):
log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text)
#Initialize BibTeX class
bibtexc = BibTeX()
#Entries writing after Bibtex formating (or not)
if bibfile_enc != 'ascii' :
bibtexc.ascii_bibtex = False
else :
bibtexc.ascii_bibtex = True
#Check citation choice and go to default in case of bad CLI
if isinstance(opts.impcit, (StringType, UnicodeType)) :
if opts.impcit == 'False' :
citation_bibtex= False
elif opts.impcit == 'True' :
citation_bibtex= True
else :
log.warn("Incorrect --create-citation, revert to default")
citation_bibtex= True
else :
citation_bibtex= opts.impcit
#Check add file entry and go to default in case of bad CLI
if isinstance(opts.addfiles, (StringType, UnicodeType)) :
if opts.addfiles == 'False' :
addfiles_bibtex = False
elif opts.addfiles == 'True' :
addfiles_bibtex = True
else :
log.warn("Incorrect --add-files-path, revert to default")
addfiles_bibtex= True
else :
addfiles_bibtex = opts.addfiles
#Preprocess for error and light correction
template_citation = preprocess_template(opts.bib_cit)
#Open output and write entries
with codecs.open(path_to_output, 'w', bibfile_enc, bibfile_enctag)\
as outfile:
#File header
nb_entries = len(data)
#check in book strict if all is ok else throw a warning into log
if bib_entry == 'book' :
nb_books = len(filter(check_entry_book_valid, data))
if nb_books < nb_entries :
log.warn("Only %d entries in %d are book compatible" % (nb_books, nb_entries))
nb_entries = nb_books
# If connected device, add 'On Device' values to data
if opts.connected_device['is_device_connected'] and 'ondevice' in fields:
for entry in data:
entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[entry['id']]['ondevice']
outfile.write(u'%%%Calibre catalog\n%%%{0} entries in catalog\n\n'.format(nb_entries))
outfile.write(u'@preamble{"This catalog of %d entries was generated by calibre on %s"}\n\n'
% (nb_entries, nowf().strftime("%A, %d. %B %Y %H:%M").decode(preferred_encoding)))
for entry in data:
outfile.write(create_bibtex_entry(entry, fields, bib_entry, template_citation,
bibtexc, db, citation_bibtex, addfiles_bibtex))
