def search(self, query, max_results=10, timeout=60):
# Note! Not all results returned from this request get shown by Calibre:
# it filters down results by looking for search keywords
# in the different fields according to what user entered in the UI.
try:
for s in ModifiedOpenSearchOPDSStore.search(self, query, max_results, timeout, create_browser=self.create_browser):
s.price = '$0.00'
s.drm = SearchResult.DRM_UNLOCKED
yield s
except Exception, e:
default_log.error("Error executing search request (maybe bad login credentials?): %s" % (e,))
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
if len(args) != 2:
parser.print_help()
print
default_log.error(_('You must specify an epub file'))
return
epub = os.path.abspath(args[1])
try:
run(epub, opts, default_log)
except ParseError as err:
default_log.error(unicode(err))
raise SystemExit(1)
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
if len(args) != 2:
parser.print_help()
print
default_log.error(_('You must specify an epub file'))
return
epub = os.path.abspath(args[1])
try:
run(epub, opts, default_log)
except ParseError as err:
default_log.error(unicode(err))
raise SystemExit(1)
def run(self, path_to_output, opts, db, notification=DummyReporter()):
from calibre.library import current_library_name
from calibre.utils.date import isoformat
from calibre.utils.html2text import html2text
from calibre.utils.logging import default_log as log
from lxml import etree
from calibre.ebooks.metadata import authors_to_string
self.fmt = path_to_output.rpartition('.')[2]
self.notification = notification
current_library = current_library_name()
if getattr(opts, 'library_path', None):
current_library = os.path.basename(opts.library_path)
if opts.verbose:
opts_dict = vars(opts)
log("%s('%s'): Generating %s" %
(self.name, current_library, self.fmt.upper()))
if opts.connected_device['is_device_connected']:
log(" connected_device: %s" % opts.connected_device['name'])
if opts_dict['search_text']:
log(" --search='%s'" % opts_dict['search_text'])
if opts_dict['ids']:
log(" Book count: %d" % len(opts_dict['ids']))
if opts_dict['search_text']:
log(" (--search ignored when a subset of the database is specified)"
)
if opts_dict['fields']:
if opts_dict['fields'] == 'all':
log(" Fields: %s" % ', '.join(FIELDS[1:]))
else:
log(" Fields: %s" % opts_dict['fields'])
# If a list of ids are provided, don't use search_text
if opts.ids:
opts.search_text = None
data = self.search_sort_db(db, opts)
if not len(data):
log.error(
"\nNo matching database entries for search criteria '%s'" %
opts.search_text)
# raise SystemExit(1)
# Get the requested output fields as a list
fields = self.get_output_fields(db, opts)
# If connected device, add 'On Device' values to data
if opts.connected_device[
'is_device_connected'] and 'ondevice' in fields:
for entry in data:
entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[
entry['id']]['ondevice']
fm = {x: db.field_metadata.get(x, {}) for x in fields}
if self.fmt == 'csv':
outfile = codecs.open(path_to_output, 'w', 'utf8')
# Write a UTF-8 BOM
outfile.write('\ufeff')
# Output the field headers
outfile.write('%s\n' % ','.join(fields))
# Output the entry fields
for entry in data:
outstr = []
for field in fields:
if field.startswith('#'):
item = db.get_field(entry['id'],
field,
index_is_id=True)
if isinstance(item, (list, tuple)):
if fm.get(field,
{}).get('display',
{}).get('is_names', False):
item = ' & '.join(item)
else:
item = ', '.join(item)
elif field == 'library_name':
item = current_library
elif field == 'title_sort':
item = entry['sort']
else:
item = entry[field]
if item is None:
outstr.append('""')
continue
elif field == 'formats':
fmt_list = []
for format in item:
fmt_list.append(format.rpartition('.')[2].lower())
item = ', '.join(fmt_list)
elif field == 'authors':
item = authors_to_string(item)
elif field == 'tags':
item = ', '.join(item)
elif field == 'isbn':
# Could be 9, 10 or 13 digits, with hyphens, possibly ending in 'X'
item = '%s' % re.sub(r'[^\dX-]', '', item)
elif fm.get(field, {}).get('datatype') == 'datetime':
item = isoformat(item, as_utc=False)
elif field == 'comments':
item = item.replace('\r\n', ' ')
item = item.replace('\n', ' ')
elif fm.get(field, {}).get('datatype',
None) == 'rating' and item:
item = '%.2g' % (item / 2)
# Convert HTML to markdown text
if isinstance(item, str):
opening_tag = re.search(r'<(\w+)( |>)', item)
if opening_tag:
closing_tag = re.search(
r'<\/%s>$' % opening_tag.group(1), item)
if closing_tag:
item = html2text(item)
outstr.append('"%s"' % str(item).replace('"', '""'))
outfile.write(','.join(outstr) + '\n')
outfile.close()
elif self.fmt == 'xml':
from lxml.builder import E
if getattr(opts, 'catalog_title', None):
root = E.calibredb(title=opts.catalog_title)
else:
root = E.calibredb()
for r in data:
record = E.record()
root.append(record)
for field in fields:
if field.startswith('#'):
val = db.get_field(r['id'], field, index_is_id=True)
if not isinstance(val, str):
val = str(val)
item = getattr(E, field.replace('#', '_'))(val)
record.append(item)
for field in ('id', 'uuid', 'publisher', 'rating', 'size',
'isbn', 'ondevice', 'identifiers'):
if field in fields:
val = r[field]
if not val:
continue
if not isinstance(val, (bytes, str)):
if (fm.get(field, {}).get('datatype', None)
== 'rating' and val):
val = '%.2g' % (val / 2)
val = str(val)
item = getattr(E, field)(val)
record.append(item)
if 'title' in fields:
title = E.title(r['title'], sort=r['sort'])
record.append(title)
if 'authors' in fields:
aus = E.authors(sort=r['author_sort'])
for au in r['authors']:
aus.append(E.author(au))
record.append(aus)
for field in ('timestamp', 'pubdate'):
if field in fields:
record.append(
getattr(E, field)(isoformat(r[field],
as_utc=False)))
if 'tags' in fields and r['tags']:
tags = E.tags()
for tag in r['tags']:
tags.append(E.tag(tag))
record.append(tags)
if 'comments' in fields and r['comments']:
record.append(E.comments(r['comments']))
if 'series' in fields and r['series']:
record.append(
E.series(r['series'], index=str(r['series_index'])))
if 'languages' in fields and r['languages']:
record.append(E.languages(r['languages']))
if 'cover' in fields and r['cover']:
record.append(E.cover(r['cover'].replace(os.sep, '/')))
if 'formats' in fields and r['formats']:
fmt = E.formats()
for f in r['formats']:
fmt.append(E.format(f.replace(os.sep, '/')))
record.append(fmt)
if 'library_name' in fields:
record.append(E.library_name(current_library))
with open(path_to_output, 'wb') as f:
f.write(
etree.tostring(root,
encoding='utf-8',
xml_declaration=True,
pretty_print=True))
def run(self, path_to_output, opts, db, notification=DummyReporter()):
from calibre.library.catalogs.epub_mobi_builder import CatalogBuilder
from calibre.utils.logging import default_log as log
# If preset specified from the cli, insert stored options from JSON file
if hasattr(opts, 'preset') and opts.preset:
available_presets = JSONConfig("catalog_presets")
if not opts.preset in available_presets:
if available_presets:
print(_('Error: Preset "%s" not found.' % opts.preset))
print(
_('Stored presets: %s' % ', '.join(
[p for p in sorted(available_presets.keys())])))
else:
print(_('Error: No stored presets.'))
return 1
# Copy the relevant preset values to the opts object
for item in available_presets[opts.preset]:
if not item in [
'exclusion_rules_tw', 'format', 'prefix_rules_tw'
]:
setattr(opts, item, available_presets[opts.preset][item])
# Provide an unconnected device
opts.connected_device = {
'is_device_connected': False,
'kind': None,
'name': None,
'save_template': None,
'serial': None,
'storage': None,
}
# Convert prefix_rules and exclusion_rules from JSON lists to tuples
prs = []
for rule in opts.prefix_rules:
prs.append(tuple(rule))
opts.prefix_rules = tuple(prs)
ers = []
for rule in opts.exclusion_rules:
ers.append(tuple(rule))
opts.exclusion_rules = tuple(ers)
opts.log = log
opts.fmt = self.fmt = path_to_output.rpartition('.')[2]
# Add local options
opts.creator = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'),
strftime('%d').lstrip('0'),
strftime('%Y'))
opts.creator_sort_as = '%s %s' % ('calibre', strftime('%Y-%m-%d'))
opts.connected_kindle = False
# Finalize output_profile
op = opts.output_profile
if op is None:
op = 'default'
if opts.connected_device['name'] and 'kindle' in opts.connected_device[
'name'].lower():
opts.connected_kindle = True
if opts.connected_device['serial'] and \
opts.connected_device['serial'][:4] in ['B004', 'B005']:
op = "kindle_dx"
else:
op = "kindle"
opts.description_clip = 380 if op.endswith(
'dx') or 'kindle' not in op else 100
opts.author_clip = 100 if op.endswith(
'dx') or 'kindle' not in op else 60
opts.output_profile = op
opts.basename = "Catalog"
opts.cli_environment = not hasattr(opts, 'sync')
# Hard-wired to always sort descriptions by author, with series after non-series
opts.sort_descriptions_by_author = True
build_log = []
build_log.append(
u"%s('%s'): Generating %s %sin %s environment, locale: '%s'" %
(self.name, current_library_name(), self.fmt,
'for %s ' % opts.output_profile if opts.output_profile else '',
'CLI' if opts.cli_environment else 'GUI',
calibre_langcode_to_name(canonicalize_lang(get_lang()),
localize=False)))
# If exclude_genre is blank, assume user wants all tags as genres
if opts.exclude_genre.strip() == '':
#opts.exclude_genre = '\[^.\]'
#build_log.append(" converting empty exclude_genre to '\[^.\]'")
opts.exclude_genre = 'a^'
build_log.append(" converting empty exclude_genre to 'a^'")
if opts.connected_device['is_device_connected'] and \
opts.connected_device['kind'] == 'device':
if opts.connected_device['serial']:
build_log.append(u" connected_device: '%s' #%s%s " % \
(opts.connected_device['name'],
opts.connected_device['serial'][0:4],
'x' * (len(opts.connected_device['serial']) - 4)))
for storage in opts.connected_device['storage']:
if storage:
build_log.append(u" mount point: %s" % storage)
else:
build_log.append(u" connected_device: '%s'" %
opts.connected_device['name'])
try:
for storage in opts.connected_device['storage']:
if storage:
build_log.append(u" mount point: %s" % storage)
except:
build_log.append(u" (no mount points)")
else:
build_log.append(u" connected_device: '%s'" %
opts.connected_device['name'])
opts_dict = vars(opts)
if opts_dict['ids']:
build_log.append(" book count: %d" % len(opts_dict['ids']))
sections_list = []
if opts.generate_authors:
sections_list.append('Authors')
if opts.generate_titles:
sections_list.append('Titles')
if opts.generate_series:
sections_list.append('Series')
if opts.generate_genres:
sections_list.append('Genres')
if opts.generate_recently_added:
sections_list.append('Recently Added')
if opts.generate_descriptions:
sections_list.append('Descriptions')
if not sections_list:
if opts.cli_environment:
opts.log.warn(
'*** No Section switches specified, enabling all Sections ***'
)
opts.generate_authors = True
opts.generate_titles = True
opts.generate_series = True
opts.generate_genres = True
opts.generate_recently_added = True
opts.generate_descriptions = True
sections_list = [
'Authors', 'Titles', 'Series', 'Genres', 'Recently Added',
'Descriptions'
]
else:
opts.log.warn(
'\n*** No enabled Sections, terminating catalog generation ***'
)
return [
"No Included Sections",
"No enabled Sections.\nCheck E-book options tab\n'Included sections'\n"
]
if opts.fmt == 'mobi' and sections_list == ['Descriptions']:
warning = _(
"\n*** Adding 'By Authors' Section required for MOBI output ***"
)
opts.log.warn(warning)
sections_list.insert(0, 'Authors')
opts.generate_authors = True
opts.log(u" Sections: %s" % ', '.join(sections_list))
opts.section_list = sections_list
# Limit thumb_width to 1.0" - 2.0"
try:
if float(opts.thumb_width) < float(self.THUMB_SMALLEST):
log.warning("coercing thumb_width from '%s' to '%s'" %
(opts.thumb_width, self.THUMB_SMALLEST))
opts.thumb_width = self.THUMB_SMALLEST
if float(opts.thumb_width) > float(self.THUMB_LARGEST):
log.warning("coercing thumb_width from '%s' to '%s'" %
(opts.thumb_width, self.THUMB_LARGEST))
opts.thumb_width = self.THUMB_LARGEST
opts.thumb_width = "%.2f" % float(opts.thumb_width)
except:
log.error("coercing thumb_width from '%s' to '%s'" %
(opts.thumb_width, self.THUMB_SMALLEST))
opts.thumb_width = "1.0"
# eval prefix_rules if passed from command line
if type(opts.prefix_rules) is not tuple:
try:
opts.prefix_rules = eval(opts.prefix_rules)
except:
log.error("malformed --prefix-rules: %s" % opts.prefix_rules)
raise
for rule in opts.prefix_rules:
if len(rule) != 4:
log.error(
"incorrect number of args for --prefix-rules: %s" %
repr(rule))
# eval exclusion_rules if passed from command line
if type(opts.exclusion_rules) is not tuple:
try:
opts.exclusion_rules = eval(opts.exclusion_rules)
except:
log.error("malformed --exclusion-rules: %s" %
opts.exclusion_rules)
raise
for rule in opts.exclusion_rules:
if len(rule) != 3:
log.error(
"incorrect number of args for --exclusion-rules: %s" %
repr(rule))
# Display opts
keys = sorted(opts_dict.keys())
build_log.append(" opts:")
for key in keys:
if key in [
'catalog_title', 'author_clip', 'connected_kindle',
'creator', 'cross_reference_authors', 'description_clip',
'exclude_book_marker', 'exclude_genre', 'exclude_tags',
'exclusion_rules', 'fmt', 'genre_source_field',
'header_note_source_field', 'merge_comments_rule',
'output_profile', 'prefix_rules', 'preset',
'read_book_marker', 'search_text', 'sort_by',
'sort_descriptions_by_author', 'sync', 'thumb_width',
'use_existing_cover', 'wishlist_tag'
]:
build_log.append(" %s: %s" % (key, repr(opts_dict[key])))
if opts.verbose:
log('\n'.join(line for line in build_log))
# Capture start_time
opts.start_time = time.time()
self.opts = opts
if opts.verbose:
log.info(" Begin catalog source generation (%s)" % str(
datetime.timedelta(seconds=int(time.time() -
opts.start_time))))
# Launch the Catalog builder
catalog = CatalogBuilder(db, opts, self, report_progress=notification)
try:
catalog.build_sources()
if opts.verbose:
log.info(" Completed catalog source generation (%s)\n" % str(
datetime.timedelta(seconds=int(time.time() -
opts.start_time))))
except (AuthorSortMismatchException, EmptyCatalogException), e:
log.error(" *** Terminated catalog generation: %s ***" % e)
datetime.timedelta(seconds=int(time.time() -
opts.start_time))))
# Launch the Catalog builder
catalog = CatalogBuilder(db, opts, self, report_progress=notification)
try:
catalog.build_sources()
if opts.verbose:
log.info(" Completed catalog source generation (%s)\n" % str(
datetime.timedelta(seconds=int(time.time() -
opts.start_time))))
except (AuthorSortMismatchException, EmptyCatalogException), e:
log.error(" *** Terminated catalog generation: %s ***" % e)
except:
log.error(" unhandled exception in catalog generator")
raise
else:
recommendations = []
recommendations.append(
('remove_fake_margins', False, OptionRecommendation.HIGH))
recommendations.append(('comments', '', OptionRecommendation.HIGH))
"""
>>> Use to debug generated catalog code before pipeline conversion <<<
"""
GENERATE_DEBUG_EPUB = False
if GENERATE_DEBUG_EPUB:
catalog_debug_path = os.path.join(os.path.expanduser('~'),
'Desktop', 'Catalog debug')
setattr(opts, 'debug_pipeline',
def run(self, path_to_output, opts, db, notification=DummyReporter()):
from calibre.utils.date import isoformat
from calibre.utils.html2text import html2text
from calibre.utils.bibtex import BibTeX
from calibre.library.save_to_disk import preprocess_template
from calibre.utils.date import now as nowf
from calibre.utils.logging import default_log as log
library_name = os.path.basename(db.library_path)
def create_bibtex_entry(entry,
fields,
mode,
template_citation,
bibtexdict,
db,
citation_bibtex=True,
calibre_files=True):
#Bibtex doesn't like UTF-8 but keep unicode until writing
#Define starting chain or if book valid strict and not book return a Fail string
bibtex_entry = []
if mode != "misc" and check_entry_book_valid(entry):
bibtex_entry.append(u'@book{')
elif mode != "book":
bibtex_entry.append(u'@misc{')
else:
#case strict book
return ''
if citation_bibtex:
# Citation tag
bibtex_entry.append(
make_bibtex_citation(entry, template_citation, bibtexdict))
bibtex_entry = [u' '.join(bibtex_entry)]
for field in fields:
if field.startswith('#'):
item = db.get_field(entry['id'], field, index_is_id=True)
if isinstance(item, (bool, float, int)):
item = repr(item)
elif field == 'title_sort':
item = entry['sort']
elif field == 'library_name':
item = library_name
else:
item = entry[field]
#check if the field should be included (none or empty)
if item is None:
continue
try:
if len(item) == 0:
continue
except TypeError:
pass
if field == 'authors':
bibtex_entry.append(u'author = "%s"' %
bibtexdict.bibtex_author_format(item))
elif field == 'id':
bibtex_entry.append(u'calibreid = "%s"' % int(item))
elif field == 'rating':
bibtex_entry.append(u'rating = "%s"' % int(item))
elif field == 'size':
bibtex_entry.append(u'%s = "%s octets"' %
(field, int(item)))
elif field == 'tags':
#A list to flatten
bibtex_entry.append(
u'tags = "%s"' %
bibtexdict.utf8ToBibtex(u', '.join(item)))
elif field == 'comments':
#\n removal
item = item.replace(u'\r\n', u' ')
item = item.replace(u'\n', u' ')
# unmatched brace removal (users should use \leftbrace or \rightbrace for single braces)
item = bibtexdict.stripUnmatchedSyntax(item, u'{', u'}')
#html to text
try:
item = html2text(item)
except:
log.warn("Failed to convert comments to text")
bibtex_entry.append(u'note = "%s"' %
bibtexdict.utf8ToBibtex(item))
elif field == 'isbn':
# Could be 9, 10 or 13 digits
bibtex_entry.append(u'isbn = "%s"' % format_isbn(item))
elif field == 'formats':
#Add file path if format is selected
formats = [
format.rpartition('.')[2].lower() for format in item
]
bibtex_entry.append(u'formats = "%s"' %
u', '.join(formats))
if calibre_files:
files = [u':%s:%s' % (format, format.rpartition('.')[2].upper())\
for format in item]
bibtex_entry.append(u'file = "%s"' % u', '.join(files))
elif field == 'series_index':
bibtex_entry.append(u'volume = "%s"' % int(item))
elif field == 'timestamp':
bibtex_entry.append(u'timestamp = "%s"' %
isoformat(item).partition('T')[0])
elif field == 'pubdate':
bibtex_entry.append(u'year = "%s"' % item.year)
bibtex_entry.append(
u'month = "%s"' %
bibtexdict.utf8ToBibtex(strftime("%b", item)))
elif field.startswith('#') and isinstance(item, basestring):
bibtex_entry.append(
u'custom_%s = "%s"' %
(field[1:], bibtexdict.utf8ToBibtex(item)))
elif isinstance(item, basestring):
# elif field in ['title', 'publisher', 'cover', 'uuid', 'ondevice',
# 'author_sort', 'series', 'title_sort'] :
bibtex_entry.append(u'%s = "%s"' %
(field, bibtexdict.utf8ToBibtex(item)))
bibtex_entry = u',\n '.join(bibtex_entry)
bibtex_entry += u' }\n\n'
return bibtex_entry
def check_entry_book_valid(entry):
#Check that the required fields are ok for a book entry
for field in ['title', 'authors', 'publisher']:
if entry[field] is None or len(entry[field]) == 0:
return False
if entry['pubdate'] is None:
return False
else:
return True
def make_bibtex_citation(entry, template_citation, bibtexclass):
#define a function to replace the template entry by its value
def tpl_replace(objtplname):
tpl_field = re.sub(u'[\{\}]', u'', objtplname.group())
if tpl_field in TEMPLATE_ALLOWED_FIELDS:
if tpl_field in ['pubdate', 'timestamp']:
tpl_field = isoformat(
entry[tpl_field]).partition('T')[0]
elif tpl_field in ['tags', 'authors']:
tpl_field = entry[tpl_field][0]
elif tpl_field in ['id', 'series_index']:
tpl_field = str(entry[tpl_field])
else:
tpl_field = entry[tpl_field]
return tpl_field
else:
return u''
if len(template_citation) > 0:
tpl_citation = bibtexclass.utf8ToBibtex(
bibtexclass.ValidateCitationKey(
re.sub(u'\{[^{}]*\}', tpl_replace, template_citation)))
if len(tpl_citation) > 0:
return tpl_citation
if len(entry["isbn"]) > 0:
template_citation = u'%s' % re.sub(u'[\D]', u'', entry["isbn"])
else:
template_citation = u'%s' % str(entry["id"])
return bibtexclass.ValidateCitationKey(template_citation)
self.fmt = path_to_output.rpartition('.')[2]
self.notification = notification
# Combobox options
bibfile_enc = ['utf8', 'cp1252', 'ascii']
bibfile_enctag = ['strict', 'replace', 'ignore', 'backslashreplace']
bib_entry = ['mixed', 'misc', 'book']
# Needed beacause CLI return str vs int by widget
try:
bibfile_enc = bibfile_enc[opts.bibfile_enc]
bibfile_enctag = bibfile_enctag[opts.bibfile_enctag]
bib_entry = bib_entry[opts.bib_entry]
except:
if opts.bibfile_enc in bibfile_enc:
bibfile_enc = opts.bibfile_enc
else:
log.warn("Incorrect --choose-encoding flag, revert to default")
bibfile_enc = bibfile_enc[0]
if opts.bibfile_enctag in bibfile_enctag:
bibfile_enctag = opts.bibfile_enctag
else:
log.warn(
"Incorrect --choose-encoding-configuration flag, revert to default"
)
bibfile_enctag = bibfile_enctag[0]
if opts.bib_entry in bib_entry:
bib_entry = opts.bib_entry
else:
log.warn("Incorrect --entry-type flag, revert to default")
bib_entry = bib_entry[0]
if opts.verbose:
opts_dict = vars(opts)
log("%s(): Generating %s" % (self.name, self.fmt))
if opts.connected_device['is_device_connected']:
log(" connected_device: %s" % opts.connected_device['name'])
if opts_dict['search_text']:
log(" --search='%s'" % opts_dict['search_text'])
if opts_dict['ids']:
log(" Book count: %d" % len(opts_dict['ids']))
if opts_dict['search_text']:
log(" (--search ignored when a subset of the database is specified)"
)
if opts_dict['fields']:
if opts_dict['fields'] == 'all':
log(" Fields: %s" % ', '.join(FIELDS[1:]))
else:
log(" Fields: %s" % opts_dict['fields'])
log(" Output file will be encoded in %s with %s flag" %
(bibfile_enc, bibfile_enctag))
log(" BibTeX entry type is %s with a citation like '%s' flag" %
(bib_entry, opts_dict['bib_cit']))
# If a list of ids are provided, don't use search_text
if opts.ids:
opts.search_text = None
data = self.search_sort_db(db, opts)
if not len(data):
log.error(
"\nNo matching database entries for search criteria '%s'" %
opts.search_text)
# Get the requested output fields as a list
fields = self.get_output_fields(db, opts)
if not len(data):
log.error(
"\nNo matching database entries for search criteria '%s'" %
opts.search_text)
#Initialize BibTeX class
bibtexc = BibTeX()
#Entries writing after Bibtex formating (or not)
if bibfile_enc != 'ascii':
bibtexc.ascii_bibtex = False
else:
bibtexc.ascii_bibtex = True
#Check citation choice and go to default in case of bad CLI
if isinstance(opts.impcit, (StringType, UnicodeType)):
if opts.impcit == 'False':
citation_bibtex = False
elif opts.impcit == 'True':
citation_bibtex = True
else:
log.warn("Incorrect --create-citation, revert to default")
citation_bibtex = True
else:
citation_bibtex = opts.impcit
#Check add file entry and go to default in case of bad CLI
if isinstance(opts.addfiles, (StringType, UnicodeType)):
if opts.addfiles == 'False':
addfiles_bibtex = False
elif opts.addfiles == 'True':
addfiles_bibtex = True
else:
log.warn("Incorrect --add-files-path, revert to default")
addfiles_bibtex = True
else:
addfiles_bibtex = opts.addfiles
#Preprocess for error and light correction
template_citation = preprocess_template(opts.bib_cit)
#Open output and write entries
with codecs.open(path_to_output, 'w', bibfile_enc, bibfile_enctag)\
as outfile:
#File header
nb_entries = len(data)
#check in book strict if all is ok else throw a warning into log
if bib_entry == 'book':
nb_books = len(filter(check_entry_book_valid, data))
if nb_books < nb_entries:
log.warn("Only %d entries in %d are book compatible" %
(nb_books, nb_entries))
nb_entries = nb_books
# If connected device, add 'On Device' values to data
if opts.connected_device[
'is_device_connected'] and 'ondevice' in fields:
for entry in data:
entry[
'ondevice'] = db.catalog_plugin_on_device_temp_mapping[
entry['id']]['ondevice']
outfile.write(
u'%%%Calibre catalog\n%%%{0} entries in catalog\n\n'.format(
nb_entries))
outfile.write(
u'@preamble{"This catalog of %d entries was generated by calibre on %s"}\n\n'
% (nb_entries, nowf().strftime("%A, %d. %B %Y %H:%M").decode(
preferred_encoding)))
for entry in data:
outfile.write(
create_bibtex_entry(entry, fields, bib_entry,
template_citation, bibtexc, db,
citation_bibtex, addfiles_bibtex))
def run(self, path_to_output, opts, db, notification=DummyReporter()):
from calibre.library.catalogs.epub_mobi_builder import CatalogBuilder
from calibre.utils.logging import default_log as log
from calibre.utils.config import JSONConfig
# If preset specified from the cli, insert stored options from JSON file
if hasattr(opts, 'preset') and opts.preset:
available_presets = JSONConfig("catalog_presets")
if opts.preset not in available_presets:
if available_presets:
print(_('Error: Preset "%s" not found.' % opts.preset))
print(_('Stored presets: %s' % ', '.join([p for p in sorted(available_presets.keys())])))
else:
print(_('Error: No stored presets.'))
return 1
# Copy the relevant preset values to the opts object
for item in available_presets[opts.preset]:
if item not in ['exclusion_rules_tw', 'format', 'prefix_rules_tw']:
setattr(opts, item, available_presets[opts.preset][item])
# Provide an unconnected device
opts.connected_device = {
'is_device_connected': False,
'kind': None,
'name': None,
'save_template': None,
'serial': None,
'storage': None,
}
# Convert prefix_rules and exclusion_rules from JSON lists to tuples
prs = []
for rule in opts.prefix_rules:
prs.append(tuple(rule))
opts.prefix_rules = tuple(prs)
ers = []
for rule in opts.exclusion_rules:
ers.append(tuple(rule))
opts.exclusion_rules = tuple(ers)
opts.log = log
opts.fmt = self.fmt = path_to_output.rpartition('.')[2]
# Add local options
opts.creator = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'), strftime('%d').lstrip('0'), strftime('%Y'))
opts.creator_sort_as = '%s %s' % ('calibre', strftime('%Y-%m-%d'))
opts.connected_kindle = False
# Finalize output_profile
op = opts.output_profile
if op is None:
op = 'default'
if opts.connected_device['name'] and 'kindle' in opts.connected_device['name'].lower():
opts.connected_kindle = True
if opts.connected_device['serial'] and \
opts.connected_device['serial'][:4] in ['B004', 'B005']:
op = "kindle_dx"
else:
op = "kindle"
opts.description_clip = 380 if op.endswith('dx') or 'kindle' not in op else 100
opts.author_clip = 100 if op.endswith('dx') or 'kindle' not in op else 60
opts.output_profile = op
opts.basename = "Catalog"
opts.cli_environment = not hasattr(opts, 'sync')
# Hard-wired to always sort descriptions by author, with series after non-series
opts.sort_descriptions_by_author = True
build_log = []
build_log.append(u"%s('%s'): Generating %s %sin %s environment, locale: '%s'" %
(self.name,
current_library_name(),
self.fmt,
'for %s ' % opts.output_profile if opts.output_profile else '',
'CLI' if opts.cli_environment else 'GUI',
calibre_langcode_to_name(canonicalize_lang(get_lang()), localize=False))
)
# If exclude_genre is blank, assume user wants all tags as genres
if opts.exclude_genre.strip() == '':
# opts.exclude_genre = '\[^.\]'
# build_log.append(" converting empty exclude_genre to '\[^.\]'")
opts.exclude_genre = 'a^'
build_log.append(" converting empty exclude_genre to 'a^'")
if opts.connected_device['is_device_connected'] and \
opts.connected_device['kind'] == 'device':
if opts.connected_device['serial']:
build_log.append(u" connected_device: '%s' #%s%s " %
(opts.connected_device['name'],
opts.connected_device['serial'][0:4],
'x' * (len(opts.connected_device['serial']) - 4)))
for storage in opts.connected_device['storage']:
if storage:
build_log.append(u" mount point: %s" % storage)
else:
build_log.append(u" connected_device: '%s'" % opts.connected_device['name'])
try:
for storage in opts.connected_device['storage']:
if storage:
build_log.append(u" mount point: %s" % storage)
except:
build_log.append(u" (no mount points)")
else:
build_log.append(u" connected_device: '%s'" % opts.connected_device['name'])
opts_dict = vars(opts)
if opts_dict['ids']:
build_log.append(" book count: %d" % len(opts_dict['ids']))
sections_list = []
if opts.generate_authors:
sections_list.append('Authors')
if opts.generate_titles:
sections_list.append('Titles')
if opts.generate_series:
sections_list.append('Series')
if opts.generate_genres:
sections_list.append('Genres')
if opts.generate_recently_added:
sections_list.append('Recently Added')
if opts.generate_descriptions:
sections_list.append('Descriptions')
if not sections_list:
if opts.cli_environment:
opts.log.warn('*** No Section switches specified, enabling all Sections ***')
opts.generate_authors = True
opts.generate_titles = True
opts.generate_series = True
opts.generate_genres = True
opts.generate_recently_added = True
opts.generate_descriptions = True
sections_list = ['Authors', 'Titles', 'Series', 'Genres', 'Recently Added', 'Descriptions']
else:
opts.log.warn('\n*** No enabled Sections, terminating catalog generation ***')
return ["No Included Sections", "No enabled Sections.\nCheck E-book options tab\n'Included sections'\n"]
if opts.fmt == 'mobi' and sections_list == ['Descriptions']:
warning = _("\n*** Adding 'By authors' section required for MOBI output ***")
opts.log.warn(warning)
sections_list.insert(0, 'Authors')
opts.generate_authors = True
opts.log(u" Sections: %s" % ', '.join(sections_list))
opts.section_list = sections_list
# Limit thumb_width to 1.0" - 2.0"
try:
if float(opts.thumb_width) < float(self.THUMB_SMALLEST):
log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width, self.THUMB_SMALLEST))
opts.thumb_width = self.THUMB_SMALLEST
if float(opts.thumb_width) > float(self.THUMB_LARGEST):
log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width, self.THUMB_LARGEST))
opts.thumb_width = self.THUMB_LARGEST
opts.thumb_width = "%.2f" % float(opts.thumb_width)
except:
log.error("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width, self.THUMB_SMALLEST))
opts.thumb_width = "1.0"
# eval prefix_rules if passed from command line
if type(opts.prefix_rules) is not tuple:
try:
opts.prefix_rules = eval(opts.prefix_rules)
except:
log.error("malformed --prefix-rules: %s" % opts.prefix_rules)
raise
for rule in opts.prefix_rules:
if len(rule) != 4:
log.error("incorrect number of args for --prefix-rules: %s" % repr(rule))
# eval exclusion_rules if passed from command line
if type(opts.exclusion_rules) is not tuple:
try:
opts.exclusion_rules = eval(opts.exclusion_rules)
except:
log.error("malformed --exclusion-rules: %s" % opts.exclusion_rules)
raise
for rule in opts.exclusion_rules:
if len(rule) != 3:
log.error("incorrect number of args for --exclusion-rules: %s" % repr(rule))
# Display opts
keys = sorted(opts_dict.keys())
build_log.append(" opts:")
for key in keys:
if key in ['catalog_title', 'author_clip', 'connected_kindle', 'creator',
'cross_reference_authors', 'description_clip', 'exclude_book_marker',
'exclude_genre', 'exclude_tags', 'exclusion_rules', 'fmt',
'genre_source_field', 'header_note_source_field', 'merge_comments_rule',
'output_profile', 'prefix_rules', 'preset', 'read_book_marker',
'search_text', 'sort_by', 'sort_descriptions_by_author', 'sync',
'thumb_width', 'use_existing_cover', 'wishlist_tag']:
build_log.append(" %s: %s" % (key, repr(opts_dict[key])))
if opts.verbose:
log('\n'.join(line for line in build_log))
# Capture start_time
opts.start_time = time.time()
self.opts = opts
if opts.verbose:
log.info(" Begin catalog source generation (%s)" %
str(datetime.timedelta(seconds=int(time.time() - opts.start_time))))
# Launch the Catalog builder
catalog = CatalogBuilder(db, opts, self, report_progress=notification)
try:
catalog.build_sources()
if opts.verbose:
log.info(" Completed catalog source generation (%s)\n" %
str(datetime.timedelta(seconds=int(time.time() - opts.start_time))))
except (AuthorSortMismatchException, EmptyCatalogException) as e:
log.error(" *** Terminated catalog generation: %s ***" % e)
except:
log.error(" unhandled exception in catalog generator")
raise
else:
recommendations = []
recommendations.append(('remove_fake_margins', False,
OptionRecommendation.HIGH))
recommendations.append(('comments', '', OptionRecommendation.HIGH))
"""
>>> Use to debug generated catalog code before pipeline conversion <<<
"""
GENERATE_DEBUG_EPUB = False
if GENERATE_DEBUG_EPUB:
catalog_debug_path = os.path.join(os.path.expanduser('~'), 'Desktop', 'Catalog debug')
setattr(opts, 'debug_pipeline', os.path.expanduser(catalog_debug_path))
dp = getattr(opts, 'debug_pipeline', None)
if dp is not None:
recommendations.append(('debug_pipeline', dp,
OptionRecommendation.HIGH))
if opts.output_profile and opts.output_profile.startswith("kindle"):
recommendations.append(('output_profile', opts.output_profile,
OptionRecommendation.HIGH))
recommendations.append(('book_producer', opts.output_profile,
OptionRecommendation.HIGH))
if opts.fmt == 'mobi':
recommendations.append(('no_inline_toc', True,
OptionRecommendation.HIGH))
recommendations.append(('verbose', 2,
OptionRecommendation.HIGH))
# Use existing cover or generate new cover
cpath = None
existing_cover = False
try:
search_text = 'title:"%s" author:%s' % (
opts.catalog_title.replace('"', '\\"'), 'calibre')
matches = db.search(search_text, return_matches=True, sort_results=False)
if matches:
cpath = db.cover(matches[0], index_is_id=True, as_path=True)
if cpath and os.path.exists(cpath):
existing_cover = True
except:
pass
if self.opts.use_existing_cover and not existing_cover:
log.warning("no existing catalog cover found")
if self.opts.use_existing_cover and existing_cover:
recommendations.append(('cover', cpath, OptionRecommendation.HIGH))
log.info("using existing catalog cover")
else:
from calibre.ebooks.covers import calibre_cover2
log.info("replacing catalog cover")
new_cover_path = PersistentTemporaryFile(suffix='.jpg')
new_cover = calibre_cover2(opts.catalog_title, 'calibre')
new_cover_path.write(new_cover)
new_cover_path.close()
recommendations.append(('cover', new_cover_path.name, OptionRecommendation.HIGH))
# Run ebook-convert
from calibre.ebooks.conversion.plumber import Plumber
plumber = Plumber(os.path.join(catalog.catalog_path, opts.basename + '.opf'),
path_to_output, log, report_progress=notification,
abort_after_input_dump=False)
plumber.merge_ui_recommendations(recommendations)
plumber.run()
try:
os.remove(cpath)
except:
pass
if GENERATE_DEBUG_EPUB:
from calibre.ebooks.epub import initialize_container
from calibre.ebooks.tweak import zip_rebuilder
from calibre.utils.zipfile import ZipFile
input_path = os.path.join(catalog_debug_path, 'input')
epub_shell = os.path.join(catalog_debug_path, 'epub_shell.zip')
initialize_container(epub_shell, opf_name='content.opf')
with ZipFile(epub_shell, 'r') as zf:
zf.extractall(path=input_path)
os.remove(epub_shell)
zip_rebuilder(input_path, os.path.join(catalog_debug_path, 'input.epub'))
if opts.verbose:
log.info(" Catalog creation complete (%s)\n" %
str(datetime.timedelta(seconds=int(time.time() - opts.start_time))))
# returns to gui2.actions.catalog:catalog_generated()
return catalog.error
def run(self, path_to_output, opts, db, notification=DummyReporter()):
from calibre.library import current_library_name
from calibre.utils.date import isoformat
from calibre.utils.html2text import html2text
from calibre.utils.logging import default_log as log
from lxml import etree
from calibre.ebooks.metadata import authors_to_string
self.fmt = path_to_output.rpartition('.')[2]
self.notification = notification
current_library = current_library_name()
if getattr(opts, 'library_path', None):
current_library = os.path.basename(opts.library_path)
if opts.verbose:
opts_dict = vars(opts)
log("%s('%s'): Generating %s" % (self.name, current_library, self.fmt.upper()))
if opts.connected_device['is_device_connected']:
log(" connected_device: %s" % opts.connected_device['name'])
if opts_dict['search_text']:
log(" --search='%s'" % opts_dict['search_text'])
if opts_dict['ids']:
log(" Book count: %d" % len(opts_dict['ids']))
if opts_dict['search_text']:
log(" (--search ignored when a subset of the database is specified)")
if opts_dict['fields']:
if opts_dict['fields'] == 'all':
log(" Fields: %s" % ', '.join(FIELDS[1:]))
else:
log(" Fields: %s" % opts_dict['fields'])
# If a list of ids are provided, don't use search_text
if opts.ids:
opts.search_text = None
data = self.search_sort_db(db, opts)
if not len(data):
log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text)
# raise SystemExit(1)
# Get the requested output fields as a list
fields = self.get_output_fields(db, opts)
# If connected device, add 'On Device' values to data
if opts.connected_device['is_device_connected'] and 'ondevice' in fields:
for entry in data:
entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[entry['id']]['ondevice']
fm = {x: db.field_metadata.get(x, {}) for x in fields}
if self.fmt == 'csv':
outfile = codecs.open(path_to_output, 'w', 'utf8')
# Write a UTF-8 BOM
outfile.write('\xef\xbb\xbf')
# Output the field headers
outfile.write(u'%s\n' % u','.join(fields))
# Output the entry fields
for entry in data:
outstr = []
for field in fields:
if field.startswith('#'):
item = db.get_field(entry['id'], field, index_is_id=True)
if isinstance(item, (list, tuple)):
if fm.get(field, {}).get('display', {}).get('is_names', False):
item = ' & '.join(item)
else:
item = ', '.join(item)
elif field == 'library_name':
item = current_library
elif field == 'title_sort':
item = entry['sort']
else:
item = entry[field]
if item is None:
outstr.append('""')
continue
elif field == 'formats':
fmt_list = []
for format in item:
fmt_list.append(format.rpartition('.')[2].lower())
item = ', '.join(fmt_list)
elif field == 'authors':
item = authors_to_string(item)
elif field == 'tags':
item = ', '.join(item)
elif field == 'isbn':
# Could be 9, 10 or 13 digits, with hyphens, possibly ending in 'X'
item = u'%s' % re.sub(r'[^\dX-]', '', item)
elif fm.get(field, {}).get('datatype') == 'datetime':
item = isoformat(item, as_utc=False)
elif field == 'comments':
item = item.replace(u'\r\n', u' ')
item = item.replace(u'\n', u' ')
elif fm.get(field, {}).get('datatype', None) == 'rating' and item:
item = u'%.2g' % (item / 2.0)
# Convert HTML to markdown text
if type(item) is unicode:
opening_tag = re.search('<(\w+)(\x20|>)', item)
if opening_tag:
closing_tag = re.search('<\/%s>$' % opening_tag.group(1), item)
if closing_tag:
item = html2text(item)
outstr.append(u'"%s"' % unicode(item).replace('"', '""'))
outfile.write(u','.join(outstr) + u'\n')
outfile.close()
elif self.fmt == 'xml':
from lxml.builder import E
root = E.calibredb()
for r in data:
record = E.record()
root.append(record)
for field in fields:
if field.startswith('#'):
val = db.get_field(r['id'], field, index_is_id=True)
if not isinstance(val, (str, unicode)):
val = unicode(val)
item = getattr(E, field.replace('#', '_'))(val)
record.append(item)
for field in ('id', 'uuid', 'publisher', 'rating', 'size',
'isbn', 'ondevice', 'identifiers'):
if field in fields:
val = r[field]
if not val:
continue
if not isinstance(val, (str, unicode)):
if (fm.get(field, {}).get('datatype', None) ==
'rating' and val):
val = u'%.2g' % (val / 2.0)
val = unicode(val)
item = getattr(E, field)(val)
record.append(item)
if 'title' in fields:
title = E.title(r['title'], sort=r['sort'])
record.append(title)
if 'authors' in fields:
aus = E.authors(sort=r['author_sort'])
for au in r['authors']:
aus.append(E.author(au))
record.append(aus)
for field in ('timestamp', 'pubdate'):
if field in fields:
record.append(getattr(E, field)(isoformat(r[field], as_utc=False)))
if 'tags' in fields and r['tags']:
tags = E.tags()
for tag in r['tags']:
tags.append(E.tag(tag))
record.append(tags)
if 'comments' in fields and r['comments']:
record.append(E.comments(r['comments']))
if 'series' in fields and r['series']:
record.append(E.series(r['series'],
index=str(r['series_index'])))
if 'cover' in fields and r['cover']:
record.append(E.cover(r['cover'].replace(os.sep, '/')))
if 'formats' in fields and r['formats']:
fmt = E.formats()
for f in r['formats']:
fmt.append(E.format(f.replace(os.sep, '/')))
record.append(fmt)
if 'library_name' in fields:
record.append(E.library_name(current_library))
with open(path_to_output, 'w') as f:
f.write(etree.tostring(root, encoding='utf-8',
xml_declaration=True, pretty_print=True))
def run(self, path_to_output, opts, db, notification=DummyReporter()):
from calibre.library.catalogs.epub_mobi_builder import CatalogBuilder
from calibre.utils.logging import default_log as log
opts.log = log
opts.fmt = self.fmt = path_to_output.rpartition('.')[2]
# Add local options
opts.creator = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'), strftime('%d').lstrip('0'), strftime('%Y'))
opts.creator_sort_as = '%s %s' % ('calibre', strftime('%Y-%m-%d'))
opts.connected_kindle = False
# Finalize output_profile
op = opts.output_profile
if op is None:
op = 'default'
if opts.connected_device['name'] and 'kindle' in opts.connected_device['name'].lower():
opts.connected_kindle = True
if opts.connected_device['serial'] and \
opts.connected_device['serial'][:4] in ['B004', 'B005']:
op = "kindle_dx"
else:
op = "kindle"
opts.description_clip = 380 if op.endswith('dx') or 'kindle' not in op else 100
opts.author_clip = 100 if op.endswith('dx') or 'kindle' not in op else 60
opts.output_profile = op
opts.basename = "Catalog"
opts.cli_environment = not hasattr(opts, 'sync')
# Hard-wired to always sort descriptions by author, with series after non-series
opts.sort_descriptions_by_author = True
build_log = []
build_log.append(u"%s('%s'): Generating %s %sin %s environment, locale: '%s'" %
(self.name,
current_library_name(),
self.fmt,
'for %s ' % opts.output_profile if opts.output_profile else '',
'CLI' if opts.cli_environment else 'GUI',
calibre_langcode_to_name(canonicalize_lang(get_lang()), localize=False))
)
# If exclude_genre is blank, assume user wants all tags as genres
if opts.exclude_genre.strip() == '':
#opts.exclude_genre = '\[^.\]'
#build_log.append(" converting empty exclude_genre to '\[^.\]'")
opts.exclude_genre = 'a^'
build_log.append(" converting empty exclude_genre to 'a^'")
if opts.connected_device['is_device_connected'] and \
opts.connected_device['kind'] == 'device':
if opts.connected_device['serial']:
build_log.append(u" connected_device: '%s' #%s%s " % \
(opts.connected_device['name'],
opts.connected_device['serial'][0:4],
'x' * (len(opts.connected_device['serial']) - 4)))
for storage in opts.connected_device['storage']:
if storage:
build_log.append(u" mount point: %s" % storage)
else:
build_log.append(u" connected_device: '%s'" % opts.connected_device['name'])
try:
for storage in opts.connected_device['storage']:
if storage:
build_log.append(u" mount point: %s" % storage)
except:
build_log.append(u" (no mount points)")
else:
build_log.append(u" connected_device: '%s'" % opts.connected_device['name'])
opts_dict = vars(opts)
if opts_dict['ids']:
build_log.append(" book count: %d" % len(opts_dict['ids']))
sections_list = []
if opts.generate_authors:
sections_list.append('Authors')
if opts.generate_titles:
sections_list.append('Titles')
if opts.generate_series:
sections_list.append('Series')
if opts.generate_genres:
sections_list.append('Genres')
if opts.generate_recently_added:
sections_list.append('Recently Added')
if opts.generate_descriptions:
sections_list.append('Descriptions')
if not sections_list:
if opts.cli_environment:
opts.log.warn('*** No Section switches specified, enabling all Sections ***')
opts.generate_authors = True
opts.generate_titles = True
opts.generate_series = True
opts.generate_genres = True
opts.generate_recently_added = True
opts.generate_descriptions = True
sections_list = ['Authors', 'Titles', 'Series', 'Genres', 'Recently Added', 'Descriptions']
else:
opts.log.warn('\n*** No enabled Sections, terminating catalog generation ***')
return ["No Included Sections", "No enabled Sections.\nCheck E-book options tab\n'Included sections'\n"]
if opts.fmt == 'mobi' and sections_list == ['Descriptions']:
warning = _("\n*** Adding 'By Authors' Section required for MOBI output ***")
opts.log.warn(warning)
sections_list.insert(0, 'Authors')
opts.generate_authors = True
opts.log(u" Sections: %s" % ', '.join(sections_list))
opts.section_list = sections_list
# Limit thumb_width to 1.0" - 2.0"
try:
if float(opts.thumb_width) < float(self.THUMB_SMALLEST):
log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width, self.THUMB_SMALLEST))
opts.thumb_width = self.THUMB_SMALLEST
if float(opts.thumb_width) > float(self.THUMB_LARGEST):
log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width, self.THUMB_LARGEST))
opts.thumb_width = self.THUMB_LARGEST
opts.thumb_width = "%.2f" % float(opts.thumb_width)
except:
log.error("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width, self.THUMB_SMALLEST))
opts.thumb_width = "1.0"
# eval prefix_rules if passed from command line
if type(opts.prefix_rules) is not tuple:
try:
opts.prefix_rules = eval(opts.prefix_rules)
except:
log.error("malformed --prefix-rules: %s" % opts.prefix_rules)
raise
for rule in opts.prefix_rules:
if len(rule) != 4:
log.error("incorrect number of args for --prefix-rules: %s" % repr(rule))
# eval exclusion_rules if passed from command line
if type(opts.exclusion_rules) is not tuple:
try:
opts.exclusion_rules = eval(opts.exclusion_rules)
except:
log.error("malformed --exclusion-rules: %s" % opts.exclusion_rules)
raise
for rule in opts.exclusion_rules:
if len(rule) != 3:
log.error("incorrect number of args for --exclusion-rules: %s" % repr(rule))
# Display opts
keys = opts_dict.keys()
keys.sort()
build_log.append(" opts:")
for key in keys:
if key in ['catalog_title', 'author_clip', 'connected_kindle', 'creator',
'cross_reference_authors', 'description_clip', 'exclude_book_marker',
'exclude_genre', 'exclude_tags', 'exclusion_rules', 'fmt',
'genre_source_field', 'header_note_source_field', 'merge_comments_rule',
'output_profile', 'prefix_rules', 'read_book_marker',
'search_text', 'sort_by', 'sort_descriptions_by_author', 'sync',
'thumb_width', 'use_existing_cover', 'wishlist_tag']:
build_log.append(" %s: %s" % (key, repr(opts_dict[key])))
if opts.verbose:
log('\n'.join(line for line in build_log))
self.opts = opts
# Launch the Catalog builder
catalog = CatalogBuilder(db, opts, self, report_progress=notification)
if opts.verbose:
log.info(" Begin catalog source generation")
try:
catalog.build_sources()
if opts.verbose:
log.info(" Completed catalog source generation\n")
except (AuthorSortMismatchException, EmptyCatalogException), e:
log.error(" *** Terminated catalog generation: %s ***" % e)
self.opts = opts
# Launch the Catalog builder
catalog = CatalogBuilder(db, opts, self, report_progress=notification)
if opts.verbose:
log.info(" Begin catalog source generation")
try:
catalog.build_sources()
if opts.verbose:
log.info(" Completed catalog source generation\n")
except (AuthorSortMismatchException, EmptyCatalogException), e:
log.error(" *** Terminated catalog generation: %s ***" % e)
except:
log.error(" unhandled exception in catalog generator")
raise
else:
recommendations = []
recommendations.append(('remove_fake_margins', False,
OptionRecommendation.HIGH))
recommendations.append(('comments', '', OptionRecommendation.HIGH))
"""
>>> Use to debug generated catalog code before pipeline conversion <<<
"""
GENERATE_DEBUG_EPUB = False
if GENERATE_DEBUG_EPUB:
catalog_debug_path = os.path.join(os.path.expanduser('~'), 'Desktop', 'Catalog debug')
setattr(opts, 'debug_pipeline', os.path.expanduser(catalog_debug_path))
def run(self, path_to_output, opts, db, notification=DummyReporter()):
from calibre.utils.date import isoformat
from calibre.utils.html2text import html2text
from calibre.utils.bibtex import BibTeX
from calibre.library.save_to_disk import preprocess_template
from calibre.utils.date import now as nowf
from calibre.utils.logging import default_log as log
library_name = os.path.basename(db.library_path)
def create_bibtex_entry(entry, fields, mode, template_citation,
bibtexdict, db, citation_bibtex=True, calibre_files=True):
#Bibtex doesn't like UTF-8 but keep unicode until writing
#Define starting chain or if book valid strict and not book return a Fail string
bibtex_entry = []
if mode != "misc" and check_entry_book_valid(entry) :
bibtex_entry.append(u'@book{')
elif mode != "book" :
bibtex_entry.append(u'@misc{')
else :
#case strict book
return ''
if citation_bibtex :
# Citation tag
bibtex_entry.append(make_bibtex_citation(entry, template_citation,
bibtexdict))
bibtex_entry = [u' '.join(bibtex_entry)]
for field in fields:
if field.startswith('#'):
item = db.get_field(entry['id'],field,index_is_id=True)
if isinstance(item, (bool, float, int)):
item = repr(item)
elif field == 'title_sort':
item = entry['sort']
elif field == 'library_name':
item = library_name
else:
item = entry[field]
#check if the field should be included (none or empty)
if item is None:
continue
try:
if len(item) == 0 :
continue
except TypeError:
pass
if field == 'authors' :
bibtex_entry.append(u'author = "%s"' % bibtexdict.bibtex_author_format(item))
elif field == 'id' :
bibtex_entry.append(u'calibreid = "%s"' % int(item))
elif field == 'rating' :
bibtex_entry.append(u'rating = "%s"' % int(item))
elif field == 'size' :
bibtex_entry.append(u'%s = "%s octets"' % (field, int(item)))
elif field == 'tags' :
#A list to flatten
bibtex_entry.append(u'tags = "%s"' % bibtexdict.utf8ToBibtex(u', '.join(item)))
elif field == 'comments' :
#\n removal
item = item.replace(u'\r\n',u' ')
item = item.replace(u'\n',u' ')
# unmatched brace removal (users should use \leftbrace or \rightbrace for single braces)
item = bibtexdict.stripUnmatchedSyntax(item, u'{', u'}')
#html to text
try:
item = html2text(item)
except:
log.warn("Failed to convert comments to text")
bibtex_entry.append(u'note = "%s"' % bibtexdict.utf8ToBibtex(item))
elif field == 'isbn' :
# Could be 9, 10 or 13 digits
bibtex_entry.append(u'isbn = "%s"' % format_isbn(item))
elif field == 'formats' :
#Add file path if format is selected
formats = [format.rpartition('.')[2].lower() for format in item]
bibtex_entry.append(u'formats = "%s"' % u', '.join(formats))
if calibre_files:
files = [u':%s:%s' % (format, format.rpartition('.')[2].upper())\
for format in item]
bibtex_entry.append(u'file = "%s"' % u', '.join(files))
elif field == 'series_index' :
bibtex_entry.append(u'volume = "%s"' % int(item))
elif field == 'timestamp' :
bibtex_entry.append(u'timestamp = "%s"' % isoformat(item).partition('T')[0])
elif field == 'pubdate' :
bibtex_entry.append(u'year = "%s"' % item.year)
bibtex_entry.append(u'month = "%s"' % bibtexdict.utf8ToBibtex(strftime("%b", item)))
elif field.startswith('#') and isinstance(item, basestring):
bibtex_entry.append(u'custom_%s = "%s"' % (field[1:],
bibtexdict.utf8ToBibtex(item)))
elif isinstance(item, basestring):
# elif field in ['title', 'publisher', 'cover', 'uuid', 'ondevice',
# 'author_sort', 'series', 'title_sort'] :
bibtex_entry.append(u'%s = "%s"' % (field, bibtexdict.utf8ToBibtex(item)))
bibtex_entry = u',\n '.join(bibtex_entry)
bibtex_entry += u' }\n\n'
return bibtex_entry
def check_entry_book_valid(entry):
#Check that the required fields are ok for a book entry
for field in ['title', 'authors', 'publisher'] :
if entry[field] is None or len(entry[field]) == 0 :
return False
if entry['pubdate'] is None :
return False
else :
return True
def make_bibtex_citation(entry, template_citation, bibtexclass):
#define a function to replace the template entry by its value
def tpl_replace(objtplname) :
tpl_field = re.sub(u'[\{\}]', u'', objtplname.group())
if tpl_field in TEMPLATE_ALLOWED_FIELDS :
if tpl_field in ['pubdate', 'timestamp'] :
tpl_field = isoformat(entry[tpl_field]).partition('T')[0]
elif tpl_field in ['tags', 'authors'] :
tpl_field =entry[tpl_field][0]
elif tpl_field in ['id', 'series_index'] :
tpl_field = str(entry[tpl_field])
else :
tpl_field = entry[tpl_field]
return tpl_field
else:
return u''
if len(template_citation) >0 :
tpl_citation = bibtexclass.utf8ToBibtex(
bibtexclass.ValidateCitationKey(re.sub(u'\{[^{}]*\}',
tpl_replace, template_citation)))
if len(tpl_citation) >0 :
return tpl_citation
if len(entry["isbn"]) > 0 :
template_citation = u'%s' % re.sub(u'[\D]',u'', entry["isbn"])
else :
template_citation = u'%s' % str(entry["id"])
return bibtexclass.ValidateCitationKey(template_citation)
self.fmt = path_to_output.rpartition('.')[2]
self.notification = notification
# Combobox options
bibfile_enc = ['utf8', 'cp1252', 'ascii']
bibfile_enctag = ['strict', 'replace', 'ignore', 'backslashreplace']
bib_entry = ['mixed', 'misc', 'book']
# Needed beacause CLI return str vs int by widget
try:
bibfile_enc = bibfile_enc[opts.bibfile_enc]
bibfile_enctag = bibfile_enctag[opts.bibfile_enctag]
bib_entry = bib_entry[opts.bib_entry]
except:
if opts.bibfile_enc in bibfile_enc :
bibfile_enc = opts.bibfile_enc
else :
log.warn("Incorrect --choose-encoding flag, revert to default")
bibfile_enc = bibfile_enc[0]
if opts.bibfile_enctag in bibfile_enctag :
bibfile_enctag = opts.bibfile_enctag
else :
log.warn("Incorrect --choose-encoding-configuration flag, revert to default")
bibfile_enctag = bibfile_enctag[0]
if opts.bib_entry in bib_entry :
bib_entry = opts.bib_entry
else :
log.warn("Incorrect --entry-type flag, revert to default")
bib_entry = bib_entry[0]
if opts.verbose:
opts_dict = vars(opts)
log("%s(): Generating %s" % (self.name,self.fmt))
if opts.connected_device['is_device_connected']:
log(" connected_device: %s" % opts.connected_device['name'])
if opts_dict['search_text']:
log(" --search='%s'" % opts_dict['search_text'])
if opts_dict['ids']:
log(" Book count: %d" % len(opts_dict['ids']))
if opts_dict['search_text']:
log(" (--search ignored when a subset of the database is specified)")
if opts_dict['fields']:
if opts_dict['fields'] == 'all':
log(" Fields: %s" % ', '.join(FIELDS[1:]))
else:
log(" Fields: %s" % opts_dict['fields'])
log(" Output file will be encoded in %s with %s flag" % (bibfile_enc, bibfile_enctag))
log(" BibTeX entry type is %s with a citation like '%s' flag" % (bib_entry, opts_dict['bib_cit']))
# If a list of ids are provided, don't use search_text
if opts.ids:
opts.search_text = None
data = self.search_sort_db(db, opts)
if not len(data):
log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text)
# Get the requested output fields as a list
fields = self.get_output_fields(db, opts)
if not len(data):
log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text)
#Initialize BibTeX class
bibtexc = BibTeX()
#Entries writing after Bibtex formating (or not)
if bibfile_enc != 'ascii' :
bibtexc.ascii_bibtex = False
else :
bibtexc.ascii_bibtex = True
#Check citation choice and go to default in case of bad CLI
if isinstance(opts.impcit, (StringType, UnicodeType)) :
if opts.impcit == 'False' :
citation_bibtex= False
elif opts.impcit == 'True' :
citation_bibtex= True
else :
log.warn("Incorrect --create-citation, revert to default")
citation_bibtex= True
else :
citation_bibtex= opts.impcit
#Check add file entry and go to default in case of bad CLI
if isinstance(opts.addfiles, (StringType, UnicodeType)) :
if opts.addfiles == 'False' :
addfiles_bibtex = False
elif opts.addfiles == 'True' :
addfiles_bibtex = True
else :
log.warn("Incorrect --add-files-path, revert to default")
addfiles_bibtex= True
else :
addfiles_bibtex = opts.addfiles
#Preprocess for error and light correction
template_citation = preprocess_template(opts.bib_cit)
#Open output and write entries
with codecs.open(path_to_output, 'w', bibfile_enc, bibfile_enctag)\
as outfile:
#File header
nb_entries = len(data)
#check in book strict if all is ok else throw a warning into log
if bib_entry == 'book' :
nb_books = len(filter(check_entry_book_valid, data))
if nb_books < nb_entries :
log.warn("Only %d entries in %d are book compatible" % (nb_books, nb_entries))
nb_entries = nb_books
# If connected device, add 'On Device' values to data
if opts.connected_device['is_device_connected'] and 'ondevice' in fields:
for entry in data:
entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[entry['id']]['ondevice']
outfile.write(u'%%%Calibre catalog\n%%%{0} entries in catalog\n\n'.format(nb_entries))
outfile.write(u'@preamble{"This catalog of %d entries was generated by calibre on %s"}\n\n'
% (nb_entries, nowf().strftime("%A, %d. %B %Y %H:%M").decode(preferred_encoding)))
for entry in data:
outfile.write(create_bibtex_entry(entry, fields, bib_entry, template_citation,
bibtexc, db, citation_bibtex, addfiles_bibtex))
def run(self, path_to_output, opts, db, notification=DummyReporter()):
from calibre.library.catalogs.epub_mobi_builder import CatalogBuilder
from calibre.utils.logging import default_log as log
from calibre.utils.config import JSONConfig
# If preset specified from the cli, insert stored options from JSON file
if hasattr(opts, "preset") and opts.preset:
available_presets = JSONConfig("catalog_presets")
if not opts.preset in available_presets:
if available_presets:
print(_('Error: Preset "%s" not found.' % opts.preset))
print(_("Stored presets: %s" % ", ".join([p for p in sorted(available_presets.keys())])))
else:
print(_("Error: No stored presets."))
return 1
# Copy the relevant preset values to the opts object
for item in available_presets[opts.preset]:
if not item in ["exclusion_rules_tw", "format", "prefix_rules_tw"]:
setattr(opts, item, available_presets[opts.preset][item])
# Provide an unconnected device
opts.connected_device = {
"is_device_connected": False,
"kind": None,
"name": None,
"save_template": None,
"serial": None,
"storage": None,
}
# Convert prefix_rules and exclusion_rules from JSON lists to tuples
prs = []
for rule in opts.prefix_rules:
prs.append(tuple(rule))
opts.prefix_rules = tuple(prs)
ers = []
for rule in opts.exclusion_rules:
ers.append(tuple(rule))
opts.exclusion_rules = tuple(ers)
opts.log = log
opts.fmt = self.fmt = path_to_output.rpartition(".")[2]
# Add local options
opts.creator = "%s, %s %s, %s" % (strftime("%A"), strftime("%B"), strftime("%d").lstrip("0"), strftime("%Y"))
opts.creator_sort_as = "%s %s" % ("calibre", strftime("%Y-%m-%d"))
opts.connected_kindle = False
# Finalize output_profile
op = opts.output_profile
if op is None:
op = "default"
if opts.connected_device["name"] and "kindle" in opts.connected_device["name"].lower():
opts.connected_kindle = True
if opts.connected_device["serial"] and opts.connected_device["serial"][:4] in ["B004", "B005"]:
op = "kindle_dx"
else:
op = "kindle"
opts.description_clip = 380 if op.endswith("dx") or "kindle" not in op else 100
opts.author_clip = 100 if op.endswith("dx") or "kindle" not in op else 60
opts.output_profile = op
opts.basename = "Catalog"
opts.cli_environment = not hasattr(opts, "sync")
# Hard-wired to always sort descriptions by author, with series after non-series
opts.sort_descriptions_by_author = True
build_log = []
build_log.append(
"%s('%s'): Generating %s %sin %s environment, locale: '%s'"
% (
self.name,
current_library_name(),
self.fmt,
"for %s " % opts.output_profile if opts.output_profile else "",
"CLI" if opts.cli_environment else "GUI",
calibre_langcode_to_name(canonicalize_lang(get_lang()), localize=False),
)
)
# If exclude_genre is blank, assume user wants all tags as genres
if opts.exclude_genre.strip() == "":
# opts.exclude_genre = '\[^.\]'
# build_log.append(" converting empty exclude_genre to '\[^.\]'")
opts.exclude_genre = "a^"
build_log.append(" converting empty exclude_genre to 'a^'")
if opts.connected_device["is_device_connected"] and opts.connected_device["kind"] == "device":
if opts.connected_device["serial"]:
build_log.append(
" connected_device: '%s' #%s%s "
% (
opts.connected_device["name"],
opts.connected_device["serial"][0:4],
"x" * (len(opts.connected_device["serial"]) - 4),
)
)
for storage in opts.connected_device["storage"]:
if storage:
build_log.append(" mount point: %s" % storage)
else:
build_log.append(" connected_device: '%s'" % opts.connected_device["name"])
try:
for storage in opts.connected_device["storage"]:
if storage:
build_log.append(" mount point: %s" % storage)
except:
build_log.append(" (no mount points)")
else:
build_log.append(" connected_device: '%s'" % opts.connected_device["name"])
opts_dict = vars(opts)
if opts_dict["ids"]:
build_log.append(" book count: %d" % len(opts_dict["ids"]))
sections_list = []
if opts.generate_authors:
sections_list.append("Authors")
if opts.generate_titles:
sections_list.append("Titles")
if opts.generate_series:
sections_list.append("Series")
if opts.generate_genres:
sections_list.append("Genres")
if opts.generate_recently_added:
sections_list.append("Recently Added")
if opts.generate_descriptions:
sections_list.append("Descriptions")
if not sections_list:
if opts.cli_environment:
opts.log.warn("*** No Section switches specified, enabling all Sections ***")
opts.generate_authors = True
opts.generate_titles = True
opts.generate_series = True
opts.generate_genres = True
opts.generate_recently_added = True
opts.generate_descriptions = True
sections_list = ["Authors", "Titles", "Series", "Genres", "Recently Added", "Descriptions"]
else:
opts.log.warn("\n*** No enabled Sections, terminating catalog generation ***")
return ["No Included Sections", "No enabled Sections.\nCheck E-book options tab\n'Included sections'\n"]
if opts.fmt == "mobi" and sections_list == ["Descriptions"]:
warning = _("\n*** Adding 'By Authors' Section required for MOBI output ***")
opts.log.warn(warning)
sections_list.insert(0, "Authors")
opts.generate_authors = True
opts.log(" Sections: %s" % ", ".join(sections_list))
opts.section_list = sections_list
# Limit thumb_width to 1.0" - 2.0"
try:
if float(opts.thumb_width) < float(self.THUMB_SMALLEST):
log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width, self.THUMB_SMALLEST))
opts.thumb_width = self.THUMB_SMALLEST
if float(opts.thumb_width) > float(self.THUMB_LARGEST):
log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width, self.THUMB_LARGEST))
opts.thumb_width = self.THUMB_LARGEST
opts.thumb_width = "%.2f" % float(opts.thumb_width)
except:
log.error("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width, self.THUMB_SMALLEST))
opts.thumb_width = "1.0"
# eval prefix_rules if passed from command line
if type(opts.prefix_rules) is not tuple:
try:
opts.prefix_rules = eval(opts.prefix_rules)
except:
log.error("malformed --prefix-rules: %s" % opts.prefix_rules)
raise
for rule in opts.prefix_rules:
if len(rule) != 4:
log.error("incorrect number of args for --prefix-rules: %s" % repr(rule))
# eval exclusion_rules if passed from command line
if type(opts.exclusion_rules) is not tuple:
try:
opts.exclusion_rules = eval(opts.exclusion_rules)
except:
log.error("malformed --exclusion-rules: %s" % opts.exclusion_rules)
raise
for rule in opts.exclusion_rules:
if len(rule) != 3:
log.error("incorrect number of args for --exclusion-rules: %s" % repr(rule))
# Display opts
keys = sorted(opts_dict.keys())
build_log.append(" opts:")
for key in keys:
if key in [
"catalog_title",
"author_clip",
"connected_kindle",
"creator",
"cross_reference_authors",
"description_clip",
"exclude_book_marker",
"exclude_genre",
"exclude_tags",
"exclusion_rules",
"fmt",
"genre_source_field",
"header_note_source_field",
"merge_comments_rule",
"output_profile",
"prefix_rules",
"preset",
"read_book_marker",
"search_text",
"sort_by",
"sort_descriptions_by_author",
"sync",
"thumb_width",
"use_existing_cover",
"wishlist_tag",
]:
build_log.append(" %s: %s" % (key, repr(opts_dict[key])))
if opts.verbose:
log("\n".join(line for line in build_log))
# Capture start_time
opts.start_time = time.time()
self.opts = opts
if opts.verbose:
log.info(
" Begin catalog source generation (%s)"
% str(datetime.timedelta(seconds=int(time.time() - opts.start_time)))
)
# Launch the Catalog builder
catalog = CatalogBuilder(db, opts, self, report_progress=notification)
try:
catalog.build_sources()
if opts.verbose:
log.info(
" Completed catalog source generation (%s)\n"
% str(datetime.timedelta(seconds=int(time.time() - opts.start_time)))
)
except (AuthorSortMismatchException, EmptyCatalogException), e:
log.error(" *** Terminated catalog generation: %s ***" % e)
def run(self, path_to_output, opts, db, notification=DummyReporter()):
from calibre.library.catalogs.epub_mobi_builder import CatalogBuilder
from calibre.utils.logging import default_log as log
opts.log = log
opts.fmt = self.fmt = path_to_output.rpartition('.')[2]
# Add local options
opts.creator = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'), strftime('%d').lstrip('0'), strftime('%Y'))
opts.creator_sort_as = '%s %s' % ('calibre', strftime('%Y-%m-%d'))
opts.connected_kindle = False
# Finalize output_profile
op = opts.output_profile
if op is None:
op = 'default'
if opts.connected_device['name'] and 'kindle' in opts.connected_device['name'].lower():
opts.connected_kindle = True
if opts.connected_device['serial'] and \
opts.connected_device['serial'][:4] in ['B004','B005']:
op = "kindle_dx"
else:
op = "kindle"
opts.descriptionClip = 380 if op.endswith('dx') or 'kindle' not in op else 100
opts.authorClip = 100 if op.endswith('dx') or 'kindle' not in op else 60
opts.output_profile = op
opts.basename = "Catalog"
opts.cli_environment = not hasattr(opts,'sync')
# Hard-wired to always sort descriptions by author, with series after non-series
opts.sort_descriptions_by_author = True
build_log = []
build_log.append(u"%s(): Generating %s %sin %s environment" %
(self.name,self.fmt,'for %s ' % opts.output_profile if opts.output_profile else '',
'CLI' if opts.cli_environment else 'GUI'))
# If exclude_genre is blank, assume user wants all genre tags included
if opts.exclude_genre.strip() == '':
opts.exclude_genre = '\[^.\]'
build_log.append(" converting empty exclude_genre to '\[^.\]'")
if opts.connected_device['is_device_connected'] and \
opts.connected_device['kind'] == 'device':
if opts.connected_device['serial']:
build_log.append(u" connected_device: '%s' #%s%s " % \
(opts.connected_device['name'],
opts.connected_device['serial'][0:4],
'x' * (len(opts.connected_device['serial']) - 4)))
for storage in opts.connected_device['storage']:
if storage:
build_log.append(u" mount point: %s" % storage)
else:
build_log.append(u" connected_device: '%s'" % opts.connected_device['name'])
try:
for storage in opts.connected_device['storage']:
if storage:
build_log.append(u" mount point: %s" % storage)
except:
build_log.append(u" (no mount points)")
else:
build_log.append(u" connected_device: '%s'" % opts.connected_device['name'])
opts_dict = vars(opts)
if opts_dict['ids']:
build_log.append(" book count: %d" % len(opts_dict['ids']))
sections_list = []
if opts.generate_authors:
sections_list.append('Authors')
if opts.generate_titles:
sections_list.append('Titles')
if opts.generate_series:
sections_list.append('Series')
if opts.generate_genres:
sections_list.append('Genres')
if opts.generate_recently_added:
sections_list.append('Recently Added')
if opts.generate_descriptions:
sections_list.append('Descriptions')
if not sections_list:
if opts.cli_environment:
opts.log.warn('*** No Section switches specified, enabling all Sections ***')
opts.generate_authors = True
opts.generate_titles = True
opts.generate_series = True
opts.generate_genres = True
opts.generate_recently_added = True
opts.generate_descriptions = True
sections_list = ['Authors','Titles','Series','Genres','Recently Added','Descriptions']
else:
opts.log.warn('\n*** No enabled Sections, terminating catalog generation ***')
return ["No Included Sections","No enabled Sections.\nCheck E-book options tab\n'Included sections'\n"]
if opts.fmt == 'mobi' and sections_list == ['Descriptions']:
warning = _("\n*** Adding 'By Authors' Section required for MOBI output ***")
opts.log.warn(warning)
sections_list.insert(0,'Authors')
opts.generate_authors = True
opts.log(u" Sections: %s" % ', '.join(sections_list))
opts.section_list = sections_list
# Limit thumb_width to 1.0" - 2.0"
try:
if float(opts.thumb_width) < float(self.THUMB_SMALLEST):
log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_SMALLEST))
opts.thumb_width = self.THUMB_SMALLEST
if float(opts.thumb_width) > float(self.THUMB_LARGEST):
log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_LARGEST))
opts.thumb_width = self.THUMB_LARGEST
opts.thumb_width = "%.2f" % float(opts.thumb_width)
except:
log.error("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_SMALLEST))
opts.thumb_width = "1.0"
# Display opts
keys = opts_dict.keys()
keys.sort()
build_log.append(" opts:")
for key in keys:
if key in ['catalog_title','authorClip','connected_kindle','descriptionClip',
'exclude_book_marker','exclude_genre','exclude_tags',
'header_note_source_field','merge_comments',
'output_profile','read_book_marker',
'search_text','sort_by','sort_descriptions_by_author','sync',
'thumb_width','wishlist_tag']:
build_log.append(" %s: %s" % (key, repr(opts_dict[key])))
if opts.verbose:
log('\n'.join(line for line in build_log))
self.opts = opts
# Launch the Catalog builder
catalog = CatalogBuilder(db, opts, self, report_progress=notification)
if opts.verbose:
log.info(" Begin catalog source generation")
catalog.createDirectoryStructure()
catalog.copyResources()
catalog.calculateThumbnailSize()
catalog_source_built = catalog.buildSources()
if opts.verbose:
if catalog_source_built:
log.info(" Completed catalog source generation\n")
else:
log.error(" *** Terminated catalog generation, check log for details ***")
if catalog_source_built:
recommendations = []
recommendations.append(('remove_fake_margins', False,
OptionRecommendation.HIGH))
recommendations.append(('comments', '', OptionRecommendation.HIGH))
# Use to debug generated catalog code before conversion
#setattr(opts,'debug_pipeline',os.path.expanduser("~/Desktop/Catalog debug"))
dp = getattr(opts, 'debug_pipeline', None)
if dp is not None:
recommendations.append(('debug_pipeline', dp,
OptionRecommendation.HIGH))
if opts.fmt == 'mobi' and opts.output_profile and opts.output_profile.startswith("kindle"):
recommendations.append(('output_profile', opts.output_profile,
OptionRecommendation.HIGH))
recommendations.append(('no_inline_toc', True,
OptionRecommendation.HIGH))
recommendations.append(('book_producer',opts.output_profile,
OptionRecommendation.HIGH))
# If cover exists, use it
cpath = None
try:
search_text = 'title:"%s" author:%s' % (
opts.catalog_title.replace('"', '\\"'), 'calibre')
matches = db.search(search_text, return_matches=True)
if matches:
cpath = db.cover(matches[0], index_is_id=True, as_path=True)
if cpath and os.path.exists(cpath):
recommendations.append(('cover', cpath,
OptionRecommendation.HIGH))
except:
pass
# Run ebook-convert
from calibre.ebooks.conversion.plumber import Plumber
plumber = Plumber(os.path.join(catalog.catalogPath,
opts.basename + '.opf'), path_to_output, log, report_progress=notification,
abort_after_input_dump=False)
plumber.merge_ui_recommendations(recommendations)
plumber.run()
try:
os.remove(cpath)
except:
pass
# returns to gui2.actions.catalog:catalog_generated()
return catalog.error
for form in browser.forms():
browser.form = form
login_populated, password_populated = False, False
for possible_login_field in ['email', 'login']:
try:
browser[possible_login_field] = login
login_populated = True
except ControlNotFoundError, e:
pass
if login_populated:
break
try:
browser['password'] = password
password_populated = True
except ControlNotFoundError, e:
pass
if login_populated and password_populated:
response = browser.submit()
break
if "Logout" in response.read():
default_log.debug("Login seemed successful")
success = True
else:
success = False
# subsequent requests using this browser will fail. oh well. :(
# we can't do much about it here.
default_log.error("Login seemed to fail on page %s" % (config.get('auth_url')))
return success
def run(self, path_to_output, opts, db, notification=DummyReporter()):
from calibre.library import current_library_name
from calibre.utils.date import isoformat
from calibre.utils.html2text import html2text
from calibre.utils.logging import default_log as log
from lxml import etree
self.fmt = path_to_output.rpartition(".")[2]
self.notification = notification
current_library = current_library_name()
if getattr(opts, "library_path", None):
current_library = os.path.basename(opts.library_path)
if opts.verbose:
opts_dict = vars(opts)
log("%s('%s'): Generating %s" % (self.name, current_library, self.fmt.upper()))
if opts.connected_device["is_device_connected"]:
log(" connected_device: %s" % opts.connected_device["name"])
if opts_dict["search_text"]:
log(" --search='%s'" % opts_dict["search_text"])
if opts_dict["ids"]:
log(" Book count: %d" % len(opts_dict["ids"]))
if opts_dict["search_text"]:
log(" (--search ignored when a subset of the database is specified)")
if opts_dict["fields"]:
if opts_dict["fields"] == "all":
log(" Fields: %s" % ", ".join(FIELDS[1:]))
else:
log(" Fields: %s" % opts_dict["fields"])
# If a list of ids are provided, don't use search_text
if opts.ids:
opts.search_text = None
data = self.search_sort_db(db, opts)
if not len(data):
log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text)
# raise SystemExit(1)
# Get the requested output fields as a list
fields = self.get_output_fields(db, opts)
# If connected device, add 'On Device' values to data
if opts.connected_device["is_device_connected"] and "ondevice" in fields:
for entry in data:
entry["ondevice"] = db.catalog_plugin_on_device_temp_mapping[entry["id"]]["ondevice"]
fm = {x: db.field_metadata.get(x, {}) for x in fields}
if self.fmt == "csv":
outfile = codecs.open(path_to_output, "w", "utf8")
# Write a UTF-8 BOM
outfile.write("\xef\xbb\xbf")
# Output the field headers
outfile.write(u"%s\n" % u",".join(fields))
# Output the entry fields
for entry in data:
outstr = []
for field in fields:
if field.startswith("#"):
item = db.get_field(entry["id"], field, index_is_id=True)
elif field == "library_name":
item = current_library
elif field == "title_sort":
item = entry["sort"]
else:
item = entry[field]
if item is None:
outstr.append('""')
continue
elif field == "formats":
fmt_list = []
for format in item:
fmt_list.append(format.rpartition(".")[2].lower())
item = ", ".join(fmt_list)
elif field in ["authors", "tags"]:
item = ", ".join(item)
elif field == "isbn":
# Could be 9, 10 or 13 digits, with hyphens, possibly ending in 'X'
item = u"%s" % re.sub(r"[^\dX-]", "", item)
elif field in ["pubdate", "timestamp"]:
item = isoformat(item, as_utc=False)
elif field == "comments":
item = item.replace(u"\r\n", u" ")
item = item.replace(u"\n", u" ")
elif fm.get(field, {}).get("datatype", None) == "rating" and item:
item = u"%.2g" % (item / 2.0)
# Convert HTML to markdown text
if type(item) is unicode:
opening_tag = re.search("<(\w+)(\x20|>)", item)
if opening_tag:
closing_tag = re.search("<\/%s>$" % opening_tag.group(1), item)
if closing_tag:
item = html2text(item)
outstr.append(u'"%s"' % unicode(item).replace('"', '""'))
outfile.write(u",".join(outstr) + u"\n")
outfile.close()
elif self.fmt == "xml":
from lxml.builder import E
root = E.calibredb()
for r in data:
record = E.record()
root.append(record)
for field in fields:
if field.startswith("#"):
val = db.get_field(r["id"], field, index_is_id=True)
if not isinstance(val, (str, unicode)):
val = unicode(val)
item = getattr(E, field.replace("#", "_"))(val)
record.append(item)
for field in ("id", "uuid", "publisher", "rating", "size", "isbn", "ondevice", "identifiers"):
if field in fields:
val = r[field]
if not val:
continue
if not isinstance(val, (str, unicode)):
if fm.get(field, {}).get("datatype", None) == "rating" and val:
val = u"%.2g" % (val / 2.0)
val = unicode(val)
item = getattr(E, field)(val)
record.append(item)
if "title" in fields:
title = E.title(r["title"], sort=r["sort"])
record.append(title)
if "authors" in fields:
aus = E.authors(sort=r["author_sort"])
for au in r["authors"]:
aus.append(E.author(au))
record.append(aus)
for field in ("timestamp", "pubdate"):
if field in fields:
record.append(getattr(E, field)(isoformat(r[field], as_utc=False)))
if "tags" in fields and r["tags"]:
tags = E.tags()
for tag in r["tags"]:
tags.append(E.tag(tag))
record.append(tags)
if "comments" in fields and r["comments"]:
record.append(E.comments(r["comments"]))
if "series" in fields and r["series"]:
record.append(E.series(r["series"], index=str(r["series_index"])))
if "cover" in fields and r["cover"]:
record.append(E.cover(r["cover"].replace(os.sep, "/")))
if "formats" in fields and r["formats"]:
fmt = E.formats()
for f in r["formats"]:
fmt.append(E.format(f.replace(os.sep, "/")))
record.append(fmt)
if "library_name" in fields:
record.append(E.library_name(current_library))
with open(path_to_output, "w") as f:
f.write(etree.tostring(root, encoding="utf-8", xml_declaration=True, pretty_print=True))
