def strftime(fmt='%Y/%m/%d %H:%M:%S', dt=None): if not hasattr(dt, 'timetuple'): dt = nowf() dt = dt.timetuple() try: return _strftime(fmt, dt) except: return _strftime(fmt, nowf().timetuple())
def recipe_needs_to_be_downloaded(self, recipe): try: typ, sch, ld = self.un_serialize_schedule(recipe) except: return False def is_time(now, hour, minute): return now.hour > hour or \ (now.hour == hour and now.minute >= minute) def is_weekday(day, now): return day < 0 or day > 6 or \ day == calendar.weekday(now.year, now.month, now.day) def was_downloaded_already_today(ld_local, now): return ld_local.date() == now.date() if typ == 'interval': return utcnow() - ld > timedelta(sch) elif typ == 'day/time': now = nowf() try: ld_local = ld.astimezone(local_tz) except Exception: return False day, hour, minute = sch return is_weekday(day, now) and \ not was_downloaded_already_today(ld_local, now) and \ is_time(now, hour, minute) elif typ == 'days_of_week': now = nowf() try: ld_local = ld.astimezone(local_tz) except Exception: return False days, hour, minute = sch have_day = False for day in days: if is_weekday(day, now): have_day = True break return have_day and \ not was_downloaded_already_today(ld_local, now) and \ is_time(now, hour, minute) elif typ == 'days_of_month': now = nowf() try: ld_local = ld.astimezone(local_tz) except Exception: return False days, hour, minute = sch have_day = now.day in days return have_day and \ not was_downloaded_already_today(ld_local, now) and \ is_time(now, hour, minute) return False
def update_last_modified(self, book_ids, now=None): if now is None: now = nowf() if book_ids: f = self.fields['last_modified'] f.writer.set_books({book_id: now for book_id in book_ids}, self.backend)
def update(self, mi): mi.title = normalize(mi.title) def update_exth_record(rec): recs.append(rec) if rec[0] in self.original_exth_records: self.original_exth_records.pop(rec[0]) if self.type != "BOOKMOBI": raise MobiError("Setting metadata only supported for MOBI files of type 'BOOK'.\n" "\tThis is a %r file of type %r" % (self.type[0:4], self.type[4:8])) recs = [] added_501 = False try: from calibre.ebooks.conversion.config import load_defaults prefs = load_defaults('mobi_output') pas = prefs.get('prefer_author_sort', False) kindle_pdoc = prefs.get('personal_doc', None) share_not_sync = prefs.get('share_not_sync', False) except: pas = False kindle_pdoc = None share_not_sync = False if mi.author_sort and pas: # We want an EXTH field per author... authors = mi.author_sort.split(' & ') for author in authors: update_exth_record((100, normalize(author).encode(self.codec, 'replace'))) elif mi.authors: authors = mi.authors for author in authors: update_exth_record((100, normalize(author).encode(self.codec, 'replace'))) if mi.publisher: update_exth_record((101, normalize(mi.publisher).encode(self.codec, 'replace'))) if mi.comments: # Strip user annotations a_offset = mi.comments.find('<div class="user_annotations">') ad_offset = mi.comments.find('<hr class="annotations_divider" />') if a_offset >= 0: mi.comments = mi.comments[:a_offset] if ad_offset >= 0: mi.comments = mi.comments[:ad_offset] update_exth_record((103, normalize(mi.comments).encode(self.codec, 'replace'))) if mi.isbn: update_exth_record((104, mi.isbn.encode(self.codec, 'replace'))) if mi.tags: # FIXME: Keep a single subject per EXTH field? subjects = '; '.join(mi.tags) update_exth_record((105, normalize(subjects).encode(self.codec, 'replace'))) if kindle_pdoc and kindle_pdoc in mi.tags: added_501 = True update_exth_record((501, b'PDOC')) if mi.pubdate: update_exth_record((106, str(mi.pubdate).encode(self.codec, 'replace'))) elif mi.timestamp: update_exth_record((106, str(mi.timestamp).encode(self.codec, 'replace'))) elif self.timestamp: update_exth_record((106, self.timestamp)) else: update_exth_record((106, nowf().isoformat().encode(self.codec, 'replace'))) if self.cover_record is not None: update_exth_record((201, pack('>I', self.cover_rindex))) update_exth_record((203, pack('>I', 0))) if self.thumbnail_record is not None: update_exth_record((202, pack('>I', self.thumbnail_rindex))) # Add a 113 record if not present to allow Amazon syncing if (113 not in self.original_exth_records and self.original_exth_records.get(501, None) == 'EBOK' and not added_501 and not share_not_sync): from uuid import uuid4 update_exth_record((113, str(uuid4()))) # Add a 112 record with actual UUID if getattr(mi, 'uuid', None): update_exth_record((112, (u"calibre:%s" % mi.uuid).encode(self.codec, 'replace'))) if 503 in self.original_exth_records: update_exth_record((503, mi.title.encode(self.codec, 'replace'))) # Update book producer if getattr(mi, 'book_producer', False): update_exth_record((108, mi.book_producer.encode(self.codec, 'replace'))) # Set langcode in EXTH header if not mi.is_null('language'): lang = canonicalize_lang(mi.language) lang = lang_as_iso639_1(lang) or lang if lang: update_exth_record((524, lang.encode(self.codec, 'replace'))) # Include remaining original EXTH fields for id in sorted(self.original_exth_records): recs.append((id, self.original_exth_records[id])) recs = sorted(recs, key=lambda x:(x[0],x[0])) exth = StringIO() for code, data in recs: exth.write(pack('>II', code, len(data) + 8)) exth.write(data) exth = exth.getvalue() trail = len(exth) % 4 pad = '\0' * (4 - trail) # Always pad w/ at least 1 byte exth = ['EXTH', pack('>II', len(exth) + 12, len(recs)), exth, pad] exth = ''.join(exth) if getattr(self, 'exth', None) is None: raise MobiError('No existing EXTH record. Cannot update metadata.') if not mi.is_null('language'): self.record0[92:96] = iana2mobi(mi.language) self.create_exth(exth=exth, new_title=mi.title) # Fetch updated timestamp, cover_record, thumbnail_record self.fetchEXTHFields() if mi.cover_data[1] or mi.cover: try: data = mi.cover_data[1] if mi.cover_data[1] else open(mi.cover, 'rb').read() except: pass else: if is_image(self.cover_record): size = len(self.cover_record) cover = rescale_image(data, size) if len(cover) <= size: cover += b'\0' * (size - len(cover)) self.cover_record[:] = cover if is_image(self.thumbnail_record): size = len(self.thumbnail_record) thumbnail = rescale_image(data, size, dimen=MAX_THUMB_DIMEN) if len(thumbnail) <= size: thumbnail += b'\0' * (size - len(thumbnail)) self.thumbnail_record[:] = thumbnail return
def update(self, mi, asin=None): mi.title = normalize(mi.title) def update_exth_record(rec): recs.append(rec) if rec[0] in self.original_exth_records: self.original_exth_records.pop(rec[0]) if self.type != b"BOOKMOBI": raise MobiError( "Setting metadata only supported for MOBI files of type 'BOOK'.\n" "\tThis is a %r file of type %r" % (self.type[0:4], self.type[4:8])) recs = [] added_501 = False try: from calibre.ebooks.conversion.config import load_defaults prefs = load_defaults('mobi_output') pas = prefs.get('prefer_author_sort', False) kindle_pdoc = prefs.get('personal_doc', None) share_not_sync = prefs.get('share_not_sync', False) except: pas = False kindle_pdoc = None share_not_sync = False if mi.author_sort and pas: # We want an EXTH field per author... authors = mi.author_sort.split(' & ') for author in authors: update_exth_record( (100, normalize(author).encode(self.codec, 'replace'))) elif mi.authors: authors = mi.authors for author in authors: update_exth_record( (100, normalize(author).encode(self.codec, 'replace'))) if mi.publisher: update_exth_record( (101, normalize(mi.publisher).encode(self.codec, 'replace'))) if mi.comments: # Strip user annotations a_offset = mi.comments.find('<div class="user_annotations">') ad_offset = mi.comments.find('<hr class="annotations_divider" />') if a_offset >= 0: mi.comments = mi.comments[:a_offset] if ad_offset >= 0: mi.comments = mi.comments[:ad_offset] update_exth_record( (103, normalize(mi.comments).encode(self.codec, 'replace'))) if mi.isbn: update_exth_record((104, mi.isbn.encode(self.codec, 'replace'))) if mi.tags: # FIXME: Keep a single subject per EXTH field? subjects = '; '.join(mi.tags) update_exth_record( (105, normalize(subjects).encode(self.codec, 'replace'))) if kindle_pdoc and kindle_pdoc in mi.tags: added_501 = True update_exth_record((501, b'PDOC')) if mi.pubdate: update_exth_record( (106, unicode_type(mi.pubdate).encode(self.codec, 'replace'))) elif mi.timestamp: update_exth_record( (106, unicode_type(mi.timestamp).encode(self.codec, 'replace'))) elif self.timestamp: update_exth_record((106, self.timestamp)) else: update_exth_record( (106, nowf().isoformat().encode(self.codec, 'replace'))) if self.cover_record is not None: update_exth_record((201, pack('>I', self.cover_rindex))) update_exth_record((203, pack('>I', 0))) if self.thumbnail_record is not None: update_exth_record((202, pack('>I', self.thumbnail_rindex))) # Add a 113 record if not present to allow Amazon syncing if (113 not in self.original_exth_records and self.original_exth_records.get(501, None) == 'EBOK' and not added_501 and not share_not_sync): from uuid import uuid4 update_exth_record((113, unicode_type(uuid4()).encode(self.codec))) if asin is not None: update_exth_record((113, asin.encode(self.codec))) update_exth_record((504, asin.encode(self.codec))) # Add a 112 record with actual UUID if getattr(mi, 'uuid', None): update_exth_record( (112, ("calibre:%s" % mi.uuid).encode(self.codec, 'replace'))) if 503 in self.original_exth_records: update_exth_record((503, mi.title.encode(self.codec, 'replace'))) # Update book producer if getattr(mi, 'book_producer', False): update_exth_record( (108, mi.book_producer.encode(self.codec, 'replace'))) # Set langcode in EXTH header if not mi.is_null('language'): lang = canonicalize_lang(mi.language) lang = lang_as_iso639_1(lang) or lang if lang: update_exth_record((524, lang.encode(self.codec, 'replace'))) # Include remaining original EXTH fields for id in sorted(self.original_exth_records): recs.append((id, self.original_exth_records[id])) recs = sorted(recs, key=lambda x: (x[0], x[0])) exth = io.BytesIO() for code, data in recs: exth.write(pack('>II', code, len(data) + 8)) exth.write(data) exth = exth.getvalue() trail = len(exth) % 4 pad = b'\0' * (4 - trail) # Always pad w/ at least 1 byte exth = [b'EXTH', pack('>II', len(exth) + 12, len(recs)), exth, pad] exth = b''.join(exth) if getattr(self, 'exth', None) is None: raise MobiError('No existing EXTH record. Cannot update metadata.') if not mi.is_null('language'): self.record0[92:96] = iana2mobi(mi.language) self.create_exth(exth=exth, new_title=mi.title) # Fetch updated timestamp, cover_record, thumbnail_record self.fetchEXTHFields() if mi.cover_data[1] or mi.cover: try: data = mi.cover_data[1] if not data: with open(mi.cover, 'rb') as f: data = f.read() except: pass else: if is_image(self.cover_record): size = len(self.cover_record) cover = rescale_image(data, size) if len(cover) <= size: cover += b'\0' * (size - len(cover)) self.cover_record[:] = cover if is_image(self.thumbnail_record): size = len(self.thumbnail_record) thumbnail = rescale_image(data, size, dimen=MAX_THUMB_DIMEN) if len(thumbnail) <= size: thumbnail += b'\0' * (size - len(thumbnail)) self.thumbnail_record[:] = thumbnail return
def run(self, path_to_output, opts, db, notification=DummyReporter()): from calibre.utils.date import isoformat from calibre.utils.html2text import html2text from calibre.utils.bibtex import BibTeX from calibre.library.save_to_disk import preprocess_template from calibre.utils.date import now as nowf from calibre.utils.logging import default_log as log library_name = os.path.basename(db.library_path) def create_bibtex_entry(entry, fields, mode, template_citation, bibtexdict, db, citation_bibtex=True, calibre_files=True): #Bibtex doesn't like UTF-8 but keep unicode until writing #Define starting chain or if book valid strict and not book return a Fail string bibtex_entry = [] if mode != "misc" and check_entry_book_valid(entry): bibtex_entry.append(u'@book{') elif mode != "book": bibtex_entry.append(u'@misc{') else: #case strict book return '' if citation_bibtex: # Citation tag bibtex_entry.append( make_bibtex_citation(entry, template_citation, bibtexdict)) bibtex_entry = [u' '.join(bibtex_entry)] for field in fields: if field.startswith('#'): item = db.get_field(entry['id'], field, index_is_id=True) if isinstance(item, (bool, float, int)): item = repr(item) elif field == 'title_sort': item = entry['sort'] elif field == 'library_name': item = library_name else: item = entry[field] #check if the field should be included (none or empty) if item is None: continue try: if len(item) == 0: continue except TypeError: pass if field == 'authors': bibtex_entry.append(u'author = "%s"' % bibtexdict.bibtex_author_format(item)) elif field == 'id': bibtex_entry.append(u'calibreid = "%s"' % int(item)) elif field == 'rating': bibtex_entry.append(u'rating = "%s"' % int(item)) elif field == 'size': bibtex_entry.append(u'%s = "%s octets"' % (field, int(item))) elif field == 'tags': #A list to flatten bibtex_entry.append( u'tags = "%s"' % bibtexdict.utf8ToBibtex(u', '.join(item))) elif field == 'comments': #\n removal item = item.replace(u'\r\n', u' ') item = item.replace(u'\n', u' ') # unmatched brace removal (users should use \leftbrace or \rightbrace for single braces) item = bibtexdict.stripUnmatchedSyntax(item, u'{', u'}') #html to text try: item = html2text(item) except: log.warn("Failed to convert comments to text") bibtex_entry.append(u'note = "%s"' % bibtexdict.utf8ToBibtex(item)) elif field == 'isbn': # Could be 9, 10 or 13 digits bibtex_entry.append(u'isbn = "%s"' % format_isbn(item)) elif field == 'formats': #Add file path if format is selected formats = [ format.rpartition('.')[2].lower() for format in item ] bibtex_entry.append(u'formats = "%s"' % u', '.join(formats)) if calibre_files: files = [u':%s:%s' % (format, format.rpartition('.')[2].upper())\ for format in item] bibtex_entry.append(u'file = "%s"' % u', '.join(files)) elif field == 'series_index': bibtex_entry.append(u'volume = "%s"' % int(item)) elif field == 'timestamp': bibtex_entry.append(u'timestamp = "%s"' % isoformat(item).partition('T')[0]) elif field == 'pubdate': bibtex_entry.append(u'year = "%s"' % item.year) bibtex_entry.append( u'month = "%s"' % bibtexdict.utf8ToBibtex(strftime("%b", item))) elif field.startswith('#') and isinstance(item, basestring): bibtex_entry.append( u'custom_%s = "%s"' % (field[1:], bibtexdict.utf8ToBibtex(item))) elif isinstance(item, basestring): # elif field in ['title', 'publisher', 'cover', 'uuid', 'ondevice', # 'author_sort', 'series', 'title_sort'] : bibtex_entry.append(u'%s = "%s"' % (field, bibtexdict.utf8ToBibtex(item))) bibtex_entry = u',\n '.join(bibtex_entry) bibtex_entry += u' }\n\n' return bibtex_entry def check_entry_book_valid(entry): #Check that the required fields are ok for a book entry for field in ['title', 'authors', 'publisher']: if entry[field] is None or len(entry[field]) == 0: return False if entry['pubdate'] is None: return False else: return True def make_bibtex_citation(entry, template_citation, bibtexclass): #define a function to replace the template entry by its value def tpl_replace(objtplname): tpl_field = re.sub(u'[\{\}]', u'', objtplname.group()) if tpl_field in TEMPLATE_ALLOWED_FIELDS: if tpl_field in ['pubdate', 'timestamp']: tpl_field = isoformat( entry[tpl_field]).partition('T')[0] elif tpl_field in ['tags', 'authors']: tpl_field = entry[tpl_field][0] elif tpl_field in ['id', 'series_index']: tpl_field = str(entry[tpl_field]) else: tpl_field = entry[tpl_field] return tpl_field else: return u'' if len(template_citation) > 0: tpl_citation = bibtexclass.utf8ToBibtex( bibtexclass.ValidateCitationKey( re.sub(u'\{[^{}]*\}', tpl_replace, template_citation))) if len(tpl_citation) > 0: return tpl_citation if len(entry["isbn"]) > 0: template_citation = u'%s' % re.sub(u'[\D]', u'', entry["isbn"]) else: template_citation = u'%s' % str(entry["id"]) return bibtexclass.ValidateCitationKey(template_citation) self.fmt = path_to_output.rpartition('.')[2] self.notification = notification # Combobox options bibfile_enc = ['utf8', 'cp1252', 'ascii'] bibfile_enctag = ['strict', 'replace', 'ignore', 'backslashreplace'] bib_entry = ['mixed', 'misc', 'book'] # Needed beacause CLI return str vs int by widget try: bibfile_enc = bibfile_enc[opts.bibfile_enc] bibfile_enctag = bibfile_enctag[opts.bibfile_enctag] bib_entry = bib_entry[opts.bib_entry] except: if opts.bibfile_enc in bibfile_enc: bibfile_enc = opts.bibfile_enc else: log.warn("Incorrect --choose-encoding flag, revert to default") bibfile_enc = bibfile_enc[0] if opts.bibfile_enctag in bibfile_enctag: bibfile_enctag = opts.bibfile_enctag else: log.warn( "Incorrect --choose-encoding-configuration flag, revert to default" ) bibfile_enctag = bibfile_enctag[0] if opts.bib_entry in bib_entry: bib_entry = opts.bib_entry else: log.warn("Incorrect --entry-type flag, revert to default") bib_entry = bib_entry[0] if opts.verbose: opts_dict = vars(opts) log("%s(): Generating %s" % (self.name, self.fmt)) if opts.connected_device['is_device_connected']: log(" connected_device: %s" % opts.connected_device['name']) if opts_dict['search_text']: log(" --search='%s'" % opts_dict['search_text']) if opts_dict['ids']: log(" Book count: %d" % len(opts_dict['ids'])) if opts_dict['search_text']: log(" (--search ignored when a subset of the database is specified)" ) if opts_dict['fields']: if opts_dict['fields'] == 'all': log(" Fields: %s" % ', '.join(FIELDS[1:])) else: log(" Fields: %s" % opts_dict['fields']) log(" Output file will be encoded in %s with %s flag" % (bibfile_enc, bibfile_enctag)) log(" BibTeX entry type is %s with a citation like '%s' flag" % (bib_entry, opts_dict['bib_cit'])) # If a list of ids are provided, don't use search_text if opts.ids: opts.search_text = None data = self.search_sort_db(db, opts) if not len(data): log.error( "\nNo matching database entries for search criteria '%s'" % opts.search_text) # Get the requested output fields as a list fields = self.get_output_fields(db, opts) if not len(data): log.error( "\nNo matching database entries for search criteria '%s'" % opts.search_text) #Initialize BibTeX class bibtexc = BibTeX() #Entries writing after Bibtex formating (or not) if bibfile_enc != 'ascii': bibtexc.ascii_bibtex = False else: bibtexc.ascii_bibtex = True #Check citation choice and go to default in case of bad CLI if isinstance(opts.impcit, (StringType, UnicodeType)): if opts.impcit == 'False': citation_bibtex = False elif opts.impcit == 'True': citation_bibtex = True else: log.warn("Incorrect --create-citation, revert to default") citation_bibtex = True else: citation_bibtex = opts.impcit #Check add file entry and go to default in case of bad CLI if isinstance(opts.addfiles, (StringType, UnicodeType)): if opts.addfiles == 'False': addfiles_bibtex = False elif opts.addfiles == 'True': addfiles_bibtex = True else: log.warn("Incorrect --add-files-path, revert to default") addfiles_bibtex = True else: addfiles_bibtex = opts.addfiles #Preprocess for error and light correction template_citation = preprocess_template(opts.bib_cit) #Open output and write entries with codecs.open(path_to_output, 'w', bibfile_enc, bibfile_enctag)\ as outfile: #File header nb_entries = len(data) #check in book strict if all is ok else throw a warning into log if bib_entry == 'book': nb_books = len(filter(check_entry_book_valid, data)) if nb_books < nb_entries: log.warn("Only %d entries in %d are book compatible" % (nb_books, nb_entries)) nb_entries = nb_books # If connected device, add 'On Device' values to data if opts.connected_device[ 'is_device_connected'] and 'ondevice' in fields: for entry in data: entry[ 'ondevice'] = db.catalog_plugin_on_device_temp_mapping[ entry['id']]['ondevice'] outfile.write( u'%%%Calibre catalog\n%%%{0} entries in catalog\n\n'.format( nb_entries)) outfile.write( u'@preamble{"This catalog of %d entries was generated by calibre on %s"}\n\n' % (nb_entries, nowf().strftime("%A, %d. %B %Y %H:%M").decode( preferred_encoding))) for entry in data: outfile.write( create_bibtex_entry(entry, fields, bib_entry, template_citation, bibtexc, db, citation_bibtex, addfiles_bibtex))
def _get_metadata(self, book_id, get_user_categories=True): # {{{ mi = Metadata(None, template_cache=self.formatter_template_cache) author_ids = self._field_ids_for('authors', book_id) aut_list = [self._author_data(i) for i in author_ids] aum = [] aus = {} aul = {} for rec in aut_list: aut = rec['name'] aum.append(aut) aus[aut] = rec['sort'] aul[aut] = rec['link'] mi.title = self._field_for('title', book_id, default_value=_('Unknown')) mi.authors = aum mi.author_sort = self._field_for('author_sort', book_id, default_value=_('Unknown')) mi.author_sort_map = aus mi.author_link_map = aul mi.comments = self._field_for('comments', book_id) mi.publisher = self._field_for('publisher', book_id) n = nowf() mi.timestamp = self._field_for('timestamp', book_id, default_value=n) mi.pubdate = self._field_for('pubdate', book_id, default_value=n) mi.uuid = self._field_for('uuid', book_id, default_value='dummy') mi.title_sort = self._field_for('sort', book_id, default_value=_('Unknown')) mi.book_size = self._field_for('size', book_id, default_value=0) mi.ondevice_col = self._field_for('ondevice', book_id, default_value='') mi.last_modified = self._field_for('last_modified', book_id, default_value=n) formats = self._field_for('formats', book_id) mi.format_metadata = {} mi.languages = list(self._field_for('languages', book_id)) if not formats: good_formats = None else: mi.format_metadata = FormatMetadata(self, book_id, formats) good_formats = FormatsList(formats, mi.format_metadata) mi.formats = good_formats mi.has_cover = _('Yes') if self._field_for( 'cover', book_id, default_value=False) else '' mi.tags = list(self._field_for('tags', book_id, default_value=())) mi.series = self._field_for('series', book_id) if mi.series: mi.series_index = self._field_for('series_index', book_id, default_value=1.0) mi.rating = self._field_for('rating', book_id) mi.set_identifiers( self._field_for('identifiers', book_id, default_value={})) mi.application_id = book_id mi.id = book_id composites = [] for key, meta in self.field_metadata.custom_iteritems(): mi.set_user_metadata(key, meta) if meta['datatype'] == 'composite': composites.append(key) else: val = self._field_for(key, book_id) if isinstance(val, tuple): val = list(val) extra = self._field_for(key + '_index', book_id) mi.set(key, val=val, extra=extra) for key in composites: mi.set(key, val=self._composite_for(key, book_id, mi)) user_cat_vals = {} if get_user_categories: user_cats = self.backend.prefs['user_categories'] for ucat in user_cats: res = [] for name, cat, ign in user_cats[ucat]: v = mi.get(cat, None) if isinstance(v, list): if name in v: res.append([name, cat]) elif name == v: res.append([name, cat]) user_cat_vals[ucat] = res mi.user_categories = user_cat_vals return mi
def update_last_modified(self, book_ids, now=None): if now is None: now = nowf() if book_ids: f = self.fields['last_modified'] f.writer.set_books({book_id:now for book_id in book_ids}, self.backend)
def _get_metadata(self, book_id, get_user_categories=True): # {{{ mi = Metadata(None, template_cache=self.formatter_template_cache) author_ids = self._field_ids_for('authors', book_id) aut_list = [self._author_data(i) for i in author_ids] aum = [] aus = {} aul = {} for rec in aut_list: aut = rec['name'] aum.append(aut) aus[aut] = rec['sort'] aul[aut] = rec['link'] mi.title = self._field_for('title', book_id, default_value=_('Unknown')) mi.authors = aum mi.author_sort = self._field_for('author_sort', book_id, default_value=_('Unknown')) mi.author_sort_map = aus mi.author_link_map = aul mi.comments = self._field_for('comments', book_id) mi.publisher = self._field_for('publisher', book_id) n = nowf() mi.timestamp = self._field_for('timestamp', book_id, default_value=n) mi.pubdate = self._field_for('pubdate', book_id, default_value=n) mi.uuid = self._field_for('uuid', book_id, default_value='dummy') mi.title_sort = self._field_for('sort', book_id, default_value=_('Unknown')) mi.book_size = self._field_for('size', book_id, default_value=0) mi.ondevice_col = self._field_for('ondevice', book_id, default_value='') mi.last_modified = self._field_for('last_modified', book_id, default_value=n) formats = self._field_for('formats', book_id) mi.format_metadata = {} mi.languages = list(self._field_for('languages', book_id)) if not formats: good_formats = None else: mi.format_metadata = FormatMetadata(self, book_id, formats) good_formats = FormatsList(formats, mi.format_metadata) mi.formats = good_formats mi.has_cover = _('Yes') if self._field_for('cover', book_id, default_value=False) else '' mi.tags = list(self._field_for('tags', book_id, default_value=())) mi.series = self._field_for('series', book_id) if mi.series: mi.series_index = self._field_for('series_index', book_id, default_value=1.0) mi.rating = self._field_for('rating', book_id) mi.set_identifiers(self._field_for('identifiers', book_id, default_value={})) mi.application_id = book_id mi.id = book_id composites = [] for key, meta in self.field_metadata.custom_iteritems(): mi.set_user_metadata(key, meta) if meta['datatype'] == 'composite': composites.append(key) else: val = self._field_for(key, book_id) if isinstance(val, tuple): val = list(val) extra = self._field_for(key+'_index', book_id) mi.set(key, val=val, extra=extra) for key in composites: mi.set(key, val=self._composite_for(key, book_id, mi)) user_cat_vals = {} if get_user_categories: user_cats = self.backend.prefs['user_categories'] for ucat in user_cats: res = [] for name,cat,ign in user_cats[ucat]: v = mi.get(cat, None) if isinstance(v, list): if name in v: res.append([name,cat]) elif name == v: res.append([name,cat]) user_cat_vals[ucat] = res mi.user_categories = user_cat_vals return mi
def run(self, path_to_output, opts, db, notification=DummyReporter()): from calibre.utils.date import isoformat from calibre.utils.html2text import html2text from calibre.utils.bibtex import BibTeX from calibre.library.save_to_disk import preprocess_template from calibre.utils.date import now as nowf from calibre.utils.logging import default_log as log library_name = os.path.basename(db.library_path) def create_bibtex_entry(entry, fields, mode, template_citation, bibtexdict, db, citation_bibtex=True, calibre_files=True): #Bibtex doesn't like UTF-8 but keep unicode until writing #Define starting chain or if book valid strict and not book return a Fail string bibtex_entry = [] if mode != "misc" and check_entry_book_valid(entry) : bibtex_entry.append(u'@book{') elif mode != "book" : bibtex_entry.append(u'@misc{') else : #case strict book return '' if citation_bibtex : # Citation tag bibtex_entry.append(make_bibtex_citation(entry, template_citation, bibtexdict)) bibtex_entry = [u' '.join(bibtex_entry)] for field in fields: if field.startswith('#'): item = db.get_field(entry['id'],field,index_is_id=True) if isinstance(item, (bool, float, int)): item = repr(item) elif field == 'title_sort': item = entry['sort'] elif field == 'library_name': item = library_name else: item = entry[field] #check if the field should be included (none or empty) if item is None: continue try: if len(item) == 0 : continue except TypeError: pass if field == 'authors' : bibtex_entry.append(u'author = "%s"' % bibtexdict.bibtex_author_format(item)) elif field == 'id' : bibtex_entry.append(u'calibreid = "%s"' % int(item)) elif field == 'rating' : bibtex_entry.append(u'rating = "%s"' % int(item)) elif field == 'size' : bibtex_entry.append(u'%s = "%s octets"' % (field, int(item))) elif field == 'tags' : #A list to flatten bibtex_entry.append(u'tags = "%s"' % bibtexdict.utf8ToBibtex(u', '.join(item))) elif field == 'comments' : #\n removal item = item.replace(u'\r\n',u' ') item = item.replace(u'\n',u' ') # unmatched brace removal (users should use \leftbrace or \rightbrace for single braces) item = bibtexdict.stripUnmatchedSyntax(item, u'{', u'}') #html to text try: item = html2text(item) except: log.warn("Failed to convert comments to text") bibtex_entry.append(u'note = "%s"' % bibtexdict.utf8ToBibtex(item)) elif field == 'isbn' : # Could be 9, 10 or 13 digits bibtex_entry.append(u'isbn = "%s"' % format_isbn(item)) elif field == 'formats' : #Add file path if format is selected formats = [format.rpartition('.')[2].lower() for format in item] bibtex_entry.append(u'formats = "%s"' % u', '.join(formats)) if calibre_files: files = [u':%s:%s' % (format, format.rpartition('.')[2].upper())\ for format in item] bibtex_entry.append(u'file = "%s"' % u', '.join(files)) elif field == 'series_index' : bibtex_entry.append(u'volume = "%s"' % int(item)) elif field == 'timestamp' : bibtex_entry.append(u'timestamp = "%s"' % isoformat(item).partition('T')[0]) elif field == 'pubdate' : bibtex_entry.append(u'year = "%s"' % item.year) bibtex_entry.append(u'month = "%s"' % bibtexdict.utf8ToBibtex(strftime("%b", item))) elif field.startswith('#') and isinstance(item, basestring): bibtex_entry.append(u'custom_%s = "%s"' % (field[1:], bibtexdict.utf8ToBibtex(item))) elif isinstance(item, basestring): # elif field in ['title', 'publisher', 'cover', 'uuid', 'ondevice', # 'author_sort', 'series', 'title_sort'] : bibtex_entry.append(u'%s = "%s"' % (field, bibtexdict.utf8ToBibtex(item))) bibtex_entry = u',\n '.join(bibtex_entry) bibtex_entry += u' }\n\n' return bibtex_entry def check_entry_book_valid(entry): #Check that the required fields are ok for a book entry for field in ['title', 'authors', 'publisher'] : if entry[field] is None or len(entry[field]) == 0 : return False if entry['pubdate'] is None : return False else : return True def make_bibtex_citation(entry, template_citation, bibtexclass): #define a function to replace the template entry by its value def tpl_replace(objtplname) : tpl_field = re.sub(u'[\{\}]', u'', objtplname.group()) if tpl_field in TEMPLATE_ALLOWED_FIELDS : if tpl_field in ['pubdate', 'timestamp'] : tpl_field = isoformat(entry[tpl_field]).partition('T')[0] elif tpl_field in ['tags', 'authors'] : tpl_field =entry[tpl_field][0] elif tpl_field in ['id', 'series_index'] : tpl_field = str(entry[tpl_field]) else : tpl_field = entry[tpl_field] return tpl_field else: return u'' if len(template_citation) >0 : tpl_citation = bibtexclass.utf8ToBibtex( bibtexclass.ValidateCitationKey(re.sub(u'\{[^{}]*\}', tpl_replace, template_citation))) if len(tpl_citation) >0 : return tpl_citation if len(entry["isbn"]) > 0 : template_citation = u'%s' % re.sub(u'[\D]',u'', entry["isbn"]) else : template_citation = u'%s' % str(entry["id"]) return bibtexclass.ValidateCitationKey(template_citation) self.fmt = path_to_output.rpartition('.')[2] self.notification = notification # Combobox options bibfile_enc = ['utf8', 'cp1252', 'ascii'] bibfile_enctag = ['strict', 'replace', 'ignore', 'backslashreplace'] bib_entry = ['mixed', 'misc', 'book'] # Needed beacause CLI return str vs int by widget try: bibfile_enc = bibfile_enc[opts.bibfile_enc] bibfile_enctag = bibfile_enctag[opts.bibfile_enctag] bib_entry = bib_entry[opts.bib_entry] except: if opts.bibfile_enc in bibfile_enc : bibfile_enc = opts.bibfile_enc else : log.warn("Incorrect --choose-encoding flag, revert to default") bibfile_enc = bibfile_enc[0] if opts.bibfile_enctag in bibfile_enctag : bibfile_enctag = opts.bibfile_enctag else : log.warn("Incorrect --choose-encoding-configuration flag, revert to default") bibfile_enctag = bibfile_enctag[0] if opts.bib_entry in bib_entry : bib_entry = opts.bib_entry else : log.warn("Incorrect --entry-type flag, revert to default") bib_entry = bib_entry[0] if opts.verbose: opts_dict = vars(opts) log("%s(): Generating %s" % (self.name,self.fmt)) if opts.connected_device['is_device_connected']: log(" connected_device: %s" % opts.connected_device['name']) if opts_dict['search_text']: log(" --search='%s'" % opts_dict['search_text']) if opts_dict['ids']: log(" Book count: %d" % len(opts_dict['ids'])) if opts_dict['search_text']: log(" (--search ignored when a subset of the database is specified)") if opts_dict['fields']: if opts_dict['fields'] == 'all': log(" Fields: %s" % ', '.join(FIELDS[1:])) else: log(" Fields: %s" % opts_dict['fields']) log(" Output file will be encoded in %s with %s flag" % (bibfile_enc, bibfile_enctag)) log(" BibTeX entry type is %s with a citation like '%s' flag" % (bib_entry, opts_dict['bib_cit'])) # If a list of ids are provided, don't use search_text if opts.ids: opts.search_text = None data = self.search_sort_db(db, opts) if not len(data): log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text) # Get the requested output fields as a list fields = self.get_output_fields(db, opts) if not len(data): log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text) #Initialize BibTeX class bibtexc = BibTeX() #Entries writing after Bibtex formating (or not) if bibfile_enc != 'ascii' : bibtexc.ascii_bibtex = False else : bibtexc.ascii_bibtex = True #Check citation choice and go to default in case of bad CLI if isinstance(opts.impcit, (StringType, UnicodeType)) : if opts.impcit == 'False' : citation_bibtex= False elif opts.impcit == 'True' : citation_bibtex= True else : log.warn("Incorrect --create-citation, revert to default") citation_bibtex= True else : citation_bibtex= opts.impcit #Check add file entry and go to default in case of bad CLI if isinstance(opts.addfiles, (StringType, UnicodeType)) : if opts.addfiles == 'False' : addfiles_bibtex = False elif opts.addfiles == 'True' : addfiles_bibtex = True else : log.warn("Incorrect --add-files-path, revert to default") addfiles_bibtex= True else : addfiles_bibtex = opts.addfiles #Preprocess for error and light correction template_citation = preprocess_template(opts.bib_cit) #Open output and write entries with codecs.open(path_to_output, 'w', bibfile_enc, bibfile_enctag)\ as outfile: #File header nb_entries = len(data) #check in book strict if all is ok else throw a warning into log if bib_entry == 'book' : nb_books = len(filter(check_entry_book_valid, data)) if nb_books < nb_entries : log.warn("Only %d entries in %d are book compatible" % (nb_books, nb_entries)) nb_entries = nb_books # If connected device, add 'On Device' values to data if opts.connected_device['is_device_connected'] and 'ondevice' in fields: for entry in data: entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[entry['id']]['ondevice'] outfile.write(u'%%%Calibre catalog\n%%%{0} entries in catalog\n\n'.format(nb_entries)) outfile.write(u'@preamble{"This catalog of %d entries was generated by calibre on %s"}\n\n' % (nb_entries, nowf().strftime("%A, %d. %B %Y %H:%M").decode(preferred_encoding))) for entry in data: outfile.write(create_bibtex_entry(entry, fields, bib_entry, template_citation, bibtexc, db, citation_bibtex, addfiles_bibtex))