def to_html(self):
'''
A HTML representation of this object.
'''
from calibre.ebooks.metadata import authors_to_string
from calibre.utils.date import isoformat
ans = [(_('Title'), unicode(self.title))]
ans += [(_('Author(s)'), (authors_to_string(self.authors) if self.authors else _('Unknown')))]
ans += [(_('Publisher'), unicode(self.publisher))]
ans += [(_('Producer'), unicode(self.book_producer))]
ans += [(_('Comments'), unicode(self.comments))]
ans += [('ISBN', unicode(self.isbn))]
ans += [(_('Tags'), u', '.join([unicode(t) for t in self.tags]))]
if self.series:
ans += [(_('Series'), unicode(self.series) + ' #%s'%self.format_series_index())]
ans += [(_('Languages'), u', '.join(self.languages))]
if self.timestamp is not None:
ans += [(_('Timestamp'), unicode(isoformat(self.timestamp, as_utc=False, sep=' ')))]
if self.pubdate is not None:
ans += [(_('Published'), unicode(isoformat(self.pubdate, as_utc=False, sep=' ')))]
if self.rights is not None:
ans += [(_('Rights'), unicode(self.rights))]
for key in self.custom_field_keys():
val = self.get(key, None)
if val:
(name, val) = self.format_field(key)
ans += [(name, val)]
for i, x in enumerate(ans):
ans[i] = u'<tr><td><b>%s</b></td><td>%s</td></tr>'%x
return u'<table>%s</table>'%u'\n'.join(ans)
def __unicode__representation__(self):
'''
A string representation of this object, suitable for printing to
console
'''
from calibre.utils.date import isoformat
from calibre.ebooks.metadata import authors_to_string
ans = []
def fmt(x, y):
ans.append(u'%-20s: %s'%(unicode_type(x), unicode_type(y)))
fmt('Title', self.title)
if self.title_sort:
fmt('Title sort', self.title_sort)
if self.authors:
fmt('Author(s)', authors_to_string(self.authors) +
((' [' + self.author_sort + ']')
if self.author_sort and self.author_sort != _('Unknown') else ''))
if self.publisher:
fmt('Publisher', self.publisher)
if getattr(self, 'book_producer', False):
fmt('Book Producer', self.book_producer)
if self.tags:
fmt('Tags', u', '.join([unicode_type(t) for t in self.tags]))
if self.series:
fmt('Series', self.series + ' #%s'%self.format_series_index())
if not self.is_null('languages'):
fmt('Languages', ', '.join(self.languages))
if self.rating is not None:
fmt('Rating', (u'%.2g'%(float(self.rating)/2.0)) if self.rating
else u'')
if self.timestamp is not None:
fmt('Timestamp', isoformat(self.timestamp))
if self.pubdate is not None:
fmt('Published', isoformat(self.pubdate))
if self.rights is not None:
fmt('Rights', unicode_type(self.rights))
if self.identifiers:
fmt('Identifiers', u', '.join(['%s:%s'%(k, v) for k, v in
iteritems(self.identifiers)]))
if self.comments:
fmt('Comments', self.comments)
for key in self.custom_field_keys():
val = self.get(key, None)
if val:
(name, val) = self.format_field(key)
fmt(name, unicode_type(val))
return u'\n'.join(ans)
def _update_drive_info(self, storage, location_code, name=None):
from calibre.utils.date import isoformat, now
from calibre.utils.config import from_json, to_json
import uuid
f = storage.find_path(self.calibre_file_paths['driveinfo'].split('/'))
dinfo = {}
if f is not None:
try:
stream = self.get_mtp_file(f)
dinfo = json.load(stream, object_hook=from_json)
except:
prints('Failed to load existing driveinfo.calibre file, with error:')
traceback.print_exc()
dinfo = {}
if dinfo.get('device_store_uuid', None) is None:
dinfo['device_store_uuid'] = unicode(uuid.uuid4())
if dinfo.get('device_name', None) is None:
dinfo['device_name'] = self.current_friendly_name
if name is not None:
dinfo['device_name'] = name
dinfo['location_code'] = location_code
dinfo['last_library_uuid'] = getattr(self, 'current_library_uuid', None)
dinfo['calibre_version'] = '.'.join([unicode(i) for i in numeric_version])
dinfo['date_last_connected'] = isoformat(now())
dinfo['mtp_prefix'] = storage.storage_prefix
raw = json.dumps(dinfo, default=to_json)
self.put_calibre_file(storage, 'driveinfo', BytesIO(raw), len(raw))
self.driveinfo[location_code] = dinfo
def stringify(data, metadata, for_machine):
for field, m in iteritems(metadata):
if field == 'authors':
data[field] = {
k: authors_to_string(v)
for k, v in iteritems(data[field])
}
else:
dt = m['datatype']
if dt == 'datetime':
data[field] = {
k: isoformat(v, as_utc=for_machine) if v else 'None'
for k, v in iteritems(data[field])
}
elif not for_machine:
ism = m['is_multiple']
if ism:
data[field] = {
k: ism['list_to_ui'].join(v)
for k, v in iteritems(data[field])
}
if field == 'formats':
data[field] = {
k: '[' + v + ']'
for k, v in iteritems(data[field])
}
def set_pubdate(root, prefixes, refines, val):
for date in XPath('./opf:metadata/dc:date')(root):
remove_element(date, refines)
if not is_date_undefined(val):
val = isoformat(val)
m = XPath('./opf:metadata')(root)[0]
d = m.makeelement(DC('date'))
d.text = val
m.append(d)
def to_json(obj):
if isinstance(obj, bytearray):
return {'__class__': 'bytearray',
'__value__': base64.standard_b64encode(bytes(obj))}
if isinstance(obj, datetime.datetime):
from calibre.utils.date import isoformat
return {'__class__': 'datetime.datetime',
'__value__': isoformat(obj, as_utc=True)}
raise TypeError(repr(obj) + ' is not JSON serializable')
def encode_datetime(dateval):
if dateval is None:
return "None"
if not isinstance(dateval, datetime):
dateval = datetime.combine(dateval, time())
if hasattr(dateval, 'tzinfo') and dateval.tzinfo is None:
dateval = dateval.replace(tzinfo=local_tz)
if dateval <= UNDEFINED_DATE:
return None
return isoformat(dateval)
def datetime_to_string(dateval):
from calibre.utils.date import isoformat, UNDEFINED_DATE, local_tz
if dateval is None:
return "None"
if not isinstance(dateval, datetime):
dateval = datetime.combine(dateval, time())
if hasattr(dateval, 'tzinfo') and dateval.tzinfo is None:
dateval = dateval.replace(tzinfo=local_tz)
if dateval <= UNDEFINED_DATE:
return "None"
return isoformat(dateval)
def open(self, device, library_uuid):
self.current_library_uuid = library_uuid
self.location_paths = None
self.driveinfo = {}
BASE.open(self, device, library_uuid)
h = self.prefs['history']
if self.current_serial_num:
h[self.current_serial_num] = (self.current_friendly_name,
isoformat(utcnow()))
self.prefs['history'] = h
self.current_device_defaults = self.device_defaults(device, self)
def set_timestamp(root, prefixes, refines, val):
ensure_prefix(root, prefixes, 'calibre', CALIBRE_PREFIX)
ensure_prefix(root, prefixes, 'dcterms')
pq = '%s:timestamp' % CALIBRE_PREFIX
for meta in XPath('./opf:metadata/opf:meta')(root):
prop = expand_prefix(meta.get('property'), prefixes)
if prop.lower() == pq or meta.get('name') == 'calibre:timestamp':
remove_element(meta, refines)
if not is_date_undefined(val):
val = isoformat(val)
m = XPath('./opf:metadata')(root)[0]
d = m.makeelement(OPF('meta'), attrib={'property':'calibre:timestamp', 'scheme':'dcterms:W3CDTF'})
d.text = val
m.append(d)
def open(self, device, library_uuid):
from calibre.utils.date import isoformat, utcnow
self.current_library_uuid = library_uuid
self.location_paths = None
self.driveinfo = {}
BASE.open(self, device, library_uuid)
h = self.prefs['history']
if self.current_serial_num:
h[self.current_serial_num] = (self.current_friendly_name,
isoformat(utcnow()))
self.prefs['history'] = h
self.current_device_defaults = self.device_defaults(device, self)
self.calibre_file_paths = self.current_device_defaults.get(
'calibre_file_paths', {'metadata':self.METADATA_CACHE, 'driveinfo':self.DRIVEINFO})
def tpl_replace(objtplname) :
tpl_field = re.sub(u'[\{\}]', u'', objtplname.group())
if tpl_field in TEMPLATE_ALLOWED_FIELDS :
if tpl_field in ['pubdate', 'timestamp'] :
tpl_field = isoformat(entry[tpl_field]).partition('T')[0]
elif tpl_field in ['tags', 'authors'] :
tpl_field =entry[tpl_field][0]
elif tpl_field in ['id', 'series_index'] :
tpl_field = str(entry[tpl_field])
else :
tpl_field = entry[tpl_field]
return tpl_field
else:
return u''
def _update_driveinfo_record(self, dinfo, prefix, location_code, name=None):
import uuid
if not isinstance(dinfo, dict):
dinfo = {}
if dinfo.get('device_store_uuid', None) is None:
dinfo['device_store_uuid'] = unicode(uuid.uuid4())
if dinfo.get('device_name') is None:
dinfo['device_name'] = self.get_gui_name()
if name is not None:
dinfo['device_name'] = name
dinfo['location_code'] = location_code
dinfo['last_library_uuid'] = getattr(self, 'current_library_uuid', None)
dinfo['calibre_version'] = '.'.join([unicode(i) for i in numeric_version])
dinfo['date_last_connected'] = isoformat(now())
dinfo['prefix'] = prefix.replace('\\', '/')
return dinfo
def update_last_downloaded(self, recipe_id):
with self.lock:
now = utcnow()
for x in self.iter_recipes():
if x.get('id', False) == recipe_id:
typ, sch, last_downloaded = self.un_serialize_schedule(x)
if typ == 'interval':
# Prevent downloads more frequent than once an hour
actual_interval = now - last_downloaded
nominal_interval = timedelta(days=sch)
if abs(actual_interval - nominal_interval) < \
timedelta(hours=1):
now = last_downloaded + nominal_interval
x.set('last_downloaded', isoformat(now))
break
self.write_scheduler_file()
def to_json(obj):
import datetime
if isinstance(obj, bytearray):
from base64 import standard_b64encode
return {'__class__': 'bytearray',
'__value__': standard_b64encode(bytes(obj)).decode('ascii')}
if isinstance(obj, datetime.datetime):
from calibre.utils.date import isoformat
return {'__class__': 'datetime.datetime',
'__value__': isoformat(obj, as_utc=True)}
if isinstance(obj, (set, frozenset)):
return {'__class__': 'set', '__value__': tuple(obj)}
if isinstance(obj, bytes):
return obj.decode('utf-8')
if hasattr(obj, 'toBase64'): # QByteArray
return {'__class__': 'bytearray',
'__value__': bytes(obj.toBase64()).decode('ascii')}
raise TypeError(repr(obj) + ' is not JSON serializable')
def schedule_recipe(self, recipe, schedule_type, schedule, last_downloaded=None):
with self.lock:
for x in list(self.iter_recipes()):
if x.get('id', False) == recipe.get('id'):
ld = x.get('last_downloaded', None)
if ld and last_downloaded is None:
try:
last_downloaded = parse_date(ld)
except:
pass
self.root.remove(x)
break
if last_downloaded is None:
last_downloaded = fromordinal(1)
sr = E.scheduled_recipe({
'id' : recipe.get('id'),
'title': recipe.get('title'),
'last_downloaded':isoformat(last_downloaded),
}, self.serialize_schedule(schedule_type, schedule))
self.root.append(sr)
self.write_scheduler_file()
def set_metadata(self, stream, mi, type_):
from calibre_plugins.kfx_output.kfxlib import (set_logger, YJ_Book, YJ_Metadata)
from calibre.ebooks import normalize as normalize_unicode
from calibre.ebooks.metadata import author_to_author_sort
from calibre.utils.config_base import tweaks
from calibre.utils.date import (is_date_undefined, isoformat)
from calibre.utils.logging import Log
from calibre.utils.localization import (canonicalize_lang, lang_as_iso639_1)
def mapped_author_to_author_sort(author):
if hasattr(mi, "author_sort_map"):
author_sort = mi.author_sort_map.get(author) # use mapping if provided
if author_sort:
return author_sort
return author_to_author_sort(author)
def normalize(s):
if not isinstance(s, type("")):
s = s.decode("utf8", "ignore")
return normalize_unicode(s)
log = set_logger(Log())
filename = stream.name if hasattr(stream, "name") else "stream"
log.info("KFX metadata writer activated for %s" % filename)
try:
from calibre.ebooks.conversion.config import load_defaults
prefs = load_defaults('kfx_output')
except Exception:
prefs = {}
log.info("Failed to read default KFX Output preferences")
md = YJ_Metadata(author_sort_fn=mapped_author_to_author_sort)
md.title = normalize(mi.title)
md.authors = [normalize(author) for author in mi.authors]
if mi.publisher:
md.publisher = normalize(mi.publisher)
if mi.pubdate and not is_date_undefined(mi.pubdate):
md.issue_date = str(isoformat(mi.pubdate)[:10])
if mi.comments:
# Strip user annotations
a_offset = mi.comments.find('<div class="user_annotations">')
ad_offset = mi.comments.find('<hr class="annotations_divider" />')
if a_offset >= 0:
mi.comments = mi.comments[:a_offset]
if ad_offset >= 0:
mi.comments = mi.comments[:ad_offset]
md.description = normalize(mi.comments)
if not mi.is_null('language'):
lang = canonicalize_lang(mi.language)
lang = lang_as_iso639_1(lang) or lang
if lang:
md.language = normalize(lang)
if mi.cover_data[1]:
md.cover_image_data = mi.cover_data
elif mi.cover:
md.cover_image_data = ("jpg", open(mi.cover, 'rb').read())
if not tweaks.get("kfx_output_ignore_asin_metadata", False):
value = mi.identifiers.get("mobi-asin")
if value is not None and re.match(ASIN_RE, value):
md.asin = value
else:
for ident, value in mi.identifiers.items():
if ident.startswith("amazon") and re.match(ASIN_RE, value):
md.asin = value
break
else:
value = mi.identifiers.get("asin")
if value is not None and re.match(ASIN_RE, value):
md.asin = value
if md.asin:
md.cde_content_type = "EBOK"
if prefs.get("approximate_pages", False):
page_count = 0
number_of_pages_field = prefs.get("number_of_pages_field", AUTO_PAGES)
if number_of_pages_field and number_of_pages_field != AUTO_PAGES:
number_of_pages = mi.get(number_of_pages_field, "")
try:
page_count = int(number_of_pages)
except Exception:
pass
else:
page_count = -1
book = YJ_Book(stream, log)
book.decode_book(set_metadata=md, set_approximate_pages=page_count)
new_data = book.convert_to_single_kfx()
set_logger()
stream.seek(0)
stream.truncate()
stream.write(new_data)
stream.seek(0)
def upgrade_version_18(self):
'''
Add a library UUID.
Add an identifiers table.
Add a languages table.
Add a last_modified column.
NOTE: You cannot downgrade after this update, if you do
any changes you make to book isbns will be lost.
'''
script = '''
DROP TABLE IF EXISTS library_id;
CREATE TABLE library_id ( id INTEGER PRIMARY KEY,
uuid TEXT NOT NULL,
UNIQUE(uuid)
);
DROP TABLE IF EXISTS identifiers;
CREATE TABLE identifiers ( id INTEGER PRIMARY KEY,
book INTEGER NON NULL,
type TEXT NON NULL DEFAULT "isbn" COLLATE NOCASE,
val TEXT NON NULL COLLATE NOCASE,
UNIQUE(book, type)
);
DROP TABLE IF EXISTS languages;
CREATE TABLE languages ( id INTEGER PRIMARY KEY,
lang_code TEXT NON NULL COLLATE NOCASE,
UNIQUE(lang_code)
);
DROP TABLE IF EXISTS books_languages_link;
CREATE TABLE books_languages_link ( id INTEGER PRIMARY KEY,
book INTEGER NOT NULL,
lang_code INTEGER NOT NULL,
item_order INTEGER NOT NULL DEFAULT 0,
UNIQUE(book, lang_code)
);
DROP TRIGGER IF EXISTS fkc_delete_on_languages;
CREATE TRIGGER fkc_delete_on_languages
BEFORE DELETE ON languages
BEGIN
SELECT CASE
WHEN (SELECT COUNT(id) FROM books_languages_link WHERE lang_code=OLD.id) > 0
THEN RAISE(ABORT, 'Foreign key violation: language is still referenced')
END;
END;
DROP TRIGGER IF EXISTS fkc_delete_on_languages_link;
CREATE TRIGGER fkc_delete_on_languages_link
BEFORE INSERT ON books_languages_link
BEGIN
SELECT CASE
WHEN (SELECT id from books WHERE id=NEW.book) IS NULL
THEN RAISE(ABORT, 'Foreign key violation: book not in books')
WHEN (SELECT id from languages WHERE id=NEW.lang_code) IS NULL
THEN RAISE(ABORT, 'Foreign key violation: lang_code not in languages')
END;
END;
DROP TRIGGER IF EXISTS fkc_update_books_languages_link_a;
CREATE TRIGGER fkc_update_books_languages_link_a
BEFORE UPDATE OF book ON books_languages_link
BEGIN
SELECT CASE
WHEN (SELECT id from books WHERE id=NEW.book) IS NULL
THEN RAISE(ABORT, 'Foreign key violation: book not in books')
END;
END;
DROP TRIGGER IF EXISTS fkc_update_books_languages_link_b;
CREATE TRIGGER fkc_update_books_languages_link_b
BEFORE UPDATE OF lang_code ON books_languages_link
BEGIN
SELECT CASE
WHEN (SELECT id from languages WHERE id=NEW.lang_code) IS NULL
THEN RAISE(ABORT, 'Foreign key violation: lang_code not in languages')
END;
END;
DROP INDEX IF EXISTS books_languages_link_aidx;
CREATE INDEX books_languages_link_aidx ON books_languages_link (lang_code);
DROP INDEX IF EXISTS books_languages_link_bidx;
CREATE INDEX books_languages_link_bidx ON books_languages_link (book);
DROP INDEX IF EXISTS languages_idx;
CREATE INDEX languages_idx ON languages (lang_code COLLATE NOCASE);
DROP TRIGGER IF EXISTS books_delete_trg;
CREATE TRIGGER books_delete_trg
AFTER DELETE ON books
BEGIN
DELETE FROM books_authors_link WHERE book=OLD.id;
DELETE FROM books_publishers_link WHERE book=OLD.id;
DELETE FROM books_ratings_link WHERE book=OLD.id;
DELETE FROM books_series_link WHERE book=OLD.id;
DELETE FROM books_tags_link WHERE book=OLD.id;
DELETE FROM books_languages_link WHERE book=OLD.id;
DELETE FROM data WHERE book=OLD.id;
DELETE FROM comments WHERE book=OLD.id;
DELETE FROM conversion_options WHERE book=OLD.id;
DELETE FROM books_plugin_data WHERE book=OLD.id;
DELETE FROM identifiers WHERE book=OLD.id;
END;
INSERT INTO identifiers (book, val) SELECT id,isbn FROM books WHERE isbn;
ALTER TABLE books ADD COLUMN last_modified TIMESTAMP NOT NULL DEFAULT "%s";
''' % isoformat(DEFAULT_DATE, sep=' ')
# Sqlite does not support non constant default values in alter
# statements
self.conn.executescript(script)
def sqlite_datetime(x):
return isoformat(x, sep=' ') if isinstance(x, datetime) else x
def run(self, path_to_output, opts, db, notification=DummyReporter()):
from calibre.library import current_library_name
from calibre.utils.date import isoformat
from calibre.utils.html2text import html2text
from calibre.utils.logging import default_log as log
from lxml import etree
from calibre.ebooks.metadata import authors_to_string
self.fmt = path_to_output.rpartition('.')[2]
self.notification = notification
current_library = current_library_name()
if getattr(opts, 'library_path', None):
current_library = os.path.basename(opts.library_path)
if opts.verbose:
opts_dict = vars(opts)
log("%s('%s'): Generating %s" % (self.name, current_library, self.fmt.upper()))
if opts.connected_device['is_device_connected']:
log(" connected_device: %s" % opts.connected_device['name'])
if opts_dict['search_text']:
log(" --search='%s'" % opts_dict['search_text'])
if opts_dict['ids']:
log(" Book count: %d" % len(opts_dict['ids']))
if opts_dict['search_text']:
log(" (--search ignored when a subset of the database is specified)")
if opts_dict['fields']:
if opts_dict['fields'] == 'all':
log(" Fields: %s" % ', '.join(FIELDS[1:]))
else:
log(" Fields: %s" % opts_dict['fields'])
# If a list of ids are provided, don't use search_text
if opts.ids:
opts.search_text = None
data = self.search_sort_db(db, opts)
if not len(data):
log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text)
# raise SystemExit(1)
# Get the requested output fields as a list
fields = self.get_output_fields(db, opts)
# If connected device, add 'On Device' values to data
if opts.connected_device['is_device_connected'] and 'ondevice' in fields:
for entry in data:
entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[entry['id']]['ondevice']
fm = {x: db.field_metadata.get(x, {}) for x in fields}
if self.fmt == 'csv':
outfile = codecs.open(path_to_output, 'w', 'utf8')
# Write a UTF-8 BOM
outfile.write('\xef\xbb\xbf')
# Output the field headers
outfile.write(u'%s\n' % u','.join(fields))
# Output the entry fields
for entry in data:
outstr = []
for field in fields:
if field.startswith('#'):
item = db.get_field(entry['id'], field, index_is_id=True)
if isinstance(item, (list, tuple)):
if fm.get(field, {}).get('display', {}).get('is_names', False):
item = ' & '.join(item)
else:
item = ', '.join(item)
elif field == 'library_name':
item = current_library
elif field == 'title_sort':
item = entry['sort']
else:
item = entry[field]
if item is None:
outstr.append('""')
continue
elif field == 'formats':
fmt_list = []
for format in item:
fmt_list.append(format.rpartition('.')[2].lower())
item = ', '.join(fmt_list)
elif field == 'authors':
item = authors_to_string(item)
elif field == 'tags':
item = ', '.join(item)
elif field == 'isbn':
# Could be 9, 10 or 13 digits, with hyphens, possibly ending in 'X'
item = u'%s' % re.sub(r'[^\dX-]', '', item)
elif fm.get(field, {}).get('datatype') == 'datetime':
item = isoformat(item, as_utc=False)
elif field == 'comments':
item = item.replace(u'\r\n', u' ')
item = item.replace(u'\n', u' ')
elif fm.get(field, {}).get('datatype', None) == 'rating' and item:
item = u'%.2g' % (item / 2.0)
# Convert HTML to markdown text
if type(item) is unicode:
opening_tag = re.search('<(\w+)(\x20|>)', item)
if opening_tag:
closing_tag = re.search('<\/%s>$' % opening_tag.group(1), item)
if closing_tag:
item = html2text(item)
outstr.append(u'"%s"' % unicode(item).replace('"', '""'))
outfile.write(u','.join(outstr) + u'\n')
outfile.close()
elif self.fmt == 'xml':
from lxml.builder import E
root = E.calibredb()
for r in data:
record = E.record()
root.append(record)
for field in fields:
if field.startswith('#'):
val = db.get_field(r['id'], field, index_is_id=True)
if not isinstance(val, (str, unicode)):
val = unicode(val)
item = getattr(E, field.replace('#', '_'))(val)
record.append(item)
for field in ('id', 'uuid', 'publisher', 'rating', 'size',
'isbn', 'ondevice', 'identifiers'):
if field in fields:
val = r[field]
if not val:
continue
if not isinstance(val, (str, unicode)):
if (fm.get(field, {}).get('datatype', None) ==
'rating' and val):
val = u'%.2g' % (val / 2.0)
val = unicode(val)
item = getattr(E, field)(val)
record.append(item)
if 'title' in fields:
title = E.title(r['title'], sort=r['sort'])
record.append(title)
if 'authors' in fields:
aus = E.authors(sort=r['author_sort'])
for au in r['authors']:
aus.append(E.author(au))
record.append(aus)
for field in ('timestamp', 'pubdate'):
if field in fields:
record.append(getattr(E, field)(isoformat(r[field], as_utc=False)))
if 'tags' in fields and r['tags']:
tags = E.tags()
for tag in r['tags']:
tags.append(E.tag(tag))
record.append(tags)
if 'comments' in fields and r['comments']:
record.append(E.comments(r['comments']))
if 'series' in fields and r['series']:
record.append(E.series(r['series'],
index=str(r['series_index'])))
if 'cover' in fields and r['cover']:
record.append(E.cover(r['cover'].replace(os.sep, '/')))
if 'formats' in fields and r['formats']:
fmt = E.formats()
for f in r['formats']:
fmt.append(E.format(f.replace(os.sep, '/')))
record.append(fmt)
if 'library_name' in fields:
record.append(E.library_name(current_library))
with open(path_to_output, 'w') as f:
f.write(etree.tostring(root, encoding='utf-8',
xml_declaration=True, pretty_print=True))
def upgrade_version_18(self):
'''
Add a library UUID.
Add an identifiers table.
Add a languages table.
Add a last_modified column.
NOTE: You cannot downgrade after this update, if you do
any changes you make to book isbns will be lost.
'''
script = '''
DROP TABLE IF EXISTS library_id;
CREATE TABLE library_id ( id INTEGER PRIMARY KEY,
uuid TEXT NOT NULL,
UNIQUE(uuid)
);
DROP TABLE IF EXISTS identifiers;
CREATE TABLE identifiers ( id INTEGER PRIMARY KEY,
book INTEGER NOT NULL,
type TEXT NOT NULL DEFAULT "isbn" COLLATE NOCASE,
val TEXT NOT NULL COLLATE NOCASE,
UNIQUE(book, type)
);
DROP TABLE IF EXISTS languages;
CREATE TABLE languages ( id INTEGER PRIMARY KEY,
lang_code TEXT NOT NULL COLLATE NOCASE,
UNIQUE(lang_code)
);
DROP TABLE IF EXISTS books_languages_link;
CREATE TABLE books_languages_link ( id INTEGER PRIMARY KEY,
book INTEGER NOT NULL,
lang_code INTEGER NOT NULL,
item_order INTEGER NOT NULL DEFAULT 0,
UNIQUE(book, lang_code)
);
DROP TRIGGER IF EXISTS fkc_delete_on_languages;
CREATE TRIGGER fkc_delete_on_languages
BEFORE DELETE ON languages
BEGIN
SELECT CASE
WHEN (SELECT COUNT(id) FROM books_languages_link WHERE lang_code=OLD.id) > 0
THEN RAISE(ABORT, 'Foreign key violation: language is still referenced')
END;
END;
DROP TRIGGER IF EXISTS fkc_delete_on_languages_link;
CREATE TRIGGER fkc_delete_on_languages_link
BEFORE INSERT ON books_languages_link
BEGIN
SELECT CASE
WHEN (SELECT id from books WHERE id=NEW.book) IS NULL
THEN RAISE(ABORT, 'Foreign key violation: book not in books')
WHEN (SELECT id from languages WHERE id=NEW.lang_code) IS NULL
THEN RAISE(ABORT, 'Foreign key violation: lang_code not in languages')
END;
END;
DROP TRIGGER IF EXISTS fkc_update_books_languages_link_a;
CREATE TRIGGER fkc_update_books_languages_link_a
BEFORE UPDATE OF book ON books_languages_link
BEGIN
SELECT CASE
WHEN (SELECT id from books WHERE id=NEW.book) IS NULL
THEN RAISE(ABORT, 'Foreign key violation: book not in books')
END;
END;
DROP TRIGGER IF EXISTS fkc_update_books_languages_link_b;
CREATE TRIGGER fkc_update_books_languages_link_b
BEFORE UPDATE OF lang_code ON books_languages_link
BEGIN
SELECT CASE
WHEN (SELECT id from languages WHERE id=NEW.lang_code) IS NULL
THEN RAISE(ABORT, 'Foreign key violation: lang_code not in languages')
END;
END;
DROP INDEX IF EXISTS books_languages_link_aidx;
CREATE INDEX books_languages_link_aidx ON books_languages_link (lang_code);
DROP INDEX IF EXISTS books_languages_link_bidx;
CREATE INDEX books_languages_link_bidx ON books_languages_link (book);
DROP INDEX IF EXISTS languages_idx;
CREATE INDEX languages_idx ON languages (lang_code COLLATE NOCASE);
DROP TRIGGER IF EXISTS books_delete_trg;
CREATE TRIGGER books_delete_trg
AFTER DELETE ON books
BEGIN
DELETE FROM books_authors_link WHERE book=OLD.id;
DELETE FROM books_publishers_link WHERE book=OLD.id;
DELETE FROM books_ratings_link WHERE book=OLD.id;
DELETE FROM books_series_link WHERE book=OLD.id;
DELETE FROM books_tags_link WHERE book=OLD.id;
DELETE FROM books_languages_link WHERE book=OLD.id;
DELETE FROM data WHERE book=OLD.id;
DELETE FROM comments WHERE book=OLD.id;
DELETE FROM conversion_options WHERE book=OLD.id;
DELETE FROM books_plugin_data WHERE book=OLD.id;
DELETE FROM identifiers WHERE book=OLD.id;
END;
INSERT INTO identifiers (book, val) SELECT id,isbn FROM books WHERE isbn;
ALTER TABLE books ADD COLUMN last_modified TIMESTAMP NOT NULL DEFAULT "%s";
'''%isoformat(DEFAULT_DATE, sep=' ')
# Sqlite does not support non constant default values in alter
# statements
self.db.execute(script)
def metadata_to_xmp_packet(mi):
A = ElementMaker(namespace=NS_MAP["x"], nsmap=nsmap("x"))
R = ElementMaker(namespace=NS_MAP["rdf"], nsmap=nsmap("rdf"))
root = A.xmpmeta(R.RDF)
rdf = root[0]
dc = rdf.makeelement(expand("rdf:Description"), nsmap=nsmap("dc"))
dc.set(expand("rdf:about"), "")
rdf.append(dc)
for prop, tag in {"title": "dc:title", "comments": "dc:description"}.iteritems():
val = mi.get(prop) or ""
create_alt_property(dc, tag, val)
for prop, (tag, ordered) in {
"authors": ("dc:creator", True),
"tags": ("dc:subject", False),
"publisher": ("dc:publisher", False),
}.iteritems():
val = mi.get(prop) or ()
if isinstance(val, basestring):
val = [val]
create_sequence_property(dc, tag, val, ordered)
if not mi.is_null("pubdate"):
create_sequence_property(
dc, "dc:date", [isoformat(mi.pubdate, as_utc=False)]
) # Adobe spec recommends local time
if not mi.is_null("languages"):
langs = filter(None, map(lambda x: lang_as_iso639_1(x) or canonicalize_lang(x), mi.languages))
if langs:
create_sequence_property(dc, "dc:language", langs, ordered=False)
xmp = rdf.makeelement(expand("rdf:Description"), nsmap=nsmap("xmp", "xmpidq"))
xmp.set(expand("rdf:about"), "")
rdf.append(xmp)
extra_ids = {}
for x in ("prism", "pdfx"):
p = extra_ids[x] = rdf.makeelement(expand("rdf:Description"), nsmap=nsmap(x))
p.set(expand("rdf:about"), "")
rdf.append(p)
identifiers = mi.get_identifiers()
if identifiers:
create_identifiers(xmp, identifiers)
for scheme, val in identifiers.iteritems():
if scheme in {"isbn", "doi"}:
for prefix, parent in extra_ids.iteritems():
ie = parent.makeelement(expand("%s:%s" % (prefix, scheme)))
ie.text = val
parent.append(ie)
d = xmp.makeelement(expand("xmp:MetadataDate"))
d.text = isoformat(now(), as_utc=False)
xmp.append(d)
calibre = rdf.makeelement(expand("rdf:Description"), nsmap=nsmap("calibre", "calibreSI", "calibreCC"))
calibre.set(expand("rdf:about"), "")
rdf.append(calibre)
if not mi.is_null("rating"):
try:
r = float(mi.rating)
except (TypeError, ValueError):
pass
else:
create_simple_property(calibre, "calibre:rating", "%g" % r)
if not mi.is_null("series"):
create_series(calibre, mi.series, mi.series_index)
if not mi.is_null("timestamp"):
create_simple_property(calibre, "calibre:timestamp", isoformat(mi.timestamp, as_utc=False))
for x in ("author_link_map", "user_categories"):
val = getattr(mi, x, None)
if val:
create_simple_property(calibre, "calibre:" + x, dump_dict(val))
for x in ("title_sort", "author_sort"):
if not mi.is_null(x):
create_simple_property(calibre, "calibre:" + x, getattr(mi, x))
all_user_metadata = mi.get_all_user_metadata(True)
if all_user_metadata:
create_user_metadata(calibre, all_user_metadata)
return serialize_xmp_packet(root)
def sqlite_datetime(x):
return isoformat(x, sep=' ') if isinstance(x, datetime) else x
def book_to_json(ctx, rd, db, book_id,
get_category_urls=True, device_compatible=False, device_for_template=None):
mi = db.get_metadata(book_id, get_cover=False)
codec = JsonCodec(db.field_metadata)
if not device_compatible:
try:
mi.rating = mi.rating/2.
except Exception:
mi.rating = 0.0
data = codec.encode_book_metadata(mi)
for x in ('publication_type', 'size', 'db_id', 'lpath', 'mime',
'rights', 'book_producer'):
data.pop(x, None)
get = partial(ctx.url_for, get_content, book_id=book_id, library_id=db.server_library_id)
data['cover'] = get(what='cover')
data['thumbnail'] = get(what='thumb')
if not device_compatible:
mi.format_metadata = {k.lower():dict(v) for k, v in
mi.format_metadata.iteritems()}
for v in mi.format_metadata.itervalues():
mtime = v.get('mtime', None)
if mtime is not None:
v['mtime'] = isoformat(mtime, as_utc=True)
data['format_metadata'] = mi.format_metadata
fmts = set(x.lower() for x in mi.format_metadata.iterkeys())
pf = prefs['output_format'].lower()
other_fmts = list(fmts)
try:
fmt = pf if pf in fmts else other_fmts[0]
except:
fmt = None
if fmts and fmt:
other_fmts = [x for x in fmts if x != fmt]
data['formats'] = sorted(fmts)
if fmt:
data['main_format'] = {fmt:get(what=fmt)}
else:
data['main_format'] = None
data['other_formats'] = {fmt:get(what=fmt) for fmt in other_fmts}
if get_category_urls:
category_urls = data['category_urls'] = {}
all_cats = ctx.get_categories(rd, db)
for key in mi.all_field_keys():
fm = mi.metadata_for_field(key)
if (fm and fm['is_category'] and not fm['is_csp'] and
key != 'formats' and fm['datatype'] != 'rating'):
categories = mi.get(key) or []
if isinstance(categories, basestring):
categories = [categories]
category_urls[key] = dbtags = {}
for category in categories:
for tag in all_cats.get(key, ()):
if tag.original_name == category:
dbtags[category] = ctx.url_for(
books_in,
encoded_category=encode_name(tag.category if tag.category else key),
encoded_item=encode_name(tag.original_name if tag.id is None else unicode(tag.id)),
library_id=db.server_library_id
)
break
else:
series = data.get('series', None) or ''
if series:
tsorder = tweaks['save_template_title_series_sorting']
series = title_sort(series, order=tsorder)
data['_series_sort_'] = series
if device_for_template:
import posixpath
from calibre.devices.utils import create_upload_path
from calibre.utils.filenames import ascii_filename as sanitize
from calibre.customize.ui import device_plugins
for device_class in device_plugins():
if device_class.__class__.__name__ == device_for_template:
template = device_class.save_template()
data['_filename_'] = create_upload_path(mi, book_id,
template, sanitize, path_type=posixpath)
break
return data, mi.last_modified
def do_list(db, fields, afields, sort_by, ascending, search_text, line_width, separator,
prefix, subtitle='Books in the calibre database'):
from calibre.constants import terminal_controller as tc
terminal_controller = tc()
if sort_by:
db.sort(sort_by, ascending)
if search_text:
db.search(search_text)
data = db.get_data_as_dict(prefix, authors_as_string=True)
fields = ['id'] + fields
title_fields = fields
def field_name(f):
ans = f
if f[0] == '*':
if f.endswith('_index'):
fkey = f[1:-len('_index')]
num = db.custom_column_label_map[fkey]['num']
ans = '%d_index'%num
else:
ans = db.custom_column_label_map[f[1:]]['num']
return ans
fields = list(map(field_name, fields))
for f in data:
fmts = [x for x in f['formats'] if x is not None]
f['formats'] = u'[%s]'%u','.join(fmts)
widths = list(map(lambda x : 0, fields))
for record in data:
for f in record.keys():
if hasattr(record[f], 'isoformat'):
record[f] = isoformat(record[f], as_utc=False)
else:
record[f] = unicode(record[f])
record[f] = record[f].replace('\n', ' ')
for i in data:
for j, field in enumerate(fields):
widths[j] = max(widths[j], len(unicode(i[field])))
screen_width = terminal_controller.COLS if line_width < 0 else line_width
if not screen_width:
screen_width = 80
field_width = screen_width//len(fields)
base_widths = map(lambda x: min(x+1, field_width), widths)
while sum(base_widths) < screen_width:
adjusted = False
for i in range(len(widths)):
if base_widths[i] < widths[i]:
base_widths[i] += min(screen_width-sum(base_widths), widths[i]-base_widths[i])
adjusted = True
break
if not adjusted:
break
widths = list(base_widths)
titles = map(lambda x, y: '%-*s%s'%(x-len(separator), y, separator),
widths, title_fields)
print terminal_controller.GREEN + ''.join(titles)+terminal_controller.NORMAL
wrappers = map(lambda x: TextWrapper(x-1), widths)
o = cStringIO.StringIO()
for record in data:
text = [wrappers[i].wrap(unicode(record[field]).encode('utf-8')) for i, field in enumerate(fields)]
lines = max(map(len, text))
for l in range(lines):
for i, field in enumerate(text):
ft = text[i][l] if l < len(text[i]) else ''
filler = '%*s'%(widths[i]-len(ft)-1, '')
o.write(ft)
o.write(filler+separator)
print >>o
return o.getvalue()
def meta_info_to_oeb_metadata(mi, m, log, override_input_metadata=False):
from calibre.ebooks.oeb.base import OPF
if not mi.is_null('title'):
m.clear('title')
m.add('title', mi.title)
if mi.title_sort:
if not m.title:
m.add('title', mi.title_sort)
m.clear('title_sort')
m.add('title_sort', mi.title_sort)
if not mi.is_null('authors'):
m.filter('creator', lambda x: x.role.lower() in ['aut', ''])
for a in mi.authors:
attrib = {'role': 'aut'}
if mi.author_sort:
attrib[OPF('file-as')] = mi.author_sort
m.add('creator', a, attrib=attrib)
if not mi.is_null('book_producer'):
m.filter('contributor', lambda x: x.role.lower() == 'bkp')
m.add('contributor', mi.book_producer, role='bkp')
elif override_input_metadata:
m.filter('contributor', lambda x: x.role.lower() == 'bkp')
if not mi.is_null('comments'):
m.clear('description')
m.add('description', mi.comments)
elif override_input_metadata:
m.clear('description')
if not mi.is_null('publisher'):
m.clear('publisher')
m.add('publisher', mi.publisher)
elif override_input_metadata:
m.clear('publisher')
if not mi.is_null('series'):
m.clear('series')
m.add('series', mi.series)
elif override_input_metadata:
m.clear('series')
identifiers = mi.get_identifiers()
set_isbn = False
for typ, val in identifiers.iteritems():
has = False
if typ.lower() == 'isbn':
set_isbn = True
for x in m.identifier:
if x.scheme.lower() == typ.lower():
x.content = val
has = True
if not has:
m.add('identifier', val, scheme=typ.upper())
if override_input_metadata and not set_isbn:
m.filter('identifier', lambda x: x.scheme.lower() == 'isbn')
if not mi.is_null('languages'):
m.clear('language')
for lang in mi.languages:
if lang and lang.lower() not in ('und', ''):
m.add('language', lang)
if not mi.is_null('series_index'):
m.clear('series_index')
m.add('series_index', mi.format_series_index())
elif override_input_metadata:
m.clear('series_index')
if not mi.is_null('rating'):
m.clear('rating')
m.add('rating', '%.2f' % mi.rating)
elif override_input_metadata:
m.clear('rating')
if not mi.is_null('tags'):
m.clear('subject')
for t in mi.tags:
m.add('subject', t)
elif override_input_metadata:
m.clear('subject')
if not mi.is_null('pubdate'):
m.clear('date')
m.add('date', isoformat(mi.pubdate))
if not mi.is_null('timestamp'):
m.clear('timestamp')
m.add('timestamp', isoformat(mi.timestamp))
if not mi.is_null('rights'):
m.clear('rights')
m.add('rights', mi.rights)
if not mi.is_null('publication_type'):
m.clear('publication_type')
m.add('publication_type', mi.publication_type)
if not m.timestamp:
m.add('timestamp', isoformat(now()))
def run(self, path_to_output, opts, db, notification=DummyReporter()):
from calibre.library import current_library_name
from calibre.utils.date import isoformat
from calibre.utils.html2text import html2text
from calibre.utils.logging import default_log as log
from lxml import etree
from calibre.ebooks.metadata import authors_to_string
self.fmt = path_to_output.rpartition('.')[2]
self.notification = notification
current_library = current_library_name()
if getattr(opts, 'library_path', None):
current_library = os.path.basename(opts.library_path)
if opts.verbose:
opts_dict = vars(opts)
log("%s('%s'): Generating %s" % (self.name, current_library, self.fmt.upper()))
if opts.connected_device['is_device_connected']:
log(" connected_device: %s" % opts.connected_device['name'])
if opts_dict['search_text']:
log(" --search='%s'" % opts_dict['search_text'])
if opts_dict['ids']:
log(" Book count: %d" % len(opts_dict['ids']))
if opts_dict['search_text']:
log(" (--search ignored when a subset of the database is specified)")
if opts_dict['fields']:
if opts_dict['fields'] == 'all':
log(" Fields: %s" % ', '.join(FIELDS[1:]))
else:
log(" Fields: %s" % opts_dict['fields'])
# If a list of ids are provided, don't use search_text
if opts.ids:
opts.search_text = None
data = self.search_sort_db(db, opts)
if not len(data):
log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text)
# raise SystemExit(1)
# Get the requested output fields as a list
fields = self.get_output_fields(db, opts)
# If connected device, add 'On Device' values to data
if opts.connected_device['is_device_connected'] and 'ondevice' in fields:
for entry in data:
entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[entry['id']]['ondevice']
fm = {x: db.field_metadata.get(x, {}) for x in fields}
if self.fmt == 'csv':
outfile = codecs.open(path_to_output, 'w', 'utf8')
# Write a UTF-8 BOM
outfile.write('\xef\xbb\xbf')
# Output the field headers
outfile.write(u'%s\n' % u','.join(fields))
# Output the entry fields
for entry in data:
outstr = []
for field in fields:
if field.startswith('#'):
item = db.get_field(entry['id'], field, index_is_id=True)
if isinstance(item, (list, tuple)):
if fm.get(field, {}).get('display', {}).get('is_names', False):
item = ' & '.join(item)
else:
item = ', '.join(item)
elif field == 'library_name':
item = current_library
elif field == 'title_sort':
item = entry['sort']
else:
item = entry[field]
if item is None:
outstr.append('""')
continue
elif field == 'formats':
fmt_list = []
for format in item:
fmt_list.append(format.rpartition('.')[2].lower())
item = ', '.join(fmt_list)
elif field == 'authors':
item = authors_to_string(item)
elif field == 'tags':
item = ', '.join(item)
elif field == 'isbn':
# Could be 9, 10 or 13 digits, with hyphens, possibly ending in 'X'
item = u'%s' % re.sub(r'[^\dX-]', '', item)
elif fm.get(field, {}).get('datatype') == 'datetime':
item = isoformat(item, as_utc=False)
elif field == 'comments':
item = item.replace(u'\r\n', u' ')
item = item.replace(u'\n', u' ')
elif fm.get(field, {}).get('datatype', None) == 'rating' and item:
item = u'%.2g' % (item / 2.0)
# Convert HTML to markdown text
if isinstance(item, unicode_type):
opening_tag = re.search('<(\\w+)(\x20|>)', item)
if opening_tag:
closing_tag = re.search('<\\/%s>$' % opening_tag.group(1), item)
if closing_tag:
item = html2text(item)
outstr.append(u'"%s"' % unicode_type(item).replace('"', '""'))
outfile.write(u','.join(outstr) + u'\n')
outfile.close()
elif self.fmt == 'xml':
from lxml.builder import E
root = E.calibredb()
for r in data:
record = E.record()
root.append(record)
for field in fields:
if field.startswith('#'):
val = db.get_field(r['id'], field, index_is_id=True)
if not isinstance(val, unicode_type):
val = unicode_type(val)
item = getattr(E, field.replace('#', '_'))(val)
record.append(item)
for field in ('id', 'uuid', 'publisher', 'rating', 'size',
'isbn', 'ondevice', 'identifiers'):
if field in fields:
val = r[field]
if not val:
continue
if not isinstance(val, (bytes, unicode_type)):
if (fm.get(field, {}).get('datatype', None) ==
'rating' and val):
val = u'%.2g' % (val / 2.0)
val = unicode_type(val)
item = getattr(E, field)(val)
record.append(item)
if 'title' in fields:
title = E.title(r['title'], sort=r['sort'])
record.append(title)
if 'authors' in fields:
aus = E.authors(sort=r['author_sort'])
for au in r['authors']:
aus.append(E.author(au))
record.append(aus)
for field in ('timestamp', 'pubdate'):
if field in fields:
record.append(getattr(E, field)(isoformat(r[field], as_utc=False)))
if 'tags' in fields and r['tags']:
tags = E.tags()
for tag in r['tags']:
tags.append(E.tag(tag))
record.append(tags)
if 'comments' in fields and r['comments']:
record.append(E.comments(r['comments']))
if 'series' in fields and r['series']:
record.append(E.series(r['series'],
index=str(r['series_index'])))
if 'cover' in fields and r['cover']:
record.append(E.cover(r['cover'].replace(os.sep, '/')))
if 'formats' in fields and r['formats']:
fmt = E.formats()
for f in r['formats']:
fmt.append(E.format(f.replace(os.sep, '/')))
record.append(fmt)
if 'library_name' in fields:
record.append(E.library_name(current_library))
with open(path_to_output, 'w') as f:
f.write(etree.tostring(root, encoding='utf-8',
xml_declaration=True, pretty_print=True))
def meta_info_to_oeb_metadata(mi, m, log, override_input_metadata=False):
from calibre.ebooks.oeb.base import OPF
if not mi.is_null('title'):
m.clear('title')
m.add('title', mi.title)
if mi.title_sort:
if not m.title:
m.add('title', mi.title_sort)
m.clear('title_sort')
m.add('title_sort', mi.title_sort)
if not mi.is_null('authors'):
m.filter('creator', lambda x : x.role.lower() in ['aut', ''])
for a in mi.authors:
attrib = {'role':'aut'}
if mi.author_sort:
attrib[OPF('file-as')] = mi.author_sort
m.add('creator', a, attrib=attrib)
if not mi.is_null('book_producer'):
m.filter('contributor', lambda x : x.role.lower() == 'bkp')
m.add('contributor', mi.book_producer, role='bkp')
elif override_input_metadata:
m.filter('contributor', lambda x : x.role.lower() == 'bkp')
if not mi.is_null('comments'):
m.clear('description')
m.add('description', mi.comments)
elif override_input_metadata:
m.clear('description')
if not mi.is_null('publisher'):
m.clear('publisher')
m.add('publisher', mi.publisher)
elif override_input_metadata:
m.clear('publisher')
if not mi.is_null('series'):
m.clear('series')
m.add('series', mi.series)
elif override_input_metadata:
m.clear('series')
identifiers = mi.get_identifiers()
set_isbn = False
for typ, val in identifiers.iteritems():
has = False
if typ.lower() == 'isbn':
set_isbn = True
for x in m.identifier:
if x.scheme.lower() == typ.lower():
x.content = val
has = True
if not has:
m.add('identifier', val, scheme=typ.upper())
if override_input_metadata and not set_isbn:
m.filter('identifier', lambda x: x.scheme.lower() == 'isbn')
if not mi.is_null('languages'):
m.clear('language')
for lang in mi.languages:
if lang and lang.lower() not in ('und', ''):
m.add('language', lang)
if not mi.is_null('series_index'):
m.clear('series_index')
m.add('series_index', mi.format_series_index())
elif override_input_metadata:
m.clear('series_index')
if not mi.is_null('rating'):
m.clear('rating')
m.add('rating', '%.2f'%mi.rating)
elif override_input_metadata:
m.clear('rating')
if not mi.is_null('tags'):
m.clear('subject')
for t in mi.tags:
m.add('subject', t)
elif override_input_metadata:
m.clear('subject')
if not mi.is_null('pubdate'):
m.clear('date')
m.add('date', isoformat(mi.pubdate))
if not mi.is_null('timestamp'):
m.clear('timestamp')
m.add('timestamp', isoformat(mi.timestamp))
if not mi.is_null('rights'):
m.clear('rights')
m.add('rights', mi.rights)
if not mi.is_null('publication_type'):
m.clear('publication_type')
m.add('publication_type', mi.publication_type)
if not m.timestamp:
m.add('timestamp', isoformat(now()))
def ajax_book_to_json(self, book_id, get_category_urls=True,
device_compatible=False, device_for_template=None):
mi = self.db.get_metadata(book_id, index_is_id=True)
if not device_compatible:
try:
mi.rating = mi.rating/2.
except:
mi.rating = 0.0
data = self.ajax_json_codec.encode_book_metadata(mi)
for x in ('publication_type', 'size', 'db_id', 'lpath', 'mime',
'rights', 'book_producer'):
data.pop(x, None)
data['cover'] = absurl(self.opts.url_prefix, u'/get/cover/%d'%book_id)
data['thumbnail'] = absurl(self.opts.url_prefix, u'/get/thumb/%d'%book_id)
if not device_compatible:
mi.format_metadata = {k.lower():dict(v) for k, v in
mi.format_metadata.iteritems()}
for v in mi.format_metadata.itervalues():
mtime = v.get('mtime', None)
if mtime is not None:
v['mtime'] = isoformat(mtime, as_utc=True)
data['format_metadata'] = mi.format_metadata
fmts = set(x.lower() for x in mi.format_metadata.iterkeys())
pf = prefs['output_format'].lower()
other_fmts = list(fmts)
try:
fmt = pf if pf in fmts else other_fmts[0]
except:
fmt = None
if fmts and fmt:
other_fmts = [x for x in fmts if x != fmt]
data['formats'] = sorted(fmts)
if fmt:
data['main_format'] = {fmt: absurl(self.opts.url_prefix, u'/get/%s/%d'%(fmt, book_id))}
else:
data['main_format'] = None
data['other_formats'] = {fmt: absurl(self.opts.url_prefix, u'/get/%s/%d'%(fmt, book_id)) for fmt
in other_fmts}
if get_category_urls:
category_urls = data['category_urls'] = {}
ccache = self.categories_cache()
for key in mi.all_field_keys():
fm = mi.metadata_for_field(key)
if (fm and fm['is_category'] and not fm['is_csp'] and
key != 'formats' and fm['datatype'] not in ['rating']):
categories = mi.get(key)
if isinstance(categories, basestring):
categories = [categories]
if categories is None:
categories = []
dbtags = {}
for category in categories:
for tag in ccache.get(key, []):
if tag.original_name == category:
dbtags[category] = books_in_url(self.opts.url_prefix,
tag.category if tag.category else key,
tag.original_name if tag.id is None else
unicode(tag.id))
break
category_urls[key] = dbtags
else:
series = data.get('series', None)
if series:
tsorder = tweaks['save_template_title_series_sorting']
series = title_sort(series, order=tsorder)
else:
series = ''
data['_series_sort_'] = series
if device_for_template:
import posixpath
from calibre.devices.utils import create_upload_path
from calibre.utils.filenames import ascii_filename as sanitize
from calibre.customize.ui import device_plugins
for device_class in device_plugins():
if device_class.__class__.__name__ == device_for_template:
template = device_class.save_template()
data['_filename_'] = create_upload_path(mi, book_id,
template, sanitize, path_type=posixpath)
break
return data, mi.last_modified
def create_bibtex_entry(entry, fields, mode, template_citation,
bibtexdict, db, citation_bibtex=True, calibre_files=True):
#Bibtex doesn't like UTF-8 but keep unicode until writing
#Define starting chain or if book valid strict and not book return a Fail string
bibtex_entry = []
if mode != "misc" and check_entry_book_valid(entry) :
bibtex_entry.append(u'@book{')
elif mode != "book" :
bibtex_entry.append(u'@misc{')
else :
#case strict book
return ''
if citation_bibtex :
# Citation tag
bibtex_entry.append(make_bibtex_citation(entry, template_citation,
bibtexdict))
bibtex_entry = [u' '.join(bibtex_entry)]
for field in fields:
if field.startswith('#'):
item = db.get_field(entry['id'],field,index_is_id=True)
if isinstance(item, (bool, float, int)):
item = repr(item)
elif field == 'title_sort':
item = entry['sort']
elif field == 'library_name':
item = library_name
else:
item = entry[field]
#check if the field should be included (none or empty)
if item is None:
continue
try:
if len(item) == 0 :
continue
except TypeError:
pass
if field == 'authors' :
bibtex_entry.append(u'author = "%s"' % bibtexdict.bibtex_author_format(item))
elif field == 'id' :
bibtex_entry.append(u'calibreid = "%s"' % int(item))
elif field == 'rating' :
bibtex_entry.append(u'rating = "%s"' % int(item))
elif field == 'size' :
bibtex_entry.append(u'%s = "%s octets"' % (field, int(item)))
elif field == 'tags' :
#A list to flatten
bibtex_entry.append(u'tags = "%s"' % bibtexdict.utf8ToBibtex(u', '.join(item)))
elif field == 'comments' :
#\n removal
item = item.replace(u'\r\n',u' ')
item = item.replace(u'\n',u' ')
# unmatched brace removal (users should use \leftbrace or \rightbrace for single braces)
item = bibtexdict.stripUnmatchedSyntax(item, u'{', u'}')
#html to text
try:
item = html2text(item)
except:
log.warn("Failed to convert comments to text")
bibtex_entry.append(u'note = "%s"' % bibtexdict.utf8ToBibtex(item))
elif field == 'isbn' :
# Could be 9, 10 or 13 digits
bibtex_entry.append(u'isbn = "%s"' % format_isbn(item))
elif field == 'formats' :
#Add file path if format is selected
formats = [format.rpartition('.')[2].lower() for format in item]
bibtex_entry.append(u'formats = "%s"' % u', '.join(formats))
if calibre_files:
files = [u':%s:%s' % (format, format.rpartition('.')[2].upper())\
for format in item]
bibtex_entry.append(u'file = "%s"' % u', '.join(files))
elif field == 'series_index' :
bibtex_entry.append(u'volume = "%s"' % int(item))
elif field == 'timestamp' :
bibtex_entry.append(u'timestamp = "%s"' % isoformat(item).partition('T')[0])
elif field == 'pubdate' :
bibtex_entry.append(u'year = "%s"' % item.year)
bibtex_entry.append(u'month = "%s"' % bibtexdict.utf8ToBibtex(strftime("%b", item)))
elif field.startswith('#') and isinstance(item, basestring):
bibtex_entry.append(u'custom_%s = "%s"' % (field[1:],
bibtexdict.utf8ToBibtex(item)))
elif isinstance(item, basestring):
# elif field in ['title', 'publisher', 'cover', 'uuid', 'ondevice',
# 'author_sort', 'series', 'title_sort'] :
bibtex_entry.append(u'%s = "%s"' % (field, bibtexdict.utf8ToBibtex(item)))
bibtex_entry = u',\n '.join(bibtex_entry)
bibtex_entry += u' }\n\n'
return bibtex_entry
def ajax_book_to_json(self, book_id, get_category_urls=True):
mi = self.db.get_metadata(book_id, index_is_id=True)
try:
mi.rating = mi.rating/2.
except:
mi.rating = 0.0
data = self.ajax_json_codec.encode_book_metadata(mi)
for x in ('publication_type', 'size', 'db_id', 'lpath', 'mime',
'rights', 'book_producer'):
data.pop(x, None)
data['cover'] = absurl(self.opts.url_prefix, u'/get/cover/%d'%book_id)
data['thumbnail'] = absurl(self.opts.url_prefix, u'/get/thumb/%d'%book_id)
mi.format_metadata = {k.lower():dict(v) for k, v in
mi.format_metadata.iteritems()}
for v in mi.format_metadata.itervalues():
mtime = v.get('mtime', None)
if mtime is not None:
v['mtime'] = isoformat(mtime, as_utc=True)
data['format_metadata'] = mi.format_metadata
fmts = set(x.lower() for x in mi.format_metadata.iterkeys())
pf = prefs['output_format'].lower()
other_fmts = list(fmts)
try:
fmt = pf if pf in fmts else other_fmts[0]
except:
fmt = None
if fmts and fmt:
other_fmts = [x for x in fmts if x != fmt]
data['formats'] = sorted(fmts)
if fmt:
data['main_format'] = {fmt: absurl(self.opts.url_prefix, u'/get/%s/%d'%(fmt, book_id))}
else:
data['main_format'] = None
data['other_formats'] = {fmt: absurl(self.opts.url_prefix, u'/get/%s/%d'%(fmt, book_id)) for fmt
in other_fmts}
if get_category_urls:
category_urls = data['category_urls'] = {}
ccache = self.categories_cache()
for key in mi.all_field_keys():
fm = mi.metadata_for_field(key)
if (fm and fm['is_category'] and not fm['is_csp'] and
key != 'formats' and fm['datatype'] not in ['rating']):
categories = mi.get(key)
if isinstance(categories, basestring):
categories = [categories]
if categories is None:
categories = []
dbtags = {}
for category in categories:
for tag in ccache.get(key, []):
if tag.original_name == category:
dbtags[category] = books_in_url(self.opts.url_prefix,
tag.category if tag.category else key,
tag.original_name if tag.id is None else
unicode(tag.id))
break
category_urls[key] = dbtags
return data, mi.last_modified
def metadata_to_xmp_packet(mi):
A = ElementMaker(namespace=NS_MAP['x'], nsmap=nsmap('x'))
R = ElementMaker(namespace=NS_MAP['rdf'], nsmap=nsmap('rdf'))
root = A.xmpmeta(R.RDF)
rdf = root[0]
dc = rdf.makeelement(expand('rdf:Description'), nsmap=nsmap('dc'))
dc.set(expand('rdf:about'), '')
rdf.append(dc)
for prop, tag in iteritems({
'title': 'dc:title',
'comments': 'dc:description'
}):
val = mi.get(prop) or ''
create_alt_property(dc, tag, val)
for prop, (tag, ordered) in iteritems({
'authors': ('dc:creator', True),
'tags': ('dc:subject', False),
'publisher': ('dc:publisher', False),
}):
val = mi.get(prop) or ()
if isinstance(val, string_or_bytes):
val = [val]
create_sequence_property(dc, tag, val, ordered)
if not mi.is_null('pubdate'):
create_sequence_property(dc, 'dc:date',
[isoformat(mi.pubdate, as_utc=False)
]) # Adobe spec recommends local time
if not mi.is_null('languages'):
langs = list(
filter(
None,
map(lambda x: lang_as_iso639_1(x) or canonicalize_lang(x),
mi.languages)))
if langs:
create_sequence_property(dc, 'dc:language', langs, ordered=False)
xmp = rdf.makeelement(expand('rdf:Description'),
nsmap=nsmap('xmp', 'xmpidq'))
xmp.set(expand('rdf:about'), '')
rdf.append(xmp)
extra_ids = {}
for x in ('prism', 'pdfx'):
p = extra_ids[x] = rdf.makeelement(expand('rdf:Description'),
nsmap=nsmap(x))
p.set(expand('rdf:about'), '')
rdf.append(p)
identifiers = mi.get_identifiers()
if identifiers:
create_identifiers(xmp, identifiers)
for scheme, val in iteritems(identifiers):
if scheme in {'isbn', 'doi'}:
for prefix, parent in iteritems(extra_ids):
ie = parent.makeelement(expand('%s:%s' % (prefix, scheme)))
ie.text = val
parent.append(ie)
d = xmp.makeelement(expand('xmp:MetadataDate'))
d.text = isoformat(now(), as_utc=False)
xmp.append(d)
calibre = rdf.makeelement(expand('rdf:Description'),
nsmap=nsmap('calibre', 'calibreSI', 'calibreCC'))
calibre.set(expand('rdf:about'), '')
rdf.append(calibre)
if not mi.is_null('rating'):
try:
r = float(mi.rating)
except (TypeError, ValueError):
pass
else:
create_simple_property(calibre, 'calibre:rating', '%g' % r)
if not mi.is_null('series'):
create_series(calibre, mi.series, mi.series_index)
if not mi.is_null('timestamp'):
create_simple_property(calibre, 'calibre:timestamp',
isoformat(mi.timestamp, as_utc=False))
for x in ('author_link_map', 'user_categories'):
val = getattr(mi, x, None)
if val:
create_simple_property(calibre, 'calibre:' + x, dump_dict(val))
for x in ('title_sort', 'author_sort'):
if not mi.is_null(x):
create_simple_property(calibre, 'calibre:' + x, getattr(mi, x))
all_user_metadata = mi.get_all_user_metadata(True)
if all_user_metadata:
create_user_metadata(calibre, all_user_metadata)
return serialize_xmp_packet(root)
def adapt_datetime(dt):
return isoformat(dt, sep=' ')
def ajax_book_to_json(self, book_id, get_category_urls=True,
device_compatible=False):
mi = self.db.get_metadata(book_id, index_is_id=True)
if not device_compatible:
try:
mi.rating = mi.rating/2.
except:
mi.rating = 0.0
data = self.ajax_json_codec.encode_book_metadata(mi)
for x in ('publication_type', 'size', 'db_id', 'lpath', 'mime',
'rights', 'book_producer'):
data.pop(x, None)
data['cover'] = absurl(self.opts.url_prefix, u'/get/cover/%d'%book_id)
data['thumbnail'] = absurl(self.opts.url_prefix, u'/get/thumb/%d'%book_id)
if not device_compatible:
mi.format_metadata = {k.lower():dict(v) for k, v in
mi.format_metadata.iteritems()}
for v in mi.format_metadata.itervalues():
mtime = v.get('mtime', None)
if mtime is not None:
v['mtime'] = isoformat(mtime, as_utc=True)
data['format_metadata'] = mi.format_metadata
fmts = set(x.lower() for x in mi.format_metadata.iterkeys())
pf = prefs['output_format'].lower()
other_fmts = list(fmts)
try:
fmt = pf if pf in fmts else other_fmts[0]
except:
fmt = None
if fmts and fmt:
other_fmts = [x for x in fmts if x != fmt]
data['formats'] = sorted(fmts)
if fmt:
data['main_format'] = {fmt: absurl(self.opts.url_prefix, u'/get/%s/%d'%(fmt, book_id))}
else:
data['main_format'] = None
data['other_formats'] = {fmt: absurl(self.opts.url_prefix, u'/get/%s/%d'%(fmt, book_id)) for fmt
in other_fmts}
if get_category_urls:
category_urls = data['category_urls'] = {}
ccache = self.categories_cache()
for key in mi.all_field_keys():
fm = mi.metadata_for_field(key)
if (fm and fm['is_category'] and not fm['is_csp'] and
key != 'formats' and fm['datatype'] not in ['rating']):
categories = mi.get(key)
if isinstance(categories, basestring):
categories = [categories]
if categories is None:
categories = []
dbtags = {}
for category in categories:
for tag in ccache.get(key, []):
if tag.original_name == category:
dbtags[category] = books_in_url(self.opts.url_prefix,
tag.category if tag.category else key,
tag.original_name if tag.id is None else
unicode(tag.id))
break
category_urls[key] = dbtags
else:
series = data.get('series', None)
if series:
tsorder = tweaks['save_template_title_series_sorting']
series = title_sort(series, order=tsorder)
else:
series = ''
data['_series_sort_'] = series
return data, mi.last_modified
def do_list(db, fields, afields, sort_by, ascending, search_text, line_width, separator,
prefix, subtitle='Books in the calibre database'):
from calibre.constants import terminal_controller as tc
terminal_controller = tc()
if sort_by:
db.sort(sort_by, ascending)
if search_text:
db.search(search_text)
data = db.get_data_as_dict(prefix, authors_as_string=True)
fields = ['id'] + fields
title_fields = fields
def field_name(f):
ans = f
if f[0] == '*':
if f.endswith('_index'):
fkey = f[1:-len('_index')]
num = db.custom_column_label_map[fkey]['num']
ans = '%d_index'%num
else:
ans = db.custom_column_label_map[f[1:]]['num']
return ans
fields = list(map(field_name, fields))
for f in data:
fmts = [x for x in f['formats'] if x is not None]
f['formats'] = u'[%s]'%u','.join(fmts)
widths = list(map(lambda x : 0, fields))
for record in data:
for f in record.keys():
if hasattr(record[f], 'isoformat'):
record[f] = isoformat(record[f], as_utc=False)
else:
record[f] = unicode(record[f])
record[f] = record[f].replace('\n', ' ')
for i in data:
for j, field in enumerate(fields):
widths[j] = max(widths[j], len(unicode(i[field])))
screen_width = terminal_controller.COLS if line_width < 0 else line_width
if not screen_width:
screen_width = 80
field_width = screen_width//len(fields)
base_widths = map(lambda x: min(x+1, field_width), widths)
while sum(base_widths) < screen_width:
adjusted = False
for i in range(len(widths)):
if base_widths[i] < widths[i]:
base_widths[i] += min(screen_width-sum(base_widths), widths[i]-base_widths[i])
adjusted = True
break
if not adjusted:
break
widths = list(base_widths)
titles = map(lambda x, y: '%-*s%s'%(x-len(separator), y, separator),
widths, title_fields)
print terminal_controller.GREEN + ''.join(titles)+terminal_controller.NORMAL
wrappers = map(lambda x: TextWrapper(x-1), widths)
o = cStringIO.StringIO()
for record in data:
text = [wrappers[i].wrap(unicode(record[field]).encode('utf-8')) for i, field in enumerate(fields)]
lines = max(map(len, text))
for l in range(lines):
for i, field in enumerate(text):
ft = text[i][l] if l < len(text[i]) else ''
filler = '%*s'%(widths[i]-len(ft)-1, '')
o.write(ft)
o.write(filler+separator)
print >>o
return o.getvalue()
def metadata_to_xmp_packet(mi):
A = ElementMaker(namespace=NS_MAP['x'], nsmap=nsmap('x'))
R = ElementMaker(namespace=NS_MAP['rdf'], nsmap=nsmap('rdf'))
root = A.xmpmeta(R.RDF)
rdf = root[0]
dc = rdf.makeelement(expand('rdf:Description'), nsmap=nsmap('dc'))
dc.set(expand('rdf:about'), '')
rdf.append(dc)
for prop, tag in {'title':'dc:title', 'comments':'dc:description'}.iteritems():
val = mi.get(prop) or ''
create_alt_property(dc, tag, val)
for prop, (tag, ordered) in {
'authors':('dc:creator', True), 'tags':('dc:subject', False), 'publisher':('dc:publisher', False),
}.iteritems():
val = mi.get(prop) or ()
if isinstance(val, basestring):
val = [val]
create_sequence_property(dc, tag, val, ordered)
if not mi.is_null('pubdate'):
create_sequence_property(dc, 'dc:date', [isoformat(mi.pubdate, as_utc=False)]) # Adobe spec recommends local time
if not mi.is_null('languages'):
langs = filter(None, map(lambda x:lang_as_iso639_1(x) or canonicalize_lang(x), mi.languages))
if langs:
create_sequence_property(dc, 'dc:language', langs, ordered=False)
xmp = rdf.makeelement(expand('rdf:Description'), nsmap=nsmap('xmp', 'xmpidq'))
xmp.set(expand('rdf:about'), '')
rdf.append(xmp)
extra_ids = {}
for x in ('prism', 'pdfx'):
p = extra_ids[x] = rdf.makeelement(expand('rdf:Description'), nsmap=nsmap(x))
p.set(expand('rdf:about'), '')
rdf.append(p)
identifiers = mi.get_identifiers()
if identifiers:
create_identifiers(xmp, identifiers)
for scheme, val in identifiers.iteritems():
if scheme in {'isbn', 'doi'}:
for prefix, parent in extra_ids.iteritems():
ie = parent.makeelement(expand('%s:%s'%(prefix, scheme)))
ie.text = val
parent.append(ie)
d = xmp.makeelement(expand('xmp:MetadataDate'))
d.text = isoformat(now(), as_utc=False)
xmp.append(d)
calibre = rdf.makeelement(expand('rdf:Description'), nsmap=nsmap('calibre', 'calibreSI', 'calibreCC'))
calibre.set(expand('rdf:about'), '')
rdf.append(calibre)
if not mi.is_null('rating'):
try:
r = float(mi.rating)
except (TypeError, ValueError):
pass
else:
create_simple_property(calibre, 'calibre:rating', '%g' % r)
if not mi.is_null('series'):
create_series(calibre, mi.series, mi.series_index)
if not mi.is_null('timestamp'):
create_simple_property(calibre, 'calibre:timestamp', isoformat(mi.timestamp, as_utc=False))
for x in ('author_link_map', 'user_categories'):
val = getattr(mi, x, None)
if val:
create_simple_property(calibre, 'calibre:'+x, dump_dict(val))
for x in ('title_sort', 'author_sort'):
if not mi.is_null(x):
create_simple_property(calibre, 'calibre:'+x, getattr(mi, x))
all_user_metadata = mi.get_all_user_metadata(True)
if all_user_metadata:
create_user_metadata(calibre, all_user_metadata)
return serialize_xmp_packet(root)
def book_to_json(ctx, rd, db, book_id,
get_category_urls=True, device_compatible=False, device_for_template=None):
mi = db.get_metadata(book_id, get_cover=False)
codec = JsonCodec(db.field_metadata)
if not device_compatible:
try:
mi.rating = mi.rating/2.
except Exception:
mi.rating = 0.0
data = codec.encode_book_metadata(mi)
for x in ('publication_type', 'size', 'db_id', 'lpath', 'mime',
'rights', 'book_producer'):
data.pop(x, None)
get = partial(ctx.url_for, get_content, book_id=book_id, library_id=db.server_library_id)
data['cover'] = get(what='cover')
data['thumbnail'] = get(what='thumb')
if not device_compatible:
mi.format_metadata = {k.lower():dict(v) for k, v in
mi.format_metadata.iteritems()}
for v in mi.format_metadata.itervalues():
mtime = v.get('mtime', None)
if mtime is not None:
v['mtime'] = isoformat(mtime, as_utc=True)
data['format_metadata'] = mi.format_metadata
fmts = set(x.lower() for x in mi.format_metadata.iterkeys())
pf = prefs['output_format'].lower()
other_fmts = list(fmts)
try:
fmt = pf if pf in fmts else other_fmts[0]
except:
fmt = None
if fmts and fmt:
other_fmts = [x for x in fmts if x != fmt]
data['formats'] = sorted(fmts)
if fmt:
data['main_format'] = {fmt:get(what=fmt)}
else:
data['main_format'] = None
data['other_formats'] = {fmt:get(what=fmt) for fmt in other_fmts}
if get_category_urls:
category_urls = data['category_urls'] = {}
all_cats = ctx.get_categories(rd, db)
for key in mi.all_field_keys():
fm = mi.metadata_for_field(key)
if (fm and fm['is_category'] and not fm['is_csp'] and
key != 'formats' and fm['datatype'] != 'rating'):
categories = mi.get(key) or []
if isinstance(categories, basestring):
categories = [categories]
category_urls[key] = dbtags = {}
for category in categories:
for tag in all_cats.get(key, ()):
if tag.original_name == category:
dbtags[category] = ctx.url_for(
books_in,
encoded_category=encode_name(tag.category if tag.category else key),
encoded_item=encode_name(tag.original_name if tag.id is None else unicode(tag.id)),
library_id=db.server_library_id
)
break
else:
series = data.get('series', None) or ''
if series:
tsorder = tweaks['save_template_title_series_sorting']
series = title_sort(series, order=tsorder)
data['_series_sort_'] = series
if device_for_template:
import posixpath
from calibre.devices.utils import create_upload_path
from calibre.utils.filenames import ascii_filename as sanitize
from calibre.customize.ui import device_plugins
for device_class in device_plugins():
if device_class.__class__.__name__ == device_for_template:
template = device_class.save_template()
data['_filename_'] = create_upload_path(mi, book_id,
template, sanitize, path_type=posixpath)
break
return data, mi.last_modified
def create_bibtex_entry(entry, fields, mode, template_citation,
bibtexdict, db, citation_bibtex=True, calibre_files=True):
# Bibtex doesn't like UTF-8 but keep unicode until writing
# Define starting chain or if book valid strict and not book return a Fail string
bibtex_entry = []
if mode != "misc" and check_entry_book_valid(entry) :
bibtex_entry.append('@book{')
elif mode != "book" :
bibtex_entry.append('@misc{')
else :
# case strict book
return ''
if citation_bibtex :
# Citation tag
bibtex_entry.append(make_bibtex_citation(entry, template_citation,
bibtexdict))
bibtex_entry = [' '.join(bibtex_entry)]
for field in fields:
if field.startswith('#'):
item = db.get_field(entry['id'],field,index_is_id=True)
if isinstance(item, (bool, numbers.Number)):
item = repr(item)
elif field == 'title_sort':
item = entry['sort']
elif field == 'library_name':
item = library_name
else:
item = entry[field]
# check if the field should be included (none or empty)
if item is None:
continue
try:
if len(item) == 0 :
continue
except TypeError:
pass
if field == 'authors' :
bibtex_entry.append('author = "%s"' % bibtexdict.bibtex_author_format(item))
elif field == 'id' :
bibtex_entry.append('calibreid = "%s"' % int(item))
elif field == 'rating' :
bibtex_entry.append('rating = "%s"' % int(item))
elif field == 'size' :
bibtex_entry.append('%s = "%s octets"' % (field, int(item)))
elif field == 'tags' :
# A list to flatten
bibtex_entry.append('tags = "%s"' % bibtexdict.utf8ToBibtex(', '.join(item)))
elif field == 'comments' :
# \n removal
item = item.replace('\r\n', ' ')
item = item.replace('\n', ' ')
# unmatched brace removal (users should use \leftbrace or \rightbrace for single braces)
item = bibtexdict.stripUnmatchedSyntax(item, '{', '}')
# html to text
try:
item = html2text(item)
except:
log.warn("Failed to convert comments to text")
bibtex_entry.append('note = "%s"' % bibtexdict.utf8ToBibtex(item))
elif field == 'isbn' :
# Could be 9, 10 or 13 digits
bibtex_entry.append('isbn = "%s"' % format_isbn(item))
elif field == 'formats' :
# Add file path if format is selected
formats = [format.rpartition('.')[2].lower() for format in item]
bibtex_entry.append('formats = "%s"' % ', '.join(formats))
if calibre_files:
files = [':%s:%s' % (format, format.rpartition('.')[2].upper())
for format in item]
bibtex_entry.append('file = "%s"' % ', '.join(files))
elif field == 'series_index' :
bibtex_entry.append('volume = "%s"' % int(item))
elif field == 'timestamp' :
bibtex_entry.append('timestamp = "%s"' % isoformat(item).partition('T')[0])
elif field == 'pubdate' :
bibtex_entry.append('year = "%s"' % item.year)
bibtex_entry.append('month = "%s"' % bibtexdict.utf8ToBibtex(strftime("%b", item)))
elif field.startswith('#') and isinstance(item, string_or_bytes):
bibtex_entry.append('custom_%s = "%s"' % (field[1:],
bibtexdict.utf8ToBibtex(item)))
elif isinstance(item, string_or_bytes):
# elif field in ['title', 'publisher', 'cover', 'uuid', 'ondevice',
# 'author_sort', 'series', 'title_sort'] :
bibtex_entry.append('%s = "%s"' % (field, bibtexdict.utf8ToBibtex(item)))
bibtex_entry = ',\n '.join(bibtex_entry)
bibtex_entry += ' }\n\n'
return bibtex_entry
def adapt_datetime(dt):
return isoformat(dt, sep=' ')
def run(self, path_to_output, opts, db, notification=DummyReporter()):
from calibre.library import current_library_name
from calibre.utils.date import isoformat
from calibre.utils.html2text import html2text
from calibre.utils.logging import default_log as log
from lxml import etree
self.fmt = path_to_output.rpartition(".")[2]
self.notification = notification
current_library = current_library_name()
if getattr(opts, "library_path", None):
current_library = os.path.basename(opts.library_path)
if opts.verbose:
opts_dict = vars(opts)
log("%s('%s'): Generating %s" % (self.name, current_library, self.fmt.upper()))
if opts.connected_device["is_device_connected"]:
log(" connected_device: %s" % opts.connected_device["name"])
if opts_dict["search_text"]:
log(" --search='%s'" % opts_dict["search_text"])
if opts_dict["ids"]:
log(" Book count: %d" % len(opts_dict["ids"]))
if opts_dict["search_text"]:
log(" (--search ignored when a subset of the database is specified)")
if opts_dict["fields"]:
if opts_dict["fields"] == "all":
log(" Fields: %s" % ", ".join(FIELDS[1:]))
else:
log(" Fields: %s" % opts_dict["fields"])
# If a list of ids are provided, don't use search_text
if opts.ids:
opts.search_text = None
data = self.search_sort_db(db, opts)
if not len(data):
log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text)
# raise SystemExit(1)
# Get the requested output fields as a list
fields = self.get_output_fields(db, opts)
# If connected device, add 'On Device' values to data
if opts.connected_device["is_device_connected"] and "ondevice" in fields:
for entry in data:
entry["ondevice"] = db.catalog_plugin_on_device_temp_mapping[entry["id"]]["ondevice"]
fm = {x: db.field_metadata.get(x, {}) for x in fields}
if self.fmt == "csv":
outfile = codecs.open(path_to_output, "w", "utf8")
# Write a UTF-8 BOM
outfile.write("\xef\xbb\xbf")
# Output the field headers
outfile.write(u"%s\n" % u",".join(fields))
# Output the entry fields
for entry in data:
outstr = []
for field in fields:
if field.startswith("#"):
item = db.get_field(entry["id"], field, index_is_id=True)
elif field == "library_name":
item = current_library
elif field == "title_sort":
item = entry["sort"]
else:
item = entry[field]
if item is None:
outstr.append('""')
continue
elif field == "formats":
fmt_list = []
for format in item:
fmt_list.append(format.rpartition(".")[2].lower())
item = ", ".join(fmt_list)
elif field in ["authors", "tags"]:
item = ", ".join(item)
elif field == "isbn":
# Could be 9, 10 or 13 digits, with hyphens, possibly ending in 'X'
item = u"%s" % re.sub(r"[^\dX-]", "", item)
elif field in ["pubdate", "timestamp"]:
item = isoformat(item, as_utc=False)
elif field == "comments":
item = item.replace(u"\r\n", u" ")
item = item.replace(u"\n", u" ")
elif fm.get(field, {}).get("datatype", None) == "rating" and item:
item = u"%.2g" % (item / 2.0)
# Convert HTML to markdown text
if type(item) is unicode:
opening_tag = re.search("<(\w+)(\x20|>)", item)
if opening_tag:
closing_tag = re.search("<\/%s>$" % opening_tag.group(1), item)
if closing_tag:
item = html2text(item)
outstr.append(u'"%s"' % unicode(item).replace('"', '""'))
outfile.write(u",".join(outstr) + u"\n")
outfile.close()
elif self.fmt == "xml":
from lxml.builder import E
root = E.calibredb()
for r in data:
record = E.record()
root.append(record)
for field in fields:
if field.startswith("#"):
val = db.get_field(r["id"], field, index_is_id=True)
if not isinstance(val, (str, unicode)):
val = unicode(val)
item = getattr(E, field.replace("#", "_"))(val)
record.append(item)
for field in ("id", "uuid", "publisher", "rating", "size", "isbn", "ondevice", "identifiers"):
if field in fields:
val = r[field]
if not val:
continue
if not isinstance(val, (str, unicode)):
if fm.get(field, {}).get("datatype", None) == "rating" and val:
val = u"%.2g" % (val / 2.0)
val = unicode(val)
item = getattr(E, field)(val)
record.append(item)
if "title" in fields:
title = E.title(r["title"], sort=r["sort"])
record.append(title)
if "authors" in fields:
aus = E.authors(sort=r["author_sort"])
for au in r["authors"]:
aus.append(E.author(au))
record.append(aus)
for field in ("timestamp", "pubdate"):
if field in fields:
record.append(getattr(E, field)(isoformat(r[field], as_utc=False)))
if "tags" in fields and r["tags"]:
tags = E.tags()
for tag in r["tags"]:
tags.append(E.tag(tag))
record.append(tags)
if "comments" in fields and r["comments"]:
record.append(E.comments(r["comments"]))
if "series" in fields and r["series"]:
record.append(E.series(r["series"], index=str(r["series_index"])))
if "cover" in fields and r["cover"]:
record.append(E.cover(r["cover"].replace(os.sep, "/")))
if "formats" in fields and r["formats"]:
fmt = E.formats()
for f in r["formats"]:
fmt.append(E.format(f.replace(os.sep, "/")))
record.append(fmt)
if "library_name" in fields:
record.append(E.library_name(current_library))
with open(path_to_output, "w") as f:
f.write(etree.tostring(root, encoding="utf-8", xml_declaration=True, pretty_print=True))
