def get_metadata(stream, cover=True):
with TemporaryDirectory('_pdf_metadata_read') as pdfpath:
stream.seek(0)
with open(os.path.join(pdfpath, 'src.pdf'), 'wb') as f:
shutil.copyfileobj(stream, f)
info = read_info(pdfpath, bool(cover))
if info is None:
raise ValueError('Could not read info dict from PDF')
covpath = os.path.join(pdfpath, 'cover.jpg')
cdata = None
if cover and os.path.exists(covpath):
with open(covpath, 'rb') as f:
cdata = f.read()
title = info.get('Title', None) or 'Unknown'
au = info.get('Author', None)
if au is None:
au = ['Unknown']
else:
au = string_to_authors(au)
mi = MetaInformation(title, au)
creator = info.get('Creator', None)
if creator:
mi.book_producer = creator
keywords = info.get('Keywords', None)
mi.tags = []
if keywords:
mi.tags = [x.strip() for x in keywords.split(',')]
isbn = [check_isbn(x) for x in mi.tags if check_isbn(x)]
if isbn:
mi.isbn = isbn = isbn[0]
mi.tags = [x for x in mi.tags if check_isbn(x) != isbn]
subject = info.get('Subject', None)
if subject:
mi.tags.insert(0, subject)
if 'xmp_metadata' in info:
from ebook_converter.ebooks.metadata.xmp import consolidate_metadata
mi = consolidate_metadata(mi, info)
# Look for recognizable identifiers in the info dict, if they were not
# found in the XMP metadata
for scheme, check_func in {'doi': check_doi, 'isbn': check_isbn}.items():
if scheme not in mi.get_identifiers():
for k, v in info.items():
if k != 'xmp_metadata':
val = check_func(v)
if val:
mi.set_identifier(scheme, val)
break
if cdata:
mi.cover_data = ('jpeg', cdata)
return mi
def get_metadata(stream):
"""
Return basic meta-data about the LRF file in C{stream} as a
L{MetaInformation} object.
@param stream: A file like object or an instance of L{LRFMetaFile}
"""
lrf = stream if isinstance(stream, LRFMetaFile) else LRFMetaFile(stream)
authors = string_to_authors(lrf.author)
mi = MetaInformation(lrf.title.strip(), authors)
mi.author = lrf.author.strip()
mi.comments = lrf.free_text.strip()
mi.category = lrf.category.strip() + ', ' + lrf.classification.strip()
tags = [x.strip() for x in mi.category.split(',') if x.strip()]
if tags:
mi.tags = tags
if mi.category.strip() == ',':
mi.category = None
mi.publisher = lrf.publisher.strip()
mi.cover_data = lrf.get_cover()
try:
mi.title_sort = lrf.title_reading.strip()
if not mi.title_sort:
mi.title_sort = None
except Exception:
pass
try:
mi.author_sort = lrf.author_reading.strip()
if not mi.author_sort:
mi.author_sort = None
except Exception:
pass
if not mi.title or 'unknown' in mi.title.lower():
mi.title = None
if not mi.authors:
mi.authors = None
if not mi.author or 'unknown' in mi.author.lower():
mi.author = None
if not mi.category or 'unknown' in mi.category.lower():
mi.category = None
if not mi.publisher or 'unknown' in mi.publisher.lower() or \
'some publisher' in mi.publisher.lower():
mi.publisher = None
return mi
def read_doc_props(raw, mi, XPath):
root = etree.fromstring(raw)
titles = XPath('//dc:title')(root)
if titles:
title = titles[0].text
if title and title.strip():
mi.title = title.strip()
tags = []
for subject in XPath('//dc:subject')(root):
if subject.text and subject.text.strip():
tags.append(subject.text.strip().replace(',', '_'))
for keywords in XPath('//cp:keywords')(root):
if keywords.text and keywords.text.strip():
for x in keywords.text.split():
tags.extend(y.strip() for y in x.split(',') if y.strip())
if tags:
mi.tags = tags
authors = XPath('//dc:creator')(root)
aut = []
for author in authors:
if author.text and author.text.strip():
aut.extend(string_to_authors(author.text))
if aut:
mi.authors = aut
mi.author_sort = authors_to_sort_string(aut)
desc = XPath('//dc:description')(root)
if desc:
raw = etree.tostring(desc[0], method='text', encoding='unicode')
# Word 2007 mangles newlines in the summary
raw = raw.replace('_x000d_', '')
mi.comments = raw.strip()
langs = []
for lang in XPath('//dc:language')(root):
if lang.text and lang.text.strip():
canonic_lang = canonicalize_lang(lang.text)
if canonic_lang:
langs.append(canonic_lang)
if langs:
mi.languages = langs
def create_oebbook(self, htmlpath, basedir, opts, log, mi):
import uuid
from ebook_converter.ebooks.conversion.plumber import create_oebbook
from ebook_converter.ebooks.oeb.base import (DirContainer,
rewrite_links, urlnormalize, BINARY_MIME, OEB_STYLES,
xpath, urlquote)
from ebook_converter.ebooks.oeb.transforms.metadata import \
meta_info_to_oeb_metadata
from ebook_converter.ebooks.html.input import get_filelist
from ebook_converter.ebooks.metadata import string_to_authors
from ebook_converter.utils.localization import canonicalize_lang
import css_parser, logging
css_parser.log.setLevel(logging.WARN)
self.OEB_STYLES = OEB_STYLES
oeb = create_oebbook(log, None, opts, self,
encoding=opts.input_encoding, populate=False)
self.oeb = oeb
metadata = oeb.metadata
meta_info_to_oeb_metadata(mi, metadata, log)
if not metadata.language:
l = canonicalize_lang(getattr(opts, 'language', None))
if not l:
oeb.logger.warn('Language not specified')
l = get_lang().replace('_', '-')
metadata.add('language', l)
if not metadata.creator:
a = getattr(opts, 'authors', None)
if a:
a = string_to_authors(a)
if not a:
oeb.logger.warn('Creator not specified')
a = [self.oeb.translate('Unknown')]
for aut in a:
metadata.add('creator', aut)
if not metadata.title:
oeb.logger.warn('Title not specified')
metadata.add('title', self.oeb.translate('Unknown'))
bookid = str(uuid.uuid4())
metadata.add('identifier', bookid, id='uuid_id', scheme='uuid')
for ident in metadata.identifier:
if 'id' in ident.attrib:
self.oeb.uid = metadata.identifier[0]
break
filelist = get_filelist(htmlpath, basedir, opts, log)
filelist = [f for f in filelist if not f.is_binary]
htmlfile_map = {}
for f in filelist:
path = f.path
oeb.container = DirContainer(os.path.dirname(path), log,
ignore_opf=True)
bname = os.path.basename(path)
id, href = oeb.manifest.generate(id='html', href=sanitize_file_name(bname))
htmlfile_map[path] = href
item = oeb.manifest.add(id, href, 'text/html')
if path == htmlpath and '%' in path:
bname = urlquote(bname)
item.html_input_href = bname
oeb.spine.add(item, True)
self.added_resources = {}
self.log = log
self.log('Normalizing filename cases')
for path, href in htmlfile_map.items():
self.added_resources[path] = href
self.urlnormalize, self.DirContainer = urlnormalize, DirContainer
self.urldefrag = urllib.parse.urldefrag
self.BINARY_MIME = BINARY_MIME
self.log('Rewriting HTML links')
for f in filelist:
path = f.path
dpath = os.path.dirname(path)
oeb.container = DirContainer(dpath, log, ignore_opf=True)
href = htmlfile_map[path]
try:
item = oeb.manifest.hrefs[href]
except KeyError:
item = oeb.manifest.hrefs[urlnormalize(href)]
rewrite_links(item.data,
functools.partial(self.resource_adder, base=dpath))
for item in oeb.manifest.values():
if item.media_type in self.OEB_STYLES:
dpath = None
for path, href in self.added_resources.items():
if href == item.href:
dpath = os.path.dirname(path)
break
css_parser.replaceUrls(item.data,
functools.partial(self.resource_adder, base=dpath))
toc = self.oeb.toc
self.oeb.auto_generated_toc = True
titles = []
headers = []
for item in self.oeb.spine:
if not item.linear:
continue
html = item.data
title = ''.join(xpath(html, '/h:html/h:head/h:title/text()'))
title = re.sub(r'\s+', ' ', title.strip())
if title:
titles.append(title)
headers.append('(unlabled)')
for tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'strong'):
expr = '/h:html/h:body//h:%s[position()=1]/text()'
header = ''.join(xpath(html, expr % tag))
header = re.sub(r'\s+', ' ', header.strip())
if header:
headers[-1] = header
break
use = titles
if len(titles) > len(set(titles)):
use = headers
for title, item in zip(use, self.oeb.spine):
if not item.linear:
continue
toc.add(title, item.href)
oeb.container = DirContainer(os.getcwd(), oeb.log, ignore_opf=True)
return oeb
def get_metadata_(src, encoding=None):
# Meta data definitions as in
# https://www.mobileread.com/forums/showpost.php?p=712544&postcount=9
if isinstance(src, bytes):
if not encoding:
src = xml_to_unicode(src)[0]
else:
src = src.decode(encoding, 'replace')
src = src[:150000] # Searching shouldn't take too long
comment_tags, meta_tags, meta_tag_ids, title_tag = parse_metadata(src)
def get_all(field):
ans = comment_tags.get(field, meta_tags.get(field, None))
if ans:
ans = [x.strip() for x in ans if x.strip()]
if not ans:
ans = None
return ans
def get(field):
ans = get_all(field)
if ans:
ans = ans[0]
return ans
# Title
title = get('title') or title_tag.strip() or 'Unknown'
# Author
authors = authors_to_string(get_all('authors')) or 'Unknown'
# Create MetaInformation with Title and Author
mi = Metadata(title, string_to_authors(authors))
# Single-value text fields
for field in ('publisher', 'isbn'):
val = get(field)
if val:
setattr(mi, field, val)
# Multi-value text fields
for field in ('languages',):
val = get_all(field)
if val:
setattr(mi, field, val)
# HTML fields
for field in ('comments',):
val = get(field)
if val:
setattr(mi, field, val.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"').replace("'", '''))
# Date fields
for field in ('pubdate', 'timestamp'):
try:
val = parse_date(get(field))
except:
pass
else:
if not is_date_undefined(val):
setattr(mi, field, val)
# SERIES
series = get('series')
if series:
pat = re.compile(r'\[([.0-9]+)\]$')
match = pat.search(series)
series_index = None
if match is not None:
try:
series_index = float(match.group(1))
except:
pass
series = series.replace(match.group(), '').strip()
mi.series = series
if series_index is None:
series_index = get('series_index')
try:
series_index = float(series_index)
except:
pass
if series_index is not None:
mi.series_index = series_index
# RATING
rating = get('rating')
if rating:
try:
mi.rating = float(rating)
if mi.rating < 0:
mi.rating = 0
if mi.rating > 10:
mi.rating = 0
except:
pass
# TAGS
tags = get_all('tags')
if tags:
tags = [x.strip() for s in tags for x in s.split(',') if x.strip()]
if tags:
mi.tags = tags
# IDENTIFIERS
for (k,v) in meta_tag_ids.items():
v = [x.strip() for x in v if x.strip()]
if v:
mi.set_identifier(k, v[0])
return mi
def authors_from_string(self, raw):
from ebook_converter.ebooks.metadata import string_to_authors
self.authors = string_to_authors(raw)
def metadata_from_filename(name, pat=None, fallback_pat=None):
if isinstance(name, bytes):
name = name.decode(filesystem_encoding, 'replace')
name = name.rpartition('.')[0]
mi = MetaInformation(None, None)
if pat is None:
pat = re.compile(prefs.get('filename_pattern'))
name = name.replace('_', ' ')
match = pat.search(name)
if match is None and fallback_pat is not None:
match = fallback_pat.search(name)
if match is not None:
try:
mi.title = match.group('title')
except IndexError:
pass
try:
au = match.group('author')
aus = string_to_authors(au)
if aus:
mi.authors = aus
if prefs['swap_author_names'] and mi.authors:
def swap(a):
if ',' in a:
parts = a.split(',', 1)
else:
parts = a.split(None, 1)
if len(parts) > 1:
t = parts[-1]
parts = parts[:-1]
parts.insert(0, t)
return ' '.join(parts)
mi.authors = [swap(x) for x in mi.authors]
except (IndexError, ValueError):
pass
try:
mi.series = match.group('series')
except IndexError:
pass
try:
si = match.group('series_index')
mi.series_index = float(si)
except (IndexError, ValueError, TypeError):
pass
try:
si = match.group('isbn')
mi.isbn = si
except (IndexError, ValueError):
pass
try:
publisher = match.group('publisher')
mi.publisher = publisher
except (IndexError, ValueError):
pass
try:
pubdate = match.group('published')
if pubdate:
from ebook_converter.utils.date import parse_only_date
mi.pubdate = parse_only_date(pubdate)
except:
pass
try:
comments = match.group('comments')
mi.comments = comments
except (IndexError, ValueError):
pass
if mi.is_null('title'):
mi.title = name
return mi
def apply_metadata(root,
mi,
cover_prefix='',
cover_data=None,
apply_null=False,
update_timestamp=False,
force_identifiers=False,
add_missing_cover=True):
prefixes, refines = read_prefixes(root), read_refines(root)
current_mi = read_metadata(root)
if apply_null:
def ok(x):
return True
else:
def ok(x):
return not mi.is_null(x)
if ok('identifiers'):
set_identifiers(root,
prefixes,
refines,
mi.identifiers,
force_identifiers=force_identifiers)
if ok('title'):
set_title(root, prefixes, refines, mi.title, mi.title_sort)
if ok('languages'):
set_languages(root, prefixes, refines, mi.languages)
if ok('book_producer'):
set_book_producers(root, prefixes, refines, (mi.book_producer, ))
aus = string_to_authors(mi.author_sort or '')
authors = []
for i, aut in enumerate(mi.authors):
authors.append(Author(aut, aus[i] if i < len(aus) else None))
if authors or apply_null:
set_authors(root, prefixes, refines, authors)
if ok('pubdate'):
set_pubdate(root, prefixes, refines, mi.pubdate)
if update_timestamp and mi.timestamp is not None:
set_timestamp(root, prefixes, refines, mi.timestamp)
if ok('comments'):
set_comments(root, prefixes, refines, mi.comments)
if ok('publisher'):
set_publisher(root, prefixes, refines, mi.publisher)
if ok('tags'):
set_tags(root, prefixes, refines, mi.tags)
if ok('rating') and mi.rating is not None and mi.rating > 0.1:
set_rating(root, prefixes, refines, mi.rating)
if ok('series'):
set_series(root, prefixes, refines, mi.series, mi.series_index or 1)
if ok('author_link_map'):
set_author_link_map(root, prefixes, refines,
getattr(mi, 'author_link_map', None))
if ok('user_categories'):
set_user_categories(root, prefixes, refines,
getattr(mi, 'user_categories', None))
# We ignore apply_null for the next two to match the behavior with opf2.py
if mi.application_id:
set_application_id(root, prefixes, refines, mi.application_id)
if mi.uuid:
set_uuid(root, prefixes, refines, mi.uuid)
new_user_metadata = mi.get_all_user_metadata(True)
current_user_metadata = current_mi.get_all_user_metadata(True)
missing = object()
for key in tuple(new_user_metadata):
meta = new_user_metadata.get(key)
if meta is None:
if apply_null:
new_user_metadata[key] = None
continue
dt = meta.get('datatype')
if dt == 'text' and meta.get('is_multiple'):
val = mi.get(key, [])
if val or apply_null:
current_user_metadata[key] = meta
elif dt in {'int', 'float', 'bool'}:
val = mi.get(key, missing)
if val is missing:
if apply_null:
current_user_metadata[key] = meta
elif apply_null or val is not None:
current_user_metadata[key] = meta
elif apply_null or not mi.is_null(key):
current_user_metadata[key] = meta
set_user_metadata(root, prefixes, refines, current_user_metadata)
raster_cover = read_raster_cover(root, prefixes, refines)
if not raster_cover and cover_data and add_missing_cover:
if cover_prefix and not cover_prefix.endswith('/'):
cover_prefix += '/'
name = cover_prefix + 'cover.jpg'
i = create_manifest_item(root, name, 'cover')
if i is not None:
ensure_is_only_raster_cover(root, prefixes, refines, name)
raster_cover = name
pretty_print_opf(root)
return raster_cover
def get_metadata(stream, extract_cover=True):
whitespace = re.compile(r'\s+')
def normalize(s):
return whitespace.sub(' ', s).strip()
with ZipFile(stream) as zf:
meta = zf.read('meta.xml')
root = fromstring(meta)
def find(field):
ns, tag = fields[field]
ans = root.xpath('//ns0:{}'.format(tag), namespaces={'ns0': ns})
if ans:
return normalize(
tostring(ans[0],
method='text',
encoding='unicode',
with_tail=False)).strip()
mi = MetaInformation(None, [])
title = find('title')
if title:
mi.title = title
creator = find('initial-creator') or find('creator')
if creator:
mi.authors = string_to_authors(creator)
desc = find('description')
if desc:
mi.comments = desc
lang = find('language')
if lang and canonicalize_lang(lang):
mi.languages = [canonicalize_lang(lang)]
kw = find('keyword') or find('keywords')
if kw:
mi.tags = [x.strip() for x in kw.split(',') if x.strip()]
data = {}
for tag in root.xpath('//ns0:user-defined',
namespaces={'ns0': fields['user-defined'][0]}):
name = (tag.get('{%s}name' % METANS) or '').lower()
vtype = tag.get('{%s}value-type' % METANS) or 'string'
val = tag.text
if name and val:
if vtype == 'boolean':
val = val == 'true'
data[name] = val
opfmeta = False # we need this later for the cover
opfnocover = False
if data.get('opf.metadata'):
# custom metadata contains OPF information
opfmeta = True
if data.get('opf.titlesort', ''):
mi.title_sort = data['opf.titlesort']
if data.get('opf.authors', ''):
mi.authors = string_to_authors(data['opf.authors'])
if data.get('opf.authorsort', ''):
mi.author_sort = data['opf.authorsort']
if data.get('opf.isbn', ''):
isbn = check_isbn(data['opf.isbn'])
if isbn is not None:
mi.isbn = isbn
if data.get('opf.publisher', ''):
mi.publisher = data['opf.publisher']
if data.get('opf.pubdate', ''):
mi.pubdate = parse_date(data['opf.pubdate'], assume_utc=True)
if data.get('opf.identifiers'):
try:
mi.identifiers = json.loads(data['opf.identifiers'])
except Exception:
pass
if data.get('opf.rating'):
try:
mi.rating = max(0, min(float(data['opf.rating']), 10))
except Exception:
pass
if data.get('opf.series', ''):
mi.series = data['opf.series']
if data.get('opf.seriesindex', ''):
try:
mi.series_index = float(data['opf.seriesindex'])
except Exception:
mi.series_index = 1.0
if data.get('opf.language', ''):
cl = canonicalize_lang(data['opf.language'])
if cl:
mi.languages = [cl]
opfnocover = data.get('opf.nocover', False)
if not opfnocover:
try:
read_cover(stream, zf, mi, opfmeta, extract_cover)
except Exception:
pass # Do not let an error reading the cover prevent reading other data
return mi