def to_metadata(self, browser, log, entry_, timeout): # {{{
from lxml import etree
from calibre.ebooks.chardet import xml_to_unicode
from calibre.utils.date import parse_date, utcnow
from calibre.utils.cleantext import clean_ascii_chars
# log.info('entry_ is: ',entry_)
id_url = entry_['url']
douban_id = entry_['id']
title_ = entry_['title']
subtitle = entry_['subtitle']
authors = [x.strip() for x in entry_['author'] if x]
if not authors:
authors = [_('Unknown')]
mi = Metadata(title_, authors)
mi.identifiers = {'douban': douban_id}
mi.comments = entry_['summary']
mi.publisher = entry_['publisher']
# ISBN
mi.isbn = entry_['isbn10']
mi.all_isbns = [entry_['isbn10'], entry_['isbn13']]
# Tags
mi.tags = [x['name'].strip() for x in entry_['tags']]
# pubdate
pubdate = entry_['pubdate']
if pubdate:
try:
default = utcnow().replace(day=15)
mi.pubdate = parse_date(pubdate,
assume_utc=True,
default=default)
except:
log.error('Failed to parse pubdate %r' % pubdate)
# Ratings
mi.rating = float(entry_['rating']['average']) / 2.0
# Cover
mi.has_douban_cover = entry_['image']
return mi
def to_metadata(self, browser, log, entry_, timeout): # {{{
from calibre.utils.date import parse_date, utcnow
douban_id = entry_.get('id')
title = entry_.get('title')
description = entry_.get('summary')
# subtitle = entry_.get('subtitle') # TODO: std metada doesn't have this field
publisher = entry_.get('publisher')
isbn = entry_.get('isbn13') # ISBN11 is obsolute, use ISBN13
pubdate = entry_.get('pubdate')
authors = entry_.get('author')
book_tags = entry_.get('tags')
rating = entry_.get('rating')
cover_url = entry_.get('images', {}).get('large')
series = entry_.get('series')
if not authors:
authors = [_('Unknown')]
if not douban_id or not title:
# Silently discard this entry
return None
mi = Metadata(title, authors)
mi.identifiers = {'douban': douban_id}
mi.publisher = publisher
mi.comments = description
# mi.subtitle = subtitle
# ISBN
isbns = []
if isinstance(isbn, (type(''), bytes)):
if check_isbn(isbn):
isbns.append(isbn)
else:
for x in isbn:
if check_isbn(x):
isbns.append(x)
if isbns:
mi.isbn = sorted(isbns, key=len)[-1]
mi.all_isbns = isbns
# Tags
mi.tags = [tag['name'] for tag in book_tags]
# pubdate
if pubdate:
try:
default = utcnow().replace(day=15)
mi.pubdate = parse_date(pubdate, assume_utc=True, default=default)
except:
log.error('Failed to parse pubdate %r' % pubdate)
# Ratings
if rating:
try:
mi.rating = float(rating['average']) / 2.0
except:
log.exception('Failed to parse rating')
mi.rating = 0
# Cover
mi.has_douban_cover = None
u = cover_url
if u:
# If URL contains "book-default", the book doesn't have a cover
if u.find('book-default') == -1:
mi.has_douban_cover = u
# Series
if series:
mi.series = series['title']
return mi
def to_metadata(browser, log, entry_, timeout): # {{{
from lxml import etree
XPath = partial(etree.XPath, namespaces=NAMESPACES)
# total_results = XPath('//openSearch:totalResults')
# start_index = XPath('//openSearch:startIndex')
# items_per_page = XPath('//openSearch:itemsPerPage')
entry = XPath('//atom:entry')
entry_id = XPath('descendant::atom:id')
creator = XPath('descendant::dc:creator')
identifier = XPath('descendant::dc:identifier')
title = XPath('descendant::dc:title')
date = XPath('descendant::dc:date')
publisher = XPath('descendant::dc:publisher')
subject = XPath('descendant::dc:subject')
description = XPath('descendant::dc:description')
language = XPath('descendant::dc:language')
rating = XPath('descendant::gd:rating[@average]')
def get_text(extra, x):
try:
ans = x(extra)
if ans:
ans = ans[0].text
if ans and ans.strip():
return ans.strip()
except:
log.exception('Programming error:')
return None
id_url = entry_id(entry_)[0].text
google_id = id_url.split('/')[-1]
title_ = ': '.join([x.text for x in title(entry_)]).strip()
authors = [x.text.strip() for x in creator(entry_) if x.text]
if not authors:
authors = [_('Unknown')]
if not id_url or not title:
# Silently discard this entry
return None
mi = Metadata(title_, authors)
mi.identifiers = {'google':google_id}
try:
raw = get_details(browser, id_url, timeout)
feed = etree.fromstring(xml_to_unicode(clean_ascii_chars(raw),
strip_encoding_pats=True)[0])
extra = entry(feed)[0]
except:
log.exception('Failed to get additional details for', mi.title)
return mi
mi.comments = get_text(extra, description)
lang = canonicalize_lang(get_text(extra, language))
if lang:
mi.language = lang
mi.publisher = get_text(extra, publisher)
# ISBN
isbns = []
for x in identifier(extra):
t = str(x.text).strip()
if t[:5].upper() in ('ISBN:', 'LCCN:', 'OCLC:'):
if t[:5].upper() == 'ISBN:':
t = check_isbn(t[5:])
if t:
isbns.append(t)
if isbns:
mi.isbn = sorted(isbns, key=len)[-1]
mi.all_isbns = isbns
# Tags
try:
btags = [x.text for x in subject(extra) if x.text]
tags = []
for t in btags:
atags = [y.strip() for y in t.split('/')]
for tag in atags:
if tag not in tags:
tags.append(tag)
except:
log.exception('Failed to parse tags:')
tags = []
if tags:
mi.tags = [x.replace(',', ';') for x in tags]
# pubdate
pubdate = get_text(extra, date)
if pubdate:
from calibre.utils.date import parse_date, utcnow
try:
default = utcnow().replace(day=15)
mi.pubdate = parse_date(pubdate, assume_utc=True, default=default)
except:
log.error('Failed to parse pubdate %r'%pubdate)
# Ratings
for x in rating(extra):
try:
mi.rating = float(x.get('average'))
if mi.rating > 5:
mi.rating /= 2
except:
log.exception('Failed to parse rating')
# Cover
mi.has_google_cover = None
for x in extra.xpath(
'//*[@href and @rel="http://schemas.google.com/books/2008/thumbnail"]'):
mi.has_google_cover = x.get('href')
break
return mi
def to_metadata(browser, log, entry_, timeout): # {{{
from lxml import etree
from calibre.ebooks.chardet import xml_to_unicode
from calibre.utils.date import parse_date, utcnow
from calibre.utils.cleantext import clean_ascii_chars
XPath = partial(etree.XPath, namespaces=NAMESPACES)
entry = XPath('//atom:entry')
entry_id = XPath('descendant::atom:id')
title = XPath('descendant::atom:title')
description = XPath('descendant::atom:summary')
publisher = XPath("descendant::db:attribute[@name='publisher']")
isbn = XPath("descendant::db:attribute[@name='isbn13']")
date = XPath("descendant::db:attribute[@name='pubdate']")
creator = XPath("descendant::db:attribute[@name='author']")
booktag = XPath("descendant::db:tag/attribute::name")
rating = XPath("descendant::gd:rating/attribute::average")
cover_url = XPath("descendant::atom:link[@rel='image']/attribute::href")
def get_text(extra, x):
try:
ans = x(extra)
if ans:
ans = ans[0].text
if ans and ans.strip():
return ans.strip()
except:
log.exception('Programming error:')
return None
id_url = entry_id(entry_)[0].text.replace('http://', 'https://')
douban_id = id_url.split('/')[-1]
title_ = ': '.join([x.text for x in title(entry_)]).strip()
authors = [x.text.strip() for x in creator(entry_) if x.text]
if not authors:
authors = [_('Unknown')]
if not id_url or not title:
# Silently discard this entry
return None
mi = Metadata(title_, authors)
mi.identifiers = {'douban':douban_id}
try:
raw = get_details(browser, id_url, timeout)
feed = etree.fromstring(xml_to_unicode(clean_ascii_chars(raw),
strip_encoding_pats=True)[0])
extra = entry(feed)[0]
except:
log.exception('Failed to get additional details for', mi.title)
return mi
mi.comments = get_text(extra, description)
mi.publisher = get_text(extra, publisher)
# ISBN
isbns = []
for x in [t.text for t in isbn(extra)]:
if check_isbn(x):
isbns.append(x)
if isbns:
mi.isbn = sorted(isbns, key=len)[-1]
mi.all_isbns = isbns
# Tags
try:
btags = [x for x in booktag(extra) if x]
tags = []
for t in btags:
atags = [y.strip() for y in t.split('/')]
for tag in atags:
if tag not in tags:
tags.append(tag)
except:
log.exception('Failed to parse tags:')
tags = []
if tags:
mi.tags = [x.replace(',', ';') for x in tags]
# pubdate
pubdate = get_text(extra, date)
if pubdate:
try:
default = utcnow().replace(day=15)
mi.pubdate = parse_date(pubdate, assume_utc=True, default=default)
except:
log.error('Failed to parse pubdate %r'%pubdate)
# Ratings
if rating(extra):
try:
mi.rating = float(rating(extra)[0]) / 2.0
except:
log.exception('Failed to parse rating')
mi.rating = 0
# Cover
mi.has_douban_cover = None
u = cover_url(extra)
if u:
u = u[0].replace('/spic/', '/lpic/')
# If URL contains "book-default", the book doesn't have a cover
if u.find('book-default') == -1:
mi.has_douban_cover = u
return mi
def to_metadata(self, log, entry_, timeout): # {{{
from calibre.utils.date import parse_date, utcnow
log.info("to_metadata")
douban_id = entry_.get("id")
title = entry_.get("title")
description = entry_.get("summary")
# subtitle = entry_.get('subtitle') # TODO: std metada doesn't have this field
publisher = entry_.get("publisher")
isbn = entry_.get("isbn13") # ISBN11 is obsolute, use ISBN13
pubdate = entry_.get("pubdate")
authors = entry_.get("author")
# authors = "author"
book_tags = entry_.get("tags")
rating = entry_.get("rating")
cover_url = entry_.get("cover")
series = entry_.get("series")
if not authors:
authors = [("Unknown")]
if not douban_id or not title:
# Silently discard this entry
return None
mi = Metadata(title, authors)
mi.identifiers = {"douban": douban_id}
mi.publisher = publisher
mi.comments = description
# mi.subtitle = subtitle
# ISBN
isbns = []
if isinstance(isbn, (type(""), bytes)):
if check_isbn(isbn):
isbns.append(isbn)
else:
for x in isbn:
if check_isbn(x):
isbns.append(x)
if isbns:
mi.isbn = sorted(isbns, key=len)[-1]
mi.all_isbns = isbns
# Tags
mi.tags = book_tags
# pubdate
if pubdate:
try:
default = utcnow().replace(day=15)
mi.pubdate = parse_date(pubdate,
assume_utc=True,
default=default)
except BaseException:
log.error("Failed to parse pubdate %r" % pubdate)
if rating:
try:
# mi.publisher += "#PrB.rating#" + str(rating)
mi.rating = rating / 2.0
except BaseException:
log.exception("Failed to parse rating")
mi.rating = 0
# Cover
mi.has_douban_cover = None
u = cover_url
if u:
# If URL contains "book-default", the book doesn't have a cover
if u.find("book-default") == -1:
mi.has_douban_cover = u
# Series
if series:
mi.series = series
return mi
def to_metadata(browser, log, entry_, timeout): # {{{
from lxml import etree
from calibre.ebooks.chardet import xml_to_unicode
from calibre.utils.date import parse_date, utcnow
from calibre.utils.cleantext import clean_ascii_chars
XPath = partial(etree.XPath, namespaces=NAMESPACES)
entry = XPath('//atom:entry')
entry_id = XPath('descendant::atom:id')
title = XPath('descendant::atom:title')
description = XPath('descendant::atom:summary')
publisher = XPath("descendant::db:attribute[@name='publisher']")
isbn = XPath("descendant::db:attribute[@name='isbn13']")
date = XPath("descendant::db:attribute[@name='pubdate']")
creator = XPath("descendant::db:attribute[@name='author']")
booktag = XPath("descendant::db:tag/attribute::name")
rating = XPath("descendant::gd:rating/attribute::average")
cover_url = XPath("descendant::atom:link[@rel='image']/attribute::href")
def get_text(extra, x):
try:
ans = x(extra)
if ans:
ans = ans[0].text
if ans and ans.strip():
return ans.strip()
except:
log.exception('Programming error:')
return None
id_url = entry_id(entry_)[0].text
douban_id = id_url.split('/')[-1]
title_ = ': '.join([x.text for x in title(entry_)]).strip()
authors = [x.text.strip() for x in creator(entry_) if x.text]
if not authors:
authors = [_('Unknown')]
if not id_url or not title:
# Silently discard this entry
return None
mi = Metadata(title_, authors)
mi.identifiers = {'douban': douban_id}
try:
raw = get_details(browser, id_url, timeout)
feed = etree.fromstring(
xml_to_unicode(clean_ascii_chars(raw),
strip_encoding_pats=True)[0])
extra = entry(feed)[0]
except:
log.exception('Failed to get additional details for', mi.title)
return mi
mi.comments = get_text(extra, description)
mi.publisher = get_text(extra, publisher)
# ISBN
isbns = []
for x in [t.text for t in isbn(extra)]:
if check_isbn(x):
isbns.append(x)
if isbns:
mi.isbn = sorted(isbns, key=len)[-1]
mi.all_isbns = isbns
# Tags
try:
btags = [x for x in booktag(extra) if x]
tags = []
for t in btags:
atags = [y.strip() for y in t.split('/')]
for tag in atags:
if tag not in tags:
tags.append(tag)
except:
log.exception('Failed to parse tags:')
tags = []
if tags:
mi.tags = [x.replace(',', ';') for x in tags]
# pubdate
pubdate = get_text(extra, date)
if pubdate:
try:
default = utcnow().replace(day=15)
mi.pubdate = parse_date(pubdate, assume_utc=True, default=default)
except:
log.error('Failed to parse pubdate %r' % pubdate)
# Ratings
if rating(extra):
try:
mi.rating = float(rating(extra)[0]) / 2.0
except:
log.exception('Failed to parse rating')
mi.rating = 0
# Cover
mi.has_douban_cover = None
u = cover_url(extra)
if u:
u = u[0].replace('/spic/', '/lpic/')
# If URL contains "book-default", the book doesn't have a cover
if u.find('book-default') == -1:
mi.has_douban_cover = u
return mi
