def _generate(self,
bottom,
feed,
art,
number_of_articles_in_feed,
two_levels,
url,
__appname__,
prefix='',
center=True,
extra_css=None,
style=None):
head = HEAD(TITLE('navbar'))
if style:
head.append(STYLE(style, type='text/css'))
if extra_css:
head.append(STYLE(extra_css, type='text/css'))
if prefix and not prefix.endswith('/'):
prefix += '/'
align = 'center' if center else 'left'
navbar = DIV(
attrs('calibre_navbar', rescale=70, style='text-align:' + align))
if bottom:
if not url.startswith('file://'):
navbar.append(HR())
text = 'This article was downloaded by '
p = PT(
text,
STRONG(__appname__),
A(url, href=url, rel='calibre-downloaded-from'),
style='text-align:left; max-width: 100%; overflow: hidden;'
)
p[0].tail = ' from '
navbar.append(p)
navbar.append(BR())
navbar.append(BR())
else:
next_art = 'feed_%d'%(feed+1) if art == number_of_articles_in_feed - 1 \
else 'article_%d'%(art+1)
up = '../..' if art == number_of_articles_in_feed - 1 else '..'
href = '%s%s/%s/index.html' % (prefix, up, next_art)
navbar.text = '| '
navbar.append(A(_('Next'), href=href))
href = '%s../index.html#article_%d' % (prefix, art)
next(navbar.iterchildren(reversed=True)).tail = ' | '
navbar.append(A(_('Section menu'), href=href))
href = '%s../../index.html#feed_%d' % (prefix, feed)
next(navbar.iterchildren(reversed=True)).tail = ' | '
navbar.append(A(_('Main menu'), href=href))
if art > 0 and not bottom:
href = '%s../article_%d/index.html' % (prefix, art - 1)
next(navbar.iterchildren(reversed=True)).tail = ' | '
navbar.append(A(_('Previous'), href=href))
next(navbar.iterchildren(reversed=True)).tail = ' | '
if not bottom:
navbar.append(HR())
self.root = HTML(head, BODY(navbar))
def convert_run(self, run):
ans = SPAN()
self.object_map[ans] = run
text = Text(ans, 'text', [])
for child in run:
if is_tag(child, 'w:t'):
if not child.text:
continue
space = child.get(XML('space'), None)
preserve = False
if space == 'preserve':
# Only use a <span> with white-space:pre-wrap if this element
# actually needs it, i.e. if it has more than one
# consecutive space or it has newlines or tabs.
multi_spaces = self.ms_pat.search(child.text) is not None
preserve = multi_spaces or self.ws_pat.search(child.text) is not None
if preserve:
text.add_elem(SPAN(child.text, style="white-space:pre-wrap"))
ans.append(text.elem)
else:
text.buf.append(child.text)
elif is_tag(child, 'w:cr'):
text.add_elem(BR())
ans.append(text.elem)
elif is_tag(child, 'w:br'):
typ = get(child, 'w:type')
if typ in {'column', 'page'}:
br = BR(style='page-break-after:always')
else:
clear = child.get('clear', None)
if clear in {'all', 'left', 'right'}:
br = BR(style='clear:%s'%('both' if clear == 'all' else clear))
else:
br = BR()
text.add_elem(br)
ans.append(text.elem)
elif is_tag(child, 'w:drawing') or is_tag(child, 'w:pict'):
for img in self.images.to_html(child, self.current_page, self.docx, self.dest_dir):
text.add_elem(img)
ans.append(text.elem)
elif is_tag(child, 'w:footnoteReference') or is_tag(child, 'w:endnoteReference'):
anchor, name = self.footnotes.get_ref(child)
if anchor and name:
l = SUP(A(name, href='#' + anchor, title=name), id='back_%s' % anchor)
l.set('class', 'noteref')
text.add_elem(l)
ans.append(text.elem)
if text.buf:
setattr(text.elem, text.attr, ''.join(text.buf))
style = self.styles.resolve_run(run)
if style.vert_align in {'superscript', 'subscript'}:
ans.tag = 'sub' if style.vert_align == 'subscript' else 'sup'
if style.lang is not inherit:
ans.lang = style.lang
return ans
def convert_run(self, run):
ans = SPAN()
self.object_map[ans] = run
text = Text(ans, 'text', [])
for child in run:
if is_tag(child, 'w:t'):
if not child.text:
continue
space = child.get(XML('space'), None)
if space == 'preserve':
text.add_elem(SPAN(child.text, style="white-space:pre-wrap"))
ans.append(text.elem)
else:
text.buf.append(child.text)
elif is_tag(child, 'w:cr'):
text.add_elem(BR())
ans.append(text.elem)
elif is_tag(child, 'w:br'):
typ = child.get('type', None)
if typ in {'column', 'page'}:
br = BR(style='page-break-after:always')
else:
clear = child.get('clear', None)
if clear in {'all', 'left', 'right'}:
br = BR(style='clear:%s'%('both' if clear == 'all' else clear))
else:
br = BR()
text.add_elem(br)
ans.append(text.elem)
elif is_tag(child, 'w:drawing') or is_tag(child, 'w:pict'):
for img in self.images.to_html(child, self.current_page, self.docx, self.dest_dir):
text.add_elem(img)
ans.append(text.elem)
elif is_tag(child, 'w:footnoteReference') or is_tag(child, 'w:endnoteReference'):
anchor, name = self.footnotes.get_ref(child)
if anchor and name:
l = SUP(A(name, href='#' + anchor, title=name), id='back_%s' % anchor)
l.set('class', 'noteref')
text.add_elem(l)
ans.append(text.elem)
elif is_tag(child, 'w:fldChar') and get(child, 'w:fldCharType') == 'separate':
text.buf.append('\xa0')
if text.buf:
setattr(text.elem, text.attr, ''.join(text.buf))
style = self.styles.resolve_run(run)
if style.vert_align in {'superscript', 'subscript'}:
ans.tag = 'sub' if style.vert_align == 'subscript' else 'sup'
if style.lang is not inherit:
ans.lang = style.lang
return ans
def _generate(self, f, feeds, cutoff, extra_css=None, style=None):
from calibre.utils.cleantext import clean_xml_chars
feed = feeds[f]
head = HEAD(TITLE(feed.title))
if style:
head.append(STYLE(style, type='text/css'))
if extra_css:
head.append(STYLE(extra_css, type='text/css'))
body = BODY()
body.append(self.get_navbar(f, feeds))
div = DIV(
H2(feed.title,
CLASS('calibre_feed_title', 'calibre_rescale_160')),
CLASS('calibre_rescale_100')
)
body.append(div)
if getattr(feed, 'image', None):
div.append(DIV(IMG(
alt=feed.image_alt if feed.image_alt else '',
src=feed.image_url
),
CLASS('calibre_feed_image')))
if getattr(feed, 'description', None):
d = DIV(clean_xml_chars(feed.description), CLASS('calibre_feed_description',
'calibre_rescale_80'))
d.append(BR())
div.append(d)
ul = UL(CLASS('calibre_article_list'))
for i, article in enumerate(feed.articles):
if not getattr(article, 'downloaded', False):
continue
li = LI(
A(article.title, CLASS('article calibre_rescale_120',
href=article.url)),
SPAN(article.formatted_date, CLASS('article_date')),
CLASS('calibre_rescale_100', id='article_%d'%i,
style='padding-bottom:0.5em')
)
if article.summary:
li.append(DIV(clean_xml_chars(cutoff(article.text_summary)),
CLASS('article_description', 'calibre_rescale_70')))
ul.append(li)
div.append(ul)
div.append(self.get_navbar(f, feeds, top=False))
self.root = HTML(head, body)
if self.html_lang:
self.root.set('lang', self.html_lang)
def _generate(self, bottom, feed, art, number_of_articles_in_feed,
two_levels, url, __appname__, prefix='', center=True,
extra_css=None, style=None):
head = HEAD(TITLE('navbar'))
if style:
head.append(STYLE(style, type='text/css'))
if extra_css:
head.append(STYLE(extra_css, type='text/css'))
navbar = DIV()
navbar_t = TABLE(CLASS('touchscreen_navbar'))
navbar_tr = TR()
if bottom and not url.startswith('file://'):
navbar.append(HR())
text = 'This article was downloaded by '
p = PT(text, STRONG(__appname__), A(url, href=url, rel='calibre-downloaded-from'),
style='text-align:left; max-width: 100%; overflow: hidden;')
p[0].tail = ' from '
navbar.append(p)
navbar.append(BR())
# | Previous
if art > 0:
link = A(CLASS('article_link'),_('Previous'),href='%s../article_%d/index.html'%(prefix, art-1))
navbar_tr.append(TD(CLASS('article_prev'),link))
else:
navbar_tr.append(TD(CLASS('article_prev'),''))
# | Articles | Sections |
link = A(CLASS('articles_link'),_('Articles'), href='%s../index.html#article_%d'%(prefix, art))
navbar_tr.append(TD(CLASS('article_articles_list'),link))
link = A(CLASS('sections_link'),_('Sections'), href='%s../../index.html#feed_%d'%(prefix, feed))
navbar_tr.append(TD(CLASS('article_sections_list'),link))
# | Next
next = 'feed_%d'%(feed+1) if art == number_of_articles_in_feed - 1 \
else 'article_%d'%(art+1)
up = '../..' if art == number_of_articles_in_feed - 1 else '..'
link = A(CLASS('article_link'), _('Next'), href='%s%s/%s/index.html'%(prefix, up, next))
navbar_tr.append(TD(CLASS('article_next'),link))
navbar_t.append(navbar_tr)
navbar.append(navbar_t)
# print "\n%s\n" % etree.tostring(navbar, pretty_print=True)
self.root = HTML(head, BODY(navbar))
def convert_run(self, run):
ans = SPAN()
self.object_map[ans] = run
text = Text(ans, 'text', [])
for child in run:
if self.namespace.is_tag(child, 'w:t'):
if not child.text:
continue
space = child.get(XML('space'), None)
preserve = False
ctext = child.text
if space != 'preserve':
# Remove leading and trailing whitespace. Word ignores
# leading and trailing whitespace without preserve
ctext = ctext.strip(' \n\r\t')
# Only use a <span> with white-space:pre-wrap if this element
# actually needs it, i.e. if it has more than one
# consecutive space or it has newlines or tabs.
multi_spaces = self.ms_pat.search(ctext) is not None
preserve = multi_spaces or self.ws_pat.search(ctext) is not None
if preserve:
text.add_elem(SPAN(ctext, style="white-space:pre-wrap"))
ans.append(text.elem)
else:
text.buf.append(ctext)
elif self.namespace.is_tag(child, 'w:cr'):
text.add_elem(BR())
ans.append(text.elem)
elif self.namespace.is_tag(child, 'w:br'):
typ = self.namespace.get(child, 'w:type')
if typ in {'column', 'page'}:
br = BR(style='page-break-after:always')
else:
clear = child.get('clear', None)
if clear in {'all', 'left', 'right'}:
br = BR(style='clear:%s'%('both' if clear == 'all' else clear))
else:
br = BR()
text.add_elem(br)
ans.append(text.elem)
elif self.namespace.is_tag(child, 'w:drawing') or self.namespace.is_tag(child, 'w:pict'):
for img in self.images.to_html(child, self.current_page, self.docx, self.dest_dir):
text.add_elem(img)
ans.append(text.elem)
elif self.namespace.is_tag(child, 'w:footnoteReference') or self.namespace.is_tag(child, 'w:endnoteReference'):
anchor, name = self.footnotes.get_ref(child)
if anchor and name:
l = A(SUP(name, id='back_%s' % anchor), href='#' + anchor, title=name)
l.set('class', 'noteref')
text.add_elem(l)
ans.append(text.elem)
elif self.namespace.is_tag(child, 'w:tab'):
spaces = int(math.ceil((self.settings.default_tab_stop / 36) * 6))
text.add_elem(SPAN(NBSP * spaces))
ans.append(text.elem)
ans[-1].set('class', 'tab')
elif self.namespace.is_tag(child, 'w:noBreakHyphen'):
text.buf.append('\u2011')
elif self.namespace.is_tag(child, 'w:softHyphen'):
text.buf.append('\u00ad')
if text.buf:
setattr(text.elem, text.attr, ''.join(text.buf))
style = self.styles.resolve_run(run)
if style.vert_align in {'superscript', 'subscript'}:
ans.tag = 'sub' if style.vert_align == 'subscript' else 'sup'
if style.lang is not inherit:
lang = html_lang(style.lang)
if lang is not None and lang != self.doc_lang:
ans.set('lang', lang)
if style.rtl is True:
ans.set('dir', 'rtl')
if is_symbol_font(style.font_family):
for elem in text:
if elem.text:
elem.text = map_symbol_text(elem.text, style.font_family)
if elem.tail:
elem.tail = map_symbol_text(elem.tail, style.font_family)
style.font_family = 'sans-serif'
return ans
def _generate(self, f, feeds, cutoff, extra_css=None, style=None):
from calibre.utils.cleantext import clean_xml_chars
def trim_title(title,clip=18):
if len(title)>clip:
tokens = title.split(' ')
new_title_tokens = []
new_title_len = 0
if len(tokens[0]) > clip:
return tokens[0][:clip] + '...'
for token in tokens:
if len(token) + new_title_len < clip:
new_title_tokens.append(token)
new_title_len += len(token)
else:
new_title_tokens.append('...')
title = ' '.join(new_title_tokens)
break
return title
self.IS_HTML = False
feed = feeds[f]
# Construct the navbar
navbar_t = TABLE(CLASS('touchscreen_navbar'))
navbar_tr = TR()
# Previous Section
link = ''
if f > 0:
link = A(CLASS('feed_link'),
trim_title(feeds[f-1].title),
href='../feed_%d/index.html' % int(f-1))
navbar_tr.append(TD(CLASS('feed_prev'),link))
# Up to Sections
link = A(_('Sections'), href="../index.html")
navbar_tr.append(TD(CLASS('feed_up'),link))
# Next Section
link = ''
if f < len(feeds)-1:
link = A(CLASS('feed_link'),
trim_title(feeds[f+1].title),
href='../feed_%d/index.html' % int(f+1))
navbar_tr.append(TD(CLASS('feed_next'),link))
navbar_t.append(navbar_tr)
top_navbar = navbar_t
bottom_navbar = copy.copy(navbar_t)
# print "\n%s\n" % etree.tostring(navbar_t, pretty_print=True)
# Build the page
head = HEAD(TITLE(feed.title))
if style:
head.append(STYLE(style, type='text/css'))
if extra_css:
head.append(STYLE(extra_css, type='text/css'))
body = BODY()
div = DIV(
top_navbar,
H2(feed.title, CLASS('feed_title'))
)
body.append(div)
if getattr(feed, 'image', None):
div.append(DIV(IMG(
alt=feed.image_alt if feed.image_alt else '',
src=feed.image_url
),
CLASS('calibre_feed_image')))
if getattr(feed, 'description', None):
d = DIV(clean_xml_chars(feed.description), CLASS('calibre_feed_description',
'calibre_rescale_80'))
d.append(BR())
div.append(d)
for i, article in enumerate(feed.articles):
if not getattr(article, 'downloaded', False):
continue
div_td = DIV(CLASS('article_summary'),
A(article.title, CLASS('summary_headline','calibre_rescale_120',
href=article.url)))
if article.author:
div_td.append(DIV(article.author,
CLASS('summary_byline', 'calibre_rescale_100')))
if article.summary:
div_td.append(DIV(cutoff(article.text_summary),
CLASS('summary_text', 'calibre_rescale_100')))
div.append(div_td)
div.append(bottom_navbar)
self.root = HTML(head, body)
if self.html_lang:
self.root.set('lang', self.html_lang)
def handle_filemaker(self, root, **options):
# Prefix to append to import IDs.
ID_PREFIX = 'inglis:'
# Load user to create/update documents as.
try:
user = User.objects.get(username=options['username'])
except User.DoesNotExist:
raise CommandError('unknown user: %s' % options['username'])
# Verify that we have the fields we expect.
f = expected_fields = [
'CardID', 'Export', 'CardHeading', 'Transcription', 'CardType',
'CardFormat', 'Language', 'ContributorJoin::Contributors',
'SubjectJoin::Subject', 'OrganizationJoin::Organizations',
'Citation', 'CatalogLink', 'ResourceLink', 'AdditionalNotes',
'DateEntered'
]
fields = root.xpath('./fmp:METADATA/fmp:FIELD/@NAME', namespaces=NS)
if not fields == expected_fields:
new_fields = [f for f in fields if not f in expected_fields]
missing_fields = [f for f in expected_fields if not f in fields]
message = 'fields have changed:\n'
message += '\n'.join(ndiff(expected_fields, fields)) + '\n\n'
if missing_fields:
message += ('Missing fields:\n ' +
'\n '.join(missing_fields))
if new_fields:
message += ('Unexpected fields:\n ' + '\n '.join(new_fields))
raise CommandError(message)
# Utility functions for accessing XML data.
def text(e):
text = e.text or ''
for child in e:
if not child.tag == '{%s}BR' % NS['fmp']:
raise CommandError('Unexpected element: %s' % child)
text += ('\n%s' % (child.tail or ''))
return text.strip()
def values(row, field):
return list(
set([v for v in [text(e) for e in row[f.index(field)]] if v]))
def value(row, field):
v = values(row, field)
if len(v) == 0:
return None
elif len(v) == 1:
return v[0]
else:
raise CommandError('multiple values for %s in record %s' %
(field, row.get('RECORDID')))
def row_to_dict(row):
d = {}
for field in f:
if 'Join::' in field:
d[field] = values(row, field)
else:
d[field] = value(row, field)
return d
# Statistics.
created_count = collections_created_count = topics_created_count = changed_count = unchanged_count = skipped_count = deleted_count = 0
for row in root.xpath('./fmp:RESULTSET/fmp:ROW', namespaces=NS):
try:
md = row_to_dict(row)
for field in [
'CardID', 'CardHeading', 'CardType', 'Transcription'
]:
if md[field] is None:
raise CommandError('missing %s value in record %s' %
(field, row.get('RECORDID')))
except CommandError as e:
self.stderr.write(self.style.ERROR('Warning: %s\n' % e))
skipped_count += 1
continue
if md['Export'] == 'No':
exists = Document.objects.filter(import_id__exact='%s%s' %
(ID_PREFIX, md['CardID']))
if exists:
exists[0].delete()
deleted_count += 1
continue
else:
skipped_count += 1
continue
collection_id = ID_PREFIX + (':%s' % md['CardHeading'])
collection_description = P('%s (Agnes Inglis cards)' %
md['CardHeading'])
collection, collection_created = Document.objects.get_or_create(
import_id=collection_id,
defaults={
'description': collection_description,
'creator': user,
'last_updater': user
})
if collection_created:
collections_created_count += 1
description = P('%s -- %s (Agnes Inglis card #%s)' %
(md['CardHeading'], md['CardType'], md['CardID']))
document, created = Document.objects.get_or_create(
import_id=(ID_PREFIX + md['CardID']),
defaults={
'description': description,
'creator': user,
'last_updater': user
})
document.description = description
document.collection = collection
document.language = md['Language']
document.save()
# Set document topics.
for topic_assignment in document.related_topics.all():
topic_assignment.delete()
def assign_topic(document, user, topic_name, topic_type=''):
topic, topic_created = Topic.objects.get_or_create(
slug=Topic.make_slug(topic_name),
defaults={
'preferred_name': topic_name,
'creator': user,
'last_updater': user
})
topic.type = topic_type
topic.save()
TopicAssignment.objects.create(content_object=document,
topic=topic,
creator=user)
if topic_created:
return 1
else:
return 0
for topic_name in md['ContributorJoin::Contributors']:
topics_created_count += assign_topic(document, user,
topic_name, 'PER')
for topic_name in md['OrganizationJoin::Organizations']:
topics_created_count += assign_topic(document, user,
topic_name, 'ORG')
for topic_name in md['SubjectJoin::Subject']:
topics_created_count += assign_topic(document, user,
topic_name)
# Set document links.
for link in document.links.all():
link.delete()
for url in [md['CatalogLink'], md['ResourceLink']]:
if url is not None:
document.links.create(url=url, creator=user)
# Set document metadata.
changed = document.set_metadata(md, user)
# Create or update document transcript.
transcript_html = P(*list(
chain.from_iterable((
(line, BR())
for line in md['Transcription'].split('\n'))))[:-1])
if created:
Transcript.objects.create(document=document,
content=transcript_html,
creator=user,
last_updater=user)
created_count += 1
elif changed or options['force_update']:
document.transcript.content = transcript_html
document.transcript.last_updater = user
document.transcript.save()
document.last_updater = user
document.save()
changed_count += 1
else:
unchanged_count += 1
self.stderr.write('%s records skipped.\n' % skipped_count)
self.stderr.write('%s records deleted.\n' % deleted_count)
self.stderr.write('%s new documents created.\n' % created_count)
self.stderr.write('%s new collections created.\n' %
collections_created_count)
self.stderr.write('%s new topics created.\n' % topics_created_count)
self.stderr.write('%s documents updated.\n' % changed_count)
self.stderr.write('%s documents unchanged.\n' % unchanged_count)
