def fetch_selected_annotations(self):
'''
Invoked by 'Import annotations' button in show_annotated_books_dialog()
Populate a list of books by Reader App:
{ 'iBooks': [{'title':, 'author':, 'uuid'}, ...],
'Marvin': [{'title':, 'author':, 'uuid'}, ...] }
'''
self.selected_books = {}
for i in range(len(self.tabledata)):
self.tv.selectRow(i)
enabled = bool(self.tm.arraydata[i][self.ENABLED_COL].checkState())
if not enabled:
continue
reader_app = str(self.tm.arraydata[i][self.READER_APP_COL].text())
if not reader_app in self.selected_books:
self.selected_books[reader_app] = []
author = str(self.tm.arraydata[i][self.AUTHOR_COL].text())
book_id = self.tm.arraydata[i][self.annotations_header.index(
'book_id')]
genre = self.tm.arraydata[i][self.annotations_header.index(
'genre')]
title = str(self.tm.arraydata[i][self.TITLE_COL].text())
uuid = self.tm.arraydata[i][self.annotations_header.index('uuid')]
book_mi = BookStruct()
book_mi.author = author
book_mi.book_id = book_id
book_mi.genre = genre
book_mi.reader_app = reader_app
book_mi.title = title
book_mi.uuid = uuid
self.selected_books[reader_app].append(book_mi)
def fetch_selected_annotations(self):
'''
Invoked by 'Import annotations' button in show_annotated_books_dialog()
Populate a list of books by Reader App:
{ 'iBooks': [{'title':, 'author':, 'uuid'}, ...],
'Marvin': [{'title':, 'author':, 'uuid'}, ...] }
'''
self.selected_books = {}
for i in range(len(self.tabledata)):
self.tv.selectRow(i)
enabled = bool(self.tm.arraydata[i][self.ENABLED_COL].checkState())
if not enabled:
continue
reader_app = str(self.tm.arraydata[i][self.annotations_header.index('Reader App')].text())
if not reader_app in self.selected_books:
self.selected_books[reader_app] = []
author = str(self.tm.arraydata[i][self.annotations_header.index('Author')].text())
book_id = self.tm.arraydata[i][self.annotations_header.index('book_id')]
genre = self.tm.arraydata[i][self.annotations_header.index('genre')]
title = str(self.tm.arraydata[i][self.annotations_header.index('Title')].text())
uuid = self.tm.arraydata[i][self.annotations_header.index('uuid')]
book_mi = BookStruct()
book_mi.author = author
book_mi.book_id = book_id
book_mi.genre = genre
book_mi.reader_app = reader_app
book_mi.title = title
book_mi.uuid = uuid
self.selected_books[reader_app].append(book_mi)
def preview_annotations(self):
"""
The listed annotations are in annotations.db.
AnnotationsDB:annotations_to_HTML() needs title, book_id, reader_app
"""
i = self.tvSelectionModel.currentIndex().row()
reader_app = str(self.tm.arraydata[i][self.annotations_header.index('Reader App')].text())
title = str(self.tm.arraydata[i][self.annotations_header.index('Title')].text())
book_mi = BookStruct()
book_mi.book_id = self.tm.arraydata[i][self.annotations_header.index('book_id')]
book_mi.reader_app = reader_app
book_mi.title = title
# Render annotations from db
annotations_db = ReaderApp.generate_annotations_db_name(reader_app, self.source)
annotations = self.get_annotations_as_HTML(annotations_db, book_mi)
PreviewDialog(book_mi, annotations, parent=self.opts.gui).exec_()
def preview_annotations(self):
"""
The listed annotations are in annotations.db.
AnnotationsDB:annotations_to_HTML() needs title, book_id, reader_app
"""
i = self.tvSelectionModel.currentIndex().row()
reader_app = str(self.tm.arraydata[i][self.READER_APP_COL].text())
title = str(self.tm.arraydata[i][self.TITLE_COL].text())
book_mi = BookStruct()
book_mi.book_id = self.tm.arraydata[i][self.annotations_header.index(
'book_id')]
book_mi.reader_app = reader_app
book_mi.title = title
# Render annotations from db
annotations_db = ReaderApp.generate_annotations_db_name(
reader_app, self.source)
annotations = self.get_annotations_as_HTML(annotations_db, book_mi)
PreviewDialog(book_mi, annotations, parent=self.opts.gui).exec_()
def get_installed_books(self):
'''
For each book, construct a BookStruct object with the book's metadata.
Starred items are minimally required.
BookStruct properties:
*active: [True|False]
*author: "John Smith"
author_sort: (if known)
*book_id: an int uniquely identifying the book.
Highlights are associated with books through book_id
genre: "Fiction" (if known)
*reader_app: self.app_name
*title: "The Story of John Smith"
title_sort: "Story of John Smith, The" (if known)
uuid: Calibre's uuid for this book, if known
'''
# Sample installed books indexed by book_id
dict_of_books = {}
dict_of_books[1] = {
'author': 'John Smith',
'author_sort': 'Smith, John',
'title': 'The Book by John Smith',
'title_sort': 'Book by John Smith, The'
}
dict_of_books[2] = {
'author': 'William Jones',
'author_sort': 'Jones, William',
'title': 'Learning Programming',
'title_sort': 'Learning Programming'
}
dict_of_books[3] = {
'author': 'Matthew Williams',
'author_sort': 'Williams, Matthew',
'title': 'A Book With No Annotations',
'title_sort': 'Book With No Annotations, A'
}
self._log("%s:get_installed_books()" % self.app_name)
self.installed_books = []
# Don't change the template of books_db string
books_db = "%s_books_%s" % (re.sub(
' ', '_', self.app_name), re.sub(' ', '_', self.opts.device_name))
installed_books = set([])
# Create the books table
self.create_books_table(books_db)
# Initialize the progress bar
self.opts.pb.set_label("Getting installed books from %s" %
self.app_name)
self.opts.pb.set_value(0)
self.opts.pb.set_maximum(len(dict_of_books))
# Add installed books to the database
for book_id in dict_of_books:
# Add book_id to list of installed_books (make this a sql function)
installed_books.add(book_id)
# Populate a BookStruct with available metadata
book_mi = BookStruct()
# Required items
book_mi.active = True
book_mi.author = dict_of_books[book_id]['author']
book_mi.book_id = book_id
book_mi.reader_app = self.app_name
book_mi.title = dict_of_books[book_id]['title']
# Optional items
if 'author_sort' in dict_of_books[book_id]:
book_mi.author_sort = dict_of_books[book_id]['author_sort']
if 'genre' in dict_of_books[book_id]:
book_mi.genre = dict_of_books[book_id]['genre']
if 'title_sort' in dict_of_books[book_id]:
book_mi.title_sort = dict_of_books[book_id]['title_sort']
if 'uuid' in dict_of_books[book_id]:
book_mi.uuid = dict_of_books[book_id]['uuid']
# Add book to books_db
self.add_to_books_db(books_db, book_mi)
# Increment the progress bar
self.opts.pb.increment()
# Update the timestamp
self.update_timestamp(books_db)
self.commit()
self.installed_books = list(installed_books)
def get_installed_books(self):
"""
Fetch installed books from mainDb.sqlite or cached_db
Populate self.tocs: {book_id: {toc_entries} ...}
"""
self._log("%s:get_installed_books()" % self.app_name)
self.opts.pb.set_label("Getting installed books from %s" %
self.app_name)
self.opts.pb.set_value(0)
db_profile = self._localize_database_path(self.app_id,
self.books_subpath)
self.books_db = db_profile['path']
cached_db = self.generate_books_db_name(self.app_name_,
self.ios.device_name)
if self.opts.disable_caching or not self._cache_is_current(
db_profile['stats'], cached_db):
# (Re)load installed books from device
self._log(" fetching installed books from %s on %s" %
(self.app_name, self.ios.device_name))
# Mount the ios container
self.ios.mount_ios_app(app_id=self.app_id)
installed_books = set([])
self.tocs = {}
# Create the books table as needed
self.create_books_table(cached_db)
con = sqlite3.connect(self.books_db)
with con:
con.row_factory = sqlite3.Row
cur = con.cursor()
cur.execute('''SELECT
Author,
AuthorSort,
Title,
CalibreTitleSort,
FileName,
Books.ID AS id_,
UUID
FROM Books
''')
rows = cur.fetchall()
self.opts.pb.set_maximum(len(rows))
for row in rows:
self.opts.pb.increment()
this_is_news = False
path = self._fix_Marvin_path(row[b'FileName'])
book_id = row[b'id_']
# Get the genre(s) for this book
genre_cur = con.cursor()
genre_cur.execute("""SELECT
Subject
FROM BookSubjects
WHERE BookID = '{0}'
""".format(book_id))
genres = None
genre_rows = genre_cur.fetchall()
if genre_rows is not None:
genres = ', '.join(
[genre[b'Subject'] for genre in genre_rows])
genre_cur.close()
if 'News' in genres:
if not self.collect_news_clippings:
continue
this_is_news = True
installed_books.add(book_id)
# Populate a BookStruct
b_mi = BookStruct()
b_mi.active = True
b_mi.author = row[b'Author']
b_mi.author_sort = row[b'AuthorSort']
b_mi.book_id = book_id
b_mi.genre = genres
b_mi.title = row[b'Title']
b_mi.title_sort = row[b'CalibreTitleSort']
b_mi.uuid = row[b'UUID']
# Add book to books_db
self.add_to_books_db(cached_db, b_mi)
# Get the library cid, confidence
toc_entries = None
if this_is_news:
cid = self.news_clippings_cid
confidence = 5
if path is not None:
toc_entries = self._get_epub_toc(
path=path, prepend_title=b_mi.title)
else:
cid, confidence = self.parent.generate_confidence(b_mi)
if confidence >= 2:
toc_entries = self._get_epub_toc(cid=cid,
path=path)
elif path is not None:
toc_entries = self._get_epub_toc(path=path)
self.tocs[book_id] = toc_entries
# Update the timestamp
self.update_timestamp(cached_db)
self.commit()
self.ios.disconnect_idevice()
installed_books = list(installed_books)
else:
# Load installed books from cache
self._log(" retrieving cached books from %s" % cached_db)
self.opts.pb.set_maximum(2)
self.opts.pb.set_value(1)
Application.processEvents()
installed_books = self._get_cached_books(cached_db)
self.installed_books = installed_books
def get_installed_books(self):
'''
For each book, construct a BookStruct object with the book's metadata.
Starred items are minimally required.
BookStruct properties:
*active: [True|False]
*author: "John Smith"
author_sort: (if known)
*book_id: an int uniquely identifying the book.
Highlights are associated with books through book_id
genre: "Fiction" (if known)
*reader_app: self.app_name
*title: "The Story of John Smith"
title_sort: "Story of John Smith, The" (if known)
uuid: Calibre's uuid for this book, if known
'''
self._log("%s:get_installed_books()" % self.app_name)
self.installed_books = []
self.device = self.opts.gui.device_manager.device
path_map = self.get_path_map()
# Get books added to Kindle by calibre
resolved_path_map = self._get_installed_books(path_map)
# Add books added to Kindle by WhisperNet or download
resolved_path_map = self._get_imported_books(resolved_path_map)
self.books_db = self.generate_books_db_name(self.app_name_, self.opts.device_name)
installed_books = set([])
# Used by get_active_annotations() to look up metadata based on title
self.installed_books_by_title = {}
# Create the books table
self.create_books_table(self.books_db)
# Initialize the progress bar
self.opts.pb.set_label("Getting installed books from %s" % self.app_name)
self.opts.pb.set_value(0)
self.opts.pb.show()
self.opts.pb.set_maximum(len(resolved_path_map))
# Add installed books to the database
for book_id in resolved_path_map:
try:
mi = self._get_metadata(resolved_path_map[book_id])
except Exception as e:
self._log("Unable to get metadata from book. path='%s'" % (resolved_path_map[book_id]))
self._log(" Exception thrown was=%s" % (str(e)))
continue
self._log("Book on device title: '%s'" % (mi.title))
if 'News' in mi.tags:
if not self.collect_news_clippings:
continue
installed_books.add(self.news_clippings_cid)
else:
installed_books.add(book_id)
#self._log(mi.standard_field_keys())
# Populate a BookStruct with available metadata
book_mi = BookStruct()
book_mi.path = resolved_path_map[book_id]
# Required items
book_mi.active = True
# Massage last, first authors back to normalcy
book_mi.author = ''
for i, author in enumerate(mi.authors):
this_author = author.split(', ')
this_author.reverse()
book_mi.author += ' '.join(this_author)
if i < len(mi.authors) - 1:
book_mi.author += ' & '
book_mi.book_id = book_id
book_mi.reader_app = self.app_name
book_mi.title = mi.title.strip()
# Optional items
if mi.tags:
book_mi.genre = ', '.join([tag for tag in mi.tags])
if 'News' in mi.tags:
book_mi.book_id = self.news_clippings_cid
if hasattr(mi, 'author_sort'):
book_mi.author_sort = mi.author_sort
if hasattr(mi, 'title_sort'):
book_mi.title_sort = mi.title_sort
else:
book_mi.title_sort = re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', mi.title).rstrip()
if hasattr(mi, 'uuid'):
book_mi.uuid = mi.uuid
# Add book to self.books_db
self.add_to_books_db(self.books_db, book_mi)
# Add book to indexed_books
self.installed_books_by_title[mi.title.strip()] = {'book_id': book_id, 'author_sort': mi.author_sort}
# Increment the progress bar
self.opts.pb.increment()
self.opts.pb.hide()
# Update the timestamp
self.update_timestamp(self.books_db)
self.commit()
self.installed_books = list(installed_books)
def parse_exported_highlights(self, raw):
"""
Extract highlights from pasted Annotations summary, add them to selected book
in calibre library
Construct a BookStruct object with the book's metadata.
Starred items are minimally required.
BookStruct properties:
*active: [True|False]
*author: "John Smith"
author_sort: (if known)
*book_id: an int uniquely identifying the book.
Highlights are associated with books through book_id
genre: "Fiction" (if known)
*title: "The Story of John Smith"
title_sort: "Story of John Smith, The" (if known)
uuid: Calibre's uuid for this book, if known
Construct an AnnotationStruct object with the
highlight's metadata. Starred items are minimally required. Dashed items
(highlight_text and note_text) may be one or both.
AnnotationStruct properties:
annotation_id: an int uniquely identifying the annotation
*book_id: The book this annotation is associated with
highlight_color: [Blue|Gray|Green|Pink|Purple|Underline|Yellow]
-highlight_text: A list of paragraphs constituting the highlight
last_modification: The timestamp of the annotation
location: location of highlight in the book
-note_text: A list of paragraphs constituting the note
*timestamp: Unique timestamp of highlight's creation/modification time
"""
self._log("%s:parse_exported_highlight()" % self.app_name)
# Create the annotations, books table as needed
self.annotations_db = "%s_imported_annotations" % self.app_name_
self.create_annotations_table(self.annotations_db)
self.books_db = "%s_imported_books" % self.app_name_
self.create_books_table(self.books_db)
self.annotated_book_list = []
self.selected_books = None
# Generate the book metadata from the selected book
row = self.opts.gui.library_view.currentIndex()
book_id = self.opts.gui.library_view.model().id(row)
db = self.opts.gui.current_db
mi = db.get_metadata(book_id, index_is_id=True)
# Populate author, title at a minimum
title = "A Book With Some Exported Annotations"
author = "John Smith"
# Populate a BookStruct
book_mi = BookStruct()
book_mi.active = True
book_mi.author = author
book_mi.book_id = mi.id
book_mi.title = title
book_mi.uuid = None
book_mi.last_update = time.mktime(time.localtime())
book_mi.reader_app = self.app_name
book_mi.cid = mi.id
book_mi.annotations = len(self.highlights)
# Add annotations to the database
for timestamp in sorted(self.highlights.keys()):
book_mi.last_update = timestamp
# Populate an AnnotationStruct
ann_mi = AnnotationStruct()
# Required items
ann_mi.book_id = book_mi['book_id']
ann_mi.last_modification = timestamp
# Optional items
if 'annotation_id' in self.highlights[timestamp]:
ann_mi.annotation_id = self.highlights[timestamp]['annotation_id']
if 'highlight_color' in self.highlights[timestamp]:
ann_mi.highlight_color = self.highlights[timestamp]['highlight_color']
if 'highlight_text' in self.highlights[timestamp]:
highlight_text = '\n'.join(self.highlights[timestamp]['highlight_text'])
ann_mi.highlight_text = highlight_text
if 'note_text' in self.highlights[timestamp]:
note_text = '\n'.join(self.highlights[timestamp]['note_text'])
ann_mi.note_text = note_text
# Add annotation to annotations_db
self.add_to_annotations_db(self.annotations_db, ann_mi)
# Increment the progress bar
self.opts.pb.increment()
# Update last_annotation in books_db
self.update_book_last_annotation(self.books_db, timestamp, ann_mi.book_id)
# Add book to books_db
self.add_to_books_db(self.books_db, book_mi)
self.annotated_book_list.append(book_mi)
# Update the timestamp
self.update_timestamp(self.annotations_db)
self.update_timestamp(self.books_db)
self.commit()
# Return True if successful
return True
def get_installed_books(self):
'''
For each book, construct a BookStruct object with the book's metadata.
Starred items are minimally required.
BookStruct properties:
*active: [True|False]
*author: "John Smith"
author_sort: (if known)
*book_id: an int uniquely identifying the book.
Highlights are associated with books through book_id
genre: "Fiction" (if known)
*reader_app: self.app_name
*title: "The Story of John Smith"
title_sort: "Story of John Smith, The" (if known)
uuid: Calibre's uuid for this book, if known
'''
self._log_location("Start!!!!")
self._log("%s:get_installed_books()" % self.app_name)
self.installed_books = []
self.device = self.opts.gui.device_manager.device
# Calibre already knows what books are on the device, so use it.
db = self.opts.gui.library_view.model().db
self.onDeviceIds = set(db.search_getting_ids('ondevice:True', None, sort_results=False, use_virtual_library=False))
self._log("%s:get_installed_books() - self.onDeviceIds=" % self.onDeviceIds)
self._log("%s:get_installed_books() - about to call self.generate_books_db_name" % self.app_name)
self.books_db = self.generate_books_db_name(self.app_name_, self.opts.device_name)
installed_books = set([])
# Used by get_active_annotations() to look up metadata based on title
self.installed_books_by_title = {}
# Create the books table
self.create_books_table(self.books_db)
# Initialize the progress bar
self.opts.pb.set_label("Getting installed books from %s" % self.app_name)
self.opts.pb.set_value(0)
self.opts.pb.set_maximum(len(self.onDeviceIds))
self._log("Number of books on the device=%d" % len(self.onDeviceIds))
# Add installed books to the database
for book_id in self.onDeviceIds:
mi = db.get_metadata(book_id, index_is_id=True)
# self._log_location("book: {0} - {1}".format(mi.authors, mi.title))
# self._log("mi={0}".format(mi))
installed_books.add(book_id)
# Populate a BookStruct with available metadata
book_mi = BookStruct()
# book_mi.path = resolved_path_map[book_id] # Add book_id to list of installed_books (make this a sql function)
installed_books.add(book_id)
# Populate a BookStruct with available metadata
book_mi = BookStruct()
# Required items
book_mi.active = True
# Massage last, first authors back to normalcy
book_mi.author = ''
for i, author in enumerate(mi.authors):
# self._log_location("author=%s, author.__class__=%s" % (author, author.__class__))
this_author = author.split(', ')
this_author.reverse()
book_mi.author += ' '.join(this_author)
if i < len(mi.authors) - 1:
book_mi.author += ' & '
book_mi.book_id = book_id
book_mi.reader_app = self.app_name
book_mi.title = mi.title
if hasattr(mi, 'author_sort'):
book_mi.author_sort = mi.author_sort
if hasattr(mi, 'title_sort'):
book_mi.title_sort = mi.title_sort
else:
book_mi.title_sort = re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', mi.title).rstrip()
if hasattr(mi, 'uuid'):
book_mi.uuid = mi.uuid
# Add book to self.books_db
self.add_to_books_db(self.books_db, book_mi)
# Add book to indexed_books
self.installed_books_by_title[mi.title] = {'book_id': book_id, 'author_sort': mi.author_sort}
# Increment the progress bar
self.opts.pb.increment()
# Update the timestamp
self.update_timestamp(self.books_db)
self.commit()
self.installed_books = list(installed_books)
self._log_location("Finish!!!!")
def parse_exported_highlights(self, raw, log_failure=True):
"""
Extract highlights from pasted Annotation summary email
Return True if no problems
Return False if error
"""
# Create the annotations, books table as needed
self.annotations_db = "%s_imported_annotations" % self.app_name_
self.create_annotations_table(self.annotations_db)
self.books_db = "%s_imported_books" % self.app_name_
self.create_books_table(self.books_db)
self.annotated_book_list = []
self.selected_books = None
self._log("raw highlights: {0}".format(raw))
# Generate the book metadata from the selected book
row = self.opts.gui.library_view.currentIndex()
book_id = self.opts.gui.library_view.model().id(row)
db = self.opts.gui.current_db
mi = db.get_metadata(book_id, index_is_id=True)
# Grab the title from the front of raw
try:
title = re.match(r'(?m)File: (?P<title>.*)$', raw).group('title')
self._log("title='{0}".format(title))
# Populate a BookStruct
book_mi = BookStruct()
book_mi.active = True
book_mi.author = 'Unknown'
book_mi.book_id = mi.id
book_mi.title = title
book_mi.uuid = None
book_mi.last_update = time.mktime(time.localtime())
book_mi.reader_app = self.app_name
book_mi.cid = mi.id
gr_annotations = raw.split('\n')
num_lines = len(gr_annotations)
highlights = {}
# Find the first annotation
i = 0
line = gr_annotations[i]
self._log("Looking for Page: Line number={0} line='{1}'".format(
i, line))
while not line.startswith('--- Page'):
self._log(" unable to parse GoodReader Annotation summary")
i += 1
line = gr_annotations[i]
self._log(
"Looking for Page: Line number={0} line='{1}'".format(
i, line))
while i < num_lines and not line.startswith(
'(report generated by GoodReader)'):
# Extract the page number
page_num = re.search('--- (Page \w+) ---', line)
self._log("regex result: page_num={0}".format(page_num))
if page_num:
page_num = page_num.group(1)
self._log("page_num={0}".format(page_num))
# Extract the highlight
i += 1
line = gr_annotations[i]
self._log(
"Looking for annotation start: Line number={0} line='{1}'"
.format(i, line))
prefix = None
while True:
prefix = re.search(
'^(?P<ann_type>{0})'.format(
'|'.join(self.ANNOTATION_TYPES +
self.SKIP_TYPES)), line)
self._log("Searched for prefix={0}".format(prefix))
if prefix and prefix.group(
'ann_type') in self.SKIP_TYPES:
i += 1
line = gr_annotations[i]
self._log(
"Looking for annotation start: Line number={0} line='{1}'"
.format(i, line))
while not re.search(
'^(?P<ann_type>{0})'.format('|'.join(
self.ANNOTATION_TYPES)), line):
i += 1
line = gr_annotations[i]
self._log(
"Looking for annotation start after a SKIP type: Line number={0} line='{1}'"
.format(i, line))
continue
elif prefix:
self._log(
"Have annotation start: Line number={0} line='{1}' prefix={2}"
.format(i, line, prefix))
break
else:
i += 1
line = gr_annotations[i]
self._log(
"Looking for annotation start 2: Line number={0} line='{1}'"
.format(i, line))
annotation = self._extract_highlight(
line, prefix.group('ann_type'))
annotation.page_num = page_num
self._log(
"Started annotation: page_num={0} annotation='{1}'".
format(page_num, annotation))
# Get the annotation(s)
i += 1
line = gr_annotations[i]
self._log(
"Reading annotation text 1: Line number={0} line='{1}'"
.format(i, line))
ann = ''
while i < num_lines \
and not line.startswith('--- Page') \
and not line.startswith('(report generated by GoodReader)'):
if line:
prefix = re.search(
'^(?P<ann_type>{0})'.format(
'|'.join(self.ANNOTATION_TYPES +
self.SKIP_TYPES)), line)
if prefix and prefix.group(
'ann_type') in self.SKIP_TYPES:
# Continue until next ann_type
i += 1
line = gr_annotations[i]
while not re.search(
'^(?P<ann_type>{0})'.format('|'.join(
self.ANNOTATION_TYPES)), line):
i += 1
if i == num_lines:
break
line = gr_annotations[i]
continue
elif prefix:
# Additional highlight on the same page
# write current annotation, start new annotation
self._store_annotation(highlights, annotation)
annotation = self._extract_highlight(
line, prefix.group('ann_type'))
annotation.page_num = page_num
annotation.ann_type = prefix.group('ann_type')
ann = ''
i += 1
line = gr_annotations[i]
continue
if not ann:
ann = line
else:
ann += '\n' + line
i += 1
line = gr_annotations[i]
annotation.ann = ann
# Back up so that the next line is '--- Page' or '(report generated'
i -= 1
self._store_annotation(highlights, annotation)
i += 1
if i == num_lines:
break
line = gr_annotations[i]
except Exception as e:
import traceback
self._log("Exception parsing GoodReader Annotation summary: %s" %
e)
traceback.print_exc()
if log_failure:
self._log(" unable to parse GoodReader Annotation summary")
self._log("{:~^80}".format(" Imported Annotation summary "))
self._log(raw)
self._log(
"{:~^80}".format(" end imported Annotations summary "))
import traceback
traceback.print_exc()
msg = ('Unable to parse Annotation summary from %s. ' %
self.app_name +
'Paste entire contents of emailed summary.')
MessageBox(MessageBox.WARNING,
'Error importing annotations',
msg,
show_copy_button=False,
parent=self.opts.gui).exec_()
self._log_location("WARNING: %s" % msg)
return False
# Finalize book_mi
book_mi.annotations = len(highlights)
# Add book to books_db
self.add_to_books_db(self.books_db, book_mi)
self.annotated_book_list.append(book_mi)
sorted_keys = sorted(list(highlights.keys()))
for dt in sorted_keys:
highlight_text = None
if 'text' in highlights[dt]:
highlight_text = highlights[dt]['text']
note_text = None
if 'note' in highlights[dt]:
note_text = highlights[dt]['note']
# Populate an AnnotationStruct
a_mi = AnnotationStruct()
a_mi.annotation_id = dt
a_mi.book_id = book_mi['book_id']
a_mi.highlight_color = highlights[dt]['color']
a_mi.highlight_text = highlight_text
a_mi.location = highlights[dt]['page']
a_mi.last_modification = dt
a_mi.note_text = note_text
# Location sort
page_literal = re.match(r'^Page (?P<page>[0-9ivx]+).*$',
a_mi.location).group('page')
if re.match('[IXVL]', page_literal.upper()):
whole = 0
decimal = self._roman_to_int(page_literal)
else:
whole = int(page_literal)
decimal = 0
a_mi.location_sort = "%05d.%05d" % (whole, decimal)
# Add annotation
self.add_to_annotations_db(self.annotations_db, a_mi)
self.update_book_last_annotation(self.books_db, dt,
book_mi['book_id'])
# Update the timestamp
self.update_timestamp(self.annotations_db)
self.update_timestamp(self.books_db)
self.commit()
return True
def get_installed_books(self):
'''
For each book, construct a BookStruct object with the book's metadata.
Starred items are minimally required.
BookStruct properties:
*active: [True|False]
*author: "John Smith"
author_sort: (if known)
*book_id: an int uniquely identifying the book.
Highlights are associated with books through book_id
genre: "Fiction" (if known)
*reader_app: self.app_name
*title: "The Story of John Smith"
title_sort: "Story of John Smith, The" (if known)
uuid: Calibre's uuid for this book, if known
'''
self._log("%s:get_installed_books()" % self.app_name)
self.opts.pb.set_label("Getting installed books from %s" %
self.app_name)
self.opts.pb.set_value(0)
db_profile = self._localize_database_path(self.app_id,
self.books_subpath)
self.book_db = db_profile['path']
cached_db = self.generate_books_db_name(self.app_name_,
self.ios.device_name)
if self.opts.disable_caching or not self._cache_is_current(
db_profile['stats'], cached_db):
self._log(" fetching installed books from %s on %s" %
(self.app_name, self.ios.device_name))
# Mount the ios container
self.ios.mount_ios_app(app_id=self.app_id)
installed_books = set([])
self.tocs = {}
# Create the books table as needed
self.create_books_table(cached_db)
con = sqlite3.connect(self.books_db)
with con:
con.row_factory = sqlite3.Row
cur = con.cursor()
cur.execute('''SELECT DISTINCT
author,
oid,
subject,
source,
title,
uuidstr
FROM book
JOIN book_subjects ON book.oid = book_subjects.book_oid
''')
rows = cur.fetchall()
self.opts.pb.set_maximum(len(rows))
for row in rows:
self.opts.pb.increment()
book_id = row[b'oid']
installed_books.add(book_id)
path = self._get_stanza_path(row[b'title'])
# Populate a BookStruct
b_mi = BookStruct()
b_mi.active = True
b_mi.author = row[b'author']
b_mi.book_id = book_id
b_mi.genre = row[b'subject']
b_mi.path = path
b_mi.title = row[b'title']
b_mi.uuid = self._get_uuid(row[b'uuidstr'])
# Add book to books_db
self.add_to_books_db(cached_db, b_mi)
# Get the library cid, confidence
cid, confidence = self.parent.generate_confidence(b_mi)
toc_entries = None
if confidence >= 2:
toc_entries = self._get_epub_toc(cid=cid, path=path)
elif path is not None:
toc_entries = self._get_epub_toc(path=path)
self.tocs[book_id] = toc_entries
# Update the timestamp
self.update_timestamp(cached_db)
self.commit()
installed_books = list(installed_books)
else:
# Load installed books from cache
self._log(" retrieving cached books from %s" % cached_db)
installed_books = self._get_cached_books(cached_db)
self.installed_books = installed_books
def get_installed_books(self):
"""
Fetch installed books from mainDb.sqlite or cached_db
Populate self.tocs: {book_id: {toc_entries} ...}
"""
self._log("%s:get_installed_books()" % self.app_name)
self.opts.pb.set_label("Getting installed books from %s" % self.app_name)
self.opts.pb.set_value(0)
db_profile = self._localize_database_path(self.app_id, self.books_subpath)
self.books_db = db_profile['path']
cached_db = self.generate_books_db_name(self.app_name_, self.ios.device_name)
if self.opts.disable_caching or not self._cache_is_current(db_profile['stats'], cached_db):
# (Re)load installed books from device
self._log(" fetching installed books from %s on %s" % (self.app_name, self.ios.device_name))
# Mount the ios container
self.ios.mount_ios_app(app_id=self.app_id)
installed_books = set([])
self.tocs = {}
# Create the books table as needed
self.create_books_table(cached_db)
con = sqlite3.connect(self.books_db)
with con:
con.row_factory = sqlite3.Row
cur = con.cursor()
cur.execute('''SELECT
Author,
AuthorSort,
Title,
CalibreTitleSort,
FileName,
Books.ID AS id_,
UUID
FROM Books
''')
rows = cur.fetchall()
self.opts.pb.set_maximum(len(rows))
for row in rows:
self.opts.pb.increment()
this_is_news = False
path = self._fix_Marvin_path(row[b'FileName'])
book_id = row[b'id_']
# Get the genre(s) for this book
genre_cur = con.cursor()
genre_cur.execute("""SELECT
Subject
FROM BookSubjects
WHERE BookID = '{0}'
""".format(book_id))
genres = None
genre_rows = genre_cur.fetchall()
if genre_rows is not None:
genres = ', '.join([genre[b'Subject'] for genre in genre_rows])
genre_cur.close()
if 'News' in genres:
if not self.collect_news_clippings:
continue
this_is_news = True
installed_books.add(book_id)
# Populate a BookStruct
b_mi = BookStruct()
b_mi.active = True
b_mi.author = row[b'Author']
b_mi.author_sort = row[b'AuthorSort']
b_mi.book_id = book_id
b_mi.genre = genres
b_mi.title = row[b'Title']
b_mi.title_sort = row[b'CalibreTitleSort']
b_mi.uuid = row[b'UUID']
# Add book to books_db
self.add_to_books_db(cached_db, b_mi)
# Get the library cid, confidence
toc_entries = None
if this_is_news:
cid = self.news_clippings_cid
confidence = 5
if path is not None:
toc_entries = self._get_epub_toc(path=path, prepend_title=b_mi.title)
else:
cid, confidence = self.parent.generate_confidence(b_mi)
if confidence >= 2:
toc_entries = self._get_epub_toc(cid=cid, path=path)
elif path is not None:
toc_entries = self._get_epub_toc(path=path)
self.tocs[book_id] = toc_entries
# Update the timestamp
self.update_timestamp(cached_db)
self.commit()
self.ios.disconnect_idevice()
installed_books = list(installed_books)
else:
# Load installed books from cache
self._log(" retrieving cached books from %s" % cached_db)
self.opts.pb.set_maximum(2)
self.opts.pb.set_value(1)
Application.processEvents()
installed_books = self._get_cached_books(cached_db)
self.installed_books = installed_books
def get_installed_books(self):
"""
Fetch installed books from iBooks_*.sqlite or cache
"""
self._log("%s:get_installed_books()" % self.app_name)
self.opts.pb.set_label("Getting installed books from %s" % self.app_name)
self.opts.pb.set_value(0)
db_profile = self._localize_database_path(self.app_id, self.books_subpath)
self.books_db = db_profile['path']
cached_db = self.generate_books_db_name(self.app_name_, self.ios.device_name)
# Test timestamp against cached value
if self.opts.disable_caching or not self._cache_is_current(db_profile['stats'], cached_db):
# (Re)load installed books from device
self._log(" fetching installed books from %s on %s" % (self.app_name, self.ios.device_name))
# Mount the Media folder
self.ios.mount_ios_media_folder()
installed_books = set([])
self.tocs = {}
# Create the books table as needed
self.create_books_table(cached_db)
con = sqlite3.connect(self.books_db)
with con:
con.row_factory = sqlite3.Row
cur = con.cursor()
cur.execute('''SELECT ZASSETURL,
ZBOOKAUTHOR,
ZSORTAUTHOR,
ZBOOKTITLE,
ZSORTTITLE,
ZDATABASEKEY
FROM ZBKBOOKINFO
WHERE ZASSETURL LIKE 'file://localhost%' AND
ZASSETURL LIKE '%.epub/'
''')
rows = cur.fetchall()
self.opts.pb.set_maximum(len(rows))
for row in rows:
self.opts.pb.increment()
this_is_news = False
path = self._fix_iBooks_path(row[b'ZASSETURL'])
mi = self._get_metadata(path, row[b'ZBOOKTITLE'])
genres = mi['genre'].split(', ')
if 'News' in genres:
if not self.collect_news_clippings:
continue
this_is_news = True
book_id = row[b'ZDATABASEKEY']
installed_books.add(book_id)
# Populate a BookStruct
b_mi = BookStruct()
b_mi.active = True
b_mi.author = row[b'ZBOOKAUTHOR']
b_mi.author_sort = row[b'ZSORTAUTHOR']
b_mi.book_id = book_id
b_mi.genre = mi['genre']
b_mi.title = row[b'ZBOOKTITLE']
b_mi.title_sort = row[b'ZSORTTITLE']
b_mi.uuid = mi['uuid']
# Add book to books_db
self.add_to_books_db(cached_db, b_mi)
# Get the library cid, confidence
toc_entries = None
if this_is_news:
cid = self.news_clippings_cid
confidence = 5
if path is not None:
toc_entries = self._get_epub_toc(path=path, prepend_title=b_mi.title)
elif self.ios.exists(path):
cid, confidence = self.parent.generate_confidence(b_mi)
if confidence >= 2:
toc_entries = self._get_epub_toc(cid=cid, path=path)
elif path is not None:
toc_entries = self._get_epub_toc(path=path)
self.tocs[book_id] = toc_entries
# Update the timestamp
self.update_timestamp(cached_db)
self.commit()
self.ios.dismount_ios_media_folder()
installed_books = list(installed_books)
else:
self._log(" retrieving cached books from %s" % cached_db)
self.opts.pb.set_maximum(2)
self.opts.pb.set_value(1)
Application.processEvents()
installed_books = self._get_cached_books(cached_db)
self.installed_books = installed_books
def parse_exported_highlights(self, raw, log_failure=True):
"""
Extract highlights from pasted Annotations summary, add them to selected book
in calibre library
Construct a BookStruct object with the book's metadata.
Starred items are minimally required.
BookStruct properties:
*active: [True|False]
*author: "John Smith"
author_sort: (if known)
*book_id: an int uniquely identifying the book.
Highlights are associated with books through book_id
genre: "Fiction" (if known)
*title: "The Story of John Smith"
title_sort: "Story of John Smith, The" (if known)
uuid: Calibre's uuid for this book, if known
Construct an AnnotationStruct object with the
highlight's metadata. Starred items are minimally required. Dashed items
(highlight_text and note_text) may be one or both.
AnnotationStruct properties:
annotation_id: an int uniquely identifying the annotation
*book_id: The book this annotation is associated with
highlight_color: [Blue|Gray|Green|Pink|Purple|Underline|Yellow]
-highlight_text: A list of paragraphs constituting the highlight
last_modification: The timestamp of the annotation
location: location of highlight in the book
-note_text: A list of paragraphs constituting the note
*timestamp: Unique timestamp of highlight's creation/modification time
"""
# Create the annotations, books table as needed
self.annotations_db = "%s_imported_annotations" % self.app_name_
self.create_annotations_table(self.annotations_db)
self.books_db = "%s_imported_books" % self.app_name_
self.create_books_table(self.books_db)
self.annotated_book_list = []
self.selected_books = None
# Generate the book metadata from the selected book
row = self.opts.gui.library_view.currentIndex()
book_id = self.opts.gui.library_view.model().id(row)
db = self.opts.gui.current_db
mi = db.get_metadata(book_id, index_is_id=True)
try:
lines = raw.split('\n')
if len(lines) < 5:
raise AnnotationsException("Invalid annotations summary")
index = 0
annotations = {}
# Get the title, author, publisher from the first three lines
title = lines[index]
index += 1
author = lines[index]
index += 1
publisher = lines[index]
index += 1
# Next line should be the first timestamp/location
while index < len(lines):
tsl = re.match(r'^(?P<timestamp>.*) \((?P<location>Page .*)\)', lines[index])
if tsl:
ts = tsl.group('timestamp')
isoformat = parse_date(ts, as_utc=False)
isoformat = isoformat.replace(hour=12)
timestamp = mktime(isoformat.timetuple())
while timestamp in annotations:
timestamp += 60
location = tsl.group('location')
index += 1
# Continue with highlight
highlight_text = lines[index]
index += 1
# Next line is either Note: or a new tsl
note = re.match(r'^Notes: (?P<note_text>.*)', lines[index])
note_text = None
if note:
note_text = note.group('note_text')
index += 1
if re.match(r'^(?P<timestamp>.*) \((?P<location>Page .*)\)', lines[index]):
# New note - store the old one, continue
ann = AnnotationStruct()
ann.book_id = mi.id
ann.annotation_id = index
ann.highlight_color = 'Yellow'
ann.highlight_text = highlight_text
ann.location = location
ann.location_sort = "%05d" % int(re.match(r'^Page (?P<page>\d+).*$', location).group('page'))
ann.note_text = note_text
ann.last_modification = timestamp
# Add annotation to db
annotations[timestamp] = ann
continue
else:
# Store the last one
ann = AnnotationStruct()
ann.book_id = mi.id
ann.annotation_id = index
ann.highlight_color = 'Yellow'
ann.highlight_text = highlight_text
ann.location = location
ann.location_sort = "%05d" % int(re.match(r'^Page (?P<page>\d+).*$', location).group('page'))
ann.note_text = note_text
ann.last_modification = timestamp
annotations[timestamp] = ann
break
except:
if log_failure:
self._log(" unable to parse %s Annotations" % self.app_name)
self._log("{:~^80}".format(" Imported Annotation summary "))
self._log(raw)
self._log("{:~^80}".format(" end imported Annotations summary "))
import traceback
traceback.print_exc()
msg = ('Unable to parse Annotation summary from %s. ' % self.app_name +
'Paste entire contents of emailed summary.')
MessageBox(MessageBox.WARNING,
'Error importing annotations',
msg,
show_copy_button=False,
parent=self.opts.gui).exec_()
self._log_location("WARNING: %s" % msg)
return False
# Populate a BookStruct
book_mi = BookStruct()
book_mi.active = True
book_mi.author = author
book_mi.book_id = mi.id
book_mi.title = title
book_mi.uuid = None
book_mi.last_update = time.mktime(time.localtime())
book_mi.reader_app = self.app_name
book_mi.cid = mi.id
book_mi.annotations = len(annotations)
# Add book to books_db
self.add_to_books_db(self.books_db, book_mi)
self.annotated_book_list.append(book_mi)
# Add the annotations
for timestamp in sorted(annotations.keys()):
self.add_to_annotations_db(self.annotations_db, annotations[timestamp])
self.update_book_last_annotation(self.books_db, timestamp, mi.id)
self.opts.pb.increment()
self.update_book_last_annotation(self.books_db, timestamp, mi.id)
# Update the timestamp
self.update_timestamp(self.annotations_db)
self.update_timestamp(self.books_db)
self.commit()
# Return True if successful
return True
def parse_exported_highlights(self, raw, log_failure=True):
"""
Extract highlights from pasted Annotation summary email
Return True if no problems
Return False if error
"""
# Create the annotations, books table as needed
self.annotations_db = "%s_imported_annotations" % self.app_name_
self.create_annotations_table(self.annotations_db)
self.books_db = "%s_imported_books" % self.app_name_
self.create_books_table(self.books_db)
self.annotated_book_list = []
self.selected_books = None
# Generate the book metadata from the selected book
row = self.opts.gui.library_view.currentIndex()
book_id = self.opts.gui.library_view.model().id(row)
db = self.opts.gui.current_db
mi = db.get_metadata(book_id, index_is_id=True)
# Grab the title from the front of raw
try:
title = re.match(r'(?m)File: (?P<title>.*)$', raw).group('title')
# Populate a BookStruct
book_mi = BookStruct()
book_mi.active = True
book_mi.author = 'Unknown'
book_mi.book_id = mi.id
book_mi.title = title
book_mi.uuid = None
book_mi.last_update = time.mktime(time.localtime())
book_mi.reader_app = self.app_name
book_mi.cid = mi.id
gr_annotations = raw.split('\n')
num_lines = len(gr_annotations)
highlights = {}
# Find the first annotation
i = 0
line = gr_annotations[i]
while not line.startswith('--- Page'):
i += 1
line = gr_annotations[i]
while i < num_lines and not line.startswith('(report generated by GoodReader)'):
# Extract the page number
page_num = re.search('--- (Page \w+) ---', line)
if page_num:
page_num = page_num.group(1)
# Extract the highlight
i += 1
line = gr_annotations[i]
prefix = None
while True:
prefix = re.search('^(?P<ann_type>{0})'.format('|'.join(self.ANNOTATION_TYPES + self.SKIP_TYPES)), line)
if prefix and prefix.group('ann_type') in self.SKIP_TYPES:
i += 1
line = gr_annotations[i]
while not re.search('^(?P<ann_type>{0})'.format('|'.join(self.ANNOTATION_TYPES)), line):
i += 1
line = gr_annotations[i]
continue
elif prefix:
break
else:
i += 1
line = gr_annotations[i]
annotation = self._extract_highlight(line, prefix.group('ann_type'))
annotation.page_num = page_num
# Get the annotation(s)
i += 1
line = gr_annotations[i]
ann = ''
while i < num_lines \
and not line.startswith('--- Page') \
and not line.startswith('(report generated by GoodReader)'):
if line:
prefix = re.search('^(?P<ann_type>{0})'.format('|'.join(self.ANNOTATION_TYPES + self.SKIP_TYPES)), line)
if prefix and prefix.group('ann_type') in self.SKIP_TYPES:
# Continue until next ann_type
i += 1
line = gr_annotations[i]
while not re.search('^(?P<ann_type>{0})'.format('|'.join(self.ANNOTATION_TYPES)), line):
i += 1
if i == num_lines:
break
line = gr_annotations[i]
continue
elif prefix:
# Additional highlight on the same page
# write current annotation, start new annotation
self._store_annotation(highlights, annotation)
annotation = self._extract_highlight(line, prefix.group('ann_type'))
annotation.page_num = page_num
annotation.ann_type = prefix.group('ann_type')
ann = ''
i += 1
line = gr_annotations[i]
continue
if not ann:
ann = line
else:
ann += '\n' + line
i += 1
line = gr_annotations[i]
annotation.ann = ann
# Back up so that the next line is '--- Page' or '(report generated'
i -= 1
self._store_annotation(highlights, annotation)
i += 1
if i == num_lines:
break
line = gr_annotations[i]
except:
if log_failure:
self._log(" unable to parse GoodReader Annotation summary")
self._log("{:~^80}".format(" Imported Annotation summary "))
self._log(raw)
self._log("{:~^80}".format(" end imported Annotations summary "))
import traceback
traceback.print_exc()
msg = ('Unable to parse Annotation summary from %s. ' % self.app_name +
'Paste entire contents of emailed summary.')
MessageBox(MessageBox.WARNING,
'Error importing annotations',
msg,
show_copy_button=False,
parent=self.opts.gui).exec_()
self._log_location("WARNING: %s" % msg)
return False
# Finalize book_mi
book_mi.annotations = len(highlights)
# Add book to books_db
self.add_to_books_db(self.books_db, book_mi)
self.annotated_book_list.append(book_mi)
sorted_keys = sorted(highlights.iterkeys())
for dt in sorted_keys:
highlight_text = None
if 'text' in highlights[dt]:
highlight_text = highlights[dt]['text']
note_text = None
if 'note' in highlights[dt]:
note_text = highlights[dt]['note']
# Populate an AnnotationStruct
a_mi = AnnotationStruct()
a_mi.annotation_id = dt
a_mi.book_id = book_mi['book_id']
a_mi.highlight_color = highlights[dt]['color']
a_mi.highlight_text = highlight_text
a_mi.location = highlights[dt]['page']
a_mi.last_modification = dt
a_mi.note_text = note_text
# Location sort
page_literal = re.match(r'^Page (?P<page>[0-9ivx]+).*$', a_mi.location).group('page')
if re.match('[IXVL]', page_literal.upper()):
whole = 0
decimal = self._roman_to_int(page_literal)
else:
whole = int(page_literal)
decimal = 0
a_mi.location_sort = "%05d.%05d" % (whole, decimal)
# Add annotation
self.add_to_annotations_db(self.annotations_db, a_mi)
self.update_book_last_annotation(self.books_db, dt, book_mi['book_id'])
# Update the timestamp
self.update_timestamp(self.annotations_db)
self.update_timestamp(self.books_db)
self.commit()
return True
def get_installed_books(self):
'''
For each book, construct a BookStruct object with the book's metadata.
Starred items are minimally required.
BookStruct properties:
*active: [True|False]
*author: "John Smith"
author_sort: (if known)
*book_id: an int uniquely identifying the book.
Highlights are associated with books through book_id
genre: "Fiction" (if known)
*reader_app: self.app_name
*title: "The Story of John Smith"
title_sort: "Story of John Smith, The" (if known)
uuid: Calibre's uuid for this book, if known
'''
# Sample installed books indexed by book_id
dict_of_books = {}
dict_of_books[1] = {'author': 'John Smith',
'author_sort': 'Smith, John',
'title': 'The Book by John Smith',
'title_sort': 'Book by John Smith, The'}
dict_of_books[2] = {'author': 'William Jones',
'author_sort': 'Jones, William',
'title': 'Learning Programming',
'title_sort': 'Learning Programming'}
dict_of_books[3] = {'author': 'Matthew Williams',
'author_sort': 'Williams, Matthew',
'title': 'A Book With No Annotations',
'title_sort': 'Book With No Annotations, A'}
self._log("%s:get_installed_books()" % self.app_name)
self.installed_books = []
# Don't change the template of books_db string
books_db = "%s_books_%s" % (re.sub(' ', '_', self.app_name), re.sub(' ', '_', self.opts.device_name))
installed_books = set([])
# Create the books table
self.create_books_table(books_db)
# Initialize the progress bar
self.opts.pb.set_label("Getting installed books from %s" % self.app_name)
self.opts.pb.set_value(0)
self.opts.pb.set_maximum(len(dict_of_books))
# Add installed books to the database
for book_id in dict_of_books:
# Add book_id to list of installed_books (make this a sql function)
installed_books.add(book_id)
# Populate a BookStruct with available metadata
book_mi = BookStruct()
# Required items
book_mi.active = True
book_mi.author = dict_of_books[book_id]['author']
book_mi.book_id = book_id
book_mi.reader_app = self.app_name
book_mi.title = dict_of_books[book_id]['title']
# Optional items
if 'author_sort' in dict_of_books[book_id]:
book_mi.author_sort = dict_of_books[book_id]['author_sort']
if 'genre' in dict_of_books[book_id]:
book_mi.genre = dict_of_books[book_id]['genre']
if 'title_sort' in dict_of_books[book_id]:
book_mi.title_sort = dict_of_books[book_id]['title_sort']
if 'uuid' in dict_of_books[book_id]:
book_mi.uuid = dict_of_books[book_id]['uuid']
# Add book to books_db
self.add_to_books_db(books_db, book_mi)
# Increment the progress bar
self.opts.pb.increment()
# Update the timestamp
self.update_timestamp(books_db)
self.commit()
self.installed_books = list(installed_books)
def get_installed_books(self):
'''
For each book, construct a BookStruct object with the book's metadata.
Starred items are minimally required.
BookStruct properties:
*active: [True|False]
*author: "John Smith"
author_sort: (if known)
*book_id: an int uniquely identifying the book.
Highlights are associated with books through book_id
genre: "Fiction" (if known)
*reader_app: self.app_name
*title: "The Story of John Smith"
title_sort: "Story of John Smith, The" (if known)
uuid: Calibre's uuid for this book, if known
'''
self._log("%s:get_installed_books()" % self.app_name)
self.installed_books = []
self.device = self.opts.gui.device_manager.device
path_map = self.get_path_map()
# Calibre already knows what books are on the device, so use it.
db = self.opts.gui.library_view.model().db
self.onDeviceIds = set(
db.search_getting_ids('ondevice:True',
None,
sort_results=False,
use_virtual_library=False))
# Add books added to Tolino by WhisperNet or download
# resolved_path_map = self._get_imported_books(resolved_path_map)
self.books_db = self.generate_books_db_name(self.app_name_,
self.opts.device_name)
installed_books = set([])
# Used by get_active_annotations() to look up metadata based on title
self.installed_books_by_title = {}
# Create the books table
self.create_books_table(self.books_db)
# Initialize the progress bar
self.opts.pb.set_label("Getting installed books from %s" %
self.app_name)
self.opts.pb.set_value(0)
self.opts.pb.show()
self.opts.pb.set_maximum(len(self.onDeviceIds))
# Add installed books to the database
for book_id in self.onDeviceIds:
try:
library_mi = mi = db.get_metadata(book_id, index_is_id=True)
except Exception as e:
self._log("Unable to get metadata from book. book_id='%s'" %
(book_id))
self._log(" Exception thrown was=%s" % (str(e)))
continue
self._log("Book on device title: '%s'" % (mi.title))
for model in (self.opts.gui.memory_view.model(),
self.opts.gui.card_a_view.model(),
self.opts.gui.card_b_view.model()):
model_paths = model.paths_for_db_ids(set([book_id]),
as_map=True)[book_id]
if model_paths:
device_path = model_paths[0].path
self._log(" Book on device path: '%s'" % (device_path, ))
mi = self._get_metadata(device_path)
# self._log(" Book on device path: '%s'" % (mi,))
break
if 'News' in mi.tags:
if not self.collect_news_clippings:
continue
installed_books.add(self.news_clippings_cid)
else:
installed_books.add(book_id)
#self._log(mi.standard_field_keys())
# Populate a BookStruct with available metadata
book_mi = BookStruct()
# book_mi.path = resolved_path_map[book_id]
# Required items
book_mi.active = True
# Massage last, first authors back to normalcy
book_mi.author = ''
for i, author in enumerate(mi.authors):
this_author = author.split(', ')
this_author.reverse()
book_mi.author += ' '.join(this_author)
if i < len(mi.authors) - 1:
book_mi.author += ' & '
book_mi.book_id = book_id
book_mi.reader_app = self.app_name
book_mi.title = mi.title.strip()
# Add book to indexed_books
self.installed_books_by_title[mi.title] = {
'book_id': book_id,
'author_sort': mi.author_sort
}
# Optional items
if mi.tags:
book_mi.genre = ', '.join([tag for tag in mi.tags])
if 'News' in mi.tags:
book_mi.book_id = self.news_clippings_cid
if hasattr(mi, 'author_sort'):
book_mi.author_sort = mi.author_sort
self.installed_books_by_title[
mi.title]['author_sort'] = mi.author_sort
if hasattr(mi, 'title_sort'):
book_mi.title_sort = mi.title_sort
else:
book_mi.title_sort = re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+',
'', mi.title).rstrip()
if hasattr(library_mi, 'uuid'):
self._log(" Book on has uuid: '%s'" % (library_mi.uuid, ))
book_mi.uuid = library_mi.uuid
self.installed_books_by_title[mi.title]['uuid'] = book_mi.uuid
# Add book to self.books_db
self.add_to_books_db(self.books_db, book_mi)
# Increment the progress bar
self.opts.pb.increment()
self.opts.pb.hide()
# Update the timestamp
self.update_timestamp(self.books_db)
self.commit()
self.installed_books = list(installed_books)
def get_installed_books(self):
"""
Fetch installed books from iBooks_*.sqlite or cache
"""
self._log("%s:get_installed_books()" % self.app_name)
self.opts.pb.set_label("Getting installed books from %s" %
self.app_name)
self.opts.pb.set_value(0)
db_profile = self._localize_database_path(self.app_id,
self.books_subpath)
self.books_db = db_profile['path']
cached_db = self.generate_books_db_name(self.app_name_,
self.ios.device_name)
# Test timestamp against cached value
if self.opts.disable_caching or not self._cache_is_current(
db_profile['stats'], cached_db):
# (Re)load installed books from device
self._log(" fetching installed books from %s on %s" %
(self.app_name, self.ios.device_name))
# Mount the Media folder
self.ios.mount_ios_media_folder()
installed_books = set([])
self.tocs = {}
# Create the books table as needed
self.create_books_table(cached_db)
con = sqlite3.connect(self.books_db)
with con:
con.row_factory = sqlite3.Row
cur = con.cursor()
cur.execute('''SELECT ZASSETURL,
ZBOOKAUTHOR,
ZSORTAUTHOR,
ZBOOKTITLE,
ZSORTTITLE,
ZDATABASEKEY
FROM ZBKBOOKINFO
WHERE ZASSETURL LIKE 'file://localhost%' AND
ZASSETURL LIKE '%.epub/'
''')
rows = cur.fetchall()
self.opts.pb.set_maximum(len(rows))
for row in rows:
self.opts.pb.increment()
this_is_news = False
path = self._fix_iBooks_path(row[b'ZASSETURL'])
mi = self._get_metadata(path, row[b'ZBOOKTITLE'])
genres = mi['genre'].split(', ')
if 'News' in genres:
if not self.collect_news_clippings:
continue
this_is_news = True
book_id = row[b'ZDATABASEKEY']
installed_books.add(book_id)
# Populate a BookStruct
b_mi = BookStruct()
b_mi.active = True
b_mi.author = row[b'ZBOOKAUTHOR']
b_mi.author_sort = row[b'ZSORTAUTHOR']
b_mi.book_id = book_id
b_mi.genre = mi['genre']
b_mi.title = row[b'ZBOOKTITLE']
b_mi.title_sort = row[b'ZSORTTITLE']
b_mi.uuid = mi['uuid']
# Add book to books_db
self.add_to_books_db(cached_db, b_mi)
# Get the library cid, confidence
toc_entries = None
if this_is_news:
cid = self.news_clippings_cid
confidence = 5
if path is not None:
toc_entries = self._get_epub_toc(
path=path, prepend_title=b_mi.title)
elif self.ios.exists(path):
cid, confidence = self.parent.generate_confidence(b_mi)
if confidence >= 2:
toc_entries = self._get_epub_toc(cid=cid,
path=path)
elif path is not None:
toc_entries = self._get_epub_toc(path=path)
self.tocs[book_id] = toc_entries
# Update the timestamp
self.update_timestamp(cached_db)
self.commit()
self.ios.dismount_ios_media_folder()
installed_books = list(installed_books)
else:
self._log(" retrieving cached books from %s" % cached_db)
self.opts.pb.set_maximum(2)
self.opts.pb.set_value(1)
Application.processEvents()
installed_books = self._get_cached_books(cached_db)
self.installed_books = installed_books
def get_installed_books(self):
'''
For each book, construct a BookStruct object with the book's metadata.
Starred items are minimally required.
BookStruct properties:
*active: [True|False]
*author: "John Smith"
author_sort: (if known)
*book_id: an int uniquely identifying the book.
Highlights are associated with books through book_id
genre: "Fiction" (if known)
*reader_app: self.app_name
*title: "The Story of John Smith"
title_sort: "Story of John Smith, The" (if known)
uuid: Calibre's uuid for this book, if known
'''
self._log_location("Start!!!!")
self._log("%s:get_installed_books()" % self.app_name)
self.installed_books = []
self.device = self.opts.gui.device_manager.device
self._log("%s:get_installed_books() - about to call self.get_path_map" % self.app_name)
path_map = self.get_path_map()
# self._log(path_map)
# Get books added to Kindle by calibre
self._log("%s:get_installed_books() - about to call self._get_installed_books" % self.app_name)
# resolved_path_map = self._get_installed_books(path_map)
# Calibre already knows what books are on the device, so use it.
db = self.opts.gui.library_view.model().db
self.onDeviceIds = set(db.search_getting_ids('ondevice:True', None, sort_results=False, use_virtual_library=False))
self._log("%s:get_installed_books() - about to call self.generate_books_db_name" % self.app_name)
self.books_db = self.generate_books_db_name(self.app_name_, self.opts.device_name)
installed_books = set([])
# Used by get_active_annotations() to look up metadata based on title
self.installed_books_by_title = {}
# Create the books table
self.create_books_table(self.books_db)
# Initialize the progress bar
self.opts.pb.set_label("Getting installed books from %s" % self.app_name)
self.opts.pb.set_value(0)
self.opts.pb.set_maximum(len(self.onDeviceIds))
self._log("Number of books on the device=%d" % len(self.onDeviceIds))
# Add installed books to the database
for book_id in self.onDeviceIds:
mi = db.get_metadata(book_id, index_is_id=True)
# self._log_location("book: {0} - {1}".format(mi.authors, mi.title))
# self._log("mi={0}".format(mi))
installed_books.add(book_id)
#self._log(mi.standard_field_keys())
# Populate a BookStruct with available metadata
book_mi = BookStruct()
# book_mi.path = resolved_path_map[book_id] # Add book_id to list of installed_books (make this a sql function)
installed_books.add(book_id)
# Populate a BookStruct with available metadata
book_mi = BookStruct()
# Required items
book_mi.active = True
# Massage last, first authors back to normalcy
book_mi.author = ''
for i, author in enumerate(mi.authors):
# self._log_location("author=%s, author.__class__=%s" % (author, author.__class__))
this_author = author.split(', ')
this_author.reverse()
book_mi.author += ' '.join(this_author)
if i < len(mi.authors) - 1:
book_mi.author += ' & '
book_mi.book_id = book_id
book_mi.reader_app = self.app_name
book_mi.title = mi.title
# Optional items
# if mi.tags:
# book_mi.genre = ', '.join([tag for tag in mi.tags])
# if 'News' in mi.tags:
# book_mi.book_id = self.news_clippings_cid
if hasattr(mi, 'author_sort'):
book_mi.author_sort = mi.author_sort
if hasattr(mi, 'title_sort'):
book_mi.title_sort = mi.title_sort
else:
book_mi.title_sort = re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', mi.title).rstrip()
if hasattr(mi, 'uuid'):
book_mi.uuid = mi.uuid
# Add book to self.books_db
self.add_to_books_db(self.books_db, book_mi)
# Add book to indexed_books
self.installed_books_by_title[mi.title] = {'book_id': book_id, 'author_sort': mi.author_sort}
# Increment the progress bar
self.opts.pb.increment()
# Update the timestamp
self.update_timestamp(self.books_db)
self.commit()
self.installed_books = list(installed_books)
self._log_location("Finish!!!!")
def parse_exported_highlights(self, raw):
"""
Extract highlights from pasted Annotations summary, add them to selected book
in calibre library
Construct a BookStruct object with the book's metadata.
Starred items are minimally required.
BookStruct properties:
*active: [True|False]
*author: "John Smith"
author_sort: (if known)
*book_id: an int uniquely identifying the book.
Highlights are associated with books through book_id
genre: "Fiction" (if known)
*title: "The Story of John Smith"
title_sort: "Story of John Smith, The" (if known)
uuid: Calibre's uuid for this book, if known
Construct an AnnotationStruct object with the
highlight's metadata. Starred items are minimally required. Dashed items
(highlight_text and note_text) may be one or both.
AnnotationStruct properties:
annotation_id: an int uniquely identifying the annotation
*book_id: The book this annotation is associated with
highlight_color: [Blue|Gray|Green|Pink|Purple|Underline|Yellow]
-highlight_text: A list of paragraphs constituting the highlight
last_modification: The timestamp of the annotation
location: location of highlight in the book
-note_text: A list of paragraphs constituting the note
*timestamp: Unique timestamp of highlight's creation/modification time
"""
self._log("%s:parse_exported_highlight()" % self.app_name)
# Create the annotations, books table as needed
self.annotations_db = "%s_imported_annotations" % self.app_name_
self.create_annotations_table(self.annotations_db)
self.books_db = "%s_imported_books" % self.app_name_
self.create_books_table(self.books_db)
self.annotated_book_list = []
self.selected_books = None
# Generate the book metadata from the selected book
row = self.opts.gui.library_view.currentIndex()
book_id = self.opts.gui.library_view.model().id(row)
db = self.opts.gui.current_db
mi = db.get_metadata(book_id, index_is_id=True)
# Populate author, title at a minimum
title = "A Book With Some Exported Annotations"
author = "John Smith"
# Populate a BookStruct
book_mi = BookStruct()
book_mi.active = True
book_mi.author = author
book_mi.book_id = mi.id
book_mi.title = title
book_mi.uuid = None
book_mi.last_update = time.mktime(time.localtime())
book_mi.reader_app = self.app_name
book_mi.cid = mi.id
book_mi.annotations = len(self.highlights)
# Add annotations to the database
for timestamp in sorted(self.highlights.iterkeys()):
book_mi.last_update = timestamp
# Populate an AnnotationStruct
ann_mi = AnnotationStruct()
# Required items
ann_mi.book_id = book_mi['book_id']
ann_mi.last_modification = timestamp
# Optional items
if 'annotation_id' in self.highlights[timestamp]:
ann_mi.annotation_id = self.highlights[timestamp]['annotation_id']
if 'highlight_color' in self.highlights[timestamp]:
ann_mi.highlight_color = self.highlights[timestamp]['highlight_color']
if 'highlight_text' in self.highlights[timestamp]:
highlight_text = '\n'.join(self.highlights[timestamp]['highlight_text'])
ann_mi.highlight_text = highlight_text
if 'note_text' in self.highlights[timestamp]:
note_text = '\n'.join(self.highlights[timestamp]['note_text'])
ann_mi.note_text = note_text
# Add annotation to annotations_db
self.add_to_annotations_db(self.annotations_db, ann_mi)
# Increment the progress bar
self.opts.pb.increment()
# Update last_annotation in books_db
self.update_book_last_annotation(self.books_db, timestamp, ann_mi.book_id)
# Add book to books_db
self.add_to_books_db(self.books_db, book_mi)
self.annotated_book_list.append(book_mi)
# Update the timestamp
self.update_timestamp(self.annotations_db)
self.update_timestamp(self.books_db)
self.commit()
# Return True if successful
return True