def parse_exported_highlights(self, raw, log_failure=True):
"""
Extract highlights from pasted Annotation summary email
Return True if no problems
Return False if error
"""
# Create the annotations, books table as needed
self.annotations_db = "%s_imported_annotations" % self.app_name_
self.create_annotations_table(self.annotations_db)
self.books_db = "%s_imported_books" % self.app_name_
self.create_books_table(self.books_db)
self.annotated_book_list = []
self.selected_books = None
# Generate the book metadata from the selected book
row = self.opts.gui.library_view.currentIndex()
book_id = self.opts.gui.library_view.model().id(row)
db = self.opts.gui.current_db
mi = db.get_metadata(book_id, index_is_id=True)
# Grab the title from the front of raw
try:
title = re.match(r'(?m)File: (?P<title>.*)$', raw).group('title')
# Populate a BookStruct
book_mi = BookStruct()
book_mi.active = True
book_mi.author = 'Unknown'
book_mi.book_id = mi.id
book_mi.title = title
book_mi.uuid = None
book_mi.last_update = time.mktime(time.localtime())
book_mi.reader_app = self.app_name
book_mi.cid = mi.id
gr_annotations = raw.split('\n')
num_lines = len(gr_annotations)
highlights = {}
# Find the first annotation
i = 0
line = gr_annotations[i]
while not line.startswith('--- Page'):
i += 1
line = gr_annotations[i]
while i < num_lines and not line.startswith('(report generated by GoodReader)'):
# Extract the page number
page_num = re.search('--- (Page \w+) ---', line)
if page_num:
page_num = page_num.group(1)
# Extract the highlight
i += 1
line = gr_annotations[i]
prefix = None
while True:
prefix = re.search('^(?P<ann_type>{0})'.format('|'.join(self.ANNOTATION_TYPES + self.SKIP_TYPES)), line)
if prefix and prefix.group('ann_type') in self.SKIP_TYPES:
i += 1
line = gr_annotations[i]
while not re.search('^(?P<ann_type>{0})'.format('|'.join(self.ANNOTATION_TYPES)), line):
i += 1
line = gr_annotations[i]
continue
elif prefix:
break
else:
i += 1
line = gr_annotations[i]
annotation = self._extract_highlight(line, prefix.group('ann_type'))
annotation.page_num = page_num
# Get the annotation(s)
i += 1
line = gr_annotations[i]
ann = ''
while i < num_lines \
and not line.startswith('--- Page') \
and not line.startswith('(report generated by GoodReader)'):
if line:
prefix = re.search('^(?P<ann_type>{0})'.format('|'.join(self.ANNOTATION_TYPES + self.SKIP_TYPES)), line)
if prefix and prefix.group('ann_type') in self.SKIP_TYPES:
# Continue until next ann_type
i += 1
line = gr_annotations[i]
while not re.search('^(?P<ann_type>{0})'.format('|'.join(self.ANNOTATION_TYPES)), line):
i += 1
if i == num_lines:
break
line = gr_annotations[i]
continue
elif prefix:
# Additional highlight on the same page
# write current annotation, start new annotation
self._store_annotation(highlights, annotation)
annotation = self._extract_highlight(line, prefix.group('ann_type'))
annotation.page_num = page_num
annotation.ann_type = prefix.group('ann_type')
ann = ''
i += 1
line = gr_annotations[i]
continue
if not ann:
ann = line
else:
ann += '\n' + line
i += 1
line = gr_annotations[i]
annotation.ann = ann
# Back up so that the next line is '--- Page' or '(report generated'
i -= 1
self._store_annotation(highlights, annotation)
i += 1
if i == num_lines:
break
line = gr_annotations[i]
except:
if log_failure:
self._log(" unable to parse GoodReader Annotation summary")
self._log("{:~^80}".format(" Imported Annotation summary "))
self._log(raw)
self._log("{:~^80}".format(" end imported Annotations summary "))
import traceback
traceback.print_exc()
msg = ('Unable to parse Annotation summary from %s. ' % self.app_name +
'Paste entire contents of emailed summary.')
MessageBox(MessageBox.WARNING,
'Error importing annotations',
msg,
show_copy_button=False,
parent=self.opts.gui).exec_()
self._log_location("WARNING: %s" % msg)
return False
# Finalize book_mi
book_mi.annotations = len(highlights)
# Add book to books_db
self.add_to_books_db(self.books_db, book_mi)
self.annotated_book_list.append(book_mi)
sorted_keys = sorted(highlights.iterkeys())
for dt in sorted_keys:
highlight_text = None
if 'text' in highlights[dt]:
highlight_text = highlights[dt]['text']
note_text = None
if 'note' in highlights[dt]:
note_text = highlights[dt]['note']
# Populate an AnnotationStruct
a_mi = AnnotationStruct()
a_mi.annotation_id = dt
a_mi.book_id = book_mi['book_id']
a_mi.highlight_color = highlights[dt]['color']
a_mi.highlight_text = highlight_text
a_mi.location = highlights[dt]['page']
a_mi.last_modification = dt
a_mi.note_text = note_text
# Location sort
page_literal = re.match(r'^Page (?P<page>[0-9ivx]+).*$', a_mi.location).group('page')
if re.match('[IXVL]', page_literal.upper()):
whole = 0
decimal = self._roman_to_int(page_literal)
else:
whole = int(page_literal)
decimal = 0
a_mi.location_sort = "%05d.%05d" % (whole, decimal)
# Add annotation
self.add_to_annotations_db(self.annotations_db, a_mi)
self.update_book_last_annotation(self.books_db, dt, book_mi['book_id'])
# Update the timestamp
self.update_timestamp(self.annotations_db)
self.update_timestamp(self.books_db)
self.commit()
return True
def get_active_annotations(self):
'''
For each annotation, construct an AnnotationStruct object with the
highlight's metadata. Starred items are minimally required. Dashed items
(highlight_text and note_text) may be one or both.
AnnotationStruct properties:
annotation_id: an int uniquely identifying the annotation
*book_id: The book this annotation is associated with
highlight_color: [Blue|Gray|Green|Pink|Purple|Underline|Yellow]
-highlight_text: A list of paragraphs constituting the highlight
last_modification: The timestamp of the annotation
location: location of highlight in the book
-note_text: A list of paragraphs constituting the note
*timestamp: Unique timestamp of highlight's creation/modification time
'''
self._log("%s:get_active_annotations()" % self.app_name)
self.active_annotations = {}
self.opts.pb.set_label("Getting active annotations for %s" %
self.app_name)
self.opts.pb.set_value(0)
# Don't change the template of the _db strings
#self.books_db = "%s_books_%s" % (re.sub(' ', '_', self.app_name), re.sub(' ', '_', self.opts.device_name))
#self.annotations_db = "%s_annotations_%s" % (re.sub(' ', '_', self.app_name), re.sub(' ', '_', self.opts.device_name))
self.annotations_db = self.generate_annotations_db_name(
self.app_name_, self.opts.device_name)
self.books_db = self.generate_books_db_name(self.app_name_,
self.opts.device_name)
# Create the annotations table
self.create_annotations_table(self.annotations_db)
# Parse MyClippings.txt for entries matching installed_books
self._parse_tolino_notes()
# Initialize the progress bar
self.opts.pb.set_label("Getting highlights from %s" % self.app_name)
self.opts.pb.set_value(0)
self.opts.pb.show()
self.opts.pb.set_maximum(len(self.active_annotations))
# Add annotations to the database
for timestamp in sorted(self.active_annotations.keys()):
# Populate an AnnotationStruct with available data
ann_mi = AnnotationStruct()
# Required items
ann_mi.book_id = self.active_annotations[timestamp]['book_id']
ann_mi.last_modification = timestamp
this_is_news = self.collect_news_clippings and 'News' in self.get_genres(
self.books_db, ann_mi.book_id)
# Optional items
if 'annotation_id' in self.active_annotations[timestamp]:
ann_mi.annotation_id = self.active_annotations[timestamp][
'annotation_id']
if 'highlight_color' in self.active_annotations[timestamp]:
ann_mi.highlight_color = self.active_annotations[timestamp][
'highlight_color']
if 'highlight_text' in self.active_annotations[timestamp]:
highlight_text = '\n'.join(
self.active_annotations[timestamp]['highlight_text'])
ann_mi.highlight_text = highlight_text
if this_is_news:
ann_mi.location = self.get_title(self.books_db, ann_mi.book_id)
ann_mi.location_sort = timestamp
else:
if 'location' in self.active_annotations[timestamp]:
ann_mi.location = self.active_annotations[timestamp][
'location']
if 'location_sort' in self.active_annotations[timestamp]:
ann_mi.location_sort = self.active_annotations[timestamp][
'location_sort']
if 'note_text' in self.active_annotations[timestamp]:
note_text = '\n'.join(
self.active_annotations[timestamp]['note_text'])
ann_mi.note_text = note_text
# Add annotation to self.annotations_db
self.add_to_annotations_db(self.annotations_db, ann_mi)
# Increment the progress bar
self.opts.pb.increment()
# Update last_annotation in self.books_db
self.update_book_last_annotation(self.books_db, timestamp,
ann_mi.book_id)
self.opts.pb.hide()
# Update the timestamp
self.update_timestamp(self.annotations_db)
self.commit()
def get_active_annotations(self):
self._log("%s:get_active_annotations()" % self.app_name)
self.opts.pb.set_label("Getting active annotations for %s" %
self.app_name)
self.opts.pb.set_value(0)
db_profile = self._localize_database_path(self.app_id,
self.annotations_subpath)
self.annotations_db = db_profile['path']
# Test timestamp against cached value
cached_db = self.generate_annotations_db_name(self.app_name_,
self.ios.device_name)
books_db = self.generate_books_db_name(self.app_name_,
self.ios.device_name)
if self.opts.disable_caching or not self._cache_is_current(
db_profile['stats'], cached_db):
self._log(" fetching annotations from %s on %s" %
(self.app_name, self.ios.device_name))
# Create the annotations table as needed
self.create_annotations_table(cached_db)
obsolete_bookmarks = 0
deleted_bookmarks = 0
con = sqlite3.connect(self.annotations_db)
with con:
con.row_factory = sqlite3.Row
cur = con.cursor()
cur.execute('''SELECT * FROM Highlights
ORDER BY NoteDateTime
''')
rows = cur.fetchall()
self.opts.pb.set_maximum(len(rows))
for row in rows:
self.opts.pb.increment()
book_id = row[b'BookID']
if not book_id in self.installed_books:
obsolete_bookmarks += 1
continue
# Collect the markup/highlight count for all installed books
if row[b'Deleted'] == 1:
deleted_bookmarks += 1
continue
this_is_news = self.collect_news_clippings and 'News' in self.get_genres(
books_db, book_id)
# Sanitize text, note to unicode
highlight_text = re.sub('\xa0', ' ', row[b'Text'])
highlight_text = UnicodeDammit(highlight_text).unicode
highlight_text = highlight_text.rstrip('\n').split('\n')
while highlight_text.count(''):
highlight_text.remove('')
highlight_text = [line.strip() for line in highlight_text]
note_text = None
if row[b'Note']:
ntu = UnicodeDammit(row[b'Note']).unicode
note_text = ntu.rstrip('\n')
# Populate an AnnotationStruct
a_mi = AnnotationStruct()
a_mi.annotation_id = row[b'UUID']
a_mi.book_id = book_id
a_mi.highlight_color = self.HIGHLIGHT_COLORS[
row[b'Colour']]
a_mi.highlight_text = '\n'.join(highlight_text)
a_mi.last_modification = row[b'NoteDateTime']
section = str(int(row[b'Section']) - 1)
try:
a_mi.location = self.tocs[book_id][section]
except:
if this_is_news:
a_mi.location = self.get_title(books_db, book_id)
else:
a_mi.location = "Section %s" % row[b'Section']
a_mi.note_text = note_text
# If empty highlight_text and empty note_text, not a useful annotation
if not highlight_text and not note_text:
continue
# Generate location_sort
if this_is_news:
a_mi.location_sort = row[b'NoteDateTime']
else:
interior = self._generate_interior_location_sort(
row[b'StartXPath'])
if not interior:
self._log("Marvin: unable to parse xpath:")
self._log(row[b'StartXPath'])
self._log(a_mi)
continue
a_mi.location_sort = "%04d.%s.%04d" % (int(
row[b'Section']), interior, int(
row[b'StartOffset']))
# Add annotation
self.add_to_annotations_db(cached_db, a_mi)
# Update last_annotation in books_db
self.update_book_last_annotation(books_db,
row[b'NoteDateTime'],
book_id)
# Update the timestamp
self.update_timestamp(cached_db)
self.commit()
else:
self._log(" retrieving cached annotations from %s" % cached_db)
def _process_individual_book(book):
book_mi = BookStruct()
book_mi['reader_app'] = self.app_name
book_mi['cid'] = None
for md in xl:
book_mi[xl[md]] = book.get(md)
book_mi['active'] = True
book_mi['annotations'] = 0
subjects = book.find('subjects')
if subjects is not None:
sl = [s.text for s in subjects]
book_mi['genre'] = ', '.join(sl)
this_is_news = False
if 'News' in book_mi['genre']:
if not self.collect_news_clippings:
return
this_is_news = True
# Get the last update, count active annotations
last_update = 0
hls = book.find('highlights')
for hl in hls:
this_ts = hl.get('datetime')
if this_ts > last_update:
last_update = this_ts
if hl.get('deleted') == '0':
book_mi['annotations'] += 1
book_mi['last_update'] = float(last_update)
# Get the library cid, confidence
toc_entries = None
if this_is_news:
cid = self.news_clippings_cid
confidence = 5
else:
cid, confidence = self.parent.generate_confidence(book_mi)
if confidence >= 2:
toc_entries = self._get_epub_toc(cid=cid)
# Add annotated book to the db, master_list
if len(hls):
self.add_to_books_db(self.books_db, book_mi)
self.annotated_book_list.append(book_mi)
# Add the active annotations for this book to the db
highlights = {}
for hl in hls:
if hl.get('deleted') == '1':
continue
datetime = hl.get('datetime')
highlights[datetime] = {}
for md in [
'text', 'note', 'color', 'key', 'deleted',
'section', 'startx', 'startoffset'
]:
highlights[datetime][md] = hl.get(md)
sorted_keys = sorted(highlights.iterkeys())
for datetime in sorted_keys:
highlight_text = highlights[datetime]['text']
note_text = highlights[datetime]['note']
# Populate an AnnotationStruct
a_mi = AnnotationStruct()
a_mi.annotation_id = highlights[datetime]['key']
a_mi.book_id = book_mi['book_id']
a_mi.highlight_color = self.HIGHLIGHT_COLORS[int(
highlights[datetime]['color'])]
a_mi.highlight_text = highlight_text
a_mi.last_modification = datetime
try:
section = str(int(highlights[datetime]['section']) - 1)
a_mi.location = toc_entries[section]
except:
if this_is_news:
a_mi.location = book_mi['title']
else:
a_mi.location = "Section %s" % highlights[
datetime]['section']
a_mi.note_text = note_text
# If empty highlight_text and empty note_text, not a useful annotation
if (not highlight_text.strip() and not note_text.strip()):
continue
# Generate location_sort
if this_is_news:
a_mi.location_sort = datetime
else:
interior = self._generate_interior_location_sort(
highlights[datetime]['startx'])
if not interior:
self._log("Marvin: unable to parse xpath:")
self._log(" %s" % highlights[datetime]['startx'])
self._log(a_mi)
continue
a_mi.location_sort = "%04d.%s.%04d" % (
int(highlights[datetime]['section']), interior,
int(highlights[datetime]['startoffset']))
self.add_to_annotations_db(self.annotations_db, a_mi)
self.update_book_last_annotation(self.books_db, datetime,
book_mi['book_id'])
# Update the timestamps
self.update_timestamp(self.annotations_db)
self.update_timestamp(self.books_db)
self.commit()
def get_active_annotations(self):
'''
For each annotation, construct an AnnotationStruct object with the
highlight's metadata. Starred items are minimally required. Dashed items
(highlight_text and note_text) may be one or both.
AnnotationStruct properties:
annotation_id: an int uniquely identifying the annotation
*book_id: The book this annotation is associated with
highlight_color: [Blue|Gray|Green|Pink|Purple|Underline|Yellow]
-highlight_text: A list of paragraphs constituting the highlight
last_modification: The timestamp of the annotation
location: location of highlight in the book
-note_text: A list of paragraphs constituting the note
*timestamp: Unique timestamp of highlight's creation/modification time
'''
# Sample annotations, indexed by timestamp. Note that annotations may have
# highlight_text, note_text, or both.
dict_of_anns = {}
'''
ts = datetime.datetime(2012, 12, 4, 8, 15, 0)
dict_of_anns[time.mktime(ts.timetuple())] = {'book_id': 1,
'highlight_color': 'Gray',
'highlight_text': ['The first paragraph of the first highlight.',
'The second paragaph of the first highlight.'],
}
ts = ts.replace(minute=16)
dict_of_anns[time.mktime(ts.timetuple())] = {'book_id': 1,
'highlight_color': 'Gray',
'highlight_text': ['The first paragraph of the second highlight.',
'The second paragaph of the second highlight.'],
'note_text': ['A note added to the second highlight']
}
ts = ts.replace(minute=17)
dict_of_anns[time.mktime(ts.timetuple())] = {'book_id': 1,
'highlight_color': 'Gray',
'note_text': ['A note added to the third highlight']
}
ts = datetime.datetime(2012, 12, 10, 9, 0, 0)
dict_of_anns[time.mktime(ts.timetuple())] = {'book_id': 2,
'highlight_color': 'Gray',
'highlight_text': ['The first paragraph of the first highlight.',
'The second paragaph of the first highlight.']
}
ts = ts.replace(minute=1)
dict_of_anns[time.mktime(ts.timetuple())] = {'book_id': 2,
'highlight_color': 'Gray',
'highlight_text': ['The first paragraph of the second highlight.',
'The second paragaph of the second highlight.'],
'note_text': ['A note added to the second highlight']
}
ts = ts.replace(minute=2)
dict_of_anns[time.mktime(ts.timetuple())] = {'book_id': 2,
'highlight_color': 'Gray',
'note_text': ['A note added to the third highlight']
}
ts = datetime.datetime(2012, 12, 31, 23, 59, 0)
dict_of_anns[time.mktime(ts.timetuple())] = {'book_id': 999,
'highlight_color': 'Gray',
'highlight_text': ['An orphan annotation (no book)']
}
'''
conn = self._get_db()
bookmark_cursor = conn.execute(
'SELECT _id, content_id, added_date, marked_text, name, page FROM annotation'
)
for row in bookmark_cursor:
print('adding annotation: ')
print(row)
ts = datetime.datetime.fromtimestamp(row[2] / 1e3)
dict_of_anns[time.mktime(ts.timetuple())] = {
'book_id': row[1],
'highlight_color': 'Yellow',
'highlight_text': [row[3]],
'note_text': [row[4]],
'timestamp': [time.mktime(ts.timetuple())],
'location': [row[5]]
}
conn.close()
self._log("%s:get_active_annotations()" % self.app_name)
self.opts.pb.set_label("Getting active annotations for %s" %
self.app_name)
self.opts.pb.set_value(0)
annotations_db = self.generate_annotations_db_name(
self.app_name_, self.opts.device_name)
self.books_db = self.generate_books_db_name(self.app_name_,
self.opts.device_name)
# Create the annotations table
self.create_annotations_table(annotations_db)
# Initialize the progress bar
self.opts.pb.set_label("Getting highlights from %s" % self.app_name)
self.opts.pb.set_value(0)
self.opts.pb.set_maximum(len(dict_of_anns))
# Add annotations to the database
for timestamp in sorted(dict_of_anns.keys()):
# Populate an AnnotationStruct with available data
ann_mi = AnnotationStruct()
# Required items
ann_mi.book_id = dict_of_anns[timestamp]['book_id']
ann_mi.last_modification = timestamp
# Optional items
if 'annotation_id' in dict_of_anns[timestamp]:
ann_mi.annotation_id = dict_of_anns[timestamp]['annotation_id']
if 'highlight_color' in dict_of_anns[timestamp]:
ann_mi.highlight_color = dict_of_anns[timestamp][
'highlight_color']
if 'highlight_text' in dict_of_anns[timestamp]:
highlight_text = '\n'.join(
dict_of_anns[timestamp]['highlight_text'])
ann_mi.highlight_text = highlight_text
if 'note_text' in dict_of_anns[timestamp]:
note_text = '\n'.join(dict_of_anns[timestamp]['note_text'])
if note_text != ann_mi.highlight_text:
ann_mi.note_text = note_text
else:
ann_mi.note_text = ''
if 'location' in dict_of_anns[timestamp]:
ann_mi.location = str(
int(
next(iter(dict_of_anns[timestamp]['location'] or []),
None)))
ann_mi.location = (
'p. ' + ann_mi.location) if ann_mi.location != None else ''
# Add annotation to annotations_db
self.add_to_annotations_db(annotations_db, ann_mi)
# Increment the progress bar
self.opts.pb.increment()
# Update last_annotation in books_db
self.update_book_last_annotation(self.books_db, timestamp,
ann_mi.book_id)
# Update the timestamp
self.update_timestamp(annotations_db)
self.commit()
def get_active_annotations(self):
'''
For each annotation, construct an AnnotationStruct object with the
highlight's metadata. Starred items are minimally required. Dashed items
(highlight_text and note_text) may be one or both.
AnnotationStruct properties:
annotation_id: an int uniquely identifying the annotation
*book_id: The book this annotation is associated with
highlight_color: [Blue|Gray|Green|Pink|Purple|Underline|Yellow]
-highlight_text: A list of paragraphs constituting the highlight
last_modification: The timestamp of the annotation
location: location of highlight in the book
-note_text: A list of paragraphs constituting the note
*timestamp: Unique timestamp of highlight's creation/modification time
'''
self._log("%s:get_active_annotations()" % self.app_name)
self.active_annotations = {}
self.opts.pb.set_label("Getting active annotations for %s" % self.app_name)
self.opts.pb.set_value(0)
# Don't change the template of the _db strings
#self.books_db = "%s_books_%s" % (re.sub(' ', '_', self.app_name), re.sub(' ', '_', self.opts.device_name))
#self.annotations_db = "%s_annotations_%s" % (re.sub(' ', '_', self.app_name), re.sub(' ', '_', self.opts.device_name))
self.annotations_db = self.generate_annotations_db_name(self.app_name_, self.opts.device_name)
self.books_db = self.generate_books_db_name(self.app_name_, self.opts.device_name)
# Create the annotations table
self.create_annotations_table(self.annotations_db)
# Parse MyClippings.txt for entries matching installed_books
self._parse_my_clippings()
# Initialize the progress bar
self.opts.pb.set_label("Getting highlights from %s" % self.app_name)
self.opts.pb.set_value(0)
self.opts.pb.show()
self.opts.pb.set_maximum(len(self.active_annotations))
# Add annotations to the database
for timestamp in sorted(self.active_annotations.iterkeys()):
# Populate an AnnotationStruct with available data
ann_mi = AnnotationStruct()
# Required items
ann_mi.book_id = self.active_annotations[timestamp]['book_id']
ann_mi.last_modification = timestamp
this_is_news = self.collect_news_clippings and 'News' in self.get_genres(self.books_db, ann_mi.book_id)
# Optional items
if 'annotation_id' in self.active_annotations[timestamp]:
ann_mi.annotation_id = self.active_annotations[timestamp]['annotation_id']
if 'highlight_color' in self.active_annotations[timestamp]:
ann_mi.highlight_color = self.active_annotations[timestamp]['highlight_color']
if 'highlight_text' in self.active_annotations[timestamp]:
highlight_text = '\n'.join(self.active_annotations[timestamp]['highlight_text'])
ann_mi.highlight_text = highlight_text
if this_is_news:
ann_mi.location = self.get_title(self.books_db, ann_mi.book_id)
ann_mi.location_sort = timestamp
else:
if 'location' in self.active_annotations[timestamp]:
ann_mi.location = self.active_annotations[timestamp]['location']
if 'location_sort' in self.active_annotations[timestamp]:
ann_mi.location_sort = self.active_annotations[timestamp]['location_sort']
if 'note_text' in self.active_annotations[timestamp]:
note_text = '\n'.join(self.active_annotations[timestamp]['note_text'])
ann_mi.note_text = note_text
# Add annotation to self.annotations_db
self.add_to_annotations_db(self.annotations_db, ann_mi)
# Increment the progress bar
self.opts.pb.increment()
# Update last_annotation in self.books_db
self.update_book_last_annotation(self.books_db, timestamp, ann_mi.book_id)
self.opts.pb.hide()
# Update the timestamp
self.update_timestamp(self.annotations_db)
self.commit()
def get_active_annotations(self):
"""
Fetch active iBooks annotations from AEAnnotation_*.sqlite
"""
self._log("%s:get_active_annotations()" % self.app_name)
self.opts.pb.set_label("Getting active annotations for %s" %
self.app_name)
self.opts.pb.set_value(0)
db_profile = self._localize_database_path(self.app_id,
self.annotations_subpath)
self.annotations_db = db_profile['path']
# Test timestamp against cached value
cached_db = self.generate_annotations_db_name(self.app_name_,
self.ios.device_name)
books_db = self.generate_books_db_name(self.app_name_,
self.ios.device_name)
if self.opts.disable_caching or not self._cache_is_current(
db_profile['stats'], cached_db):
self._log(" fetching annotations from %s on %s" %
(self.app_name, self.ios.device_name))
# Create the annotations table as needed
self.create_annotations_table(cached_db)
con = sqlite3.connect(self.annotations_db)
with con:
con.row_factory = sqlite3.Row
cur = con.cursor()
cur.execute('''SELECT
ZANNOTATIONASSETID,
ZANNOTATIONLOCATION,
ZANNOTATIONMODIFICATIONDATE,
ZANNOTATIONNOTE,
ZANNOTATIONSELECTEDTEXT,
ZANNOTATIONSTYLE,
ZANNOTATIONUUID
FROM ZAEANNOTATION
WHERE ZANNOTATIONDELETED = 0 and ZANNOTATIONTYPE = 2
ORDER BY ZANNOTATIONMODIFICATIONDATE
''')
rows = cur.fetchall()
self.opts.pb.set_maximum(len(rows))
for row in rows:
self.opts.pb.increment()
book_id = row[b'ZANNOTATIONASSETID']
if not book_id in self.installed_books:
continue
# Collect the metadata
# Sanitize text, note to unicode
highlight_text = re.sub('\xa0', ' ',
row[b'ZANNOTATIONSELECTEDTEXT'])
highlight_text = UnicodeDammit(highlight_text).unicode
highlight_text = highlight_text.rstrip('\n').split('\n')
while highlight_text.count(''):
highlight_text.remove('')
highlight_text = [line.strip() for line in highlight_text]
note_text = None
if row[b'ZANNOTATIONNOTE']:
note_text = UnicodeDammit(
row[b'ZANNOTATIONNOTE']).unicode
note_text = note_text.rstrip('\n').split('\n')[0]
# Populate an AnnotationStruct
a_mi = AnnotationStruct()
a_mi.annotation_id = row[b'ZANNOTATIONUUID']
a_mi.book_id = book_id
a_mi.epubcfi = row[b'ZANNOTATIONLOCATION']
a_mi.highlight_color = self.HIGHLIGHT_COLORS[
row[b'ZANNOTATIONSTYLE']]
a_mi.highlight_text = '\n'.join(highlight_text)
a_mi.last_modification = row[
b'ZANNOTATIONMODIFICATIONDATE'] + self.NSTimeIntervalSince1970
if a_mi.epubcfi:
section = self._get_spine_index(a_mi.epubcfi)
try:
a_mi.location = self.tocs[book_id]["%.0f" %
(section - 1)]
except:
a_mi.location = "Section %d" % section
if self.collect_news_clippings and 'News' in self.get_genres(
books_db, book_id):
a_mi.location_sort = a_mi.last_modification
else:
a_mi.location_sort = self._generate_location_sort(
a_mi.epubcfi)
else:
if self.collect_news_clippings and 'News' in self.get_genres(
books_db, book_id):
a_mi.location = self.get_title(books_db, book_id)
a_mi.location_sort = a_mi.last_modification
a_mi.note_text = note_text
# Add annotation
self.add_to_annotations_db(cached_db, a_mi)
# Update last_annotation in books_db
self.update_book_last_annotation(
books_db, row[b'ZANNOTATIONMODIFICATIONDATE'] +
self.NSTimeIntervalSince1970, book_id)
self.update_timestamp(cached_db)
self.commit()
else:
self._log(" retrieving cached annotations from %s" % cached_db)
def parse_exported_highlights(self, raw, log_failure=True):
"""
Extract highlights from pasted Annotations summary, add them to selected book
in calibre library
Construct a BookStruct object with the book's metadata.
Starred items are minimally required.
BookStruct properties:
*active: [True|False]
*author: "John Smith"
author_sort: (if known)
*book_id: an int uniquely identifying the book.
Highlights are associated with books through book_id
genre: "Fiction" (if known)
*title: "The Story of John Smith"
title_sort: "Story of John Smith, The" (if known)
uuid: Calibre's uuid for this book, if known
Construct an AnnotationStruct object with the
highlight's metadata. Starred items are minimally required. Dashed items
(highlight_text and note_text) may be one or both.
AnnotationStruct properties:
annotation_id: an int uniquely identifying the annotation
*book_id: The book this annotation is associated with
highlight_color: [Blue|Gray|Green|Pink|Purple|Underline|Yellow]
-highlight_text: A list of paragraphs constituting the highlight
last_modification: The timestamp of the annotation
location: location of highlight in the book
-note_text: A list of paragraphs constituting the note
*timestamp: Unique timestamp of highlight's creation/modification time
"""
# Create the annotations, books table as needed
self.annotations_db = "%s_imported_annotations" % self.app_name_
self.create_annotations_table(self.annotations_db)
self.books_db = "%s_imported_books" % self.app_name_
self.create_books_table(self.books_db)
self.annotated_book_list = []
self.selected_books = None
# Generate the book metadata from the selected book
row = self.opts.gui.library_view.currentIndex()
book_id = self.opts.gui.library_view.model().id(row)
db = self.opts.gui.current_db
mi = db.get_metadata(book_id, index_is_id=True)
try:
lines = raw.split('\n')
if len(lines) < 5:
raise AnnotationsException("Invalid annotations summary")
index = 0
annotations = {}
# Get the title, author, publisher from the first three lines
title = lines[index]
index += 1
author = lines[index]
index += 1
publisher = lines[index]
index += 1
# Next line should be the first timestamp/location
while index < len(lines):
tsl = re.match(r'^(?P<timestamp>.*) \((?P<location>Page .*)\)', lines[index])
if tsl:
ts = tsl.group('timestamp')
isoformat = parse_date(ts, as_utc=False)
isoformat = isoformat.replace(hour=12)
timestamp = mktime(isoformat.timetuple())
while timestamp in annotations:
timestamp += 60
location = tsl.group('location')
index += 1
# Continue with highlight
highlight_text = lines[index]
index += 1
# Next line is either Note: or a new tsl
note = re.match(r'^Notes: (?P<note_text>.*)', lines[index])
note_text = None
if note:
note_text = note.group('note_text')
index += 1
if re.match(r'^(?P<timestamp>.*) \((?P<location>Page .*)\)', lines[index]):
# New note - store the old one, continue
ann = AnnotationStruct()
ann.book_id = mi.id
ann.annotation_id = index
ann.highlight_color = 'Yellow'
ann.highlight_text = highlight_text
ann.location = location
ann.location_sort = "%05d" % int(re.match(r'^Page (?P<page>\d+).*$', location).group('page'))
ann.note_text = note_text
ann.last_modification = timestamp
# Add annotation to db
annotations[timestamp] = ann
continue
else:
# Store the last one
ann = AnnotationStruct()
ann.book_id = mi.id
ann.annotation_id = index
ann.highlight_color = 'Yellow'
ann.highlight_text = highlight_text
ann.location = location
ann.location_sort = "%05d" % int(re.match(r'^Page (?P<page>\d+).*$', location).group('page'))
ann.note_text = note_text
ann.last_modification = timestamp
annotations[timestamp] = ann
break
except:
if log_failure:
self._log(" unable to parse %s Annotations" % self.app_name)
self._log("{:~^80}".format(" Imported Annotation summary "))
self._log(raw)
self._log("{:~^80}".format(" end imported Annotations summary "))
import traceback
traceback.print_exc()
msg = ('Unable to parse Annotation summary from %s. ' % self.app_name +
'Paste entire contents of emailed summary.')
MessageBox(MessageBox.WARNING,
'Error importing annotations',
msg,
show_copy_button=False,
parent=self.opts.gui).exec_()
self._log_location("WARNING: %s" % msg)
return False
# Populate a BookStruct
book_mi = BookStruct()
book_mi.active = True
book_mi.author = author
book_mi.book_id = mi.id
book_mi.title = title
book_mi.uuid = None
book_mi.last_update = time.mktime(time.localtime())
book_mi.reader_app = self.app_name
book_mi.cid = mi.id
book_mi.annotations = len(annotations)
# Add book to books_db
self.add_to_books_db(self.books_db, book_mi)
self.annotated_book_list.append(book_mi)
# Add the annotations
for timestamp in sorted(annotations.keys()):
self.add_to_annotations_db(self.annotations_db, annotations[timestamp])
self.update_book_last_annotation(self.books_db, timestamp, mi.id)
self.opts.pb.increment()
self.update_book_last_annotation(self.books_db, timestamp, mi.id)
# Update the timestamp
self.update_timestamp(self.annotations_db)
self.update_timestamp(self.books_db)
self.commit()
# Return True if successful
return True
def get_active_annotations(self):
self._log("%s:get_active_annotations()" % self.app_name)
self.opts.pb.set_label("Getting active annotations for %s" % self.app_name)
self.opts.pb.set_value(0)
db_profile = self._localize_database_path(self.app_id, self.annotations_subpath)
self.annotations_db = db_profile['path']
# Test timestamp against cached value
cached_db = self.generate_annotations_db_name(self.app_name_, self.ios.device_name)
books_db = self.generate_books_db_name(self.app_name_, self.ios.device_name)
if self.opts.disable_caching or not self._cache_is_current(db_profile['stats'], cached_db):
self._log(" fetching annotations from %s on %s" % (self.app_name, self.ios.device_name))
# Create the annotations table as needed
self.create_annotations_table(cached_db)
obsolete_bookmarks = 0
deleted_bookmarks = 0
con = sqlite3.connect(self.annotations_db)
with con:
con.row_factory = sqlite3.Row
cur = con.cursor()
cur.execute('''SELECT * FROM Highlights
ORDER BY NoteDateTime
''')
rows = cur.fetchall()
self.opts.pb.set_maximum(len(rows))
for row in rows:
self.opts.pb.increment()
book_id = row[b'BookID']
if not book_id in self.installed_books:
obsolete_bookmarks += 1
continue
# Collect the markup/highlight count for all installed books
if row[b'Deleted'] == 1:
deleted_bookmarks += 1
continue
this_is_news = self.collect_news_clippings and 'News' in self.get_genres(books_db, book_id)
# Sanitize text, note to unicode
highlight_text = re.sub('\xa0', ' ', row[b'Text'])
highlight_text = UnicodeDammit(highlight_text).unicode
highlight_text = highlight_text.rstrip('\n').split('\n')
while highlight_text.count(''):
highlight_text.remove('')
highlight_text = [line.strip() for line in highlight_text]
note_text = None
if row[b'Note']:
ntu = UnicodeDammit(row[b'Note']).unicode
note_text = ntu.rstrip('\n')
# Populate an AnnotationStruct
a_mi = AnnotationStruct()
a_mi.annotation_id = row[b'UUID']
a_mi.book_id = book_id
a_mi.highlight_color = self.HIGHLIGHT_COLORS[row[b'Colour']]
a_mi.highlight_text = '\n'.join(highlight_text)
a_mi.last_modification = row[b'NoteDateTime']
section = str(int(row[b'Section']) - 1)
try:
a_mi.location = self.tocs[book_id][section]
except:
if this_is_news:
a_mi.location = self.get_title(books_db, book_id)
else:
a_mi.location = "Section %s" % row[b'Section']
a_mi.note_text = note_text
# If empty highlight_text and empty note_text, not a useful annotation
if not highlight_text and not note_text:
continue
# Generate location_sort
if this_is_news:
a_mi.location_sort = row[b'NoteDateTime']
else:
interior = self._generate_interior_location_sort(row[b'StartXPath'])
if not interior:
self._log("Marvin: unable to parse xpath:")
self._log(row[b'StartXPath'])
self._log(a_mi)
continue
a_mi.location_sort = "%04d.%s.%04d" % (
int(row[b'Section']),
interior,
int(row[b'StartOffset']))
# Add annotation
self.add_to_annotations_db(cached_db, a_mi)
# Update last_annotation in books_db
self.update_book_last_annotation(books_db, row[b'NoteDateTime'], book_id)
# Update the timestamp
self.update_timestamp(cached_db)
self.commit()
else:
self._log(" retrieving cached annotations from %s" % cached_db)
def _process_individual_book(book):
book_mi = BookStruct()
book_mi['reader_app'] = self.app_name
book_mi['cid'] = None
for md in xl:
book_mi[xl[md]] = book.get(md)
book_mi['active'] = True
book_mi['annotations'] = 0
subjects = book.find('subjects')
if subjects is not None:
sl = [s.text for s in subjects]
book_mi['genre'] = ', '.join(sl)
this_is_news = False
if 'News' in book_mi['genre']:
if not self.collect_news_clippings:
return
this_is_news = True
# Get the last update, count active annotations
last_update = 0
hls = book.find('highlights')
for hl in hls:
this_ts = hl.get('datetime')
if this_ts > last_update:
last_update = this_ts
if hl.get('deleted') == '0':
book_mi['annotations'] += 1
book_mi['last_update'] = float(last_update)
# Get the library cid, confidence
toc_entries = None
if this_is_news:
cid = self.news_clippings_cid
confidence = 5
else:
cid, confidence = self.parent.generate_confidence(book_mi)
if confidence >= 2:
toc_entries = self._get_epub_toc(cid=cid)
# Add annotated book to the db, master_list
if len(hls):
self.add_to_books_db(self.books_db, book_mi)
self.annotated_book_list.append(book_mi)
# Add the active annotations for this book to the db
highlights = {}
for hl in hls:
if hl.get('deleted') == '1':
continue
datetime = hl.get('datetime')
highlights[datetime] = {}
for md in ['text', 'note', 'color', 'key', 'deleted', 'section',
'startx', 'startoffset']:
highlights[datetime][md] = hl.get(md)
sorted_keys = sorted(highlights.iterkeys())
for datetime in sorted_keys:
highlight_text = highlights[datetime]['text']
note_text = highlights[datetime]['note']
# Populate an AnnotationStruct
a_mi = AnnotationStruct()
a_mi.annotation_id = highlights[datetime]['key']
a_mi.book_id = book_mi['book_id']
a_mi.highlight_color = self.HIGHLIGHT_COLORS[int(highlights[datetime]['color'])]
a_mi.highlight_text = highlight_text
a_mi.last_modification = datetime
try:
section = str(int(highlights[datetime]['section']) - 1)
a_mi.location = toc_entries[section]
except:
if this_is_news:
a_mi.location = book_mi['title']
else:
a_mi.location = "Section %s" % highlights[datetime]['section']
a_mi.note_text = note_text
# If empty highlight_text and empty note_text, not a useful annotation
if (not highlight_text.strip() and not note_text.strip()):
continue
# Generate location_sort
if this_is_news:
a_mi.location_sort = datetime
else:
interior = self._generate_interior_location_sort(highlights[datetime]['startx'])
if not interior:
self._log("Marvin: unable to parse xpath:")
self._log(" %s" % highlights[datetime]['startx'])
self._log(a_mi)
continue
a_mi.location_sort = "%04d.%s.%04d" % (
int(highlights[datetime]['section']),
interior,
int(highlights[datetime]['startoffset']))
self.add_to_annotations_db(self.annotations_db, a_mi)
self.update_book_last_annotation(self.books_db, datetime, book_mi['book_id'])
# Update the timestamps
self.update_timestamp(self.annotations_db)
self.update_timestamp(self.books_db)
self.commit()
def get_active_annotations(self):
"""
Fetch active iBooks annotations from AEAnnotation_*.sqlite
"""
self._log("%s:get_active_annotations()" % self.app_name)
self.opts.pb.set_label("Getting active annotations for %s" % self.app_name)
self.opts.pb.set_value(0)
db_profile = self._localize_database_path(self.app_id, self.annotations_subpath)
self.annotations_db = db_profile['path']
# Test timestamp against cached value
cached_db = self.generate_annotations_db_name(self.app_name_, self.ios.device_name)
books_db = self.generate_books_db_name(self.app_name_, self.ios.device_name)
if self.opts.disable_caching or not self._cache_is_current(db_profile['stats'], cached_db):
self._log(" fetching annotations from %s on %s" % (self.app_name, self.ios.device_name))
# Create the annotations table as needed
self.create_annotations_table(cached_db)
con = sqlite3.connect(self.annotations_db)
with con:
con.row_factory = sqlite3.Row
cur = con.cursor()
cur.execute('''SELECT
ZANNOTATIONASSETID,
ZANNOTATIONLOCATION,
ZANNOTATIONMODIFICATIONDATE,
ZANNOTATIONNOTE,
ZANNOTATIONSELECTEDTEXT,
ZANNOTATIONSTYLE,
ZANNOTATIONUUID
FROM ZAEANNOTATION
WHERE ZANNOTATIONDELETED = 0 and ZANNOTATIONTYPE = 2
ORDER BY ZANNOTATIONMODIFICATIONDATE
''')
rows = cur.fetchall()
self.opts.pb.set_maximum(len(rows))
for row in rows:
self.opts.pb.increment()
book_id = row[b'ZANNOTATIONASSETID']
if not book_id in self.installed_books:
continue
# Collect the metadata
# Sanitize text, note to unicode
highlight_text = re.sub('\xa0', ' ', row[b'ZANNOTATIONSELECTEDTEXT'])
highlight_text = UnicodeDammit(highlight_text).unicode
highlight_text = highlight_text.rstrip('\n').split('\n')
while highlight_text.count(''):
highlight_text.remove('')
highlight_text = [line.strip() for line in highlight_text]
note_text = None
if row[b'ZANNOTATIONNOTE']:
note_text = UnicodeDammit(row[b'ZANNOTATIONNOTE']).unicode
note_text = note_text.rstrip('\n').split('\n')[0]
# Populate an AnnotationStruct
a_mi = AnnotationStruct()
a_mi.annotation_id = row[b'ZANNOTATIONUUID']
a_mi.book_id = book_id
a_mi.epubcfi = row[b'ZANNOTATIONLOCATION']
a_mi.highlight_color = self.HIGHLIGHT_COLORS[row[b'ZANNOTATIONSTYLE']]
a_mi.highlight_text = '\n'.join(highlight_text)
a_mi.last_modification = row[b'ZANNOTATIONMODIFICATIONDATE'] + self.NSTimeIntervalSince1970
if a_mi.epubcfi:
section = self._get_spine_index(a_mi.epubcfi)
try:
a_mi.location = self.tocs[book_id]["%.0f" % (section - 1)]
except:
a_mi.location = "Section %d" % section
if self.collect_news_clippings and 'News' in self.get_genres(books_db, book_id):
a_mi.location_sort = a_mi.last_modification
else:
a_mi.location_sort = self._generate_location_sort(a_mi.epubcfi)
else:
if self.collect_news_clippings and 'News' in self.get_genres(books_db, book_id):
a_mi.location = self.get_title(books_db, book_id)
a_mi.location_sort = a_mi.last_modification
a_mi.note_text = note_text
# Add annotation
self.add_to_annotations_db(cached_db, a_mi)
# Update last_annotation in books_db
self.update_book_last_annotation(books_db,
row[b'ZANNOTATIONMODIFICATIONDATE'] + self.NSTimeIntervalSince1970,
book_id)
self.update_timestamp(cached_db)
self.commit()
else:
self._log(" retrieving cached annotations from %s" % cached_db)
def capture_content(self, uas, book_id, transient_db):
'''
Store a set of annotations to the transient table
'''
self.create_annotations_transient_table(transient_db)
self._log_location(book_id, uas)
for ua in uas:
if isinstance(ua, NavigableString):
continue
if ua.name != 'div' or ua['class'] != "annotation":
continue
this_ua = AnnotationStruct()
this_ua.book_id = book_id
this_ua.hash = ua['hash']
try:
this_ua.genre = ua['genre']
except:
this_ua.genre = None
try:
this_ua.highlight_color = ua.find('table')['color']
except:
this_ua.highlight_color = 'gray'
try:
this_ua.reader = ua['reader']
except:
this_ua.reader = ''
try:
this_ua.last_modification = ua.find('td', 'timestamp')['uts']
except:
this_ua.last_modification = "0"
try:
this_ua.location = ua.find('td', 'location').string
except:
this_ua.location = ""
try:
this_ua.location_sort = ua['location_sort']
except:
this_ua.location_sort = ""
try:
pels = ua.findAll('p', 'highlight')
self._log_location(book_id, "highlight pels={0}".format(pels))
this_ua.highlight_text = '\n'.join([p.string for p in pels])
self._log_location(book_id, "highlight - this_ua.highlight_text={0}".format(this_ua.highlight_text))
except:
pass
try:
nels = ua.findAll('p', 'note')
self._log_location(book_id, "note nels={0}".format(nels))
this_ua.note_text = '\n'.join([n.string for n in nels])
self._log_location(book_id, "highlight - this_ua.note_text={0}".format(this_ua.note_text))
except:
pass
self.add_to_transient_db(transient_db, this_ua)
def capture_content(self, uas, book_id, transient_db):
'''
Store a set of annotations to the transient table
'''
self.create_annotations_transient_table(transient_db)
self._log_location(book_id, uas)
annotation_list = []
for ua in uas:
self._log_location(book_id, ua)
if isinstance(ua, NavigableString):
continue
if ua.name != 'div' or ua['class'][0] != "annotation":
continue
this_ua = AnnotationStruct()
this_ua.book_id = book_id
this_ua.hash = ua['hash']
try:
this_ua.genre = ua['genre']
except:
this_ua.genre = None
try:
this_ua.highlight_color = ua.find('table')['color']
except:
this_ua.highlight_color = 'gray'
try:
this_ua.reader = ua['reader']
except:
this_ua.reader = ''
try:
this_ua.last_modification = ua.find('td', 'timestamp')['uts']
except:
this_ua.last_modification = "0"
try:
this_ua.location = ua.find('td', 'location').string
except:
this_ua.location = ""
try:
this_ua.location_sort = ua['location_sort']
except:
this_ua.location_sort = ""
try:
pels = ua.findAll('p', 'highlight')
self._log_location(book_id, "highlight pels={0}".format(pels))
this_ua.highlight_text = '\n'.join([p.string or '' for p in pels])
self._log_location(book_id, "highlight - this_ua.highlight_text={0}".format(this_ua.highlight_text))
except:
pass
try:
nels = ua.findAll('p', 'note')
self._log_location(book_id, "note nels={0}".format(nels))
this_ua.note_text = '\n'.join([n.string or '' for n in nels])
self._log_location(book_id, "highlight - this_ua.note_text={0}".format(this_ua.note_text))
except:
pass
self._log_location(book_id, this_ua)
annotation_list.append(this_ua)
return annotation_list
def get_active_annotations(self):
'''
For each annotation, construct an AnnotationStruct object with the
highlight's metadata. Starred items are minimally required. Dashed items
(highlight_text and note_text) may be one or both.
AnnotationStruct properties:
annotation_id: an int uniquely identifying the annotation
*book_id: The book this annotation is associated with
highlight_color: [Blue|Gray|Green|Pink|Purple|Underline|Yellow]
-highlight_text: A list of paragraphs constituting the highlight
last_modification: The timestamp of the annotation
location: location of highlight in the book
-note_text: A list of paragraphs constituting the note
*timestamp: Unique timestamp of highlight's creation/modification time
'''
self._log("%s:get_active_annotations()" % self.app_name)
self.opts.pb.set_label("Getting active annotations for %s" %
self.app_name)
self.opts.pb.set_value(0)
db_profile = self._localize_database_path(self.app_id,
self.annotations_subpath)
self.annotations_db = db_profile['path']
# Test timestamp against cached value
cached_db = self.generate_annotations_db_name(self.app_name_,
self.ios.device_name)
books_db = self.generate_books_db_name(self.app_name_,
self.ios.device_name)
if self.opts.disable_caching or not self._cache_is_current(
db_profile['stats'], cached_db):
self._log(" fetching annotations from %s on %s" %
(self.app_name, self.ios.device_name))
# Create the annotations table as needed
self.create_annotations_table(cached_db)
con = sqlite3.connect(self.annotations_db)
with con:
con.row_factory = sqlite3.Row
cur = con.cursor()
cur.execute('''SELECT
book_oid,
last_access,
book_annotations.location,
book_annotations.book_position,
note,
book_annotations.oid as ba_oid
FROM book_annotations
JOIN book ON book.oid = book_annotations.book_oid
ORDER BY book_annotations.book_position
''')
rows = cur.fetchall()
self.opts.pb.set_maximum(len(rows))
annotations = {}
timestamp = None
for row in rows:
self.opts.pb.increment()
book_id = row[b'book_oid']
if not book_id in self.installed_books:
continue
# Annotations are quoted. Anything afterwards is a note.
# Assuming that the user hasn't edited the opening/closing quotes,
# we can assume that a sequence of '"\n' is a valid split point.
full_annotation = row[b'note']
highlight_text = None
note_text = None
if full_annotation.startswith(
'"') and full_annotation.endswith('"'):
# Highlight only - strip opening/closing quotes
highlight_text = [full_annotation[1:-1]]
elif '"\n' in full_annotation:
# Presumed to be a hybrid highlight/note, separated by closing quote/LF
tokens = full_annotation.split('"\n')
highlight_text = [tokens[0][1:]]
note_text = tokens[1].split('\n')
else:
# User manually removed the quotes, assume it's just a note
note_text = full_annotation.split('\n')
# Populate an AnnotationStruct
a_mi = AnnotationStruct()
a_mi.annotation_id = row[b'ba_oid']
a_mi.book_id = book_id
a_mi.epubcfi = row[b'location']
a_mi.highlight_color = 'Yellow'
if highlight_text:
a_mi.highlight_text = '\n'.join(highlight_text)
if note_text:
a_mi.note_text = '\n'.join(note_text)
section = self._get_spine_index(a_mi.epubcfi)
try:
a_mi.location = self.tocs[book_id]["%.0f" % (section)]
except:
a_mi.location = "Section %d" % section
a_mi.location_sort = row[b'book_position']
# Stanza doesn't timestamp individual annotations
# Space them 1 second apart
timestamp = row[b'last_access']
while timestamp in annotations:
timestamp += 1
a_mi.last_modification = timestamp + self.NSTimeIntervalSince1970
annotations[timestamp] = a_mi
for timestamp in annotations:
self.add_to_annotations_db(cached_db,
annotations[timestamp])
# Update last_annotation in books_db
if timestamp:
self.update_book_last_annotation(books_db, timestamp,
book_id)
self.update_timestamp(cached_db)
self.commit()
else:
self._log(" retrieving cached annotations from %s" % cached_db)
def get_active_annotations(self):
'''
For each annotation, construct an AnnotationStruct object with the
highlight's metadata. Starred items are minimally required. Dashed items
(highlight_text and note_text) may be one or both.
AnnotationStruct properties:
annotation_id: an int uniquely identifying the annotation
*book_id: The book this annotation is associated with
highlight_color: [Blue|Gray|Green|Pink|Purple|Underline|Yellow]
-highlight_text: A list of paragraphs constituting the highlight
last_modification: The timestamp of the annotation
location: location of highlight in the book
-note_text: A list of paragraphs constituting the note
*timestamp: Unique timestamp of highlight's creation/modification time
'''
self._log_location("Start!!!!")
self._log("%s:get_active_annotations()" % self.app_name)
self.opts.pb.set_label("Getting active annotations for %s" % self.app_name)
self.opts.pb.set_value(0)
annotations_db = self.generate_annotations_db_name(self.app_name_, self.opts.device_name)
self.books_db = self.generate_books_db_name(self.app_name_, self.opts.device_name)
self._log("%s:get_active_annotations() - annotations_db=%s, books_db=%s" % (self.app_name, annotations_db, self.books_db))
# Create the annotations table
self.create_annotations_table(annotations_db)
self._fetch_annotations()
# Initialize the progress bar
self.opts.pb.set_label("Getting highlights from %s" % self.app_name)
self.opts.pb.set_value(0)
self.opts.pb.set_maximum(len(self.active_annotations))
# self._log("%s:get_active_annotations() - self.active_annotations={0}".format(self.active_annotations))
# Add annotations to the database
for annotation in sorted(list(self.active_annotations.values()), key=lambda k: (k['book_id'], k['location_sort'], k['last_modification'])):
# Populate an AnnotationStruct with available data
ann_mi = AnnotationStruct()
# Required items
ann_mi.book_id = annotation['book_id']
ann_mi.last_modification = annotation['last_modification']
# Optional items
if 'annotation_id' in annotation:
ann_mi.annotation_id = annotation['annotation_id']
if 'highlight_color' in annotation:
ann_mi.highlight_color = annotation['highlight_color']
if 'highlight_text' in annotation:
# self._log("get_active_annotations() - annotation['highlight_text']={0}".format(annotation['highlight_text']))
highlight_text = annotation['highlight_text']
ann_mi.highlight_text = highlight_text
if 'note_text' in annotation:
note_text = annotation['note_text']
ann_mi.note_text = note_text
if 'location' in annotation:
ann_mi.location = annotation['location']
if 'location_sort' in annotation:
ann_mi.location_sort = annotation['location_sort']
# self._log(ann_mi)
# Add annotation to annotations_db
self.add_to_annotations_db(annotations_db, ann_mi)
# Increment the progress bar
self.opts.pb.increment()
# self._log("%s:get_active_annotations() - books_db=%s" % (self.app_name, self.books_db))
# Update last_annotation in books_db
self.update_book_last_annotation(self.books_db, ann_mi.last_modification, ann_mi.book_id)
# Update the timestamp
self.update_timestamp(annotations_db)
self.commit()
self._log_location("Finish!!!!")
def get_active_annotations(self):
'''
For each annotation, construct an AnnotationStruct object with the
highlight's metadata. Starred items are minimally required. Dashed items
(highlight_text and note_text) may be one or both.
AnnotationStruct properties:
annotation_id: an int uniquely identifying the annotation
*book_id: The book this annotation is associated with
highlight_color: [Blue|Gray|Green|Pink|Purple|Underline|Yellow]
-highlight_text: A list of paragraphs constituting the highlight
last_modification: The timestamp of the annotation
location: location of highlight in the book
-note_text: A list of paragraphs constituting the note
*timestamp: Unique timestamp of highlight's creation/modification time
'''
self._log_location("Start!!!!")
self._log("%s:get_active_annotations()" % self.app_name)
self.opts.pb.set_label("Getting active annotations for %s" % self.app_name)
self.opts.pb.set_value(0)
annotations_db = self.generate_annotations_db_name(self.app_name_, self.opts.device_name)
self.books_db = self.generate_books_db_name(self.app_name_, self.opts.device_name)
self._log("%s:get_active_annotations() - annotations_db=%s, books_db=%s" % (self.app_name, annotations_db, self.books_db))
# Create the annotations table
self.create_annotations_table(annotations_db)
self._fetch_annotations()
# Initialize the progress bar
self.opts.pb.set_label("Getting highlights from %s" % self.app_name)
self.opts.pb.set_value(0)
self.opts.pb.set_maximum(len(self.active_annotations))
# Add annotations to the database
for annotation_id in sorted(self.active_annotations.iterkeys()):
# Populate an AnnotationStruct with available data
ann_mi = AnnotationStruct()
# Required items
ann_mi.book_id = self.active_annotations[annotation_id]['book_id']
ann_mi.last_modification = self.active_annotations[annotation_id]['last_modification']
# Optional items
if 'annotation_id' in self.active_annotations[annotation_id]:
ann_mi.annotation_id = self.active_annotations[annotation_id]['annotation_id']
if 'highlight_color' in self.active_annotations[annotation_id]:
ann_mi.highlight_color = self.active_annotations[annotation_id]['highlight_color']
if 'highlight_text' in self.active_annotations[annotation_id]:
self._log("get_active_annotations() - self.active_annotations[annotation_id]['highlight_text']={0}".format(self.active_annotations[annotation_id]['highlight_text']))
highlight_text = self.active_annotations[annotation_id]['highlight_text']
ann_mi.highlight_text = highlight_text
if 'note_text' in self.active_annotations[annotation_id]:
note_text = self.active_annotations[annotation_id]['note_text']
ann_mi.note_text = note_text
if 'location' in self.active_annotations[annotation_id]:
ann_mi.location = self.active_annotations[annotation_id]['location']
if 'location_sort' in self.active_annotations[annotation_id]:
ann_mi.location_sort = self.active_annotations[annotation_id]['location_sort']
# self._log(ann_mi)
# Add annotation to annotations_db
self.add_to_annotations_db(annotations_db, ann_mi)
# Increment the progress bar
self.opts.pb.increment()
self._log("%s:get_active_annotations() - books_db=%s" % (self.app_name, self.books_db))
# Update last_annotation in books_db
self.update_book_last_annotation(self.books_db, ann_mi.last_modification, ann_mi.book_id)
# Update the timestamp
self.update_timestamp(annotations_db)
self.commit()
self._log_location("Finish!!!!")
def parse_exported_highlights(self, raw, log_failure=True):
"""
Extract highlights from pasted Annotation summary email
Return True if no problems
Return False if error
"""
# Create the annotations, books table as needed
self.annotations_db = "%s_imported_annotations" % self.app_name_
self.create_annotations_table(self.annotations_db)
self.books_db = "%s_imported_books" % self.app_name_
self.create_books_table(self.books_db)
self.annotated_book_list = []
self.selected_books = None
self._log("raw highlights: {0}".format(raw))
# Generate the book metadata from the selected book
row = self.opts.gui.library_view.currentIndex()
book_id = self.opts.gui.library_view.model().id(row)
db = self.opts.gui.current_db
mi = db.get_metadata(book_id, index_is_id=True)
# Grab the title from the front of raw
try:
title = re.match(r'(?m)File: (?P<title>.*)$', raw).group('title')
self._log("title='{0}".format(title))
# Populate a BookStruct
book_mi = BookStruct()
book_mi.active = True
book_mi.author = 'Unknown'
book_mi.book_id = mi.id
book_mi.title = title
book_mi.uuid = None
book_mi.last_update = time.mktime(time.localtime())
book_mi.reader_app = self.app_name
book_mi.cid = mi.id
gr_annotations = raw.split('\n')
num_lines = len(gr_annotations)
highlights = {}
# Find the first annotation
i = 0
line = gr_annotations[i]
self._log("Looking for Page: Line number={0} line='{1}'".format(
i, line))
while not line.startswith('--- Page'):
self._log(" unable to parse GoodReader Annotation summary")
i += 1
line = gr_annotations[i]
self._log(
"Looking for Page: Line number={0} line='{1}'".format(
i, line))
while i < num_lines and not line.startswith(
'(report generated by GoodReader)'):
# Extract the page number
page_num = re.search('--- (Page \w+) ---', line)
self._log("regex result: page_num={0}".format(page_num))
if page_num:
page_num = page_num.group(1)
self._log("page_num={0}".format(page_num))
# Extract the highlight
i += 1
line = gr_annotations[i]
self._log(
"Looking for annotation start: Line number={0} line='{1}'"
.format(i, line))
prefix = None
while True:
prefix = re.search(
'^(?P<ann_type>{0})'.format(
'|'.join(self.ANNOTATION_TYPES +
self.SKIP_TYPES)), line)
self._log("Searched for prefix={0}".format(prefix))
if prefix and prefix.group(
'ann_type') in self.SKIP_TYPES:
i += 1
line = gr_annotations[i]
self._log(
"Looking for annotation start: Line number={0} line='{1}'"
.format(i, line))
while not re.search(
'^(?P<ann_type>{0})'.format('|'.join(
self.ANNOTATION_TYPES)), line):
i += 1
line = gr_annotations[i]
self._log(
"Looking for annotation start after a SKIP type: Line number={0} line='{1}'"
.format(i, line))
continue
elif prefix:
self._log(
"Have annotation start: Line number={0} line='{1}' prefix={2}"
.format(i, line, prefix))
break
else:
i += 1
line = gr_annotations[i]
self._log(
"Looking for annotation start 2: Line number={0} line='{1}'"
.format(i, line))
annotation = self._extract_highlight(
line, prefix.group('ann_type'))
annotation.page_num = page_num
self._log(
"Started annotation: page_num={0} annotation='{1}'".
format(page_num, annotation))
# Get the annotation(s)
i += 1
line = gr_annotations[i]
self._log(
"Reading annotation text 1: Line number={0} line='{1}'"
.format(i, line))
ann = ''
while i < num_lines \
and not line.startswith('--- Page') \
and not line.startswith('(report generated by GoodReader)'):
if line:
prefix = re.search(
'^(?P<ann_type>{0})'.format(
'|'.join(self.ANNOTATION_TYPES +
self.SKIP_TYPES)), line)
if prefix and prefix.group(
'ann_type') in self.SKIP_TYPES:
# Continue until next ann_type
i += 1
line = gr_annotations[i]
while not re.search(
'^(?P<ann_type>{0})'.format('|'.join(
self.ANNOTATION_TYPES)), line):
i += 1
if i == num_lines:
break
line = gr_annotations[i]
continue
elif prefix:
# Additional highlight on the same page
# write current annotation, start new annotation
self._store_annotation(highlights, annotation)
annotation = self._extract_highlight(
line, prefix.group('ann_type'))
annotation.page_num = page_num
annotation.ann_type = prefix.group('ann_type')
ann = ''
i += 1
line = gr_annotations[i]
continue
if not ann:
ann = line
else:
ann += '\n' + line
i += 1
line = gr_annotations[i]
annotation.ann = ann
# Back up so that the next line is '--- Page' or '(report generated'
i -= 1
self._store_annotation(highlights, annotation)
i += 1
if i == num_lines:
break
line = gr_annotations[i]
except Exception as e:
import traceback
self._log("Exception parsing GoodReader Annotation summary: %s" %
e)
traceback.print_exc()
if log_failure:
self._log(" unable to parse GoodReader Annotation summary")
self._log("{:~^80}".format(" Imported Annotation summary "))
self._log(raw)
self._log(
"{:~^80}".format(" end imported Annotations summary "))
import traceback
traceback.print_exc()
msg = ('Unable to parse Annotation summary from %s. ' %
self.app_name +
'Paste entire contents of emailed summary.')
MessageBox(MessageBox.WARNING,
'Error importing annotations',
msg,
show_copy_button=False,
parent=self.opts.gui).exec_()
self._log_location("WARNING: %s" % msg)
return False
# Finalize book_mi
book_mi.annotations = len(highlights)
# Add book to books_db
self.add_to_books_db(self.books_db, book_mi)
self.annotated_book_list.append(book_mi)
sorted_keys = sorted(list(highlights.keys()))
for dt in sorted_keys:
highlight_text = None
if 'text' in highlights[dt]:
highlight_text = highlights[dt]['text']
note_text = None
if 'note' in highlights[dt]:
note_text = highlights[dt]['note']
# Populate an AnnotationStruct
a_mi = AnnotationStruct()
a_mi.annotation_id = dt
a_mi.book_id = book_mi['book_id']
a_mi.highlight_color = highlights[dt]['color']
a_mi.highlight_text = highlight_text
a_mi.location = highlights[dt]['page']
a_mi.last_modification = dt
a_mi.note_text = note_text
# Location sort
page_literal = re.match(r'^Page (?P<page>[0-9ivx]+).*$',
a_mi.location).group('page')
if re.match('[IXVL]', page_literal.upper()):
whole = 0
decimal = self._roman_to_int(page_literal)
else:
whole = int(page_literal)
decimal = 0
a_mi.location_sort = "%05d.%05d" % (whole, decimal)
# Add annotation
self.add_to_annotations_db(self.annotations_db, a_mi)
self.update_book_last_annotation(self.books_db, dt,
book_mi['book_id'])
# Update the timestamp
self.update_timestamp(self.annotations_db)
self.update_timestamp(self.books_db)
self.commit()
return True