def add_evernote_notes(self, evernote_notes, update=False, log_update_if_unchanged=True): """ Add Notes to or Update Notes in Anki Database :param evernote_notes: :param update: :param log_update_if_unchanged: :type evernote_notes: list[EvernoteNotePrototype.EvernoteNotePrototype] :type update: bool :return: Count of notes successfully added or updated """ new_nids=[] action_str_base = ['Add', 'Update'][update] action_str = ['Adding', 'Updating'][update] action_preposition = ['To', 'In'][update] info = stopwatch.ActionInfo(action_str + ' Of', 'Evernote Notes', action_preposition + ' Anki', report_if_empty=False) tmr = stopwatch.Timer(evernote_notes, 10, info=info, label='Add\\Anki-%sEvernoteNotes' % action_str_base) for ankiNote in evernote_notes: try: title = ankiNote.FullTitle content = decode(ankiNote.Content) anki_field_info = { FIELDS.TITLE: title, FIELDS.CONTENT: content, FIELDS.EVERNOTE_GUID: FIELDS.EVERNOTE_GUID_PREFIX + ankiNote.Guid, FIELDS.UPDATE_SEQUENCE_NUM: str(ankiNote.UpdateSequenceNum), FIELDS.SEE_ALSO: u'' } except Exception: log_error("Unable to set field info for: Note '%s': '%s'" % (ankiNote.FullTitle, ankiNote.Guid)) log_dump(ankiNote.Content, " NOTE CONTENTS ") # log_dump(encode(ankiNote.Content), " NOTE CONTENTS ") raise tmr.step(title) baseNote = None if update: baseNote = self.get_anki_note_from_evernote_guid(ankiNote.Guid) if not baseNote: log_error('Updating note %s: COULD NOT FIND BASE NOTE FOR ANKI NOTE ID' % ankiNote.Guid) tmr.reportStatus(EvernoteAPIStatus.MissingDataError) continue if ankiNote.Tags is None: log_error("Could note find tags object for note %s: %s. " % (ankiNote.Guid, ankiNote.FullTitle)) tmr.reportStatus(EvernoteAPIStatus.MissingDataError) continue anki_note_prototype = AnkiNotePrototype(self, anki_field_info, ankiNote.TagNames, baseNote, notebookGuid=ankiNote.NotebookGuid, count=tmr.count, count_update=tmr.counts.updated.completed.val, max_count=tmr.max) anki_note_prototype._log_update_if_unchanged_ = log_update_if_unchanged nid = tmr.autoStep(anki_note_prototype.update_note() if update else anki_note_prototype.add_note(), ankiNote.FullTitle, update) if tmr.status.IsSuccess and not update: new_nids.append([nid, ankiNote.Guid]) elif tmr.status.IsError: log("ANKI ERROR WHILE %s EVERNOTE NOTES: " % action_str.upper() + str(tmr.status), tmr.label + '-Error') tmr.Report() if new_nids: ankDB().executemany("UPDATE {n} SET nid = ? WHERE guid = ?", new_nids) return tmr.counts.success
def process_see_also_content(self, anki_note_ids): log = Logger('See Also\\1-process_unadded_see_also_notes\\', rm_path=True) tmr = stopwatch.Timer(anki_note_ids, infoStr='Processing Unadded See Also Notes', label=log.base_path) tmr.info.BannerHeader('error') for a_id in anki_note_ids: ankiNote = self.collection().getNote(a_id) try: items = ankiNote.items() except Exception: log.error("Unable to get note items for Note ID: %d for %s" % (a_id, tmr.base_name)) raise fields = {} for key, value in items: fields[key] = value if fields[FIELDS.SEE_ALSO]: tmr.reportSkipped() continue anki_note_prototype = AnkiNotePrototype(self, fields, ankiNote.tags, ankiNote, count=tmr.count, count_update=tmr.counts.updated.completed.val, max_count=tmr.max, light_processing=True) if not anki_note_prototype.Fields[FIELDS.SEE_ALSO]: tmr.reportSkipped() continue log.go("Detected see also contents for Note '%s': %s" % ( get_evernote_guid_from_anki_fields(fields), fields[FIELDS.TITLE])) log.go(u" ::: %s " % strip_tags_and_new_lines(fields[FIELDS.SEE_ALSO])) tmr.autoStep(anki_note_prototype.update_note(), fields[FIELDS.TITLE], update=True)
def process_see_also_content(self, anki_note_ids): log = Logger('See Also\\1-process_unadded_see_also_notes\\', rm_path=True) tmr = stopwatch.Timer(anki_note_ids, infoStr='Processing Unadded See Also Notes', label=log.base_path) tmr.info.BannerHeader('error') for a_id in anki_note_ids: ankiNote = self.collection().getNote(a_id) try: items = ankiNote.items() except Exception: log.error("Unable to get note items for Note ID: %d for %s" % (a_id, tmr.base_name)) raise fields = {} for key, value in items: fields[key] = value if fields[FIELDS.SEE_ALSO]: tmr.reportSkipped() continue anki_note_prototype = AnkiNotePrototype( self, fields, ankiNote.tags, ankiNote, count=tmr.count, count_update=tmr.counts.updated.completed.val, max_count=tmr.max, light_processing=True) if not anki_note_prototype.Fields[FIELDS.SEE_ALSO]: tmr.reportSkipped() continue log.go("Detected see also contents for Note '%s': %s" % (get_evernote_guid_from_anki_fields(fields), fields[FIELDS.TITLE])) log.go(u" ::: %s " % strip_tags_and_new_lines(fields[FIELDS.SEE_ALSO])) tmr.autoStep(anki_note_prototype.update_note(), fields[FIELDS.TITLE], update=True)
def test_anki(self, title, evernote_guid, filename=""): if not filename: filename = title fields = { FIELDS.TITLE: title, FIELDS.CONTENT: file( os.path.join(FOLDERS.LOGS, filename.replace('.enex', '') + ".enex"), 'r').read(), FIELDS.EVERNOTE_GUID: FIELDS.EVERNOTE_GUID_PREFIX + evernote_guid } tags = ['NoTags', 'NoTagsToRemove'] return AnkiNotePrototype(self.anki, fields, tags)
def add_evernote_notes(self, evernote_notes, update=False, log_update_if_unchanged=True): """ Add Notes to or Update Notes in Anki Database :param evernote_notes: :param update: :param log_update_if_unchanged: :type evernote_notes: list[EvernoteNotePrototype.EvernoteNotePrototype] :type update: bool :return: Count of notes successfully added or updated """ new_nids = [] action_str_base = ['Add', 'Update'][update] action_str = ['Adding', 'Updating'][update] action_preposition = ['To', 'In'][update] info = stopwatch.ActionInfo(action_str + ' Of', 'Evernote Notes', action_preposition + ' Anki', report_if_empty=False) tmr = stopwatch.Timer(evernote_notes, 10, info=info, label='Add\\Anki-%sEvernoteNotes' % action_str_base) for ankiNote in evernote_notes: try: title = ankiNote.FullTitle content = decode(ankiNote.Content) anki_field_info = { FIELDS.TITLE: title, FIELDS.CONTENT: content, FIELDS.EVERNOTE_GUID: FIELDS.EVERNOTE_GUID_PREFIX + ankiNote.Guid, FIELDS.UPDATE_SEQUENCE_NUM: str(ankiNote.UpdateSequenceNum), FIELDS.SEE_ALSO: u'' } except Exception: log_error("Unable to set field info for: Note '%s': '%s'" % (ankiNote.FullTitle, ankiNote.Guid)) log_dump(ankiNote.Content, " NOTE CONTENTS ") # log_dump(encode(ankiNote.Content), " NOTE CONTENTS ") raise tmr.step(title) baseNote = None if update: baseNote = self.get_anki_note_from_evernote_guid(ankiNote.Guid) if not baseNote: log_error( 'Updating note %s: COULD NOT FIND BASE NOTE FOR ANKI NOTE ID' % ankiNote.Guid) tmr.reportStatus(EvernoteAPIStatus.MissingDataError) continue if ankiNote.Tags is None: log_error("Could note find tags object for note %s: %s. " % (ankiNote.Guid, ankiNote.FullTitle)) tmr.reportStatus(EvernoteAPIStatus.MissingDataError) continue anki_note_prototype = AnkiNotePrototype( self, anki_field_info, ankiNote.TagNames, baseNote, notebookGuid=ankiNote.NotebookGuid, count=tmr.count, count_update=tmr.counts.updated.completed.val, max_count=tmr.max) anki_note_prototype._log_update_if_unchanged_ = log_update_if_unchanged nid = tmr.autoStep( anki_note_prototype.update_note() if update else anki_note_prototype.add_note(), ankiNote.FullTitle, update) if tmr.status.IsSuccess and not update: new_nids.append([nid, ankiNote.Guid]) elif tmr.status.IsError: log( "ANKI ERROR WHILE %s EVERNOTE NOTES: " % action_str.upper() + str(tmr.status), tmr.label + '-Error') tmr.Report() if new_nids: ankDB().executemany("UPDATE {n} SET nid = ? WHERE guid = ?", new_nids) return tmr.counts.success
def insert_toc_into_see_also(self): db = ankDB() db._db.row_factory = None results = db.all( "SELECT s.target_evernote_guid, s.source_evernote_guid, target_note.title, toc_note.title " "FROM {s} as s, {n} as target_note, {n} as toc_note " "WHERE s.source_evernote_guid != s.target_evernote_guid AND target_note.guid = s.target_evernote_guid " "AND toc_note.guid = s.source_evernote_guid AND s.from_toc == 1 " "ORDER BY target_note.title ASC") # results_bad = db.all( # "SELECT s.target_evernote_guid, s.source_evernote_guid FROM {t_see} as s WHERE s.source_evernote_guid COUNT(SELECT * FROM {tn} WHERE guid = s.source_evernote_guid) )" % ( # TABLES.SEE_ALSO, TABLES.EVERNOTE.NOTES, TABLES.EVERNOTE.NOTES)) all_child_guids = db.list("tagNames NOT LIKE '{t_toc}'", columns='guid') all_toc_guids = db.list("tagNames LIKE '{t_toc}'", columns='guid') grouped_results = {} toc_titles = {} for row in results: target_guid = row[0] toc_guid = row[1] if toc_guid not in all_toc_guids: continue if target_guid not in all_toc_guids and target_guid not in all_child_guids: continue if target_guid not in grouped_results: grouped_results[target_guid] = [row[2], []] toc_titles[toc_guid] = row[3] grouped_results[target_guid][1].append(toc_guid) action_title = 'INSERT TOCS INTO ANKI NOTES' info = stopwatch.ActionInfo('Inserting TOC Links into', 'Anki Notes', 'Anki Notes\' See Only Field') log = Logger('See Also\\5-insert_toc_links_into_see_also\\', rm_path=True) tmr = stopwatch.Timer(len(grouped_results), info=info, label=log.base_path) tmr.info.BannerHeader('new', crosspost=['invalid', 'error']) toc_separator = generate_evernote_span(u' | ', u'Links', u'See Also', bold=False) log.add(' <h1>%s: %d TOTAL NOTES</h1> <HR><BR><BR>' % (action_title, tmr.max), 'see_also_html', timestamp=False, clear=True, extension='htm') logged_missing_anki_note = False sorted_results = sorted(grouped_results.items(), key=lambda s: s[1][0]) for target_guid, target_guid_info in sorted_results: note_title, toc_guids = target_guid_info ankiNote = self.get_anki_note_from_evernote_guid(target_guid) # if tmr.step(): # log.add("INSERTING TOC LINKS INTO NOTE %5s: %s: %s" % ('#' + str(tmr.count), tmr.progress, note_title), # 'progress') if not ankiNote: log.dump(toc_guids, 'Missing Anki Note for ' + target_guid, tmr.label, timestamp=False, crosspost_to_default=False) if not logged_missing_anki_note: log.error( '%s: Missing Anki Note(s) for TOC entry. See %s dump log for more details' % (action_title, tmr.label)) logged_missing_anki_note = True tmr.reportStatus(EvernoteAPIStatus.NotFoundError, title=note_title) continue fields = get_dict_from_list(ankiNote.items()) see_also_html = fields[FIELDS.SEE_ALSO] content_links = find_evernote_links_as_guids( fields[FIELDS.CONTENT]) see_also_whole_links = find_evernote_links(see_also_html) see_also_links = {x.Guid for x in see_also_whole_links} invalid_see_also_links = { x for x in see_also_links if x not in all_child_guids and x not in all_toc_guids } new_tocs = set(toc_guids) - see_also_links if TAGS.TOC_AUTO in ankiNote.tags: new_tocs -= set(content_links) log.dump([ new_tocs, toc_guids, invalid_see_also_links, see_also_links, content_links ], 'TOCs for %s' % fields[FIELDS.TITLE] + ' vs ' + note_title, 'new_tocs', crosspost_to_default=False) new_toc_count = len(new_tocs) invalid_see_also_links_count = len(invalid_see_also_links) if invalid_see_also_links_count > 0: for link in see_also_whole_links: if link.Guid in invalid_see_also_links: see_also_html = remove_evernote_link( link, see_also_html) see_also_links -= invalid_see_also_links see_also_count = len(see_also_links) if new_toc_count > 0: has_ol = u'<ol' in see_also_html has_ul = u'<ul' in see_also_html has_list = has_ol or has_ul see_also_new = " " flat_links = (new_toc_count + see_also_count < 3 and not has_list) toc_delimiter = u' ' if see_also_count is 0 else toc_separator for toc_guid in toc_guids: toc_title = toc_titles[toc_guid] if flat_links: toc_title = u'[%s]' % toc_title toc_link = generate_evernote_link(toc_guid, toc_title, value='TOC') see_also_new += (toc_delimiter + toc_link) if flat_links else ( u'\n<li>%s</li>' % toc_link) toc_delimiter = toc_separator if flat_links: find_div_end = see_also_html.rfind('</div>') if find_div_end > -1: see_also_html = see_also_html[: find_div_end] + see_also_new + '\n' + see_also_html[ find_div_end:] see_also_new = '' else: see_also_toc_headers = { 'ol': u'<br><div style="margin-top:5px;">\n%s</div><ol style="margin-top:3px;">' % generate_evernote_span('<u>TABLE OF CONTENTS</u>:', 'Levels', 'Auto TOC', escape=False) } see_also_toc_headers['ul'] = see_also_toc_headers[ 'ol'].replace('<ol ', '<ul ') if see_also_toc_headers['ul'] in see_also_html: find_ul_end = see_also_html.rfind('</ul>') see_also_html = see_also_html[: find_ul_end] + '</ol>' + see_also_html[ find_ul_end + 5:] see_also_html = see_also_html.replace( see_also_toc_headers['ul'], see_also_toc_headers['ol']) if see_also_toc_headers['ol'] in see_also_html: find_ol_end = see_also_html.rfind('</ol>') see_also_html = see_also_html[: find_ol_end] + see_also_new + '\n' + see_also_html[ find_ol_end:] see_also_new = '' else: header_type = 'ul' if new_toc_count is 1 else 'ol' see_also_new = see_also_toc_headers[ header_type] + u'%s\n</%s>' % (see_also_new, header_type) if see_also_count == 0: see_also_html = generate_evernote_span( u'See Also:', 'Links', 'See Also') see_also_html += see_also_new see_also_html = see_also_html.replace( '<ol>', '<ol style="margin-top:3px;">') log.add( '<h3>%s</h3><br>' % generate_evernote_span(fields[FIELDS.TITLE], 'Links', 'TOC') + see_also_html + u'<HR>', 'see_also_html', crosspost='see_also_html\\' + note_title, timestamp=False, extension='htm') see_also_html = see_also_html.replace('evernote:///', 'evernote://') changed = see_also_html != fields[FIELDS.SEE_ALSO] fields[FIELDS.SEE_ALSO] = see_also_html anki_note_prototype = AnkiNotePrototype( self, fields, ankiNote.tags, ankiNote, count=tmr.counts.handled, count_update=tmr.counts.updated.completed.val, max_count=tmr.max, light_processing=True, steps=[0, 1, 7]) anki_note_prototype._log_update_if_unchanged_ = ( changed or new_toc_count + invalid_see_also_links_count > 0) tmr.autoStep( anki_note_prototype.update_note(error_if_unchanged=changed), note_title, True) crosspost = [] if new_toc_count: crosspost.append('new') if invalid_see_also_links: crosspost.append('invalid') if tmr.status.IsError: crosspost.append('error') log.go(' %s | %2d TOTAL TOC' 's | %s | %s | %s%s' % (format_count('%2d NEW TOC' 's', new_toc_count), len(toc_guids), format_count('%2d EXISTING LINKS', see_also_count), format_count('%2d INVALID LINKS', invalid_see_also_links_count), ('*' if changed else ' ') * 3, note_title), crosspost=crosspost, timestamp=False) db._db.row_factory = sqlite.Row
def insert_toc_into_see_also(self): db = ankDB() db._db.row_factory = None results = db.all( "SELECT s.target_evernote_guid, s.source_evernote_guid, target_note.title, toc_note.title " "FROM {s} as s, {n} as target_note, {n} as toc_note " "WHERE s.source_evernote_guid != s.target_evernote_guid AND target_note.guid = s.target_evernote_guid " "AND toc_note.guid = s.source_evernote_guid AND s.from_toc == 1 " "ORDER BY target_note.title ASC") # results_bad = db.all( # "SELECT s.target_evernote_guid, s.source_evernote_guid FROM {t_see} as s WHERE s.source_evernote_guid COUNT(SELECT * FROM {tn} WHERE guid = s.source_evernote_guid) )" % ( # TABLES.SEE_ALSO, TABLES.EVERNOTE.NOTES, TABLES.EVERNOTE.NOTES)) all_child_guids = db.list("tagNames NOT LIKE '{t_toc}'", columns='guid') all_toc_guids = db.list("tagNames LIKE '{t_toc}'", columns='guid') grouped_results = {} toc_titles = {} for row in results: target_guid = row[0] toc_guid = row[1] if toc_guid not in all_toc_guids: continue if target_guid not in all_toc_guids and target_guid not in all_child_guids: continue if target_guid not in grouped_results: grouped_results[target_guid] = [row[2], []] toc_titles[toc_guid] = row[3] grouped_results[target_guid][1].append(toc_guid) action_title = 'INSERT TOCS INTO ANKI NOTES' info = stopwatch.ActionInfo('Inserting TOC Links into', 'Anki Notes', 'Anki Notes\' See Only Field') log = Logger('See Also\\5-insert_toc_links_into_see_also\\', rm_path=True) tmr = stopwatch.Timer(len(grouped_results), info=info, label=log.base_path) tmr.info.BannerHeader('new', crosspost=['invalid', 'error']) toc_separator = generate_evernote_span(u' | ', u'Links', u'See Also', bold=False) log.add(' <h1>%s: %d TOTAL NOTES</h1> <HR><BR><BR>' % (action_title, tmr.max), 'see_also_html', timestamp=False, clear=True, extension='htm') logged_missing_anki_note = False sorted_results = sorted(grouped_results.items(), key=lambda s: s[1][0]) for target_guid, target_guid_info in sorted_results: note_title, toc_guids = target_guid_info ankiNote = self.get_anki_note_from_evernote_guid(target_guid) # if tmr.step(): # log.add("INSERTING TOC LINKS INTO NOTE %5s: %s: %s" % ('#' + str(tmr.count), tmr.progress, note_title), # 'progress') if not ankiNote: log.dump(toc_guids, 'Missing Anki Note for ' + target_guid, tmr.label, timestamp=False, crosspost_to_default=False) if not logged_missing_anki_note: log.error('%s: Missing Anki Note(s) for TOC entry. See %s dump log for more details' % (action_title, tmr.label)) logged_missing_anki_note = True tmr.reportStatus(EvernoteAPIStatus.NotFoundError, title=note_title) continue fields = get_dict_from_list(ankiNote.items()) see_also_html = fields[FIELDS.SEE_ALSO] content_links = find_evernote_links_as_guids(fields[FIELDS.CONTENT]) see_also_whole_links = find_evernote_links(see_also_html) see_also_links = {x.Guid for x in see_also_whole_links} invalid_see_also_links = {x for x in see_also_links if x not in all_child_guids and x not in all_toc_guids} new_tocs = set(toc_guids) - see_also_links if TAGS.TOC_AUTO in ankiNote.tags: new_tocs -= set(content_links) log.dump([new_tocs, toc_guids, invalid_see_also_links, see_also_links, content_links], 'TOCs for %s' % fields[FIELDS.TITLE] + ' vs ' + note_title, 'new_tocs', crosspost_to_default=False) new_toc_count = len(new_tocs) invalid_see_also_links_count = len(invalid_see_also_links) if invalid_see_also_links_count > 0: for link in see_also_whole_links: if link.Guid in invalid_see_also_links: see_also_html = remove_evernote_link(link, see_also_html) see_also_links -= invalid_see_also_links see_also_count = len(see_also_links) if new_toc_count > 0: has_ol = u'<ol' in see_also_html has_ul = u'<ul' in see_also_html has_list = has_ol or has_ul see_also_new = " " flat_links = (new_toc_count + see_also_count < 3 and not has_list) toc_delimiter = u' ' if see_also_count is 0 else toc_separator for toc_guid in toc_guids: toc_title = toc_titles[toc_guid] if flat_links: toc_title = u'[%s]' % toc_title toc_link = generate_evernote_link(toc_guid, toc_title, value='TOC') see_also_new += (toc_delimiter + toc_link) if flat_links else (u'\n<li>%s</li>' % toc_link) toc_delimiter = toc_separator if flat_links: find_div_end = see_also_html.rfind('</div>') if find_div_end > -1: see_also_html = see_also_html[:find_div_end] + see_also_new + '\n' + see_also_html[ find_div_end:] see_also_new = '' else: see_also_toc_headers = { 'ol': u'<br><div style="margin-top:5px;">\n%s</div><ol style="margin-top:3px;">' % generate_evernote_span('<u>TABLE OF CONTENTS</u>:', 'Levels', 'Auto TOC', escape=False) } see_also_toc_headers['ul'] = see_also_toc_headers['ol'].replace('<ol ', '<ul ') if see_also_toc_headers['ul'] in see_also_html: find_ul_end = see_also_html.rfind('</ul>') see_also_html = see_also_html[:find_ul_end] + '</ol>' + see_also_html[find_ul_end + 5:] see_also_html = see_also_html.replace(see_also_toc_headers['ul'], see_also_toc_headers['ol']) if see_also_toc_headers['ol'] in see_also_html: find_ol_end = see_also_html.rfind('</ol>') see_also_html = see_also_html[:find_ol_end] + see_also_new + '\n' + see_also_html[find_ol_end:] see_also_new = '' else: header_type = 'ul' if new_toc_count is 1 else 'ol' see_also_new = see_also_toc_headers[header_type] + u'%s\n</%s>' % (see_also_new, header_type) if see_also_count == 0: see_also_html = generate_evernote_span(u'See Also:', 'Links', 'See Also') see_also_html += see_also_new see_also_html = see_also_html.replace('<ol>', '<ol style="margin-top:3px;">') log.add('<h3>%s</h3><br>' % generate_evernote_span(fields[FIELDS.TITLE], 'Links', 'TOC') + see_also_html + u'<HR>', 'see_also_html', crosspost='see_also_html\\' + note_title, timestamp=False, extension='htm') see_also_html = see_also_html.replace('evernote:///', 'evernote://') changed = see_also_html != fields[FIELDS.SEE_ALSO] fields[FIELDS.SEE_ALSO] = see_also_html anki_note_prototype = AnkiNotePrototype(self, fields, ankiNote.tags, ankiNote, count=tmr.counts.handled, count_update=tmr.counts.updated.completed.val, max_count=tmr.max, light_processing=True, steps=[0, 1, 7]) anki_note_prototype._log_update_if_unchanged_ = ( changed or new_toc_count + invalid_see_also_links_count > 0) tmr.autoStep(anki_note_prototype.update_note(error_if_unchanged=changed), note_title, True) crosspost = [] if new_toc_count: crosspost.append('new') if invalid_see_also_links: crosspost.append('invalid') if tmr.status.IsError: crosspost.append('error') log.go(' %s | %2d TOTAL TOC''s | %s | %s | %s%s' % ( format_count('%2d NEW TOC''s', new_toc_count), len(toc_guids), format_count('%2d EXISTING LINKS', see_also_count), format_count('%2d INVALID LINKS', invalid_see_also_links_count), ('*' if changed else ' ') * 3, note_title), crosspost=crosspost, timestamp=False) db._db.row_factory = sqlite.Row