def try_print(full_path, content, prefix='', line_prefix=u'\n ', attempt=0, clear=False, timestamp=True, **kwargs): try: st = '[%s]: ' % datetime.now().strftime(ANKNOTES.DATE_FORMAT) if timestamp else '' print_content = line_prefix + (u' <%d>' % attempt if attempt > 0 else u'') + u' ' + st if attempt is 0: print_content += content elif attempt is 1: print_content += decode(content) elif attempt is 2: print_content += encode(content) elif attempt is 3: print_content = encode(print_content) + encode(content) elif attempt is 4: print_content = decode(print_content) + decode(content) elif attempt is 5: print_content += "Error printing content: " + str_safe(content) elif attempt is 6: print_content += "Error printing content: " + content[:10] elif attempt is 7: print_content += "Unable to print content." with open(full_path, 'w+' if clear else 'a+') as fileLog: print>> fileLog, print_content except Exception as e: if attempt < 8: try_print(full_path, content, prefix=prefix, line_prefix=line_prefix, attempt=attempt + 1, clear=clear) else: log("Try print error to %s: %s" % (os.path.split(full_path)[1], str(e)))
def write_file_contents(content, full_path, clear=False, try_encode=True, do_print=False, print_timestamp=True, print_content=None, wfc_timestamp=True, wfc_crosspost=None, get_log_full_path=None, **kwargs): all_args = locals() if wfc_crosspost: del all_args['kwargs'], all_args['wfc_crosspost'], all_args['content'], all_args['full_path'] all_args.update(kwargs) for cp in item_to_list(wfc_crosspost): write_file_contents(content, cp, **all_args) orig_path = full_path if not os.path.exists(os.path.dirname(full_path)): if callable(get_log_full_path): full_path = get_log_full_path(full_path) if full_path is False: return else: if not filter_logs(full_path): return full_path = os.path.abspath(os.path.join(FOLDERS.LOGS, full_path + '.log')) base_path = os.path.dirname(full_path) if not os.path.exists(base_path): os.makedirs(base_path) if wfc_timestamp: print_content = content content = '[%s]: ' % datetime.now().strftime(ANKNOTES.DATE_FORMAT) + content with open(full_path, 'w+' if clear else 'a+') as fileLog: try: print>> fileLog, content except UnicodeEncodeError: content = encode(content) print>> fileLog, content if do_print: print content if print_timestamp or not print_content else print_content
def encode_log_text(content, encode_text=True, **kwargs): if not encode_text: return content try: return encode(content) except Exception: return content
def try_print(full_path, content, prefix='', line_prefix=u'\n ', attempt=0, clear=False, timestamp=True, **kwargs): try: st = '[%s]: ' % datetime.now().strftime( ANKNOTES.DATE_FORMAT) if timestamp else '' print_content = line_prefix + (u' <%d>' % attempt if attempt > 0 else u'') + u' ' + st if attempt is 0: print_content += content elif attempt is 1: print_content += decode(content) elif attempt is 2: print_content += encode(content) elif attempt is 3: print_content = encode(print_content) + encode(content) elif attempt is 4: print_content = decode(print_content) + decode(content) elif attempt is 5: print_content += "Error printing content: " + str_safe(content) elif attempt is 6: print_content += "Error printing content: " + content[:10] elif attempt is 7: print_content += "Unable to print content." with open(full_path, 'w+' if clear else 'a+') as fileLog: print >> fileLog, print_content except Exception as e: if attempt < 8: try_print(full_path, content, prefix=prefix, line_prefix=line_prefix, attempt=attempt + 1, clear=clear) else: log("Try print error to %s: %s" % (os.path.split(full_path)[1], str(e)))
def get_anknotes_potential_root_titles(upper_case=False, encode=False, **kwargs): global generateTOCTitle from anknotes.EvernoteNoteTitle import generateTOCTitle def mapper(x): return generateTOCTitle(x) if upper_case: mapper = lambda x, f=mapper: f(x).upper() if encode: mapper = lambda x, f=mapper: encode(f(x)) data = get_cached_data(get_anknotes_potential_root_titles, lambda: ankDB().list( "SELECT DISTINCT SUBSTR(title, 0, INSTR(title, ':')) FROM {n} WHERE title LIKE '%:%'")) return map(mapper, data)
def makeNoteBody(content, resources=None, encode=True): ## Build body of note if resources is None: resources = [] nBody = content if not nBody.startswith("<?xml"): nBody = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" nBody += "<!DOCTYPE en-note SYSTEM \"http://xml.evernote.com/pub/enml2.dtd\">" nBody += "<en-note>%s" % content + "</en-note>" if encode: nBody = encode(nBody) return nBody
def extract_links_from_toc(self): db = ankDB(TABLES.SEE_ALSO) db.setrowfactory() toc_entries = db.all("SELECT * FROM {n} WHERE tagNames LIKE '{t_toc}' ORDER BY title ASC") db.execute("DELETE FROM {t} WHERE from_toc = 1") log = Logger('See Also\\4-extract_links_from_toc\\', timestamp=False, crosspost_to_default=False, rm_path=True) tmr = stopwatch.Timer(toc_entries, 20, infoStr='Extracting Links', label=log.base_path) tmr.info.BannerHeader('error') toc_guids = [] for toc_entry in toc_entries: toc_evernote_guid, toc_link_title = toc_entry['guid'], toc_entry['title'] toc_guids.append("'%s'" % toc_evernote_guid) # toc_link_html = generate_evernote_span(toc_link_title, 'Links', 'TOC') enLinks = find_evernote_links(toc_entry['content']) tmr.increment(toc_link_title) for enLink in enLinks: target_evernote_guid = enLink.Guid if not check_evernote_guid_is_valid(target_evernote_guid): log.go("Invalid Target GUID for %-70s %s" % (toc_link_title + ':', target_evernote_guid), 'error') continue base = { 'child_guid': target_evernote_guid, 'uid': enLink.Uid, 'shard': enLink.Shard, 'toc_guid': toc_evernote_guid, 'l1': 'source', 'l2': 'source', 'from_toc': 0, 'is_toc': 0 } query_count = "select COUNT(*) from {t} WHERE source_evernote_guid = '{%s_guid}'" toc = { 'num': 1 + db.scalar(fmt(query_count % 'toc', base)), 'html': enLink.HTML.replace(u'\'', u'\'\''), 'title': enLink.FullTitle.replace(u'\'', u'\'\''), 'l1': 'target', 'from_toc': 1 } # child = {1 + db.scalar(fmt(query_count % 'child', base)), # 'html': toc_link_html.replace(u'\'', u'\'\''), # 'title': toc_link_title.replace(u'\'', u'\'\''), # 'l2': 'target', # 'is_toc': 1 # } query = (u"INSERT OR REPLACE INTO `{t}`(`{l1}_evernote_guid`, `number`, `uid`, `shard`, " u"`{l2}_evernote_guid`, `html`, `title`, `from_toc`, `is_toc`) " u"VALUES('{child_guid}', {num}, {uid}, '{shard}', " u"'{toc_guid}', '{html}', '{title}', {from_toc}, {is_toc})") query_toc = fmt(query, base, toc) db.execute(query_toc) log.go("\t\t - Added %2d child link(s) from TOC %s" % (len(enLinks), encode(toc_link_title))) db.update("is_toc = 1", where="target_evernote_guid IN (%s)" % ', '.join(toc_guids)) db.commit()
def write_file_contents(content, full_path, clear=False, try_encode=True, do_print=False, print_timestamp=True, print_content=None, wfc_timestamp=True, wfc_crosspost=None, get_log_full_path=None, **kwargs): all_args = locals() if wfc_crosspost: del all_args['kwargs'], all_args['wfc_crosspost'], all_args[ 'content'], all_args['full_path'] all_args.update(kwargs) for cp in item_to_list(wfc_crosspost): write_file_contents(content, cp, **all_args) orig_path = full_path if not os.path.exists(os.path.dirname(full_path)): if callable(get_log_full_path): full_path = get_log_full_path(full_path) if full_path is False: return else: if not filter_logs(full_path): return full_path = os.path.abspath( os.path.join(FOLDERS.LOGS, full_path + '.log')) base_path = os.path.dirname(full_path) if not os.path.exists(base_path): os.makedirs(base_path) if wfc_timestamp: print_content = content content = '[%s]: ' % datetime.now().strftime( ANKNOTES.DATE_FORMAT) + content with open(full_path, 'w+' if clear else 'a+') as fileLog: try: print >> fileLog, content except UnicodeEncodeError: content = encode(content) print >> fileLog, content if do_print: print content if print_timestamp or not print_content else print_content
def __init__(self, path=None, text=None, timeout=0, init_db=True, table=None): self._table_ = table self.ankdb_lastquery = None self.echo = False if not init_db: return encpath = path if isinstance(encpath, unicode): encpath = encode(path) if path: log('Creating local ankDB instance from path: ' + path, 'sql\\ankDB') self._db = sqlite.connect(encpath, timeout=timeout) self._db.row_factory = sqlite.Row if text: self._db.text_factory = text self._path = path else: log('Creating local ankDB instance from Anki DB instance at: ' + mw.col.db._path, 'sql\\ankDB') self._db = mw.col.db._db """ :type : sqlite.Connection """ self._db.row_factory = sqlite.Row self._path = mw.col.db._path
def makeNote(self, noteTitle=None, noteContents=None, tagNames=None, parentNotebook=None, resources=None, noteType=None, guid=None, validated=None, enNote=None): """ Create or Update a Note instance with title and body Send Note object to user's account :type noteTitle: str :param noteContents: Valid ENML without the <en-note></en-note> tags. Will be processed by makeNoteBody :type enNote : EvernoteNotePrototype :rtype : (EvernoteAPIStatus, EvernoteNote) :returns Status and Note """ if tagNames is None: tagNames = [] if enNote: guid, noteTitle, noteContents, tagNames, parentNotebook = enNote.Guid, enNote.FullTitle, enNote.Content, enNote.Tags, enNote.NotebookGuid or parentNotebook if resources is None: resources = [] callType = "create" validation_status = EvernoteAPIStatus.Uninitialized if validated is None: if not EVERNOTE.UPLOAD.VALIDATION.ENABLED: validated = True else: validation_status = self.addNoteToMakeNoteQueue(noteTitle, noteContents, tagNames, parentNotebook, resources, guid) if not validation_status.IsSuccess and not self.hasValidator: return validation_status, None log('%s%s: %s: ' % ('+VALIDATOR ' if self.hasValidator else '' + noteType, str(validation_status), noteTitle), 'validation') ourNote = EvernoteNote() ourNote.title = encode(noteTitle) if guid: callType = "update"; ourNote.guid = guid ## Build body of note nBody = self.makeNoteBody(noteContents, resources) if validated is not True and not validation_status.IsSuccess: status, errors = self.validateNoteBody(nBody, ourNote.title) assert isinstance(status, EvernoteAPIStatus) if not status.IsSuccess: return status, None ourNote.content = nBody notestore_status = self.initialize_note_store() if not notestore_status.IsSuccess: return notestore_status, None while '' in tagNames: tagNames.remove('') if tagNames: if EVERNOTE.API.IS_SANDBOXED and not '#Sandbox' in tagNames: tagNames.append("#Sandbox") ourNote.tagNames = tagNames ## parentNotebook is optional; if omitted, default notebook is used if parentNotebook: if hasattr(parentNotebook, 'guid'): ourNote.notebookGuid = parentNotebook.guid elif hasattr(parentNotebook, 'Guid'): ourNote.notebookGuid = parentNotebook.Guid elif is_str(parentNotebook): ourNote.notebookGuid = parentNotebook ## Attempt to create note in Evernote account api_action_str = u'trying to %s a note' % callType log_api(callType + "Note", "'%s'" % noteTitle) try: note = getattr(self.noteStore, callType + 'Note')(self.token, ourNote) except EDAMSystemException as e: if not HandleEDAMRateLimitError(e, api_action_str) or EVERNOTE.API.DEBUG_RAISE_ERRORS: raise return EvernoteAPIStatus.RateLimitError, None except socket.error as v: if not HandleSocketError(v, api_action_str) or EVERNOTE.API.DEBUG_RAISE_ERRORS: raise return EvernoteAPIStatus.SocketError, None except EDAMUserException as edue: ## Something was wrong with the note data ## See EDAMErrorCode enumeration for error code explanation ## http://dev.evernote.com/documentation/reference/Errors.html#Enum_EDAMErrorCode print "EDAMUserException:", edue log_error("-" * 50, crosspost_to_default=False) log_error("EDAMUserException: " + str(edue), crosspost='api') log_error(str(ourNote.tagNames), crosspost_to_default=False) log_error(str(ourNote.content), crosspost_to_default=False) log_error("-" * 50 + "\r\n", crosspost_to_default=False) if EVERNOTE.API.DEBUG_RAISE_ERRORS: raise return EvernoteAPIStatus.UserError, None except EDAMNotFoundException as ednfe: print "EDAMNotFoundException:", ednfe log_error("-" * 50, crosspost_to_default=False) log_error("EDAMNotFoundException: " + str(ednfe), crosspost='api') if callType is "update": log_error('GUID: ' + str(ourNote.guid), crosspost_to_default=False) if ourNote.notebookGuid: log_error('Notebook GUID: ' + str(ourNote.notebookGuid), crosspost_to_default=False) log_error("-" * 50 + "\r\n", crosspost_to_default=False) if EVERNOTE.API.DEBUG_RAISE_ERRORS: raise return EvernoteAPIStatus.NotFoundError, None except Exception as e: print "Unknown Exception:", e log_error("-" * 50, crosspost_to_default=False) log_error("Unknown Exception: " + str(e)) log_error(str(ourNote.tagNames), crosspost_to_default=False) log_error(str(ourNote.content), crosspost_to_default=False) log_error("-" * 50 + "\r\n", crosspost_to_default=False) # return EvernoteAPIStatus.UnhandledError, None raise # noinspection PyUnboundLocalVariable note.content = nBody return EvernoteAPIStatus.Success, note
def processAllRootNotesMissing(self): """:rtype : list[EvernoteTOCEntry]""" DEBUG_HTML = False # log (" CREATING TOC's " , 'tocList', clear=True, timestamp=False) # log ("------------------------------------------------" , 'tocList', timestamp=False) # if DEBUG_HTML: log('<h1>CREATING TOCs</h1>', 'extra\\logs\\toc-ols\\toc-index.htm', timestamp=False, clear=True, extension='htm') ols = [] dbRows = [] returns = [] """:type : list[EvernoteTOCEntry]""" db = ankDB(TABLES.TOC_AUTO) db.delete("1", table=db.table) db.commit() # olsz = None tmr = stopwatch.Timer(self.RootNotesMissing.TitlesList, infoStr='Processing Root Notes', label='RootTitles\\') for rootTitleStr in self.RootNotesMissing.TitlesList: count_child = 0 childTitlesDictSortedKeys = sorted(self.RootNotesMissing.ChildTitlesDict[rootTitleStr], key=lambda s: s.lower()) total_child = len(childTitlesDictSortedKeys) tags = [] outline = self.getNoteFromDB("UPPER(title) = '%s' AND tagNames LIKE '%%,%s,%%'" % ( escape_text_sql(rootTitleStr.upper()), TAGS.OUTLINE)) currentAutoNote = self.getNoteFromDB("UPPER(title) = '%s' AND tagNames LIKE '%%,%s,%%'" % ( escape_text_sql(rootTitleStr.upper()), TAGS.TOC_AUTO)) notebookGuids = {} childGuid = None is_isolated = total_child is 1 and not outline if is_isolated: tmr.counts.isolated.step() childBaseTitle = childTitlesDictSortedKeys[0] childGuid = self.RootNotesMissing.ChildTitlesDict[rootTitleStr][childBaseTitle] enChildNote = self.RootNotesMissing.ChildNotesDict[rootTitleStr][childGuid] # tags = enChildNote.Tags log(" > ISOLATED ROOT TITLE: [%-3d]: %-60s --> %-40s: %s" % ( tmr.counts.isolated.val, rootTitleStr + ':', childBaseTitle, childGuid), tmr.label + 'Isolated', timestamp=False) else: tmr.counts.created.completed.step() log_blank(tmr.label + 'TOC') log(" [%-3d] %s %s" % (tmr.count, rootTitleStr, '(O)' if outline else ' '), tmr.label + 'TOC', timestamp=False) tmr.step(rootTitleStr) if is_isolated: continue tocHierarchy = TOCHierarchyClass(rootTitleStr) if outline: tocHierarchy.Outline = TOCHierarchyClass(note=outline) tocHierarchy.Outline.parent = tocHierarchy for childBaseTitle in childTitlesDictSortedKeys: count_child += 1 childGuid = self.RootNotesMissing.ChildTitlesDict[rootTitleStr][childBaseTitle] enChildNote = self.RootNotesMissing.ChildNotesDict[rootTitleStr][childGuid] if count_child == 1: tags = enChildNote.Tags else: tags = [x for x in tags if x in enChildNote.Tags] if not enChildNote.NotebookGuid in notebookGuids: notebookGuids[enChildNote.NotebookGuid] = 0 notebookGuids[enChildNote.NotebookGuid] += 1 level = enChildNote.Title.Level # childName = enChildNote.Title.Name # childTitle = enChildNote.FullTitle log(" %2d: %d. --> %-60s" % (count_child, level, childBaseTitle), tmr.label + 'TOC', timestamp=False) # tocList.generateEntry(childTitle, enChildNote) tocHierarchy.addNote(enChildNote) realTitle = get_evernote_title_from_guid(childGuid) realTitle = realTitle[0:realTitle.index(':')] # realTitleUTF8 = realTitle.encode('utf8') notebookGuid = sorted(notebookGuids.items(), key=itemgetter(1), reverse=True)[0][0] real_root_title = generateTOCTitle(realTitle) ol = tocHierarchy.GetOrderedList() tocEntry = EvernoteTOCEntry(real_root_title, ol, ',' + ','.join(tags) + ',', notebookGuid) returns.append(tocEntry) dbRows.append(tocEntry.items()) if not DEBUG_HTML: continue # ols.append(ol) # olutf8 = encode(ol) # fn = 'toc-ols\\toc-' + str(tmr.count) + '-' + rootTitleStr.replace('\\', '_') + '.htm' # full_path = os.path.join(FOLDERS.LOGS, fn) # if not os.path.exists(os.path.dirname(full_path)): # os.mkdir(os.path.dirname(full_path)) # file_object = open(full_path, 'w') # file_object.write(olutf8) # file_object.close() # if DEBUG_HTML: log(ol, 'toc-ols\\toc-' + str(count) + '-' + rootTitleStr.replace('\\', '_'), timestamp=False, clear=True, extension='htm') # log("Created TOC #%d:\n%s\n\n" % (count, str_), 'tocList', timestamp=False) if DEBUG_HTML: ols_html = u'\r\n<BR><BR><HR><BR><BR>\r\n'.join(ols) fn = 'toc-ols\\toc-index.htm' file_object = open(os.path.join(FOLDERS.LOGS, fn), 'w') try: file_object.write(u'<h1>CREATING TOCs</h1>\n\n' + ols_html) except Exception: try: file_object.write(u'<h1>CREATING TOCs</h1>\n\n' + encode(ols_html)) except Exception: pass file_object.close() db.executemany("INSERT INTO {t} (root_title, contents, tagNames, notebookGuid) VALUES(?, ?, ?, ?)", dbRows) db.commit() return returns
def makeNote(self, noteTitle=None, noteContents=None, tagNames=None, parentNotebook=None, resources=None, noteType=None, guid=None, validated=None, enNote=None): """ Create or Update a Note instance with title and body Send Note object to user's account :type noteTitle: str :param noteContents: Valid ENML without the <en-note></en-note> tags. Will be processed by makeNoteBody :type enNote : EvernoteNotePrototype :rtype : (EvernoteAPIStatus, EvernoteNote) :returns Status and Note """ if tagNames is None: tagNames = [] if enNote: guid, noteTitle, noteContents, tagNames, parentNotebook = enNote.Guid, enNote.FullTitle, enNote.Content, enNote.Tags, enNote.NotebookGuid or parentNotebook if resources is None: resources = [] callType = "create" validation_status = EvernoteAPIStatus.Uninitialized if validated is None: if not EVERNOTE.UPLOAD.VALIDATION.ENABLED: validated = True else: validation_status = self.addNoteToMakeNoteQueue( noteTitle, noteContents, tagNames, parentNotebook, resources, guid) if not validation_status.IsSuccess and not self.hasValidator: return validation_status, None log( '%s%s: %s: ' % ('+VALIDATOR ' if self.hasValidator else '' + noteType, str(validation_status), noteTitle), 'validation') ourNote = EvernoteNote() ourNote.title = encode(noteTitle) if guid: callType = "update" ourNote.guid = guid ## Build body of note nBody = self.makeNoteBody(noteContents, resources) if validated is not True and not validation_status.IsSuccess: status, errors = self.validateNoteBody(nBody, ourNote.title) assert isinstance(status, EvernoteAPIStatus) if not status.IsSuccess: return status, None ourNote.content = nBody notestore_status = self.initialize_note_store() if not notestore_status.IsSuccess: return notestore_status, None while '' in tagNames: tagNames.remove('') if tagNames: if EVERNOTE.API.IS_SANDBOXED and not '#Sandbox' in tagNames: tagNames.append("#Sandbox") ourNote.tagNames = tagNames ## parentNotebook is optional; if omitted, default notebook is used if parentNotebook: if hasattr(parentNotebook, 'guid'): ourNote.notebookGuid = parentNotebook.guid elif hasattr(parentNotebook, 'Guid'): ourNote.notebookGuid = parentNotebook.Guid elif is_str(parentNotebook): ourNote.notebookGuid = parentNotebook ## Attempt to create note in Evernote account api_action_str = u'trying to %s a note' % callType log_api(callType + "Note", "'%s'" % noteTitle) try: note = getattr(self.noteStore, callType + 'Note')(self.token, ourNote) except EDAMSystemException as e: if not HandleEDAMRateLimitError( e, api_action_str) or EVERNOTE.API.DEBUG_RAISE_ERRORS: raise return EvernoteAPIStatus.RateLimitError, None except socket.error as v: if not HandleSocketError( v, api_action_str) or EVERNOTE.API.DEBUG_RAISE_ERRORS: raise return EvernoteAPIStatus.SocketError, None except EDAMUserException as edue: ## Something was wrong with the note data ## See EDAMErrorCode enumeration for error code explanation ## http://dev.evernote.com/documentation/reference/Errors.html#Enum_EDAMErrorCode print "EDAMUserException:", edue log_error("-" * 50, crosspost_to_default=False) log_error("EDAMUserException: " + str(edue), crosspost='api') log_error(str(ourNote.tagNames), crosspost_to_default=False) log_error(str(ourNote.content), crosspost_to_default=False) log_error("-" * 50 + "\r\n", crosspost_to_default=False) if EVERNOTE.API.DEBUG_RAISE_ERRORS: raise return EvernoteAPIStatus.UserError, None except EDAMNotFoundException as ednfe: print "EDAMNotFoundException:", ednfe log_error("-" * 50, crosspost_to_default=False) log_error("EDAMNotFoundException: " + str(ednfe), crosspost='api') if callType is "update": log_error('GUID: ' + str(ourNote.guid), crosspost_to_default=False) if ourNote.notebookGuid: log_error('Notebook GUID: ' + str(ourNote.notebookGuid), crosspost_to_default=False) log_error("-" * 50 + "\r\n", crosspost_to_default=False) if EVERNOTE.API.DEBUG_RAISE_ERRORS: raise return EvernoteAPIStatus.NotFoundError, None except Exception as e: print "Unknown Exception:", e log_error("-" * 50, crosspost_to_default=False) log_error("Unknown Exception: " + str(e)) log_error(str(ourNote.tagNames), crosspost_to_default=False) log_error(str(ourNote.content), crosspost_to_default=False) log_error("-" * 50 + "\r\n", crosspost_to_default=False) # return EvernoteAPIStatus.UnhandledError, None raise # noinspection PyUnboundLocalVariable note.content = nBody return EvernoteAPIStatus.Success, note
def extract_links_from_toc(self): db = ankDB(TABLES.SEE_ALSO) db.setrowfactory() toc_entries = db.all( "SELECT * FROM {n} WHERE tagNames LIKE '{t_toc}' ORDER BY title ASC" ) db.execute("DELETE FROM {t} WHERE from_toc = 1") log = Logger('See Also\\4-extract_links_from_toc\\', timestamp=False, crosspost_to_default=False, rm_path=True) tmr = stopwatch.Timer(toc_entries, 20, infoStr='Extracting Links', label=log.base_path) tmr.info.BannerHeader('error') toc_guids = [] for toc_entry in toc_entries: toc_evernote_guid, toc_link_title = toc_entry['guid'], toc_entry[ 'title'] toc_guids.append("'%s'" % toc_evernote_guid) # toc_link_html = generate_evernote_span(toc_link_title, 'Links', 'TOC') enLinks = find_evernote_links(toc_entry['content']) tmr.increment(toc_link_title) for enLink in enLinks: target_evernote_guid = enLink.Guid if not check_evernote_guid_is_valid(target_evernote_guid): log.go( "Invalid Target GUID for %-70s %s" % (toc_link_title + ':', target_evernote_guid), 'error') continue base = { 'child_guid': target_evernote_guid, 'uid': enLink.Uid, 'shard': enLink.Shard, 'toc_guid': toc_evernote_guid, 'l1': 'source', 'l2': 'source', 'from_toc': 0, 'is_toc': 0 } query_count = "select COUNT(*) from {t} WHERE source_evernote_guid = '{%s_guid}'" toc = { 'num': 1 + db.scalar(fmt(query_count % 'toc', base)), 'html': enLink.HTML.replace(u'\'', u'\'\''), 'title': enLink.FullTitle.replace(u'\'', u'\'\''), 'l1': 'target', 'from_toc': 1 } # child = {1 + db.scalar(fmt(query_count % 'child', base)), # 'html': toc_link_html.replace(u'\'', u'\'\''), # 'title': toc_link_title.replace(u'\'', u'\'\''), # 'l2': 'target', # 'is_toc': 1 # } query = ( u"INSERT OR REPLACE INTO `{t}`(`{l1}_evernote_guid`, `number`, `uid`, `shard`, " u"`{l2}_evernote_guid`, `html`, `title`, `from_toc`, `is_toc`) " u"VALUES('{child_guid}', {num}, {uid}, '{shard}', " u"'{toc_guid}', '{html}', '{title}', {from_toc}, {is_toc})" ) query_toc = fmt(query, base, toc) db.execute(query_toc) log.go("\t\t - Added %2d child link(s) from TOC %s" % (len(enLinks), encode(toc_link_title))) db.update("is_toc = 1", where="target_evernote_guid IN (%s)" % ', '.join(toc_guids)) db.commit()
def processAllRootNotesMissing(self): """:rtype : list[EvernoteTOCEntry]""" DEBUG_HTML = False # log (" CREATING TOC's " , 'tocList', clear=True, timestamp=False) # log ("------------------------------------------------" , 'tocList', timestamp=False) # if DEBUG_HTML: log('<h1>CREATING TOCs</h1>', 'extra\\logs\\toc-ols\\toc-index.htm', timestamp=False, clear=True, extension='htm') ols = [] dbRows = [] returns = [] """:type : list[EvernoteTOCEntry]""" db = ankDB(TABLES.TOC_AUTO) db.delete("1", table=db.table) db.commit() # olsz = None tmr = stopwatch.Timer(self.RootNotesMissing.TitlesList, infoStr='Processing Root Notes', label='RootTitles\\') for rootTitleStr in self.RootNotesMissing.TitlesList: count_child = 0 childTitlesDictSortedKeys = sorted( self.RootNotesMissing.ChildTitlesDict[rootTitleStr], key=lambda s: s.lower()) total_child = len(childTitlesDictSortedKeys) tags = [] outline = self.getNoteFromDB( "UPPER(title) = '%s' AND tagNames LIKE '%%,%s,%%'" % (escape_text_sql(rootTitleStr.upper()), TAGS.OUTLINE)) currentAutoNote = self.getNoteFromDB( "UPPER(title) = '%s' AND tagNames LIKE '%%,%s,%%'" % (escape_text_sql(rootTitleStr.upper()), TAGS.TOC_AUTO)) notebookGuids = {} childGuid = None is_isolated = total_child is 1 and not outline if is_isolated: tmr.counts.isolated.step() childBaseTitle = childTitlesDictSortedKeys[0] childGuid = self.RootNotesMissing.ChildTitlesDict[ rootTitleStr][childBaseTitle] enChildNote = self.RootNotesMissing.ChildNotesDict[ rootTitleStr][childGuid] # tags = enChildNote.Tags log(" > ISOLATED ROOT TITLE: [%-3d]: %-60s --> %-40s: %s" % (tmr.counts.isolated.val, rootTitleStr + ':', childBaseTitle, childGuid), tmr.label + 'Isolated', timestamp=False) else: tmr.counts.created.completed.step() log_blank(tmr.label + 'TOC') log(" [%-3d] %s %s" % (tmr.count, rootTitleStr, '(O)' if outline else ' '), tmr.label + 'TOC', timestamp=False) tmr.step(rootTitleStr) if is_isolated: continue tocHierarchy = TOCHierarchyClass(rootTitleStr) if outline: tocHierarchy.Outline = TOCHierarchyClass(note=outline) tocHierarchy.Outline.parent = tocHierarchy for childBaseTitle in childTitlesDictSortedKeys: count_child += 1 childGuid = self.RootNotesMissing.ChildTitlesDict[ rootTitleStr][childBaseTitle] enChildNote = self.RootNotesMissing.ChildNotesDict[ rootTitleStr][childGuid] if count_child == 1: tags = enChildNote.Tags else: tags = [x for x in tags if x in enChildNote.Tags] if not enChildNote.NotebookGuid in notebookGuids: notebookGuids[enChildNote.NotebookGuid] = 0 notebookGuids[enChildNote.NotebookGuid] += 1 level = enChildNote.Title.Level # childName = enChildNote.Title.Name # childTitle = enChildNote.FullTitle log(" %2d: %d. --> %-60s" % (count_child, level, childBaseTitle), tmr.label + 'TOC', timestamp=False) # tocList.generateEntry(childTitle, enChildNote) tocHierarchy.addNote(enChildNote) realTitle = get_evernote_title_from_guid(childGuid) realTitle = realTitle[0:realTitle.index(':')] # realTitleUTF8 = realTitle.encode('utf8') notebookGuid = sorted(notebookGuids.items(), key=itemgetter(1), reverse=True)[0][0] real_root_title = generateTOCTitle(realTitle) ol = tocHierarchy.GetOrderedList() tocEntry = EvernoteTOCEntry(real_root_title, ol, ',' + ','.join(tags) + ',', notebookGuid) returns.append(tocEntry) dbRows.append(tocEntry.items()) if not DEBUG_HTML: continue # ols.append(ol) # olutf8 = encode(ol) # fn = 'toc-ols\\toc-' + str(tmr.count) + '-' + rootTitleStr.replace('\\', '_') + '.htm' # full_path = os.path.join(FOLDERS.LOGS, fn) # if not os.path.exists(os.path.dirname(full_path)): # os.mkdir(os.path.dirname(full_path)) # file_object = open(full_path, 'w') # file_object.write(olutf8) # file_object.close() # if DEBUG_HTML: log(ol, 'toc-ols\\toc-' + str(count) + '-' + rootTitleStr.replace('\\', '_'), timestamp=False, clear=True, extension='htm') # log("Created TOC #%d:\n%s\n\n" % (count, str_), 'tocList', timestamp=False) if DEBUG_HTML: ols_html = u'\r\n<BR><BR><HR><BR><BR>\r\n'.join(ols) fn = 'toc-ols\\toc-index.htm' file_object = open(os.path.join(FOLDERS.LOGS, fn), 'w') try: file_object.write(u'<h1>CREATING TOCs</h1>\n\n' + ols_html) except Exception: try: file_object.write(u'<h1>CREATING TOCs</h1>\n\n' + encode(ols_html)) except Exception: pass file_object.close() db.executemany( "INSERT INTO {t} (root_title, contents, tagNames, notebookGuid) VALUES(?, ?, ?, ?)", dbRows) db.commit() return returns