def processNote(self, enNote):
""":type enNote: EvernoteNote.EvernoteNote"""
db = ankDB()
if self.processingFlags.populateRootTitlesList or self.processingFlags.populateRootTitlesDict or self.processingFlags.populateMissingRootTitlesList or self.processingFlags.populateMissingRootTitlesDict:
if enNote.IsChild:
# log([enNote.Title, enNote.Level, enNote.Title.TitleParts, enNote.IsChild])
rootTitle = enNote.Title.Root
rootTitleStr = generateTOCTitle(rootTitle)
if self.processingFlags.populateMissingRootTitlesList or self.processingFlags.populateMissingRootTitlesDict:
if not rootTitleStr in self.RootNotesExisting.TitlesList:
if not rootTitleStr in self.RootNotesMissing.TitlesList:
self.RootNotesMissing.TitlesList.append(rootTitleStr)
self.RootNotesMissing.ChildTitlesDict[rootTitleStr] = {}
self.RootNotesMissing.ChildNotesDict[rootTitleStr] = {}
if not enNote.Title.Base:
log(enNote.Title)
log(enNote.Base)
assert enNote.Title.Base
childBaseTitleStr = enNote.Title.Base.FullTitle
if childBaseTitleStr in self.RootNotesMissing.ChildTitlesDict[rootTitleStr]:
log_error("Duplicate Child Base Title String. \n%-18s%s\n%-18s%s: %s\n%-18s%s" % (
'Root Note Title: ', rootTitleStr, 'Child Note: ', enNote.Guid, childBaseTitleStr,
'Duplicate Note: ',
self.RootNotesMissing.ChildTitlesDict[rootTitleStr][childBaseTitleStr]),
crosspost_to_default=False)
if not hasattr(self, 'loggedDuplicateChildNotesWarning'):
log(
" > WARNING: Duplicate Child Notes found when processing Root Notes. See error log for more details")
self.loggedDuplicateChildNotesWarning = True
self.RootNotesMissing.ChildTitlesDict[rootTitleStr][childBaseTitleStr] = enNote.Guid
self.RootNotesMissing.ChildNotesDict[rootTitleStr][enNote.Guid] = enNote
if self.processingFlags.populateRootTitlesList or self.processingFlags.populateRootTitlesDict:
if not rootTitleStr in self.RootNotes.TitlesList:
self.RootNotes.TitlesList.append(rootTitleStr)
if self.processingFlags.populateRootTitlesDict:
self.RootNotes.TitlesDict[rootTitleStr][enNote.Guid] = enNote.Title.Base
self.RootNotes.NotesDict[rootTitleStr][enNote.Guid] = enNote
if self.processingFlags.populateChildRootTitles or self.processingFlags.populateExistingRootTitlesList or self.processingFlags.populateExistingRootTitlesDict:
if enNote.IsRoot:
rootTitle = enNote.Title
rootTitleStr = generateTOCTitle(rootTitle)
rootGuid = enNote.Guid
if self.processingFlags.populateExistingRootTitlesList or self.processingFlags.populateExistingRootTitlesDict or self.processingFlags.populateMissingRootTitlesList:
if not rootTitleStr in self.RootNotesExisting.TitlesList:
self.RootNotesExisting.TitlesList.append(rootTitleStr)
if self.processingFlags.populateChildRootTitles:
childNotes = db.execute("title LIKE ? || ':%' ORDER BY title ASC", rootTitleStr)
child_count = 0
for childDbNote in childNotes:
child_count += 1
childGuid = childDbNote['guid']
childEnNote = EvernoteNotePrototype(db_note=childDbNote)
if child_count is 1:
self.RootNotesChildren.TitlesDict[rootGuid] = {}
self.RootNotesChildren.NotesDict[rootGuid] = {}
childBaseTitle = childEnNote.Title.Base
self.RootNotesChildren.TitlesDict[rootGuid][childGuid] = childBaseTitle
self.RootNotesChildren.NotesDict[rootGuid][childGuid] = childEnNote
def getNoteFromDB(query):
"""
:param query:
:return:
:rtype : sqlite.Row
"""
dbNote = ankDB().first(query)
if not dbNote:
return None
return dbNote
def getNoteFromDB(query):
"""
:param query:
:return:
:rtype : sqlite.Row
"""
dbNote = ankDB().first(query)
if not dbNote:
return None
return dbNote
def addDbQuery(self, sql_query, order=''):
if not sql_query:
sql_query = '1'
if self.baseQuery and self.baseQuery != '1':
if sql_query == '1':
sql_query = self.baseQuery
else:
sql_query = "(%s) AND (%s) " % (self.baseQuery, sql_query)
if order:
sql_query += ' ORDER BY ' + order
dbNotes = ankDB().execute(sql_query)
self.addDBNotes(dbNotes)
def addDbQuery(self, sql_query, order=''):
if not sql_query:
sql_query = '1'
if self.baseQuery and self.baseQuery != '1':
if sql_query == '1':
sql_query = self.baseQuery
else:
sql_query = "(%s) AND (%s) " % (self.baseQuery, sql_query)
if order:
sql_query += ' ORDER BY ' + order
dbNotes = ankDB().execute(sql_query)
self.addDBNotes(dbNotes)
def processAllRootNotesMissing(self):
""":rtype : list[EvernoteTOCEntry]"""
DEBUG_HTML = False
# log (" CREATING TOC's " , 'tocList', clear=True, timestamp=False)
# log ("------------------------------------------------" , 'tocList', timestamp=False)
# if DEBUG_HTML: log('<h1>CREATING TOCs</h1>', 'extra\\logs\\toc-ols\\toc-index.htm', timestamp=False, clear=True, extension='htm')
ols = []
dbRows = []
returns = []
""":type : list[EvernoteTOCEntry]"""
db = ankDB(TABLES.TOC_AUTO)
db.delete("1", table=db.table)
db.commit()
# olsz = None
tmr = stopwatch.Timer(self.RootNotesMissing.TitlesList, infoStr='Processing Root Notes', label='RootTitles\\')
for rootTitleStr in self.RootNotesMissing.TitlesList:
count_child = 0
childTitlesDictSortedKeys = sorted(self.RootNotesMissing.ChildTitlesDict[rootTitleStr],
key=lambda s: s.lower())
total_child = len(childTitlesDictSortedKeys)
tags = []
outline = self.getNoteFromDB("UPPER(title) = '%s' AND tagNames LIKE '%%,%s,%%'" % (
escape_text_sql(rootTitleStr.upper()), TAGS.OUTLINE))
currentAutoNote = self.getNoteFromDB("UPPER(title) = '%s' AND tagNames LIKE '%%,%s,%%'" % (
escape_text_sql(rootTitleStr.upper()), TAGS.TOC_AUTO))
notebookGuids = {}
childGuid = None
is_isolated = total_child is 1 and not outline
if is_isolated:
tmr.counts.isolated.step()
childBaseTitle = childTitlesDictSortedKeys[0]
childGuid = self.RootNotesMissing.ChildTitlesDict[rootTitleStr][childBaseTitle]
enChildNote = self.RootNotesMissing.ChildNotesDict[rootTitleStr][childGuid]
# tags = enChildNote.Tags
log(" > ISOLATED ROOT TITLE: [%-3d]: %-60s --> %-40s: %s" % (
tmr.counts.isolated.val, rootTitleStr + ':', childBaseTitle, childGuid), tmr.label + 'Isolated',
timestamp=False)
else:
tmr.counts.created.completed.step()
log_blank(tmr.label + 'TOC')
log(" [%-3d] %s %s" % (tmr.count, rootTitleStr, '(O)' if outline else ' '), tmr.label + 'TOC',
timestamp=False)
tmr.step(rootTitleStr)
if is_isolated:
continue
tocHierarchy = TOCHierarchyClass(rootTitleStr)
if outline:
tocHierarchy.Outline = TOCHierarchyClass(note=outline)
tocHierarchy.Outline.parent = tocHierarchy
for childBaseTitle in childTitlesDictSortedKeys:
count_child += 1
childGuid = self.RootNotesMissing.ChildTitlesDict[rootTitleStr][childBaseTitle]
enChildNote = self.RootNotesMissing.ChildNotesDict[rootTitleStr][childGuid]
if count_child == 1:
tags = enChildNote.Tags
else:
tags = [x for x in tags if x in enChildNote.Tags]
if not enChildNote.NotebookGuid in notebookGuids:
notebookGuids[enChildNote.NotebookGuid] = 0
notebookGuids[enChildNote.NotebookGuid] += 1
level = enChildNote.Title.Level
# childName = enChildNote.Title.Name
# childTitle = enChildNote.FullTitle
log(" %2d: %d. --> %-60s" % (count_child, level, childBaseTitle),
tmr.label + 'TOC', timestamp=False)
# tocList.generateEntry(childTitle, enChildNote)
tocHierarchy.addNote(enChildNote)
realTitle = get_evernote_title_from_guid(childGuid)
realTitle = realTitle[0:realTitle.index(':')]
# realTitleUTF8 = realTitle.encode('utf8')
notebookGuid = sorted(notebookGuids.items(), key=itemgetter(1), reverse=True)[0][0]
real_root_title = generateTOCTitle(realTitle)
ol = tocHierarchy.GetOrderedList()
tocEntry = EvernoteTOCEntry(real_root_title, ol, ',' + ','.join(tags) + ',', notebookGuid)
returns.append(tocEntry)
dbRows.append(tocEntry.items())
if not DEBUG_HTML:
continue
# ols.append(ol)
# olutf8 = encode(ol)
# fn = 'toc-ols\\toc-' + str(tmr.count) + '-' + rootTitleStr.replace('\\', '_') + '.htm'
# full_path = os.path.join(FOLDERS.LOGS, fn)
# if not os.path.exists(os.path.dirname(full_path)):
# os.mkdir(os.path.dirname(full_path))
# file_object = open(full_path, 'w')
# file_object.write(olutf8)
# file_object.close()
# if DEBUG_HTML: log(ol, 'toc-ols\\toc-' + str(count) + '-' + rootTitleStr.replace('\\', '_'), timestamp=False, clear=True, extension='htm')
# log("Created TOC #%d:\n%s\n\n" % (count, str_), 'tocList', timestamp=False)
if DEBUG_HTML:
ols_html = u'\r\n<BR><BR><HR><BR><BR>\r\n'.join(ols)
fn = 'toc-ols\\toc-index.htm'
file_object = open(os.path.join(FOLDERS.LOGS, fn), 'w')
try:
file_object.write(u'<h1>CREATING TOCs</h1>\n\n' + ols_html)
except Exception:
try:
file_object.write(u'<h1>CREATING TOCs</h1>\n\n' + encode(ols_html))
except Exception:
pass
file_object.close()
db.executemany("INSERT INTO {t} (root_title, contents, tagNames, notebookGuid) VALUES(?, ?, ?, ?)", dbRows)
db.commit()
return returns
def processAllRootNotesMissing(self):
""":rtype : list[EvernoteTOCEntry]"""
DEBUG_HTML = False
# log (" CREATING TOC's " , 'tocList', clear=True, timestamp=False)
# log ("------------------------------------------------" , 'tocList', timestamp=False)
# if DEBUG_HTML: log('<h1>CREATING TOCs</h1>', 'extra\\logs\\toc-ols\\toc-index.htm', timestamp=False, clear=True, extension='htm')
ols = []
dbRows = []
returns = []
""":type : list[EvernoteTOCEntry]"""
db = ankDB(TABLES.TOC_AUTO)
db.delete("1", table=db.table)
db.commit()
# olsz = None
tmr = stopwatch.Timer(self.RootNotesMissing.TitlesList,
infoStr='Processing Root Notes',
label='RootTitles\\')
for rootTitleStr in self.RootNotesMissing.TitlesList:
count_child = 0
childTitlesDictSortedKeys = sorted(
self.RootNotesMissing.ChildTitlesDict[rootTitleStr],
key=lambda s: s.lower())
total_child = len(childTitlesDictSortedKeys)
tags = []
outline = self.getNoteFromDB(
"UPPER(title) = '%s' AND tagNames LIKE '%%,%s,%%'" %
(escape_text_sql(rootTitleStr.upper()), TAGS.OUTLINE))
currentAutoNote = self.getNoteFromDB(
"UPPER(title) = '%s' AND tagNames LIKE '%%,%s,%%'" %
(escape_text_sql(rootTitleStr.upper()), TAGS.TOC_AUTO))
notebookGuids = {}
childGuid = None
is_isolated = total_child is 1 and not outline
if is_isolated:
tmr.counts.isolated.step()
childBaseTitle = childTitlesDictSortedKeys[0]
childGuid = self.RootNotesMissing.ChildTitlesDict[
rootTitleStr][childBaseTitle]
enChildNote = self.RootNotesMissing.ChildNotesDict[
rootTitleStr][childGuid]
# tags = enChildNote.Tags
log(" > ISOLATED ROOT TITLE: [%-3d]: %-60s --> %-40s: %s" %
(tmr.counts.isolated.val, rootTitleStr + ':',
childBaseTitle, childGuid),
tmr.label + 'Isolated',
timestamp=False)
else:
tmr.counts.created.completed.step()
log_blank(tmr.label + 'TOC')
log(" [%-3d] %s %s" %
(tmr.count, rootTitleStr, '(O)' if outline else ' '),
tmr.label + 'TOC',
timestamp=False)
tmr.step(rootTitleStr)
if is_isolated:
continue
tocHierarchy = TOCHierarchyClass(rootTitleStr)
if outline:
tocHierarchy.Outline = TOCHierarchyClass(note=outline)
tocHierarchy.Outline.parent = tocHierarchy
for childBaseTitle in childTitlesDictSortedKeys:
count_child += 1
childGuid = self.RootNotesMissing.ChildTitlesDict[
rootTitleStr][childBaseTitle]
enChildNote = self.RootNotesMissing.ChildNotesDict[
rootTitleStr][childGuid]
if count_child == 1:
tags = enChildNote.Tags
else:
tags = [x for x in tags if x in enChildNote.Tags]
if not enChildNote.NotebookGuid in notebookGuids:
notebookGuids[enChildNote.NotebookGuid] = 0
notebookGuids[enChildNote.NotebookGuid] += 1
level = enChildNote.Title.Level
# childName = enChildNote.Title.Name
# childTitle = enChildNote.FullTitle
log(" %2d: %d. --> %-60s" %
(count_child, level, childBaseTitle),
tmr.label + 'TOC',
timestamp=False)
# tocList.generateEntry(childTitle, enChildNote)
tocHierarchy.addNote(enChildNote)
realTitle = get_evernote_title_from_guid(childGuid)
realTitle = realTitle[0:realTitle.index(':')]
# realTitleUTF8 = realTitle.encode('utf8')
notebookGuid = sorted(notebookGuids.items(),
key=itemgetter(1),
reverse=True)[0][0]
real_root_title = generateTOCTitle(realTitle)
ol = tocHierarchy.GetOrderedList()
tocEntry = EvernoteTOCEntry(real_root_title, ol,
',' + ','.join(tags) + ',',
notebookGuid)
returns.append(tocEntry)
dbRows.append(tocEntry.items())
if not DEBUG_HTML:
continue
# ols.append(ol)
# olutf8 = encode(ol)
# fn = 'toc-ols\\toc-' + str(tmr.count) + '-' + rootTitleStr.replace('\\', '_') + '.htm'
# full_path = os.path.join(FOLDERS.LOGS, fn)
# if not os.path.exists(os.path.dirname(full_path)):
# os.mkdir(os.path.dirname(full_path))
# file_object = open(full_path, 'w')
# file_object.write(olutf8)
# file_object.close()
# if DEBUG_HTML: log(ol, 'toc-ols\\toc-' + str(count) + '-' + rootTitleStr.replace('\\', '_'), timestamp=False, clear=True, extension='htm')
# log("Created TOC #%d:\n%s\n\n" % (count, str_), 'tocList', timestamp=False)
if DEBUG_HTML:
ols_html = u'\r\n<BR><BR><HR><BR><BR>\r\n'.join(ols)
fn = 'toc-ols\\toc-index.htm'
file_object = open(os.path.join(FOLDERS.LOGS, fn), 'w')
try:
file_object.write(u'<h1>CREATING TOCs</h1>\n\n' + ols_html)
except Exception:
try:
file_object.write(u'<h1>CREATING TOCs</h1>\n\n' +
encode(ols_html))
except Exception:
pass
file_object.close()
db.executemany(
"INSERT INTO {t} (root_title, contents, tagNames, notebookGuid) VALUES(?, ?, ?, ?)",
dbRows)
db.commit()
return returns
def processNote(self, enNote):
""":type enNote: EvernoteNote.EvernoteNote"""
db = ankDB()
if self.processingFlags.populateRootTitlesList or self.processingFlags.populateRootTitlesDict or self.processingFlags.populateMissingRootTitlesList or self.processingFlags.populateMissingRootTitlesDict:
if enNote.IsChild:
# log([enNote.Title, enNote.Level, enNote.Title.TitleParts, enNote.IsChild])
rootTitle = enNote.Title.Root
rootTitleStr = generateTOCTitle(rootTitle)
if self.processingFlags.populateMissingRootTitlesList or self.processingFlags.populateMissingRootTitlesDict:
if not rootTitleStr in self.RootNotesExisting.TitlesList:
if not rootTitleStr in self.RootNotesMissing.TitlesList:
self.RootNotesMissing.TitlesList.append(
rootTitleStr)
self.RootNotesMissing.ChildTitlesDict[
rootTitleStr] = {}
self.RootNotesMissing.ChildNotesDict[
rootTitleStr] = {}
if not enNote.Title.Base:
log(enNote.Title)
log(enNote.Base)
assert enNote.Title.Base
childBaseTitleStr = enNote.Title.Base.FullTitle
if childBaseTitleStr in self.RootNotesMissing.ChildTitlesDict[
rootTitleStr]:
log_error(
"Duplicate Child Base Title String. \n%-18s%s\n%-18s%s: %s\n%-18s%s"
% ('Root Note Title: ', rootTitleStr,
'Child Note: ', enNote.Guid,
childBaseTitleStr, 'Duplicate Note: ',
self.RootNotesMissing.ChildTitlesDict[
rootTitleStr][childBaseTitleStr]),
crosspost_to_default=False)
if not hasattr(self,
'loggedDuplicateChildNotesWarning'):
log(" > WARNING: Duplicate Child Notes found when processing Root Notes. See error log for more details"
)
self.loggedDuplicateChildNotesWarning = True
self.RootNotesMissing.ChildTitlesDict[rootTitleStr][
childBaseTitleStr] = enNote.Guid
self.RootNotesMissing.ChildNotesDict[rootTitleStr][
enNote.Guid] = enNote
if self.processingFlags.populateRootTitlesList or self.processingFlags.populateRootTitlesDict:
if not rootTitleStr in self.RootNotes.TitlesList:
self.RootNotes.TitlesList.append(rootTitleStr)
if self.processingFlags.populateRootTitlesDict:
self.RootNotes.TitlesDict[rootTitleStr][
enNote.Guid] = enNote.Title.Base
self.RootNotes.NotesDict[rootTitleStr][
enNote.Guid] = enNote
if self.processingFlags.populateChildRootTitles or self.processingFlags.populateExistingRootTitlesList or self.processingFlags.populateExistingRootTitlesDict:
if enNote.IsRoot:
rootTitle = enNote.Title
rootTitleStr = generateTOCTitle(rootTitle)
rootGuid = enNote.Guid
if self.processingFlags.populateExistingRootTitlesList or self.processingFlags.populateExistingRootTitlesDict or self.processingFlags.populateMissingRootTitlesList:
if not rootTitleStr in self.RootNotesExisting.TitlesList:
self.RootNotesExisting.TitlesList.append(rootTitleStr)
if self.processingFlags.populateChildRootTitles:
childNotes = db.execute(
"title LIKE ? || ':%' ORDER BY title ASC",
rootTitleStr)
child_count = 0
for childDbNote in childNotes:
child_count += 1
childGuid = childDbNote['guid']
childEnNote = EvernoteNotePrototype(
db_note=childDbNote)
if child_count is 1:
self.RootNotesChildren.TitlesDict[rootGuid] = {}
self.RootNotesChildren.NotesDict[rootGuid] = {}
childBaseTitle = childEnNote.Title.Base
self.RootNotesChildren.TitlesDict[rootGuid][
childGuid] = childBaseTitle
self.RootNotesChildren.NotesDict[rootGuid][
childGuid] = childEnNote
