def cleanupNoMain():
count = 0
for key, text in index.items():
if not maintext.get(key):
count += 1
dbase.removeKey(key)
print('cleanupNoMain', count)
def searchTextRaw(targets, searchCore=False):
result = searchRaw(targets, searchCore=searchCore)
result = sortAndClean(result)
result = flipFirst(
result, lambda key: searchHitAll(targets, (key, maintext.get(key))))
result = dedupResult(result, lambda key: key.split('/')[0])
return result
def searchText(text, searchCore=False):
targets = text.split()
result = searchTextRaw(targets, searchCore=searchCore)
result = dedupResult(result,
lambda key: maintext.get(key),
sendAfter=False)
result = populateMaintext(result)
return finalTouch(result)
def searchTextRaw(targets, searchCore=False):
result = searchRaw(targets, searchCore=searchCore)
result = [(timestamp.get(key, 0) - channels.get(key.split('/')[0]) * 1000,
key) for key in result]
result.sort(reverse=True)
result = [item[1] for item in result]
result = flipFirst(result,
lambda key: channels.get(key.split('/')[0]) != -2,
sendAfter=False)
result = dedupResult(result,
lambda key: maintext.get(key),
sendAfter=False)
suspects = dbase.suspect.items()
result = flipFirst(result, lambda key: (key.split('/')[0] not in suspects))
result = flipFirst(
result, lambda key: searchHitAll(targets, (key, maintext.get(key))))
result = dedupResult(result, lambda key: key.split('/')[0])
result = flipFirst(result, lambda key: shouldFlipFirst(key))
return result
def searchRelated(text):
keys = set([item.strip() for item in text.split('/') if item.strip()])
result = set()
for item in dbase.channelrefer.items():
if set(item.split(':')) & keys:
result.update(item.split(':'))
result -= keys
result = [
channel + '/0' for channel in result if maintext.get(channel + '/0')
]
result = sortAndClean(result)
result = populateMaintext(result)
return finalTouch(result)
def shouldRemove(key):
if key.endswith('/0'):
return False
if not maintext.get(key):
return True
channel = key.split('/')[0]
if channels.get(channel) in [-2, 100]:
return True
if 0 <= channels.get(channel) < 2:
return False
if noCNnoEN(index.get(key)):
return True
if matchKey(index.get(key), ['hasFile', 'hasLink']):
return False
if timestamp.get(key, 0) < dbase.getRetain(key.split('/')[0]):
return True
return False
def getChannelTitle(key):
channel = key.split('/')[0]
return maintext.get(channel + '/0')
def populateMaintext(result):
for key in result:
yield key, maintext.get(key)
def populateMaintextLoose(result):
for key in result:
yield key, (maintext.get(key)
or maintext.get(key.split('/')[0] + '/0') or key)