def filterFeeds(self, feeds): _isContainFilterReason = lambda x: len(re.findall(self.patterns, x)) > 0 firstFilter = [ e for e in feeds if _isContainFilterReason(e.title) ] result = [e for e in firstFilter if self._isValid(e)] filteredFeeds = listSub(feeds, firstFilter) + result filteredFeeds = listUnique(filteredFeeds) return filteredFeeds
def fetchingRes(self):
#TODO: tmep solution
# try:
nodes = []
for feedRes in self.resOper.getFeedResource():
reader = FeedReader(feedRes)
newNodes = reader.getFeedItems(FeedReader.noDuplicate)
if len(newNodes) == 0:
print('[Info] skip feeds {}'.format(feedRes.id), end='\t')
continue
nodes += newNodes
#nodes = listUnique(nodes, lambda x:x.title)
validFeeds = self.feedFilter.filterFeeds(nodes)
filtered = listSub(nodes, validFeeds)
self._writeToDB(validFeeds, filtered)
self.results = (validFeeds, filtered)