def getObjectKeywords(text=None, url=None): if text: mock = {'url': text[:20], 'type': 'article', 'summary': text} elif url: from entryScraper import scrapeEntry mock = scrapeEntry({'url': url, 'type': 'article'}) from keywordGenerator import generateKeywords return generateKeywords(mock)
def getObjectKeywords(text=None,url=None): if text: mock = {'url':text[:20],'type':'article','summary':text} elif url: from entryScraper import scrapeEntry mock = scrapeEntry({'url':url,'type':'article'}) from keywordGenerator import generateKeywords return generateKeywords(mock)
def _runReport(feed,storage): bag = storage.getKeywordWeights() from feedReader import readFeed from keywordGenerator import generateKeywords from entryScraper import scrapeEntry feed_keywords = feed.get('keywords',default=None) for entry, need_scrape in readFeed(feed): if storage.checkDocumentExistence(entry['url']): continue if need_scrape: scrapeEntry(entry) entry['keywords'] = generateKeywords(entry,bag=bag,keywords=feed_keywords) storage.putEntry(entry) storage.putFeed(feed)
def _runReport(feed, storage): bag = storage.getKeywordWeights() from feedReader import readFeed from keywordGenerator import generateKeywords from entryScraper import scrapeEntry feed_keywords = feed.get('keywords', default=None) for entry, need_scrape in readFeed(feed): if storage.checkDocumentExistence(entry['url']): continue if need_scrape: scrapeEntry(entry) entry['keywords'] = generateKeywords(entry, bag=bag, keywords=feed_keywords) storage.putEntry(entry) storage.putFeed(feed)