Пример #1
0
def _runReport(feed,storage):
	bag = storage.getKeywordWeights()
	
	from feedReader import readFeed
	from keywordGenerator import generateKeywords
	from entryScraper import scrapeEntry
	
	feed_keywords = feed.get('keywords',default=None)
	for entry, need_scrape in readFeed(feed):
		if storage.checkDocumentExistence(entry['url']):
			continue
		if need_scrape:
			scrapeEntry(entry)
		entry['keywords'] = generateKeywords(entry,bag=bag,keywords=feed_keywords)
		storage.putEntry(entry)
	storage.putFeed(feed)
Пример #2
0
def getObjectKeywords(text=None, url=None):
    if text:
        mock = {'url': text[:20], 'type': 'article', 'summary': text}
    elif url:
        from entryScraper import scrapeEntry
        mock = scrapeEntry({'url': url, 'type': 'article'})
    from keywordGenerator import generateKeywords
    return generateKeywords(mock)
Пример #3
0
def getObjectKeywords(text=None,url=None):
	if text:
		mock = {'url':text[:20],'type':'article','summary':text}
	elif url:
		from entryScraper import scrapeEntry
		mock = scrapeEntry({'url':url,'type':'article'})
	from keywordGenerator import generateKeywords
	return generateKeywords(mock)
Пример #4
0
def _runReport(feed, storage):
    bag = storage.getKeywordWeights()

    from feedReader import readFeed
    from keywordGenerator import generateKeywords
    from entryScraper import scrapeEntry

    feed_keywords = feed.get('keywords', default=None)
    for entry, need_scrape in readFeed(feed):
        if storage.checkDocumentExistence(entry['url']):
            continue
        if need_scrape:
            scrapeEntry(entry)
        entry['keywords'] = generateKeywords(entry,
                                             bag=bag,
                                             keywords=feed_keywords)
        storage.putEntry(entry)
    storage.putFeed(feed)