def scrapItem(self, url=None, category=None):
'''
Save news item
'''
newsSource = NewsSource(self.default_database, self.default_collection)
newsItem = NewsItem(self.default_database, self.default_collection)
if(url == None):
item = newsSource.findEmptyNewsItem()
title = item['title']
url = item['url']
category = item['category']
self.logger.debug("Item: "+title+" / "+category)
details = self.sources[category]
scraper = Scraper(url)
scrap = scraper.get()
#Include the possibility of scraping multiple elements as news content
allNews = scrap.select(details['news_item_expr'])
#print(allNews)#temp
if len(allNews) > 1:
content = " ".join(str(n) for n in allNews)
else:
content = allNews[0]
#content = scrap.select(details['news_item_expr'])[0]
newsItem.addNewsItem(url, content)
try:
newsItem.insertNewsItem()
self.logger.debug(Stripper().strip(str(content)))
except OperationFailure as of:
pass
finally:
newsItem.resetNewsItem()
info = "Item Scraped: "+url
self.logger.info(info)
print(info)
