def scrapItem(self, url=None, category=None): ''' Save news item ''' newsSource = NewsSource(self.default_database, self.default_collection) newsItem = NewsItem(self.default_database, self.default_collection) if(url == None): item = newsSource.findEmptyNewsItem() title = item['title'] url = item['url'] category = item['category'] self.logger.debug("Item: "+title+" / "+category) details = self.sources[category] scraper = Scraper(url) scrap = scraper.get() #Include the possibility of scraping multiple elements as news content allNews = scrap.select(details['news_item_expr']) #print(allNews)#temp if len(allNews) > 1: content = " ".join(str(n) for n in allNews) else: content = allNews[0] #content = scrap.select(details['news_item_expr'])[0] newsItem.addNewsItem(url, content) try: newsItem.insertNewsItem() self.logger.debug(Stripper().strip(str(content))) except OperationFailure as of: pass finally: newsItem.resetNewsItem() info = "Item Scraped: "+url self.logger.info(info) print(info)