def test_source_build(self): """ builds a source object, validates it has no errors, prints out all valid categories and feed urls """ DESC = """CNN.com delivers the latest breaking news and information on the latest top stories, weather, business, entertainment, politics, and more. For in-depth coverage, CNN.com provides special reports, video, audio, photo galleries, and interactive guides.""" BRAND = "cnn" config = Configuration() config.verbose = False s = Source("http://cnn.com", config=config) s.clean_memo_cache() s.build() assert s.brand == BRAND assert s.description == DESC # For this test case and a few more, I don't believe you can actually # assert two values to equal eachother because some values are ever changing. # Insead, i'm just going to print some stuff out so it is just as easy to take # a glance and see if it looks OK. print "\t\tWe have %d articles currently!" % s.size() print print "\t\t%s categories are: %s" % (s.url, str(s.category_urls()))
def test_source_build(self): """ builds a source object, validates it has no errors, prints out all valid categories and feed urls """ DESC = """CNN.com delivers the latest breaking news and information on the latest top stories, weather, business, entertainment, politics, and more. For in-depth coverage, CNN.com provides special reports, video, audio, photo galleries, and interactive guides.""" BRAND = 'cnn' config = Configuration() config.verbose = False s = Source('http://cnn.com', config=config) s.clean_memo_cache() s.build() assert s.brand == BRAND assert s.description == DESC # For this test case and a few more, I don't believe you can actually # assert two values to equal eachother because some values are ever changing. # Insead, i'm just going to print some stuff out so it is just as easy to take # a glance and see if it looks OK. print '\t\tWe have %d articles currently!' % s.size() print print '\t\t%s categories are: %s' % (s.url, str(s.category_urls()))
def main(): source="The Guardian" #config = Config() #config.memoize_articles = False guardian = Source("http://www.theguardian.com/world", memoize_articles=False) guardian.build() #guardian = newspaper.build('http://theguardian.com/world', memoize_articles=False) #news_pool.set([guardian], threads_per_source=2) #news_pool.join() #print(guardian.size()) for article in [x for x in guardian.articles if re.match(".*/world/.*", x.url) is not None]: url = article.url a = Article(url, language='en') a.download() for i in range(10): if a.is_downloaded: break else: a.download() try: a.parse() a.nlp() except: print("Error: Not parsed/downloaded correctly.") continue html = a.html summary = a.summary keywords = a.keywords title = a.title text = a.text date = str(a.publish_date).split()[0].split("-") date[0], date[1], date[2] = date[1], date[2], date[0] date = "/".join(date) delta = re.search(r'<span class="content__dateline-time">(.*)</span>' , html).group(1).replace(".",":").split()[0] time = datetime.now() + timedelta(hours=delta ) date_time = date + " " + time #print(title) #print(date_time) date_obj = datetime.datetime.strptime(date_time,'%m/%d/%Y %H:%M') #print(date_obj.strftime('%Y/%m/%d %I:%M %p')) #TODO: Add stuff to the DB try: article = { 'headline': title, 'url': url, 'text': text, 'date': date_obj } newspaper_article('The Guardian', article, keywords=keywords) except Exception as ex: print 'Article could not be created due to following error' print ex
def extract_from_source(self, source): news = NpSource(source, verbose=True) news.clean_memo_cache() news.build() logging.info('...build done!') for url in news.article_urls(): if self.is_available_url(url): article = self._extract_articles(url) if self.is_available_article(article): self._store_article(article)
class News: articles = [] def __init__(self, url): self.newspaper = Source(url) self.newspaper.clean_memo_cache() self.newspaper.build() self.articles = self.newspaper.articles def get_news(self, num_of_articles): return self.newspaper.articles[:num_of_articles]
def test_source_build(self): """ builds a source object, validates it has no errors, prints out all valid categories and feed urls """ DESC = ('CNN.com International delivers breaking news from across ' 'the globe and information on the latest top stories, ' 'business, sports and entertainment headlines. Follow the ' 'news as it happens through: special reports, videos, ' 'audio, photo galleries plus interactive maps and timelines.') CATEGORY_URLS = [ u'http://cnn.com/ASIA', u'http://connecttheworld.blogs.cnn.com', u'http://cnn.com/HLN', u'http://cnn.com/MIDDLEEAST', u'http://cnn.com', u'http://ireport.cnn.com', u'http://cnn.com/video', u'http://transcripts.cnn.com', u'http://cnn.com/espanol', u'http://partners.cnn.com', u'http://www.cnn.com', u'http://cnn.com/US', u'http://cnn.com/EUROPE', u'http://cnn.com/TRAVEL', u'http://cnn.com/cnni', u'http://cnn.com/SPORT', u'http://cnn.com/mostpopular', u'http://arabic.cnn.com', u'http://cnn.com/WORLD', u'http://cnn.com/LATINAMERICA', u'http://us.cnn.com', u'http://travel.cnn.com', u'http://mexico.cnn.com', u'http://cnn.com/SHOWBIZ', u'http://edition.cnn.com', u'http://amanpour.blogs.cnn.com', u'http://money.cnn.com', u'http://cnn.com/tools/index.html', u'http://cnnespanol.cnn.com', u'http://cnn.com/CNNI', u'http://business.blogs.cnn.com', u'http://cnn.com/AFRICA', u'http://cnn.com/TECH', u'http://cnn.com/BUSINESS' ] FEEDS = [u'http://rss.cnn.com/rss/edition.rss'] BRAND = 'cnn' s = Source('http://cnn.com', verbose=False, memoize_articles=False) url_re = re.compile(".*cnn\.com") mock_response_with(url_re, 'cnn_main_site') s.clean_memo_cache() s.build() assert s.brand == BRAND assert s.description == DESC assert s.size() == 241 assert s.category_urls() == CATEGORY_URLS # TODO: A lot of the feed extraction is NOT being tested because feeds # are primarly extracted from the HTML of category URLs. We lose this # effect by just mocking CNN's main page HTML. Warning: tedious fix. assert s.feed_urls() == FEEDS
def test_source_build(self): """builds a source object, validates it has no errors, prints out all valid categories and feed urls""" DESC = """CNN.com delivers the latest breaking news and information on the latest top stories, weather, business, entertainment, politics, and more. For in-depth coverage, CNN.com provides special reports, video, audio, photo galleries, and interactive guides.""" BRAND = 'cnn' configs = Configuration() configs.verbose = False s = Source('http://cnn.com', configs=configs) s.clean_memo_cache() s.build() assert s.brand == BRAND assert s.description == DESC print '\t\tWe have %d articles currently!' % s.size()
def test_source_build(self): """ builds a source object, validates it has no errors, prints out all valid categories and feed urls """ DESC = ('CNN.com International delivers breaking news from across ' 'the globe and information on the latest top stories, ' 'business, sports and entertainment headlines. Follow the ' 'news as it happens through: special reports, videos, ' 'audio, photo galleries plus interactive maps and timelines.') CATEGORY_URLS = [ u'http://cnn.com/ASIA', u'http://connecttheworld.blogs.cnn.com', u'http://cnn.com/HLN', u'http://cnn.com/MIDDLEEAST', u'http://cnn.com', u'http://ireport.cnn.com', u'http://cnn.com/video', u'http://transcripts.cnn.com', u'http://cnn.com/espanol', u'http://partners.cnn.com', u'http://www.cnn.com', u'http://cnn.com/US', u'http://cnn.com/EUROPE', u'http://cnn.com/TRAVEL', u'http://cnn.com/cnni', u'http://cnn.com/SPORT', u'http://cnn.com/mostpopular', u'http://arabic.cnn.com', u'http://cnn.com/WORLD', u'http://cnn.com/LATINAMERICA', u'http://us.cnn.com', u'http://travel.cnn.com', u'http://mexico.cnn.com', u'http://cnn.com/SHOWBIZ', u'http://edition.cnn.com', u'http://amanpour.blogs.cnn.com', u'http://money.cnn.com', u'http://cnn.com/tools/index.html', u'http://cnnespanol.cnn.com', u'http://cnn.com/CNNI', u'http://business.blogs.cnn.com', u'http://cnn.com/AFRICA', u'http://cnn.com/TECH', u'http://cnn.com/BUSINESS'] FEEDS = [u'http://rss.cnn.com/rss/edition.rss'] BRAND = 'cnn' s = Source('http://cnn.com', verbose=False, memoize_articles=False) url_re = re.compile(".*cnn\.com") mock_response_with(url_re, 'cnn_main_site') s.clean_memo_cache() s.build() assert s.brand == BRAND assert s.description == DESC assert s.size() == 241 assert s.category_urls() == CATEGORY_URLS # TODO: A lot of the feed extraction is NOT being tested because feeds # are primarly extracted from the HTML of category URLs. We lose this # effect by just mocking CNN's main page HTML. Warning: tedious fix. assert s.feed_urls() == FEEDS
def test_source_build(self): """ builds a source object, validates it has no errors, prints out all valid categories and feed urls """ DESC = ('CNN.com International delivers breaking news from across ' 'the globe and information on the latest top stories, ' 'business, sports and entertainment headlines. Follow the ' 'news as it happens through: special reports, videos, ' 'audio, photo galleries plus interactive maps and timelines.') CATEGORY_URLS = [ 'http://cnn.com/ASIA', 'http://connecttheworld.blogs.cnn.com', 'http://cnn.com/HLN', 'http://cnn.com/MIDDLEEAST', 'http://cnn.com', 'http://ireport.cnn.com', 'http://cnn.com/video', 'http://transcripts.cnn.com', 'http://cnn.com/espanol', 'http://partners.cnn.com', 'http://www.cnn.com', 'http://cnn.com/US', 'http://cnn.com/EUROPE', 'http://cnn.com/TRAVEL', 'http://cnn.com/cnni', 'http://cnn.com/SPORT', 'http://cnn.com/mostpopular', 'http://arabic.cnn.com', 'http://cnn.com/WORLD', 'http://cnn.com/LATINAMERICA', 'http://us.cnn.com', 'http://travel.cnn.com', 'http://mexico.cnn.com', 'http://cnn.com/SHOWBIZ', 'http://edition.cnn.com', 'http://amanpour.blogs.cnn.com', 'http://money.cnn.com', 'http://cnn.com/tools/index.html', 'http://cnnespanol.cnn.com', 'http://cnn.com/CNNI', 'http://business.blogs.cnn.com', 'http://cnn.com/AFRICA', 'http://cnn.com/TECH', 'http://cnn.com/BUSINESS' ] FEEDS = ['http://rss.cnn.com/rss/edition.rss'] BRAND = 'cnn' s = Source('http://cnn.com', verbose=False, memoize_articles=False) # html = mock_resource_with('http://cnn.com', 'cnn_main_site') s.clean_memo_cache() s.build()
def test_source_build(self): """ builds a source object, validates it has no errors, prints out all valid categories and feed urls """ DESC = ('CNN.com International delivers breaking news from across ' 'the globe and information on the latest top stories, ' 'business, sports and entertainment headlines. Follow the ' 'news as it happens through: special reports, videos, ' 'audio, photo galleries plus interactive maps and timelines.') CATEGORY_URLS = [ 'http://cnn.com/ASIA', 'http://connecttheworld.blogs.cnn.com', 'http://cnn.com/HLN', 'http://cnn.com/MIDDLEEAST', 'http://cnn.com', 'http://ireport.cnn.com', 'http://cnn.com/video', 'http://transcripts.cnn.com', 'http://cnn.com/espanol', 'http://partners.cnn.com', 'http://www.cnn.com', 'http://cnn.com/US', 'http://cnn.com/EUROPE', 'http://cnn.com/TRAVEL', 'http://cnn.com/cnni', 'http://cnn.com/SPORT', 'http://cnn.com/mostpopular', 'http://arabic.cnn.com', 'http://cnn.com/WORLD', 'http://cnn.com/LATINAMERICA', 'http://us.cnn.com', 'http://travel.cnn.com', 'http://mexico.cnn.com', 'http://cnn.com/SHOWBIZ', 'http://edition.cnn.com', 'http://amanpour.blogs.cnn.com', 'http://money.cnn.com', 'http://cnn.com/tools/index.html', 'http://cnnespanol.cnn.com', 'http://cnn.com/CNNI', 'http://business.blogs.cnn.com', 'http://cnn.com/AFRICA', 'http://cnn.com/TECH', 'http://cnn.com/BUSINESS'] FEEDS = ['http://rss.cnn.com/rss/edition.rss'] BRAND = 'cnn' s = Source('http://cnn.com', verbose=False, memoize_articles=False) # resp = mock_response_with('http://cnn.com', 'cnn_main_site') s.clean_memo_cache() s.build()
def main(): source = "The Guardian" #config = Config() #config.memoize_articles = False guardian = Source("http://www.theguardian.com/world", memoize_articles=False) guardian.build() #guardian = newspaper.build('http://theguardian.com/world', memoize_articles=False) #news_pool.set([guardian], threads_per_source=2) #news_pool.join() #print(guardian.size()) for article in [ x for x in guardian.articles if re.match(".*/world/.*", x.url) is not None ]: url = article.url a = Article(url, language='en') a.download() for i in range(10): if a.is_downloaded: break else: a.download() try: a.parse() a.nlp() except: print("Error: Not parsed/downloaded correctly.") continue html = a.html summary = a.summary keywords = a.keywords title = a.title text = a.text date = str(a.publish_date).split()[0].split("-") date[0], date[1], date[2] = date[1], date[2], date[0] date = "/".join(date) delta = re.search(r'<span class="content__dateline-time">(.*)</span>', html).group(1).replace(".", ":").split()[0] time = datetime.now() + timedelta(hours=delta) date_time = date + " " + time #print(title) #print(date_time) date_obj = datetime.datetime.strptime(date_time, '%m/%d/%Y %H:%M') #print(date_obj.strftime('%Y/%m/%d %I:%M %p')) #TODO: Add stuff to the DB try: article = { 'headline': title, 'url': url, 'text': text, 'date': date_obj } newspaper_article('The Guardian', article, keywords=keywords) except Exception as ex: print 'Article could not be created due to following error' print ex
import newspaper from newspaper import Source url = 'http://www.prothomalo.com/' bangla_paper = Source(url, memoize_articles=False, number_threads=20) bangla_paper.build() print(bangla_paper.size()) for article in bangla_paper.articles: try: article.download() article.parse() print(article.url) print('Title :\n' + str(article.title) + '\n') print('Content :\n' + str(article.text) + '\n') if (len(article.tags) > 0): print('Tags :\n' + str(article.tags) + '\n') else: print('Tags :\n{}\n') except Exception: print(Exception) ''' #print (newspaper.languages()) url = 'http://www.kalerkantho.com/online/Islamic-lifestylie/2017/12/29/583269'; #url = 'https://bdnews24.com/neighbours/2017/12/29/indian-state-of-assam-tense-ahead-of-citizens-list-targeting-illegal-bangladeshis' article = Article(url, language='bn') '''
def paper(self): ''' get newspaper articles, default source is `herald` newspaper defaults to articles of this month and year import newspaperzw news = newspaperzw.news() ''' if self.summary and self.nlp == False: # raise exception. `nltk` module missing raise Exception(self.error_msg) news_source = Providers().getUrl(self.provider).strip() name = Source(news_source, self.config) name.build() name.download() name.parse() name.download_articles() # do logging logging.debug(f"News Source build and downloaded. url: {news_source}") news_data = {} news_article = [] counter = 0 for article in name.article_urls(): images = "" keywords = "" try: name.articles[counter].download() name.articles[counter].parse() # log logging.debug( f"Article #{counter} downloaded and parsed successfuly") except: counter += 1 # log logging.error( f"Error download and parsing article #{counter}. continue.." ) continue # get in data title = name.articles[counter].title date_pub = name.articles[counter].publish_date top_image = name.articles[counter].top_image link = name.articles[counter].url text = name.articles[counter].text if (self.nlp): # do nlp stuff name.articles[counter].nlp() summary = name.articles[counter].summary for words in name.articles[counter].keywords: keywords += str(words) + ',' # log logging.debug( f"summary flag enabled. NLP summary obtained successfuly") # add to news pool, only add news of this year and month # data_pub format = 10-04-2018 21:28:09 data = {} if (self.nlp): data.update({ "article_id": randint(555, 999), "title": title, "published": date_pub, "image": top_image, "news": text, "summary": summary, "keywords": keywords.rstrip(','), "url": link }) # log logging.debug("article data with summary saved to news pool!") else: data.update({ "article_id": randint(555, 999), "title": title, "published": date_pub, "image": top_image, "news": text, "url": link }) # log logging.debug("article data added to news pool") news_article.append(data) data = {} # increment to next articles counter += 1 # build main news storage news_data.update({ 'source': name.brand, 'domain': name.domain, 'news': news_article }) # log logging.debug("News main data pool created on success") return news_data