Ejemplo n.º 1
0
def scrape_article(article):
    try:
        article['fulltext'] = utils.scrape(article['link'])
        logger.info('[scraped] ' + article['link'])
    except:
        logger.error(
            '[scrapper] [error] Unable to scrape %s' % article['link'],
            exc_info=True)
    return article
Ejemplo n.º 2
0
def scrape_articles(limit=100):
    articles = ArticleModel.objects.exclude(fulltext__gt='').order_by('-date')
    count = 0
    for article in articles:
        try:
            article.fulltext = utils.scrape(article.link)
            article.save()
            print '[scraped] ', article.link
            count += 1
        except:
            print '[scrapper] [error] Unable to scrape ', article.link
            article.fulltext = article.content
            article.save()

        if limit and count >= limit:
            break
Ejemplo n.º 3
0
 def test_news24_1(self):
     text = scrape('http://www.news24.com/SouthAfrica/News/R25m-worth-of-counterfeit-CDs-seized-20140320')
     print text
     self.assertTrue('producing the fake CDs' in text)
Ejemplo n.º 4
0
 def test_rt(self):
     text = scrape('http://rt.com/politics/lavrov-crimea-slander-annexation-289/')
     print text
     self.assertTrue('On Thursday the bills were passed' in text)
Ejemplo n.º 5
0
 def test_eonline(self):
     text = scrape('http://www.eonline.com/news/523549/shirtless-andrew-garfield-hits-the-waves-to-teach-autistic-kids-how-to-surf-swoon-alert?cmpid=rss-000000-rssfeed-365-topstories&utm_source=eonline&utm_medium=rssfeeds&utm_campaign=rss_topstories')
     print text
     self.assertTrue('Because his girlfriend' in text)
Ejemplo n.º 6
0
 def test_supersport(self):
     text = scrape('http://www.supersport.com/motorsport/article.aspx?Id=2367604')
     print text
     self.assertTrue('Agag told AFP that people' in text)
Ejemplo n.º 7
0
 def test_sport24(self):
     text = scrape('http://www.sport24.co.za/Golf/PGATour/Scott-sizzles-at-Bay-Hill-20140321')
     print text
     self.assertTrue('World No 2 Scott' in text)
Ejemplo n.º 8
0
 def test_moneyweb(self):
     text = scrape('http://www.moneyweb.co.za/moneyweb-financial/rand-slips-against-dollar-2')
     print text
     self.assertTrue('consumer inflation data for February' in text)
Ejemplo n.º 9
0
 def test_newage(self):
     text = scrape('http://www.thenewage.co.za/Detail.aspx?news_id=121438&cat_id=1007')
     print text
     self.assertTrue('KwaZulu-Natal transport' in text)
Ejemplo n.º 10
0
 def test_ewn(self):
     text = scrape('http://ewn.co.za/2014/03/21/Police-hunt-for-Bosmont-school-robbers')
     print text
     self.assertTrue('The pair targeted school officials' in text)
Ejemplo n.º 11
0
 def test_timeslive2(self):
     text = scrape('http://www.timeslive.co.za/politics/2014/03/20/kzn-leads-in-unauthorised-expenditure-with-r1.2billion')
     print text
     self.assertTrue('Researcher Georgina Alexander' in text)
Ejemplo n.º 12
0
 def test_timeslive(self):
     text = scrape('http://www.timeslive.co.za/world/2014/03/20/more-eu-sanctions-on-russia-g-8-suspended-merkel')
     print text
     self.assertTrue('United States slapped sanctions' in text)
Ejemplo n.º 13
0
 def test_sowetan(self):
     text = scrape('http://www.sowetanlive.co.za/news/2014/03/14/oscar-trial-first-cop-ignorant-about-window')
     print text
     self.assertTrue('Botha was replaced' in text)
Ejemplo n.º 14
0
 def test_news24_3(self):
     text = scrape('http://www.news24.com/SouthAfrica/News/Energy-DG-steps-down-for-health-reasons-20140320')
     print text
     self.assertTrue('Deputy director general of Petroleum' in text)
Ejemplo n.º 15
0
 def test_news24_2(self):
     text = scrape('http://www.news24.com/SouthAfrica/News/Transport-dept-to-probe-Kokstad-crash-20140320')
     print text
     self.assertTrue('At least two people were seriously' in text)