def test_basic(): news = client.get_news_from_source() print news assert len(news) > 0 news = client.get_news_from_source(sources=['bbc-news']) assert len(news) > 0 print 'test_basic passed!'
def test_basic(): ''' ''' news = client.get_news_from_source() print news assert len(news) > 0 news = client.get_news_from_source(sources=['cnn'], sortBy='top') assert len(news) > 0 print 'test_basic passed'
def test_basic(): """Test news API client basically""" news = client.get_news_from_source() print(news) assert news news = client.get_news_from_source(sources=['the-new-york-times'], sort_by='top') print(news) assert news print('test_basic passed!')
def run(): logging.info("[news_monitor] start running") redis_client = redis.StrictRedis(REDIS_HOST, REDIS_PORT) cloudamqp_client = CloudAMQPClient(SCRAPE_NEWS_TASK_QUEUE_URL, SCRAPE_NEWS_TASK_QUEUE_NAME) while True: news_list = news_api_client.get_news_from_source(NEWS_SOURCES) logging.info("[news_monitor] get %d from news API", len(news_list)) number_of_news = 0 for news in news_list: news_digest = hashlib.md5( news['title'].encode('utf-8')).hexdigest() if redis_client.get(news_digest) is None: number_of_news += 1 news['digest'] = news_digest if news['publishedAt'] is None: news['publishedAt'] = datetime.datetime.utcnow().strftime( "%Y-%m-%dT%H:%M:%SZ") redis_client.set(news_digest, "True") redis_client.expire(news_digest, NEWS_TIME_OUT_IN_SECONDS) cloudamqp_client.send_message(news, "[news_monitor]") logging.info("[news_monitor] fetch %d news" % number_of_news) cloudamqp_client.sleep(SLEEP_TIME_IN_SECONDS)
'bloomberg', 'cnn', 'entertainment-weekly', 'espn', 'ign', 'techcrunch', 'the-new-york-times', 'the-wall-street-journal', 'the-washington-post' ] redis_client = redis.StrictRedis(REDIS_HOST, REDIS_PORT) cloudAMQP_client = CloudAMQPClient(SCRAPE_NEWS_TASK_QUEUE_URL, SCRAPE_NEWS_TASK_QUEUE_NAME) while True: news_list = news_api_client.get_news_from_source(NEWS_SOURCES) num_of_news_news = 0 print(news_list) for news in news_list: news_digest = hashlib.md5(news['title'].encode('utf-8')).digest().encode('base64') if redis_client.get(news_digest) is None: num_of_news_news = num_of_news_news + 1 news['digest'] = news_digest if news['publishedAt'] is None: news['publishedAt'] = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") redis_client.set(news_digest, "True") redis_client.expire(news_digest, NEWS_TIME_OUT_IN_SECONDS)