def testFeedRedirect(self): self.setMockUser() test_feed_url = 'http://example.com/rss' self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=6), status_code=200) self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, include_summary=True, max_stories_per_period=2, schedule_period=5, ), headers=self.authHeaders()) feed = Feed.query().get() assert feed.feed_url == test_feed_url test_feed_url2 = 'http://example.com/rss2' self.set_rss_response(test_feed_url, content='', status_code=302, headers={'Location': test_feed_url2}) self.set_rss_response(test_feed_url2, content=self.buildRSS('test', items=6), status_code=200) self.pollUpdate() feed = Feed.query().get() assert feed.feed_url == test_feed_url self.set_rss_response(test_feed_url2, content=self.buildRSS('test1', items=6), status_code=200) self.set_rss_response(test_feed_url, content='', status_code=301, headers={'Location': test_feed_url2}) self.pollUpdate() feed = Feed.query().get() assert feed.feed_url == test_feed_url2
def testFeed(self): self.setMockUser() resp = self.app.get('/api/feeds', headers=self.authHeaders()) json_resp = json.loads(resp.data) assert len(json_resp['data']) == 0 self.set_rss_response("http://example.com/rss", content=self.buildRSS('test', items=10), status_code=200) test_feed_url = 'http://example.com/rss' # Should fail validation resp = self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, max_stories_per_period=0, schedule_period=5, ), headers=self.authHeaders()) assert 0 == Feed.query().count() resp = self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, include_summary='true', max_stories_per_period=1, schedule_period=5, ), headers=self.authHeaders()) json_resp = json.loads(resp.data) assert json_resp['data']['feed_url'] == test_feed_url assert 10 == Entry.query(Entry.published == True, Entry.overflow == True).count() feed_id = json_resp['data']['feed_id'] resp = self.app.get('/api/feeds/%s' % feed_id, headers=self.authHeaders()) json_resp = json.loads(resp.data) assert len(json_resp['data']['entries']) == 10 assert json_resp['data']['entries'][0]['guid'] == "http://example.com/buster/test_0" # Shouldn't be able to create two feeds for the same user resp = self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, include_summary='true', max_stories_per_period=1, schedule_period=5, ), headers=self.authHeaders()) json_resp = json.loads(resp.data) assert 1 == Feed.query().count() resp = self.app.get('/api/feeds', headers=self.authHeaders()) json_resp = json.loads(resp.data) assert len(json_resp['data']) == 1 self.set_rss_response("http://example.com/rss", content=self.buildRSS('test2'), status_code=200) feed = Feed.query().get() Entry.update_for_feed(feed) assert 11 == Entry.query().count()
def try_push_resub(): """Post all new items for feeds for a specific interval""" if request.headers.get("X-Appengine-Cron") != "true": raise ndb.Return(jsonify_error(message="Not a cron call")) unsubscribed_feeds = Feed.query(Feed.hub != None, Feed.subscribed_at_hub == False) # noqa qit = unsubscribed_feeds.iter() errors = 0 success = 0 count = 0 futures = [] while (yield qit.has_next_async()): feed = qit.next() futures.append((feed, Feed.subscribe_to_hub(feed))) for feed, future in futures: count += 1 try: yield future success += 1 except: errors += 1 logger.exception("Failed to PuSH subscribe feed:%s" % (feed.feed_url,)) logger.info("Tried to call hub for num_unsubscribed_feeds:%s success:%s, errors:%s", count, success, errors) raise ndb.Return(jsonify(status="ok"))
def testPush(self): self.setMockUser() self.set_response('http://pubsubhubbub.appspot.com', content='', status_code=200, method="POST") test_feed_url = 'http://example.com/rss' self.set_rss_response(test_feed_url, content=self.buildRSS('test', push_hub=True), status_code=200) resp = self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, include_summary=True, max_stories_per_period=1, schedule_period=5, ), headers=self.authHeaders()) feed = Feed.query().get() resp = self.app.get('/api/feeds/%s/subscribe' % (feed.key.urlsafe(), ), query_string={ "hub.mode": 'subscribe', "hub.topic": feed.feed_url, "hub.challenge": 'testing', "hub.verify_token": feed.verify_token, }) assert resp.data == 'testing' data = get_file_from_data('/data/df_feed.xml') resp = self.app.post('/api/feeds/%s/subscribe' % (feed.key.urlsafe(), ), data=data, headers={ 'Content-Type': 'application/xml', }) assert 2 == Entry.query().count() assert 1 == Entry.query(Entry.published == True, Entry.overflow == False).count() resp = self.app.post('/api/feeds/%s/subscribe' % (feed.key.urlsafe(), )) assert 2 == Entry.query(Entry.published == True).count()
def testLanguage(self): self.setMockUser() test_feed_url = 'http://example.com/rss' self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=6), status_code=200) self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, include_summary=True, max_stories_per_period=2, schedule_period=5, ), headers=self.authHeaders()) feed = Feed.query().get() assert feed.language is None self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=6, language='en-US'), status_code=200) self.pollUpdate() feed = Feed.query().get() assert feed.language == 'en'
def testSingleItemPublish(self): self.setMockUser() test_feed_url = 'http://example.com/rss' self.set_rss_response(test_feed_url, content=self.buildRSS('test'), status_code=200) resp = self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, include_summary=True, max_stories_per_period=2, schedule_period=5, ), headers=self.authHeaders()) entry = Entry.query().get() feed = Feed.query().get() resp = self.app.post('/api/feeds/%s/entries/%s/publish' % (feed.key.id(), entry.key.id()), headers=self.authHeaders())
def testRssFeedDetection(self): self.set_rss_response('http://techcrunch.com/feed/', content=self.buildRSS('test'), status_code=200) self.set_response('http://techcrunch.com', content=HTML_PAGE_TEMPLATE, status_code=200, headers={'Content-Type': 'text/html'}) resp = self.app.get('/api/feed/preview?feed_url=http://techcrunch.com', headers=self.authHeaders()) assert 1 == len(json.loads(resp.data)['data']) resp = self.app.post('/api/feeds', data=dict( feed_url='http://techcrunch.com', max_stories_per_period=1, schedule_period=5, ), headers=self.authHeaders()) feed = Feed.query().get() assert feed.feed_url == 'http://techcrunch.com/feed/'
def testBadFeedRemoval(self): self.setMockUser() test_feed_url = 'http://example.com/rss' self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=1), status_code=200) self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, include_summary=True, max_stories_per_period=2, schedule_period=5, ), headers=self.authHeaders()) feed = Feed.query().get() feed.last_successful_fetch = datetime.now() - timedelta(days=2) feed.put() self.set_rss_response(test_feed_url, content=self.buildRSS('test1', items=1), status_code=500) self.pollUpdate() assert feed.feed_disabled is True
def testFeedPreview(self): self.set_rss_response('http://techcrunch.com/feed/', content=self.buildRSS('test'), status_code=200) resp = self.app.get('/api/feed/preview?feed_url=http://techcrunch.com/feed/', headers=self.authHeaders()) assert 1 == len(json.loads(resp.data)['data']) self.set_rss_response('http://techcrunch.com/feed/2', content=self.buildRSS('test'), status_code=500) resp = self.app.get('/api/feed/preview?feed_url=http://techcrunch.com/feed/2', headers=self.authHeaders()) assert json.loads(resp.data)['message'] test_feed_url = 'http://example.com/rss' self.set_rss_response(test_feed_url, content=self.buildRSS('test'), status_code=200) resp = self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, include_summary=True, max_stories_per_period=2, schedule_period=5, ), headers=self.authHeaders()) assert 1 == Entry.query(Entry.published == True, Entry.overflow == True).count() feed = Feed.query().get() resp = self.app.get('/api/feeds/%s/preview' % (feed.key.id(), ), headers=self.authHeaders()) assert 'data' in json.loads(resp.data)
def testDbRaceCondition(self): self.setMockUser() test_feed_url = 'http://example.com/rss' self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=1), status_code=200) self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, include_summary=True, max_stories_per_period=2, schedule_period=5, ), headers=self.authHeaders()) feed = Feed.query().get() key = ndb.Key(Entry, '1', parent=feed.key) entry = Entry(key=key, guid='1') entry.put() entry_2 = Entry(key=key, guid='2') entry_2.put() assert Entry.query().count() == 2