def testFeedRedirect(self):
self.setMockUser()
test_feed_url = 'http://example.com/rss'
self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=6), status_code=200)
self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
include_summary=True,
max_stories_per_period=2,
schedule_period=5,
), headers=self.authHeaders())
feed = Feed.query().get()
assert feed.feed_url == test_feed_url
test_feed_url2 = 'http://example.com/rss2'
self.set_rss_response(test_feed_url, content='', status_code=302, headers={'Location': test_feed_url2})
self.set_rss_response(test_feed_url2, content=self.buildRSS('test', items=6), status_code=200)
self.pollUpdate()
feed = Feed.query().get()
assert feed.feed_url == test_feed_url
self.set_rss_response(test_feed_url2, content=self.buildRSS('test1', items=6), status_code=200)
self.set_rss_response(test_feed_url, content='', status_code=301, headers={'Location': test_feed_url2})
self.pollUpdate()
feed = Feed.query().get()
assert feed.feed_url == test_feed_url2
def testFeed(self):
self.setMockUser()
resp = self.app.get('/api/feeds', headers=self.authHeaders())
json_resp = json.loads(resp.data)
assert len(json_resp['data']) == 0
self.set_rss_response("http://example.com/rss", content=self.buildRSS('test', items=10), status_code=200)
test_feed_url = 'http://example.com/rss'
# Should fail validation
resp = self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
max_stories_per_period=0,
schedule_period=5,
), headers=self.authHeaders())
assert 0 == Feed.query().count()
resp = self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
include_summary='true',
max_stories_per_period=1,
schedule_period=5,
), headers=self.authHeaders())
json_resp = json.loads(resp.data)
assert json_resp['data']['feed_url'] == test_feed_url
assert 10 == Entry.query(Entry.published == True, Entry.overflow == True).count()
feed_id = json_resp['data']['feed_id']
resp = self.app.get('/api/feeds/%s' % feed_id, headers=self.authHeaders())
json_resp = json.loads(resp.data)
assert len(json_resp['data']['entries']) == 10
assert json_resp['data']['entries'][0]['guid'] == "http://example.com/buster/test_0"
# Shouldn't be able to create two feeds for the same user
resp = self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
include_summary='true',
max_stories_per_period=1,
schedule_period=5,
), headers=self.authHeaders())
json_resp = json.loads(resp.data)
assert 1 == Feed.query().count()
resp = self.app.get('/api/feeds', headers=self.authHeaders())
json_resp = json.loads(resp.data)
assert len(json_resp['data']) == 1
self.set_rss_response("http://example.com/rss", content=self.buildRSS('test2'), status_code=200)
feed = Feed.query().get()
Entry.update_for_feed(feed)
assert 11 == Entry.query().count()
def try_push_resub():
"""Post all new items for feeds for a specific interval"""
if request.headers.get("X-Appengine-Cron") != "true":
raise ndb.Return(jsonify_error(message="Not a cron call"))
unsubscribed_feeds = Feed.query(Feed.hub != None, Feed.subscribed_at_hub == False) # noqa
qit = unsubscribed_feeds.iter()
errors = 0
success = 0
count = 0
futures = []
while (yield qit.has_next_async()):
feed = qit.next()
futures.append((feed, Feed.subscribe_to_hub(feed)))
for feed, future in futures:
count += 1
try:
yield future
success += 1
except:
errors += 1
logger.exception("Failed to PuSH subscribe feed:%s" % (feed.feed_url,))
logger.info("Tried to call hub for num_unsubscribed_feeds:%s success:%s, errors:%s", count, success, errors)
raise ndb.Return(jsonify(status="ok"))
def testPush(self):
self.setMockUser()
self.set_response('http://pubsubhubbub.appspot.com', content='', status_code=200, method="POST")
test_feed_url = 'http://example.com/rss'
self.set_rss_response(test_feed_url, content=self.buildRSS('test', push_hub=True), status_code=200)
resp = self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
include_summary=True,
max_stories_per_period=1,
schedule_period=5,
), headers=self.authHeaders())
feed = Feed.query().get()
resp = self.app.get('/api/feeds/%s/subscribe' % (feed.key.urlsafe(), ), query_string={
"hub.mode": 'subscribe',
"hub.topic": feed.feed_url,
"hub.challenge": 'testing',
"hub.verify_token": feed.verify_token,
})
assert resp.data == 'testing'
data = get_file_from_data('/data/df_feed.xml')
resp = self.app.post('/api/feeds/%s/subscribe' % (feed.key.urlsafe(), ), data=data, headers={
'Content-Type': 'application/xml',
})
assert 2 == Entry.query().count()
assert 1 == Entry.query(Entry.published == True, Entry.overflow == False).count()
resp = self.app.post('/api/feeds/%s/subscribe' % (feed.key.urlsafe(), ))
assert 2 == Entry.query(Entry.published == True).count()
def testLanguage(self):
self.setMockUser()
test_feed_url = 'http://example.com/rss'
self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=6), status_code=200)
self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
include_summary=True,
max_stories_per_period=2,
schedule_period=5,
), headers=self.authHeaders())
feed = Feed.query().get()
assert feed.language is None
self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=6, language='en-US'), status_code=200)
self.pollUpdate()
feed = Feed.query().get()
assert feed.language == 'en'
def testSingleItemPublish(self):
self.setMockUser()
test_feed_url = 'http://example.com/rss'
self.set_rss_response(test_feed_url, content=self.buildRSS('test'), status_code=200)
resp = self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
include_summary=True,
max_stories_per_period=2,
schedule_period=5,
), headers=self.authHeaders())
entry = Entry.query().get()
feed = Feed.query().get()
resp = self.app.post('/api/feeds/%s/entries/%s/publish' % (feed.key.id(), entry.key.id()), headers=self.authHeaders())
def testRssFeedDetection(self):
self.set_rss_response('http://techcrunch.com/feed/', content=self.buildRSS('test'), status_code=200)
self.set_response('http://techcrunch.com', content=HTML_PAGE_TEMPLATE, status_code=200, headers={'Content-Type': 'text/html'})
resp = self.app.get('/api/feed/preview?feed_url=http://techcrunch.com', headers=self.authHeaders())
assert 1 == len(json.loads(resp.data)['data'])
resp = self.app.post('/api/feeds', data=dict(
feed_url='http://techcrunch.com',
max_stories_per_period=1,
schedule_period=5,
), headers=self.authHeaders())
feed = Feed.query().get()
assert feed.feed_url == 'http://techcrunch.com/feed/'
def testBadFeedRemoval(self):
self.setMockUser()
test_feed_url = 'http://example.com/rss'
self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=1), status_code=200)
self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
include_summary=True,
max_stories_per_period=2,
schedule_period=5,
), headers=self.authHeaders())
feed = Feed.query().get()
feed.last_successful_fetch = datetime.now() - timedelta(days=2)
feed.put()
self.set_rss_response(test_feed_url, content=self.buildRSS('test1', items=1), status_code=500)
self.pollUpdate()
assert feed.feed_disabled is True
def testFeedPreview(self):
self.set_rss_response('http://techcrunch.com/feed/', content=self.buildRSS('test'), status_code=200)
resp = self.app.get('/api/feed/preview?feed_url=http://techcrunch.com/feed/', headers=self.authHeaders())
assert 1 == len(json.loads(resp.data)['data'])
self.set_rss_response('http://techcrunch.com/feed/2', content=self.buildRSS('test'), status_code=500)
resp = self.app.get('/api/feed/preview?feed_url=http://techcrunch.com/feed/2', headers=self.authHeaders())
assert json.loads(resp.data)['message']
test_feed_url = 'http://example.com/rss'
self.set_rss_response(test_feed_url, content=self.buildRSS('test'), status_code=200)
resp = self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
include_summary=True,
max_stories_per_period=2,
schedule_period=5,
), headers=self.authHeaders())
assert 1 == Entry.query(Entry.published == True, Entry.overflow == True).count()
feed = Feed.query().get()
resp = self.app.get('/api/feeds/%s/preview' % (feed.key.id(), ), headers=self.authHeaders())
assert 'data' in json.loads(resp.data)
def testDbRaceCondition(self):
self.setMockUser()
test_feed_url = 'http://example.com/rss'
self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=1), status_code=200)
self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
include_summary=True,
max_stories_per_period=2,
schedule_period=5,
), headers=self.authHeaders())
feed = Feed.query().get()
key = ndb.Key(Entry, '1', parent=feed.key)
entry = Entry(key=key, guid='1')
entry.put()
entry_2 = Entry(key=key, guid='2')
entry_2.put()
assert Entry.query().count() == 2