def try_push_resub():
"""Post all new items for feeds for a specific interval"""
if request.headers.get("X-Appengine-Cron") != "true":
raise ndb.Return(jsonify_error(message="Not a cron call"))
unsubscribed_feeds = Feed.query(Feed.hub != None, Feed.subscribed_at_hub == False) # noqa
qit = unsubscribed_feeds.iter()
errors = 0
success = 0
count = 0
futures = []
while (yield qit.has_next_async()):
feed = qit.next()
futures.append((feed, Feed.subscribe_to_hub(feed)))
for feed, future in futures:
count += 1
try:
yield future
success += 1
except:
errors += 1
logger.exception("Failed to PuSH subscribe feed:%s" % (feed.feed_url,))
logger.info("Tried to call hub for num_unsubscribed_feeds:%s success:%s, errors:%s", count, success, errors)
raise ndb.Return(jsonify(status="ok"))
def testFeedRedirect(self):
self.setMockUser()
test_feed_url = 'http://example.com/rss'
self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=6), status_code=200)
self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
include_summary=True,
max_stories_per_period=2,
schedule_period=5,
), headers=self.authHeaders())
feed = Feed.query().get()
assert feed.feed_url == test_feed_url
test_feed_url2 = 'http://example.com/rss2'
self.set_rss_response(test_feed_url, content='', status_code=302, headers={'Location': test_feed_url2})
self.set_rss_response(test_feed_url2, content=self.buildRSS('test', items=6), status_code=200)
self.pollUpdate()
feed = Feed.query().get()
assert feed.feed_url == test_feed_url
self.set_rss_response(test_feed_url2, content=self.buildRSS('test1', items=6), status_code=200)
self.set_rss_response(test_feed_url, content='', status_code=301, headers={'Location': test_feed_url2})
self.pollUpdate()
feed = Feed.query().get()
assert feed.feed_url == test_feed_url2
def try_push_resub():
"""Post all new items for feeds for a specific interval"""
if request.headers.get('X-Appengine-Cron') != 'true':
raise ndb.Return(jsonify_error(message='Not a cron call'))
unsubscribed_feeds = Feed.query(Feed.hub != None, Feed.subscribed_at_hub == False) # noqa
qit = unsubscribed_feeds.iter()
errors = 0
success = 0
count = 0
futures = []
while (yield qit.has_next_async()):
feed = qit.next()
futures.append((feed, Feed.subscribe_to_hub(feed)))
for feed, future in futures:
count += 1
try:
yield future
success += 1
except:
errors += 1
logger.exception('Failed to PuSH subscribe feed:%s' % (feed.feed_url, ))
logger.info('Tried to call hub for num_unsubscribed_feeds:%s success:%s, errors:%s', count, success, errors)
raise ndb.Return(jsonify(status='ok'))
def testFeed(self):
self.setMockUser()
resp = self.app.get('/api/feeds', headers=self.authHeaders())
json_resp = json.loads(resp.data)
assert len(json_resp['data']) == 0
self.set_rss_response("http://example.com/rss", content=self.buildRSS('test', items=10), status_code=200)
test_feed_url = 'http://example.com/rss'
# Should fail validation
resp = self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
max_stories_per_period=0,
schedule_period=5,
), headers=self.authHeaders())
assert 0 == Feed.query().count()
resp = self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
include_summary='true',
max_stories_per_period=1,
schedule_period=5,
), headers=self.authHeaders())
json_resp = json.loads(resp.data)
assert json_resp['data']['feed_url'] == test_feed_url
assert 10 == Entry.query(Entry.published == True, Entry.overflow == True).count()
feed_id = json_resp['data']['feed_id']
resp = self.app.get('/api/feeds/%s' % feed_id, headers=self.authHeaders())
json_resp = json.loads(resp.data)
assert len(json_resp['data']['entries']) == 10
assert json_resp['data']['entries'][0]['guid'] == "http://example.com/buster/test_0"
# Shouldn't be able to create two feeds for the same user
resp = self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
include_summary='true',
max_stories_per_period=1,
schedule_period=5,
), headers=self.authHeaders())
json_resp = json.loads(resp.data)
assert 1 == Feed.query().count()
resp = self.app.get('/api/feeds', headers=self.authHeaders())
json_resp = json.loads(resp.data)
assert len(json_resp['data']) == 1
self.set_rss_response("http://example.com/rss", content=self.buildRSS('test2'), status_code=200)
feed = Feed.query().get()
Entry.update_for_feed(feed)
assert 11 == Entry.query().count()
def limit_feeds_for_channel_id(channel_id):
"""Limit all feeds connected to a channel"""
max_stories_per_period = cast_int(
request.form.get('max_stories_per_period'), default=None)
schedule_period = cast_int(request.form.get('schedule_period'),
default=None)
dump_excess_in_period = bool(request.form.get('dump_excess_in_period'))
users_feeds = [
feed for feed in Feed.for_channel(channel_id) if feed.visible
]
futures = []
for feed in users_feeds:
if dump_excess_in_period:
feed.dump_excess_in_period = True
if max_stories_per_period or schedule_period:
feed.manual_control = True
if max_stories_per_period:
feed.max_stories_per_period = max_stories_per_period
if schedule_period:
feed.schedule_period = schedule_period
futures.append(feed.put_async())
for future in futures:
yield future
users_feeds = export_feeds_to_json(users_feeds)
raise ndb.Return(jsonify(status='ok', data=users_feeds))
def testPush(self):
self.setMockUser()
self.set_response('http://pubsubhubbub.appspot.com', content='', status_code=200, method="POST")
test_feed_url = 'http://example.com/rss'
self.set_rss_response(test_feed_url, content=self.buildRSS('test', push_hub=True), status_code=200)
resp = self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
include_summary=True,
max_stories_per_period=1,
schedule_period=5,
), headers=self.authHeaders())
feed = Feed.query().get()
resp = self.app.get('/api/feeds/%s/subscribe' % (feed.key.urlsafe(), ), query_string={
"hub.mode": 'subscribe',
"hub.topic": feed.feed_url,
"hub.challenge": 'testing',
"hub.verify_token": feed.verify_token,
})
assert resp.data == 'testing'
data = get_file_from_data('/data/df_feed.xml')
resp = self.app.post('/api/feeds/%s/subscribe' % (feed.key.urlsafe(), ), data=data, headers={
'Content-Type': 'application/xml',
})
assert 2 == Entry.query().count()
assert 1 == Entry.query(Entry.published == True, Entry.overflow == False).count()
resp = self.app.post('/api/feeds/%s/subscribe' % (feed.key.urlsafe(), ))
assert 2 == Entry.query(Entry.published == True).count()
def update_all_feeds(interval_id):
"""Update all feeds for a specific interval"""
if request.headers.get('X-Appengine-Cron') != 'true':
raise ndb.Return(jsonify_error(message='Not a cron call'))
for feed_type, feed_class in FEED_TYPE_TO_CLASS.iteritems():
feeds = Feed.for_interval(interval_id)
success = 0
more = True
cursor = None
futures = []
while more:
feeds_to_fetch, cursor, more = yield feeds.fetch_page_async(BATCH_SIZE, start_cursor=cursor)
feeds_to_fetch = filter(lambda x: getattr(x, 'external_polling_bucket', DEFAULT_POLLING_BUCKET) == DEFAULT_POLLING_BUCKET, feeds_to_fetch)
keys = ','.join([x.key.urlsafe() for x in feeds_to_fetch])
if not keys:
continue
futures.append(Queue('poll').add_async(Task(url=url_for('tq_feed_poll-canonical'), method='POST', params={'keys': keys})))
success += 1
for future in futures:
yield future
logger.info('queued poll for %d feeds at interval_id=%s', success, interval_id)
raise ndb.Return(jsonify(status='ok'))
def update_all_feeds(interval_id):
"""Update all feeds for a specific interval"""
if request.headers.get("X-Appengine-Cron") != "true":
raise ndb.Return(jsonify_error(message="Not a cron call"))
for feed_type, feed_class in FEED_TYPE_TO_CLASS.iteritems():
feeds = Feed.for_interval(interval_id)
success = 0
more = True
cursor = None
futures = []
while more:
feeds_to_fetch, cursor, more = yield feeds.fetch_page_async(BATCH_SIZE, start_cursor=cursor)
feeds_to_fetch = filter(
lambda x: getattr(x, "external_polling_bucket", DEFAULT_POLLING_BUCKET) == DEFAULT_POLLING_BUCKET,
feeds_to_fetch,
)
keys = ",".join([x.key.urlsafe() for x in feeds_to_fetch])
if not keys:
continue
futures.append(
Queue("poll").add_async(
Task(url=url_for("tq_feed_poll-canonical"), method="POST", params={"keys": keys})
)
)
success += 1
for future in futures:
yield future
logger.info("queued poll for %d feeds at interval_id=%s", success, interval_id)
raise ndb.Return(jsonify(status="ok"))
def limit_feeds_for_channel_id(channel_id):
"""Limit all feeds connected to a channel"""
max_stories_per_period = cast_int(request.form.get('max_stories_per_period'), default=None)
schedule_period = cast_int(request.form.get('schedule_period'), default=None)
dump_excess_in_period = bool(request.form.get('dump_excess_in_period'))
users_feeds = [feed for feed in Feed.for_channel(channel_id) if feed.visible]
futures = []
for feed in users_feeds:
if dump_excess_in_period:
feed.dump_excess_in_period = True
if max_stories_per_period or schedule_period:
feed.manual_control = True
if max_stories_per_period:
feed.max_stories_per_period = max_stories_per_period
if schedule_period:
feed.schedule_period = schedule_period
futures.append(feed.put_async())
for future in futures:
yield future
users_feeds = export_feeds_to_json(users_feeds)
raise ndb.Return(jsonify(status='ok', data=users_feeds))
def testLanguage(self):
self.setMockUser()
test_feed_url = 'http://example.com/rss'
self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=6), status_code=200)
self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
include_summary=True,
max_stories_per_period=2,
schedule_period=5,
), headers=self.authHeaders())
feed = Feed.query().get()
assert feed.language is None
self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=6, language='en-US'), status_code=200)
self.pollUpdate()
feed = Feed.query().get()
assert feed.language == 'en'
def feed_validate():
"""preview a feed"""
feed_type = int(request.form.get('feed_type', 1))
form = FEED_TYPE_TO_CLASS[feed_type].preview_form(request.form)
if not form.validate():
raise ndb.Return(jsonify(status='error', form_errors=form.errors))
feed = Feed()
form.populate_obj(feed)
feed.preview = True
error = None
parsed_feed = None
try:
parsed_feed, resp, feed = yield fetch_parsed_feed_for_feed(feed)
feed.update_feed_from_parsed_feed(parsed_feed)
if len(parsed_feed.entries) == 0:
error = 'The url you entred is not a valid feed.'
except FetchException, e:
error = unicode(e)
def feed_validate():
"""preview a feed"""
feed_type = int(request.form.get('feed_type', 1))
form = FEED_TYPE_TO_CLASS[feed_type].preview_form(request.form)
if not form.validate():
raise ndb.Return(jsonify(status='error', form_errors=form.errors))
feed = Feed()
form.populate_obj(feed)
feed.preview = True
error = None
parsed_feed = None
try:
parsed_feed, resp, feed = yield fetch_parsed_feed_for_feed(feed)
feed.update_feed_from_parsed_feed(parsed_feed)
if len(parsed_feed.entries) == 0:
error = 'The url you entred is not a valid feed.'
except FetchException, e:
error = unicode(e)
def testSingleItemPublish(self):
self.setMockUser()
test_feed_url = 'http://example.com/rss'
self.set_rss_response(test_feed_url, content=self.buildRSS('test'), status_code=200)
resp = self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
include_summary=True,
max_stories_per_period=2,
schedule_period=5,
), headers=self.authHeaders())
entry = Entry.query().get()
feed = Feed.query().get()
resp = self.app.post('/api/feeds/%s/entries/%s/publish' % (feed.key.id(), entry.key.id()), headers=self.authHeaders())
def testRssFeedDetection(self):
self.set_rss_response('http://techcrunch.com/feed/', content=self.buildRSS('test'), status_code=200)
self.set_response('http://techcrunch.com', content=HTML_PAGE_TEMPLATE, status_code=200, headers={'Content-Type': 'text/html'})
resp = self.app.get('/api/feed/preview?feed_url=http://techcrunch.com', headers=self.authHeaders())
assert 1 == len(json.loads(resp.data)['data'])
resp = self.app.post('/api/feeds', data=dict(
feed_url='http://techcrunch.com',
max_stories_per_period=1,
schedule_period=5,
), headers=self.authHeaders())
feed = Feed.query().get()
assert feed.feed_url == 'http://techcrunch.com/feed/'
def testBadFeedRemoval(self):
self.setMockUser()
test_feed_url = 'http://example.com/rss'
self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=1), status_code=200)
self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
include_summary=True,
max_stories_per_period=2,
schedule_period=5,
), headers=self.authHeaders())
feed = Feed.query().get()
feed.last_successful_fetch = datetime.now() - timedelta(days=2)
feed.put()
self.set_rss_response(test_feed_url, content=self.buildRSS('test1', items=1), status_code=500)
self.pollUpdate()
assert feed.feed_disabled is True
def testFeedPreview(self):
self.set_rss_response('http://techcrunch.com/feed/', content=self.buildRSS('test'), status_code=200)
resp = self.app.get('/api/feed/preview?feed_url=http://techcrunch.com/feed/', headers=self.authHeaders())
assert 1 == len(json.loads(resp.data)['data'])
self.set_rss_response('http://techcrunch.com/feed/2', content=self.buildRSS('test'), status_code=500)
resp = self.app.get('/api/feed/preview?feed_url=http://techcrunch.com/feed/2', headers=self.authHeaders())
assert json.loads(resp.data)['message']
test_feed_url = 'http://example.com/rss'
self.set_rss_response(test_feed_url, content=self.buildRSS('test'), status_code=200)
resp = self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
include_summary=True,
max_stories_per_period=2,
schedule_period=5,
), headers=self.authHeaders())
assert 1 == Entry.query(Entry.published == True, Entry.overflow == True).count()
feed = Feed.query().get()
resp = self.app.get('/api/feeds/%s/preview' % (feed.key.id(), ), headers=self.authHeaders())
assert 'data' in json.loads(resp.data)
def testDbRaceCondition(self):
self.setMockUser()
test_feed_url = 'http://example.com/rss'
self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=1), status_code=200)
self.app.post('/api/feeds', data=dict(
feed_url=test_feed_url,
include_summary=True,
max_stories_per_period=2,
schedule_period=5,
), headers=self.authHeaders())
feed = Feed.query().get()
key = ndb.Key(Entry, '1', parent=feed.key)
entry = Entry(key=key, guid='1')
entry.put()
entry_2 = Entry(key=key, guid='2')
entry_2.put()
assert Entry.query().count() == 2
def test_subscribe(self): u1 = User(username='******', email='*****@*****.**') u2 = User(username='******', email='*****@*****.**') f = Feed(name='susan_feed', owner_id=u2.id) db.session.add(u1) db.session.add(u2) db.session.add(f) db.session.commit() self.assertEqual(u1.subscriptions.all(), []) self.assertEqual(u2.subscriptions.all(), []) u1.subscribe(f) db.session.commit() self.assertTrue(u1.is_subscribed(f)) self.assertEqual(u1.subscriptions.count(), 1) self.assertEqual(u1.subscriptions.first().name, 'susan_feed') self.assertEqual(f.subscribers.count(), 1) self.assertEqual(f.subscribers.first().username, 'john') u1.unsubscribe(f) db.session.commit() self.assertFalse(u1.is_subscribed(f)) self.assertEqual(u1.subscriptions.count(), 0) self.assertEqual(f.subscribers.count(), 0)
