def try_push_resub(): """Post all new items for feeds for a specific interval""" if request.headers.get("X-Appengine-Cron") != "true": raise ndb.Return(jsonify_error(message="Not a cron call")) unsubscribed_feeds = Feed.query(Feed.hub != None, Feed.subscribed_at_hub == False) # noqa qit = unsubscribed_feeds.iter() errors = 0 success = 0 count = 0 futures = [] while (yield qit.has_next_async()): feed = qit.next() futures.append((feed, Feed.subscribe_to_hub(feed))) for feed, future in futures: count += 1 try: yield future success += 1 except: errors += 1 logger.exception("Failed to PuSH subscribe feed:%s" % (feed.feed_url,)) logger.info("Tried to call hub for num_unsubscribed_feeds:%s success:%s, errors:%s", count, success, errors) raise ndb.Return(jsonify(status="ok"))
def testFeedRedirect(self): self.setMockUser() test_feed_url = 'http://example.com/rss' self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=6), status_code=200) self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, include_summary=True, max_stories_per_period=2, schedule_period=5, ), headers=self.authHeaders()) feed = Feed.query().get() assert feed.feed_url == test_feed_url test_feed_url2 = 'http://example.com/rss2' self.set_rss_response(test_feed_url, content='', status_code=302, headers={'Location': test_feed_url2}) self.set_rss_response(test_feed_url2, content=self.buildRSS('test', items=6), status_code=200) self.pollUpdate() feed = Feed.query().get() assert feed.feed_url == test_feed_url self.set_rss_response(test_feed_url2, content=self.buildRSS('test1', items=6), status_code=200) self.set_rss_response(test_feed_url, content='', status_code=301, headers={'Location': test_feed_url2}) self.pollUpdate() feed = Feed.query().get() assert feed.feed_url == test_feed_url2
def try_push_resub(): """Post all new items for feeds for a specific interval""" if request.headers.get('X-Appengine-Cron') != 'true': raise ndb.Return(jsonify_error(message='Not a cron call')) unsubscribed_feeds = Feed.query(Feed.hub != None, Feed.subscribed_at_hub == False) # noqa qit = unsubscribed_feeds.iter() errors = 0 success = 0 count = 0 futures = [] while (yield qit.has_next_async()): feed = qit.next() futures.append((feed, Feed.subscribe_to_hub(feed))) for feed, future in futures: count += 1 try: yield future success += 1 except: errors += 1 logger.exception('Failed to PuSH subscribe feed:%s' % (feed.feed_url, )) logger.info('Tried to call hub for num_unsubscribed_feeds:%s success:%s, errors:%s', count, success, errors) raise ndb.Return(jsonify(status='ok'))
def testFeed(self): self.setMockUser() resp = self.app.get('/api/feeds', headers=self.authHeaders()) json_resp = json.loads(resp.data) assert len(json_resp['data']) == 0 self.set_rss_response("http://example.com/rss", content=self.buildRSS('test', items=10), status_code=200) test_feed_url = 'http://example.com/rss' # Should fail validation resp = self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, max_stories_per_period=0, schedule_period=5, ), headers=self.authHeaders()) assert 0 == Feed.query().count() resp = self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, include_summary='true', max_stories_per_period=1, schedule_period=5, ), headers=self.authHeaders()) json_resp = json.loads(resp.data) assert json_resp['data']['feed_url'] == test_feed_url assert 10 == Entry.query(Entry.published == True, Entry.overflow == True).count() feed_id = json_resp['data']['feed_id'] resp = self.app.get('/api/feeds/%s' % feed_id, headers=self.authHeaders()) json_resp = json.loads(resp.data) assert len(json_resp['data']['entries']) == 10 assert json_resp['data']['entries'][0]['guid'] == "http://example.com/buster/test_0" # Shouldn't be able to create two feeds for the same user resp = self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, include_summary='true', max_stories_per_period=1, schedule_period=5, ), headers=self.authHeaders()) json_resp = json.loads(resp.data) assert 1 == Feed.query().count() resp = self.app.get('/api/feeds', headers=self.authHeaders()) json_resp = json.loads(resp.data) assert len(json_resp['data']) == 1 self.set_rss_response("http://example.com/rss", content=self.buildRSS('test2'), status_code=200) feed = Feed.query().get() Entry.update_for_feed(feed) assert 11 == Entry.query().count()
def limit_feeds_for_channel_id(channel_id): """Limit all feeds connected to a channel""" max_stories_per_period = cast_int( request.form.get('max_stories_per_period'), default=None) schedule_period = cast_int(request.form.get('schedule_period'), default=None) dump_excess_in_period = bool(request.form.get('dump_excess_in_period')) users_feeds = [ feed for feed in Feed.for_channel(channel_id) if feed.visible ] futures = [] for feed in users_feeds: if dump_excess_in_period: feed.dump_excess_in_period = True if max_stories_per_period or schedule_period: feed.manual_control = True if max_stories_per_period: feed.max_stories_per_period = max_stories_per_period if schedule_period: feed.schedule_period = schedule_period futures.append(feed.put_async()) for future in futures: yield future users_feeds = export_feeds_to_json(users_feeds) raise ndb.Return(jsonify(status='ok', data=users_feeds))
def testPush(self): self.setMockUser() self.set_response('http://pubsubhubbub.appspot.com', content='', status_code=200, method="POST") test_feed_url = 'http://example.com/rss' self.set_rss_response(test_feed_url, content=self.buildRSS('test', push_hub=True), status_code=200) resp = self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, include_summary=True, max_stories_per_period=1, schedule_period=5, ), headers=self.authHeaders()) feed = Feed.query().get() resp = self.app.get('/api/feeds/%s/subscribe' % (feed.key.urlsafe(), ), query_string={ "hub.mode": 'subscribe', "hub.topic": feed.feed_url, "hub.challenge": 'testing', "hub.verify_token": feed.verify_token, }) assert resp.data == 'testing' data = get_file_from_data('/data/df_feed.xml') resp = self.app.post('/api/feeds/%s/subscribe' % (feed.key.urlsafe(), ), data=data, headers={ 'Content-Type': 'application/xml', }) assert 2 == Entry.query().count() assert 1 == Entry.query(Entry.published == True, Entry.overflow == False).count() resp = self.app.post('/api/feeds/%s/subscribe' % (feed.key.urlsafe(), )) assert 2 == Entry.query(Entry.published == True).count()
def update_all_feeds(interval_id): """Update all feeds for a specific interval""" if request.headers.get('X-Appengine-Cron') != 'true': raise ndb.Return(jsonify_error(message='Not a cron call')) for feed_type, feed_class in FEED_TYPE_TO_CLASS.iteritems(): feeds = Feed.for_interval(interval_id) success = 0 more = True cursor = None futures = [] while more: feeds_to_fetch, cursor, more = yield feeds.fetch_page_async(BATCH_SIZE, start_cursor=cursor) feeds_to_fetch = filter(lambda x: getattr(x, 'external_polling_bucket', DEFAULT_POLLING_BUCKET) == DEFAULT_POLLING_BUCKET, feeds_to_fetch) keys = ','.join([x.key.urlsafe() for x in feeds_to_fetch]) if not keys: continue futures.append(Queue('poll').add_async(Task(url=url_for('tq_feed_poll-canonical'), method='POST', params={'keys': keys}))) success += 1 for future in futures: yield future logger.info('queued poll for %d feeds at interval_id=%s', success, interval_id) raise ndb.Return(jsonify(status='ok'))
def update_all_feeds(interval_id): """Update all feeds for a specific interval""" if request.headers.get("X-Appengine-Cron") != "true": raise ndb.Return(jsonify_error(message="Not a cron call")) for feed_type, feed_class in FEED_TYPE_TO_CLASS.iteritems(): feeds = Feed.for_interval(interval_id) success = 0 more = True cursor = None futures = [] while more: feeds_to_fetch, cursor, more = yield feeds.fetch_page_async(BATCH_SIZE, start_cursor=cursor) feeds_to_fetch = filter( lambda x: getattr(x, "external_polling_bucket", DEFAULT_POLLING_BUCKET) == DEFAULT_POLLING_BUCKET, feeds_to_fetch, ) keys = ",".join([x.key.urlsafe() for x in feeds_to_fetch]) if not keys: continue futures.append( Queue("poll").add_async( Task(url=url_for("tq_feed_poll-canonical"), method="POST", params={"keys": keys}) ) ) success += 1 for future in futures: yield future logger.info("queued poll for %d feeds at interval_id=%s", success, interval_id) raise ndb.Return(jsonify(status="ok"))
def limit_feeds_for_channel_id(channel_id): """Limit all feeds connected to a channel""" max_stories_per_period = cast_int(request.form.get('max_stories_per_period'), default=None) schedule_period = cast_int(request.form.get('schedule_period'), default=None) dump_excess_in_period = bool(request.form.get('dump_excess_in_period')) users_feeds = [feed for feed in Feed.for_channel(channel_id) if feed.visible] futures = [] for feed in users_feeds: if dump_excess_in_period: feed.dump_excess_in_period = True if max_stories_per_period or schedule_period: feed.manual_control = True if max_stories_per_period: feed.max_stories_per_period = max_stories_per_period if schedule_period: feed.schedule_period = schedule_period futures.append(feed.put_async()) for future in futures: yield future users_feeds = export_feeds_to_json(users_feeds) raise ndb.Return(jsonify(status='ok', data=users_feeds))
def testLanguage(self): self.setMockUser() test_feed_url = 'http://example.com/rss' self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=6), status_code=200) self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, include_summary=True, max_stories_per_period=2, schedule_period=5, ), headers=self.authHeaders()) feed = Feed.query().get() assert feed.language is None self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=6, language='en-US'), status_code=200) self.pollUpdate() feed = Feed.query().get() assert feed.language == 'en'
def feed_validate(): """preview a feed""" feed_type = int(request.form.get('feed_type', 1)) form = FEED_TYPE_TO_CLASS[feed_type].preview_form(request.form) if not form.validate(): raise ndb.Return(jsonify(status='error', form_errors=form.errors)) feed = Feed() form.populate_obj(feed) feed.preview = True error = None parsed_feed = None try: parsed_feed, resp, feed = yield fetch_parsed_feed_for_feed(feed) feed.update_feed_from_parsed_feed(parsed_feed) if len(parsed_feed.entries) == 0: error = 'The url you entred is not a valid feed.' except FetchException, e: error = unicode(e)
def testSingleItemPublish(self): self.setMockUser() test_feed_url = 'http://example.com/rss' self.set_rss_response(test_feed_url, content=self.buildRSS('test'), status_code=200) resp = self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, include_summary=True, max_stories_per_period=2, schedule_period=5, ), headers=self.authHeaders()) entry = Entry.query().get() feed = Feed.query().get() resp = self.app.post('/api/feeds/%s/entries/%s/publish' % (feed.key.id(), entry.key.id()), headers=self.authHeaders())
def testRssFeedDetection(self): self.set_rss_response('http://techcrunch.com/feed/', content=self.buildRSS('test'), status_code=200) self.set_response('http://techcrunch.com', content=HTML_PAGE_TEMPLATE, status_code=200, headers={'Content-Type': 'text/html'}) resp = self.app.get('/api/feed/preview?feed_url=http://techcrunch.com', headers=self.authHeaders()) assert 1 == len(json.loads(resp.data)['data']) resp = self.app.post('/api/feeds', data=dict( feed_url='http://techcrunch.com', max_stories_per_period=1, schedule_period=5, ), headers=self.authHeaders()) feed = Feed.query().get() assert feed.feed_url == 'http://techcrunch.com/feed/'
def testBadFeedRemoval(self): self.setMockUser() test_feed_url = 'http://example.com/rss' self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=1), status_code=200) self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, include_summary=True, max_stories_per_period=2, schedule_period=5, ), headers=self.authHeaders()) feed = Feed.query().get() feed.last_successful_fetch = datetime.now() - timedelta(days=2) feed.put() self.set_rss_response(test_feed_url, content=self.buildRSS('test1', items=1), status_code=500) self.pollUpdate() assert feed.feed_disabled is True
def testFeedPreview(self): self.set_rss_response('http://techcrunch.com/feed/', content=self.buildRSS('test'), status_code=200) resp = self.app.get('/api/feed/preview?feed_url=http://techcrunch.com/feed/', headers=self.authHeaders()) assert 1 == len(json.loads(resp.data)['data']) self.set_rss_response('http://techcrunch.com/feed/2', content=self.buildRSS('test'), status_code=500) resp = self.app.get('/api/feed/preview?feed_url=http://techcrunch.com/feed/2', headers=self.authHeaders()) assert json.loads(resp.data)['message'] test_feed_url = 'http://example.com/rss' self.set_rss_response(test_feed_url, content=self.buildRSS('test'), status_code=200) resp = self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, include_summary=True, max_stories_per_period=2, schedule_period=5, ), headers=self.authHeaders()) assert 1 == Entry.query(Entry.published == True, Entry.overflow == True).count() feed = Feed.query().get() resp = self.app.get('/api/feeds/%s/preview' % (feed.key.id(), ), headers=self.authHeaders()) assert 'data' in json.loads(resp.data)
def testDbRaceCondition(self): self.setMockUser() test_feed_url = 'http://example.com/rss' self.set_rss_response(test_feed_url, content=self.buildRSS('test', items=1), status_code=200) self.app.post('/api/feeds', data=dict( feed_url=test_feed_url, include_summary=True, max_stories_per_period=2, schedule_period=5, ), headers=self.authHeaders()) feed = Feed.query().get() key = ndb.Key(Entry, '1', parent=feed.key) entry = Entry(key=key, guid='1') entry.put() entry_2 = Entry(key=key, guid='2') entry_2.put() assert Entry.query().count() == 2
def test_subscribe(self): u1 = User(username='******', email='*****@*****.**') u2 = User(username='******', email='*****@*****.**') f = Feed(name='susan_feed', owner_id=u2.id) db.session.add(u1) db.session.add(u2) db.session.add(f) db.session.commit() self.assertEqual(u1.subscriptions.all(), []) self.assertEqual(u2.subscriptions.all(), []) u1.subscribe(f) db.session.commit() self.assertTrue(u1.is_subscribed(f)) self.assertEqual(u1.subscriptions.count(), 1) self.assertEqual(u1.subscriptions.first().name, 'susan_feed') self.assertEqual(f.subscribers.count(), 1) self.assertEqual(f.subscribers.first().username, 'john') u1.unsubscribe(f) db.session.commit() self.assertFalse(u1.is_subscribed(f)) self.assertEqual(u1.subscriptions.count(), 0) self.assertEqual(f.subscribers.count(), 0)