def test_auto_sub(ctx): # tests autosubscription when feeds are indexed # with <link rel="hub" /> entries. from datetime import datetime from eventlet import sleep, spawn from melkman.db import RemoteFeed from melkman.fetch import push_feed_index from melkman.fetch.pubsubhubbub import WSGISubClient, callback_url_for from melkman.fetch.worker import run_feed_indexer w = WSGISubClient(ctx) client = spawn(w.run) indexer = spawn(run_feed_indexer, ctx) hub = FakeHub() hub_proc = spawn(hub.run) hub_url = 'http://localhost:%d/' % hub.port feed_url = 'http://www.example.org/feeds/12' content = """<?xml version="1.0" encoding="utf-8"?> <feed xmlns="http://www.w3.org/2005/Atom"> <id>%s</id> <title>Blah</title> <link rel="self" href="%s"/> <link rel="hub" href="%s" /> <updated>%s</updated> <author> <name>Joop Doderer</name> </author> </feed> """ % (feed_url, feed_url, hub_url, rfc3339_date(datetime.utcnow())) # push content in... push_feed_index(feed_url, content, ctx) sleep(.5) # check for automatic subscription... cb = callback_url_for(feed_url, ctx) assert hub.is_verified(cb, feed_url) rf = RemoteFeed.get_by_url(feed_url, ctx) assert rf.hub_info.enabled assert rf.hub_info.hub_url == hub_url client.kill() client.wait() indexer.kill() indexer.wait() hub_proc.kill() hub_proc.wait()
def test_disabled_unsubscribes(ctx): """ tests that if pubsub is disabled for a feed, it becomes unsubscribed from it's hub. """ from eventlet import sleep, spawn from melkman.db import RemoteFeed from melkman.fetch.pubsubhubbub import WSGISubClient, callback_url_for from melkman.fetch.pubsubhubbub import hubbub_sub, update_pubsub_state w = WSGISubClient(ctx) client = spawn(w.run) hub = FakeHub() hub_proc = spawn(hub.run) hub_url = 'http://localhost:%d/' % hub.port feed_url = 'http://example.org/feeds/99' rf = RemoteFeed.create_from_url(feed_url, ctx) rf.feed_info = {'links': [{'rel': 'self', 'href': feed_url}, {'rel': 'hub', 'href': hub_url}]} rf.save() # subscribe to the feed on the hub cb = callback_url_for(feed_url, ctx) assert not hub.is_verified(cb, feed_url) r, c = hubbub_sub(rf, ctx) assert r.status == 202, 'Expected 202, got %d' % r.status sleep(.5) assert hub.is_verified(cb, feed_url) # disable pubsub for the feed rf = RemoteFeed.get_by_url(feed_url, ctx) assert rf.hub_info.enabled == True and rf.hub_info.subscribed == True rf.hub_info.enabled = False rf.save() # trigger an update update_pubsub_state(rf, ctx) # check that it is now unsubscribed. sleep(.5) assert not hub.is_verified(cb, feed_url) rf = RemoteFeed.get_by_url(feed_url, ctx) assert rf.hub_info.enabled == False and rf.hub_info.subscribed == False client.kill() client.wait() hub_proc.kill() hub_proc.wait()
def test_hub_lease_renew(ctx): """ tests that we resubscribe with a hub within the hub specified lease window. """ from eventlet import sleep, spawn from melkman.db import RemoteFeed from melkman.fetch.pubsubhubbub import WSGISubClient, callback_url_for from melkman.fetch.pubsubhubbub import hubbub_sub, update_pubsub_state w = WSGISubClient(ctx) client = spawn(w.run) # create a hub with a very short lease time. hub = FakeHub(lease_seconds=2) hub_proc = spawn(hub.run) hub_url = 'http://localhost:%d/' % hub.port feed_url = 'http://example.org/feeds/99' rf = RemoteFeed.create_from_url(feed_url, ctx) rf.feed_info = {'links': [{'rel': 'self', 'href': feed_url}, {'rel': 'hub', 'href': hub_url}]} rf.save() # subscribe to the feed on the hub cb = callback_url_for(feed_url, ctx) assert not hub.is_verified(cb, feed_url) r, c = hubbub_sub(rf, ctx) assert r.status == 202, 'Expected 202, got %d' % r.status sleep(.5) assert hub.is_verified(cb, feed_url) rf = RemoteFeed.get_by_url(feed_url, ctx) assert hub.renewals(cb, feed_url) == 0 sleep(2) update_pubsub_state(rf, ctx) # make sure we triggered a lease renewal assert hub.renewals(cb, feed_url) == 1 client.kill() client.wait() hub_proc.kill() hub_proc.wait()
def test_hub_lease_renew_failover(ctx): """ tests that if we fail to renew a lease with a hub we will failover to a different hub if one is available. """ from eventlet import sleep, spawn from melkman.db import RemoteFeed from melkman.fetch.pubsubhubbub import WSGISubClient, callback_url_for from melkman.fetch.pubsubhubbub import hubbub_sub, update_pubsub_state w = WSGISubClient(ctx) client = spawn(w.run) # create a hub with a very short lease time. hub = FakeHub(lease_seconds=2) hub_proc = spawn(hub.run) hub_url = 'http://localhost:%d/' % hub.port hub2 = FakeHub(port=9298) hub2_proc = spawn(hub2.run) hub2_url = 'http://localhost:%d/' % hub2.port feed_url = 'http://example.org/feeds/99' rf = RemoteFeed.create_from_url(feed_url, ctx) rf.feed_info = {'links': [{'rel': 'self', 'href': feed_url}, {'rel': 'hub', 'href': hub_url}, {'rel': 'hub', 'href': hub2_url}]} rf.save() # subscribe to the feed on the hub cb = callback_url_for(feed_url, ctx) assert not hub.is_verified(cb, feed_url) r, c = hubbub_sub(rf, ctx) assert r.status == 202, 'Expected 202, got %d' % r.status sleep(.5) rf = RemoteFeed.get_by_url(feed_url, ctx) assert rf.hub_info.subscribed == True assert rf.hub_info.hub_url == hub_url assert hub.is_verified(cb, feed_url) assert not hub2.is_verified(cb, feed_url) assert hub.renewals(cb, feed_url) == 0 sleep(2) # kill the first hub so that when we update, # the renewal will fail... hub_proc.kill() # when this update is triggered, renewal should fail and # we should instead subscribe to the alternate hub. update_pubsub_state(rf, ctx) assert hub.renewals(cb, feed_url) == 0 rf = RemoteFeed.get_by_url(feed_url, ctx) assert rf.hub_info.subscribed == True assert rf.hub_info.hub_url == hub2_url assert hub2.is_verified(cb, feed_url) client.kill() client.wait() hub2_proc.kill() hub2_proc.wait()
def test_hub_invalidation_resub(ctx): """ tests that if a currently subscribed hub is no longer listed, we subscribe to a different hub if any are listed. """ from eventlet import sleep, spawn from melkman.db import RemoteFeed from melkman.fetch.pubsubhubbub import WSGISubClient, callback_url_for from melkman.fetch.pubsubhubbub import hubbub_sub, update_pubsub_state w = WSGISubClient(ctx) client = spawn(w.run) # create two hubs hub = FakeHub() hub_proc = spawn(hub.run) hub_url = 'http://localhost:%d/' % hub.port hub2 = FakeHub(port=9298) hub2_proc = spawn(hub2.run) hub2_url = 'http://localhost:%d/' % hub2.port feed_url = 'http://example.org/feeds/99' rf = RemoteFeed.create_from_url(feed_url, ctx) rf.feed_info = {'links': [{'rel': 'self', 'href': feed_url}, {'rel': 'hub', 'href': hub_url}]} rf.save() # subscribe to the feed on the hub cb = callback_url_for(feed_url, ctx) assert not hub.is_verified(cb, feed_url) r, c = hubbub_sub(rf, ctx) assert r.status == 202, 'Expected 202, got %d' % r.status sleep(.5) assert hub.is_verified(cb, feed_url) # remove the hub from the list of hubs, but replace it with another rf = RemoteFeed.get_by_url(feed_url, ctx) rf.feed_info = {'links': [{'rel': 'self', 'href': feed_url}, {'rel': 'hub', 'href': hub2_url}]} rf.save() # trigger an update update_pubsub_state(rf, ctx) # check that it is now unsubscribed from the original hub, and # is now subscribed to the new hub. sleep(2) assert not hub.is_verified(cb, feed_url) assert hub2.is_verified(cb, feed_url) rf = RemoteFeed.get_by_url(feed_url, ctx) assert rf.hub_info.enabled == True and rf.hub_info.subscribed == True client.kill() client.wait() hub_proc.kill() hub_proc.wait() hub2_proc.kill() hub2_proc.wait()
def test_sub_to_hub(ctx): """ test make_sub_request and make_unsub_request """ from httplib2 import Http from eventlet import sleep, spawn from melk.util.nonce import nonce_str import traceback from melkman.db import RemoteFeed from melkman.fetch.worker import run_feed_indexer from melkman.fetch.pubsubhubbub import WSGISubClient from melkman.fetch.pubsubhubbub import callback_url_for from melkman.fetch.pubsubhubbub import hubbub_sub from melkman.fetch.pubsubhubbub import hubbub_unsub from melkman.fetch.pubsubhubbub import psh_digest import logging logging.basicConfig(level=logging.WARN) w = WSGISubClient(ctx) client = spawn(w.run) indexer = spawn(run_feed_indexer, ctx) hub = FakeHub() hub_proc = spawn(hub.run) hub_url = 'http://localhost:%d/' % hub.port feed_url = 'http://example.org/feeds/99' rf = RemoteFeed.create_from_url(feed_url, ctx) rf.feed_info = {'links': [{'rel': 'self', 'href': feed_url}, {'rel': 'hub', 'href': hub_url}]} rf.save() cb = callback_url_for(feed_url, ctx) # subscribe to the hub assert not hub.is_verified(cb, feed_url) r, c = hubbub_sub(rf, ctx) assert r.status == 202, 'Expected 202, got %d' % r.status sleep(.5) assert hub.is_verified(cb, feed_url) secret = hub.secret_for(cb, feed_url) http = Http() # simulate hub posting to callback URL content = random_atom_feed(feed_url, 10, link=feed_url, hub_urls=[hub_url]) digest = 'sha1=%s' % psh_digest(content, secret) r, c = http.request(cb, 'POST', body=content, headers={'X-Hub-Signature': digest}) assert r.status == 200, 'Expected 200, got %d' % r.status sleep(0.5) # since we are subscribed, new items should be in the feed now rf = RemoteFeed.get_by_url(feed_url, ctx) assert len(rf.entries) == 10 for iid in melk_ids_in(content, feed_url): assert iid in rf.entries # unsubscribe from hub r, c = hubbub_unsub(rf, ctx) assert r.status == 202, 'Expected 202, got %d' % r.status sleep(.5) assert not hub.is_verified(cb, feed_url) # simulate another POST to the callback URL # this time it should fail (we are not subscribed) content = random_atom_feed(feed_url, 10, link=feed_url, hub_urls=[hub_url]) digest = "sha1=%s" % psh_digest(content, secret) r, c = http.request(cb, 'POST', body=content, headers={'X-Hub-Signature': digest}) assert r.status == 200, 'Expected 200, got %d' % r.status sleep(0.5) # items should be the same as before (not subscribed) rf = RemoteFeed.get_by_url(feed_url, ctx) assert len(rf.entries) == 10 for iid in melk_ids_in(content, feed_url): assert not iid in rf.entries client.kill() client.wait() indexer.kill() indexer.wait() hub_proc.kill() hub_proc.wait()
def test_sub_push(ctx): from httplib2 import Http from eventlet import sleep, spawn from melk.util.nonce import nonce_str from melkman.db import RemoteFeed from melkman.fetch.worker import run_feed_indexer from melkman.fetch.pubsubhubbub import WSGISubClient, callback_url_for, psh_digest import logging logging.basicConfig(level=logging.WARN) w = WSGISubClient(ctx) client = spawn(w.run) indexer = spawn(run_feed_indexer, ctx) http = Http() url = 'http://example.org/feed/0' content = random_atom_feed(url, 10) secret = nonce_str() digest = 'sha1=%s' % psh_digest(content, secret) cb = callback_url_for(url, ctx) assert RemoteFeed.get_by_url(url, ctx) == None # try posting something that is not subscribed r, c = http.request(cb, 'POST', body=content, headers={'X-Hub-Signature': digest}) assert r.status == 200, 'Expected 200, got %d' % r.status sleep(1) # nothing should happen... assert RemoteFeed.get(url, ctx) == None # set up the feed, but don't subscribe rf = RemoteFeed.create_from_url(url, ctx) rf.save() r, c = http.request(cb, 'POST', body=content, headers={'X-Hub-Signature': digest}) assert r.status == 200, 'Expected 200, got %d' % r.status sleep(1) # nothing should happen... rf = RemoteFeed.get_by_url(url, ctx) assert len(rf.entries) == 0 # now set it up rf.hub_info.enabled = True rf.hub_info.subscribed = True rf.hub_info.secret = secret rf.save() # try with wrong digest... r, c = http.request(cb, 'POST', body=content, headers={'X-Hub-Signature': 'wrong'}) assert r.status == 200, 'Expected 200, got %d' % r.status sleep(0.5) # nothing should happen... rf = RemoteFeed.get_by_url(url, ctx) assert len(rf.entries) == 0 # try with no digest r, c = http.request(cb, 'POST', body=content) assert r.status == 200, 'Expected 200, got %d' % r.status sleep(0.5) # nothing should happen... rf = RemoteFeed.get_by_url(url, ctx) assert len(rf.entries) == 0 # finally, try with correct digest r, c = http.request(cb, 'POST', body=content, headers={'X-Hub-Signature': digest}) assert r.status == 200, 'Expected 200, got %d' % r.status sleep(0.5) # nothing should happen... rf = RemoteFeed.get_by_url(url, ctx) assert len(rf.entries) == 10 for iid in melk_ids_in(content, url): assert iid in rf.entries client.kill() client.wait() indexer.kill() indexer.wait()
def test_sub_verify(ctx): from httplib2 import Http from eventlet import spawn from melk.util.nonce import nonce_str from melkman.db import RemoteFeed from melkman.fetch.pubsubhubbub import WSGISubClient, callback_url_for import logging logging.basicConfig(level=logging.WARN) w = WSGISubClient(ctx) client = spawn(w.run) http = Http() url = 'http://example.org/feed/0' challenge = nonce_str() verify_token = nonce_str() secret = nonce_str() cb = callback_url_for(url, ctx) cb += '?hub.mode=subscribe' cb += '&hub.topic=%s' % url cb += '&hub.challenge=%s' % challenge cb += '&hub.verify_token=%s' % verify_token # try verifying something that doesn't exist r, c = http.request(cb, 'GET') assert r.status == 404, 'Expected 404, got %d' % r.status # now create it rf = RemoteFeed.create_from_url(url, ctx) rf.feed_info = {"links": [{"rel": "self", "href": url}]} rf.save() # still should not verify r, c = http.request(cb, 'GET') assert r.status == 404, 'Expected 404, got %d' % r.status # now set appropriate fields on the feed object rf.hub_info.enabled = True rf.hub_info.verify_token = verify_token rf.hub_info.secret = secret rf.save() # now it should accept verification... for i in range(3): r, c = http.request(cb, 'GET') assert r.status == 200, 'Expected 200, got %d' % r.status assert c == challenge, 'expected %s, got %s' % (challence, c) # create unsubscribe callback... cb = callback_url_for(url, ctx) cb += '?hub.mode=unsubscribe' cb += '&hub.topic=%s' % url cb += '&hub.challenge=%s' % challenge cb += '&hub.verify_token=%s' % verify_token # currently it should fail, we are not unsubscribed r, c = http.request(cb, 'GET') assert r.status == 404, 'Expected 404, got %d' % r.status # after disabling, the unsub verify should be okay rf.reload() rf.hub_info.enabled = False rf.save() r, c = http.request(cb, 'GET') assert r.status == 200, 'Expected 200, got %d' % r.status assert c == challenge, 'expected %s, got %s' % (challence, c) # now destroy the feed entirely, # unsub request for stuff that # does not exist should also # verify. del ctx.db[rf.id] r, c = http.request(cb, 'GET') assert r.status == 200, 'Expected 200, got %d' % r.status assert c == challenge, 'expected %s, got %s' % (challence, c) client.kill() client.wait()