def test_deferred_in_database(ctx):
from datetime import datetime, timedelta
from carrot.messaging import Consumer
from eventlet import sleep, spawn
import logging
from melk.util.nonce import nonce_str
import sys
from melkman.context import Context
from melkman.scheduler import defer_amqp_message, cancel_deferred
from melkman.scheduler.worker import ScheduledMessageService
from melkman.scheduler.worker import DeferredAMQPMessage, view_deferred_messages_by_timestamp
sms = ScheduledMessageService(ctx)
sched = spawn(sms.run)
m1 = {'hello_world': nonce_str()}
when = no_micro(datetime.utcnow() + timedelta(hours=2))
defer_amqp_message(when, m1, 'testq', 'testx', ctx)
# give it a sec to write it out, then close it down.
sleep(1)
sched.kill()
sched.wait()
# check the database for the message we sent
count = 0
for r in view_deferred_messages_by_timestamp(ctx.db, include_docs=True):
count += 1
message = DeferredAMQPMessage.wrap(r.doc)
for (k, v) in m1.items():
assert message.message[k] == v
assert message.timestamp == when
assert count == 1
def __init__(self, *args, **kw):
if not args and not 'id' in kw:
args = [melk_id(nonce_str())]
DocumentHelper.__init__(self, *args, **kw)
self._entries = None # lazy load
self._removed = {}
self._updated = {}
def hubbub_sub(feed, context, hub_url=None):
"""
subscribe to feed on the pubsubhubbub hub
specified or the first hub listed in the
feed if none is given.
"""
topic_url = topic_url_for(feed)
if topic_url is None:
raise ValueError('No self link found in feed, cannot subscribe via pubsubhubub.')
if hub_url is None:
hub_urls = feed.find_hub_urls()
if len(hub_urls) == 0:
raise ValueError("Cannot subscribe, no hubs were specified.")
hub_url = hub_urls[0]
log.warn("Guessing hub %s for %s" % (hub_url, feed.url))
feed.hub_info.enabled = True
feed.hub_info.hub_url = hub_url
feed.hub_info.verify_token = nonce_str()
feed.hub_info.secret = nonce_str()
feed.save()
cb = callback_url_for(feed.url, context)
req = [
('hub.callback', cb),
('hub.mode', 'subscribe'),
('hub.topic', topic_url),
('hub.verify', 'sync'),
('hub.verify', 'async'),
('hub.verify_token', feed.hub_info.verify_token),
('hub.secret', feed.hub_info.secret)
]
body = urlencode(req)
headers = {'content-type': 'application/x-www-form-urlencoded'}
return Http().request(feed.hub_info.hub_url, method="POST", body=body, headers=headers)
def test_push_index_digest(ctx):
from melk.util.nonce import nonce_str
from melkman.db.remotefeed import RemoteFeed
from melkman.fetch import push_feed_index
from melkman.fetch.worker import run_feed_indexer
from eventlet import sleep, spawn
from melkman.fetch.pubsubhubbub import psh_digest
# start a feed indexer
indexer = spawn(run_feed_indexer, ctx)
url = 'http://www.example.com/feeds/2'
rf = RemoteFeed.create_from_url(url, ctx)
rf.hub_info.enabled = True
rf.hub_info.subscribed = True
rf.save()
secret = nonce_str()
content = random_atom_feed(url, 10)
ids = melk_ids_in(content, url)
correct_digest = 'sha1=%s' % psh_digest(content, secret)
wrong_digest = 'wrong digest'
#
# no hub secret is specified on the feed
#
push_feed_index(url, content, ctx, digest=wrong_digest, from_hub=True)
sleep(.5)
rf = RemoteFeed.get_by_url(url, ctx)
for iid in ids:
assert iid not in rf.entries
push_feed_index(url, content, ctx, digest=None, from_hub=True)
sleep(.5)
rf = RemoteFeed.get_by_url(url, ctx)
for iid in ids:
assert iid not in rf.entries
# even the correct digest fails as no digest has been set
push_feed_index(url, content, ctx, digest=correct_digest, from_hub=True)
sleep(.5)
rf = RemoteFeed.get_by_url(url, ctx)
for iid in ids:
assert iid not in rf.entries
#
# now set the hub secret
#
rf.hub_info.secret = secret
rf.save()
push_feed_index(url, content, ctx, digest=wrong_digest, from_hub=True)
sleep(.5)
rf = RemoteFeed.get_by_url(url, ctx)
for iid in ids:
assert iid not in rf.entries
push_feed_index(url, content, ctx, digest=None, from_hub=True)
sleep(.5)
rf = RemoteFeed.get_by_url(url, ctx)
for iid in ids:
assert iid not in rf.entries
# finally, the correct digest should work now...
push_feed_index(url, content, ctx, digest=correct_digest, from_hub=True)
sleep(.5)
rf = RemoteFeed.get_by_url(url, ctx)
for iid in ids:
assert iid in rf.entries
indexer.kill()
indexer.wait()
def __call__(self, environ, start_response):
log.debug("FakeHub got request: %s" % environ)
req = Request(environ)
res = Response()
try:
if req.method != 'POST':
res.status = 400
return
cb = req.POST['hub.callback']
mode = req.POST['hub.mode']
topic = req.POST['hub.topic']
verify_token = req.POST.get('hub.verify_token', None)
if not mode in ('subscribe', 'unsubscribe'):
res.status = 400
return
# subscribe when already subscribed
if (cb, topic) in self._verified and mode == 'subscribe':
res.status = 204
# track lease renewals
self._renewals.setdefault((cb, topic), 0)
self._renewals[(cb, topic)] += 1
return
# unsubscribe and not currently subscribed
if (cb, topic) not in self._verified and mode == 'unsubscribe':
res.status = 204
return
# do a verification...
challenge = nonce_str()
vurl = cb
vurl = append_param(vurl, 'hub.mode', mode)
vurl = append_param(vurl, 'hub.topic', topic)
vurl = append_param(vurl, 'hub.challenge', challenge)
vurl = append_param(vurl, 'hub.lease_seconds', '%d' % self.lease_seconds)
if verify_token:
vurl = append_param(vurl, 'hub.verify_token', verify_token)
http = Http()
r, c = http.request(vurl, 'GET')
if r.status != 200 or c != challenge:
log.warn("Request did not validate :/ %s" % vurl)
res.status = 400
return
# okay it was fine...
if mode == 'subscribe':
secret = req.POST.get('hub.secret', None)
self._verified[(cb, topic)] = secret
else:
try:
del self._verified[(cb, topic)]
del self._renewals[(cb, topic)]
except KeyError:
pass
res.status = 202
except:
log.error("Error handling hub request: %s" % traceback.format_exc())
res.status = 500
finally:
log.debug("Returning w/ status=%s" % res.status)
return res(environ, start_response)
def test_sub_push(ctx):
from httplib2 import Http
from eventlet import sleep, spawn
from melk.util.nonce import nonce_str
from melkman.db import RemoteFeed
from melkman.fetch.worker import run_feed_indexer
from melkman.fetch.pubsubhubbub import WSGISubClient, callback_url_for, psh_digest
import logging
logging.basicConfig(level=logging.WARN)
w = WSGISubClient(ctx)
client = spawn(w.run)
indexer = spawn(run_feed_indexer, ctx)
http = Http()
url = 'http://example.org/feed/0'
content = random_atom_feed(url, 10)
secret = nonce_str()
digest = 'sha1=%s' % psh_digest(content, secret)
cb = callback_url_for(url, ctx)
assert RemoteFeed.get_by_url(url, ctx) == None
# try posting something that is not subscribed
r, c = http.request(cb, 'POST', body=content, headers={'X-Hub-Signature': digest})
assert r.status == 200, 'Expected 200, got %d' % r.status
sleep(1)
# nothing should happen...
assert RemoteFeed.get(url, ctx) == None
# set up the feed, but don't subscribe
rf = RemoteFeed.create_from_url(url, ctx)
rf.save()
r, c = http.request(cb, 'POST', body=content, headers={'X-Hub-Signature': digest})
assert r.status == 200, 'Expected 200, got %d' % r.status
sleep(1)
# nothing should happen...
rf = RemoteFeed.get_by_url(url, ctx)
assert len(rf.entries) == 0
# now set it up
rf.hub_info.enabled = True
rf.hub_info.subscribed = True
rf.hub_info.secret = secret
rf.save()
# try with wrong digest...
r, c = http.request(cb, 'POST', body=content, headers={'X-Hub-Signature': 'wrong'})
assert r.status == 200, 'Expected 200, got %d' % r.status
sleep(0.5)
# nothing should happen...
rf = RemoteFeed.get_by_url(url, ctx)
assert len(rf.entries) == 0
# try with no digest
r, c = http.request(cb, 'POST', body=content)
assert r.status == 200, 'Expected 200, got %d' % r.status
sleep(0.5)
# nothing should happen...
rf = RemoteFeed.get_by_url(url, ctx)
assert len(rf.entries) == 0
# finally, try with correct digest
r, c = http.request(cb, 'POST', body=content, headers={'X-Hub-Signature': digest})
assert r.status == 200, 'Expected 200, got %d' % r.status
sleep(0.5)
# nothing should happen...
rf = RemoteFeed.get_by_url(url, ctx)
assert len(rf.entries) == 10
for iid in melk_ids_in(content, url):
assert iid in rf.entries
client.kill()
client.wait()
indexer.kill()
indexer.wait()
def test_sub_verify(ctx):
from httplib2 import Http
from eventlet import spawn
from melk.util.nonce import nonce_str
from melkman.db import RemoteFeed
from melkman.fetch.pubsubhubbub import WSGISubClient, callback_url_for
import logging
logging.basicConfig(level=logging.WARN)
w = WSGISubClient(ctx)
client = spawn(w.run)
http = Http()
url = 'http://example.org/feed/0'
challenge = nonce_str()
verify_token = nonce_str()
secret = nonce_str()
cb = callback_url_for(url, ctx)
cb += '?hub.mode=subscribe'
cb += '&hub.topic=%s' % url
cb += '&hub.challenge=%s' % challenge
cb += '&hub.verify_token=%s' % verify_token
# try verifying something that doesn't exist
r, c = http.request(cb, 'GET')
assert r.status == 404, 'Expected 404, got %d' % r.status
# now create it
rf = RemoteFeed.create_from_url(url, ctx)
rf.feed_info = {"links": [{"rel": "self", "href": url}]}
rf.save()
# still should not verify
r, c = http.request(cb, 'GET')
assert r.status == 404, 'Expected 404, got %d' % r.status
# now set appropriate fields on the feed object
rf.hub_info.enabled = True
rf.hub_info.verify_token = verify_token
rf.hub_info.secret = secret
rf.save()
# now it should accept verification...
for i in range(3):
r, c = http.request(cb, 'GET')
assert r.status == 200, 'Expected 200, got %d' % r.status
assert c == challenge, 'expected %s, got %s' % (challence, c)
# create unsubscribe callback...
cb = callback_url_for(url, ctx)
cb += '?hub.mode=unsubscribe'
cb += '&hub.topic=%s' % url
cb += '&hub.challenge=%s' % challenge
cb += '&hub.verify_token=%s' % verify_token
# currently it should fail, we are not unsubscribed
r, c = http.request(cb, 'GET')
assert r.status == 404, 'Expected 404, got %d' % r.status
# after disabling, the unsub verify should be okay
rf.reload()
rf.hub_info.enabled = False
rf.save()
r, c = http.request(cb, 'GET')
assert r.status == 200, 'Expected 200, got %d' % r.status
assert c == challenge, 'expected %s, got %s' % (challence, c)
# now destroy the feed entirely,
# unsub request for stuff that
# does not exist should also
# verify.
del ctx.db[rf.id]
r, c = http.request(cb, 'GET')
assert r.status == 200, 'Expected 200, got %d' % r.status
assert c == challenge, 'expected %s, got %s' % (challence, c)
client.kill()
client.wait()
def random_id():
return melk_id(nonce_str())
def create(cls, context, *args, **kw):
if not args and not 'id' in kw:
args = [melk_id(nonce_str())]
return super(NewsBucket, cls).create(context, *args, **kw)
