Example #1
0
def test_deferred_in_database(ctx):
    from datetime import datetime, timedelta
    from carrot.messaging import Consumer
    from eventlet import sleep, spawn
    import logging
    from melk.util.nonce import nonce_str
    import sys

    from melkman.context import Context
    from melkman.scheduler import defer_amqp_message, cancel_deferred
    from melkman.scheduler.worker import ScheduledMessageService
    from melkman.scheduler.worker import DeferredAMQPMessage, view_deferred_messages_by_timestamp

    sms = ScheduledMessageService(ctx)
    sched = spawn(sms.run)
    
    m1 = {'hello_world': nonce_str()}
    when = no_micro(datetime.utcnow() + timedelta(hours=2))
    defer_amqp_message(when, m1, 'testq', 'testx', ctx)
    
    # give it a sec to write it out, then close it down.
    sleep(1)
    sched.kill()
    sched.wait()
    
    # check the database for the message we sent
    count = 0
    for r in view_deferred_messages_by_timestamp(ctx.db, include_docs=True):
        count += 1
        message = DeferredAMQPMessage.wrap(r.doc)
        for (k, v) in m1.items():
            assert message.message[k] == v
        assert message.timestamp == when
    assert count == 1
Example #2
0
 def __init__(self, *args, **kw):
     if not args and not 'id' in kw:
         args = [melk_id(nonce_str())]
     
     DocumentHelper.__init__(self, *args, **kw)
     self._entries = None # lazy load
     self._removed = {}
     self._updated = {}
Example #3
0
def hubbub_sub(feed, context, hub_url=None):
    """
    subscribe to feed on the pubsubhubbub hub 
    specified or the first hub listed in the 
    feed if none is given.
    """
    topic_url = topic_url_for(feed)

    if topic_url is None:
        raise ValueError('No self link found in feed, cannot subscribe via pubsubhubub.')

    if hub_url is None:
        hub_urls = feed.find_hub_urls()
        if len(hub_urls) == 0:
            raise ValueError("Cannot subscribe, no hubs were specified.")
        hub_url = hub_urls[0]
        log.warn("Guessing hub %s for %s" % (hub_url, feed.url))

    feed.hub_info.enabled = True
    feed.hub_info.hub_url = hub_url
    feed.hub_info.verify_token = nonce_str()
    feed.hub_info.secret = nonce_str()
    feed.save()

    cb = callback_url_for(feed.url, context)
    req = [
        ('hub.callback', cb),
        ('hub.mode', 'subscribe'),
        ('hub.topic', topic_url),
        ('hub.verify', 'sync'),
        ('hub.verify', 'async'),
        ('hub.verify_token', feed.hub_info.verify_token),
        ('hub.secret', feed.hub_info.secret)
    ]
    body = urlencode(req)
    headers = {'content-type': 'application/x-www-form-urlencoded'}
    return Http().request(feed.hub_info.hub_url, method="POST", body=body, headers=headers)
Example #4
0
def test_push_index_digest(ctx):
    from melk.util.nonce import nonce_str
    from melkman.db.remotefeed import RemoteFeed
    from melkman.fetch import push_feed_index
    from melkman.fetch.worker import run_feed_indexer
    from eventlet import sleep, spawn
    from melkman.fetch.pubsubhubbub import psh_digest


    # start a feed indexer
    indexer = spawn(run_feed_indexer, ctx)

    url = 'http://www.example.com/feeds/2'
    rf = RemoteFeed.create_from_url(url, ctx)
    rf.hub_info.enabled = True
    rf.hub_info.subscribed = True
    rf.save()

    secret = nonce_str()

    content = random_atom_feed(url, 10)
    ids = melk_ids_in(content, url)

    correct_digest = 'sha1=%s' % psh_digest(content, secret)
    wrong_digest = 'wrong digest'

    #
    # no hub secret is specified on the feed
    #
    push_feed_index(url, content, ctx, digest=wrong_digest, from_hub=True)
    sleep(.5)
    rf = RemoteFeed.get_by_url(url, ctx)
    for iid in ids:
        assert iid not in rf.entries
    push_feed_index(url, content, ctx, digest=None, from_hub=True)
    sleep(.5)
    rf = RemoteFeed.get_by_url(url, ctx)
    for iid in ids:
        assert iid not in rf.entries
    # even the correct digest fails as no digest has been set 
    push_feed_index(url, content, ctx, digest=correct_digest, from_hub=True)
    sleep(.5)
    rf = RemoteFeed.get_by_url(url, ctx)
    for iid in ids:
        assert iid not in rf.entries

    #
    # now set the hub secret
    #
    rf.hub_info.secret = secret
    rf.save()

    push_feed_index(url, content, ctx, digest=wrong_digest, from_hub=True)
    sleep(.5)
    rf = RemoteFeed.get_by_url(url, ctx)
    for iid in ids:
        assert iid not in rf.entries
    push_feed_index(url, content, ctx, digest=None, from_hub=True)
    sleep(.5)
    rf = RemoteFeed.get_by_url(url, ctx)
    for iid in ids:
        assert iid not in rf.entries

    # finally, the correct digest should work now...
    push_feed_index(url, content, ctx, digest=correct_digest, from_hub=True)
    sleep(.5)
    rf = RemoteFeed.get_by_url(url, ctx)
    for iid in ids:
        assert iid in rf.entries

    indexer.kill()
    indexer.wait()
Example #5
0
    def __call__(self, environ, start_response):
        log.debug("FakeHub got request: %s" % environ)

        req = Request(environ)
        res = Response()
        
        try:
            if req.method != 'POST':
                res.status = 400
                return
        
            cb = req.POST['hub.callback']
            mode = req.POST['hub.mode']
            topic = req.POST['hub.topic']
            verify_token = req.POST.get('hub.verify_token', None)            

            if not mode in ('subscribe', 'unsubscribe'):
                res.status = 400
                return
            
            # subscribe when already subscribed
            if (cb, topic) in self._verified and mode == 'subscribe':
               res.status = 204
               # track lease renewals
               self._renewals.setdefault((cb, topic), 0)
               self._renewals[(cb, topic)] += 1
               return
                
            # unsubscribe and not currently subscribed
            if (cb, topic) not in self._verified and mode == 'unsubscribe':
                res.status = 204
                return

            # do a verification...
            challenge = nonce_str()
        
            vurl = cb
            vurl = append_param(vurl, 'hub.mode', mode)
            vurl = append_param(vurl, 'hub.topic', topic)
            vurl = append_param(vurl, 'hub.challenge', challenge)
            vurl = append_param(vurl, 'hub.lease_seconds', '%d' % self.lease_seconds)
            if verify_token:
                vurl = append_param(vurl, 'hub.verify_token', verify_token)

            http = Http()
            r, c = http.request(vurl, 'GET')
            
            if r.status != 200 or c != challenge:
                log.warn("Request did not validate :/ %s" % vurl)
                res.status = 400
                return
            
            # okay it was fine...
            if mode == 'subscribe':
                secret = req.POST.get('hub.secret', None)
                self._verified[(cb, topic)] = secret
            else:
                try:
                    del self._verified[(cb, topic)]
                    del self._renewals[(cb, topic)]
                except KeyError:
                    pass

            res.status = 202
            
        except:
            log.error("Error handling hub request: %s" % traceback.format_exc())
            res.status = 500
        finally:
            log.debug("Returning w/ status=%s" % res.status)
            return res(environ, start_response)
Example #6
0
def test_sub_push(ctx):
    from httplib2 import Http
    from eventlet import sleep, spawn
    from melk.util.nonce import nonce_str
    from melkman.db import RemoteFeed
    from melkman.fetch.worker import run_feed_indexer
    from melkman.fetch.pubsubhubbub import WSGISubClient, callback_url_for, psh_digest
    
    import logging
    logging.basicConfig(level=logging.WARN)
    
    
    w = WSGISubClient(ctx)
    client = spawn(w.run)
    indexer = spawn(run_feed_indexer, ctx)
    
    http = Http()
    url = 'http://example.org/feed/0'
    content = random_atom_feed(url, 10)
    secret = nonce_str()
    
    digest = 'sha1=%s' % psh_digest(content, secret)
    cb = callback_url_for(url, ctx)
    
    assert RemoteFeed.get_by_url(url, ctx) == None
    
    # try posting something that is not subscribed
    r, c = http.request(cb, 'POST', body=content, headers={'X-Hub-Signature': digest})
    assert r.status == 200, 'Expected 200, got %d' % r.status
    sleep(1)
    # nothing should happen...
    assert RemoteFeed.get(url, ctx) == None

    # set up the feed, but don't subscribe
    rf = RemoteFeed.create_from_url(url, ctx)
    rf.save()
    r, c = http.request(cb, 'POST', body=content, headers={'X-Hub-Signature': digest})
    assert r.status == 200, 'Expected 200, got %d' % r.status
    sleep(1)
    # nothing should happen...
    rf = RemoteFeed.get_by_url(url, ctx)
    assert len(rf.entries) == 0
    
    # now set it up 
    rf.hub_info.enabled = True
    rf.hub_info.subscribed = True
    rf.hub_info.secret = secret
    rf.save()

    # try with wrong digest...
    r, c = http.request(cb, 'POST', body=content, headers={'X-Hub-Signature': 'wrong'})
    assert r.status == 200, 'Expected 200, got %d' % r.status
    sleep(0.5)
    # nothing should happen...
    rf = RemoteFeed.get_by_url(url, ctx)
    assert len(rf.entries) == 0
    
    # try with no digest
    r, c = http.request(cb, 'POST', body=content)
    assert r.status == 200, 'Expected 200, got %d' % r.status
    sleep(0.5)
    # nothing should happen...
    rf = RemoteFeed.get_by_url(url, ctx)
    assert len(rf.entries) == 0
    
    # finally, try with correct digest
    r, c = http.request(cb, 'POST', body=content, headers={'X-Hub-Signature': digest})
    assert r.status == 200, 'Expected 200, got %d' % r.status
    sleep(0.5)
    # nothing should happen...
    rf = RemoteFeed.get_by_url(url, ctx)
    assert len(rf.entries) == 10
    for iid in melk_ids_in(content, url):
        assert iid in rf.entries 
    
    client.kill()
    client.wait()
    indexer.kill()
    indexer.wait()
Example #7
0
def test_sub_verify(ctx):
    from httplib2 import Http
    from eventlet import spawn
    from melk.util.nonce import nonce_str
    from melkman.db import RemoteFeed
    from melkman.fetch.pubsubhubbub import WSGISubClient, callback_url_for
    
    import logging
    logging.basicConfig(level=logging.WARN)
    
    
    w = WSGISubClient(ctx)
    client = spawn(w.run)
    
    http = Http()
    url = 'http://example.org/feed/0'
    challenge = nonce_str()
    verify_token = nonce_str()
    secret = nonce_str()
    
    cb = callback_url_for(url, ctx)
    cb += '?hub.mode=subscribe'
    cb += '&hub.topic=%s' % url
    cb += '&hub.challenge=%s' % challenge
    cb += '&hub.verify_token=%s' % verify_token
    
    # try verifying something that doesn't exist
    r, c = http.request(cb, 'GET')
    assert r.status == 404, 'Expected 404, got %d' % r.status
    
    # now create it
    rf = RemoteFeed.create_from_url(url, ctx)
    rf.feed_info = {"links": [{"rel": "self", "href": url}]}
    rf.save()
    
    # still should not verify
    r, c = http.request(cb, 'GET')
    assert r.status == 404, 'Expected 404, got %d' % r.status

    
    # now set appropriate fields on the feed object
    rf.hub_info.enabled = True
    rf.hub_info.verify_token = verify_token
    rf.hub_info.secret = secret
    rf.save()
    
    # now it should accept verification...
    for i in range(3):
        r, c = http.request(cb, 'GET')
        assert r.status == 200, 'Expected 200, got %d' % r.status
        assert c == challenge, 'expected %s, got %s' % (challence, c)
        
    # create unsubscribe callback...
    cb = callback_url_for(url, ctx)
    cb += '?hub.mode=unsubscribe'
    cb += '&hub.topic=%s' % url
    cb += '&hub.challenge=%s' % challenge
    cb += '&hub.verify_token=%s' % verify_token

    # currently it should fail, we are not unsubscribed
    r, c = http.request(cb, 'GET')
    assert r.status == 404, 'Expected 404, got %d' % r.status
        
    # after disabling, the unsub verify should be okay
    rf.reload()
    rf.hub_info.enabled = False
    rf.save()
    r, c = http.request(cb, 'GET')
    assert r.status == 200, 'Expected 200, got %d' % r.status
    assert c == challenge, 'expected %s, got %s' % (challence, c)

    
    # now destroy the feed entirely,
    # unsub request for stuff that 
    # does not exist should also 
    # verify.
    del ctx.db[rf.id]    
    r, c = http.request(cb, 'GET')
    assert r.status == 200, 'Expected 200, got %d' % r.status
    assert c == challenge, 'expected %s, got %s' % (challence, c)
    
    client.kill()
    client.wait()
Example #8
0
def random_id():
    return melk_id(nonce_str())
Example #9
0
 def create(cls, context, *args, **kw):
     if not args and not 'id' in kw:
         args = [melk_id(nonce_str())]
     return super(NewsBucket, cls).create(context, *args, **kw)