def test_index_request(ctx):
from melkman.db.remotefeed import RemoteFeed
from melkman.fetch import request_feed_index
from melkman.fetch.worker import run_feed_indexer
from eventlet import sleep, spawn
# start a feed indexer
indexer = spawn(run_feed_indexer, ctx)
#start a web server...
www = os.path.join(data_path(), 'www')
ts = FileServer(www)
ts_proc = spawn(ts.run)
try:
test_url = ts.url_for('good.xml')
assert RemoteFeed.get_by_url(test_url, ctx) is None
# make an index request...
request_feed_index(test_url, ctx)
sleep(.5)
rf = RemoteFeed.get_by_url(test_url, ctx)
assert rf is not None
assert len(rf.entries.keys()) == 2
finally:
indexer.kill()
indexer.wait()
ts_proc.kill()
ts_proc.wait()
def test_schedule_index(ctx):
from melkman.db.remotefeed import RemoteFeed
from melkman.fetch import schedule_feed_index
from melkman.fetch.worker import run_feed_indexer
from melkman.scheduler.worker import ScheduledMessageService
from eventlet import sleep, spawn
# start a feed indexer
indexer = spawn(run_feed_indexer, ctx)
# scheduled message service
sms = ScheduledMessageService(ctx)
sched = spawn(sms.run)
#start a web server...
www = os.path.join(data_path(), 'www')
ts = FileServer(www)
ts_proc = spawn(ts.run)
test_url = ts.url_for('good.xml')
# schedule an index request...
try:
assert RemoteFeed.get_by_url(test_url, ctx) is None
# make an index request...
when = datetime.utcnow() + timedelta(seconds=2)
schedule_feed_index(test_url, when, ctx)
sleep(3)
rf = RemoteFeed.get_by_url(test_url, ctx)
assert rf is not None
assert len(rf.entries.keys()) == 2
except:
log.error("error: %s" % traceback.format_exc())
finally:
indexer.kill()
indexer.wait()
sched.kill()
sched.wait()
ts_proc.kill()
ts_proc.wait()
def set_opml(self, id):
composite = Composite.get(id, ctx)
if composite is None:
abort(404)
opml_data = get_posted_data()
try:
feeds = set(feeds_in_opml(opml_data))
except:
import traceback
log.error(traceback.format_exc())
abort(400)
result = defaultdict(list)
oldfeeds = set(i.url for i in composite.subscriptions.itervalues())
remove = oldfeeds - feeds
for url in remove:
feed = RemoteFeed.get_by_url(url, ctx)
if feed is not None:
composite.unsubscribe(feed)
result["unsubscribed"].append(url)
log.debug('Unsubscribed composite "%s" from %s' % (id, url))
else:
result["unsubscribe_failed"].append(url)
log.error('Expected composite "%s" to have RemoteFeed for %s' % (id, url))
for url in feeds:
if url not in oldfeeds:
feed = get_or_immediate_create_by_url(url, ctx)
if feed is None:
result["subscribe_failed"].append(url)
log.warn("Could not get or create feed for %s" % url)
continue
composite.subscribe(feed)
result["subscribed"].append(url)
log.debug('Subscribed composite "%s" to %s' % (id, url))
else:
result["unchanged"].append(url)
composite.save()
log.debug('Composite "%s" saved' % id)
return json_response(result)
def test_push_index(ctx):
from melkman.db.remotefeed import RemoteFeed
from melkman.fetch import push_feed_index
from melkman.fetch.worker import run_feed_indexer
from eventlet import sleep, spawn
# start a feed indexer
indexer = spawn(run_feed_indexer, ctx)
try:
url = 'http://www.example.com/feeds/2'
content = random_atom_feed(url, 10)
ids = melk_ids_in(content, url)
push_feed_index(url, content, ctx)
sleep(.5)
rf = RemoteFeed.get_by_url(url, ctx)
for iid in ids:
assert iid in rf.entries
finally:
indexer.kill()
indexer.wait()
def _handle_new_subscriptions(message_data, message, context):
"""
helper handler called when new subscriptions are added to
a composite.
"""
try:
new_subscriptions = message_data.get('new_subscriptions', [])
if len(new_subscriptions) == 0:
log.warn("Ignoring init_subscription with no new subscriptions...")
return
cid = message_data.get('bucket_id', None)
if cid is None:
log.error("Ignoring init_subscription with no bucket_id: %s" % message_data)
return
composite = Composite.get(cid, context)
if composite is None or not 'Composite' in composite.document_types:
log.error("Ignoring subscription update for non-existent composite %s" % cid)
return
new_feeds = []
updates = 0
for sub in new_subscriptions:
if not sub in composite.subscriptions:
log.warn("ignoring subscription %s -> %s, not in composite" % (sub, cid))
continue
bucket = NewsBucket.get(sub, context)
if bucket is None:
log.warn("Ignoring init subscription to unknown object (%s)" % composite.subscriptions[sub])
continue
# try 'casting' to a RemoteFeed
if 'RemoteFeed' in bucket.document_types:
rf = RemoteFeed.from_doc(bucket.unwrap(), context)
# mark as needing immediate fetch if
# there is no history for this feed.
if len(rf.update_history) == 0:
new_feeds.append(rf.url)
continue
try:
log.debug("init subscription %s -> %s" % (sub, cid))
updates += composite.init_subscription(sub)
sleep(0) # yield control
except:
log.error("Error initializing subscription %s -> %s: %s" % (sub, cid, traceback.format_exc()))
if updates > 0:
try:
composite.save()
except ResourceConflict:
# not a big deal in this case. This basically means
# our timestamp did not become the latest -- we
# have made no alterations other than adding items.
# Our additions succeed/fail independently of this as they
# are separate documents.
pass
# request that we start indexing anything new...
for url in new_feeds:
request_feed_index(url, context)
except:
log.error("Error handling init_subscrition %s: %s" % (message_data, traceback.format_exc()))
raise