def test_direct_entries_access(ctx):
from melkman.db.bucket import NewsBucket, NewsItemRef
# if you add something to bucket.entries directly rather than going
# through the interface, make sure it's there after saving (i.e. NewsBucket
# is properly observing changes to the underlying nldict)
bucket = NewsBucket.create(ctx)
assert len(bucket.entries) == 0
item1 = NewsItemRef.create_from_info(ctx, bucket.id, item_id=random_id())
item2 = NewsItemRef.create_from_info(ctx, bucket.id, item_id=random_id())
bucket.entries[item1.item_id] = item1
bucket.entries[item2.item_id] = item2
assert len(bucket.entries) == 2
assert bucket.has_news_item(item1)
assert bucket.has_news_item(item2)
bucket.save()
bucket.reload()
assert len(bucket.entries) == 2
assert bucket.has_news_item(item1)
assert bucket.has_news_item(item2)
# directly changing the maxlen of the underlying nldict instead of using
# NewsBucket.set_maxlen is *not* supported:
bucket.entries.maxlen = 1 # causes items to be deleted in memory:
assert len(bucket.entries) == 1
# but the document's maxlen field is not updated:
assert bucket.maxlen is None
# so after re-retrieving the document the nldict's maxlen is the old value
bucket.reload()
assert bucket.entries.maxlen is None
assert len(bucket.entries) == 2 # and the bucket still has two items
# if we save after changing the nldict's maxlen...
bucket.entries.maxlen = 1 # items deleted in memory
assert len(bucket.entries) == 1
bucket.save()
# ...items are now deleted from persistent storage too
bucket.reload()
assert len(bucket.entries) == 1
# but the maxlen field on the document was still never set
assert bucket.maxlen == None
# add back both items since one of them was discarded
bucket.entries.update({item1.item_id: item1, item2.item_id: item2})
bucket.save()
bucket.reload()
assert len(bucket.entries) == 2
# if we set the maxlen field on the bucket rather than using set_maxlen...
bucket.maxlen = 1
# maxlen changes in the document but the underlying nldict is not updated:
assert len(bucket.entries) == 2
# ...until *after* saving:
bucket.save()
bucket.reload()
assert len(bucket.entries) == 1
def test_delete(ctx):
from melkman.db import RemoteFeed, NewsItem, NewsItemRef
feed_url = 'http://example.org/feeds/1'
dummy_feed = random_atom_feed(feed_url, 25)
items = melk_ids_in(dummy_feed, feed_url)
rf = RemoteFeed.create_from_url(feed_url, ctx)
rf.update_from_feed(dummy_feed, 'test')
rf.save()
bucket_id = rf.id
ref_ids = []
assert bucket_id in ctx.db
for iid in items:
assert iid in rf.entries
ref_id = NewsItemRef.dbid(bucket_id, iid)
ref_ids.append(ref_id)
assert ref_id in ctx.db
# a news item was created too...
assert iid in ctx.db
# now destroy!
rf.delete()
assert not bucket_id in ctx.db
for ref_id in ref_ids:
assert not ref_id in ctx.db
for iid in items:
assert not iid in ctx.db
def test_bucket_delete(ctx):
from melkman.db import NewsBucket, NewsItemRef
bucket = NewsBucket.create(ctx)
items = [random_id() for i in range(1000)]
for iid in items:
bucket.add_news_item(iid)
bucket.save()
bucket_id = bucket.id
ref_ids = []
assert bucket_id in ctx.db
for iid in items:
assert iid in bucket.entries
ref_id = NewsItemRef.dbid(bucket.id, iid)
ref_ids.append(ref_id)
assert ref_id in ctx.db
# now destroy!
bucket.delete()
assert not bucket_id in ctx.db
for ref_id in ref_ids:
assert not ref_id in ctx.db
def test_view_bucket_entries_by_timestamp(ctx):
from melkman.db import NewsBucket, NewsItemRef
from melkman.db.bucket import view_entries_by_timestamp
from random import shuffle
bucket_id = 'test_bucket'
bucket = NewsBucket.create(ctx, bucket_id)
first_date = datetime.today()
items = [(random_id(), first_date - timedelta(days=i)) for i in range(100)]
jumbled_items = list(items)
shuffle(jumbled_items)
for iid, timestamp in jumbled_items:
bucket.add_news_item({'item_id': iid, 'timestamp': timestamp})
bucket.save()
# make sure they're all in there
for iid, timestamp in items:
assert bucket.has_news_item(iid)
# insure they come out in the right order in the index
query = {
'startkey': [bucket_id, {}],
'endkey': [bucket_id],
'limit': 200,
'descending': True,
'include_docs': True
}
sorted_items = [NewsItemRef.from_doc(r.doc, ctx) for r in
view_entries_by_timestamp(ctx.db, **query)]
assert len(sorted_items) == 100
for i, item in enumerate(sorted_items):
assert item.item_id == items[i][0]
def test_bucket_maxlen(ctx):
"""
Test that bucket with maxlen behaves as expected
"""
from melkman.db.bucket import NewsBucket, NewsItemRef, SORTKEY
from datetime import datetime, timedelta
from operator import attrgetter
# add 10 items spaced an hour apart to a bucket of max-length 3:
maxlen = 3
sortkey = SORTKEY # for now this is hardcoded in melkman.db.bucket
bucket = NewsBucket.create(ctx, maxlen=maxlen)
assert bucket.maxlen == maxlen
items = []
timestamp = datetime.utcnow()
for i in xrange(10):
item = NewsItemRef.create_from_info(ctx, bucket.id,
item_id=random_id(),
timestamp=timestamp - timedelta(hours=i),
)
items.append(item)
bucket.add_news_item(item)
# make sure the bucket has only the maxlen latest items
# before and after saving:
idgetter = attrgetter('item_id')
sorteditemsids = map(idgetter, sorted(items, key=sortkey))
def ids_by_timestamp(entries):
return map(idgetter, sorted(entries.values(), key=sortkey))
def check_before_and_after_save(bucket):
bucketlen = len(bucket.entries)
if bucket.maxlen is not None:
assert bucketlen <= bucket.maxlen
assert ids_by_timestamp(bucket.entries) == sorteditemsids[-bucketlen:]
bucket.save()
bucket.reload()
assert ids_by_timestamp(bucket.entries) == sorteditemsids[-bucketlen:]
return bucket
bucket = check_before_and_after_save(bucket)
# decrease maxlen and make sure bucket.entries remains consistent:
maxlen -= 1
bucket.set_maxlen(maxlen)
bucket = check_before_and_after_save(bucket)
# now increase maxlen so that the bucket is under capacity and check consistency:
maxlen += 2
bucket.set_maxlen(maxlen)
bucket = check_before_and_after_save(bucket)
# fill to capacity and check that the new maxlen is maintained:
for i in items:
bucket.add_news_item(i)
bucket = check_before_and_after_save(bucket)
# now set the maxlen to None and make sure it becomes an unbounded bucket:
maxlen = None
bucket.set_maxlen(maxlen)
for i in items:
bucket.add_news_item(i)
bucket = check_before_and_after_save(bucket)