def test_lifecycle_with_worklist(self):
facets = Facets.default(self._default_library)
pagination = Pagination.default()
lane = WorkList()
lane.initialize(self._default_library)
# Fetch a cached feed from the database. It comes out updated.
refresher = MockFeedGenerator()
args = (self._db, lane, facets, pagination, refresher)
feed = CachedFeed.fetch(*args, max_age=0, raw=True)
assert "This is feed #1" == feed.content
assert pagination.query_string == feed.pagination
assert facets.query_string == feed.facets
assert None == feed.lane_id
assert lane.unique_key == feed.unique_key
# Fetch it again, with a high max_age, and it's cached!
feed = CachedFeed.fetch(*args, max_age=1000, raw=True)
assert "This is feed #1" == feed.content
# Fetch it with a low max_age, and it gets updated again.
feed = CachedFeed.fetch(*args, max_age=0, raw=True)
assert "This is feed #2" == feed.content
# The special constant CACHE_FOREVER means it's always cached.
feed = CachedFeed.fetch(*args, max_age=CachedFeed.CACHE_FOREVER, raw=True)
assert "This is feed #2" == feed.content
def suppressed(cls, _db, title, url, annotator, pagination=None):
pagination = pagination or Pagination.default()
q = _db.query(LicensePool).filter(
LicensePool.suppressed == True).order_by(
LicensePool.id
)
pools = pagination.apply(q).all()
works = [pool.work for pool in pools]
feed = cls(_db, title, url, works, annotator)
# Render a 'start' link
top_level_title = annotator.top_level_title()
start_uri = annotator.groups_url(None)
feed.add_link(href=start_uri, rel="start", title=top_level_title)
# Render an 'up' link, same as the 'start' link to indicate top-level feed
feed.add_link(href=start_uri, rel="up", title=top_level_title)
if len(works) > 0:
# There are works in this list. Add a 'next' link.
feed.add_link(rel="next", href=annotator.suppressed_url(pagination.next_page))
if pagination.offset > 0:
feed.add_link(rel="first", href=annotator.suppressed_url(pagination.first_page))
previous_page = pagination.previous_page
if previous_page:
feed.add_link(rel="previous", href=annotator.suppressed_url(previous_page))
annotator.annotate_feed(feed)
return unicode(feed)
def test_pagination(self):
script = CacheFacetListsPerLane(self._db, manager=object(), cmd_args=[])
script.pages = 3
lane = self._lane()
p1, p2, p3 = script.pagination(lane)
pagination = Pagination.default()
eq_(pagination.query_string, p1.query_string)
eq_(pagination.next_page.query_string, p2.query_string)
eq_(pagination.next_page.next_page.query_string, p3.query_string)
def test_pagination(self):
script = CacheFacetListsPerLane(self._db, manager=object(), cmd_args=[])
script.pages = 3
lane = self._lane()
p1, p2, p3 = script.pagination(lane)
pagination = Pagination.default()
eq_(pagination.query_string, p1.query_string)
eq_(pagination.next_page.query_string, p2.query_string)
eq_(pagination.next_page.next_page.query_string, p3.query_string)
def complaints(cls, library, title, url, annotator, pagination=None):
_db = Session.object_session(library)
facets = Facets.default(library)
pagination = pagination or Pagination.default()
q = LicensePool.with_complaint(library)
results = pagination.modify_database_query(_db, q).all()
if len(results) > 0:
(pools, counts) = list(zip(*results))
else:
pools = ()
works = [pool.work for pool in pools]
feed = cls(_db, title, url, works, annotator)
# Render a 'start' link
top_level_title = annotator.top_level_title()
start_uri = annotator.groups_url(None)
AdminFeed.add_link_to_feed(feed.feed,
href=start_uri,
rel="start",
title=top_level_title)
# Render an 'up' link, same as the 'start' link to indicate top-level feed
AdminFeed.add_link_to_feed(feed.feed,
href=start_uri,
rel="up",
title=top_level_title)
if len(works) > 0:
# There are works in this list. Add a 'next' link.
AdminFeed.add_link_to_feed(
feed.feed,
rel="next",
href=annotator.complaints_url(facets, pagination.next_page),
)
if pagination.offset > 0:
AdminFeed.add_link_to_feed(
feed.feed,
rel="first",
href=annotator.complaints_url(facets, pagination.first_page),
)
previous_page = pagination.previous_page
if previous_page:
AdminFeed.add_link_to_feed(
feed.feed,
rel="previous",
href=annotator.complaints_url(facets, previous_page),
)
annotator.annotate_feed(feed)
return str(feed)
def suppressed(cls, _db, title, url, annotator, pagination=None):
pagination = pagination or Pagination.default()
q = (_db.query(LicensePool).filter(
and_(
LicensePool.suppressed == True,
LicensePool.superceded == False,
)).order_by(LicensePool.id))
pools = pagination.modify_database_query(_db, q).all()
works = [pool.work for pool in pools]
feed = cls(_db, title, url, works, annotator)
# Render a 'start' link
top_level_title = annotator.top_level_title()
start_uri = annotator.groups_url(None)
AdminFeed.add_link_to_feed(feed.feed,
href=start_uri,
rel="start",
title=top_level_title)
# Render an 'up' link, same as the 'start' link to indicate top-level feed
AdminFeed.add_link_to_feed(feed.feed,
href=start_uri,
rel="up",
title=top_level_title)
if len(works) > 0:
# There are works in this list. Add a 'next' link.
AdminFeed.add_link_to_feed(
feed.feed,
rel="next",
href=annotator.suppressed_url(pagination.next_page),
)
if pagination.offset > 0:
AdminFeed.add_link_to_feed(
feed.feed,
rel="first",
href=annotator.suppressed_url(pagination.first_page),
)
previous_page = pagination.previous_page
if previous_page:
AdminFeed.add_link_to_feed(
feed.feed,
rel="previous",
href=annotator.suppressed_url(previous_page))
annotator.annotate_feed(feed)
return str(feed)
def test_do_generate(self):
# When it's time to generate a feed, AcquisitionFeed.page
# is called with the right arguments.
class MockAcquisitionFeed(object):
called_with = None
@classmethod
def page(cls, **kwargs):
cls.called_with = kwargs
return "here's your feed"
# Test our ability to generate a single feed.
script = CacheFacetListsPerLane(self._db, testing=True, cmd_args=[])
facets = Facets.default(self._default_library)
pagination = Pagination.default()
with script.app.test_request_context("/"):
lane = self._lane()
result = script.do_generate(
lane, facets, pagination, feed_class=MockAcquisitionFeed
)
eq_("here's your feed", result)
args = MockAcquisitionFeed.called_with
eq_(self._db, args['_db'])
eq_(lane, args['lane'])
eq_(lane.display_name, args['title'])
eq_(True, args['force_refresh'])
# The Pagination object was passed into
# MockAcquisitionFeed.page, and it was also used to make the
# feed URL (see below).
eq_(pagination, args['pagination'])
# The Facets object was passed into
# MockAcquisitionFeed.page, and it was also used to make
# the feed URL and to create the feed annotator.
eq_(facets, args['facets'])
annotator = args['annotator']
eq_(facets, annotator.facets)
eq_(
args['url'],
annotator.feed_url(lane, facets=facets, pagination=pagination)
)
# Try again without mocking AcquisitionFeed to verify that
# we get something that looks like an OPDS feed.
result = script.do_generate(lane, facets, pagination)
assert result.startswith('<feed')
def test_do_generate(self):
# When it's time to generate a feed, AcquisitionFeed.page
# is called with the right arguments.
class MockAcquisitionFeed(object):
called_with = None
@classmethod
def page(cls, **kwargs):
cls.called_with = kwargs
return "here's your feed"
# Test our ability to generate a single feed.
script = CacheFacetListsPerLane(self._db, testing=True, cmd_args=[])
facets = Facets.default(self._default_library)
pagination = Pagination.default()
with script.app.test_request_context("/"):
lane = self._lane()
result = script.do_generate(
lane, facets, pagination, feed_class=MockAcquisitionFeed
)
eq_("here's your feed", result)
args = MockAcquisitionFeed.called_with
eq_(self._db, args['_db'])
eq_(lane, args['lane'])
eq_(lane.display_name, args['title'])
eq_(True, args['force_refresh'])
# The Pagination object was passed into
# MockAcquisitionFeed.page, and it was also used to make the
# feed URL (see below).
eq_(pagination, args['pagination'])
# The Facets object was passed into
# MockAcquisitionFeed.page, and it was also used to make
# the feed URL and to create the feed annotator.
eq_(facets, args['facets'])
annotator = args['annotator']
eq_(facets, annotator.facets)
eq_(
args['url'],
annotator.feed_url(lane, facets=facets, pagination=pagination)
)
# Try again without mocking AcquisitionFeed to verify that
# we get something that looks like an OPDS feed.
result = script.do_generate(lane, facets, pagination)
assert result.startswith('<feed')
def test_process_lane(self):
# process_lane() calls do_generate() once for every
# combination of items yielded by facets() and pagination().
class MockFacets(object):
def __init__(self, query):
self.query = query
@property
def query_string(self):
return self.query
facets1 = MockFacets("facets1")
facets2 = MockFacets("facets2")
page1 = Pagination.default()
page2 = page1.next_page
class Mock(CacheRepresentationPerLane):
generated = []
def do_generate(self, lane, facets, pagination):
value = (lane, facets, pagination)
self.generated.append(value)
return value
def facets(self, lane):
yield facets1
yield facets2
def pagination(self, lane):
yield page1
yield page2
lane = self._lane()
script = Mock(self._db, manager=object(), cmd_args=[])
generated = script.process_lane(lane)
eq_(generated, script.generated)
c1, c2, c3, c4 = script.generated
eq_((lane, facets1, page1), c1)
eq_((lane, facets1, page2), c2)
eq_((lane, facets2, page1), c3)
eq_((lane, facets2, page2), c4)
def test_process_lane(self):
# process_lane() calls do_generate() once for every
# combination of items yielded by facets() and pagination().
class MockFacets(object):
def __init__(self, query):
self.query = query
@property
def query_string(self):
return self.query
facets1 = MockFacets("facets1")
facets2 = MockFacets("facets2")
page1 = Pagination.default()
page2 = page1.next_page
class Mock(CacheRepresentationPerLane):
generated = []
def do_generate(self, lane, facets, pagination):
value = (lane, facets, pagination)
self.generated.append(value)
return value
def facets(self, lane):
yield facets1
yield facets2
def pagination(self, lane):
yield page1
yield page2
lane = self._lane()
script = Mock(self._db, manager=object(), cmd_args=[])
generated = script.process_lane(lane)
eq_(generated, script.generated)
c1, c2, c3, c4 = script.generated
eq_((lane, facets1, page1), c1)
eq_((lane, facets1, page2), c2)
eq_((lane, facets2, page1), c3)
eq_((lane, facets2, page2), c4)
def do_generate(self, lane):
feeds = []
annotator = self.app.manager.annotator(lane)
if isinstance(lane, Lane):
languages = lane.language_key
lane_name = None
else:
languages = None
lane_name = None
url = self.app.manager.cdn_url_for(
"feed", languages=lane.languages, lane_name=lane_name
)
order_facets = Configuration.enabled_facets(
Facets.ORDER_FACET_GROUP_NAME
)
availability = Configuration.default_facet(
Facets.AVAILABILITY_FACET_GROUP_NAME
)
collection = Configuration.default_facet(
Facets.COLLECTION_FACET_GROUP_NAME
)
for sort_order in order_facets:
pagination = Pagination.default()
facets = Facets(
collection=collection, availability=availability,
order=sort_order, order_ascending=True
)
title = lane.display_name
for pagenum in (0, 2):
feeds.append(
AcquisitionFeed.page(
self._db, title, url, lane, annotator,
facets=facets, pagination=pagination,
force_refresh=True
)
)
pagination = pagination.next_page
return feeds
def do_generate(self, lane):
feeds = []
annotator = self.app.manager.annotator(lane)
if isinstance(lane, Lane) and lane.parent:
languages = lane.language_key
lane_name = lane.name
else:
languages = None
lane_name = None
url = self.app.manager.cdn_url_for("feed",
languages=lane.languages,
lane_name=lane_name)
order_facets = Configuration.enabled_facets(
Facets.ORDER_FACET_GROUP_NAME)
availability = Configuration.default_facet(
Facets.AVAILABILITY_FACET_GROUP_NAME)
collection = Configuration.default_facet(
Facets.COLLECTION_FACET_GROUP_NAME)
for sort_order in self.orders:
for availability in self.availabilities:
for collection in self.collections:
pagination = Pagination.default()
facets = Facets(collection=collection,
availability=availability,
order=sort_order,
order_ascending=True)
title = lane.display_name
for pagenum in range(0, self.pages):
yield AcquisitionFeed.page(self._db,
title,
url,
lane,
annotator,
facets=facets,
pagination=pagination,
force_refresh=True)
pagination = pagination.next_page
def complaints(cls, library, title, url, annotator, pagination=None):
_db = Session.object_session(library)
facets = Facets.default(library)
pagination = pagination or Pagination.default()
q = LicensePool.with_complaint(library)
results = pagination.apply(q).all()
if len(results) > 0:
(pools, counts) = zip(*results)
else:
pools = ()
works = [pool.work for pool in pools]
feed = cls(_db, title, url, works, annotator)
# Render a 'start' link
top_level_title = annotator.top_level_title()
start_uri = annotator.groups_url(None)
AdminFeed.add_link_to_feed(feed.feed, href=start_uri, rel="start", title=top_level_title)
# Render an 'up' link, same as the 'start' link to indicate top-level feed
AdminFeed.add_link_to_feed(feed.feed, href=start_uri, rel="up", title=top_level_title)
if len(works) > 0:
# There are works in this list. Add a 'next' link.
AdminFeed.add_link_to_feed(feed.feed, rel="next", href=annotator.complaints_url(facets, pagination.next_page))
if pagination.offset > 0:
AdminFeed.add_link_to_feed(feed.feed, rel="first", href=annotator.complaints_url(facets, pagination.first_page))
previous_page = pagination.previous_page
if previous_page:
AdminFeed.add_link_to_feed(feed.feed, rel="previous", href=annotator.complaints_url(facets, previous_page))
annotator.annotate_feed(feed)
return unicode(feed)
def do_generate(self, lane):
feeds = []
annotator = self.app.manager.annotator(lane)
if isinstance(lane, Lane):
lane_id = lane.id
else:
# Presumably this is the top-level WorkList.
lane_id = None
library = lane.get_library(self._db)
url = self.app.manager.cdn_url_for(
"feed", lane_identifier=lane_id,
library_short_name=library.short_name
)
default_order = library.default_facet(Facets.ORDER_FACET_GROUP_NAME)
allowed_orders = library.enabled_facets(Facets.ORDER_FACET_GROUP_NAME)
chosen_orders = self.orders or [default_order]
default_availability = library.default_facet(
Facets.AVAILABILITY_FACET_GROUP_NAME
)
allowed_availabilities = library.enabled_facets(
Facets.AVAILABILITY_FACET_GROUP_NAME
)
chosen_availabilities = self.availabilities or [default_availability]
default_collection = library.default_facet(
Facets.COLLECTION_FACET_GROUP_NAME
)
allowed_collections = library.enabled_facets(
Facets.COLLECTION_FACET_GROUP_NAME
)
chosen_collections = self.collections or [default_collection]
for order in chosen_orders:
if order not in allowed_orders:
logging.warn("Ignoring unsupported ordering %s" % order)
continue
for availability in chosen_availabilities:
if availability not in allowed_availabilities:
logging.warn("Ignoring unsupported availability %s" % availability)
continue
for collection in chosen_collections:
if collection not in allowed_collections:
logging.warn("Ignoring unsupported collection %s" % collection)
continue
pagination = Pagination.default()
facets = Facets(
library=library, collection=collection,
availability=availability,
order=order, order_ascending=True
)
title = lane.display_name
for pagenum in range(0, self.pages):
yield AcquisitionFeed.page(
self._db, title, url, lane, annotator,
facets=facets, pagination=pagination,
force_refresh=True
)
pagination = pagination.next_page
def test_response_format(self):
# Verify that fetch() can be told to return an appropriate
# OPDSFeedResponse object. This is the default behavior, since
# it preserves some useful information that would otherwise be
# lost.
facets = Facets.default(self._default_library)
pagination = Pagination.default()
wl = WorkList()
wl.initialize(self._default_library)
def refresh():
return "Here's a feed."
private = object()
r = CachedFeed.fetch(
self._db, wl, facets, pagination, refresh, max_age=102, private=private
)
assert isinstance(r, OPDSFeedResponse)
assert 200 == r.status_code
assert OPDSFeed.ACQUISITION_FEED_TYPE == r.content_type
assert 102 == r.max_age
assert "Here's a feed." == str(r)
# The extra argument `private`, not used by CachedFeed.fetch, was
# passed on to the OPDSFeedResponse constructor.
assert private == r.private
# The CachedFeed was created; just not returned.
cf = self._db.query(CachedFeed).one()
assert "Here's a feed." == cf.content
# Try it again as a cache hit.
r = CachedFeed.fetch(
self._db, wl, facets, pagination, refresh, max_age=102, private=private
)
assert isinstance(r, OPDSFeedResponse)
assert 200 == r.status_code
assert OPDSFeed.ACQUISITION_FEED_TYPE == r.content_type
assert 102 == r.max_age
assert "Here's a feed." == str(r)
# If we tell CachedFeed to cache its feed 'forever', that only
# applies to the _database_ cache. The client is told to cache
# the feed for the default period.
r = CachedFeed.fetch(
self._db,
wl,
facets,
pagination,
refresh,
max_age=CachedFeed.CACHE_FOREVER,
private=private,
)
assert isinstance(r, OPDSFeedResponse)
assert OPDSFeed.DEFAULT_MAX_AGE == r.max_age
# If the Library associated with the WorkList used in the feed
# has root lanes, `private` is always set to True, even if we
# asked for the opposite.
from core.model import Library
Library._has_root_lane_cache[self._default_library.id] = True
r = CachedFeed.fetch(self._db, wl, facets, pagination, refresh, private=False)
assert isinstance(r, OPDSFeedResponse)
assert True == r.private
def test_no_race_conditions(self):
# Why do we look up a CachedFeed again after feed generation?
# Well, let's see what happens if someone else messes around
# with the CachedFeed object _while the refresher is running_.
#
# This is a race condition that happens in real life. Rather
# than setting up a multi-threaded test, we can have the
# refresher itself simulate a background modification by
# messing around with the CachedFeed object we know will
# eventually be returned.
#
# The most up-to-date feed always wins, so background
# modifications will take effect only if they made the
# CachedFeed look _newer_ than the foreground process does.
facets = Facets.default(self._default_library)
pagination = Pagination.default()
wl = WorkList()
wl.initialize(self._default_library)
m = CachedFeed.fetch
# In this case, two simulated threads try to create the same
# CachedFeed at the same time. We end up with a single
# CachedFeed containing the result of the last code that ran.
def simultaneous_refresher():
# This refresher method simulates another thread creating
# a CachedFeed for this feed while this thread's
# refresher is running.
def other_thread_refresher():
return "Another thread made a feed."
m(self._db, wl, facets, pagination, other_thread_refresher, 0, raw=True)
return "Then this thread made a feed."
# This will call simultaneous_refresher(), which will call
# CachedFeed.fetch() _again_, which will call
# other_thread_refresher().
result = m(
self._db, wl, facets, pagination, simultaneous_refresher, 0, raw=True
)
# We ended up with a single CachedFeed containing the
# latest information.
assert [result] == self._db.query(CachedFeed).all()
assert "Then this thread made a feed." == result.content
# If two threads contend for an existing CachedFeed, the one that
# sets CachedFeed.timestamp to the later value wins.
#
# Here, the other thread wins by setting .timestamp on the
# existing CachedFeed to a date in the future.
now = utc_now()
tomorrow = now + datetime.timedelta(days=1)
yesterday = now - datetime.timedelta(days=1)
def tomorrow_vs_now():
result.content = "Someone in the background set tomorrow's content."
result.timestamp = tomorrow
return "Today's content can't compete."
tomorrow_result = m(
self._db, wl, facets, pagination, tomorrow_vs_now, 0, raw=True
)
assert tomorrow_result == result
assert (
"Someone in the background set tomorrow's content."
== tomorrow_result.content
)
assert tomorrow_result.timestamp == tomorrow
# Here, the other thread sets .timestamp to a date in the past, and
# it loses out to the (apparently) newer feed.
def yesterday_vs_now():
result.content = "Someone in the background set yesterday's content."
result.timestamp = yesterday
return "Today's content is fresher."
now_result = m(self._db, wl, facets, pagination, yesterday_vs_now, 0, raw=True)
# We got the same CachedFeed we've been getting this whole
# time, but the outdated data set by the 'background thread'
# has been fixed.
assert result == now_result
assert "Today's content is fresher." == result.content
assert result.timestamp > yesterday
# This shouldn't happen, but if the CachedFeed's timestamp or
# content are *cleared out* in the background, between the
# time the CacheFeed is fetched and the time the refresher
# finishes, then we don't know what's going on and we don't
# take chances. We create a whole new CachedFeed object for
# the updated version of the feed.
# First, try the situation where .timestamp is cleared out in
# the background.
def timestamp_cleared_in_background():
result.content = "Someone else sets content and clears timestamp."
result.timestamp = None
return "Non-weird content."
result2 = m(
self._db,
wl,
facets,
pagination,
timestamp_cleared_in_background,
0,
raw=True,
)
now = utc_now()
# result2 is a brand new CachedFeed.
assert result2 != result
assert "Non-weird content." == result2.content
assert (now - result2.timestamp).total_seconds() < 2
# We let the background process do whatever it wants to do
# with the old one.
assert "Someone else sets content and clears timestamp." == result.content
assert None == result.timestamp
# Next, test the situation where .content is cleared out.
def content_cleared_in_background():
result2.content = None
result2.timestamp = tomorrow
return "Non-weird content."
result3 = m(
self._db, wl, facets, pagination, content_cleared_in_background, 0, raw=True
)
now = utc_now()
# Again, a brand new CachedFeed.
assert result3 != result2
assert result3 != result
assert "Non-weird content." == result3.content
assert (now - result3.timestamp).total_seconds() < 2
# Again, we let the background process have the old one for
# whatever weird thing it wants to do.
assert None == result2.content
assert tomorrow == result2.timestamp
