def test_initialization(self):
with pytest.raises(ValueError) as excinfo:
ContributorLane(self._default_library, None)
assert "ContributorLane can't be created without contributor" in str(
excinfo.value
)
parent = WorkList()
parent.initialize(self._default_library)
lane = ContributorLane(
self._default_library,
self.contributor,
parent,
languages=["a"],
audiences=["b"],
)
assert self.contributor == lane.contributor
assert ["a"] == lane.languages
assert ["b"] == lane.audiences
assert [lane] == parent.children
# The contributor_key will be used in links to other pages
# of this Lane and so on.
assert "Lois Lane" == lane.contributor_key
# If the contributor used to create a ContributorLane has no
# display name, their sort name is used as the
# contributor_key.
contributor = ContributorData(sort_name="Lane, Lois")
lane = ContributorLane(self._default_library, contributor)
assert contributor == lane.contributor
assert "Lane, Lois" == lane.contributor_key
def test_lifecycle_with_worklist(self):
facets = Facets.default(self._default_library)
pagination = Pagination.default()
lane = WorkList()
lane.initialize(self._default_library)
# Fetch a cached feed from the database. It comes out updated.
refresher = MockFeedGenerator()
args = (self._db, lane, facets, pagination, refresher)
feed = CachedFeed.fetch(*args, max_age=0, raw=True)
assert "This is feed #1" == feed.content
assert pagination.query_string == feed.pagination
assert facets.query_string == feed.facets
assert None == feed.lane_id
assert lane.unique_key == feed.unique_key
# Fetch it again, with a high max_age, and it's cached!
feed = CachedFeed.fetch(*args, max_age=1000, raw=True)
assert "This is feed #1" == feed.content
# Fetch it with a low max_age, and it gets updated again.
feed = CachedFeed.fetch(*args, max_age=0, raw=True)
assert "This is feed #2" == feed.content
# The special constant CACHE_FOREVER means it's always cached.
feed = CachedFeed.fetch(*args, max_age=CachedFeed.CACHE_FOREVER, raw=True)
assert "This is feed #2" == feed.content
def test_initialization(self):
assert_raises_regexp(
ValueError, "ContributorLane can't be created without contributor",
ContributorLane, self._default_library, None)
parent = WorkList()
parent.initialize(self._default_library)
lane = ContributorLane(
self._default_library,
self.contributor,
parent,
languages=['a'],
audiences=['b'],
)
eq_(self.contributor, lane.contributor)
eq_(['a'], lane.languages)
eq_(['b'], lane.audiences)
eq_([lane], parent.children)
# The contributor_key will be used in links to other pages
# of this Lane and so on.
eq_("Lois Lane", lane.contributor_key)
# If the contributor used to create a ContributorLane has no
# display name, their sort name is used as the
# contributor_key.
contributor = ContributorData(sort_name="Lane, Lois")
lane = ContributorLane(self._default_library, contributor)
eq_(contributor, lane.contributor)
eq_("Lane, Lois", lane.contributor_key)
def make_child():
# Set up a WorkList with settings that contradict the
# settings of the work we'll be using as the basis for our
# WorkBasedLane.
child = WorkList()
child.initialize(self._default_library,
'sublane',
languages=['eng'],
audiences=[Classifier.AUDIENCE_ADULT])
return child
def test_load_facets_from_request(self):
# The library has two EntryPoints enabled.
self._default_library.setting(
EntryPoint.ENABLED_SETTING).value = json.dumps([
EbooksEntryPoint.INTERNAL_NAME,
AudiobooksEntryPoint.INTERNAL_NAME
])
with self.app.test_request_context("/?order=%s" % Facets.ORDER_TITLE):
flask.request.library = self._default_library
facets = load_facets_from_request()
assert Facets.ORDER_TITLE == facets.order
# Enabled facets are passed in to the newly created Facets,
# in case the load method received a custom config.
assert facets.facets_enabled_at_init != None
with self.app.test_request_context("/?order=bad_facet"):
flask.request.library = self._default_library
problemdetail = load_facets_from_request()
assert INVALID_INPUT.uri == problemdetail.uri
# An EntryPoint will be picked up from the request and passed
# into the Facets object, assuming the EntryPoint is
# configured on the present library.
worklist = WorkList()
worklist.initialize(self._default_library)
with self.app.test_request_context("/?entrypoint=Audio"):
flask.request.library = self._default_library
facets = load_facets_from_request(worklist=worklist)
assert AudiobooksEntryPoint == facets.entrypoint
assert False == facets.entrypoint_is_default
# If the requested EntryPoint not configured, the default
# EntryPoint is used.
with self.app.test_request_context("/?entrypoint=NoSuchEntryPoint"):
flask.request.library = self._default_library
default_entrypoint = object()
facets = load_facets_from_request(
worklist=worklist, default_entrypoint=default_entrypoint)
assert default_entrypoint == facets.entrypoint
assert True == facets.entrypoint_is_default
# Load a SearchFacets object that pulls information from an
# HTTP header.
with self.app.test_request_context("/",
headers={"Accept-Language": "ja"}):
flask.request.library = self._default_library
facets = load_facets_from_request(base_class=SearchFacets)
assert ["jpn"] == facets.languages
def load_lanes(_db, library):
"""Return a WorkList that reflects the current lane structure of the
Library.
If no top-level visible lanes are configured, the WorkList will be
configured to show every book in the collection.
If a single top-level Lane is configured, it will returned as the
WorkList.
Otherwise, a WorkList containing the visible top-level lanes is
returned.
"""
top_level = WorkList.top_level_for_library(_db, library)
# It's likely this WorkList will be used across sessions, so
# expunge any data model objects from the database session.
#
# TODO: This is the cause of a lot of problems in the cached OPDS
# feed generator. There, these Lanes are used in a normal database
# session and we end up needing hacks to merge them back into the
# session.
if isinstance(top_level, Lane):
to_expunge = [top_level]
else:
to_expunge = [x for x in top_level.children if isinstance(x, Lane)]
map(_db.expunge, to_expunge)
return top_level
def test_should_process_lane(self):
parent = self._lane()
parent.size = 100
child = self._lane(parent=parent)
child.size = 10
grandchild = self._lane(parent=child)
grandchild.size = 1
wl = WorkList()
empty = self._lane(fiction=False)
empty.size = 0
script = CacheMARCFiles(self._db, cmd_args=[])
script.max_depth = 1
eq_(True, script.should_process_lane(parent))
eq_(True, script.should_process_lane(child))
eq_(False, script.should_process_lane(grandchild))
eq_(True, script.should_process_lane(wl))
eq_(False, script.should_process_lane(empty))
script.max_depth = 0
eq_(True, script.should_process_lane(parent))
eq_(False, script.should_process_lane(child))
eq_(False, script.should_process_lane(grandchild))
eq_(True, script.should_process_lane(wl))
eq_(False, script.should_process_lane(empty))
def load_lanes(_db, library):
"""Return a WorkList that reflects the current lane structure of the
Library.
If no top-level visible lanes are configured, the WorkList will be
configured to show every book in the collection.
If a single top-level Lane is configured, it will returned as the
WorkList.
Otherwise, a WorkList containing the visible top-level lanes is
returned.
"""
top_level = WorkList.top_level_for_library(_db, library)
# It's likely this WorkList will be used across sessions, so
# expunge any data model objects from the database session.
if isinstance(top_level, Lane):
to_expunge = [top_level]
else:
to_expunge = [x for x in top_level.children if isinstance(x, Lane)]
map(_db.expunge, to_expunge)
return top_level
def load_lanes(_db, library):
"""Return a WorkList that reflects the current lane structure of the
Library.
If no top-level visible lanes are configured, the WorkList will be
configured to show every book in the collection.
If a single top-level Lane is configured, it will returned as the
WorkList.
Otherwise, a WorkList containing the visible top-level lanes is
returned.
"""
top_level = WorkList.top_level_for_library(_db, library)
# It's likely this WorkList will be used across sessions, so
# expunge any data model objects from the database session.
if isinstance(top_level, Lane):
to_expunge = [top_level]
else:
to_expunge = [x for x in top_level.children if isinstance(x, Lane)]
map(_db.expunge, to_expunge)
return top_level
def test_do_generate_handles_all_entrypoints(self):
self.called_with = []
@classmethod
def mock_groups(cls, *args, **kwargs):
self.called_with.append((args, kwargs))
old_groups = AcquisitionFeed.groups
AcquisitionFeed.groups = mock_groups
# Here's a normal WorkList with no EntryPoints.
worklist = WorkList()
library = self._default_library
worklist.initialize(library)
script = CacheOPDSGroupFeedPerLane(self._db, cmd_args=[])
with script.app.test_request_context("/"):
list(script.do_generate(worklist))
# AcquisitionFeed.groups was called once, with a FeaturedFacets
# object that did not include an EntryPoint.
args, kwargs = self.called_with.pop()
facets = kwargs['facets']
assert isinstance(facets, FeaturedFacets)
eq_(library.minimum_featured_quality, facets.minimum_featured_quality)
eq_(worklist.uses_customlists, facets.uses_customlists)
eq_(None, facets.entrypoint)
# Now give the WorkList some EntryPoints.
worklist.initialize(
library, entrypoints=[AudiobooksEntryPoint, EbooksEntryPoint])
with script.app.test_request_context("/"):
list(script.do_generate(worklist))
# AcquisitionFeed.groups was called once for each
# EntryPoint available to the WorkList.
eq_([AudiobooksEntryPoint, EbooksEntryPoint], [
kwargs['facets'].entrypoint for (args, kwargs) in self.called_with
])
AcquisitionFeed.groups = old_groups
def test__prepare_keys(self):
# Verify the method that turns WorkList, Facets, and Pagination
# into a unique set of values for CachedFeed fields.
# First, prepare some mock classes.
class MockCachedFeed(CachedFeed):
feed_type_called_with = None
@classmethod
def feed_type(cls, worklist, facets):
cls.feed_type_called_with = (worklist, facets)
return "mock type"
class MockFacets(object):
query_string = b"facets query string"
class MockPagination(object):
query_string = b"pagination query string"
m = MockCachedFeed._prepare_keys
# A WorkList of some kind is required.
with pytest.raises(ValueError) as excinfo:
m(self._db, None, MockFacets, MockPagination)
assert "Cannot prepare a CachedFeed without a WorkList." in str(excinfo.value)
# Basic Lane case, no facets or pagination.
lane = self._lane()
# The response object is a named tuple. feed_type, library and
# lane_id are the only members set.
keys = m(self._db, lane, None, None)
assert "mock type" == keys.feed_type
assert lane.library == keys.library
assert None == keys.work
assert lane.id == keys.lane_id
assert None == keys.unique_key
assert "" == keys.facets_key
assert "" == keys.pagination_key
# When pagination and/or facets are available, facets_key and
# pagination_key are set appropriately.
keys = m(self._db, lane, MockFacets, MockPagination)
assert "facets query string" == keys.facets_key
assert "pagination query string" == keys.pagination_key
# Now we can check that feed_type was obtained by passing
# `worklist` and `facets` into MockCachedFeed.feed_type.
assert "mock type" == keys.feed_type
assert (lane, MockFacets) == MockCachedFeed.feed_type_called_with
# When a WorkList is used instead of a Lane, keys.lane_id is None
# but keys.unique_key is set to worklist.unique_key.
worklist = WorkList()
worklist.initialize(
library=self._default_library,
display_name="wl",
languages=["eng", "spa"],
audiences=[Classifier.AUDIENCE_CHILDREN],
)
keys = m(self._db, worklist, None, None)
assert "mock type" == keys.feed_type
assert worklist.get_library(self._db) == keys.library
assert None == keys.work
assert None == keys.lane_id
assert "wl-eng,spa-Children" == keys.unique_key
assert keys.unique_key == worklist.unique_key
assert "" == keys.facets_key
assert "" == keys.pagination_key
# When a WorkList is associated with a specific .work,
# that information is included as keys.work.
work = object()
worklist.work = work
keys = m(self._db, worklist, None, None)
assert work == keys.work
def test_response_format(self):
# Verify that fetch() can be told to return an appropriate
# OPDSFeedResponse object. This is the default behavior, since
# it preserves some useful information that would otherwise be
# lost.
facets = Facets.default(self._default_library)
pagination = Pagination.default()
wl = WorkList()
wl.initialize(self._default_library)
def refresh():
return "Here's a feed."
private = object()
r = CachedFeed.fetch(
self._db, wl, facets, pagination, refresh, max_age=102, private=private
)
assert isinstance(r, OPDSFeedResponse)
assert 200 == r.status_code
assert OPDSFeed.ACQUISITION_FEED_TYPE == r.content_type
assert 102 == r.max_age
assert "Here's a feed." == str(r)
# The extra argument `private`, not used by CachedFeed.fetch, was
# passed on to the OPDSFeedResponse constructor.
assert private == r.private
# The CachedFeed was created; just not returned.
cf = self._db.query(CachedFeed).one()
assert "Here's a feed." == cf.content
# Try it again as a cache hit.
r = CachedFeed.fetch(
self._db, wl, facets, pagination, refresh, max_age=102, private=private
)
assert isinstance(r, OPDSFeedResponse)
assert 200 == r.status_code
assert OPDSFeed.ACQUISITION_FEED_TYPE == r.content_type
assert 102 == r.max_age
assert "Here's a feed." == str(r)
# If we tell CachedFeed to cache its feed 'forever', that only
# applies to the _database_ cache. The client is told to cache
# the feed for the default period.
r = CachedFeed.fetch(
self._db,
wl,
facets,
pagination,
refresh,
max_age=CachedFeed.CACHE_FOREVER,
private=private,
)
assert isinstance(r, OPDSFeedResponse)
assert OPDSFeed.DEFAULT_MAX_AGE == r.max_age
# If the Library associated with the WorkList used in the feed
# has root lanes, `private` is always set to True, even if we
# asked for the opposite.
from core.model import Library
Library._has_root_lane_cache[self._default_library.id] = True
r = CachedFeed.fetch(self._db, wl, facets, pagination, refresh, private=False)
assert isinstance(r, OPDSFeedResponse)
assert True == r.private
def test_no_race_conditions(self):
# Why do we look up a CachedFeed again after feed generation?
# Well, let's see what happens if someone else messes around
# with the CachedFeed object _while the refresher is running_.
#
# This is a race condition that happens in real life. Rather
# than setting up a multi-threaded test, we can have the
# refresher itself simulate a background modification by
# messing around with the CachedFeed object we know will
# eventually be returned.
#
# The most up-to-date feed always wins, so background
# modifications will take effect only if they made the
# CachedFeed look _newer_ than the foreground process does.
facets = Facets.default(self._default_library)
pagination = Pagination.default()
wl = WorkList()
wl.initialize(self._default_library)
m = CachedFeed.fetch
# In this case, two simulated threads try to create the same
# CachedFeed at the same time. We end up with a single
# CachedFeed containing the result of the last code that ran.
def simultaneous_refresher():
# This refresher method simulates another thread creating
# a CachedFeed for this feed while this thread's
# refresher is running.
def other_thread_refresher():
return "Another thread made a feed."
m(self._db, wl, facets, pagination, other_thread_refresher, 0, raw=True)
return "Then this thread made a feed."
# This will call simultaneous_refresher(), which will call
# CachedFeed.fetch() _again_, which will call
# other_thread_refresher().
result = m(
self._db, wl, facets, pagination, simultaneous_refresher, 0, raw=True
)
# We ended up with a single CachedFeed containing the
# latest information.
assert [result] == self._db.query(CachedFeed).all()
assert "Then this thread made a feed." == result.content
# If two threads contend for an existing CachedFeed, the one that
# sets CachedFeed.timestamp to the later value wins.
#
# Here, the other thread wins by setting .timestamp on the
# existing CachedFeed to a date in the future.
now = utc_now()
tomorrow = now + datetime.timedelta(days=1)
yesterday = now - datetime.timedelta(days=1)
def tomorrow_vs_now():
result.content = "Someone in the background set tomorrow's content."
result.timestamp = tomorrow
return "Today's content can't compete."
tomorrow_result = m(
self._db, wl, facets, pagination, tomorrow_vs_now, 0, raw=True
)
assert tomorrow_result == result
assert (
"Someone in the background set tomorrow's content."
== tomorrow_result.content
)
assert tomorrow_result.timestamp == tomorrow
# Here, the other thread sets .timestamp to a date in the past, and
# it loses out to the (apparently) newer feed.
def yesterday_vs_now():
result.content = "Someone in the background set yesterday's content."
result.timestamp = yesterday
return "Today's content is fresher."
now_result = m(self._db, wl, facets, pagination, yesterday_vs_now, 0, raw=True)
# We got the same CachedFeed we've been getting this whole
# time, but the outdated data set by the 'background thread'
# has been fixed.
assert result == now_result
assert "Today's content is fresher." == result.content
assert result.timestamp > yesterday
# This shouldn't happen, but if the CachedFeed's timestamp or
# content are *cleared out* in the background, between the
# time the CacheFeed is fetched and the time the refresher
# finishes, then we don't know what's going on and we don't
# take chances. We create a whole new CachedFeed object for
# the updated version of the feed.
# First, try the situation where .timestamp is cleared out in
# the background.
def timestamp_cleared_in_background():
result.content = "Someone else sets content and clears timestamp."
result.timestamp = None
return "Non-weird content."
result2 = m(
self._db,
wl,
facets,
pagination,
timestamp_cleared_in_background,
0,
raw=True,
)
now = utc_now()
# result2 is a brand new CachedFeed.
assert result2 != result
assert "Non-weird content." == result2.content
assert (now - result2.timestamp).total_seconds() < 2
# We let the background process do whatever it wants to do
# with the old one.
assert "Someone else sets content and clears timestamp." == result.content
assert None == result.timestamp
# Next, test the situation where .content is cleared out.
def content_cleared_in_background():
result2.content = None
result2.timestamp = tomorrow
return "Non-weird content."
result3 = m(
self._db, wl, facets, pagination, content_cleared_in_background, 0, raw=True
)
now = utc_now()
# Again, a brand new CachedFeed.
assert result3 != result2
assert result3 != result
assert "Non-weird content." == result3.content
assert (now - result3.timestamp).total_seconds() < 2
# Again, we let the background process have the old one for
# whatever weird thing it wants to do.
assert None == result2.content
assert tomorrow == result2.timestamp
def test_records(self):
integration = self._integration()
now = utc_now()
exporter = MARCExporter.from_config(self._default_library)
annotator = Annotator()
lane = self._lane("Test Lane", genres=["Mystery"])
w1 = self._work(genre="Mystery", with_open_access_download=True)
w2 = self._work(genre="Mystery", with_open_access_download=True)
search_engine = MockExternalSearchIndex()
search_engine.bulk_update([w1, w2])
# If there's a storage protocol but not corresponding storage integration,
# it raises an exception.
pytest.raises(Exception, exporter.records, lane, annotator)
# If there is a storage integration, the output file is mirrored.
mirror_integration = self._external_integration(
ExternalIntegration.S3,
ExternalIntegration.STORAGE_GOAL,
username="******",
password="******",
)
mirror = MockS3Uploader()
exporter.records(
lane,
annotator,
mirror_integration,
mirror=mirror,
query_batch_size=1,
upload_batch_size=1,
search_engine=search_engine,
)
# The file was mirrored and a CachedMARCFile was created to track the mirrored file.
assert 1 == len(mirror.uploaded)
[cache] = self._db.query(CachedMARCFile).all()
assert self._default_library == cache.library
assert lane == cache.lane
assert mirror.uploaded[0] == cache.representation
assert None == cache.representation.content
assert ("https://test-marc-bucket.s3.amazonaws.com/%s/%s/%s.mrc" % (
self._default_library.short_name,
quote(str(cache.representation.fetched_at)),
quote(lane.display_name),
) == mirror.uploaded[0].mirror_url)
assert None == cache.start_time
assert cache.end_time > now
# The content was uploaded in two parts.
assert 2 == len(mirror.content[0])
complete_file = b"".join(mirror.content[0])
records = list(MARCReader(complete_file))
assert 2 == len(records)
title_fields = [record.get_fields("245") for record in records]
titles = [fields[0].get_subfields("a")[0] for fields in title_fields]
assert set([w1.title, w2.title]) == set(titles)
assert w1.title in w1.marc_record
assert w2.title in w2.marc_record
self._db.delete(cache)
# It also works with a WorkList instead of a Lane, in which case
# there will be no lane in the CachedMARCFile.
worklist = WorkList()
worklist.initialize(self._default_library, display_name="All Books")
mirror = MockS3Uploader()
exporter.records(
worklist,
annotator,
mirror_integration,
mirror=mirror,
query_batch_size=1,
upload_batch_size=1,
search_engine=search_engine,
)
assert 1 == len(mirror.uploaded)
[cache] = self._db.query(CachedMARCFile).all()
assert self._default_library == cache.library
assert None == cache.lane
assert mirror.uploaded[0] == cache.representation
assert None == cache.representation.content
assert ("https://test-marc-bucket.s3.amazonaws.com/%s/%s/%s.mrc" % (
self._default_library.short_name,
quote(str(cache.representation.fetched_at)),
quote(worklist.display_name),
) == mirror.uploaded[0].mirror_url)
assert None == cache.start_time
assert cache.end_time > now
assert 2 == len(mirror.content[0])
complete_file = b"".join(mirror.content[0])
records = list(MARCReader(complete_file))
assert 2 == len(records)
self._db.delete(cache)
# If a start time is set, it's used in the mirror url.
#
# (Our mock search engine returns everthing in its 'index',
# so this doesn't test that the start time is actually used to
# find works -- that's in the search index tests and the
# tests of MARCExporterFacets.)
start_time = now - datetime.timedelta(days=3)
mirror = MockS3Uploader()
exporter.records(
lane,
annotator,
mirror_integration,
start_time=start_time,
mirror=mirror,
query_batch_size=2,
upload_batch_size=2,
search_engine=search_engine,
)
[cache] = self._db.query(CachedMARCFile).all()
assert self._default_library == cache.library
assert lane == cache.lane
assert mirror.uploaded[0] == cache.representation
assert None == cache.representation.content
assert ("https://test-marc-bucket.s3.amazonaws.com/%s/%s-%s/%s.mrc" % (
self._default_library.short_name,
quote(str(start_time)),
quote(str(cache.representation.fetched_at)),
quote(lane.display_name),
) == mirror.uploaded[0].mirror_url)
assert start_time == cache.start_time
assert cache.end_time > now
self._db.delete(cache)
# If the search engine returns no contents for the lane,
# nothing will be mirrored, but a CachedMARCFile is still
# created to track that we checked for updates.
empty_search_engine = MockExternalSearchIndex()
mirror = MockS3Uploader()
exporter.records(
lane,
annotator,
mirror_integration,
mirror=mirror,
search_engine=empty_search_engine,
)
assert [] == mirror.content[0]
[cache] = self._db.query(CachedMARCFile).all()
assert cache.representation == mirror.uploaded[0]
assert self._default_library == cache.library
assert lane == cache.lane
assert None == cache.representation.content
assert None == cache.start_time
assert cache.end_time > now
self._db.delete(cache)