def test_run_once(self):
overdrive, isbn, axis_360 = self.prep_feed_identifiers()
# Give one of the identifiers a full work.
self._work(presentation_edition=overdrive.primarily_identifies[0])
# And another identifier a work without entries.
w = self._work(presentation_edition=isbn.primarily_identifies[0])
w.simple_opds_entry = w.verbose_opds_entry = None
# Queue some response feeds.
feed1 = sample_data('metadata_data_needed_response.opds', 'opds')
feed2 = sample_data('metadata_data_needed_empty_response.opds', 'opds')
for feed in [feed1, feed2]:
self.lookup.queue_response(
200, {'content-type': OPDSFeed.ACQUISITION_FEED_TYPE}, feed)
self.monitor.run_once(None, None)
# Only the identifier with a work has been given coverage.
record = CoverageRecord.lookup(
overdrive,
self.monitor.provider.data_source,
operation=self.monitor.provider.operation)
assert record
for identifier in [axis_360, isbn]:
record = CoverageRecord.lookup(
identifier,
self.monitor.provider.data_source,
operation=self.monitor.provider.operation)
eq_(None, record)
def add_coverage_record_for(self, item):
"""Record this CoverageProvider's coverage for the given
Edition/Identifier in the known Collection
"""
return CoverageRecord.add_for(item,
data_source=self.data_source,
operation=self.operation,
collection=self.collection)
def register_identifier_as_unresolved(self, urn, identifier):
# This identifier could have a presentation-ready Work
# associated with it, but it doesn't. We need to make sure the
# work gets done eventually by creating a CoverageRecord
# representing the work that needs to be done.
source = DataSource.lookup(self._db, DataSource.INTERNAL_PROCESSING)
record = CoverageRecord.lookup(identifier, source, self.OPERATION)
is_new = False
if not record:
# There is no existing CoverageRecord for this Identifier.
# Create one, but put it in a state of transient failure
# to represent the fact that work needs to be done.
record, is_new = CoverageRecord.add_for(
identifier,
source,
self.OPERATION,
status=CoverageRecord.TRANSIENT_FAILURE)
record.exception = self.NO_WORK_DONE_EXCEPTION
if is_new:
# The CoverageRecord was just created. Tell the client to
# come back later.
return self.add_message(urn, HTTP_CREATED,
self.IDENTIFIER_REGISTERED)
else:
# There is a pending attempt to resolve this identifier.
# Tell the client we're working on it, or if the
# pending attempt resulted in an exception,
# tell the client about the exception.
message = record.exception
if not message or message == self.NO_WORK_DONE_EXCEPTION:
message = self.WORKING_TO_RESOLVE_IDENTIFIER
status = HTTP_ACCEPTED
if record.status == record.PERSISTENT_FAILURE:
# Apparently we just can't provide coverage of this
# identifier.
status = HTTP_INTERNAL_SERVER_ERROR
elif record.status == record.SUCCESS:
# This shouldn't happen, since success in providing
# this sort of coverage means creating a presentation
# ready work. Something weird is going on.
status = HTTP_INTERNAL_SERVER_ERROR
message = self.SUCCESS_DID_NOT_RESULT_IN_PRESENTATION_READY_WORK
return self.add_message(urn, status, message)
def test_process_urn_isbn(self):
# Create a new ISBN identifier.
# Ask online providers for metadata to turn into an opds feed about this identifier.
# Make sure a coverage record was created, and a 201 status obtained from provider.
# Ask online provider again, and make sure we're now getting a 202 "working on it" status.
# Ask again, this time getting a result. Make sure know that got a result.
isbn, ignore = Identifier.for_foreign_id(
self._db, Identifier.ISBN, self._isbn
)
# The first time we look up an ISBN a CoverageRecord is created
# representing the work to be done.
self.controller.process_urn(isbn.urn)
self.assert_one_message(
isbn.urn, HTTP_CREATED, self.controller.IDENTIFIER_REGISTERED
)
[record] = isbn.coverage_records
eq_(record.exception, self.controller.NO_WORK_DONE_EXCEPTION)
eq_(record.status, CoverageRecord.TRANSIENT_FAILURE)
# So long as the necessary coverage is not provided,
# future lookups will not provide useful information
self.controller.precomposed_entries = []
self.controller.process_urn(isbn.urn)
self.assert_one_message(
isbn.urn, HTTP_ACCEPTED, self.controller.WORKING_TO_RESOLVE_IDENTIFIER
)
# Let's provide the coverage.
metadata_sources = DataSource.metadata_sources_for(
self._db, isbn
)
for source in metadata_sources:
CoverageRecord.add_for(isbn, source)
# Process the ISBN again, and we get an <entry> tag with the
# information.
self.controller.precomposed_entries = []
self.controller.process_urn(isbn.urn)
expect = isbn.opds_entry()
[actual] = self.controller.precomposed_entries
eq_(etree.tostring(expect), etree.tostring(actual))
def register_identifier_as_unresolved(self, urn, identifier):
# This identifier could have a presentation-ready Work
# associated with it, but it doesn't. We need to make sure the
# work gets done eventually by creating a CoverageRecord
# representing the work that needs to be done.
source = DataSource.lookup(self._db, DataSource.INTERNAL_PROCESSING)
record = CoverageRecord.lookup(identifier, source, self.OPERATION)
is_new = False
if not record:
# There is no existing CoverageRecord for this Identifier.
# Create one, but put it in a state of transient failure
# to represent the fact that work needs to be done.
record, is_new = CoverageRecord.add_for(
identifier, source, self.OPERATION,
status=CoverageRecord.TRANSIENT_FAILURE
)
record.exception = self.NO_WORK_DONE_EXCEPTION
if is_new:
# The CoverageRecord was just created. Tell the client to
# come back later.
return self.add_message(urn, HTTP_CREATED, self.IDENTIFIER_REGISTERED)
else:
# There is a pending attempt to resolve this identifier.
# Tell the client we're working on it, or if the
# pending attempt resulted in an exception,
# tell the client about the exception.
message = record.exception
if not message or message == self.NO_WORK_DONE_EXCEPTION:
message = self.WORKING_TO_RESOLVE_IDENTIFIER
status = HTTP_ACCEPTED
if record.status == record.PERSISTENT_FAILURE:
# Apparently we just can't provide coverage of this
# identifier.
status = HTTP_INTERNAL_SERVER_ERROR
elif record.status == record.SUCCESS:
# This shouldn't happen, since success in providing
# this sort of coverage means creating a presentation
# ready work. Something weird is going on.
status = HTTP_INTERNAL_SERVER_ERROR
message = self.SUCCESS_DID_NOT_RESULT_IN_PRESENTATION_READY_WORK
return self.add_message(urn, status, message)
def test_process_urn_exception_during_resolve_attempt(self):
identifier = self._identifier(Identifier.GUTENBERG_ID)
record, is_new = CoverageRecord.add_for(
identifier, self.source, self.controller.OPERATION,
status=CoverageRecord.PERSISTENT_FAILURE
)
record.exception = "foo"
self.controller.process_urn(identifier.urn)
self.assert_one_message(
identifier.urn, HTTP_INTERNAL_SERVER_ERROR, "foo"
)
def test_process_urn_exception_during_resolve_attempt(self):
identifier = self._identifier(Identifier.GUTENBERG_ID)
record, is_new = CoverageRecord.add_for(
identifier,
self.source,
self.controller.OPERATION,
status=CoverageRecord.PERSISTENT_FAILURE)
record.exception = "foo"
self.controller.process_urn(identifier.urn)
self.assert_one_message(identifier.urn, HTTP_INTERNAL_SERVER_ERROR,
"foo")
def test_run_once(self):
overdrive, isbn, axis_360 = self.prep_feed_identifiers()
# Give one of the identifiers a full work.
self._work(presentation_edition=overdrive.primarily_identifies[0])
# And another identifier a work without entries.
w = self._work(presentation_edition=isbn.primarily_identifies[0])
w.simple_opds_entry = w.verbose_opds_entry = None
# Queue some response feeds.
feed1 = sample_data("metadata_data_needed_response.opds", "opds")
feed2 = sample_data("metadata_data_needed_empty_response.opds", "opds")
for feed in [feed1, feed2]:
self.lookup.queue_response(
200, {"content-type": OPDSFeed.ACQUISITION_FEED_TYPE}, feed)
progress = self.monitor.run_once(self.ts)
# Only the identifier with a work has been given coverage.
assert "Identifiers processed: 1" == progress.achievements
# The TimestampData returned by run_once() does not include
# any timing information -- that will be applied by run().
assert None == progress.start
assert None == progress.finish
record = CoverageRecord.lookup(
overdrive,
self.monitor.provider.data_source,
operation=self.monitor.provider.operation,
)
assert record
for identifier in [axis_360, isbn]:
record = CoverageRecord.lookup(
identifier,
self.monitor.provider.data_source,
operation=self.monitor.provider.operation,
)
assert None == record
def test_process_urn_no_presentation_ready_work(self):
identifier = self._identifier(Identifier.GUTENBERG_ID)
# There's a record of success, but no presentation-ready work.
record, is_new = CoverageRecord.add_for(identifier,
self.source,
self.controller.OPERATION,
status=CoverageRecord.SUCCESS)
self.controller.process_urn(identifier.urn)
self.assert_one_message(
identifier.urn, HTTP_INTERNAL_SERVER_ERROR,
self.controller.SUCCESS_DID_NOT_RESULT_IN_PRESENTATION_READY_WORK)
def test_run_once(self):
overdrive, isbn, axis_360 = self.prep_feed_identifiers()
# Give one of the identifiers a full work.
self._work(presentation_edition=overdrive.primarily_identifies[0])
# And another identifier a work without entries.
w = self._work(presentation_edition=isbn.primarily_identifies[0])
w.simple_opds_entry = w.verbose_opds_entry = None
# Queue some response feeds.
feed1 = sample_data('metadata_data_needed_response.opds', 'opds')
feed2 = sample_data('metadata_data_needed_empty_response.opds', 'opds')
for feed in [feed1, feed2]:
self.lookup.queue_response(
200, {'content-type' : OPDSFeed.ACQUISITION_FEED_TYPE}, feed
)
progress = self.monitor.run_once(self.ts)
# Only the identifier with a work has been given coverage.
eq_("Identifiers processed: 1", progress.achievements)
# The TimestampData returned by run_once() does not include
# any timing information -- that will be applied by run().
eq_(None, progress.start)
eq_(None, progress.finish)
record = CoverageRecord.lookup(
overdrive, self.monitor.provider.data_source,
operation=self.monitor.provider.operation
)
assert record
for identifier in [axis_360, isbn]:
record = CoverageRecord.lookup(
identifier, self.monitor.provider.data_source,
operation=self.monitor.provider.operation
)
eq_(None, record)
def test_process_urn_isbn(self):
# Create a new ISBN identifier.
# Ask online providers for metadata to turn into an opds feed about this identifier.
# Make sure a coverage record was created, and a 201 status obtained from provider.
# Ask online provider again, and make sure we're now getting a 202 "working on it" status.
# Ask again, this time getting a result. Make sure know that got a result.
isbn, ignore = Identifier.for_foreign_id(self._db, Identifier.ISBN,
self._isbn)
# The first time we look up an ISBN a CoverageRecord is created
# representing the work to be done.
self.controller.process_urn(isbn.urn)
self.assert_one_message(isbn.urn, HTTP_CREATED,
self.controller.IDENTIFIER_REGISTERED)
[record] = isbn.coverage_records
eq_(record.exception, self.controller.NO_WORK_DONE_EXCEPTION)
eq_(record.status, CoverageRecord.TRANSIENT_FAILURE)
# So long as the necessary coverage is not provided,
# future lookups will not provide useful information
self.controller.precomposed_entries = []
self.controller.process_urn(isbn.urn)
self.assert_one_message(isbn.urn, HTTP_ACCEPTED,
self.controller.WORKING_TO_RESOLVE_IDENTIFIER)
# Let's provide the coverage.
metadata_sources = DataSource.metadata_sources_for(self._db, isbn)
for source in metadata_sources:
CoverageRecord.add_for(isbn, source)
# Process the ISBN again, and we get an <entry> tag with the
# information.
self.controller.precomposed_entries = []
self.controller.process_urn(isbn.urn)
expect = isbn.opds_entry()
[actual] = self.controller.precomposed_entries
eq_(etree.tostring(expect), etree.tostring(actual))
def test_process_urn_no_presentation_ready_work(self):
identifier = self._identifier(Identifier.GUTENBERG_ID)
# There's a record of success, but no presentation-ready work.
record, is_new = CoverageRecord.add_for(
identifier, self.source, self.controller.OPERATION,
status=CoverageRecord.SUCCESS
)
self.controller.process_urn(identifier.urn)
self.assert_one_message(
identifier.urn, HTTP_INTERNAL_SERVER_ERROR,
self.controller.SUCCESS_DID_NOT_RESULT_IN_PRESENTATION_READY_WORK
)
def test_process_urn_pending_resolve_attempt(self):
# Simulate calling process_urn twice, and make sure the
# second call results in an "I'm working on it, hold your horses" message.
identifier = self._identifier(Identifier.GUTENBERG_ID)
record, is_new = CoverageRecord.add_for(
identifier, self.source, self.controller.OPERATION,
status=CoverageRecord.TRANSIENT_FAILURE
)
record.exception = self.controller.NO_WORK_DONE_EXCEPTION
self.controller.process_urn(identifier.urn)
self.assert_one_message(
identifier.urn, HTTP_ACCEPTED,
URNLookupController.WORKING_TO_RESOLVE_IDENTIFIER
)
def test_process_urn_pending_resolve_attempt(self):
# Simulate calling process_urn twice, and make sure the
# second call results in an "I'm working on it, hold your horses" message.
identifier = self._identifier(Identifier.GUTENBERG_ID)
record, is_new = CoverageRecord.add_for(
identifier,
self.source,
self.controller.OPERATION,
status=CoverageRecord.TRANSIENT_FAILURE)
record.exception = self.controller.NO_WORK_DONE_EXCEPTION
self.controller.process_urn(identifier.urn)
self.assert_one_message(
identifier.urn, HTTP_ACCEPTED,
URNLookupController.WORKING_TO_RESOLVE_IDENTIFIER)
def test_refresh_metadata(self):
wrangler = DataSource.lookup(self._db, DataSource.METADATA_WRANGLER)
success_provider = AlwaysSuccessfulCoverageProvider(
"Always successful", [Identifier.GUTENBERG_ID], wrangler
)
failure_provider = NeverSuccessfulCoverageProvider(
"Never successful", [Identifier.GUTENBERG_ID], wrangler
)
with self.app.test_request_context('/'):
[lp] = self.english_1.license_pools
response = self.manager.admin_work_controller.refresh_metadata(
lp.data_source.name, lp.identifier.type, lp.identifier.identifier, provider=success_provider
)
eq_(200, response.status_code)
# Also, the work has a coverage record now for the wrangler.
assert CoverageRecord.lookup(lp.identifier, wrangler)
response = self.manager.admin_work_controller.refresh_metadata(
lp.data_source.name, lp.identifier.type, lp.identifier.identifier, provider=failure_provider
)
eq_(METADATA_REFRESH_FAILURE.status_code, response.status_code)
eq_(METADATA_REFRESH_FAILURE.detail, response.detail)