def test_delta(self):
with temp_config() as config:
config[Configuration.INTEGRATIONS]['OneClick'] = {
'library_id' : '1931',
'username' : 'username_123',
'password' : 'password_123',
'remote_stage' : 'qa',
'base_url' : 'www.oneclickapi.test',
'basic_token' : 'abcdef123hijklm',
"ebook_loan_length" : '21',
"eaudio_loan_length" : '21'
}
cmd_args = ["--mock"]
# first, load a sample library
importer = OneClickImportScript(_db=self._db, cmd_args=cmd_args)
datastr, datadict = self.get_data("response_catalog_all_sample.json")
importer.api.queue_response(status_code=200, content=datastr)
importer.run()
# set license numbers on test pool
pool, made_new = LicensePool.for_foreign_id(self._db, DataSource.ONECLICK, Identifier.ONECLICK_ID, "9781615730186")
eq_(False, made_new)
pool.licenses_owned = 10
pool.licenses_available = 9
pool.licenses_reserved = 2
pool.patrons_in_hold_queue = 1
# now update that library with a sample delta
cmd_args = ["--mock"]
delta_runner = OneClickDeltaScript(_db=self._db, cmd_args=cmd_args)
datastr, datadict = self.get_data("response_catalog_delta.json")
delta_runner.api.queue_response(status_code=200, content=datastr)
delta_runner.run()
# "Tricks" did not get deleted, but did get its pools set to "nope".
# "Emperor Mage: The Immortals" got new metadata.
works = self._db.query(Work).all()
work_titles = [work.title for work in works]
expected_titles = ["Tricks", "Emperor Mage: The Immortals",
"In-Flight Russian", "Road, The", "Private Patient, The",
"Year of Magical Thinking, The", "Junkyard Bot: Robots Rule, Book 1, The",
"Challenger Deep"]
eq_(set(expected_titles), set(work_titles))
eq_("Tricks", pool.presentation_edition.title)
eq_(0, pool.licenses_owned)
eq_(0, pool.licenses_available)
eq_(0, pool.licenses_reserved)
eq_(0, pool.patrons_in_hold_queue)
assert (datetime.datetime.utcnow() - pool.last_checked) < datetime.timedelta(seconds=20)
# make sure we updated fields
edition = Edition.for_foreign_id(self._db, DataSource.ONECLICK, Identifier.ONECLICK_ID, "9781934180723", create_if_not_exists=False)
eq_("Recorded Books, Inc.", edition.publisher)
# make sure there are still 8 LicensePools
pools = self._db.query(LicensePool).all()
eq_(8, len(pools))
def _edition(self, data_source_name=DataSource.GUTENBERG,
identifier_type=Identifier.GUTENBERG_ID,
with_license_pool=False, with_open_access_download=False,
title=None, language="eng", authors=None, identifier_id=None):
id = identifier_id or self._str
source = DataSource.lookup(self._db, data_source_name)
wr = Edition.for_foreign_id(
self._db, source, identifier_type, id)[0]
if not title:
title = self._str
wr.title = unicode(title)
if language:
wr.language = language
if authors is None:
authors = self._str
if isinstance(authors, basestring):
authors = [authors]
if authors != []:
wr.add_contributor(unicode(authors[0]), Contributor.PRIMARY_AUTHOR_ROLE)
wr.author = unicode(authors[0])
for author in authors[1:]:
wr.add_contributor(unicode(author), Contributor.AUTHOR_ROLE)
if with_license_pool or with_open_access_download:
pool = self._licensepool(wr, data_source_name=data_source_name,
with_open_access_download=with_open_access_download)
pool.set_presentation_edition()
return wr, pool
return wr
def edition(self, identifier):
"""Finds or creates an Edition representing this coverage provider's
view of a given Identifier.
"""
edition, ignore = Edition.for_foreign_id(
self._db, self.data_source, identifier.type,
identifier.identifier
)
return edition
def test_import(self):
with temp_config() as config:
config[Configuration.INTEGRATIONS]['OneClick'] = {
'library_id' : '1931',
'username' : 'username_123',
'password' : 'password_123',
'remote_stage' : 'qa',
'base_url' : 'www.oneclickapi.test',
'basic_token' : 'abcdef123hijklm',
"ebook_loan_length" : '21',
"eaudio_loan_length" : '21'
}
cmd_args = ["--mock"]
importer = OneClickImportScript(_db=self._db, cmd_args=cmd_args)
datastr, datadict = self.get_data("response_catalog_all_sample.json")
importer.api.queue_response(status_code=200, content=datastr)
importer.run()
# verify that we created Works, Editions, LicensePools
works = self._db.query(Work).all()
work_titles = [work.title for work in works]
expected_titles = ["Tricks", "Emperor Mage: The Immortals",
"In-Flight Russian", "Road, The", "Private Patient, The",
"Year of Magical Thinking, The", "Junkyard Bot: Robots Rule, Book 1, The",
"Challenger Deep"]
eq_(set(expected_titles), set(work_titles))
# make sure we created some Editions
edition = Edition.for_foreign_id(self._db, DataSource.ONECLICK, Identifier.ONECLICK_ID, "9780062231727", create_if_not_exists=False)
assert(edition is not None)
edition = Edition.for_foreign_id(self._db, DataSource.ONECLICK, Identifier.ONECLICK_ID, "9781615730186", create_if_not_exists=False)
assert(edition is not None)
# make sure we created some LicensePools
pool, made_new = LicensePool.for_foreign_id(self._db, DataSource.ONECLICK, Identifier.ONECLICK_ID, "9780062231727")
eq_(False, made_new)
pool, made_new = LicensePool.for_foreign_id(self._db, DataSource.ONECLICK, Identifier.ONECLICK_ID, "9781615730186")
eq_(False, made_new)
# make sure there are 8 LicensePools
pools = self._db.query(LicensePool).all()
eq_(8, len(pools))
def test_register_equivalency(self):
data_source = DataSource.lookup(self._db, DataSource.GUTENBERG)
id = "549"
# We've got a record.
record, was_new = Edition.for_foreign_id(self._db, data_source,
Identifier.GUTENBERG_ID, id)
# Then we look it up and discover another identifier for it.
data_source_2 = DataSource.lookup(self._db, DataSource.OCLC)
record2, was_new = Edition.for_foreign_id(self._db, data_source_2,
Identifier.OCLC_NUMBER, "22")
eq = record.primary_identifier.equivalent_to(
data_source_2, record2.primary_identifier, 1)
eq_(eq.input, record.primary_identifier)
eq_(eq.output, record2.primary_identifier)
eq_(eq.data_source, data_source_2)
eq_([eq], record.primary_identifier.equivalencies)
eq_(set([record, record2]), set(record.equivalent_editions().all()))
def edition(self, identifier):
"""Finds or creates the Edition for a given Identifier."""
license_pool = self.license_pool(identifier)
if not license_pool:
e = "No license pool available"
return CoverageFailure(identifier,
e,
data_source=self.output_source,
transient=True)
edition, ignore = Edition.for_foreign_id(self._db,
license_pool.data_source,
identifier.type,
identifier.identifier)
return edition
def create_books_editions(books, gr_id, shelf_name):
""" Takes in a list of book/edition dictionaries and creates the corresponding
Book/Edition objects as needed. """
books_to_shelve = []
# write query to get shelf_id from gr_id and shelf_name
user = get_user_by_gr_id(gr_id)
shelf = db.session.query(Shelf).filter(Shelf.name == shelf_name, Shelf.user == user).one()
print shelf
for book in books:
try:
# check work id to see if any edition is in db already
db_book = db.session.query(Book).filter(Book.gr_work_id == book['gr_work_id']).one()
print "book found! added id to shelving list."
except:
# create new Book object
db_book = Book(title=book['title'],
author_name=book['author_name'],
author_gr_id=book['author_gr_id'],
description=book['description'],
gr_work_id=book['gr_work_id'])
# add book to db.session
db.session.add(db_book)
db.session.commit()
try:
db_edition = db.session.query(Edition).filter(Edition.gr_id == book['edition']['gr_id']).one()
except:
print book
db_edition = Edition(format_id=book['edition']['format_id'],
book_id=db_book.book_id,
isbn=book['edition']['isbn'],
pic_url=book['edition']['pic_url'],
publisher=book['edition']['publisher'],
date=book['edition']['date'],
gr_url=book['edition']['gr_url'],
gr_id=book['edition']['gr_id'],
num_pages=book['edition']['num_pages'])
db.session.add(db_edition)
db.session.commit()
# add ed_id to books_to_shelve list
books_to_shelve.append(db_edition.ed_id)
add_shelf_books(books_to_shelve, shelf)
def _edition(self, data_source_name=DataSource.GUTENBERG,
identifier_type=Identifier.GUTENBERG_ID,
with_license_pool=False, with_open_access_download=False,
title=None, language="eng", authors=None, identifier_id=None,
series=None, collection=None, publicationDate=None
):
id = identifier_id or self._str
source = DataSource.lookup(self._db, data_source_name)
wr = Edition.for_foreign_id(
self._db, source, identifier_type, id)[0]
if not title:
title = self._str
wr.title = unicode(title)
wr.medium = Edition.BOOK_MEDIUM
if series:
wr.series = series
if language:
wr.language = language
if authors is None:
authors = self._str
if isinstance(authors, basestring):
authors = [authors]
if authors != []:
wr.add_contributor(unicode(authors[0]), Contributor.PRIMARY_AUTHOR_ROLE)
wr.author = unicode(authors[0])
for author in authors[1:]:
wr.add_contributor(unicode(author), Contributor.AUTHOR_ROLE)
if publicationDate:
wr.published = publicationDate
if with_license_pool or with_open_access_download:
pool = self._licensepool(
wr, data_source_name=data_source_name,
with_open_access_download=with_open_access_download,
collection=collection
)
pool.set_presentation_edition()
return wr, pool
return wr
def saveEdition(self, edition):
newEd = Edition(items=[], links=[])
# Set Titles
try:
newEd.title = edition['title'].most_common(1)[0][0].strip(' .:/')
except AttributeError:
logger.warning('Unable to read title for edition')
if len(edition['sub_title']):
newEd.sub_title = edition['sub_title'].most_common(1)[0][0]
newEd.alt_titles = [t[0] for t in edition['alt_titles'].most_common()]
# Set Publication Date
newEd.publication_date = SFRRecordManager.publicationDateCheck(edition)
# Set Publication Place
newEd.publication_place = edition['publication_place'].most_common(
1)[0][0]
# Set Abstract Data
newEd.table_of_contents = edition['table_of_contents'].most_common(
1)[0][0]
newEd.extent = edition['extent'].most_common(1)[0][0]
newEd.summary = edition['summary'].most_common(1)[0][0]
# Set Edition Data
if len(edition['edition_data']):
editionStmt, editionNo = tuple(
edition['edition_data'].most_common(1)[0][0].split('|'))
newEd.edition_statement = editionStmt
try:
newEd.edition = int(editionNo)
except ValueError:
pass
# Set Volume Data
if len(edition['volume_data']):
volume, volumeNo, _ = tuple(
edition['volume_data'].most_common(1)[0][0].split('|'))
newEd.volume = '{}, {}'.format(volume, volumeNo)
# Set Agents
newEd.contributors = self.agentParser(
edition['contributors'], ['name', 'viaf', 'lcnaf', 'role'])
newEd.publishers = self.agentParser(edition['publishers'],
['name', 'viaf', 'lcnaf'])
# Set Identifiers
newEd.identifiers = SFRRecordManager.setPipeDelimitedData(
edition['identifiers'], ['identifier', 'authority'], Identifier)
# Set Languages
newEd.languages = SFRRecordManager.setPipeDelimitedData(
edition['languages'], ['language', 'iso_2', 'iso_3'],
dParser=self.getLanguage)
# Set Measurements
newEd.measurements = SFRRecordManager.setPipeDelimitedData(
edition['measurements'], ['value', 'type'])
# Set Dates
newEd.dates = SFRRecordManager.setPipeDelimitedData(
edition['dates'], ['date', 'type'])
# Set Links
newEd.links = SFRRecordManager.setPipeDelimitedData(
edition['links'], ['url', 'media_type', 'flags'],
Link,
dParser=SFRRecordManager.parseLinkFlags)
# Add Items
for item in edition['items']:
newEd.items.append(self.saveItem(item))
# Set DCDW UUIDs
newEd.dcdw_uuids = list(edition['dcdw_uuids'])
return newEd
def test_import(self):
# Create a OneClickImportMonitor, which will take the current
# state of a OneClick collection and mirror the whole thing to
# a local database.
monitor = OneClickImportMonitor(
self._db,
self.collection,
api_class=MockOneClickAPI,
api_class_kwargs=dict(base_path=self.base_path))
datastr, datadict = self.get_data("response_catalog_all_sample.json")
monitor.api.queue_response(status_code=200, content=datastr)
monitor.run()
# verify that we created Works, Editions, LicensePools
works = self._db.query(Work).all()
work_titles = [work.title for work in works]
expected_titles = [
"Tricks", "Emperor Mage: The Immortals", "In-Flight Russian",
"Road, The", "Private Patient, The",
"Year of Magical Thinking, The",
"Junkyard Bot: Robots Rule, Book 1, The", "Challenger Deep"
]
eq_(set(expected_titles), set(work_titles))
# make sure we created some Editions
edition = Edition.for_foreign_id(self._db,
DataSource.ONECLICK,
Identifier.ONECLICK_ID,
"9780062231727",
create_if_not_exists=False)
assert (edition is not None)
edition = Edition.for_foreign_id(self._db,
DataSource.ONECLICK,
Identifier.ONECLICK_ID,
"9781615730186",
create_if_not_exists=False)
assert (edition is not None)
# make sure we created some LicensePools
pool, made_new = LicensePool.for_foreign_id(self._db,
DataSource.ONECLICK,
Identifier.ONECLICK_ID,
"9780062231727",
collection=self.collection)
eq_(1, pool.licenses_owned)
eq_(1, pool.licenses_available)
eq_(False, made_new)
pool, made_new = LicensePool.for_foreign_id(self._db,
DataSource.ONECLICK,
Identifier.ONECLICK_ID,
"9781615730186",
collection=self.collection)
eq_(False, made_new)
eq_(1, pool.licenses_owned)
eq_(1, pool.licenses_available)
# make sure there are 8 LicensePools
pools = self._db.query(LicensePool).all()
eq_(8, len(pools))
#
# Now we're going to run the delta monitor to change things
# around a bit.
#
# set license numbers on test pool to match what's in the
# delta document.
pool, made_new = LicensePool.for_foreign_id(self._db,
DataSource.ONECLICK,
Identifier.ONECLICK_ID,
"9781615730186",
collection=self.collection)
eq_(False, made_new)
pool.licenses_owned = 10
pool.licenses_available = 9
pool.licenses_reserved = 2
pool.patrons_in_hold_queue = 1
# now update that library with a sample delta
delta_monitor = OneClickDeltaMonitor(
self._db,
self.collection,
api_class=MockOneClickAPI,
api_class_kwargs=dict(base_path=self.base_path))
datastr, datadict = self.get_data("response_catalog_delta.json")
delta_monitor.api.queue_response(status_code=200, content=datastr)
delta_monitor.run()
# "Tricks" did not get deleted, but did get its pools set to "nope".
# "Emperor Mage: The Immortals" got new metadata.
works = self._db.query(Work).all()
work_titles = [work.title for work in works]
expected_titles = [
"Tricks", "Emperor Mage: The Immortals", "In-Flight Russian",
"Road, The", "Private Patient, The",
"Year of Magical Thinking, The",
"Junkyard Bot: Robots Rule, Book 1, The", "Challenger Deep"
]
eq_(set(expected_titles), set(work_titles))
eq_("Tricks", pool.presentation_edition.title)
eq_(0, pool.licenses_owned)
eq_(0, pool.licenses_available)
eq_(0, pool.licenses_reserved)
eq_(0, pool.patrons_in_hold_queue)
assert (datetime.datetime.utcnow() -
pool.last_checked) < datetime.timedelta(seconds=20)
# make sure we updated fields
edition = Edition.for_foreign_id(self._db,
DataSource.ONECLICK,
Identifier.ONECLICK_ID,
"9781934180723",
create_if_not_exists=False)
eq_("Recorded Books, Inc.", edition.publisher)
# make sure there are still 8 LicensePools
pools = self._db.query(LicensePool).all()
eq_(8, len(pools))
# Running the monitor again does nothing. Since no more responses
# are queued, doing any work at this point would crash the test.
eq_((0, 0), monitor.invoke())
from nose.tools import set_trace
import os
import site
import sys
import datetime
d = os.path.split(__file__)[0]
site.addsitedir(os.path.join(d, ".."))
from integration.threem import (
ThreeMAPI, )
from integration.overdrive import (
OverdriveAPI, )
from model import (
production_session,
DataSource,
Edition,
Identifier,
)
if __name__ == '__main__':
type, identifier_name = sys.argv[1:3]
db = production_session()
identifier, is_new = Identifier.for_foreign_id(db, type, identifier_name)
if identifier.type == Identifier.THREEM_ID:
source = DataSource.lookup(db, DataSource.THREEM)
api = ThreeMAPI(db)
edition, ignore = Edition.for_foreign_id(db, source, type,
identifier_name)
data = api.get_bibliographic_info_for([edition])
import site
import sys
import datetime
d = os.path.split(__file__)[0]
site.addsitedir(os.path.join(d, ".."))
from integration.threem import (
ThreeMAPI,
)
from integration.overdrive import (
OverdriveAPI,
)
from model import (
production_session,
DataSource,
Edition,
Identifier,
)
if __name__ == '__main__':
type, identifier_name = sys.argv[1:3]
db = production_session()
identifier, is_new = Identifier.for_foreign_id(db, type, identifier_name)
if identifier.type==Identifier.THREEM_ID:
source = DataSource.lookup(db, DataSource.THREEM)
api = ThreeMAPI(db)
edition, ignore = Edition.for_foreign_id(
db, source, type, identifier_name)
data = api.get_bibliographic_info_for([edition])
def test_recursively_equivalent_identifiers(self):
# We start with a Gutenberg book.
gutenberg = DataSource.lookup(self._db, DataSource.GUTENBERG)
record, ignore = Edition.for_foreign_id(self._db, gutenberg,
Identifier.GUTENBERG_ID, "100")
gutenberg_id = record.primary_identifier
# We use OCLC Classify to do a title/author lookup.
oclc = DataSource.lookup(self._db, DataSource.OCLC)
search_id, ignore = Identifier.for_foreign_id(self._db,
Identifier.OCLC_WORK,
"60010")
gutenberg_id.equivalent_to(oclc, search_id, 1)
# The title/author lookup associates the search term with two
# different OCLC Numbers.
oclc_id, ignore = Identifier.for_foreign_id(self._db,
Identifier.OCLC_NUMBER,
"9999")
oclc_id_2, ignore = Identifier.for_foreign_id(self._db,
Identifier.OCLC_NUMBER,
"1000")
search_id.equivalent_to(oclc, oclc_id, 1)
search_id.equivalent_to(oclc, oclc_id_2, 1)
# We then use OCLC Linked Data to connect one of the OCLC
# Numbers with an ISBN.
linked_data = DataSource.lookup(self._db, DataSource.OCLC_LINKED_DATA)
isbn_id, ignore = Identifier.for_foreign_id(self._db, Identifier.ISBN,
"900100434X")
oclc_id.equivalent_to(linked_data, isbn_id, 1)
# As it turns out, we have an Overdrive work record...
overdrive = DataSource.lookup(self._db, DataSource.OVERDRIVE)
overdrive_record, ignore = Edition.for_foreign_id(
self._db, overdrive, Identifier.OVERDRIVE_ID, "{111-222}")
overdrive_id = overdrive_record.primary_identifier
# ...which is tied (by Overdrive) to the same ISBN.
overdrive_id.equivalent_to(overdrive, isbn_id, 1)
# Finally, here's a completely unrelated Edition, which
# will not be showing up.
gutenberg2, ignore = Edition.for_foreign_id(self._db, gutenberg,
Identifier.GUTENBERG_ID,
"200")
gutenberg2.title = "Unrelated Gutenberg record."
levels = [record.equivalent_identifiers(i) for i in range(0, 5)]
# At level 0, the only identifier found is the Gutenberg ID.
eq_(set([gutenberg_id]), set(levels[0]))
# At level 1, we pick up the title/author lookup.
eq_(set([gutenberg_id, search_id]), set(levels[1]))
# At level 2, we pick up the title/author lookup and the two
# OCLC Numbers.
eq_(set([gutenberg_id, search_id, oclc_id, oclc_id_2]), set(levels[2]))
# At level 3, we also pick up the ISBN.
eq_(set([gutenberg_id, search_id, oclc_id, oclc_id_2, isbn_id]),
set(levels[3]))
# At level 4, the recursion starts to go in the other
# direction: we pick up the Overdrive ID that's equivalent to
# the same ISBN as the OCLC Number.
eq_(
set([
gutenberg_id, search_id, oclc_id, oclc_id_2, isbn_id,
overdrive_id
]), set(levels[4]))