def test_circulation_events(self):
[lp] = self.english_1.license_pools
patron_id = "patronid"
types = [
CirculationEvent.CHECKIN,
CirculationEvent.CHECKOUT,
CirculationEvent.HOLD_PLACE,
CirculationEvent.HOLD_RELEASE,
CirculationEvent.TITLE_ADD
]
time = datetime.now() - timedelta(minutes=len(types))
for type in types:
get_one_or_create(
self._db, CirculationEvent,
license_pool=lp, type=type, start=time, end=time,
foreign_patron_id=patron_id)
time += timedelta(minutes=1)
with self.app.test_request_context("/"):
response = self.manager.admin_feed_controller.circulation_events()
url = AdminAnnotator(self.manager.circulation).permalink_for(self.english_1, lp, lp.identifier)
events = response['circulation_events']
eq_(types[::-1], [event['type'] for event in events])
eq_([self.english_1.title]*len(types), [event['book']['title'] for event in events])
eq_([url]*len(types), [event['book']['url'] for event in events])
eq_([patron_id]*len(types), [event['patron_id'] for event in events])
# request fewer events
with self.app.test_request_context("/?num=2"):
response = self.manager.admin_feed_controller.circulation_events()
url = AdminAnnotator(self.manager.circulation).permalink_for(self.english_1, lp, lp.identifier)
eq_(2, len(response['circulation_events']))
def test_disable(self):
# This reaper can be disabled with a configuration setting
enabled = ConfigurationSetting.sitewide(
self._db, Configuration.MEASUREMENT_REAPER)
enabled.value = False
measurement1, created = get_one_or_create(
self._db,
Measurement,
quantity_measured="answer",
value=12,
is_most_recent=True,
)
measurement2, created = get_one_or_create(
self._db,
Measurement,
quantity_measured="answer",
value=42,
is_most_recent=False,
)
reaper = MeasurementReaper(self._db)
reaper.run()
assert [measurement1,
measurement2] == self._db.query(Measurement).all()
enabled.value = True
reaper.run()
assert [measurement1] == self._db.query(Measurement).all()
def handle_event(self, threem_id, isbn, foreign_patron_id, start_time,
end_time, internal_event_type):
# Find or lookup the LicensePool for this event.
license_pool, is_new = LicensePool.for_foreign_id(
self._db, self.api.source, Identifier.THREEM_ID, threem_id)
if is_new:
# Immediately acquire bibliographic coverage for this book.
# This will set the DistributionMechanisms and make the
# book presentation-ready. However, its circulation information
# might not be up to date until we process some more events.
record = self.bibliographic_coverage_provider.ensure_coverage(
license_pool.identifier, force=True)
threem_identifier = license_pool.identifier
isbn, ignore = Identifier.for_foreign_id(self._db, Identifier.ISBN,
isbn)
edition, ignore = Edition.for_foreign_id(self._db, self.api.source,
Identifier.THREEM_ID,
threem_id)
# The ISBN and the 3M identifier are exactly equivalent.
threem_identifier.equivalent_to(self.api.source, isbn, strength=1)
# Log the event.
event, was_new = get_one_or_create(self._db,
CirculationEvent,
license_pool=license_pool,
type=internal_event_type,
start=start_time,
foreign_patron_id=foreign_patron_id,
create_method_kwargs=dict(
delta=1, end=end_time))
# If this is our first time seeing this LicensePool, log its
# occurance as a separate event
if is_new:
event = get_one_or_create(
self._db,
CirculationEvent,
type=CirculationEvent.DISTRIBUTOR_TITLE_ADD,
license_pool=license_pool,
create_method_kwargs=dict(
start=license_pool.last_checked or start_time,
delta=1,
end=license_pool.last_checked or end_time,
))
title = edition.title or "[no title]"
self.log.info("%r %s: %s", start_time, title, internal_event_type)
return start_time
def handle_event(self, threem_id, isbn, foreign_patron_id,
start_time, end_time, internal_event_type):
# Find or lookup the LicensePool for this event.
license_pool, is_new = LicensePool.for_foreign_id(
self._db, self.api.source, Identifier.THREEM_ID, threem_id)
if is_new:
# Immediately acquire bibliographic coverage for this book.
# This will set the DistributionMechanisms and make the
# book presentation-ready. However, its circulation information
# might not be up to date until we process some more events.
record = self.bibliographic_coverage_provider.ensure_coverage(
license_pool.identifier, force=True
)
threem_identifier = license_pool.identifier
isbn, ignore = Identifier.for_foreign_id(
self._db, Identifier.ISBN, isbn)
edition, ignore = Edition.for_foreign_id(
self._db, self.api.source, Identifier.THREEM_ID, threem_id)
# The ISBN and the 3M identifier are exactly equivalent.
threem_identifier.equivalent_to(self.api.source, isbn, strength=1)
# Log the event.
event, was_new = get_one_or_create(
self._db, CirculationEvent, license_pool=license_pool,
type=internal_event_type, start=start_time,
foreign_patron_id=foreign_patron_id,
create_method_kwargs=dict(delta=1,end=end_time)
)
# If this is our first time seeing this LicensePool, log its
# occurance as a separate event
if is_new:
event = get_one_or_create(
self._db, CirculationEvent,
type=CirculationEvent.TITLE_ADD,
license_pool=license_pool,
create_method_kwargs=dict(
start=license_pool.last_checked or start_time,
delta=1,
end=license_pool.last_checked or end_time,
)
)
title = edition.title or "[no title]"
self.log.info("%r %s: %s", start_time, title, internal_event_type)
return start_time
def process_post(self):
email = flask.request.form.get("email")
error = self.validate_form_fields(email)
if error:
return error
# If there are no admins yet, anyone can create the first system admin.
settingUp = (self._db.query(Admin).count() == 0)
if settingUp and not flask.request.form.get("password"):
return INCOMPLETE_CONFIGURATION.detailed(
_("The password field cannot be blank."))
admin, is_new = get_one_or_create(self._db, Admin, email=email)
self.check_permissions(admin, settingUp)
roles = flask.request.form.get("roles")
if roles:
roles = json.loads(roles)
else:
roles = []
roles_error = self.handle_roles(admin, roles, settingUp)
if roles_error:
return roles_error
password = flask.request.form.get("password")
self.handle_password(password, admin, is_new, settingUp)
return self.response(admin, is_new)
def process_post(self):
protocol = flask.request.form.get("protocol")
id = flask.request.form.get("id")
auth_service = ExternalIntegration.admin_authentication(self._db)
fields = {"protocol": protocol, "id": id, "auth_service": auth_service}
error = self.validate_form_fields(**fields)
if error:
return error
is_new = False
if not auth_service:
if protocol:
auth_service, is_new = get_one_or_create(
self._db,
ExternalIntegration,
protocol=protocol,
goal=ExternalIntegration.ADMIN_AUTH_GOAL,
)
else:
return NO_PROTOCOL_FOR_NEW_SERVICE
name = flask.request.form.get("name")
auth_service.name = name
[protocol] = [p for p in self.protocols if p.get("name") == protocol]
result = self._set_integration_settings_and_libraries(auth_service, protocol)
if isinstance(result, ProblemDetail):
return result
if is_new:
return Response(str(auth_service.protocol), 201)
else:
return Response(str(auth_service.protocol), 200)
def test_collect_event_without_work(self):
ga = MockGoogleAnalyticsProvider("faketrackingid")
identifier = self._identifier()
source = DataSource.lookup(self._db, DataSource.GUTENBERG)
pool, is_new = get_one_or_create(
self._db, LicensePool,
identifier=identifier, data_source=source)
now = datetime.datetime.utcnow()
ga.collect_event(self._db, pool, CirculationEvent.DISTRIBUTOR_CHECKIN, now)
params = urlparse.parse_qs(ga.params)
eq_(1, ga.count)
eq_("http://www.google-analytics.com/collect", ga.url)
eq_("faketrackingid", params['tid'][0])
eq_("event", params['t'][0])
eq_("circulation", params['ec'][0])
eq_(CirculationEvent.DISTRIBUTOR_CHECKIN, params['ea'][0])
eq_(str(now), params['cd1'][0])
eq_(pool.identifier.identifier, params['cd2'][0])
eq_(pool.identifier.type, params['cd3'][0])
eq_(None, params.get('cd4'))
eq_(None, params.get('cd5'))
eq_(None, params.get('cd6'))
eq_(None, params.get('cd7'))
eq_(None, params.get('cd8'))
eq_(None, params.get('cd9'))
eq_(None, params.get('cd10'))
eq_(None, params.get('cd11'))
eq_(None, params.get('cd12'))
def process_post(self):
email = flask.request.form.get("email")
error = self.validate_form_fields(email)
if error:
return error
# If there are no admins yet, anyone can create the first system admin.
settingUp = (self._db.query(Admin).count() == 0)
if settingUp and not flask.request.form.get("password"):
return INCOMPLETE_CONFIGURATION.detailed(_("The password field cannot be blank."))
admin, is_new = get_one_or_create(self._db, Admin, email=email)
self.check_permissions(admin, settingUp)
roles = flask.request.form.get("roles")
if roles:
roles = json.loads(roles)
else:
roles = []
roles_error = self.handle_roles(admin, roles, settingUp)
if roles_error:
return roles_error
password = flask.request.form.get("password")
self.handle_password(password, admin, is_new, settingUp)
return self.response(admin, is_new)
def make_patron_auth_integration(_db, provider):
integration, ignore = get_one_or_create(
_db,
ExternalIntegration,
protocol=provider.get('module'),
goal=ExternalIntegration.PATRON_AUTH_GOAL)
# If any of the common Basic Auth-type settings were provided, set them
# as ConfigurationSettings on the ExternalIntegration.
test_identifier = provider.get('test_username')
test_password = provider.get('test_password')
if test_identifier:
integration.setting(BasicAuthenticationProvider.TEST_IDENTIFIER
).value = test_identifier
if test_password:
integration.setting(
BasicAuthenticationProvider.TEST_PASSWORD).value = test_password
identifier_re = provider.get('identifier_regular_expression')
password_re = provider.get('password_regular_expression')
if identifier_re:
integration.setting(
BasicAuthenticationProvider.IDENTIFIER_REGULAR_EXPRESSION
).value = identifier_re
if password_re:
integration.setting(BasicAuthenticationProvider.
PASSWORD_REGULAR_EXPRESSION).value = password_re
return integration
def authenticated_patron(self, _db, header):
identifier = header.get('username')
password = header.get('password')
# If they fail basic validation, there is no authenticated patron.
if not self.server_side_validation(identifier, password):
return None
# All FirstBook credentials are in upper-case.
identifier = identifier.upper()
# If they fail a PIN test, there is no authenticated patron.
if not self.pintest(identifier, password):
return None
# First Book thinks this is a valid patron. Find or create a
# corresponding Patron in our database.
kwargs = {Patron.authorization_identifier.name: identifier}
__transaction = _db.begin_nested()
patron, is_new = get_one_or_create(
_db, Patron, external_identifier=identifier,
authorization_identifier=identifier,
)
__transaction.commit()
return patron
def from_dict(self, data):
_db = self._db
# Identify the source of the event.
source_name = data["source"]
source = DataSource.lookup(_db, source_name)
# Identify which LicensePool the event is talking about.
foreign_id = data["id"]
identifier_type = source.primary_identifier_type
collection = data["collection"]
license_pool, was_new = LicensePool.for_foreign_id(
_db, source, identifier_type, foreign_id, collection=collection)
# Finally, gather some information about the event itself.
type = data.get("type")
start = self._get_datetime(data, "start")
end = self._get_datetime(data, "end")
old_value = self._get_int(data, "old_value")
new_value = self._get_int(data, "new_value")
delta = self._get_int(data, "delta")
event, was_new = get_one_or_create(
_db,
CirculationEvent,
license_pool=license_pool,
type=type,
start=start,
create_method_kwargs=dict(old_value=old_value,
new_value=new_value,
delta=delta,
end=end),
)
return event, was_new
def place_hold(self, patron, pin, licensepool, notification_email_address):
"""Create a new hold."""
_db = Session.object_session(patron)
# Make sure pool info is updated.
self.update_hold_queue(licensepool)
if licensepool.licenses_available > 0:
raise CurrentlyAvailable()
# Create local hold.
hold, is_new = get_one_or_create(
_db,
Hold,
license_pool=licensepool,
patron=patron,
create_method_kwargs=dict(start=datetime.datetime.utcnow()),
)
if not is_new:
raise AlreadyOnHold()
licensepool.patrons_in_hold_queue += 1
self._update_hold_end_date(hold)
return HoldInfo(
licensepool.collection,
licensepool.data_source.name,
licensepool.identifier.type,
licensepool.identifier.identifier,
start_date=hold.start,
end_date=hold.end,
hold_position=hold.position,
)
def test_create_external_integration(self):
# A newly created Collection has no associated ExternalIntegration.
collection, ignore = get_one_or_create(self._db, Collection, name=self._str)
assert None == collection.external_integration_id
with pytest.raises(ValueError) as excinfo:
getattr(collection, "external_integration")
assert "No known external integration for collection" in str(excinfo.value)
# We can create one with create_external_integration().
overdrive = ExternalIntegration.OVERDRIVE
integration = collection.create_external_integration(protocol=overdrive)
assert integration.id == collection.external_integration_id
assert overdrive == integration.protocol
# If we call create_external_integration() again we get the same
# ExternalIntegration as before.
integration2 = collection.create_external_integration(protocol=overdrive)
assert integration == integration2
# If we try to initialize an ExternalIntegration with a different
# protocol, we get an error.
with pytest.raises(ValueError) as excinfo:
collection.create_external_integration(protocol="blah")
assert (
"Located ExternalIntegration, but its protocol (Overdrive) does not match desired protocol (blah)."
in str(excinfo.value)
)
def _set_external_integration_link(self, service):
"""Either set or delete the external integration link between the
service and the storage integration.
"""
mirror_integration_id = flask.request.form.get('mirror_integration_id')
# If no storage integration was selected, then delete the existing
# external integration link.
current_integration_link, ignore = get_one_or_create(
self._db,
ExternalIntegrationLink,
library_id=None,
external_integration_id=service.id,
purpose="MARC")
if mirror_integration_id == self.NO_MIRROR_INTEGRATION:
if current_integration_link:
self._db.delete(current_integration_link)
else:
storage_integration = get_one(self._db,
ExternalIntegration,
id=mirror_integration_id)
# Only get storage integrations that have a MARC file option set
if not storage_integration or not storage_integration.setting(
S3Uploader.MARC_BUCKET_KEY).value:
return MISSING_INTEGRATION
current_integration_link.other_integration_id = storage_integration.id
def test_query(self):
# This reaper is looking for measurements that are not current.
measurement, created = get_one_or_create(self._db,
Measurement,
is_most_recent=True)
reaper = MeasurementReaper(self._db)
assert [] == reaper.query().all()
measurement.is_most_recent = False
assert [measurement] == reaper.query().all()
def set_up_default_registry(self):
"""Set up the default library registry; no other registries exist yet."""
service, is_new = get_one_or_create(self._db,
ExternalIntegration,
protocol=self.opds_registration,
goal=self.goal)
if is_new:
service.url = (RemoteRegistry.DEFAULT_LIBRARY_REGISTRY_URL)
def test_neglected_source_cannot_be_normalized(self):
obj, new = get_one_or_create(self._db,
DataSource,
name="Neglected source")
neglected_source = obj
p = self._popularity(100, neglected_source)
assert None == p.normalized_value
r = self._rating(100, neglected_source)
assert None == r.normalized_value
def authenticated_admin(self, admin_details):
"""Creates or updates an admin with the given details"""
admin, ignore = get_one_or_create(
self._db, Admin, email=admin_details['email']
)
admin.update_credentials(
self._db, admin_details['access_token'], admin_details['credentials']
)
return admin
def process_post(self, protocols, goal, multiple_sitewide_services_detail):
name = flask.request.form.get("name")
protocol = flask.request.form.get("protocol")
fields = {"name": name, "protocol": protocol}
form_field_error = self.validate_form_fields(protocols, **fields)
if form_field_error:
return form_field_error
settings = protocols[0].get("settings")
wrong_format = self.validate_formats(settings)
if wrong_format:
return wrong_format
is_new = False
id = flask.request.form.get("id")
if id:
# Find an existing service in order to edit it
service = self.look_up_service_by_id(id, protocol, goal)
else:
if protocol:
service, is_new = get_one_or_create(
self._db, ExternalIntegration, protocol=protocol,
goal=goal
)
# There can only be one of each sitewide service.
if not is_new:
self._db.rollback()
return MULTIPLE_SITEWIDE_SERVICES.detailed(
multiple_sitewide_services_detail
)
else:
return NO_PROTOCOL_FOR_NEW_SERVICE
if isinstance(service, ProblemDetail):
self._db.rollback()
return service
name_error = self.check_name_unique(service, name)
if name_error:
self._db.rollback()
return name_error
protocol_error = self.set_protocols(service, protocol, protocols)
if protocol_error:
self._db.rollback()
return protocol_error
service.name = name
if is_new:
return Response(unicode(service.id), 201)
else:
return Response(unicode(service.id), 200)
def process_post(self, protocols, goal, multiple_sitewide_services_detail):
name = flask.request.form.get("name")
protocol = flask.request.form.get("protocol")
fields = {"name": name, "protocol": protocol}
form_field_error = self.validate_form_fields(protocols, **fields)
if form_field_error:
return form_field_error
settings = protocols[0].get("settings")
wrong_format = self.validate_formats(settings)
if wrong_format:
return wrong_format
is_new = False
id = flask.request.form.get("id")
if id:
# Find an existing service in order to edit it
service = self.look_up_service_by_id(id, protocol, goal)
else:
if protocol:
service, is_new = get_one_or_create(
self._db, ExternalIntegration, protocol=protocol,
goal=goal
)
# There can only be one of each sitewide service.
if not is_new:
self._db.rollback()
return MULTIPLE_SITEWIDE_SERVICES.detailed(
multiple_sitewide_services_detail
)
else:
return NO_PROTOCOL_FOR_NEW_SERVICE
if isinstance(service, ProblemDetail):
self._db.rollback()
return service
name_error = self.check_name_unique(service, name)
if name_error:
self._db.rollback()
return name_error
protocol_error = self.set_protocols(service, protocol, protocols)
if protocol_error:
self._db.rollback()
return protocol_error
service.name = name
if is_new:
return Response(unicode(service.id), 201)
else:
return Response(unicode(service.id), 200)
def set_up_default_registry(self):
"""Set up the default library registry; no other registries exist yet."""
service, is_new = get_one_or_create(
self._db, ExternalIntegration, protocol=self.opds_registration,
goal=self.goal
)
if is_new:
service.url = (
RemoteRegistry.DEFAULT_LIBRARY_REGISTRY_URL
)
def test_run_once(self):
# End-to-end test
measurement1, created = get_one_or_create(
self._db,
Measurement,
quantity_measured="answer",
value=12,
is_most_recent=True,
)
measurement2, created = get_one_or_create(
self._db,
Measurement,
quantity_measured="answer",
value=42,
is_most_recent=False,
)
reaper = MeasurementReaper(self._db)
result = reaper.run_once()
assert [measurement1] == self._db.query(Measurement).all()
assert "Items deleted: 1" == result.achievements
def convert_content_server(_db, library):
config = Configuration.integration("Content Server")
if not config:
print u"No content server configuration, not creating a Collection for it."
return
url = config.get('url')
collection, ignore = get_one_or_create(_db,
Collection,
protocol=Collection.OPDS_IMPORT,
name="Open Access Content Server")
library.collections.append(collection)
collection.url = url
def test_collect_event_without_work(self):
integration, ignore = create(
self._db,
ExternalIntegration,
goal=ExternalIntegration.ANALYTICS_GOAL,
protocol="api.google_analytics_provider",
)
integration.url = self._str
ConfigurationSetting.for_library_and_externalintegration(
self._db,
GoogleAnalyticsProvider.TRACKING_ID,
self._default_library,
integration,
).value = "faketrackingid"
ga = MockGoogleAnalyticsProvider(integration, self._default_library)
identifier = self._identifier()
source = DataSource.lookup(self._db, DataSource.GUTENBERG)
pool, is_new = get_one_or_create(
self._db,
LicensePool,
identifier=identifier,
data_source=source,
collection=self._default_collection,
)
now = utc_now()
ga.collect_event(self._default_library, pool,
CirculationEvent.DISTRIBUTOR_CHECKIN, now)
params = urllib.parse.parse_qs(ga.params)
assert 1 == ga.count
assert integration.url == ga.url
assert "faketrackingid" == params["tid"][0]
assert "event" == params["t"][0]
assert "circulation" == params["ec"][0]
assert CirculationEvent.DISTRIBUTOR_CHECKIN == params["ea"][0]
assert str(now) == params["cd1"][0]
assert pool.identifier.identifier == params["cd2"][0]
assert pool.identifier.type == params["cd3"][0]
assert None == params.get("cd4")
assert None == params.get("cd5")
assert None == params.get("cd6")
assert None == params.get("cd7")
assert None == params.get("cd8")
assert None == params.get("cd9")
assert None == params.get("cd10")
assert None == params.get("cd11")
assert None == params.get("cd12")
assert [source.name] == params.get("cd13")
assert None == params.get("cd14")
assert [self._default_library.short_name] == params.get("cd15")
def convert_content_server(_db, library):
config = Configuration.integration("Content Server")
if not config:
print u"No content server configuration, not creating a Collection for it."
return
url = config.get('url')
collection, ignore = get_one_or_create(_db,
Collection,
protocol=Collection.OPDS_IMPORT,
name="Open Access Content Server")
collection.external_integration.setting(
"data_source").value = DataSource.OA_CONTENT_SERVER
library.collections.append(collection)
def to_customlist(self, _db):
"""Turn this NYTBestSeller list into a CustomList object."""
data_source = DataSource.lookup(_db, DataSource.NYT)
l, was_new = get_one_or_create(
_db,
CustomList,
data_source=data_source,
foreign_identifier=self.foreign_identifier,
create_method_kwargs=dict(created=self.created, ))
l.name = self.name
l.updated = self.updated
self.update_custom_list(l)
return l
def convert_content_server(_db, library):
config = Configuration.integration("Content Server")
if not config:
print u"No content server configuration, not creating a Collection for it."
return
url = config.get('url')
collection, ignore = get_one_or_create(
_db, Collection,
protocol=Collection.OPDS_IMPORT,
name="Open Access Content Server"
)
collection.external_integration.setting("data_source").value = DataSource.OA_CONTENT_SERVER
library.collections.append(collection)
def test_custom_lists(self):
# A Collection can be associated with one or more CustomLists.
list1, ignore = get_one_or_create(self._db, CustomList, name=self._str)
list2, ignore = get_one_or_create(self._db, CustomList, name=self._str)
self.collection.customlists = [list1, list2]
assert 0 == len(list1.entries)
assert 0 == len(list2.entries)
# When a new pool is added to the collection and its presentation edition is
# calculated for the first time, it's automatically added to the lists.
work = self._work(collection=self.collection, with_license_pool=True)
assert 1 == len(list1.entries)
assert 1 == len(list2.entries)
assert work == list1.entries[0].work
assert work == list2.entries[0].work
# Now remove it from one of the lists. If its presentation edition changes
# again or its pool changes works, it's not added back.
self._db.delete(list1.entries[0])
self._db.commit()
assert 0 == len(list1.entries)
assert 1 == len(list2.entries)
pool = work.license_pools[0]
identifier = pool.identifier
staff_data_source = DataSource.lookup(self._db, DataSource.LIBRARY_STAFF)
staff_edition, ignore = Edition.for_foreign_id(
self._db, staff_data_source, identifier.type, identifier.identifier
)
staff_edition.title = self._str
work.calculate_presentation()
assert 0 == len(list1.entries)
assert 1 == len(list2.entries)
new_work = self._work(collection=self.collection)
pool.work = new_work
assert 0 == len(list1.entries)
assert 1 == len(list2.entries)
def oauth_callback(self, _db, params):
code = params.get('code')
payload = dict(
code=code,
grant_type='authorization_code',
redirect_uri=url_for('oauth_callback', _external=True),
)
headers = {
'Authorization': 'Basic %s' % base64.b64encode(self.client_id + ":" + self.client_secret),
'Content-Type': 'application/json',
}
response = requests.post(self.CLEVER_TOKEN_URL, data=json.dumps(payload), headers=headers).json()
token = response['access_token']
bearer_headers = {
'Authorization': 'Bearer %s' % token
}
result = requests.get(self.CLEVER_API_BASE_URL + '/me', headers=bearer_headers).json()
data = result['data']
# TODO: which teachers and admins should have access?
if result['type'] != 'student':
return INVALID_CREDENTIALS
identifier = data['id']
student = requests.get(self.CLEVER_API_BASE_URL + '/v1.1/students/%s' % identifier, headers=bearer_headers).json()
# TODO: check student free and reduced lunch status and/or school's NCES ID
student_data = student['data']
school_id = student_data['school']
school = requests.get(self.CLEVER_API_BASE_URL + '/v1.1/schools/%s' % school_id, headers=bearer_headers).json()
grade = student_data.get('grade')
external_type = None
if grade in ["Kindergarten", "1", "2", "3"]:
external_type = "E"
elif grade in ["4", "5", "6", "7", "8"]:
external_type = "M"
elif grade in ["9", "10", "11", "12"]:
external_type = "H"
patron, is_new = get_one_or_create(
_db, Patron, external_identifier=identifier,
authorization_identifier=identifier,
)
patron._external_type = external_type
return token
def run(self, subset=None):
added_books = 0
for edition, license_pool in self.source.create_missing_books(subset):
# Log a circulation event for this title.
event = get_one_or_create(
self._db, CirculationEvent,
type=CirculationEvent.TITLE_ADD,
license_pool=license_pool,
create_method_kwargs=dict(
start=license_pool.last_checked
)
)
self._db.commit()
def mock_collection(self, _db):
"""Create a mock OPDS For Distributors collection to use in tests."""
library = DatabaseTest.make_default_library(_db)
collection, ignore = get_one_or_create(
_db,
Collection,
name="Test OPDS For Distributors Collection",
create_method_kwargs=dict(external_account_id=u"http://opds", ))
integration = collection.create_external_integration(
protocol=OPDSForDistributorsAPI.NAME)
integration.username = u'a'
integration.password = u'b'
library.collections.append(collection)
return collection
def process_post(self):
self.require_system_admin()
protocols = self._get_collection_protocols()
is_new = False
collection = None
name = flask.request.form.get("name")
protocol_name = flask.request.form.get("protocol")
fields = {"name": name, "protocol": protocol_name}
id = flask.request.form.get("id")
if id:
collection = get_one(self._db, Collection, id=id)
fields["collection"] = collection
error = self.validate_form_fields(is_new, protocols, **fields)
if error:
return error
if protocol_name and not collection:
collection, is_new = get_one_or_create(self._db,
Collection,
name=name)
if not is_new:
self._db.rollback()
return COLLECTION_NAME_ALREADY_IN_USE
collection.create_external_integration(protocol_name)
collection.name = name
[protocol_dict
] = [p for p in protocols if p.get("name") == protocol_name]
settings = self.validate_parent(protocol_dict, collection)
if isinstance(settings, ProblemDetail):
self._db.rollback()
return settings
settings_error = self.process_settings(settings, collection)
if settings_error:
self._db.rollback()
return settings_error
libraries_error = self.process_libraries(protocol_dict, collection)
if libraries_error:
return libraries_error
if is_new:
return Response(unicode(collection.id), 201)
else:
return Response(unicode(collection.id), 200)
def _get_token_data_source(self, db):
"""Returns a token data source
:param db: Database session
:type db: sqlalchemy.orm.session.Session
:return: Token data source
:rtype: DataSource
"""
# FIXME: This code will probably not work in a situation where a library has multiple SAML
# authentication mechanisms for its patrons.
# It'll look up a Credential from this data source but it won't be able to tell which IdP it came from.
return get_one_or_create(db,
DataSource,
name=self.TOKEN_DATA_SOURCE_NAME)
def to_customlist(self, _db):
"""Turn this NYTBestSeller list into a CustomList object."""
data_source = DataSource.lookup(_db, DataSource.NYT)
l, was_new = get_one_or_create(
_db,
CustomList,
data_source=data_source,
foreign_identifier=self.foreign_identifier,
create_method_kwargs = dict(
created=self.created,
)
)
l.name = self.name
l.updated = self.updated
self.update_custom_list(l)
return l
def mock_collection(self, _db):
"""Create a mock OPDS For Distributors collection to use in tests."""
library = DatabaseTest.make_default_library(_db)
collection, ignore = get_one_or_create(
_db, Collection,
name="Test OPDS For Distributors Collection", create_method_kwargs=dict(
external_account_id=u"http://opds",
)
)
integration = collection.create_external_integration(
protocol=OPDSForDistributorsAPI.NAME
)
integration.username = u'a'
integration.password = u'b'
library.collections.append(collection)
return collection
def mock_collection(cls, _db):
library = DatabaseTest.make_default_library(_db)
collection, ignore = get_one_or_create(
_db,
Collection,
name="Test Odilo Collection",
create_method_kwargs=dict(external_account_id=u'library_id_123', ))
integration = collection.create_external_integration(
protocol=ExternalIntegration.ODILO)
integration.username = u'username'
integration.password = u'password'
integration.setting(OdiloAPI.LIBRARY_API_BASE_URL
).value = u'http://library_api_base_url/api/v2'
library.collections.append(collection)
return collection
def test_sort_name(self):
bob, new = get_one_or_create(self._db, Contributor, sort_name=None)
assert None == bob.sort_name
bob, ignore = self._contributor(sort_name="Bob Bitshifter")
bob.sort_name = None
assert None == bob.sort_name
bob, ignore = self._contributor(sort_name="Bob Bitshifter")
assert "Bitshifter, Bob" == bob.sort_name
bob, ignore = self._contributor(sort_name="Bitshifter, Bob")
assert "Bitshifter, Bob" == bob.sort_name
# test that human name parser doesn't die badly on foreign names
bob, ignore = self._contributor(sort_name="Боб Битшифтер")
assert "Битшифтер, Боб" == bob.sort_name
def convert_bibliotheca(_db, library):
config = Configuration.integration('3M')
if not config:
print u"No Bibliotheca configuration, not creating a Collection for it."
return
print u"Creating Collection object for Bibliotheca collection."
username = config.get('account_id')
password = config.get('account_key')
library_id = config.get('library_id')
collection, ignore = get_one_or_create(_db,
Collection,
protocol=Collection.BIBLIOTHECA,
name="Bibliotheca")
library.collections.append(collection)
collection.external_integration.username = username
collection.external_integration.password = password
collection.external_account_id = library_id
def test_collect_event_without_work(self):
integration, ignore = create(
self._db, ExternalIntegration,
goal=ExternalIntegration.ANALYTICS_GOAL,
protocol="api.google_analytics_provider",
)
integration.url = self._str
ConfigurationSetting.for_library_and_externalintegration(
self._db, GoogleAnalyticsProvider.TRACKING_ID, self._default_library, integration
).value = "faketrackingid"
ga = MockGoogleAnalyticsProvider(integration, self._default_library)
identifier = self._identifier()
source = DataSource.lookup(self._db, DataSource.GUTENBERG)
pool, is_new = get_one_or_create(
self._db, LicensePool,
identifier=identifier, data_source=source,
collection=self._default_collection
)
now = datetime.datetime.utcnow()
ga.collect_event(self._default_library, pool, CirculationEvent.DISTRIBUTOR_CHECKIN, now)
params = urlparse.parse_qs(ga.params)
eq_(1, ga.count)
eq_(integration.url, ga.url)
eq_("faketrackingid", params['tid'][0])
eq_("event", params['t'][0])
eq_("circulation", params['ec'][0])
eq_(CirculationEvent.DISTRIBUTOR_CHECKIN, params['ea'][0])
eq_(str(now), params['cd1'][0])
eq_(pool.identifier.identifier, params['cd2'][0])
eq_(pool.identifier.type, params['cd3'][0])
eq_(None, params.get('cd4'))
eq_(None, params.get('cd5'))
eq_(None, params.get('cd6'))
eq_(None, params.get('cd7'))
eq_(None, params.get('cd8'))
eq_(None, params.get('cd9'))
eq_(None, params.get('cd10'))
eq_(None, params.get('cd11'))
eq_(None, params.get('cd12'))
eq_([source.name], params.get('cd13'))
eq_(None, params.get('cd14'))
eq_([self._default_library.short_name], params.get('cd15'))
def convert_bibliotheca(_db, library):
config = Configuration.integration('3M')
if not config:
print u"No Bibliotheca configuration, not creating a Collection for it."
return
print u"Creating Collection object for Bibliotheca collection."
username = config.get('account_id')
password = config.get('account_key')
library_id = config.get('library_id')
collection, ignore = get_one_or_create(
_db, Collection,
protocol=Collection.BIBLIOTHECA,
name="Bibliotheca"
)
library.collections.append(collection)
collection.external_integration.username = username
collection.external_integration.password = password
collection.external_account_id = library_id
def _set_external_integration_link(
self,
_db,
key,
value,
collection,
):
"""Find or create a ExternalIntegrationLink and either delete it
or update the other external integration it links to.
"""
collection_service = get_one(_db,
ExternalIntegration,
id=collection.external_integration_id)
storage_service = None
other_integration_id = None
purpose = key.rsplit("_", 2)[0]
external_integration_link, ignore = get_one_or_create(
_db,
ExternalIntegrationLink,
library_id=None,
external_integration_id=collection_service.id,
purpose=purpose,
)
if not external_integration_link:
return MISSING_INTEGRATION
if value == self.NO_MIRROR_INTEGRATION:
_db.delete(external_integration_link)
else:
storage_service = get_one(_db, ExternalIntegration, id=value)
if storage_service:
if storage_service.goal != ExternalIntegration.STORAGE_GOAL:
return INTEGRATION_GOAL_CONFLICT
other_integration_id = storage_service.id
else:
return MISSING_SERVICE
external_integration_link.other_integration_id = other_integration_id
return external_integration_link
def convert_overdrive(_db, library):
config = Configuration.integration('Overdrive')
if not config:
print u"No Overdrive configuration, not creating a Collection for it."
print u"Creating Collection object for Overdrive collection."
username = config.get('client_key')
password = config.get('client_secret')
library_id = config.get('library_id')
website_id = config.get('website_id')
collection, ignore = get_one_or_create(_db,
Collection,
protocol=Collection.OVERDRIVE,
name="Overdrive")
library.collections.append(collection)
collection.username = username
collection.password = password
collection.external_account_id = library_id
collection.set_setting("website_id", website_id)
def convert_overdrive(_db, library):
config = Configuration.integration('Overdrive')
if not config:
print u"No Overdrive configuration, not creating a Collection for it."
return
print u"Creating Collection object for Overdrive collection."
username = config.get('client_key')
password = config.get('client_secret')
library_id = config.get('library_id')
website_id = config.get('website_id')
collection, ignore = get_one_or_create(
_db, Collection,
protocol=Collection.OVERDRIVE,
name="Overdrive"
)
library.collections.append(collection)
collection.external_integration.username = username
collection.external_integration.password = password
collection.external_account_id = library_id
collection.external_integration.set_setting("website_id", website_id)
def convert_axis(_db, library):
config = Configuration.integration('Axis 360')
if not config:
print u"No Axis 360 configuration, not creating a Collection for it."
return
print u"Creating Collection object for Axis 360 collection."
username = config.get('username')
password = config.get('password')
library_id = config.get('library_id')
# This is not technically a URL, it's u"production" or u"staging",
# but it's converted into a URL internally.
url = config.get('server')
collection, ignore = get_one_or_create(
_db, Collection,
protocol=Collection.AXIS_360,
name="Axis 360"
)
library.collections.append(collection)
collection.external_integration.username = username
collection.external_integration.password = password
collection.external_account_id = library_id
collection.external_integration.url = url
def convert_one_click(_db, library):
config = Configuration.integration('OneClick')
if not config:
print u"No OneClick configuration, not creating a Collection for it."
return
print u"Creating Collection object for OneClick collection."
basic_token = config.get('basic_token')
library_id = config.get('library_id')
url = config.get('url')
ebook_loan_length = config.get('ebook_loan_length')
eaudio_loan_length = config.get('eaudio_loan_length')
collection, ignore = get_one_or_create(
_db, Collection,
protocol=Collection.ONECLICK,
name="OneClick"
)
library.collections.append(collection)
collection.external_integration.password = basic_token
collection.external_account_id = library_id
collection.external_integration.url = url
collection.external_integration.set_setting("ebook_loan_length", ebook_loan_length)
collection.external_integration.set_setting("eaudio_loan_length", eaudio_loan_length)
def __init__(self, _db, batch_size=10, cutoff_time=None,
uploader=None, providers=None, **kwargs):
output_source, made_new = get_one_or_create(
_db, DataSource,
name=DataSource.INTERNAL_PROCESSING
)
# Other components don't have INTERNAL_PROCESSING as offering
# licenses, but we do, because we're responsible for managing
# LicensePools.
output_source.offers_licenses=True
input_identifier_types = [Identifier.OVERDRIVE_ID, Identifier.ISBN]
super(IdentifierResolutionCoverageProvider, self).__init__(
service_name="Identifier Resolution Coverage Provider",
input_identifier_types=input_identifier_types,
output_source=output_source,
batch_size=batch_size,
operation=CoverageRecord.RESOLVE_IDENTIFIER_OPERATION,
)
# Since we are the metadata wrangler, any resources we find,
# we mirror to S3.
mirror = uploader or S3Uploader()
# We're going to be aggressive about recalculating the presentation
# for this work because either the work is currently not set up
# at all, or something went wrong trying to set it up.
presentation_calculation_policy = PresentationCalculationPolicy(
regenerate_opds_entries=True,
update_search_index=True
)
policy = ReplacementPolicy.from_metadata_source(
mirror=mirror, even_if_not_apparently_updated=True,
presentation_calculation_policy=presentation_calculation_policy
)
if providers:
# For testing purposes. Initializing the real coverage providers
# during tests can cause requests to third-parties.
(self.required_coverage_providers,
self.optional_coverage_providers) = providers
else:
overdrive = OverdriveBibliographicCoverageProvider(
_db, metadata_replacement_policy=policy
)
content_cafe = ContentCafeCoverageProvider(self._db)
content_server = ContentServerCoverageProvider(self._db)
oclc_classify = OCLCClassifyCoverageProvider(self._db)
self.required_coverage_providers = [
overdrive, content_cafe, content_server, oclc_classify
]
self.optional_coverage_providers = []
self.viaf = VIAFClient(self._db)
self.image_mirrors = {
DataSource.OVERDRIVE : OverdriveCoverImageMirror(
self._db, uploader=uploader
)
}
self.image_scaler = ImageScaler(
self._db, self.image_mirrors.values(), uploader=uploader
)
self.oclc_linked_data = LinkedDataCoverageProvider(self._db)
LicensePool,
Patron,
)
from threem import ThreeMAPI
from overdrive import OverdriveAPI
from axis import Axis360API
from circulation import CirculationAPI
from circulation_exceptions import *
barcode, pin, borrow_urn, hold_urn = sys.argv[1:5]
email = os.environ.get('DEFAULT_NOTIFICATION_EMAIL_ADDRESS', '*****@*****.**')
_db = production_session()
patron, ignore = get_one_or_create(
_db, Patron, authorization_identifier=barcode)
borrow_identifier = Identifier.parse_urn(_db, borrow_urn, True)[0]
hold_identifier = Identifier.parse_urn(_db, hold_urn, True)[0]
borrow_pool = borrow_identifier.licensed_through
hold_pool = hold_identifier.licensed_through
if any(x.type == Identifier.THREEM_ID for x in [borrow_identifier, hold_identifier]):
threem = ThreeMAPI(_db)
else:
threem = None
if any(x.type == Identifier.OVERDRIVE_ID for x in [borrow_identifier, hold_identifier]):
overdrive = OverdriveAPI(_db)
else:
overdrive = None
def authenticated_patron(self, db, identifier, password):
# If they fail basic validation, there is no authenticated patron.
if not self.server_side_validation(identifier, password):
return None
# If they fail a PIN test, it's very simple: there is
# no authenticated patron.
if not self.pintest(identifier, password):
return None
now = datetime.datetime.utcnow()
# Now it gets more complicated. There is *some* authenticated
# patron, but it might not correspond to a Patron in our
# database, and if it does, that Patron's
# authorization_identifier might be different from the
# identifier passed in to this method.
# Let's start with a simple lookup based on identifier.
kwargs = {Patron.authorization_identifier.name: identifier}
patron = get_one(db, Patron, **kwargs)
if not patron:
# The patron might have used a username instead of a barcode.
kwargs = {Patron.username.name: identifier}
patron = get_one(db, Patron, *kwargs)
__transaction = db.begin_nested()
if patron:
# We found them!
if (not patron.last_external_sync
or (now - patron.last_external_sync) > self.MAX_STALE_TIME):
# Sync our internal Patron record with what the API
# says.
self.update_patron(patron, identifier)
patron.last_external_sync = now
__transaction.commit()
return patron
# We didn't find them. Now the question is: _why_ doesn't this
# patron show up in our database? Have we never seen them
# before, has their authorization identifier (barcode)
# changed, or do they not exist in Millenium either?
dump = self.dump(identifier)
if dump.get('ERRNUM') in ('1', '2'):
# The patron does not exist in Millenium. This is a bad
# barcode. How we passed the PIN test is a mystery, but
# ours not to reason why. There is no authenticated
# patron.
# TODO: EXCEPT, this might be a test patron dynamically
# created by the test code.
if len(identifier) != 14:
print "Creating test patron!"
patron, is_new = get_one_or_create(
db, Patron, external_identifier=identifier,
)
patron.authorization_identifier = identifier
__transaction.commit()
return patron
__transaction.commit()
return None
# If we've gotten this far, the patron does exist in
# Millenium.
permanent_id = dump.get(self.RECORD_NUMBER_FIELD)
if not permanent_id:
# We have no reliable way of identifying this patron.
# This should never happen, but if it does, we can't
# create a Patron record.
__transaction.commit()
return None
# Look up the Patron record by the permanent record ID. If
# there is no such patron, we've never seen them
# before--create a new Patron record for them.
#
# If there is such a patron, their barcode has changed,
# probably because their old barcode was reported lost. We
# will update their barcode in the next step.
patron, is_new = get_one_or_create(
db, Patron, external_identifier=permanent_id)
# Update the new/out-of-date Patron record with information
# from the data dump.
self.update_patron(patron, identifier, dump)
__transaction.commit()
return patron
if not config:
print u"No content server configuration, not creating a Collection for it."
return
url = config.get('url')
collection, ignore = get_one_or_create(
_db, Collection,
protocol=Collection.OPDS_IMPORT,
name="Open Access Content Server"
)
collection.external_integration.setting("data_source").value = DataSource.OA_CONTENT_SERVER
library.collections.append(collection)
# This is the point in the migration where we first create a Library
# for this system.
library = get_one_or_create(
_db, Library,
create_method_kwargs=dict(
name="Default Library",
short_name="default",
uuid=unicode(uuid.uuid4())
)
)
copy_library_registry_information(_db, library)
convert_overdrive(_db, library)
convert_bibliotheca(_db, library)
convert_axis(_db, library)
convert_one_click(_db, library)
convert_content_server(_db, library)
_db.commit()
def process_item(self, identifier):
try:
new_info_counter = Counter()
self.log.info("Processing identifier %r", identifier)
for metadata in self.api.info_for(identifier):
other_identifier, ignore = metadata.primary_identifier.load(self._db)
oclc_editions = other_identifier.primarily_identifies
# Keep track of the number of editions OCLC associates
# with this identifier.
other_identifier.add_measurement(
self.output_source, Measurement.PUBLISHED_EDITIONS,
len(oclc_editions)
)
self.apply_viaf_to_contributor_data(metadata)
# When metadata is applied, it must be given a client that can
# response to 'canonicalize_author_name'. Usually this is an
# OPDSImporter that reaches out to the Metadata Wrangler, but
# in the case of being _on_ the Metadata Wrangler...:
from canonicalize import AuthorNameCanonicalizer
metadata_client = AuthorNameCanonicalizer(
self._db, oclcld=self.api, viaf=self.viaf
)
num_new_isbns = self.new_isbns(metadata)
new_info_counter['isbns'] += num_new_isbns
if oclc_editions:
# There are existing OCLC editions. Apply any new information to them.
for edition in oclc_editions:
metadata, new_info_counter = self.apply_metadata_to_edition(
edition, metadata, metadata_client, new_info_counter
)
elif num_new_isbns:
# Create a new OCLC edition to hold the information.
edition, ignore = get_one_or_create(
self._db, Edition, data_source=self.output_source,
primary_identifier=other_identifier
)
metadata, new_info_counter = self.apply_metadata_to_edition(
edition, metadata, metadata_client, new_info_counter
)
# Set the new OCLC edition's identifier equivalent to this
# identifier so we know they're related.
self.set_equivalence(identifier, metadata)
self.log.info(
"Total: %(editions)d editions, %(isbns)d ISBNs, "\
"%(descriptions)d descriptions, %(subjects)d classifications.",
new_info_counter
)
except IOError as e:
if ", but couldn't find location" in e.message:
exception = "OCLC doesn't know about this ISBN: %r" % e
transient = False
else:
exception = "OCLC raised an error: %r" % e
transient = True
return CoverageFailure(
identifier, exception, data_source=self.output_source,
transient=transient
)
return identifier
DataSource,
UnresolvedIdentifier,
)
from controller import URNLookupController
_db = production_session()
source = DataSource.lookup(_db, DataSource.INTERNAL_PROCESSING)
unresolved_identifiers = _db.query(UnresolvedIdentifier).all()
print "Replacing %d UnresolvedIdentifiers with CoverageRecords" % len(unresolved_identifiers)
for unresolved in unresolved_identifiers:
identifier = unresolved.identifier
record, is_new = get_one_or_create(
_db, CoverageRecord,
identifier=identifier, data_source=source,
operation=CoverageRecord.RESOLVE_IDENTIFIER_OPERATION
)
if is_new:
# This CoverageRecord wasn't created from a lookup prior to this
# migration, so it should duplicate as much of the
# UnresolvedIdentifier's data as it can.
record.timestamp = unresolved.most_recent_attempt
record.status = BaseCoverageRecord.TRANSIENT_FAILURE
record.exception = unresolved.exception
if not record.exception:
# The UnresolvedIdentifier didn't have an exception, so this
# identifier hadn't been run at all yet.
# We'll give it the default lookup message.
def process_item(self, identifier):
# Books are not looked up in OCLC Linked Data directly, since
# there is no Collection that identifies a book by its OCLC Number.
# However, when a book is looked up through OCLC Classify, some
# OCLC Numbers may be associated with it, and _those_ numbers
# can be run through OCLC Linked Data.
try:
new_info_counter = Counter()
self.log.info("Processing identifier %r", identifier)
metadatas = [m for m in self.api.info_for(identifier)]
if identifier.type==Identifier.ISBN:
# Currently info_for seeks the results of OCLC Work IDs only
# This segment will get the metadata of any equivalent OCLC Numbers
# as well.
equivalents = Identifier.recursively_equivalent_identifier_ids(
self._db, [identifier.id]
)
oclc_numbers = self._db.query(Identifier).\
filter(Identifier.id.in_(equivalents)).\
filter(Identifier.type==Identifier.OCLC_NUMBER).all()
for oclc_number in oclc_numbers:
more_metadata = [m for m in self.api.info_for(oclc_number)]
metadatas += more_metadata
metadatas = [m for m in metadatas if m]
for metadata in metadatas:
other_identifier, ignore = metadata.primary_identifier.load(self._db)
oclc_editions = other_identifier.primarily_identifies
# Keep track of the number of editions OCLC associates
# with this identifier.
other_identifier.add_measurement(
self.data_source, Measurement.PUBLISHED_EDITIONS,
len(oclc_editions)
)
# Clean up contributor information.
self.apply_viaf_to_contributor_data(metadata)
# Remove any empty ContributorData objects that may have
# been created.
metadata.contributors = filter(
lambda c: c.sort_name or c.display_name,
metadata.contributors
)
# When metadata is applied, it must be given a client that can
# response to 'canonicalize_author_name'. Usually this is an
# OPDSImporter that reaches out to the Metadata Wrangler, but
# in the case of being _on_ the Metadata Wrangler...:
from canonicalize import AuthorNameCanonicalizer
metadata_client = AuthorNameCanonicalizer(
self._db, oclcld=self.api, viaf=self.viaf
)
num_new_isbns = self.new_isbns(metadata)
new_info_counter['isbns'] += num_new_isbns
if oclc_editions:
# There are existing OCLC editions. Apply any new information to them.
for edition in oclc_editions:
metadata, new_info_counter = self.apply_metadata_to_edition(
edition, metadata, metadata_client, new_info_counter
)
else:
# Create a new OCLC edition to hold the information.
edition, ignore = get_one_or_create(
self._db, Edition, data_source=self.data_source,
primary_identifier=other_identifier
)
metadata, new_info_counter = self.apply_metadata_to_edition(
edition, metadata, metadata_client, new_info_counter
)
# Set the new OCLC edition's identifier equivalent to this
# identifier so we know they're related.
self.set_equivalence(identifier, metadata)
self.log.info(
"Total: %(editions)d editions, %(isbns)d ISBNs, "\
"%(descriptions)d descriptions, %(subjects)d classifications.",
new_info_counter
)
except IOError as e:
if ", but couldn't find location" in e.message:
exception = "OCLC doesn't know about this ISBN: %r" % e
transient = False
else:
exception = "OCLC raised an error: %r" % e
transient = True
return self.failure(identifier, exception, transient=transient)
# Try to calculate or recalculate a work for ISBNs.
#
# We won't do this for other Identifier types because we don't want
# to overwrite the high-quality metadata direct from the source.
# With ISBNs, that higher-quality metadata is not available, so we
# depend on OCLC for title and author information.
if identifier.type == Identifier.ISBN:
self.calculate_work_for_isbn(identifier)
return identifier
def extract_edition(cls, _db, work_tag, existing_authors, **restrictions):
"""Create a new Edition object with information about a
work (identified by OCLC Work ID).
"""
# TODO: 'pswid' is what it's called in older representations.
# That code can be removed once we replace all representations.
oclc_work_id = unicode(work_tag.get('owi') or work_tag.get('pswid'))
# if oclc_work_id:
# print " owi: %s" % oclc_work_id
# else:
# print " No owi in %s" % etree.tostring(work_tag)
if not oclc_work_id:
raise ValueError("Work has no owi")
item_type = work_tag.get("itemtype")
if (item_type.startswith('itemtype-book')
or item_type.startswith('itemtype-compfile')):
medium = Edition.BOOK_MEDIUM
elif item_type.startswith('itemtype-audiobook') or item_type.startswith('itemtype-music'):
# Pretty much all Gutenberg texts, even the audio texts,
# are based on a book, and the ones that aren't
# (recordings of individual songs) probably aren't in OCLC
# anyway. So we just want to get the books.
medium = Edition.AUDIO_MEDIUM
medium = None
elif item_type.startswith('itemtype-video'):
#medium = Edition.VIDEO_MEDIUM
medium = None
elif item_type in cls.UNUSED_MEDIA:
medium = None
else:
medium = None
# Only create Editions for books with a recognized medium
if medium is None:
return None, False
result = cls._extract_basic_info(_db, work_tag, existing_authors, **restrictions)
if not result:
# This record did not meet one of the restrictions.
return None, False
title, authors_and_roles, language = result
# Record some extra OCLC-specific information
editions = work_tag.get('editions')
holdings = work_tag.get('holdings')
# Get an identifier for this work.
identifier, ignore = Identifier.for_foreign_id(
_db, Identifier.OCLC_WORK, oclc_work_id
)
data_source = DataSource.lookup(_db, DataSource.OCLC)
identifier.add_measurement(data_source, Measurement.HOLDINGS, holdings)
identifier.add_measurement(
data_source, Measurement.PUBLISHED_EDITIONS, editions)
# Create a Edition for source + identifier
edition, new = get_one_or_create(
_db, Edition,
data_source=data_source,
primary_identifier=identifier,
create_method_kwargs=dict(
title=title,
language=language,
)
)
# Get the most popular Dewey and LCC classification for this
# work.
for tag_name, subject_type in (
("ddc", Subject.DDC),
("lcc", Subject.LCC)):
tag = cls._xpath1(
work_tag,
"//oclc:%s/oclc:mostPopular" % tag_name)
if tag is not None:
id = tag.get('nsfa') or tag.get('sfa')
weight = int(tag.get('holdings'))
identifier.classify(
data_source, subject_type, id, weight=weight)
# Find FAST subjects for the work.
for heading in cls._xpath(
work_tag, "//oclc:fast//oclc:heading"):
id = heading.get('ident')
weight = int(heading.get('heldby'))
value = heading.text
identifier.classify(
data_source, Subject.FAST, id, value, weight)
# Associate the authors with the Edition.
for contributor, roles in authors_and_roles:
edition.add_contributor(contributor, roles)
return edition, new
def test_libraries_post_errors(self):
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("name", "Brooklyn Public Library"),
])
response = self.manager.admin_library_settings_controller.process_post()
eq_(response, MISSING_LIBRARY_SHORT_NAME)
self.admin.remove_role(AdminRole.SYSTEM_ADMIN)
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("name", "Brooklyn Public Library"),
("short_name", "bpl"),
])
assert_raises(AdminNotAuthorized,
self.manager.admin_library_settings_controller.process_post)
library = self._library()
self.admin.add_role(AdminRole.LIBRARIAN, library)
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("uuid", library.uuid),
("name", "Brooklyn Public Library"),
("short_name", library.short_name),
])
assert_raises(AdminNotAuthorized,
self.manager.admin_library_settings_controller.process_post)
self.admin.add_role(AdminRole.SYSTEM_ADMIN)
with self.request_context_with_admin("/", method="POST"):
flask.request.form = self.library_form(library, {"uuid": "1234"})
response = self.manager.admin_library_settings_controller.process_post()
eq_(response.uri, LIBRARY_NOT_FOUND.uri)
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("name", "Brooklyn Public Library"),
("short_name", library.short_name),
])
response = self.manager.admin_library_settings_controller.process_post()
eq_(response, LIBRARY_SHORT_NAME_ALREADY_IN_USE)
bpl, ignore = get_one_or_create(
self._db, Library, short_name="bpl"
)
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("uuid", bpl.uuid),
("name", "Brooklyn Public Library"),
("short_name", library.short_name),
])
response = self.manager.admin_library_settings_controller.process_post()
eq_(response, LIBRARY_SHORT_NAME_ALREADY_IN_USE)
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("uuid", library.uuid),
("name", "The New York Public Library"),
("short_name", library.short_name),
])
response = self.manager.admin_library_settings_controller.process_post()
eq_(response.uri, INCOMPLETE_CONFIGURATION.uri)
# Test a bad contrast ratio between the web foreground and
# web background colors.
with self.request_context_with_admin("/", method="POST"):
flask.request.form = self.library_form(
library, {Configuration.WEB_BACKGROUND_COLOR: "#000000",
Configuration.WEB_FOREGROUND_COLOR: "#010101"}
)
response = self.manager.admin_library_settings_controller.process_post()
eq_(response.uri, INVALID_CONFIGURATION_OPTION.uri)
assert "contrast-ratio.com/#%23010101-on-%23000000" in response.detail
# Test a list of web header links and a list of labels that
# aren't the same length.
library = self._library()
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("uuid", library.uuid),
("name", "The New York Public Library"),
("short_name", library.short_name),
(Configuration.WEBSITE_URL, "https://library.library/"),
(Configuration.DEFAULT_NOTIFICATION_EMAIL_ADDRESS, "*****@*****.**"),
(Configuration.HELP_EMAIL, "*****@*****.**"),
(Configuration.WEB_HEADER_LINKS, "http://library.com/1"),
(Configuration.WEB_HEADER_LINKS, "http://library.com/2"),
(Configuration.WEB_HEADER_LABELS, "One"),
])
response = self.manager.admin_library_settings_controller.process_post()
eq_(response.uri, INVALID_CONFIGURATION_OPTION.uri)
