def post_process(self, metadata):
"""Post process the data.
add extra data such as title statement.
:param metadata: dictionary version of a record
:return: the modified dictionary
"""
metadata = Document.post_process(metadata)
titles = metadata.get('title', [])
text_title = title_format_text_head(titles, with_subtitle=False)
if text_title:
metadata['ui_title_text'] = text_title
responsibility = metadata.get('responsibilityStatement', [])
text_title = title_format_text_head(titles,
responsibility,
with_subtitle=False)
if text_title:
metadata['ui_title_text_responsibility'] = text_title
contributions = metadata.get('contribution', [])
new_contributions = []
for contribution in contributions:
agent = contribution['agent']
agent_type = agent['type']
agent_data = JsonRef.replace_refs(agent,
loader=None).get('metadata')
if agent_data:
agent_data.pop('$schema', None)
agent = agent_data
agent['type'] = agent_type
new_contributions.append({'agent': agent})
if new_contributions:
metadata['contribution'] = create_contributions(new_contributions)
return metadata
def test_title_format_text_head():
"""Test title format text head."""
data = [{
'mainTitle': [{
'value': 'Dingding lixianji'
}, {
'value': '\u4e01\u4e01\u5386\u9669\u8bb0',
'language': 'und-hani'
}],
'type': 'bf:Title'
}]
assert '\u4e01\u4e01\u5386\u9669\u8bb0' == title_format_text_head(data)
data = [{
'mainTitle': [{
'value': 'Die russischen orthodoxen Bischöfe von 1893',
}],
'subtitle': [{
'value': 'Bio-Bibliographie'
}],
'type': 'bf:Title'
}]
assert 'Die russischen orthodoxen Bischöfe von 1893 ' \
': Bio-Bibliographie' == title_format_text_head(data)
data = [{
'mainTitle': [{
'value': 'Die russischen orthodoxen Bischöfe von 1893',
}, {
'value': 'The Russian Orthodox Bishops of 1893',
'language': 'eng'
}],
'subtitle': [{
'value': 'Bio-Bibliographie'
}],
'type': 'bf:Title'
}]
assert 'The Russian Orthodox Bishops of 1893' == \
title_format_text_head(data)
data = [{
'mainTitle': [{
'value': 'main_title_text',
}],
'subtitle': [{
'value': 'subtitle_text'
}],
'part': [
{'partName': [{'value': 'part1'}, {'value': 'part1.1'}]},
{'partName': [{'value': 'part2'}]}
],
'type': 'bf:Title'
}]
assert 'main_title_text : subtitle_text. part1, part1.1, part2' == \
title_format_text_head(data)
def post_process_serialize_search(self, results, pid_fetcher):
"""Post process the search results.
:param results: Elasticsearch search result.
:param pid_fetcher: Persistent identifier fetcher.
"""
records = results.get('hits', {}).get('hits', {})
for record in records:
metadata = record.get('metadata', {})
document = search_document_by_pid(
metadata.get('document').get('pid'))
metadata['ui_title_text'] = title_format_text_head(
document['title'], with_subtitle=True)
# Add library name
for lib_term in results.get('aggregations',
{}).get('library', {}).get('buckets', []):
lib = Library.get_record_by_pid(lib_term.get('key'))
lib_term['name'] = lib.get('name')
# Add location name
for loc_term in results.get('aggregations',
{}).get('location', {}).get('buckets', []):
loc = Location.get_record_by_pid(loc_term.get('key'))
loc_term['name'] = loc.get('name')
# Add library name
for item_type_term in results.get('aggregations',
{}).get('item_type',
{}).get('buckets', []):
item_type = ItemType.get_record_by_pid(item_type_term.get('key'))
item_type_term['name'] = item_type.get('name')
return super(ItemsJSONSerializer,
self).post_process_serialize_search(results, pid_fetcher)
def dump(self, record, data):
"""Dump an AcqReceiptLine instance for ElasticSearch.
For ElasticSearch integration, we need to dump basic informations from
a `AcqReceiptLine` object instance, and add some basic data about
related.
:param record: The record to dump.
:param data: The initial dump data passed in by ``record.dumps()``.
"""
# Keep only some attributes from AcqReceiptLine object initial dump.
for attr in ['pid', 'receipt_date', 'amount', 'quantity', 'vat_rate']:
value = record.get(attr)
if value:
data.update({attr: value})
notes = record.get('notes', [])
if notes:
data['notes'] = [note['content'] for note in notes]
# Add document informations: pid, formatted title and ISBN identifiers.
# (remove None values from document metadata)
document = record.order_line.document
data['document'] = {
'pid': document.pid,
'title': title_format_text_head(document.get('title', [])),
'identifiers': document.get_identifier_values(filters=['bf:Isbn'])
}
data['document'] = {k: v for k, v in data['document'].items() if v}
return data
def preprocess_search_hit(self,
pid,
record_hit,
links_factory=None,
**kwargs):
"""Prepare a record hit from Elasticsearch for serialization.
:param pid: Persistent identifier instance.
:param record_hit: Record metadata retrieved via search.
:param links_factory: Factory function for record links.
"""
record = record_hit['_source']
item_pid = pid.pid_value
# process location
locations_map = kwargs.get('locations_map')
record['location_name'] = locations_map[record.get('location').get(
'pid')]
# retrieve and process document
document = search_document_by_pid(record['document']['pid'])
record['document_title'] = title_format_text_head(document.title,
with_subtitle=True)
creator = []
for contribution in document.contribution:
if any(role in contribution.role for role in role_filter):
try:
creator.append(contribution['agent']['preferred_name'])
except KeyError:
creator.append(
contribution['agent']['authorized_access_point_en'])
if creator:
record['document_creator'] = ' ; '.join(creator)
record['document_type'] = document.type
# get loans information
loans_count, loans = search_active_loans_for_item(item_pid)
record['loans_count'] = loans_count
if loans_count:
# get first loan
loan = next(loans)
record['last_transaction_date'] = format_date_filter(
loan.transaction_date,
date_format='short',
locale=current_i18n.locale.language,
)
record['created'] = format_date_filter(
record['_created'],
date_format='short',
locale=current_i18n.locale.language,
)
# prevent csv key error
# TODO: find other way to prevent csv key error
del (record['type'])
return record
def test_title_format_text_head():
"""Test title format text head."""
data = [{
"mainTitle": [{
"value": "Dingding lixianji"
}, {
"value": "\u4e01\u4e01\u5386\u9669\u8bb0",
"language": "und-hani"
}],
"type":
"bf:Title"
}]
assert "\u4e01\u4e01\u5386\u9669\u8bb0" == title_format_text_head(data)
data = [{
"mainTitle": [{
"value": "Die russischen orthodoxen Bischöfe von 1893",
}],
"subtitle": [{
"value": "Bio-Bibliographie"
}],
"type":
"bf:Title"
}]
assert "Die russischen orthodoxen Bischöfe von 1893 " \
": Bio-Bibliographie" == title_format_text_head(data)
data = [{
"mainTitle": [{
"value": "Die russischen orthodoxen Bischöfe von 1893",
}, {
"value": "The Russian Orthodox Bishops of 1893",
"language": "eng"
}],
"subtitle": [{
"value": "Bio-Bibliographie"
}],
"type":
"bf:Title"
}]
assert "The Russian Orthodox Bishops of 1893" == \
title_format_text_head(data)
def dump(self, record, data):
"""Dump a document instance with basic document informations.
:param record: The record to dump.
:param data: The initial dump data passed in by ``record.dumps()``.
"""
title_text = title_format_text_head(
record.get('title', []),
responsabilities=record.get('responsibilityStatement'))
data.update({'pid': record.get('pid'), 'title_text': title_text})
data = {k: v for k, v in data.items() if v}
return data
def do(self,
blob,
ignore_missing=True,
exception_handlers=None,
language='fr'):
"""Translate blob values and instantiate new model instance.
Raises ``MissingRule`` when no rule matched and ``ignore_missing``
is ``False``.
:param blob: ``dict``-like object on which the matching rules are
going to be applied.
:param ignore_missing: Set to ``False`` if you prefer to raise
an exception ``MissingRule`` for the first
key that it is not matching any rule.
:param exception_handlers: Give custom exception handlers to take care
of non-standard codes that are installation
specific.
:param language: Language to use.
"""
self.language = language
result = super().do(blob,
ignore_missing=ignore_missing,
exception_handlers=exception_handlers)
titles = blob.get('title', [])
bf_titles = list(filter(lambda t: t['type'] == 'bf:Title', titles))
text = title_format_text_head(titles=bf_titles,
responsabilities=blob.get(
'responsibilityStatement', []),
with_subtitle=True)
if text:
result['titles'] = [text]
pid = blob.get('pid')
if pid:
identifiers = result.get('identifiers', [])
identifiers.insert(0, f'bf:Local|{pid}')
return result
def dump(self, record, data):
"""Dump an AcqOrderLine instance for ElasticSearch.
For ElasticSearch integration, we need to dump basic informations from
a `AcqOrderLine` object instance, and add some data from related
object : related account basic informations and related document basic
informations.
:param record: The record to dump.
:param data: The initial dump data passed in by ``record.dumps()``.
"""
# Keep only some attributes from AcqOrderLine object initial dump.
for attr in ['pid', 'status', 'order_date', 'quantity']:
value = record.get(attr)
if value:
data.update({attr: value})
# Add account informations: pid, name and reference number.
# (remove None values from account metadata)
account = record.account
data['account'] = {
'pid': account.pid,
'name': account['name'],
'number': account.get('number')
}
data['account'] = {k: v for k, v in data['account'].items() if v}
# Add document informations: pid, formatted title and ISBN identifiers.
# (remove None values from document metadata)
document = record.document
data['document'] = {
'pid': document.pid,
'title': title_format_text_head(document.get('title', [])),
'identifiers': document.get_identifier_values(filters=['bf:Isbn'])
}
data['document'] = {k: v for k, v in data['document'].items() if v}
return data
def preprocess_search_hit(self,
pid,
record_hit,
links_factory=None,
**kwargs):
"""Prepare a record hit from Elasticsearch for serialization.
:param pid: Persistent identifier instance.
:param record_hit: Record metadata retrieved via search.
:param links_factory: Factory function for record links.
"""
language = kwargs.get('language')
record = record_hit['_source']
# inherit holdings call number when possible
item = Item(record)
issue_call_number = item.issue_inherited_first_call_number
if issue_call_number:
record['call_number'] = issue_call_number
item_pid = pid.pid_value
# process location
locations_map = kwargs.get('locations_map')
record['location_name'] = locations_map[record.get('location').get(
'pid')]
# retrieve and process document
document = search_document_by_pid(record['document']['pid'])
record['document_title'] = title_format_text_head(document.title,
with_subtitle=True)
# process contributions
creator = []
if 'contribution' in document:
for contribution in document.contribution:
if any(role in contribution.role for role in role_filter):
authorized_access_point = \
f'authorized_access_point_{language}'
if authorized_access_point in contribution['agent']:
creator.append(
contribution['agent'][authorized_access_point])
if creator:
record['document_creator'] = ' ; '.join(creator)
document_main_type = []
document_sub_type = []
for document_type in document.type:
data = document_type.to_dict()
document_main_type.append(data.get('main_type'))
document_sub_type.append(data.get('subtype', ''))
record['document_main_type'] = ', '.join(document_main_type)
record['document_sub_type'] = ', '.join(document_sub_type)
# get loans information
loans_count, loans = search_active_loans_for_item(item_pid)
record['loans_count'] = loans_count
if loans_count:
# get first loan
loan = next(loans)
record['last_transaction_date'] = format_date_filter(
loan.transaction_date,
date_format='short',
locale=language,
)
record['created'] = format_date_filter(
record['_created'],
date_format='short',
locale=language,
)
# prevent csv key error
# TODO: find other way to prevent csv key error
del (record['type'])
return record
def post_process_serialize_search(self, results, pid_fetcher):
"""Post process the search results.
:param results: Elasticsearch search result.
:param pid_fetcher: Persistent identifier fetcher.
"""
records = results.get('hits', {}).get('hits', {})
orgs = {}
libs = {}
locs = {}
for record in records:
metadata = record.get('metadata', {})
document = search_document_by_pid(
metadata.get('document').get('pid')
)
metadata['ui_title_text'] = title_format_text_head(
document['title'],
with_subtitle=True
)
item = Item.get_record_by_pid(metadata.get('pid'))
metadata['availability'] = {
'available': item.available,
'status': metadata['status'],
'display_text': item.availability_text,
'request': item.number_of_requests()
}
if not metadata['available']:
if metadata['status'] == ItemStatus.ON_LOAN:
metadata['availability']['due_date'] =\
item.get_item_end_date(format='long', language='en')
# Item in collection
collection = item.in_collection()
if collection:
metadata['in_collection'] = collection
# Organisation
organisation = metadata['organisation']
if organisation['pid'] not in orgs:
orgs[organisation['pid']] = Organisation \
.get_record_by_pid(organisation['pid'])
organisation['viewcode'] = orgs[organisation['pid']].get('code')
# Library
library = metadata['library']
if library['pid'] not in libs:
libs[library['pid']] = Library \
.get_record_by_pid(library['pid'])
library['name'] = libs[library['pid']].get('name')
# Location
location = metadata['location']
if location['pid'] not in locs:
locs[location['pid']] = Location \
.get_record_by_pid(location['pid'])
location['name'] = locs[location['pid']].get('name')
# Add library name
for lib_term in results.get('aggregations', {}).get(
'library', {}).get('buckets', []):
lib = Library.get_record_by_pid(lib_term.get('key'))
lib_term['name'] = lib.get('name')
# Add location name
for loc_term in results.get('aggregations', {}).get(
'location', {}).get('buckets', []):
loc = Location.get_record_by_pid(loc_term.get('key'))
loc_term['name'] = loc.get('name')
# Add item type name
for item_type_term in results.get('aggregations', {}).get(
'item_type', {}).get('buckets', []):
item_type = ItemType.get_record_by_pid(item_type_term.get('key'))
item_type_term['name'] = item_type.get('name')
# Add vendor name
for vendor_term in results.get('aggregations', {}).get(
'vendor', {}).get('buckets', []):
vendor = Vendor.get_record_by_pid(vendor_term.get('key'))
vendor_term['name'] = vendor.get('name')
# Correct document type buckets
buckets = results['aggregations']['document_type']['buckets']
results['aggregations']['document_type']['buckets'] = \
filter_document_type_buckets(buckets)
return super().post_process_serialize_search(results, pid_fetcher)
# Correct document type buckets
buckets = results['aggregations']['document_type']['buckets']
results['aggregations']['document_type']['buckets'] = \
filter_document_type_buckets(buckets)
def post_process_serialize_search(self, results, pid_fetcher):
"""Post process the search results.
:param results: Elasticsearch search result.
:param pid_fetcher: Persistent identifier fetcher.
"""
records = results.get('hits', {}).get('hits', {})
orgs = {}
libs = {}
locs = {}
# enrich library bucket
JSONSerializer.enrich_bucket_with_data(results, 'library',
LibrariesSearch, 'name')
# enrich location bucket
JSONSerializer.enrich_bucket_with_data(results, 'location',
LocationsSearch, 'name')
# enrich item type bucket
JSONSerializer.enrich_bucket_with_data(results, 'item_type',
ItemTypesSearch, 'name')
# enrich temporary item type bucket
JSONSerializer.enrich_bucket_with_data(results, 'temporary_item_type',
ItemTypesSearch, 'name')
# enrich temporary location bucket
JSONSerializer.enrich_bucket_with_data(results, 'temporary_location',
LocationsSearch, 'name')
# enrich vendor bucket
JSONSerializer.enrich_bucket_with_data(results, 'vendor',
VendorsSearch, 'name')
for record in records:
metadata = record.get('metadata', {})
document = search_document_by_pid(
metadata.get('document').get('pid'))
metadata['ui_title_text'] = title_format_text_head(
document['title'], with_subtitle=True)
item = Item.get_record_by_pid(metadata.get('pid'))
metadata['available'] = item.available
metadata['availability'] = {
'available': metadata['available'],
'status': metadata['status'],
'display_text': item.availability_text,
'request': item.number_of_requests()
}
if not metadata['available']:
if metadata['status'] == ItemStatus.ON_LOAN:
metadata['availability']['due_date'] =\
item.get_item_end_date(format='long', language='en')
# Item in collection
collection = item.in_collection()
if collection:
metadata['in_collection'] = collection
# Temporary location
temp_location = metadata.get('temporary_location')
if temp_location:
temp_location_pid = temp_location['pid']
if temp_location_pid not in locs:
locs[temp_location_pid] = Location \
.get_record_by_pid(temp_location_pid)
temp_location['name'] = locs[temp_location_pid].get('name')
# Organisation
organisation = metadata['organisation']
if organisation['pid'] not in orgs:
orgs[organisation['pid']] = Organisation \
.get_record_by_pid(organisation['pid'])
organisation['viewcode'] = orgs[organisation['pid']].get('code')
# Library
library = metadata['library']
if library['pid'] not in libs:
libs[library['pid']] = Library \
.get_record_by_pid(library['pid'])
library['name'] = libs[library['pid']].get('name')
# Location
location = metadata['location']
if location['pid'] not in locs:
locs[location['pid']] = Location \
.get_record_by_pid(location['pid'])
location['name'] = locs[location['pid']].get('name')
# Correct document type buckets
if results.get('aggregations', {}).get('document_type'):
buckets = results['aggregations']['document_type']['buckets']
results['aggregations']['document_type']['buckets'] = \
filter_document_type_buckets(buckets)
return super().post_process_serialize_search(results, pid_fetcher)