def item2_lib_martigny(
app,
document,
item2_lib_martigny_data,
loc_public_martigny,
item_type_standard_martigny):
"""Create item2 of martigny library."""
item = Item.create(
data=item2_lib_martigny_data,
delete_pid=False,
dbcommit=True,
reindex=True)
flush_index(ItemsSearch.Meta.index)
return item
def test_create(db, minimal_document_record, minimal_item_record,
minimal_member_record, minimal_location_record):
"""Test DocumentWithItems creation."""
memb = MemberWithLocations.create(minimal_member_record, dbcommit=True)
loc = Location.create(minimal_location_record, dbcommit=True)
memb.add_location(loc, dbcommit=True)
minimal_item_record['location_pid'] = loc.pid
item = Item.create(minimal_item_record, dbcommit=True)
doc = DocumentsWithItems.create(minimal_document_record, dbcommit=True)
assert doc.itemslist == []
doc.add_item(item, dbcommit=True)
assert doc.itemslist[0] == item
dump = doc.dumps()
assert dump['itemslist'][0] == item.dumps()
def test_item_create(db, es_clear, item_lib_martigny_data_tmp,
item_lib_martigny):
"""Test itemanisation creation."""
item = Item.create(item_lib_martigny_data_tmp, delete_pid=True)
del item['holding']
assert item == item_lib_martigny_data_tmp
assert item.get('pid') == '1'
assert item.can_delete
item = Item.get_record_by_pid('1')
del item['holding']
assert item == item_lib_martigny_data_tmp
fetched_pid = item_id_fetcher(item.id, item)
assert fetched_pid.pid_value == '1'
assert fetched_pid.pid_type == 'item'
def test_member_name(db, minimal_member_record, minimal_item_record,
minimal_location_record):
"""Test member names."""
member = MemberWithLocations.create(minimal_member_record, dbcommit=True)
assert member
location = Location.create(minimal_location_record, dbcommit=True)
assert location
member.add_location(location, dbcommit=True)
assert member.locations
item = Item.create({})
item.update(minimal_item_record, dbcommit=True)
assert item
data = item.dumps()
assert data.get('member_pid') == '1'
assert data.get('member_name') == 'MV Sion'
holding = data.get('_circulation').get('holdings')[1]
assert holding['pickup_member_name'] == 'MV Sion'
def test_items_availability(item_type_missing_martigny,
item_type_standard_martigny,
item_lib_martigny_data_tmp, loc_public_martigny,
lib_martigny, org_martigny, document):
"""Test availability for an item."""
# Create a temporary item with correct data for the test
item_data = deepcopy(item_lib_martigny_data_tmp)
del item_data['pid']
item_data['barcode'] = 'TEST_AVAILABILITY'
item_data['temporary_item_type'] = {
'$ref': get_ref_for_pid(ItemType, item_type_missing_martigny.pid)
}
item = Item.create(item_data, dbcommit=True, reindex=True)
# test the availability and availability_text
assert not item.available
assert len(item.availability_text) == \
len(item_type_missing_martigny.get('displayed_status', [])) + 1
del item['temporary_item_type']
item = item.update(item, dbcommit=True, reindex=True)
assert item.available
assert len(item.availability_text) == 1 # only default value
# test availability and availability_text for an issue
item['type'] = TypeOfItem.ISSUE
item['enumerationAndChronology'] = 'dummy'
item['issue'] = {
'regular': False,
'status': ItemIssueStatus.RECEIVED,
'received_date': '1970-01-01',
'expected_date': '1970-01-01'
}
item = item.update(item, dbcommit=True, reindex=True)
assert item.available
assert item.availability_text[0]['label'] == item.status
item['issue']['status'] = ItemIssueStatus.LATE
item = item.update(item, dbcommit=True, reindex=True)
assert not item.available
assert item.availability_text[0]['label'] == ItemIssueStatus.LATE
# delete the created item
item.delete()
def create_new_item_from_existing_item(item=None):
"""Create a new item as a copy of a given existing item.
:param item: the item record
:return: the newly created item
"""
data = deepcopy(item)
data.pop('barcode')
data['status'] = ItemStatus.ON_SHELF
new_item = Item.create(data=data,
dbcommit=True,
reindex=True,
delete_pid=True)
flush_index(ItemsSearch.Meta.index)
assert new_item.status == ItemStatus.ON_SHELF
assert new_item.number_of_requests() == 0
return new_item
def test_item_create(item_lib_martigny_data_tmp, item_lib_martigny):
"""Test itemanisation creation."""
item = Item.create(item_lib_martigny_data_tmp, delete_pid=True)
assert item == item_lib_martigny_data_tmp
# we have used item_lib_martigny_data_tmp two times -> pid == 2
assert item.get('pid') == '2'
assert item.can_delete
item = Item.get_record_by_pid('1')
item_lib_martigny_data_tmp['pid'] = '1'
assert item == item_lib_martigny_data_tmp
fetched_pid = item_id_fetcher(item.id, item)
assert fetched_pid.pid_value == '1'
assert fetched_pid.pid_type == 'item'
item_lib_martigny.delete_from_index()
assert not item_lib_martigny.delete_from_index()
item_lib_martigny.dbcommit(forceindex=True)
def test_item_can_request(client, document, holding_lib_martigny,
item_lib_martigny, librarian_martigny, lib_martigny,
patron_martigny, circulation_policies,
patron_type_children_martigny,
loc_public_martigny_data, system_librarian_martigny,
item_lib_martigny_data):
"""Test item can request API."""
# test no logged user
res = client.get(
url_for('api_item.can_request',
item_pid=item_lib_martigny.pid,
library_pid=lib_martigny.pid,
patron_barcode=patron_martigny.get('patron',
{}).get('barcode')[0]))
assert res.status_code == 401
can, _ = can_request(item_lib_martigny)
assert not can
login_user_via_session(client, librarian_martigny.user)
# valid test
res = client.get(
url_for('api_item.can_request',
item_pid=item_lib_martigny.pid,
library_pid=lib_martigny.pid,
patron_barcode=patron_martigny.get('patron',
{}).get('barcode')[0]))
assert res.status_code == 200
data = get_json(res)
assert data.get('can')
# test no valid item
res = client.get(
url_for('api_item.can_request',
item_pid='no_item',
library_pid=lib_martigny.pid,
patron_barcode=patron_martigny.get('patron',
{}).get('barcode')[0]))
assert res.status_code == 404
# test no valid library
res = client.get(
url_for('api_item.can_request',
item_pid=item_lib_martigny.pid,
library_pid='no_library',
patron_barcode=patron_martigny.get('patron',
{}).get('barcode')[0]))
assert res.status_code == 404
# test no valid patron
res = client.get(
url_for('api_item.can_request',
item_pid=item_lib_martigny.pid,
library_pid=lib_martigny.pid,
patron_barcode='no_barcode'))
assert res.status_code == 404
# test no valid item status
item_lib_martigny['status'] = ItemStatus.MISSING
item_lib_martigny.update(item_lib_martigny, dbcommit=True, reindex=True)
res = client.get(
url_for('api_item.can_request',
item_pid=item_lib_martigny.pid,
library_pid=lib_martigny.pid,
patron_barcode=patron_martigny.get('patron',
{}).get('barcode')[0]))
assert res.status_code == 200
data = get_json(res)
assert not data.get('can')
item_lib_martigny['status'] = ItemStatus.ON_SHELF
item_lib_martigny.update(item_lib_martigny, dbcommit=True, reindex=True)
# Location :: allow_request == false
# create a new location and set 'allow_request' to false. Assign a new
# item to this location. Chek if this item can be requested : it couldn't
# with 'Item location doesn't allow request' reason.
new_location = deepcopy(loc_public_martigny_data)
del new_location['pid']
new_location['allow_request'] = False
new_location = Location.create(new_location, dbcommit=True, reindex=True)
assert new_location
new_item = deepcopy(item_lib_martigny_data)
del new_item['pid']
new_item['barcode'] = 'dummy_barcode_allow_request'
new_item['location']['$ref'] = get_ref_for_pid(Location, new_location.pid)
new_item = Item.create(new_item, dbcommit=True, reindex=True)
assert new_item
res = client.get(url_for('api_item.can_request', item_pid=new_item.pid))
assert res.status_code == 200
data = get_json(res)
assert not data.get('can')
# remove created data
item_url = url_for('invenio_records_rest.item_item',
pid_value=new_item.pid)
hold_url = url_for('invenio_records_rest.hold_item',
pid_value=new_item.holding_pid)
loc_url = url_for('invenio_records_rest.loc_item',
pid_value=new_location.pid)
client.delete(item_url)
client.delete(hold_url)
client.delete(loc_url)
def test_operation_log_on_item(client, item_lib_martigny_data_tmp,
librarian_martigny, json_header,
item_lib_martigny):
"""Test operation log on Item."""
login_user_via_session(client, librarian_martigny.user)
# Get the operation log index
fake_data = {'date': datetime.now().isoformat()}
oplg_index = OperationLog.get_index(fake_data)
# STEP #1 : Create an item. This will generate an operation log
item_data = deepcopy(item_lib_martigny_data_tmp)
del item_data['pid']
item = Item.create(item_data, dbcommit=True, reindex=True)
flush_index(oplg_index)
q = f'record.type:item AND record.value:{item.pid}'
es_url = url_for('invenio_records_rest.oplg_list', q=q, sort='mostrecent')
res = client.get(es_url)
data = get_json(res)
assert data['hits']['total']['value'] == 1
metadata = data['hits']['hits'][0]['metadata']
assert metadata['operation'] == OperationLogOperation.CREATE
# STEP #2 : Update the item ``price`` attribute.
# As any changes on this attribute must be logged, a new operation log
# will be generated.
item['price'] = 10
item = item.update(item, dbcommit=True, reindex=True)
flush_index(oplg_index)
res = client.get(es_url)
data = get_json(res)
assert data['hits']['total']['value'] == 2
metadata = data['hits']['hits'][0]['metadata']
assert metadata['operation'] == OperationLogOperation.UPDATE
# STEP #3 : Update the item ``status`` attribute.
# This attribute doesn't need to be tracked. So if it's the only change
# on this record then no OpLog should be created.
item['status'] = ItemStatus.EXCLUDED
item = item.update(item, dbcommit=True, reindex=True)
flush_index(oplg_index)
res = client.get(es_url)
data = get_json(res)
assert data['hits']['total']['value'] == 2
# STEP #4 : Update the item ``status`` and ``price`` attributes.
# As we update at least one attribute that need to be tracked, this
# update will generate a new OpLog (UPDATE)
item['status'] = ItemStatus.AT_DESK
item['price'] = 12
item = item.update(item, dbcommit=True, reindex=True)
flush_index(oplg_index)
res = client.get(es_url)
data = get_json(res)
assert data['hits']['total']['value'] == 3
metadata = data['hits']['hits'][0]['metadata']
assert metadata['operation'] == OperationLogOperation.UPDATE
# STEP #5 : Delete the item
# This will generate the last OpLog about the item.
item.delete(dbcommit=True, delindex=True)
flush_index(oplg_index)
res = client.get(es_url)
data = get_json(res)
assert data['hits']['total']['value'] == 4
metadata = data['hits']['hits'][0]['metadata']
assert metadata['operation'] == OperationLogOperation.DELETE
def test_provisional_items_creation(client, document, org_martigny,
holding_lib_martigny_w_patterns,
provisional_item_lib_martigny_data,
json_header, item_lib_martigny,
patron_martigny,
system_librarian_martigny):
"""Test creation of provisional items."""
holding = holding_lib_martigny_w_patterns
provisional_item_lib_martigny_data['holding'] = \
{'$ref': get_ref_for_pid('hold', holding.pid)}
item = Item.create(provisional_item_lib_martigny_data,
dbcommit=True,
reindex=True)
item_url = url_for('invenio_records_rest.item_item', pid_value=item.pid)
res = client.get(item_url)
assert res.status_code == 200
item_es = Item(get_json(res).get('metadata'))
assert item_es.get('type') == TypeOfItem.PROVISIONAL
assert item_es.status == ItemStatus.ON_SHELF
assert item_es.holding_pid == holding.pid
# TEST: logged patrons can not have the provisional items in the results.
# for both global and insitutional view.
login_user_via_session(client, patron_martigny.user)
list_url = url_for('invenio_records_rest.item_list', view='org1')
response = client.get(list_url, headers=json_header)
assert response.status_code == 200
data = get_json(response)
items = data['hits']['hits']
assert len(items) == 1
assert items[0]['metadata']['pid'] == item_lib_martigny.pid
list_url = url_for('invenio_records_rest.item_list', view='global')
response = client.get(list_url, headers=json_header)
assert response.status_code == 200
data = get_json(response)
items = data['hits']['hits']
assert len(items) == 1
assert items[0]['metadata']['pid'] == item_lib_martigny.pid
# TEST: logged librarians can have the provisional items in the results.
# provisional items are still not available for the global and other views.
login_user_via_session(client, system_librarian_martigny.user)
list_url = url_for('invenio_records_rest.item_list')
response = client.get(list_url, headers=json_header)
assert response.status_code == 200
data = get_json(response)
items = data['hits']['hits']
assert len(items) == 2
list_url = url_for('invenio_records_rest.item_list', view='global')
response = client.get(list_url, headers=json_header)
assert response.status_code == 200
data = get_json(response)
items = data['hits']['hits']
assert len(items) == 1
assert items[0]['metadata']['pid'] == item_lib_martigny.pid
list_url = url_for('invenio_records_rest.item_list', view='org1')
response = client.get(list_url, headers=json_header)
assert response.status_code == 200
data = get_json(response)
items = data['hits']['hits']
assert len(items) == 1
assert items[0]['metadata']['pid'] == item_lib_martigny.pid
def test_checkout_on_item_on_shelf(
circulation_policies,
patron_martigny,
patron2_martigny,
item_lib_martigny,
loc_public_martigny,
librarian_martigny,
item_on_shelf_martigny_patron_and_loan_pending):
"""Test checkout on an ON_SHELF item."""
# Create a new item in ON_SHELF (without Loan)
data = deepcopy(item_lib_martigny)
data.pop('barcode')
data.setdefault('status', ItemStatus.ON_SHELF)
created_item = Item.create(
data=data, dbcommit=True, reindex=True, delete_pid=True)
# Check item is ON_SHELF and NO PENDING loan exist!
assert created_item.number_of_requests() == 0
assert created_item.status == ItemStatus.ON_SHELF
assert not created_item.is_requested_by_patron(
patron_martigny.get('patron', {}).get('barcode')[0])
# the following tests the circulation action CHECKOUT_1_1
# an ON_SHELF item
# WITHOUT pending loan
# CAN be CHECKOUT
params = {
'patron_pid': patron_martigny.pid,
'transaction_location_pid': loc_public_martigny.pid,
'transaction_user_pid': librarian_martigny.pid,
'pickup_location_pid': loc_public_martigny.pid
}
onloan_item, actions = created_item.checkout(**params)
loan = Loan.get_record_by_pid(actions[LoanAction.CHECKOUT].get('pid'))
# Check loan is ITEM_ON_LOAN and item is ON_LOAN
assert onloan_item.number_of_requests() == 0
assert onloan_item.status == ItemStatus.ON_LOAN
assert loan['state'] == LoanState.ITEM_ON_LOAN
# Fetch a PENDING item and loan
pending_item, patron, loan = item_on_shelf_martigny_patron_and_loan_pending
# the following tests the circulation action CHECKOUT_1_2_2
# an ON_SHELF item
# WITH pending loan
# checkout patron != patron of first PENDING loan
# can NOT be CHECKOUT
params['patron_pid'] = patron2_martigny.pid
# CHECKOUT patron is DIFFERENT from 1st PENDING LOAN patron
assert params['patron_pid'] != patron['pid']
with pytest.raises(ItemNotAvailableError):
asked_item, actions = pending_item.checkout(**params)
checkout_loan = Loan.get_record_by_pid(loan['pid'])
asked_item = Item.get_record_by_pid(pending_item.pid)
# Check loan is PENDING and item is ON_SHELF
assert asked_item.status == ItemStatus.ON_SHELF
assert checkout_loan['state'] == LoanState.PENDING
assert asked_item.number_of_requests() == 1
# the following tests the circulation action CHECKOUT_1_2_1
# an ON_SHELF item
# WITH a pending loan
# checkout patron = patron of first PENDING loan
# CAN be CHECKOUT
assert pending_item.is_requested_by_patron(
patron.patron.get('barcode')[0])
# Checkout it! CHECKOUT patron == 1st PENDING LOAN patron
assert patron.get('pid') == loan.get('patron_pid')
params['patron_pid'] = patron_martigny.pid
onloan_item, actions = pending_item.checkout(**params, pid=loan.pid)
loan = Loan.get_record_by_pid(actions[LoanAction.CHECKOUT].get('pid'))
# Check loan is ITEM_ON_LOAN and item is ON_LOAN
assert onloan_item.number_of_requests() == 0
assert onloan_item.status == ItemStatus.ON_LOAN
assert loan['state'] == LoanState.ITEM_ON_LOAN
def test_overdue_limit(
client, app, librarian_martigny, lib_martigny, item_lib_martigny,
item2_lib_martigny, patron_type_children_martigny,
item3_lib_martigny, item_lib_martigny_data, item2_lib_martigny_data,
item3_lib_martigny_data, loc_public_martigny, patron_martigny,
circ_policy_short_martigny, lib_saxon, loc_public_saxon):
"""Test overdue limit."""
item = item_lib_martigny
item_pid = item.pid
patron_pid = patron_martigny.pid
date_format = '%Y/%m/%dT%H:%M:%S.000Z'
today = datetime.utcnow()
eod = today.replace(hour=23, minute=59, second=0, microsecond=0,
tzinfo=lib_martigny.get_timezone())
# STEP 0 :: Prepare data for test
# * Update the patron_type to set a overdue_items_limit rule.
# We define than only 1 overdue items are allowed. Trying a second
# checkout is disallowed if patron has an overdue item
patron_type = patron_type_children_martigny
patron_type \
.setdefault('limits', {}) \
.setdefault('overdue_items_limits', {}) \
.setdefault('default_value', 1)
patron_type.update(patron_type, dbcommit=True, reindex=True)
patron_type = PatronType.get_record_by_pid(patron_type.pid)
assert patron_type\
.get('limits', {})\
.get('overdue_items_limits', {}) \
.get('default_value') == 1
# STEP 1 :: Create an checkout with a end_date at the current date
# * Create a checkout and set end_date to a fixed_date equals to
# current tested date. The loan should not be considered as overdue
# and a second checkout should be possible
login_user_via_session(client, librarian_martigny.user)
res, data = postdata(
client,
'api_item.checkout',
dict(
item_pid=item_pid,
patron_pid=patron_pid,
transaction_location_pid=loc_public_martigny.pid,
transaction_user_pid=librarian_martigny.pid,
end_date=eod.strftime(date_format)
)
)
assert res.status_code == 200
loan_pid = data.get('action_applied')[LoanAction.CHECKOUT].get('pid')
loan = Loan.get_record_by_pid(loan_pid)
assert not loan.is_loan_overdue()
tmp_item_data = deepcopy(item_lib_martigny_data)
del tmp_item_data['pid']
tmp_item_data['library']['$ref'] = get_ref_for_pid('lib', lib_saxon.pid)
tmp_item_data['location']['$ref'] = \
get_ref_for_pid('loc', loc_public_saxon.pid)
tmp_item = Item.create(tmp_item_data, dbcommit=True, reindex=True)
res, data = postdata(
client,
'api_item.checkout',
dict(
item_pid=tmp_item.pid,
patron_pid=patron_pid,
transaction_location_pid=loc_public_martigny.pid,
transaction_user_pid=librarian_martigny.pid
)
)
assert res.status_code == 200
res, _ = postdata(
client,
'api_item.checkin',
dict(
item_pid=tmp_item.pid,
pid=data.get('action_applied')[LoanAction.CHECKOUT].get('pid'),
transaction_location_pid=loc_public_martigny.pid,
transaction_user_pid=librarian_martigny.pid,
)
)
assert res.status_code == 200
# STEP 2 :: Set the loan as overdue and test a new checkout
# Now there is one loan in overdue, then the limit is reached and a new
# checkout shouldn't be possible
end_date = eod - timedelta(days=7)
loan['end_date'] = end_date.isoformat()
loan.update(loan, dbcommit=True, reindex=True)
overdue_loans = list(get_overdue_loans(patron_pid=patron_pid))
assert loan.is_loan_overdue()
assert loan.end_date == end_date.isoformat()
assert overdue_loans[0].get('pid') == loan_pid
assert not get_notification(loan, NotificationType.OVERDUE)
notification = loan.create_notification(
_type=NotificationType.OVERDUE).pop()
Dispatcher.dispatch_notifications([notification.get('pid')])
flush_index(NotificationsSearch.Meta.index)
flush_index(LoansSearch.Meta.index)
assert get_notification(loan, NotificationType.OVERDUE)
# Try a second checkout - limit should be reached
res, data = postdata(
client,
'api_item.checkout',
dict(
item_pid=item2_lib_martigny.pid,
patron_pid=patron_pid,
transaction_location_pid=loc_public_martigny.pid,
transaction_user_pid=librarian_martigny.pid,
)
)
assert res.status_code == 403
assert 'Checkout denied' in data['message']
# Try a request - limit should be reached
res, data = postdata(
client,
'api_item.librarian_request',
dict(
item_pid=item2_lib_martigny.pid,
patron_pid=patron_pid,
pickup_location_pid=loc_public_martigny.pid,
transaction_library_pid=lib_martigny.pid,
transaction_user_pid=librarian_martigny.pid
)
)
assert res.status_code == 403
assert 'maximal number of overdue items is reached' in data['message']
# Try to extend - limit should be reached
res, _ = postdata(
client,
'api_item.extend_loan',
dict(
item_pid=item_pid,
transaction_user_pid=librarian_martigny.pid,
transaction_location_pid=loc_public_martigny.pid
)
)
assert res.status_code == 403
assert 'maximal number of overdue items is reached' in data['message']
# reset the patron_type with default value
del patron_type['limits']
# [2] test fee amount limit
# Update the patron_type to set a fee_amount_limit rule
patron_type \
.setdefault('limits', {}) \
.setdefault('fee_amount_limits', {}) \
.setdefault('default_value', 0.5)
patron_type.update(patron_type, dbcommit=True, reindex=True)
patron_type = PatronType.get_record_by_pid(patron_type.pid)
assert patron_type.get('limits', {}).get('fee_amount_limits', {}) \
.get('default_value') == 0.5
# [2.1] test fee amount limit when we try to checkout a second item
res, data = postdata(
client,
'api_item.checkout',
dict(
item_pid=item2_lib_martigny.pid,
patron_pid=patron_pid,
transaction_location_pid=loc_public_martigny.pid,
transaction_user_pid=librarian_martigny.pid,
)
)
assert res.status_code == 403
assert 'maximal overdue fee amount is reached' in data['message']
# [2.2] test fee amount limit when we try to request another item
res, data = postdata(
client,
'api_item.librarian_request',
dict(
item_pid=item2_lib_martigny.pid,
patron_pid=patron_pid,
pickup_location_pid=loc_public_martigny.pid,
transaction_library_pid=lib_martigny.pid,
transaction_user_pid=librarian_martigny.pid
)
)
assert res.status_code == 403
assert 'maximal overdue fee amount is reached' in data['message']
# [2.3] test fee amount limit when we try to extend loan
res, _ = postdata(
client,
'api_item.extend_loan',
dict(
item_pid=item_pid,
transaction_user_pid=librarian_martigny.pid,
transaction_location_pid=loc_public_martigny.pid
)
)
assert res.status_code == 403
assert 'maximal overdue fee amount is reached' in data['message']
# reset the patron_type with default value
del patron_type['limits']
patron_type.update(patron_type, dbcommit=True, reindex=True)
patron_type = PatronType.get_record_by_pid(patron_type.pid)
assert patron_type.get('limits') is None
# # checkin the item to put it back to it's original state
res, _ = postdata(
client,
'api_item.checkin',
dict(
item_pid=item_pid,
pid=loan_pid,
transaction_location_pid=loc_public_martigny.pid,
transaction_user_pid=librarian_martigny.pid,
)
)
assert res.status_code == 200
def bulk_records(records):
"""Records creation."""
n_updated = 0
n_rejected = 0
n_created = 0
record_schema = current_jsonschemas.path_to_url('documents/document-v0.0.1.json')
item_schema = current_jsonschemas.path_to_url('items/item-v0.0.1.json')
holding_schema = current_jsonschemas.path_to_url('holdings/holding-v0.0.1.json')
host_url = current_app.config.get('RERO_ILS_APP_BASE_URL')
url_api = '{host}/api/{doc_type}/{pid}'
record_id_iterator = []
item_id_iterator = []
holding_id_iterator = []
indexer = RecordIndexer()
start_time = datetime.now()
for record in records:
try:
if record.get('frbr', False):
document = record.get('document', {})
"""
# check if already in Rero-ILS
pid = None
for identifier in document.get('identifiedBy') :
if identifier.get('source') == 'VIRTUA' :
bibid = identifier.get('value')
query = DocumentsSearch().filter(
'term',
identifiedBy__value=bibid
).source(includes=['pid'])
try:
pid = [r.pid for r in query.scan()].pop()
except IndexError:
pid = None
if pid:
# update the record
# Do nothing for the moment
continue
else:
"""
document['$schema'] = record_schema
created_time = datetime.now()
document = Document.create(
document,
dbcommit=False,
reindex=False
)
record_id_iterator.append(document.id)
uri_documents = url_api.format(host=host_url,
doc_type='documents',
pid=document.pid)
map_holdings = {}
for holding in record.get('holdings'):
holding['$schema'] = holding_schema
holding['document'] = {
'$ref': uri_documents
}
holding['circulation_category'] = {
'$ref': map_item_type(str(holding.get('circulation_category')))
}
holding['location'] = {
'$ref': map_locations(str(holding.get('location')))
}
created_time = datetime.now()
result = Holding.create(
holding,
dbcommit=False,
reindex=False
)
map_holdings.update({
'{location}#{cica}'.format(
location = holding.get('location'),
cica = holding.get('circulation_category')) : result.get('pid')
}
)
holding_id_iterator.append(result.id)
for item in record.get('items'):
item['$schema'] = item_schema
item['document'] = {
'$ref': uri_documents
}
item['item_type'] = {
'$ref': map_item_type(str(item.get('item_type')))
}
item['location'] = {
'$ref': map_locations(str(item.get('location')))
}
holding_pid = map_holdings.get(
'{location}#{cica}'.format(
location = item.get('location'),
cica = item.get('item_type')))
item['holding'] = {
'$ref': url_api.format(host=host_url,
doc_type='holdings',
pid=holding_pid)
}
result = Item.create(
item,
dbcommit=False,
reindex=False
)
item_id_iterator.append(result.id)
n_created += 1
if n_created % 1000 == 0:
execution_time = datetime.now() - start_time
click.secho('{nb} created records in {execution_time}.'
.format(nb=len(record_id_iterator),
execution_time=execution_time),
fg='white')
start_time = datetime.now()
db.session.commit()
execution_time = datetime.now() - start_time
click.secho('{nb} commited records in {execution_time}.'
.format(nb=len(record_id_iterator),
execution_time=execution_time),
fg='white')
start_time = datetime.now()
click.secho('sending {n} holdings to indexer queue.'
.format(n=len(holding_id_iterator)), fg='white')
indexer.bulk_index(holding_id_iterator)
click.secho('process queue...', fg='yellow')
indexer.process_bulk_queue()
click.secho('sending {n} items to indexer queue.'
.format(n=len(item_id_iterator)), fg='white')
indexer.bulk_index(item_id_iterator)
click.secho('process queue...', fg='yellow')
indexer.process_bulk_queue()
click.secho('sending {n} documents to indexer queue.'
.format(n=len(record_id_iterator)), fg='white')
indexer.bulk_index(record_id_iterator)
click.secho('process queue...', fg='yellow')
indexer.process_bulk_queue()
execution_time = datetime.now() - start_time
click.secho('indexing records process in {execution_time}.'
.format(execution_time=execution_time),
fg='white')
click.secho('processing next batch records.', fg='green')
record_id_iterator.clear()
holding_id_iterator.clear()
item_id_iterator.clear()
start_time = datetime.now()
except Exception as e:
n_rejected += 1
click.secho('Error processing record [{id}] : {e}'
.format(id=record.get('_id'), e=e), fg='red')
db.session.commit()
indexer.bulk_index(holding_id_iterator)
indexer.process_bulk_queue()
indexer.bulk_index(item_id_iterator)
indexer.process_bulk_queue()
indexer.bulk_index(record_id_iterator)
indexer.process_bulk_queue()
return n_created
