def test_create_holdings_with_pattern(client, librarian_martigny_no_email,
loc_public_martigny, journal,
item_type_standard_martigny, document,
json_header, holding_lib_martigny_data,
pattern_yearly_one_level_data,
holding_lib_martigny_w_patterns_data):
"""Test create holding type serial with patterns."""
login_user_via_session(client, librarian_martigny_no_email.user)
post_entrypoint = 'invenio_records_rest.hold_list'
del holding_lib_martigny_data['pid']
holding_lib_martigny_data['holdings_type'] = 'serial'
res, _ = postdata(client, post_entrypoint, holding_lib_martigny_data)
assert res.status_code == 403
holding_lib_martigny_data['patterns'] = \
pattern_yearly_one_level_data['patterns']
# test will fail when creating a serial holding for a standard document.
res, _ = postdata(client, post_entrypoint, holding_lib_martigny_data)
assert res.status_code == 403
# test will fail when creating a standard holding for a journal document.
holding_lib_martigny_w_patterns_data['holdings_type'] = 'standard'
del holding_lib_martigny_w_patterns_data['patterns']
with pytest.raises(RecordValidationError):
Holding.create(data=holding_lib_martigny_w_patterns_data,
delete_pid=True,
dbcommit=True,
reindex=True)
journal_pids = list(Document.get_all_serial_pids())
assert journal_pids == [journal.pid]
def test_holding_es_mapping(es, db, holding_lib_martigny,
holding_lib_martigny_data):
"""Test holding elasticsearch mapping."""
search = HoldingsSearch()
mapping = get_mapping(search.Meta.index)
assert mapping
Holding.create(holding_lib_martigny_data,
dbcommit=True,
reindex=True,
delete_pid=True)
assert mapping == get_mapping(search.Meta.index)
def test_holding_extended_validation(client,
journal, ebook_5,
loc_public_sion,
loc_public_martigny,
item_type_standard_martigny,
item_type_online_sion,
holding_lib_martigny_w_patterns_data,
holding_lib_sion_electronic_data):
"""Test holding extended validation."""
# instantiate serial holding
holding_tmp = Holding.create(
holding_lib_martigny_w_patterns_data, delete_pid=True)
# 1. holding type serial
# 1.1. test correct holding
holding_tmp.validate() # validation with extented_validation rules
# 1.1. test next expected date for regular frequencies
expected_date = holding_tmp['patterns']['next_expected_date']
del(holding_tmp['patterns']['next_expected_date'])
with pytest.raises(ValidationError):
holding_tmp.validate()
# reset data with original value
holding_tmp['patterns']['next_expected_date'] = expected_date
# 1.2. test multiple note with same type
holding_tmp.get('notes').append({
'type': 'general_note',
'content': 'other general_note...'
})
with pytest.raises(ValidationError):
holding_tmp.validate()
del holding_tmp['notes']
# 2. holding type electronic
# 2.1. test holding type electronic attached to wrong document type
holding_tmp['holdings_type'] = 'electronic'
with pytest.raises(ValidationError):
holding_tmp.validate()
# 2.2 test electronic holding
# instantiate electronic holding
holding_tmp = Holding.create(
holding_lib_sion_electronic_data, delete_pid=True)
holding_tmp.validate()
# 2.2 test electronic holding with enumeration and chronology
holding_tmp['enumerationAndChronology'] = 'enumerationAndChronology'
holding_tmp.validate()
def test_holding_create(db, es, document, org_martigny,
loc_public_martigny, item_type_standard_martigny,
holding_lib_martigny_data):
"""Test holding creation."""
next_pid = Holding.provider.identifier.next()
holding = Holding.create(holding_lib_martigny_data, dbcommit=True,
reindex=True, delete_pid=True)
next_pid += 1
assert holding == holding_lib_martigny_data
assert holding.get('pid') == str(next_pid)
holding = Holding.get_record_by_pid(str(next_pid))
assert holding == holding_lib_martigny_data
fetched_pid = fetcher(holding.id, holding)
assert fetched_pid.pid_value == str(next_pid)
assert fetched_pid.pid_type == 'hold'
search = HoldingsSearch()
es_hit = next(search.filter('term', pid=holding.pid).source('pid').scan())
holding_record = Holding.get_record_by_pid(es_hit.pid)
assert holding_record.organisation_pid == org_martigny.get('pid')
# holdings does not exist
assert not Holding.get_holdings_type_by_holding_pid('toto')
# clean created data
holding.delete(force=True, dbcommit=True, delindex=True)
def test_create_holdings_with_pattern(
client, librarian_martigny_no_email, loc_public_martigny,
journal, item_type_standard_martigny, document,
json_header, holding_lib_martigny_data, pattern_yearly_one_level_data,
holding_lib_martigny_w_patterns_data):
"""Test create holding type serial with patterns."""
login_user_via_session(client, librarian_martigny_no_email.user)
post_entrypoint = 'invenio_records_rest.hold_list'
del holding_lib_martigny_data['pid']
holding_lib_martigny_data['holdings_type'] = 'serial'
res, _ = postdata(
client,
post_entrypoint,
holding_lib_martigny_data
)
assert res.status_code == 403
holding_lib_martigny_data['patterns'] = \
pattern_yearly_one_level_data['patterns']
# test will fail when creating a serial holding for a standard document.
res, _ = postdata(
client,
post_entrypoint,
holding_lib_martigny_data
)
assert res.status_code == 403
# test will not fail when creating a standard holding for a journal doc.
holding_lib_martigny_w_patterns_data['holdings_type'] = 'standard'
# delete serials fields
fields = [
'enumerationAndChronology', 'notes', 'index', 'missing_issues',
'supplementaryContent', 'patterns'
]
for field in fields:
del holding_lib_martigny_w_patterns_data[field]
Holding.create(
data=holding_lib_martigny_w_patterns_data,
delete_pid=True,
dbcommit=True,
reindex=True)
journal_pids = list(Document.get_all_serial_pids())
assert journal_pids == [journal.pid]
def holding_lib_sion_w_patterns(app, journal, holding_lib_sion_w_patterns_data,
loc_public_sion, item_type_regular_sion):
"""Create holding of sion library with patterns."""
holding = Holding.create(data=holding_lib_sion_w_patterns_data,
delete_pid=False,
dbcommit=True,
reindex=True)
flush_index(HoldingsSearch.Meta.index)
return holding
def holding_lib_sion(app, document, holding_lib_sion_data, loc_public_sion,
item_type_internal_sion):
"""Create holding of sion library."""
holding = Holding.create(data=holding_lib_sion_data,
delete_pid=False,
dbcommit=True,
reindex=True)
flush_index(HoldingsSearch.Meta.index)
return holding
def holding_lib_fully(app, document, holding_lib_fully_data, loc_public_fully,
item_type_standard_martigny):
"""Create holding of fully library."""
holding = Holding.create(data=holding_lib_fully_data,
delete_pid=False,
dbcommit=True,
reindex=True)
flush_index(HoldingsSearch.Meta.index)
return holding
def holding_lib_sion_electronic(app, ebook_5, holding_lib_sion_electronic_data,
loc_public_sion, item_type_online_sion):
"""Create electronic holding of Martigny library."""
holding = Holding.create(data=holding_lib_sion_electronic_data,
delete_pid=False,
dbcommit=True,
reindex=True)
flush_index(HoldingsSearch.Meta.index)
return holding
def holding_lib_martigny_w_patterns(app, journal,
holding_lib_martigny_w_patterns_data,
loc_public_martigny,
item_type_standard_martigny):
"""Create holding of martigny library with patterns."""
holding = Holding.create(data=holding_lib_martigny_w_patterns_data,
delete_pid=False,
dbcommit=True,
reindex=True)
flush_index(HoldingsSearch.Meta.index)
return holding
def test_holding_create(db, es_clear, holding_lib_martigny_data):
"""Test holding creation."""
holding = Holding.create(holding_lib_martigny_data, delete_pid=True)
assert holding == holding_lib_martigny_data
assert holding.get('pid') == '1'
holding = Holding.get_record_by_pid('1')
assert holding == holding_lib_martigny_data
fetched_pid = fetcher(holding.id, holding)
assert fetched_pid.pid_value == '1'
assert fetched_pid.pid_type == 'hold'
def test_holding_es_mapping(es, db, loc_public_martigny,
item_type_standard_martigny, document,
holding_lib_martigny_data):
"""Test holding elasticsearch mapping."""
search = HoldingsSearch()
mapping = get_mapping(search.Meta.index)
assert mapping
holding = Holding.create(holding_lib_martigny_data,
dbcommit=True,
reindex=True,
delete_pid=True)
assert mapping == get_mapping(search.Meta.index)
# clean created data
holding.delete(force=True, dbcommit=True, delindex=True)
def test_holding_validate_next_expected_date(
client, librarian_martigny,
journal, loc_public_sion, item_type_internal_sion, document,
pattern_yearly_two_times_data, json_header,
holding_lib_sion_w_patterns_data):
"""Test create holding with regular frequency and missing
the next_expected_date.
"""
login_user_via_session(client, librarian_martigny.user)
holding = holding_lib_sion_w_patterns_data
holding['holdings_type'] = 'serial'
holding['patterns'] = \
pattern_yearly_two_times_data['patterns']
del holding['pid']
del holding['patterns']['next_expected_date']
# test will fail when the serial holding has no field
# next_expected_date for the regular frequency
with pytest.raises(ValidationError):
Holding.create(
data=holding,
delete_pid=False,
dbcommit=True,
reindex=True)
def test_holding_create(db, es_clear, document, org_martigny,
loc_public_martigny, item_type_standard_martigny,
holding_lib_martigny_data):
"""Test holding creation."""
holding = Holding.create(holding_lib_martigny_data,
dbcommit=True,
reindex=True,
delete_pid=True)
flush_index(HoldingsSearch.Meta.index)
assert holding == holding_lib_martigny_data
assert holding.get('pid') == '1'
holding = Holding.get_record_by_pid('1')
assert holding == holding_lib_martigny_data
fetched_pid = fetcher(holding.id, holding)
assert fetched_pid.pid_value == '1'
assert fetched_pid.pid_type == 'hold'
search = HoldingsSearch()
holding = next(search.filter('term', pid=holding.pid).scan())
holding_record = Holding.get_record_by_pid(holding.pid)
assert holding_record.organisation_pid == org_martigny.get('pid')
def bulk_records(records):
"""Records creation."""
n_updated = 0
n_rejected = 0
n_created = 0
record_schema = current_jsonschemas.path_to_url('documents/document-v0.0.1.json')
item_schema = current_jsonschemas.path_to_url('items/item-v0.0.1.json')
holding_schema = current_jsonschemas.path_to_url('holdings/holding-v0.0.1.json')
host_url = current_app.config.get('RERO_ILS_APP_BASE_URL')
url_api = '{host}/api/{doc_type}/{pid}'
record_id_iterator = []
item_id_iterator = []
holding_id_iterator = []
indexer = RecordIndexer()
start_time = datetime.now()
for record in records:
try:
if record.get('frbr', False):
document = record.get('document', {})
"""
# check if already in Rero-ILS
pid = None
for identifier in document.get('identifiedBy') :
if identifier.get('source') == 'VIRTUA' :
bibid = identifier.get('value')
query = DocumentsSearch().filter(
'term',
identifiedBy__value=bibid
).source(includes=['pid'])
try:
pid = [r.pid for r in query.scan()].pop()
except IndexError:
pid = None
if pid:
# update the record
# Do nothing for the moment
continue
else:
"""
document['$schema'] = record_schema
created_time = datetime.now()
document = Document.create(
document,
dbcommit=False,
reindex=False
)
record_id_iterator.append(document.id)
uri_documents = url_api.format(host=host_url,
doc_type='documents',
pid=document.pid)
map_holdings = {}
for holding in record.get('holdings'):
holding['$schema'] = holding_schema
holding['document'] = {
'$ref': uri_documents
}
holding['circulation_category'] = {
'$ref': map_item_type(str(holding.get('circulation_category')))
}
holding['location'] = {
'$ref': map_locations(str(holding.get('location')))
}
created_time = datetime.now()
result = Holding.create(
holding,
dbcommit=False,
reindex=False
)
map_holdings.update({
'{location}#{cica}'.format(
location = holding.get('location'),
cica = holding.get('circulation_category')) : result.get('pid')
}
)
holding_id_iterator.append(result.id)
for item in record.get('items'):
item['$schema'] = item_schema
item['document'] = {
'$ref': uri_documents
}
item['item_type'] = {
'$ref': map_item_type(str(item.get('item_type')))
}
item['location'] = {
'$ref': map_locations(str(item.get('location')))
}
holding_pid = map_holdings.get(
'{location}#{cica}'.format(
location = item.get('location'),
cica = item.get('item_type')))
item['holding'] = {
'$ref': url_api.format(host=host_url,
doc_type='holdings',
pid=holding_pid)
}
result = Item.create(
item,
dbcommit=False,
reindex=False
)
item_id_iterator.append(result.id)
n_created += 1
if n_created % 1000 == 0:
execution_time = datetime.now() - start_time
click.secho('{nb} created records in {execution_time}.'
.format(nb=len(record_id_iterator),
execution_time=execution_time),
fg='white')
start_time = datetime.now()
db.session.commit()
execution_time = datetime.now() - start_time
click.secho('{nb} commited records in {execution_time}.'
.format(nb=len(record_id_iterator),
execution_time=execution_time),
fg='white')
start_time = datetime.now()
click.secho('sending {n} holdings to indexer queue.'
.format(n=len(holding_id_iterator)), fg='white')
indexer.bulk_index(holding_id_iterator)
click.secho('process queue...', fg='yellow')
indexer.process_bulk_queue()
click.secho('sending {n} items to indexer queue.'
.format(n=len(item_id_iterator)), fg='white')
indexer.bulk_index(item_id_iterator)
click.secho('process queue...', fg='yellow')
indexer.process_bulk_queue()
click.secho('sending {n} documents to indexer queue.'
.format(n=len(record_id_iterator)), fg='white')
indexer.bulk_index(record_id_iterator)
click.secho('process queue...', fg='yellow')
indexer.process_bulk_queue()
execution_time = datetime.now() - start_time
click.secho('indexing records process in {execution_time}.'
.format(execution_time=execution_time),
fg='white')
click.secho('processing next batch records.', fg='green')
record_id_iterator.clear()
holding_id_iterator.clear()
item_id_iterator.clear()
start_time = datetime.now()
except Exception as e:
n_rejected += 1
click.secho('Error processing record [{id}] : {e}'
.format(id=record.get('_id'), e=e), fg='red')
db.session.commit()
indexer.bulk_index(holding_id_iterator)
indexer.process_bulk_queue()
indexer.bulk_index(item_id_iterator)
indexer.process_bulk_queue()
indexer.bulk_index(record_id_iterator)
indexer.process_bulk_queue()
return n_created
