def test_create_holdings_with_pattern(client, librarian_martigny_no_email, loc_public_martigny, journal, item_type_standard_martigny, document, json_header, holding_lib_martigny_data, pattern_yearly_one_level_data, holding_lib_martigny_w_patterns_data): """Test create holding type serial with patterns.""" login_user_via_session(client, librarian_martigny_no_email.user) post_entrypoint = 'invenio_records_rest.hold_list' del holding_lib_martigny_data['pid'] holding_lib_martigny_data['holdings_type'] = 'serial' res, _ = postdata(client, post_entrypoint, holding_lib_martigny_data) assert res.status_code == 403 holding_lib_martigny_data['patterns'] = \ pattern_yearly_one_level_data['patterns'] # test will fail when creating a serial holding for a standard document. res, _ = postdata(client, post_entrypoint, holding_lib_martigny_data) assert res.status_code == 403 # test will fail when creating a standard holding for a journal document. holding_lib_martigny_w_patterns_data['holdings_type'] = 'standard' del holding_lib_martigny_w_patterns_data['patterns'] with pytest.raises(RecordValidationError): Holding.create(data=holding_lib_martigny_w_patterns_data, delete_pid=True, dbcommit=True, reindex=True) journal_pids = list(Document.get_all_serial_pids()) assert journal_pids == [journal.pid]
def test_holding_es_mapping(es, db, holding_lib_martigny, holding_lib_martigny_data): """Test holding elasticsearch mapping.""" search = HoldingsSearch() mapping = get_mapping(search.Meta.index) assert mapping Holding.create(holding_lib_martigny_data, dbcommit=True, reindex=True, delete_pid=True) assert mapping == get_mapping(search.Meta.index)
def test_holding_extended_validation(client, journal, ebook_5, loc_public_sion, loc_public_martigny, item_type_standard_martigny, item_type_online_sion, holding_lib_martigny_w_patterns_data, holding_lib_sion_electronic_data): """Test holding extended validation.""" # instantiate serial holding holding_tmp = Holding.create( holding_lib_martigny_w_patterns_data, delete_pid=True) # 1. holding type serial # 1.1. test correct holding holding_tmp.validate() # validation with extented_validation rules # 1.1. test next expected date for regular frequencies expected_date = holding_tmp['patterns']['next_expected_date'] del(holding_tmp['patterns']['next_expected_date']) with pytest.raises(ValidationError): holding_tmp.validate() # reset data with original value holding_tmp['patterns']['next_expected_date'] = expected_date # 1.2. test multiple note with same type holding_tmp.get('notes').append({ 'type': 'general_note', 'content': 'other general_note...' }) with pytest.raises(ValidationError): holding_tmp.validate() del holding_tmp['notes'] # 2. holding type electronic # 2.1. test holding type electronic attached to wrong document type holding_tmp['holdings_type'] = 'electronic' with pytest.raises(ValidationError): holding_tmp.validate() # 2.2 test electronic holding # instantiate electronic holding holding_tmp = Holding.create( holding_lib_sion_electronic_data, delete_pid=True) holding_tmp.validate() # 2.2 test electronic holding with enumeration and chronology holding_tmp['enumerationAndChronology'] = 'enumerationAndChronology' holding_tmp.validate()
def test_holding_create(db, es, document, org_martigny, loc_public_martigny, item_type_standard_martigny, holding_lib_martigny_data): """Test holding creation.""" next_pid = Holding.provider.identifier.next() holding = Holding.create(holding_lib_martigny_data, dbcommit=True, reindex=True, delete_pid=True) next_pid += 1 assert holding == holding_lib_martigny_data assert holding.get('pid') == str(next_pid) holding = Holding.get_record_by_pid(str(next_pid)) assert holding == holding_lib_martigny_data fetched_pid = fetcher(holding.id, holding) assert fetched_pid.pid_value == str(next_pid) assert fetched_pid.pid_type == 'hold' search = HoldingsSearch() es_hit = next(search.filter('term', pid=holding.pid).source('pid').scan()) holding_record = Holding.get_record_by_pid(es_hit.pid) assert holding_record.organisation_pid == org_martigny.get('pid') # holdings does not exist assert not Holding.get_holdings_type_by_holding_pid('toto') # clean created data holding.delete(force=True, dbcommit=True, delindex=True)
def test_create_holdings_with_pattern( client, librarian_martigny_no_email, loc_public_martigny, journal, item_type_standard_martigny, document, json_header, holding_lib_martigny_data, pattern_yearly_one_level_data, holding_lib_martigny_w_patterns_data): """Test create holding type serial with patterns.""" login_user_via_session(client, librarian_martigny_no_email.user) post_entrypoint = 'invenio_records_rest.hold_list' del holding_lib_martigny_data['pid'] holding_lib_martigny_data['holdings_type'] = 'serial' res, _ = postdata( client, post_entrypoint, holding_lib_martigny_data ) assert res.status_code == 403 holding_lib_martigny_data['patterns'] = \ pattern_yearly_one_level_data['patterns'] # test will fail when creating a serial holding for a standard document. res, _ = postdata( client, post_entrypoint, holding_lib_martigny_data ) assert res.status_code == 403 # test will not fail when creating a standard holding for a journal doc. holding_lib_martigny_w_patterns_data['holdings_type'] = 'standard' # delete serials fields fields = [ 'enumerationAndChronology', 'notes', 'index', 'missing_issues', 'supplementaryContent', 'patterns' ] for field in fields: del holding_lib_martigny_w_patterns_data[field] Holding.create( data=holding_lib_martigny_w_patterns_data, delete_pid=True, dbcommit=True, reindex=True) journal_pids = list(Document.get_all_serial_pids()) assert journal_pids == [journal.pid]
def holding_lib_sion_w_patterns(app, journal, holding_lib_sion_w_patterns_data, loc_public_sion, item_type_regular_sion): """Create holding of sion library with patterns.""" holding = Holding.create(data=holding_lib_sion_w_patterns_data, delete_pid=False, dbcommit=True, reindex=True) flush_index(HoldingsSearch.Meta.index) return holding
def holding_lib_sion(app, document, holding_lib_sion_data, loc_public_sion, item_type_internal_sion): """Create holding of sion library.""" holding = Holding.create(data=holding_lib_sion_data, delete_pid=False, dbcommit=True, reindex=True) flush_index(HoldingsSearch.Meta.index) return holding
def holding_lib_fully(app, document, holding_lib_fully_data, loc_public_fully, item_type_standard_martigny): """Create holding of fully library.""" holding = Holding.create(data=holding_lib_fully_data, delete_pid=False, dbcommit=True, reindex=True) flush_index(HoldingsSearch.Meta.index) return holding
def holding_lib_sion_electronic(app, ebook_5, holding_lib_sion_electronic_data, loc_public_sion, item_type_online_sion): """Create electronic holding of Martigny library.""" holding = Holding.create(data=holding_lib_sion_electronic_data, delete_pid=False, dbcommit=True, reindex=True) flush_index(HoldingsSearch.Meta.index) return holding
def holding_lib_martigny_w_patterns(app, journal, holding_lib_martigny_w_patterns_data, loc_public_martigny, item_type_standard_martigny): """Create holding of martigny library with patterns.""" holding = Holding.create(data=holding_lib_martigny_w_patterns_data, delete_pid=False, dbcommit=True, reindex=True) flush_index(HoldingsSearch.Meta.index) return holding
def test_holding_create(db, es_clear, holding_lib_martigny_data): """Test holding creation.""" holding = Holding.create(holding_lib_martigny_data, delete_pid=True) assert holding == holding_lib_martigny_data assert holding.get('pid') == '1' holding = Holding.get_record_by_pid('1') assert holding == holding_lib_martigny_data fetched_pid = fetcher(holding.id, holding) assert fetched_pid.pid_value == '1' assert fetched_pid.pid_type == 'hold'
def test_holding_es_mapping(es, db, loc_public_martigny, item_type_standard_martigny, document, holding_lib_martigny_data): """Test holding elasticsearch mapping.""" search = HoldingsSearch() mapping = get_mapping(search.Meta.index) assert mapping holding = Holding.create(holding_lib_martigny_data, dbcommit=True, reindex=True, delete_pid=True) assert mapping == get_mapping(search.Meta.index) # clean created data holding.delete(force=True, dbcommit=True, delindex=True)
def test_holding_validate_next_expected_date( client, librarian_martigny, journal, loc_public_sion, item_type_internal_sion, document, pattern_yearly_two_times_data, json_header, holding_lib_sion_w_patterns_data): """Test create holding with regular frequency and missing the next_expected_date. """ login_user_via_session(client, librarian_martigny.user) holding = holding_lib_sion_w_patterns_data holding['holdings_type'] = 'serial' holding['patterns'] = \ pattern_yearly_two_times_data['patterns'] del holding['pid'] del holding['patterns']['next_expected_date'] # test will fail when the serial holding has no field # next_expected_date for the regular frequency with pytest.raises(ValidationError): Holding.create( data=holding, delete_pid=False, dbcommit=True, reindex=True)
def test_holding_create(db, es_clear, document, org_martigny, loc_public_martigny, item_type_standard_martigny, holding_lib_martigny_data): """Test holding creation.""" holding = Holding.create(holding_lib_martigny_data, dbcommit=True, reindex=True, delete_pid=True) flush_index(HoldingsSearch.Meta.index) assert holding == holding_lib_martigny_data assert holding.get('pid') == '1' holding = Holding.get_record_by_pid('1') assert holding == holding_lib_martigny_data fetched_pid = fetcher(holding.id, holding) assert fetched_pid.pid_value == '1' assert fetched_pid.pid_type == 'hold' search = HoldingsSearch() holding = next(search.filter('term', pid=holding.pid).scan()) holding_record = Holding.get_record_by_pid(holding.pid) assert holding_record.organisation_pid == org_martigny.get('pid')
def bulk_records(records): """Records creation.""" n_updated = 0 n_rejected = 0 n_created = 0 record_schema = current_jsonschemas.path_to_url('documents/document-v0.0.1.json') item_schema = current_jsonschemas.path_to_url('items/item-v0.0.1.json') holding_schema = current_jsonschemas.path_to_url('holdings/holding-v0.0.1.json') host_url = current_app.config.get('RERO_ILS_APP_BASE_URL') url_api = '{host}/api/{doc_type}/{pid}' record_id_iterator = [] item_id_iterator = [] holding_id_iterator = [] indexer = RecordIndexer() start_time = datetime.now() for record in records: try: if record.get('frbr', False): document = record.get('document', {}) """ # check if already in Rero-ILS pid = None for identifier in document.get('identifiedBy') : if identifier.get('source') == 'VIRTUA' : bibid = identifier.get('value') query = DocumentsSearch().filter( 'term', identifiedBy__value=bibid ).source(includes=['pid']) try: pid = [r.pid for r in query.scan()].pop() except IndexError: pid = None if pid: # update the record # Do nothing for the moment continue else: """ document['$schema'] = record_schema created_time = datetime.now() document = Document.create( document, dbcommit=False, reindex=False ) record_id_iterator.append(document.id) uri_documents = url_api.format(host=host_url, doc_type='documents', pid=document.pid) map_holdings = {} for holding in record.get('holdings'): holding['$schema'] = holding_schema holding['document'] = { '$ref': uri_documents } holding['circulation_category'] = { '$ref': map_item_type(str(holding.get('circulation_category'))) } holding['location'] = { '$ref': map_locations(str(holding.get('location'))) } created_time = datetime.now() result = Holding.create( holding, dbcommit=False, reindex=False ) map_holdings.update({ '{location}#{cica}'.format( location = holding.get('location'), cica = holding.get('circulation_category')) : result.get('pid') } ) holding_id_iterator.append(result.id) for item in record.get('items'): item['$schema'] = item_schema item['document'] = { '$ref': uri_documents } item['item_type'] = { '$ref': map_item_type(str(item.get('item_type'))) } item['location'] = { '$ref': map_locations(str(item.get('location'))) } holding_pid = map_holdings.get( '{location}#{cica}'.format( location = item.get('location'), cica = item.get('item_type'))) item['holding'] = { '$ref': url_api.format(host=host_url, doc_type='holdings', pid=holding_pid) } result = Item.create( item, dbcommit=False, reindex=False ) item_id_iterator.append(result.id) n_created += 1 if n_created % 1000 == 0: execution_time = datetime.now() - start_time click.secho('{nb} created records in {execution_time}.' .format(nb=len(record_id_iterator), execution_time=execution_time), fg='white') start_time = datetime.now() db.session.commit() execution_time = datetime.now() - start_time click.secho('{nb} commited records in {execution_time}.' .format(nb=len(record_id_iterator), execution_time=execution_time), fg='white') start_time = datetime.now() click.secho('sending {n} holdings to indexer queue.' .format(n=len(holding_id_iterator)), fg='white') indexer.bulk_index(holding_id_iterator) click.secho('process queue...', fg='yellow') indexer.process_bulk_queue() click.secho('sending {n} items to indexer queue.' .format(n=len(item_id_iterator)), fg='white') indexer.bulk_index(item_id_iterator) click.secho('process queue...', fg='yellow') indexer.process_bulk_queue() click.secho('sending {n} documents to indexer queue.' .format(n=len(record_id_iterator)), fg='white') indexer.bulk_index(record_id_iterator) click.secho('process queue...', fg='yellow') indexer.process_bulk_queue() execution_time = datetime.now() - start_time click.secho('indexing records process in {execution_time}.' .format(execution_time=execution_time), fg='white') click.secho('processing next batch records.', fg='green') record_id_iterator.clear() holding_id_iterator.clear() item_id_iterator.clear() start_time = datetime.now() except Exception as e: n_rejected += 1 click.secho('Error processing record [{id}] : {e}' .format(id=record.get('_id'), e=e), fg='red') db.session.commit() indexer.bulk_index(holding_id_iterator) indexer.process_bulk_queue() indexer.bulk_index(item_id_iterator) indexer.process_bulk_queue() indexer.bulk_index(record_id_iterator) indexer.process_bulk_queue() return n_created