def test_search_pattern_change(app, without_oaiset_signals, schema): """Test search pattern change.""" record0 = create_record(app, { '_oai': {'sets': ['a']}, 'title_statement': {'title': 'Test0'}, '$schema': schema }) rec_uuid = record0.id oaiset = OAISet(spec="a", search_pattern="title_statement.title:Test0") db.session.add(oaiset) db.session.commit() run_after_insert_oai_set() sleep(2) record = Record.get_record(rec_uuid) assert record['_oai']['sets'] == ['a'] # change search pattern: record0 will not inside it anymore oaiset = OAISet.query.first() oaiset.search_pattern = 'title_statement.title:Test1' db.session.merge(oaiset) db.session.commit() after_update_oai_set(None, None, oaiset) sleep(2) record = Record.get_record(rec_uuid) record.commit() assert record['_oai']['sets'] == []
def test_response(db, schema, client): oaiset = OAISet(spec='higgs', name='Higgs', description='...') oaiset.search_pattern = 'title:higgs' db.session.add(oaiset) db.session.add(oaiset) res = client.get('/oai2d?verb=ListSets') print(res.data)
def oai_server(sender, app=None, **kwargs): #app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite://' app.config['CELERY_TASK_ALWAYS_EAGER'] = True if not hasattr(app, 'cli'): from flask_cli import FlaskCLI ext_cli = FlaskCLI(app) ext_db = InvenioDB(app) ext_indexer = InvenioIndexer(app) ext_pidstore = InvenioPIDStore(app) ext_records = InvenioRecords(app) ext_search = InvenioSearch(app) ext_celery = FlaskCeleryExt(app) app.config['OAISERVER_RECORD_INDEX'] = ['authors', 'records'] app.config['OAISERVER_ID_PREFIX'] = 'oai:example:' app.config['OAISERVER_QUERY_PARSER_FIELDS'] = ["title"] app.config['OAISERVER_METADATA_FORMATS'] = { 'oai_dc': { 'serializer': ('invenio_oaiserver.utils:dumps_etree', { 'xslt_filename': pkg_resources.resource_filename( 'testinvenio.records', 'static/xsl/MARC21slim2OAIDC.xsl'), 'xslt_filename': pathlib.Path( "/home/alzbeta/testinvenio/testinvenio/records/static/xsl/MARC21slim2OAIDC.xsl" ) }), 'schema': 'http://json-schema.org/draft-04/schema#', 'namespace': 'http://json-schema.org/draft-04/schema#', } } ext_oaiserver = InvenioOAIServer(app) app.register_blueprint(blueprint) ctx = app.app_context() ctx.push() db.create_all() ext_search.flush_and_refresh('_all') oaiset = OAISet(spec='pattern', name='Pattern', description='...') #nefunguje pokud search pattern ma pole ktere neni v obouch modelech... oaiset.search_pattern = 'title:Some title1' try: print("1") db.session.add(oaiset) db.session.commit() except: pass
def add_set(spec, name, pattern, description='...'): """Add OAI set. :param spec: set identifier :param name: human readable name of the set :param pattern: search pattern to get records :param description: human readable description """ try: oaiset = OAISet(spec=spec, name=name, description=description) oaiset.search_pattern = pattern db.session.add(oaiset) db.session.commit() msg = 'OAIset added: {name}'.format(name=name) except Exception as err: db.session.rollback() msg = 'OAIset exist: {name}'.format(name=name) return msg
def _try_populate_oaisets(): """Try to update collections.""" schema = { 'type': 'object', 'properties': { 'title': { 'type': 'string' }, 'field': { 'type': 'boolean' }, }, 'required': ['title'], } a = OAISet(name="a") b = OAISet(name="b") e = OAISet(name="e", search_pattern="title:Test2 OR title:Test3") c = OAISet(name="c", search_pattern="title:Test0") d = OAISet(name="d", search_pattern="title:Test1") f = OAISet(name="f", search_pattern="title:Test2") g = OAISet(name="g") h = OAISet(name="h") i = OAISet(name="i", search_pattern="title:Test3") j = OAISet(name="j", search_pattern="title:Test4") with db.session.begin_nested(): for coll in [a, b, c, d, e, f, g, h, i, j]: db.session.add(coll) db.session.commit() # start tests record0 = Record.create({'title': 'Test0', '$schema': schema}) assert 'c' in record0['_oaisets'] assert len(record0['_oaisets']) == 1 record = Record.create({'title': 'TestNotFound', '$schema': schema}) assert record['_oaisets'] == [] record = Record.create({'title': 'Test1', '$schema': schema}) assert 'd' in record['_oaisets'] assert len(record['_oaisets']) == 1 record = Record.create({'title': 'Test2', '$schema': schema}) assert 'e' in record['_oaisets'] assert 'f' in record['_oaisets'] assert len(record['_oaisets']) == 2 record3 = Record.create({'title': 'Test3', '$schema': schema}) assert 'e' in record3['_oaisets'] assert 'i' in record3['_oaisets'] assert len(record3['_oaisets']) == 2 record4 = Record.create({'title': 'Test4', '$schema': schema}) assert 'j' in record4['_oaisets'] assert len(record4['_oaisets']) == 1 # test delete db.session.delete(j) db.session.commit() record4.commit() assert 'h' not in record4['_oaisets'] assert 'j' not in record4['_oaisets'] assert len(record4['_oaisets']) == 0 # test update search_pattern i.search_pattern = None db.session.add(i) db.session.commit() record3.commit() assert 'e' in record3['_oaisets'] assert len(record3['_oaisets']) == 1 # test update search_pattern i.search_pattern = 'title:Test3' db.session.add(i) db.session.commit() record3.commit() assert 'e' in record3['_oaisets'] assert 'i' in record3['_oaisets'] assert len(record3['_oaisets']) == 2 # test update name a.name = "new-a" db.session.add(a) db.session.commit() record3.commit() assert 'i' in record3['_oaisets'] assert 'e' in record3['_oaisets'] assert len(record3['_oaisets']) == 2 # test update name c.name = "new-c" db.session.add(c) db.session.commit() record0.commit() assert 'new-c' in record0['_oaisets'] assert len(record0['_oaisets']) == 1
def _try_populate_oaisets(): """Try to update collections.""" schema = { 'type': 'object', 'properties': { 'title': {'type': 'string'}, 'field': {'type': 'boolean'}, }, 'required': ['title'], } a = OAISet(spec="a") b = OAISet(spec="b") e = OAISet( spec="e", search_pattern="title:Test2 OR title:Test3") c = OAISet(spec="c", search_pattern="title:Test0") d = OAISet(spec="d", search_pattern="title:Test1") f = OAISet(spec="f", search_pattern="title:Test2") g = OAISet(spec="g") h = OAISet(spec="h") i = OAISet(spec="i", search_pattern="title:Test3") j = OAISet(spec="j", search_pattern="title:Test4") with db.session.begin_nested(): for coll in [a, b, c, d, e, f, g, h, i, j]: db.session.add(coll) db.session.commit() # start tests record0 = Record.create({'title': 'Test0', '$schema': schema}) assert 'c' in record0['_oai']['sets'] assert len(record0['_oai']['sets']) == 1 record = Record.create({'title': 'TestNotFound', '$schema': schema}) assert record['_oai']['sets'] == [] record = Record.create({'title': 'Test1', '$schema': schema}) assert 'd' in record['_oai']['sets'] assert len(record['_oai']['sets']) == 1 record = Record.create({'title': 'Test2', '$schema': schema}) assert 'e' in record['_oai']['sets'] assert 'f' in record['_oai']['sets'] assert len(record['_oai']['sets']) == 2 record3 = Record.create({'title': 'Test3', '$schema': schema}) assert 'e' in record3['_oai']['sets'] assert 'i' in record3['_oai']['sets'] assert len(record3['_oai']['sets']) == 2 record4 = Record.create({'title': 'Test4', '$schema': schema}) assert 'j' in record4['_oai']['sets'] assert len(record4['_oai']['sets']) == 1 # test delete db.session.delete(j) db.session.commit() record4.commit() assert 'h' not in record4['_oai']['sets'] assert 'j' not in record4['_oai']['sets'] assert len(record4['_oai']['sets']) == 0 # test update search_pattern i.search_pattern = None db.session.add(i) db.session.commit() record3.commit() assert 'e' in record3['_oai']['sets'] assert len(record3['_oai']['sets']) == 1 # test update search_pattern i.search_pattern = 'title:Test3' db.session.add(i) db.session.commit() record3.commit() assert 'e' in record3['_oai']['sets'] assert 'i' in record3['_oai']['sets'] assert len(record3['_oai']['sets']) == 2 # test update the spec a.spec = "new-a" db.session.add(a) db.session.commit() record3.commit() assert 'i' in record3['_oai']['sets'] assert 'e' in record3['_oai']['sets'] assert len(record3['_oai']['sets']) == 2 # test update name c.spec = "new-c" db.session.add(c) db.session.commit() record0.commit() assert 'new-c' in record0['_oai']['sets'] assert len(record0['_oai']['sets']) == 1
def test_populate_oaisets(app, with_record_signals): """Populate OAISets.""" indexer = RecordIndexer() schema = { 'allOf': [{ 'type': 'object', 'properties': { 'title': {'type': 'string'}, 'genre': {'type': 'string'}, 'field': {'type': 'boolean'}, }, 'required': ['title'], }, { '$ref': 'http://inveniosoftware.org/schemas/' 'oaiserver/internal-v1.0.0.json', }] } def create_record(item_dict, mint_oaiid=True): """Create test record.""" with app.test_request_context(): record_id = uuid.uuid4() recid_minter(record_id, item_dict) if mint_oaiid: oaiid_minter(record_id, item_dict) record = Record.create(item_dict, id_=record_id) indexer.index(record) return record a = OAISet(spec='a') b = OAISet(spec='b') e = OAISet( spec="e", search_pattern="title:Test2 OR title:Test3") c = OAISet(spec="c", search_pattern="title:Test0") d = OAISet(spec="d", search_pattern="title:Test1") f = OAISet(spec="f", search_pattern="title:Test2") g = OAISet(spec="g") h = OAISet(spec="h") i = OAISet(spec="i", search_pattern="title:Test3") j = OAISet(spec="j with space", search_pattern="title:Test4") # Note below: brackets around AND search query are required l = OAISet(spec="math", search_pattern="(title:foo AND genre:math)") m = OAISet(spec="nonmath", search_pattern="(title:foo AND -genre:math)") with db.session.begin_nested(): for oaiset in [a, b, c, d, e, f, g, h, i, j, l, m]: db.session.add(oaiset) db.session.commit() a_id = a.id i_id = i.id # start tests record0 = create_record({ '_oai': {'sets': ['a']}, 'title': 'Test0', '$schema': schema }) assert 'a' in record0['_oai']['sets'], 'Keep manually managed set "a".' assert 'c' in record0['_oai']['sets'] assert len(record0['_oai']['sets']) == 2 record_not_found = create_record( {'title': 'TestNotFound', '$schema': schema} ) # Don't create empty sets list just because of commit assert 'sets' not in record_not_found['_oai'] record1 = create_record({'title': 'Test1', '$schema': schema}) assert 'd' in record1['_oai']['sets'] assert len(record1['_oai']['sets']) == 1 record2 = create_record({'title': 'Test2', '$schema': schema}) record2_id = record2.id assert 'e' in record2['_oai']['sets'] assert 'f' in record2['_oai']['sets'] assert len(record2['_oai']['sets']) == 2 record3 = create_record({'title': 'Test3', '$schema': schema}) record3_id = record3.id assert 'e' in record3['_oai']['sets'] assert 'i' in record3['_oai']['sets'] assert len(record3['_oai']['sets']) == 2 record4 = create_record({'title': 'Test4', '$schema': schema}) record4_id = record4.id assert 'j with space' in record4['_oai']['sets'] assert len(record4['_oai']['sets']) == 1 # If record does not have '_oai', don't add any sets, # nor even the default '_oai' key record5 = create_record({'title': 'Test1', '$schema': schema}, mint_oaiid=False) assert '_oai' not in record5 # If 'sets' before and after record commit are equivalent # don't bump up the '_oai.updated' timestamp... record6 = create_record({'title': 'Test1', '$schema': schema}) assert record6['_oai']['sets'] == ['d'] prev_updated_r6 = record6['_oai']['updated'] record6.commit() assert record6['_oai']['sets'] == ['d'] assert record6['_oai']['updated'] == prev_updated_r6 # date stays the same # ...but do bump up '_oai.updated' if the sets are different record7 = create_record({'title': 'Test1', '$schema': schema}) assert record7['_oai']['sets'] == ['d'] prev_updated_r7 = record7['_oai']['updated'] sleep(1) # 'updated' timestamp is accurate to a second, hence the wait record7['_oai']['sets'] = ['d', 'f'] # 'f' should be removed after commit record7.commit() assert record7['_oai']['sets'] == ['d'] assert record7['_oai']['updated'] != prev_updated_r7 # date bumped # Test 'AND' keyword for records record8 = create_record( {'title': 'foo', 'genre': 'math', '$schema': schema}) assert record8['_oai']['sets'] == ['math', ] record9 = create_record( {'title': 'foo', 'genre': 'physics', '$schema': schema}) assert record9['_oai']['sets'] == ['nonmath', ] record10 = create_record( {'title': 'bar', 'genre': 'math', '$schema': schema}) assert 'sets' not in record10['_oai'] # title is not 'foo' # wait ElasticSearch end to index records sleep(10) # test delete current_oaiserver.unregister_signals_oaiset() with patch('invenio_oaiserver.receivers.after_delete_oai_set') as f: current_oaiserver.register_signals_oaiset() with db.session.begin_nested(): db.session.delete(j) db.session.commit() assert f.called after_delete_oai_set(None, None, j) record4_model = RecordMetadata.query.filter_by( id=record4_id).first().json assert 'j with space' not in record4_model['_oai']['sets'] assert len(record4_model['_oai']['sets']) == 0 current_oaiserver.unregister_signals_oaiset() # test update search_pattern with patch('invenio_oaiserver.receivers.after_update_oai_set') as f: current_oaiserver.register_signals_oaiset() with db.session.begin_nested(): i.search_pattern = None assert current_oaiserver.sets is None, 'Cache should be empty.' db.session.merge(i) db.session.commit() assert f.called i = OAISet.query.get(i_id) after_update_oai_set(None, None, i) record3_model = RecordMetadata.query.filter_by( id=record3_id).first().json assert 'i' in record3['_oai']['sets'], 'Set "i" is manually managed.' assert 'e' in record3_model['_oai']['sets'] assert len(record3_model['_oai']['sets']) == 2 current_oaiserver.unregister_signals_oaiset() # test update search_pattern with patch('invenio_oaiserver.receivers.after_update_oai_set') as f: current_oaiserver.register_signals_oaiset() with db.session.begin_nested(): i.search_pattern = 'title:Test3' db.session.merge(i) db.session.commit() assert f.called i = OAISet.query.get(i_id) after_update_oai_set(None, None, i) record3_model = RecordMetadata.query.filter_by( id=record3_id).first().json assert 'e' in record3_model['_oai']['sets'] assert 'i' in record3_model['_oai']['sets'] assert len(record3_model['_oai']['sets']) == 2 current_oaiserver.unregister_signals_oaiset() # test update the spec with pytest.raises(OAISetSpecUpdateError) as exc_info: a = OAISet.query.get(a_id) a.spec = 'new-a' assert exc_info.type is OAISetSpecUpdateError # test create new set with patch('invenio_oaiserver.receivers.after_insert_oai_set') as f: current_oaiserver.register_signals_oaiset() with db.session.begin_nested(): k = OAISet(spec="k", search_pattern="title:Test2") db.session.add(k) db.session.commit() assert f.called after_insert_oai_set(None, None, k) record2_model = RecordMetadata.query.filter_by( id=record2_id).first().json assert 'e' in record2_model['_oai']['sets'] assert 'f' in record2_model['_oai']['sets'] assert 'k' in record2_model['_oai']['sets'] assert len(record2_model['_oai']['sets']) == 3 current_oaiserver.register_signals_oaiset()
def _try_populate_oaisets(): """Try to update collections.""" indexer = RecordIndexer() schema = { 'type': 'object', 'properties': { 'title': {'type': 'string'}, 'field': {'type': 'boolean'}, }, 'required': ['title'], } def create_record(item_dict): """Create test record.""" record_id = uuid.uuid4() recid_minter(record_id, item_dict) oaiid_minter(record_id, item_dict) record = Record.create(item_dict, id_=record_id) indexer.index(record) return record a = OAISet(spec='a') b = OAISet(spec='b') e = OAISet( spec="e", search_pattern="title:Test2 OR title:Test3") c = OAISet(spec="c", search_pattern="title:Test0") d = OAISet(spec="d", search_pattern="title:Test1") f = OAISet(spec="f", search_pattern="title:Test2") g = OAISet(spec="g") h = OAISet(spec="h") i = OAISet(spec="i", search_pattern="title:Test3") j = OAISet(spec="j with space", search_pattern="title:Test4") with db.session.begin_nested(): for oaiset in [a, b, c, d, e, f, g, h, i, j]: db.session.add(oaiset) db.session.commit() a_id = a.id i_id = i.id # start tests record0 = create_record({ '_oai': {'sets': ['a']}, 'title': 'Test0', '$schema': schema }) assert 'a' in record0['_oai']['sets'], 'Keep manually managed set "a".' assert 'c' in record0['_oai']['sets'] assert len(record0['_oai']['sets']) == 2 record_not_found = create_record( {'title': 'TestNotFound', '$schema': schema} ) assert record_not_found['_oai']['sets'] == [] record1 = create_record({'title': 'Test1', '$schema': schema}) assert 'd' in record1['_oai']['sets'] assert len(record1['_oai']['sets']) == 1 record2 = create_record({'title': 'Test2', '$schema': schema}) record2_id = record2.id assert 'e' in record2['_oai']['sets'] assert 'f' in record2['_oai']['sets'] assert len(record2['_oai']['sets']) == 2 record3 = create_record({'title': 'Test3', '$schema': schema}) record3_id = record3.id assert 'e' in record3['_oai']['sets'] assert 'i' in record3['_oai']['sets'] assert len(record3['_oai']['sets']) == 2 record4 = create_record({'title': 'Test4', '$schema': schema}) record4_id = record4.id assert 'j with space' in record4['_oai']['sets'] assert len(record4['_oai']['sets']) == 1 # wait ElasticSearch end to index records sleep(10) # test delete current_oaiserver.unregister_signals_oaiset() with patch('invenio_oaiserver.receivers.after_delete_oai_set') as f: current_oaiserver.register_signals_oaiset() with db.session.begin_nested(): db.session.delete(j) db.session.commit() assert f.called after_delete_oai_set(None, None, j) record4_model = RecordMetadata.query.filter_by( id=record4_id).first().json assert 'j with space' not in record4_model['_oai']['sets'] assert len(record4_model['_oai']['sets']) == 0 current_oaiserver.unregister_signals_oaiset() # test update search_pattern with patch('invenio_oaiserver.receivers.after_update_oai_set') as f: current_oaiserver.register_signals_oaiset() with db.session.begin_nested(): i.search_pattern = None assert current_oaiserver.sets is None, 'Cache should be empty.' db.session.merge(i) db.session.commit() assert f.called i = OAISet.query.get(i_id) after_update_oai_set(None, None, i) record3_model = RecordMetadata.query.filter_by( id=record3_id).first().json assert 'i' in record3['_oai']['sets'], 'Set "i" is manually managed.' assert 'e' in record3_model['_oai']['sets'] assert len(record3_model['_oai']['sets']) == 2 current_oaiserver.unregister_signals_oaiset() # test update search_pattern with patch('invenio_oaiserver.receivers.after_update_oai_set') as f: current_oaiserver.register_signals_oaiset() with db.session.begin_nested(): i.search_pattern = 'title:Test3' db.session.merge(i) db.session.commit() assert f.called i = OAISet.query.get(i_id) after_update_oai_set(None, None, i) record3_model = RecordMetadata.query.filter_by( id=record3_id).first().json assert 'e' in record3_model['_oai']['sets'] assert 'i' in record3_model['_oai']['sets'] assert len(record3_model['_oai']['sets']) == 2 current_oaiserver.unregister_signals_oaiset() # test update the spec with pytest.raises(OAISetSpecUpdateError) as exc_info: a = OAISet.query.get(a_id) a.spec = 'new-a' assert exc_info.type is OAISetSpecUpdateError # test create new set with patch('invenio_oaiserver.receivers.after_insert_oai_set') as f: current_oaiserver.register_signals_oaiset() with db.session.begin_nested(): k = OAISet(spec="k", search_pattern="title:Test2") db.session.add(k) db.session.commit() assert f.called after_insert_oai_set(None, None, k) record2_model = RecordMetadata.query.filter_by( id=record2_id).first().json assert 'e' in record2_model['_oai']['sets'] assert 'f' in record2_model['_oai']['sets'] assert 'k' in record2_model['_oai']['sets'] assert len(record2_model['_oai']['sets']) == 3 current_oaiserver.register_signals_oaiset()