def test_search_pattern_change(app, without_oaiset_signals, schema): """Test search pattern change.""" record0 = create_record(app, { '_oai': {'sets': ['a']}, 'title_statement': {'title': 'Test0'}, '$schema': schema }) rec_uuid = record0.id oaiset = OAISet(spec="a", search_pattern="title_statement.title:Test0") db.session.add(oaiset) db.session.commit() run_after_insert_oai_set() sleep(2) record = Record.get_record(rec_uuid) assert record['_oai']['sets'] == ['a'] # change search pattern: record0 will not inside it anymore oaiset = OAISet.query.first() oaiset.search_pattern = 'title_statement.title:Test1' db.session.merge(oaiset) db.session.commit() after_update_oai_set(None, None, oaiset) sleep(2) record = Record.get_record(rec_uuid) record.commit() assert record['_oai']['sets'] == []
def update_sets(): """Check that each community has a corresponding oai set""" for community in Community.get_all(): dirty = False oaiset = OAISet.query.filter(OAISet.spec == str(community.id)).one_or_none() if not oaiset: click.secho('Adding new oai set {}'.format(community.name), fg='yellow', bold=True) oaiset = OAISet(spec=str(community.id), name=community.name, description=community.description) dirty = True else: if oaiset.name != community.name: oaiset.name = community.name dirty = True click.secho('Update name for set {}'.format(oaiset.spec), fg='yellow', bold=True) if oaiset.description != community.description: oaiset.description = community.description dirty = True click.secho('Update description for set {}'.format(oaiset.spec), fg='yellow', bold=True) if dirty: db.session.add(oaiset) db.session.commit()
def update_sets(): """Check that each community has a corresponding oai set""" for community in Community.get_all(): dirty = False oaiset = OAISet.query.filter( OAISet.spec == str(community.id)).one_or_none() if not oaiset: click.secho('Adding new oai set {}'.format(community.name), fg='yellow', bold=True) oaiset = OAISet(spec=str(community.id), name=community.name, description=community.description) dirty = True else: if oaiset.name != community.name: oaiset.name = community.name dirty = True click.secho('Update name for set {}'.format(oaiset.spec), fg='yellow', bold=True) if oaiset.description != community.description: oaiset.description = community.description dirty = True click.secho('Update description for set {}'.format( oaiset.spec), fg='yellow', bold=True) if dirty: db.session.add(oaiset) db.session.commit()
def test_response(db, schema, client): oaiset = OAISet(spec='higgs', name='Higgs', description='...') oaiset.search_pattern = 'title:higgs' db.session.add(oaiset) db.session.add(oaiset) res = client.get('/oai2d?verb=ListSets') print(res.data)
def test_oaiset_add_remove_record(app): """Test the API method for manual record adding.""" with app.app_context(): oaiset1 = OAISet(spec='abc') rec1 = Record.create({'title': 'Test1'}) oaiset1.add_record(rec1) assert 'abc' in rec1['_oai']['sets'] assert 'updated' in rec1['_oai'] dt1 = iso2dt(rec1['_oai']['updated']) assert dt1.year == datetime.utcnow().year # Test if parsed OK oaiset1.remove_record(rec1) assert 'abc' not in rec1['_oai']['sets'] dt2 = iso2dt(rec1['_oai']['updated']) assert dt2 >= dt1
def oai_server(sender, app=None, **kwargs): #app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite://' app.config['CELERY_TASK_ALWAYS_EAGER'] = True if not hasattr(app, 'cli'): from flask_cli import FlaskCLI ext_cli = FlaskCLI(app) ext_db = InvenioDB(app) ext_indexer = InvenioIndexer(app) ext_pidstore = InvenioPIDStore(app) ext_records = InvenioRecords(app) ext_search = InvenioSearch(app) ext_celery = FlaskCeleryExt(app) app.config['OAISERVER_RECORD_INDEX'] = ['authors', 'records'] app.config['OAISERVER_ID_PREFIX'] = 'oai:example:' app.config['OAISERVER_QUERY_PARSER_FIELDS'] = ["title"] app.config['OAISERVER_METADATA_FORMATS'] = { 'oai_dc': { 'serializer': ('invenio_oaiserver.utils:dumps_etree', { 'xslt_filename': pkg_resources.resource_filename( 'testinvenio.records', 'static/xsl/MARC21slim2OAIDC.xsl'), 'xslt_filename': pathlib.Path( "/home/alzbeta/testinvenio/testinvenio/records/static/xsl/MARC21slim2OAIDC.xsl" ) }), 'schema': 'http://json-schema.org/draft-04/schema#', 'namespace': 'http://json-schema.org/draft-04/schema#', } } ext_oaiserver = InvenioOAIServer(app) app.register_blueprint(blueprint) ctx = app.app_context() ctx.push() db.create_all() ext_search.flush_and_refresh('_all') oaiset = OAISet(spec='pattern', name='Pattern', description='...') #nefunguje pokud search pattern ma pole ktere neni v obouch modelech... oaiset.search_pattern = 'title:Some title1' try: print("1") db.session.add(oaiset) db.session.commit() except: pass
def test_list_sets_long(app): """Test listing of sets.""" from invenio_db import db from invenio_oaiserver.models import OAISet with app.app_context(): current_oaiserver.unregister_signals_oaiset() with db.session.begin_nested(): for i in range(27): oaiset = OAISet( spec='test{0}'.format(i), name='Test{0}'.format(i), description='test desc {0}'.format(i), search_pattern='title_statement.title:Test{0}'.format(i), ) db.session.add(oaiset) db.session.commit() run_after_insert_oai_set() with app.test_client() as c: # First page: result = c.get('/oai2d?verb=ListSets') tree = etree.fromstring(result.data) assert len(tree.xpath('/x:OAI-PMH/x:ListSets/x:set', namespaces=NAMESPACES)) == 10 resumption_token = tree.xpath( '/x:OAI-PMH/x:ListSets/x:resumptionToken', namespaces=NAMESPACES )[0] assert resumption_token.text # Second page: result = c.get('/oai2d?verb=ListSets&resumptionToken={0}'.format( resumption_token.text )) tree = etree.fromstring(result.data) assert len(tree.xpath('/x:OAI-PMH/x:ListSets/x:set', namespaces=NAMESPACES)) == 10 resumption_token = tree.xpath( '/x:OAI-PMH/x:ListSets/x:resumptionToken', namespaces=NAMESPACES )[0] assert resumption_token.text # Third page: result = c.get('/oai2d?verb=ListSets&resumptionToken={0}'.format( resumption_token.text )) tree = etree.fromstring(result.data) assert len(tree.xpath('/x:OAI-PMH/x:ListSets/x:set', namespaces=NAMESPACES)) == 7 resumption_token = tree.xpath( '/x:OAI-PMH/x:ListSets/x:resumptionToken', namespaces=NAMESPACES )[0] assert not resumption_token.text
def test_oai_with_for_arxiv_set(inspire_app, clean_celery_session): data = { "arxiv_eprints": [{"value": "2009.01484"}], "report_numbers": [{"value": "CERN-TH-2020-136"}], } record_data = faker.record("lit", data) record = LiteratureRecord.create(record_data) record_uuid = record.id record_marcxml = record2marcxml(record) db.session.commit() def assert_the_record_is_indexed(): current_search.flush_and_refresh("*") result = es_search("records-hep") uuids = get_value(result, "hits.hits._id") assert str(record_uuid) in uuids retry_until_pass(assert_the_record_is_indexed) set_name = inspire_app.config["OAI_SET_CERN_ARXIV"] oaiset = OAISet(spec=f"{set_name}", name="Test", description="Test") db.session.add(oaiset) db.session.commit() with inspire_app.test_client() as client: response = client.get( f"/api/oai2d?verb=ListRecords&metadataPrefix=marcxml&set={set_name}" ) assert record_marcxml in response.data
def test_oai_get_single_identifier_for_arxiv_set(inspire_app, celery_app_with_context, celery_session_worker): data = { "arxiv_eprints": [{ "value": "2009.01484" }], "report_numbers": [{ "value": "CERN-TH-2020-136" }], } record_data = faker.record("lit", data) record = LiteratureRecord.create(record_data) record_marcxml = record2marcxml(record) db.session.commit() set_name = inspire_app.config["OAI_SET_CERN_ARXIV"] oaiset = OAISet(spec=f"{set_name}", name="Test", description="Test") db.session.add(oaiset) db.session.commit() sleep(2) with inspire_app.test_client() as client: response = client.get( f"/api/oai2d?verb=GetRecord&metadataPrefix=marcxml&identifier=oai:inspirehep.net:{record['control_number']}" ) assert record_marcxml in response.data
def submit_set(): """Insert a new set.""" form = get_NewSetForm(request.form) if request.method == 'POST' and form.validate(): new_set = OAISet(spec=form.spec.data, name=form.name.data, description=form.description.data, search_pattern=form.search_pattern.data, parent=form.parent.data) db.session.add(new_set) # this shoul be moved to UPDATER (celery task) and it sould always # take care of adding records to sets. ########## query = Query(form.query.data) response = current_search_client.search( index='records', # make configurable PER SET doc_type='record', # make configurable PER SET body=query.body, fields='_id, oaiid' # path to oaiid as a configurable ) ids = [(a['_id'], a['oaiid']) for a in response['hits']['hits']] add_records_to_set(ids) ######### db.session.commit() flash(_('New set %(spec)s was added.', spec=new_set.spec)) return redirect(url_for('.manage_sets')) return render_template('make_set.html', new_set_form=form)
def test_oai_get_single_identifier_for_CDS_set(inspire_app, clean_celery_session): data = {"_export_to": {"CDS": True}} record_data = faker.record("lit", data) record = LiteratureRecord.create(record_data) record_uuid = record.id record_marcxml = record2marcxml(record) db.session.commit() def assert_the_record_is_indexed(): current_search.flush_and_refresh("*") result = es_search("records-hep") uuids = get_value(result, "hits.hits._id") assert str(record_uuid) in uuids retry_until_pass(assert_the_record_is_indexed) set_name = inspire_app.config["OAI_SET_CDS"] oaiset = OAISet(spec=f"{set_name}", name="Test", description="Test") db.session.add(oaiset) db.session.commit() with inspire_app.test_client() as client: response = client.get( f"/api/oai2d?verb=GetRecord&metadataPrefix=marcxml&identifier=oai:inspirehep.net:{record['control_number']}" ) assert record_marcxml in response.data
def create_oaiset(name, title_pattern): oaiset = OAISet(spec=name, search_pattern=f"title_statement.title:{title_pattern}") db.session.add(oaiset) db.session.commit() run_after_insert_oai_set() return oaiset
def create_oaipmh_set(mapper, connection, community): """Signal for creating OAI-PMH sets during community creation.""" from invenio_oaiserver.models import OAISet with db.session.begin_nested(): obj = OAISet(spec=community.oaiset_spec, name=community.title, description=community.description) db.session.add(obj)
def test_list_sets_long(app): """Test listing of sets.""" from invenio_db import db from invenio_oaiserver.models import OAISet with app.app_context(): with db.session.begin_nested(): for i in range(27): db.session.add(OAISet( spec='test{0}'.format(i), name='Test{0}'.format(i), description='test desc {0}'.format(i), search_pattern='title:Test{0}'.format(i), )) db.session.commit() namespaces = {'x': NS_OAIPMH} with app.test_client() as c: # First page: result = c.get('/oai2d?verb=ListSets') tree = etree.fromstring(result.data) assert len(tree.xpath('/x:OAI-PMH/x:ListSets/x:set', namespaces=namespaces)) == 10 resumption_token = tree.xpath( '/x:OAI-PMH/x:ListSets/x:resumptionToken', namespaces=namespaces )[0] assert resumption_token.text # Second page: result = c.get('/oai2d?verb=ListSets&resumptionToken={0}'.format( resumption_token.text )) tree = etree.fromstring(result.data) assert len(tree.xpath('/x:OAI-PMH/x:ListSets/x:set', namespaces=namespaces)) == 10 resumption_token = tree.xpath( '/x:OAI-PMH/x:ListSets/x:resumptionToken', namespaces=namespaces )[0] assert resumption_token.text # Third page: result = c.get('/oai2d?verb=ListSets&resumptionToken={0}'.format( resumption_token.text )) tree = etree.fromstring(result.data) assert len(tree.xpath('/x:OAI-PMH/x:ListSets/x:set', namespaces=namespaces)) == 7 resumption_token = tree.xpath( '/x:OAI-PMH/x:ListSets/x:resumptionToken', namespaces=namespaces )[0] assert not resumption_token.text
def add_set(spec, name, pattern, description='...'): """Add OAI set. :param spec: set identifier :param name: human readable name of the set :param pattern: search pattern to get records :param description: human readable description """ try: oaiset = OAISet(spec=spec, name=name, description=description) oaiset.search_pattern = pattern db.session.add(oaiset) db.session.commit() msg = 'OAIset added: {name}'.format(name=name) except Exception as err: db.session.rollback() msg = 'OAIset exist: {name}'.format(name=name) return msg
def test_admin(app): """Test Flask-Admin interace.""" admin = Admin(app, name="Test") assert "model" in set_adminview assert "modelview" in set_adminview # Register both models in admin model = set_adminview.pop("model") view = set_adminview.pop("modelview") admin.add_view(view(model, db.session, **set_adminview)) # Check if generated admin menu contains the correct items menu_items = {str(item.name): item for item in admin.menu()} assert "OAI-PMH" in menu_items assert menu_items["OAI-PMH"].is_category() submenu_items = { str(item.name): item for item in menu_items["OAI-PMH"].get_children() } assert "Sets" in submenu_items assert isinstance(submenu_items["Sets"], menu.MenuView) # Create a test set. with app.app_context(): test_set = OAISet( id=1, spec="test", name="test_name", description="some test description", search_pattern="title_statement.title:Test0", ) db.session.add(test_set) db.session.commit() with app.test_request_context(): index_view_url = url_for("oaiset.index_view") delete_view_url = url_for("oaiset.delete_view") detail_view_url = url_for("oaiset.details_view", id=1) with app.test_client() as client: # List index view and check record is there. res = client.get(index_view_url) assert res.status_code == 200 # Deletion is forbiden. res = client.post(delete_view_url, data={"id": 1}, follow_redirects=True) assert res.status_code == 200 # View the deleted record. res = client.get(detail_view_url) assert res.status_code == 200 assert 1 == OAISet.query.count()
def create_community_oaiset(community, fix=False): """Create a community's OAI set. :param community: the community for which we add the OAI set. :param fix: Enable the fixing of the database. This function doesn't fail if the oai set already exists. It will just add it if it is missing. """ oaiset = OAISet(spec=str(community.id), name=community.name, description=community.description) add_to_db(oaiset, skip_if_exists=fix)
def oaiserver(sets, records): """Initialize OAI-PMH server.""" from invenio_db import db from invenio_oaiserver.models import OAISet from invenio_records.api import Record # create a OAI Set with db.session.begin_nested(): for i in range(sets): db.session.add( OAISet( spec='test{0}'.format(i), name='Test{0}'.format(i), description='test desc {0}'.format(i), search_pattern='title_statement.title:Test{0}'.format(i), )) # create a record schema = { 'type': 'object', 'properties': { 'title_statement': { 'type': 'object', 'properties': { 'title': { 'type': 'string', }, }, }, 'field': { 'type': 'boolean' }, }, } search.client.indices.delete_alias('_all', '_all', ignore=[400, 404]) search.client.indices.delete('*') with app.app_context(): indexer = RecordIndexer() with db.session.begin_nested(): for i in range(records): record_id = uuid.uuid4() data = { 'title_statement': { 'title': 'Test{0}'.format(i) }, '$schema': schema, } recid_minter(record_id, data) oaiid_minter(record_id, data) record = Record.create(data, id_=record_id) indexer.index(record) db.session.commit()
def test_admin(app): """Test Flask-Admin interace.""" admin = Admin(app, name='Test') assert 'model' in set_adminview assert 'modelview' in set_adminview # Register both models in admin model = set_adminview.pop('model') view = set_adminview.pop('modelview') admin.add_view(view(model, db.session, **set_adminview)) # Check if generated admin menu contains the correct items menu_items = {str(item.name): item for item in admin.menu()} assert 'OAI-PMH' in menu_items assert menu_items['OAI-PMH'].is_category() submenu_items = { str(item.name): item for item in menu_items['OAI-PMH'].get_children() } assert 'Sets' in submenu_items assert isinstance(submenu_items['Sets'], menu.MenuView) # Create a test set. with app.app_context(): test_set = OAISet(id=1, spec='test', name='test_name', description='some test description', search_pattern='test search') db.session.add(test_set) db.session.commit() with app.test_request_context(): index_view_url = url_for('oaiset.index_view') delete_view_url = url_for('oaiset.delete_view') detail_view_url = url_for('oaiset.details_view', id=1) with app.test_client() as client: # List index view and check record is there. res = client.get(index_view_url) assert res.status_code == 200 # Deletion is forbiden. res = client.post(delete_view_url, data={'id': 1}, follow_redirects=True) assert res.status_code == 200 # View the deleted record. res = client.get(detail_view_url) assert res.status_code == 200 assert 1 == OAISet.query.count()
def create(cls, data, id_=None, dbcommit=False, with_bucket=True, **kwargs): """Create organisation record.""" # Create OAI set oaiset = OAISet(spec=data['code'], name=data['name']) db.session.add(oaiset) return super(OrganisationRecord, cls).create(data, id_=id_, dbcommit=dbcommit, with_bucket=with_bucket, **kwargs)
def test_oaiset_add_remove_record(app): """Test the API method for manual record adding.""" with app.app_context(): oaiset1 = OAISet(spec='abc') rec1 = Record.create({'title_statement': {'title': 'Test1'}}) assert not oaiset1.has_record(rec1) oaiset1.add_record(rec1) assert 'abc' in rec1['_oai']['sets'] assert 'updated' in rec1['_oai'] assert oaiset1.has_record(rec1) dt1 = iso2dt(rec1['_oai']['updated']) assert dt1.year == datetime.utcnow().year # Test if parsed OK oaiset1.remove_record(rec1) assert 'abc' not in rec1['_oai']['sets'] assert not oaiset1.has_record(rec1) dt2 = iso2dt(rec1['_oai']['updated']) assert dt2 >= dt1
def loadoaisets(force=False): """Load default file store location.""" sets = [ ('openaire', 'OpenAIRE', None), ('openaire_data', 'OpenAIRE data sets', None), ] try: for setid, name, pattern in sets: oset = OAISet(spec=setid, name=name, search_pattern=pattern) db.session.add(oset) db.session.commit() return len(sets) except Exception: db.session.rollback() raise
def test_listsets_invalid_name(app): """Test ListSets with invalid unicode character for XML.""" with app.test_request_context(): current_oaiserver.unregister_signals_oaiset() with db.session.begin_nested(): a = OAISet(spec='test', name=u'uni\x01co\x0bde', description='test desc') db.session.add(a) with app.test_client() as c: result = c.get('/oai2d?verb=ListSets') tree = etree.fromstring(result.data) assert tree.xpath('/x:OAI-PMH/x:ListSets/x:set/x:setName', namespaces=NAMESPACES)[0].text == 'unicode'
def test_oaiset_add_remove_record(app): """Test the API method for manual record adding.""" with app.app_context(): oaiset1 = OAISet(spec='abc') rec1 = Record.create({'title_statement': {'title': 'Test1'}}) rec1.commit() # Adding a record to an OAIset should change the record's updated date dt1 = rec1.updated assert not oaiset1.has_record(rec1) oaiset1.add_record(rec1) assert 'abc' in rec1['_oai']['sets'] assert oaiset1.has_record(rec1) rec1.commit() dt2 = rec1.updated assert dt2 > dt1 oaiset1.remove_record(rec1) rec1.commit() dt3 = rec1.updated assert 'abc' not in rec1['_oai']['sets'] assert not oaiset1.has_record(rec1) assert dt3 > dt2
def test_listsets(app): """Test ListSets.""" with app.test_request_context(): current_oaiserver.unregister_signals_oaiset() with db.session.begin_nested(): a = OAISet(spec='test', name='Test', description='test desc') db.session.add(a) with app.test_client() as c: result = c.get('/oai2d?verb=ListSets') tree = etree.fromstring(result.data) assert len(tree.xpath('/x:OAI-PMH', namespaces=NAMESPACES)) == 1 assert len(tree.xpath('/x:OAI-PMH/x:ListSets', namespaces=NAMESPACES)) == 1 assert len( tree.xpath('/x:OAI-PMH/x:ListSets/x:set', namespaces=NAMESPACES)) == 1 assert len( tree.xpath('/x:OAI-PMH/x:ListSets/x:set/x:setSpec', namespaces=NAMESPACES)) == 1 assert len( tree.xpath('/x:OAI-PMH/x:ListSets/x:set/x:setName', namespaces=NAMESPACES)) == 1 assert len( tree.xpath('/x:OAI-PMH/x:ListSets/x:set/x:setDescription', namespaces=NAMESPACES)) == 1 assert len( tree.xpath('/x:OAI-PMH/x:ListSets/x:set/x:setDescription/y:dc', namespaces=NAMESPACES)) == 1 assert len( tree.xpath( '/x:OAI-PMH/x:ListSets/x:set/x:setDescription/y:dc/' 'z:description', namespaces=NAMESPACES)) == 1 text = tree.xpath( '/x:OAI-PMH/x:ListSets/x:set/x:setDescription/y:dc/' 'z:description/text()', namespaces=NAMESPACES) assert len(text) == 1 assert text[0] == 'test desc'
def test_listsets(app): """Test ListSets.""" with app.test_request_context(): with db.session.begin_nested(): a = OAISet(spec='test', name='Test', description="test desc") db.session.add(a) with app.test_client() as c: result = c.get('/oai2d?verb=ListSets') tree = etree.fromstring(result.data) namespaces = {'x': NS_OAIPMH} assert len(tree.xpath('/x:OAI-PMH', namespaces=namespaces)) == 1 assert len(tree.xpath('/x:OAI-PMH/x:ListSets', namespaces=namespaces)) == 1 assert len(tree.xpath('/x:OAI-PMH/x:ListSets/x:set', namespaces=namespaces)) == 1 assert len(tree.xpath('/x:OAI-PMH/x:ListSets/x:set/x:setSpec', namespaces=namespaces)) == 1 assert len(tree.xpath('/x:OAI-PMH/x:ListSets/x:set/x:setName', namespaces=namespaces)) == 1 assert len(tree.xpath( '/x:OAI-PMH/x:ListSets/x:set/x:setDescription', namespaces=namespaces )) == 1 namespaces['y'] = NS_OAIDC assert len( tree.xpath('/x:OAI-PMH/x:ListSets/x:set/x:setDescription/y:dc', namespaces=namespaces) ) == 1 namespaces['z'] = NS_DC assert len( tree.xpath('/x:OAI-PMH/x:ListSets/x:set/x:setDescription/y:dc/' 'z:description', namespaces=namespaces) ) == 1 text = tree.xpath( '/x:OAI-PMH/x:ListSets/x:set/x:setDescription/y:dc/' 'z:description/text()', namespaces=namespaces) assert len(text) == 1 assert text[0] == 'test desc'
def test_oai_with_for_cds_set(inspire_app, celery_app_with_context, celery_session_worker): data = {"_export_to": {"CDS": True}} record_data = faker.record("lit", data) record = LiteratureRecord.create(record_data) record_marcxml = record2marcxml(record) db.session.commit() set_name = inspire_app.config["OAI_SET_CDS"] oaiset = OAISet(spec=f"{set_name}", name="Test", description="Test") db.session.add(oaiset) db.session.commit() sleep(2) with inspire_app.test_client() as client: response = client.get( f"/api/oai2d?verb=ListRecords&metadataPrefix=marcxml&set={set_name}" ) assert record_marcxml in response.data
def oaisets(db, communities): """Create custom OAISet objects. Those should be custom OAISet objects which are not community based. """ oaisets_data = [ { 'spec': 'extra', 'search_pattern': 'title:extra' }, { 'spec': 'user-extra', # Looks like a community-based OAISet 'search_pattern': 'title:foobar' # but has a search_pattern }, ] for oai_data in oaisets_data: obj = OAISet(**oai_data) db.session.add(obj) db.session.commit() oai_data['id'] = obj.id return oaisets_data
def oaiserver(number): """Initialize OAI-PMH server.""" from invenio_db import db from invenio_oaiserver.models import OAISet from invenio_records.api import Record # create a OAI Set with db.session.begin_nested(): for i in range(number): db.session.add( OAISet( spec='test{0}'.format(i), name='Test{0}'.format(i), description='test desc {0}'.format(i), search_pattern='title:Test{0}'.format(i), )) # create a record schema = { 'type': 'object', 'properties': { 'title': { 'type': 'string' }, 'field': { 'type': 'boolean' }, }, 'required': ['title'], } with db.session.begin_nested(): for i in range(number): record_id = uuid.uuid4() data = {'title': 'Test{0}'.format(i), '$schema': schema} recid_minter(record_id, data) oaiid_minter(record_id, data) Record.create(data, id_=record_id) db.session.commit()
def create(self, identity, data, uow=None): """Create a new OAI set.""" self.require_permission(identity, 'create') valid_data, errors = self.schema.load( data, context={"identity": identity}, raise_errors=True, ) new_set = OAISet(**valid_data) existing_set, errors = self._get_one(spec=new_set.spec, raise_error=False) if existing_set: raise OAIPMHSetSpecAlreadyExistsError(new_set.spec) uow.register(OAISetCommitOp(new_set)) return self.result_item( service=self, identity=identity, item=new_set, links_tpl=self.links_item_tpl, )
def test_oai_set_correspondence(db, users, oai_communities, oai_sets, missing_oai_set, missing_community): for c in oai_communities: with db.session.begin_nested(): new_comm = Community.create(community_id=c, user_id=1) db.session.commit() # Delete the automatically created OAI Set id required by test case if new_comm.oaiset_spec not in oai_sets: with db.session.begin_nested(): db.session.delete(new_comm.oaiset) db.session.commit() for s in oai_sets: if not OAISet.query.filter_by(spec=s).one_or_none(): with db.session.begin_nested(): db.session.add(OAISet(spec=s)) db.session.commit() check = OAICorrespondenceCheck() check.perform() assert set(check.issues.get('missing_oai_set', [])) == set(missing_oai_set) assert set(check.issues.get('missing_community', [])) == \ set(missing_community)
def _try_populate_oaisets(): """Try to update collections.""" indexer = RecordIndexer() schema = { 'type': 'object', 'properties': { 'title': {'type': 'string'}, 'field': {'type': 'boolean'}, }, 'required': ['title'], } def create_record(item_dict): """Create test record.""" record_id = uuid.uuid4() recid_minter(record_id, item_dict) oaiid_minter(record_id, item_dict) record = Record.create(item_dict, id_=record_id) indexer.index(record) return record a = OAISet(spec='a') b = OAISet(spec='b') e = OAISet( spec="e", search_pattern="title:Test2 OR title:Test3") c = OAISet(spec="c", search_pattern="title:Test0") d = OAISet(spec="d", search_pattern="title:Test1") f = OAISet(spec="f", search_pattern="title:Test2") g = OAISet(spec="g") h = OAISet(spec="h") i = OAISet(spec="i", search_pattern="title:Test3") j = OAISet(spec="j with space", search_pattern="title:Test4") with db.session.begin_nested(): for oaiset in [a, b, c, d, e, f, g, h, i, j]: db.session.add(oaiset) db.session.commit() a_id = a.id i_id = i.id # start tests record0 = create_record({ '_oai': {'sets': ['a']}, 'title': 'Test0', '$schema': schema }) assert 'a' in record0['_oai']['sets'], 'Keep manually managed set "a".' assert 'c' in record0['_oai']['sets'] assert len(record0['_oai']['sets']) == 2 record_not_found = create_record( {'title': 'TestNotFound', '$schema': schema} ) assert record_not_found['_oai']['sets'] == [] record1 = create_record({'title': 'Test1', '$schema': schema}) assert 'd' in record1['_oai']['sets'] assert len(record1['_oai']['sets']) == 1 record2 = create_record({'title': 'Test2', '$schema': schema}) record2_id = record2.id assert 'e' in record2['_oai']['sets'] assert 'f' in record2['_oai']['sets'] assert len(record2['_oai']['sets']) == 2 record3 = create_record({'title': 'Test3', '$schema': schema}) record3_id = record3.id assert 'e' in record3['_oai']['sets'] assert 'i' in record3['_oai']['sets'] assert len(record3['_oai']['sets']) == 2 record4 = create_record({'title': 'Test4', '$schema': schema}) record4_id = record4.id assert 'j with space' in record4['_oai']['sets'] assert len(record4['_oai']['sets']) == 1 # wait ElasticSearch end to index records sleep(10) # test delete current_oaiserver.unregister_signals_oaiset() with patch('invenio_oaiserver.receivers.after_delete_oai_set') as f: current_oaiserver.register_signals_oaiset() with db.session.begin_nested(): db.session.delete(j) db.session.commit() assert f.called after_delete_oai_set(None, None, j) record4_model = RecordMetadata.query.filter_by( id=record4_id).first().json assert 'j with space' not in record4_model['_oai']['sets'] assert len(record4_model['_oai']['sets']) == 0 current_oaiserver.unregister_signals_oaiset() # test update search_pattern with patch('invenio_oaiserver.receivers.after_update_oai_set') as f: current_oaiserver.register_signals_oaiset() with db.session.begin_nested(): i.search_pattern = None assert current_oaiserver.sets is None, 'Cache should be empty.' db.session.merge(i) db.session.commit() assert f.called i = OAISet.query.get(i_id) after_update_oai_set(None, None, i) record3_model = RecordMetadata.query.filter_by( id=record3_id).first().json assert 'i' in record3['_oai']['sets'], 'Set "i" is manually managed.' assert 'e' in record3_model['_oai']['sets'] assert len(record3_model['_oai']['sets']) == 2 current_oaiserver.unregister_signals_oaiset() # test update search_pattern with patch('invenio_oaiserver.receivers.after_update_oai_set') as f: current_oaiserver.register_signals_oaiset() with db.session.begin_nested(): i.search_pattern = 'title:Test3' db.session.merge(i) db.session.commit() assert f.called i = OAISet.query.get(i_id) after_update_oai_set(None, None, i) record3_model = RecordMetadata.query.filter_by( id=record3_id).first().json assert 'e' in record3_model['_oai']['sets'] assert 'i' in record3_model['_oai']['sets'] assert len(record3_model['_oai']['sets']) == 2 current_oaiserver.unregister_signals_oaiset() # test update the spec with pytest.raises(OAISetSpecUpdateError) as exc_info: a = OAISet.query.get(a_id) a.spec = 'new-a' assert exc_info.type is OAISetSpecUpdateError # test create new set with patch('invenio_oaiserver.receivers.after_insert_oai_set') as f: current_oaiserver.register_signals_oaiset() with db.session.begin_nested(): k = OAISet(spec="k", search_pattern="title:Test2") db.session.add(k) db.session.commit() assert f.called after_insert_oai_set(None, None, k) record2_model = RecordMetadata.query.filter_by( id=record2_id).first().json assert 'e' in record2_model['_oai']['sets'] assert 'f' in record2_model['_oai']['sets'] assert 'k' in record2_model['_oai']['sets'] assert len(record2_model['_oai']['sets']) == 3 current_oaiserver.register_signals_oaiset()
def test_populate_oaisets(app, with_record_signals): """Populate OAISets.""" indexer = RecordIndexer() schema = { 'allOf': [{ 'type': 'object', 'properties': { 'title': {'type': 'string'}, 'genre': {'type': 'string'}, 'field': {'type': 'boolean'}, }, 'required': ['title'], }, { '$ref': 'http://inveniosoftware.org/schemas/' 'oaiserver/internal-v1.0.0.json', }] } def create_record(item_dict, mint_oaiid=True): """Create test record.""" with app.test_request_context(): record_id = uuid.uuid4() recid_minter(record_id, item_dict) if mint_oaiid: oaiid_minter(record_id, item_dict) record = Record.create(item_dict, id_=record_id) indexer.index(record) return record a = OAISet(spec='a') b = OAISet(spec='b') e = OAISet( spec="e", search_pattern="title:Test2 OR title:Test3") c = OAISet(spec="c", search_pattern="title:Test0") d = OAISet(spec="d", search_pattern="title:Test1") f = OAISet(spec="f", search_pattern="title:Test2") g = OAISet(spec="g") h = OAISet(spec="h") i = OAISet(spec="i", search_pattern="title:Test3") j = OAISet(spec="j with space", search_pattern="title:Test4") # Note below: brackets around AND search query are required l = OAISet(spec="math", search_pattern="(title:foo AND genre:math)") m = OAISet(spec="nonmath", search_pattern="(title:foo AND -genre:math)") with db.session.begin_nested(): for oaiset in [a, b, c, d, e, f, g, h, i, j, l, m]: db.session.add(oaiset) db.session.commit() a_id = a.id i_id = i.id # start tests record0 = create_record({ '_oai': {'sets': ['a']}, 'title': 'Test0', '$schema': schema }) assert 'a' in record0['_oai']['sets'], 'Keep manually managed set "a".' assert 'c' in record0['_oai']['sets'] assert len(record0['_oai']['sets']) == 2 record_not_found = create_record( {'title': 'TestNotFound', '$schema': schema} ) # Don't create empty sets list just because of commit assert 'sets' not in record_not_found['_oai'] record1 = create_record({'title': 'Test1', '$schema': schema}) assert 'd' in record1['_oai']['sets'] assert len(record1['_oai']['sets']) == 1 record2 = create_record({'title': 'Test2', '$schema': schema}) record2_id = record2.id assert 'e' in record2['_oai']['sets'] assert 'f' in record2['_oai']['sets'] assert len(record2['_oai']['sets']) == 2 record3 = create_record({'title': 'Test3', '$schema': schema}) record3_id = record3.id assert 'e' in record3['_oai']['sets'] assert 'i' in record3['_oai']['sets'] assert len(record3['_oai']['sets']) == 2 record4 = create_record({'title': 'Test4', '$schema': schema}) record4_id = record4.id assert 'j with space' in record4['_oai']['sets'] assert len(record4['_oai']['sets']) == 1 # If record does not have '_oai', don't add any sets, # nor even the default '_oai' key record5 = create_record({'title': 'Test1', '$schema': schema}, mint_oaiid=False) assert '_oai' not in record5 # If 'sets' before and after record commit are equivalent # don't bump up the '_oai.updated' timestamp... record6 = create_record({'title': 'Test1', '$schema': schema}) assert record6['_oai']['sets'] == ['d'] prev_updated_r6 = record6['_oai']['updated'] record6.commit() assert record6['_oai']['sets'] == ['d'] assert record6['_oai']['updated'] == prev_updated_r6 # date stays the same # ...but do bump up '_oai.updated' if the sets are different record7 = create_record({'title': 'Test1', '$schema': schema}) assert record7['_oai']['sets'] == ['d'] prev_updated_r7 = record7['_oai']['updated'] sleep(1) # 'updated' timestamp is accurate to a second, hence the wait record7['_oai']['sets'] = ['d', 'f'] # 'f' should be removed after commit record7.commit() assert record7['_oai']['sets'] == ['d'] assert record7['_oai']['updated'] != prev_updated_r7 # date bumped # Test 'AND' keyword for records record8 = create_record( {'title': 'foo', 'genre': 'math', '$schema': schema}) assert record8['_oai']['sets'] == ['math', ] record9 = create_record( {'title': 'foo', 'genre': 'physics', '$schema': schema}) assert record9['_oai']['sets'] == ['nonmath', ] record10 = create_record( {'title': 'bar', 'genre': 'math', '$schema': schema}) assert 'sets' not in record10['_oai'] # title is not 'foo' # wait ElasticSearch end to index records sleep(10) # test delete current_oaiserver.unregister_signals_oaiset() with patch('invenio_oaiserver.receivers.after_delete_oai_set') as f: current_oaiserver.register_signals_oaiset() with db.session.begin_nested(): db.session.delete(j) db.session.commit() assert f.called after_delete_oai_set(None, None, j) record4_model = RecordMetadata.query.filter_by( id=record4_id).first().json assert 'j with space' not in record4_model['_oai']['sets'] assert len(record4_model['_oai']['sets']) == 0 current_oaiserver.unregister_signals_oaiset() # test update search_pattern with patch('invenio_oaiserver.receivers.after_update_oai_set') as f: current_oaiserver.register_signals_oaiset() with db.session.begin_nested(): i.search_pattern = None assert current_oaiserver.sets is None, 'Cache should be empty.' db.session.merge(i) db.session.commit() assert f.called i = OAISet.query.get(i_id) after_update_oai_set(None, None, i) record3_model = RecordMetadata.query.filter_by( id=record3_id).first().json assert 'i' in record3['_oai']['sets'], 'Set "i" is manually managed.' assert 'e' in record3_model['_oai']['sets'] assert len(record3_model['_oai']['sets']) == 2 current_oaiserver.unregister_signals_oaiset() # test update search_pattern with patch('invenio_oaiserver.receivers.after_update_oai_set') as f: current_oaiserver.register_signals_oaiset() with db.session.begin_nested(): i.search_pattern = 'title:Test3' db.session.merge(i) db.session.commit() assert f.called i = OAISet.query.get(i_id) after_update_oai_set(None, None, i) record3_model = RecordMetadata.query.filter_by( id=record3_id).first().json assert 'e' in record3_model['_oai']['sets'] assert 'i' in record3_model['_oai']['sets'] assert len(record3_model['_oai']['sets']) == 2 current_oaiserver.unregister_signals_oaiset() # test update the spec with pytest.raises(OAISetSpecUpdateError) as exc_info: a = OAISet.query.get(a_id) a.spec = 'new-a' assert exc_info.type is OAISetSpecUpdateError # test create new set with patch('invenio_oaiserver.receivers.after_insert_oai_set') as f: current_oaiserver.register_signals_oaiset() with db.session.begin_nested(): k = OAISet(spec="k", search_pattern="title:Test2") db.session.add(k) db.session.commit() assert f.called after_insert_oai_set(None, None, k) record2_model = RecordMetadata.query.filter_by( id=record2_id).first().json assert 'e' in record2_model['_oai']['sets'] assert 'f' in record2_model['_oai']['sets'] assert 'k' in record2_model['_oai']['sets'] assert len(record2_model['_oai']['sets']) == 3 current_oaiserver.register_signals_oaiset()
def _try_populate_oaisets(): """Try to update collections.""" schema = { 'type': 'object', 'properties': { 'title': { 'type': 'string' }, 'field': { 'type': 'boolean' }, }, 'required': ['title'], } a = OAISet(name="a") b = OAISet(name="b") e = OAISet(name="e", search_pattern="title:Test2 OR title:Test3") c = OAISet(name="c", search_pattern="title:Test0") d = OAISet(name="d", search_pattern="title:Test1") f = OAISet(name="f", search_pattern="title:Test2") g = OAISet(name="g") h = OAISet(name="h") i = OAISet(name="i", search_pattern="title:Test3") j = OAISet(name="j", search_pattern="title:Test4") with db.session.begin_nested(): for coll in [a, b, c, d, e, f, g, h, i, j]: db.session.add(coll) db.session.commit() # start tests record0 = Record.create({'title': 'Test0', '$schema': schema}) assert 'c' in record0['_oaisets'] assert len(record0['_oaisets']) == 1 record = Record.create({'title': 'TestNotFound', '$schema': schema}) assert record['_oaisets'] == [] record = Record.create({'title': 'Test1', '$schema': schema}) assert 'd' in record['_oaisets'] assert len(record['_oaisets']) == 1 record = Record.create({'title': 'Test2', '$schema': schema}) assert 'e' in record['_oaisets'] assert 'f' in record['_oaisets'] assert len(record['_oaisets']) == 2 record3 = Record.create({'title': 'Test3', '$schema': schema}) assert 'e' in record3['_oaisets'] assert 'i' in record3['_oaisets'] assert len(record3['_oaisets']) == 2 record4 = Record.create({'title': 'Test4', '$schema': schema}) assert 'j' in record4['_oaisets'] assert len(record4['_oaisets']) == 1 # test delete db.session.delete(j) db.session.commit() record4.commit() assert 'h' not in record4['_oaisets'] assert 'j' not in record4['_oaisets'] assert len(record4['_oaisets']) == 0 # test update search_pattern i.search_pattern = None db.session.add(i) db.session.commit() record3.commit() assert 'e' in record3['_oaisets'] assert len(record3['_oaisets']) == 1 # test update search_pattern i.search_pattern = 'title:Test3' db.session.add(i) db.session.commit() record3.commit() assert 'e' in record3['_oaisets'] assert 'i' in record3['_oaisets'] assert len(record3['_oaisets']) == 2 # test update name a.name = "new-a" db.session.add(a) db.session.commit() record3.commit() assert 'i' in record3['_oaisets'] assert 'e' in record3['_oaisets'] assert len(record3['_oaisets']) == 2 # test update name c.name = "new-c" db.session.add(c) db.session.commit() record0.commit() assert 'new-c' in record0['_oaisets'] assert len(record0['_oaisets']) == 1
def _try_populate_oaisets(): """Try to update collections.""" schema = { 'type': 'object', 'properties': { 'title': {'type': 'string'}, 'field': {'type': 'boolean'}, }, 'required': ['title'], } a = OAISet(spec="a") b = OAISet(spec="b") e = OAISet( spec="e", search_pattern="title:Test2 OR title:Test3") c = OAISet(spec="c", search_pattern="title:Test0") d = OAISet(spec="d", search_pattern="title:Test1") f = OAISet(spec="f", search_pattern="title:Test2") g = OAISet(spec="g") h = OAISet(spec="h") i = OAISet(spec="i", search_pattern="title:Test3") j = OAISet(spec="j", search_pattern="title:Test4") with db.session.begin_nested(): for coll in [a, b, c, d, e, f, g, h, i, j]: db.session.add(coll) db.session.commit() # start tests record0 = Record.create({'title': 'Test0', '$schema': schema}) assert 'c' in record0['_oai']['sets'] assert len(record0['_oai']['sets']) == 1 record = Record.create({'title': 'TestNotFound', '$schema': schema}) assert record['_oai']['sets'] == [] record = Record.create({'title': 'Test1', '$schema': schema}) assert 'd' in record['_oai']['sets'] assert len(record['_oai']['sets']) == 1 record = Record.create({'title': 'Test2', '$schema': schema}) assert 'e' in record['_oai']['sets'] assert 'f' in record['_oai']['sets'] assert len(record['_oai']['sets']) == 2 record3 = Record.create({'title': 'Test3', '$schema': schema}) assert 'e' in record3['_oai']['sets'] assert 'i' in record3['_oai']['sets'] assert len(record3['_oai']['sets']) == 2 record4 = Record.create({'title': 'Test4', '$schema': schema}) assert 'j' in record4['_oai']['sets'] assert len(record4['_oai']['sets']) == 1 # test delete db.session.delete(j) db.session.commit() record4.commit() assert 'h' not in record4['_oai']['sets'] assert 'j' not in record4['_oai']['sets'] assert len(record4['_oai']['sets']) == 0 # test update search_pattern i.search_pattern = None db.session.add(i) db.session.commit() record3.commit() assert 'e' in record3['_oai']['sets'] assert len(record3['_oai']['sets']) == 1 # test update search_pattern i.search_pattern = 'title:Test3' db.session.add(i) db.session.commit() record3.commit() assert 'e' in record3['_oai']['sets'] assert 'i' in record3['_oai']['sets'] assert len(record3['_oai']['sets']) == 2 # test update the spec a.spec = "new-a" db.session.add(a) db.session.commit() record3.commit() assert 'i' in record3['_oai']['sets'] assert 'e' in record3['_oai']['sets'] assert len(record3['_oai']['sets']) == 2 # test update name c.spec = "new-c" db.session.add(c) db.session.commit() record0.commit() assert 'new-c' in record0['_oai']['sets'] assert len(record0['_oai']['sets']) == 1