def test_search_pattern_change(app, without_oaiset_signals, schema):
"""Test search pattern change."""
record0 = create_record(app, {
'_oai': {'sets': ['a']}, 'title_statement': {'title': 'Test0'},
'$schema': schema
})
rec_uuid = record0.id
oaiset = OAISet(spec="a", search_pattern="title_statement.title:Test0")
db.session.add(oaiset)
db.session.commit()
run_after_insert_oai_set()
sleep(2)
record = Record.get_record(rec_uuid)
assert record['_oai']['sets'] == ['a']
# change search pattern: record0 will not inside it anymore
oaiset = OAISet.query.first()
oaiset.search_pattern = 'title_statement.title:Test1'
db.session.merge(oaiset)
db.session.commit()
after_update_oai_set(None, None, oaiset)
sleep(2)
record = Record.get_record(rec_uuid)
record.commit()
assert record['_oai']['sets'] == []
def update_sets():
"""Check that each community has a corresponding oai set"""
for community in Community.get_all():
dirty = False
oaiset = OAISet.query.filter(OAISet.spec == str(community.id)).one_or_none()
if not oaiset:
click.secho('Adding new oai set {}'.format(community.name),
fg='yellow', bold=True)
oaiset = OAISet(spec=str(community.id),
name=community.name,
description=community.description)
dirty = True
else:
if oaiset.name != community.name:
oaiset.name = community.name
dirty = True
click.secho('Update name for set {}'.format(oaiset.spec),
fg='yellow', bold=True)
if oaiset.description != community.description:
oaiset.description = community.description
dirty = True
click.secho('Update description for set {}'.format(oaiset.spec),
fg='yellow', bold=True)
if dirty:
db.session.add(oaiset)
db.session.commit()
def update_sets():
"""Check that each community has a corresponding oai set"""
for community in Community.get_all():
dirty = False
oaiset = OAISet.query.filter(
OAISet.spec == str(community.id)).one_or_none()
if not oaiset:
click.secho('Adding new oai set {}'.format(community.name),
fg='yellow',
bold=True)
oaiset = OAISet(spec=str(community.id),
name=community.name,
description=community.description)
dirty = True
else:
if oaiset.name != community.name:
oaiset.name = community.name
dirty = True
click.secho('Update name for set {}'.format(oaiset.spec),
fg='yellow',
bold=True)
if oaiset.description != community.description:
oaiset.description = community.description
dirty = True
click.secho('Update description for set {}'.format(
oaiset.spec),
fg='yellow',
bold=True)
if dirty:
db.session.add(oaiset)
db.session.commit()
def test_response(db, schema, client):
oaiset = OAISet(spec='higgs', name='Higgs', description='...')
oaiset.search_pattern = 'title:higgs'
db.session.add(oaiset)
db.session.add(oaiset)
res = client.get('/oai2d?verb=ListSets')
print(res.data)
def test_oaiset_add_remove_record(app):
"""Test the API method for manual record adding."""
with app.app_context():
oaiset1 = OAISet(spec='abc')
rec1 = Record.create({'title': 'Test1'})
oaiset1.add_record(rec1)
assert 'abc' in rec1['_oai']['sets']
assert 'updated' in rec1['_oai']
dt1 = iso2dt(rec1['_oai']['updated'])
assert dt1.year == datetime.utcnow().year # Test if parsed OK
oaiset1.remove_record(rec1)
assert 'abc' not in rec1['_oai']['sets']
dt2 = iso2dt(rec1['_oai']['updated'])
assert dt2 >= dt1
def oai_server(sender, app=None, **kwargs):
#app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite://'
app.config['CELERY_TASK_ALWAYS_EAGER'] = True
if not hasattr(app, 'cli'):
from flask_cli import FlaskCLI
ext_cli = FlaskCLI(app)
ext_db = InvenioDB(app)
ext_indexer = InvenioIndexer(app)
ext_pidstore = InvenioPIDStore(app)
ext_records = InvenioRecords(app)
ext_search = InvenioSearch(app)
ext_celery = FlaskCeleryExt(app)
app.config['OAISERVER_RECORD_INDEX'] = ['authors', 'records']
app.config['OAISERVER_ID_PREFIX'] = 'oai:example:'
app.config['OAISERVER_QUERY_PARSER_FIELDS'] = ["title"]
app.config['OAISERVER_METADATA_FORMATS'] = {
'oai_dc': {
'serializer': ('invenio_oaiserver.utils:dumps_etree', {
'xslt_filename':
pkg_resources.resource_filename(
'testinvenio.records', 'static/xsl/MARC21slim2OAIDC.xsl'),
'xslt_filename':
pathlib.Path(
"/home/alzbeta/testinvenio/testinvenio/records/static/xsl/MARC21slim2OAIDC.xsl"
)
}),
'schema':
'http://json-schema.org/draft-04/schema#',
'namespace':
'http://json-schema.org/draft-04/schema#',
}
}
ext_oaiserver = InvenioOAIServer(app)
app.register_blueprint(blueprint)
ctx = app.app_context()
ctx.push()
db.create_all()
ext_search.flush_and_refresh('_all')
oaiset = OAISet(spec='pattern', name='Pattern', description='...')
#nefunguje pokud search pattern ma pole ktere neni v obouch modelech...
oaiset.search_pattern = 'title:Some title1'
try:
print("1")
db.session.add(oaiset)
db.session.commit()
except:
pass
def test_list_sets_long(app):
"""Test listing of sets."""
from invenio_db import db
from invenio_oaiserver.models import OAISet
with app.app_context():
current_oaiserver.unregister_signals_oaiset()
with db.session.begin_nested():
for i in range(27):
oaiset = OAISet(
spec='test{0}'.format(i),
name='Test{0}'.format(i),
description='test desc {0}'.format(i),
search_pattern='title_statement.title:Test{0}'.format(i),
)
db.session.add(oaiset)
db.session.commit()
run_after_insert_oai_set()
with app.test_client() as c:
# First page:
result = c.get('/oai2d?verb=ListSets')
tree = etree.fromstring(result.data)
assert len(tree.xpath('/x:OAI-PMH/x:ListSets/x:set',
namespaces=NAMESPACES)) == 10
resumption_token = tree.xpath(
'/x:OAI-PMH/x:ListSets/x:resumptionToken', namespaces=NAMESPACES
)[0]
assert resumption_token.text
# Second page:
result = c.get('/oai2d?verb=ListSets&resumptionToken={0}'.format(
resumption_token.text
))
tree = etree.fromstring(result.data)
assert len(tree.xpath('/x:OAI-PMH/x:ListSets/x:set',
namespaces=NAMESPACES)) == 10
resumption_token = tree.xpath(
'/x:OAI-PMH/x:ListSets/x:resumptionToken', namespaces=NAMESPACES
)[0]
assert resumption_token.text
# Third page:
result = c.get('/oai2d?verb=ListSets&resumptionToken={0}'.format(
resumption_token.text
))
tree = etree.fromstring(result.data)
assert len(tree.xpath('/x:OAI-PMH/x:ListSets/x:set',
namespaces=NAMESPACES)) == 7
resumption_token = tree.xpath(
'/x:OAI-PMH/x:ListSets/x:resumptionToken', namespaces=NAMESPACES
)[0]
assert not resumption_token.text
def test_oai_with_for_arxiv_set(inspire_app, clean_celery_session):
data = {
"arxiv_eprints": [{"value": "2009.01484"}],
"report_numbers": [{"value": "CERN-TH-2020-136"}],
}
record_data = faker.record("lit", data)
record = LiteratureRecord.create(record_data)
record_uuid = record.id
record_marcxml = record2marcxml(record)
db.session.commit()
def assert_the_record_is_indexed():
current_search.flush_and_refresh("*")
result = es_search("records-hep")
uuids = get_value(result, "hits.hits._id")
assert str(record_uuid) in uuids
retry_until_pass(assert_the_record_is_indexed)
set_name = inspire_app.config["OAI_SET_CERN_ARXIV"]
oaiset = OAISet(spec=f"{set_name}", name="Test", description="Test")
db.session.add(oaiset)
db.session.commit()
with inspire_app.test_client() as client:
response = client.get(
f"/api/oai2d?verb=ListRecords&metadataPrefix=marcxml&set={set_name}"
)
assert record_marcxml in response.data
def test_oai_get_single_identifier_for_arxiv_set(inspire_app,
celery_app_with_context,
celery_session_worker):
data = {
"arxiv_eprints": [{
"value": "2009.01484"
}],
"report_numbers": [{
"value": "CERN-TH-2020-136"
}],
}
record_data = faker.record("lit", data)
record = LiteratureRecord.create(record_data)
record_marcxml = record2marcxml(record)
db.session.commit()
set_name = inspire_app.config["OAI_SET_CERN_ARXIV"]
oaiset = OAISet(spec=f"{set_name}", name="Test", description="Test")
db.session.add(oaiset)
db.session.commit()
sleep(2)
with inspire_app.test_client() as client:
response = client.get(
f"/api/oai2d?verb=GetRecord&metadataPrefix=marcxml&identifier=oai:inspirehep.net:{record['control_number']}"
)
assert record_marcxml in response.data
def submit_set():
"""Insert a new set."""
form = get_NewSetForm(request.form)
if request.method == 'POST' and form.validate():
new_set = OAISet(spec=form.spec.data,
name=form.name.data,
description=form.description.data,
search_pattern=form.search_pattern.data,
parent=form.parent.data)
db.session.add(new_set)
# this shoul be moved to UPDATER (celery task) and it sould always
# take care of adding records to sets.
##########
query = Query(form.query.data)
response = current_search_client.search(
index='records', # make configurable PER SET
doc_type='record', # make configurable PER SET
body=query.body,
fields='_id, oaiid' # path to oaiid as a configurable
)
ids = [(a['_id'], a['oaiid']) for a in response['hits']['hits']]
add_records_to_set(ids)
#########
db.session.commit()
flash(_('New set %(spec)s was added.', spec=new_set.spec))
return redirect(url_for('.manage_sets'))
return render_template('make_set.html', new_set_form=form)
def test_oai_get_single_identifier_for_CDS_set(inspire_app, clean_celery_session):
data = {"_export_to": {"CDS": True}}
record_data = faker.record("lit", data)
record = LiteratureRecord.create(record_data)
record_uuid = record.id
record_marcxml = record2marcxml(record)
db.session.commit()
def assert_the_record_is_indexed():
current_search.flush_and_refresh("*")
result = es_search("records-hep")
uuids = get_value(result, "hits.hits._id")
assert str(record_uuid) in uuids
retry_until_pass(assert_the_record_is_indexed)
set_name = inspire_app.config["OAI_SET_CDS"]
oaiset = OAISet(spec=f"{set_name}", name="Test", description="Test")
db.session.add(oaiset)
db.session.commit()
with inspire_app.test_client() as client:
response = client.get(
f"/api/oai2d?verb=GetRecord&metadataPrefix=marcxml&identifier=oai:inspirehep.net:{record['control_number']}"
)
assert record_marcxml in response.data
def create_oaiset(name, title_pattern):
oaiset = OAISet(spec=name,
search_pattern=f"title_statement.title:{title_pattern}")
db.session.add(oaiset)
db.session.commit()
run_after_insert_oai_set()
return oaiset
def create_oaipmh_set(mapper, connection, community):
"""Signal for creating OAI-PMH sets during community creation."""
from invenio_oaiserver.models import OAISet
with db.session.begin_nested():
obj = OAISet(spec=community.oaiset_spec,
name=community.title,
description=community.description)
db.session.add(obj)
def test_list_sets_long(app):
"""Test listing of sets."""
from invenio_db import db
from invenio_oaiserver.models import OAISet
with app.app_context():
with db.session.begin_nested():
for i in range(27):
db.session.add(OAISet(
spec='test{0}'.format(i),
name='Test{0}'.format(i),
description='test desc {0}'.format(i),
search_pattern='title:Test{0}'.format(i),
))
db.session.commit()
namespaces = {'x': NS_OAIPMH}
with app.test_client() as c:
# First page:
result = c.get('/oai2d?verb=ListSets')
tree = etree.fromstring(result.data)
assert len(tree.xpath('/x:OAI-PMH/x:ListSets/x:set',
namespaces=namespaces)) == 10
resumption_token = tree.xpath(
'/x:OAI-PMH/x:ListSets/x:resumptionToken', namespaces=namespaces
)[0]
assert resumption_token.text
# Second page:
result = c.get('/oai2d?verb=ListSets&resumptionToken={0}'.format(
resumption_token.text
))
tree = etree.fromstring(result.data)
assert len(tree.xpath('/x:OAI-PMH/x:ListSets/x:set',
namespaces=namespaces)) == 10
resumption_token = tree.xpath(
'/x:OAI-PMH/x:ListSets/x:resumptionToken', namespaces=namespaces
)[0]
assert resumption_token.text
# Third page:
result = c.get('/oai2d?verb=ListSets&resumptionToken={0}'.format(
resumption_token.text
))
tree = etree.fromstring(result.data)
assert len(tree.xpath('/x:OAI-PMH/x:ListSets/x:set',
namespaces=namespaces)) == 7
resumption_token = tree.xpath(
'/x:OAI-PMH/x:ListSets/x:resumptionToken', namespaces=namespaces
)[0]
assert not resumption_token.text
def add_set(spec, name, pattern, description='...'):
"""Add OAI set.
:param spec: set identifier
:param name: human readable name of the set
:param pattern: search pattern to get records
:param description: human readable description
"""
try:
oaiset = OAISet(spec=spec, name=name, description=description)
oaiset.search_pattern = pattern
db.session.add(oaiset)
db.session.commit()
msg = 'OAIset added: {name}'.format(name=name)
except Exception as err:
db.session.rollback()
msg = 'OAIset exist: {name}'.format(name=name)
return msg
def test_admin(app):
"""Test Flask-Admin interace."""
admin = Admin(app, name="Test")
assert "model" in set_adminview
assert "modelview" in set_adminview
# Register both models in admin
model = set_adminview.pop("model")
view = set_adminview.pop("modelview")
admin.add_view(view(model, db.session, **set_adminview))
# Check if generated admin menu contains the correct items
menu_items = {str(item.name): item for item in admin.menu()}
assert "OAI-PMH" in menu_items
assert menu_items["OAI-PMH"].is_category()
submenu_items = {
str(item.name): item
for item in menu_items["OAI-PMH"].get_children()
}
assert "Sets" in submenu_items
assert isinstance(submenu_items["Sets"], menu.MenuView)
# Create a test set.
with app.app_context():
test_set = OAISet(
id=1,
spec="test",
name="test_name",
description="some test description",
search_pattern="title_statement.title:Test0",
)
db.session.add(test_set)
db.session.commit()
with app.test_request_context():
index_view_url = url_for("oaiset.index_view")
delete_view_url = url_for("oaiset.delete_view")
detail_view_url = url_for("oaiset.details_view", id=1)
with app.test_client() as client:
# List index view and check record is there.
res = client.get(index_view_url)
assert res.status_code == 200
# Deletion is forbiden.
res = client.post(delete_view_url,
data={"id": 1},
follow_redirects=True)
assert res.status_code == 200
# View the deleted record.
res = client.get(detail_view_url)
assert res.status_code == 200
assert 1 == OAISet.query.count()
def create_community_oaiset(community, fix=False):
"""Create a community's OAI set.
:param community: the community for which we add the OAI set.
:param fix: Enable the fixing of the database. This function doesn't fail
if the oai set already exists. It will just add it if it is missing.
"""
oaiset = OAISet(spec=str(community.id), name=community.name,
description=community.description)
add_to_db(oaiset, skip_if_exists=fix)
def oaiserver(sets, records):
"""Initialize OAI-PMH server."""
from invenio_db import db
from invenio_oaiserver.models import OAISet
from invenio_records.api import Record
# create a OAI Set
with db.session.begin_nested():
for i in range(sets):
db.session.add(
OAISet(
spec='test{0}'.format(i),
name='Test{0}'.format(i),
description='test desc {0}'.format(i),
search_pattern='title_statement.title:Test{0}'.format(i),
))
# create a record
schema = {
'type': 'object',
'properties': {
'title_statement': {
'type': 'object',
'properties': {
'title': {
'type': 'string',
},
},
},
'field': {
'type': 'boolean'
},
},
}
search.client.indices.delete_alias('_all', '_all', ignore=[400, 404])
search.client.indices.delete('*')
with app.app_context():
indexer = RecordIndexer()
with db.session.begin_nested():
for i in range(records):
record_id = uuid.uuid4()
data = {
'title_statement': {
'title': 'Test{0}'.format(i)
},
'$schema': schema,
}
recid_minter(record_id, data)
oaiid_minter(record_id, data)
record = Record.create(data, id_=record_id)
indexer.index(record)
db.session.commit()
def test_admin(app):
"""Test Flask-Admin interace."""
admin = Admin(app, name='Test')
assert 'model' in set_adminview
assert 'modelview' in set_adminview
# Register both models in admin
model = set_adminview.pop('model')
view = set_adminview.pop('modelview')
admin.add_view(view(model, db.session, **set_adminview))
# Check if generated admin menu contains the correct items
menu_items = {str(item.name): item for item in admin.menu()}
assert 'OAI-PMH' in menu_items
assert menu_items['OAI-PMH'].is_category()
submenu_items = {
str(item.name): item
for item in menu_items['OAI-PMH'].get_children()
}
assert 'Sets' in submenu_items
assert isinstance(submenu_items['Sets'], menu.MenuView)
# Create a test set.
with app.app_context():
test_set = OAISet(id=1,
spec='test',
name='test_name',
description='some test description',
search_pattern='test search')
db.session.add(test_set)
db.session.commit()
with app.test_request_context():
index_view_url = url_for('oaiset.index_view')
delete_view_url = url_for('oaiset.delete_view')
detail_view_url = url_for('oaiset.details_view', id=1)
with app.test_client() as client:
# List index view and check record is there.
res = client.get(index_view_url)
assert res.status_code == 200
# Deletion is forbiden.
res = client.post(delete_view_url,
data={'id': 1},
follow_redirects=True)
assert res.status_code == 200
# View the deleted record.
res = client.get(detail_view_url)
assert res.status_code == 200
assert 1 == OAISet.query.count()
def create(cls, data, id_=None, dbcommit=False, with_bucket=True,
**kwargs):
"""Create organisation record."""
# Create OAI set
oaiset = OAISet(spec=data['code'], name=data['name'])
db.session.add(oaiset)
return super(OrganisationRecord, cls).create(data,
id_=id_,
dbcommit=dbcommit,
with_bucket=with_bucket,
**kwargs)
def test_oaiset_add_remove_record(app):
"""Test the API method for manual record adding."""
with app.app_context():
oaiset1 = OAISet(spec='abc')
rec1 = Record.create({'title_statement': {'title': 'Test1'}})
assert not oaiset1.has_record(rec1)
oaiset1.add_record(rec1)
assert 'abc' in rec1['_oai']['sets']
assert 'updated' in rec1['_oai']
assert oaiset1.has_record(rec1)
dt1 = iso2dt(rec1['_oai']['updated'])
assert dt1.year == datetime.utcnow().year # Test if parsed OK
oaiset1.remove_record(rec1)
assert 'abc' not in rec1['_oai']['sets']
assert not oaiset1.has_record(rec1)
dt2 = iso2dt(rec1['_oai']['updated'])
assert dt2 >= dt1
def loadoaisets(force=False):
"""Load default file store location."""
sets = [
('openaire', 'OpenAIRE', None),
('openaire_data', 'OpenAIRE data sets', None),
]
try:
for setid, name, pattern in sets:
oset = OAISet(spec=setid, name=name, search_pattern=pattern)
db.session.add(oset)
db.session.commit()
return len(sets)
except Exception:
db.session.rollback()
raise
def test_listsets_invalid_name(app):
"""Test ListSets with invalid unicode character for XML."""
with app.test_request_context():
current_oaiserver.unregister_signals_oaiset()
with db.session.begin_nested():
a = OAISet(spec='test', name=u'uni\x01co\x0bde',
description='test desc')
db.session.add(a)
with app.test_client() as c:
result = c.get('/oai2d?verb=ListSets')
tree = etree.fromstring(result.data)
assert tree.xpath('/x:OAI-PMH/x:ListSets/x:set/x:setName',
namespaces=NAMESPACES)[0].text == 'unicode'
def test_oaiset_add_remove_record(app):
"""Test the API method for manual record adding."""
with app.app_context():
oaiset1 = OAISet(spec='abc')
rec1 = Record.create({'title_statement': {'title': 'Test1'}})
rec1.commit()
# Adding a record to an OAIset should change the record's updated date
dt1 = rec1.updated
assert not oaiset1.has_record(rec1)
oaiset1.add_record(rec1)
assert 'abc' in rec1['_oai']['sets']
assert oaiset1.has_record(rec1)
rec1.commit()
dt2 = rec1.updated
assert dt2 > dt1
oaiset1.remove_record(rec1)
rec1.commit()
dt3 = rec1.updated
assert 'abc' not in rec1['_oai']['sets']
assert not oaiset1.has_record(rec1)
assert dt3 > dt2
def test_listsets(app):
"""Test ListSets."""
with app.test_request_context():
current_oaiserver.unregister_signals_oaiset()
with db.session.begin_nested():
a = OAISet(spec='test', name='Test', description='test desc')
db.session.add(a)
with app.test_client() as c:
result = c.get('/oai2d?verb=ListSets')
tree = etree.fromstring(result.data)
assert len(tree.xpath('/x:OAI-PMH', namespaces=NAMESPACES)) == 1
assert len(tree.xpath('/x:OAI-PMH/x:ListSets',
namespaces=NAMESPACES)) == 1
assert len(
tree.xpath('/x:OAI-PMH/x:ListSets/x:set',
namespaces=NAMESPACES)) == 1
assert len(
tree.xpath('/x:OAI-PMH/x:ListSets/x:set/x:setSpec',
namespaces=NAMESPACES)) == 1
assert len(
tree.xpath('/x:OAI-PMH/x:ListSets/x:set/x:setName',
namespaces=NAMESPACES)) == 1
assert len(
tree.xpath('/x:OAI-PMH/x:ListSets/x:set/x:setDescription',
namespaces=NAMESPACES)) == 1
assert len(
tree.xpath('/x:OAI-PMH/x:ListSets/x:set/x:setDescription/y:dc',
namespaces=NAMESPACES)) == 1
assert len(
tree.xpath(
'/x:OAI-PMH/x:ListSets/x:set/x:setDescription/y:dc/'
'z:description',
namespaces=NAMESPACES)) == 1
text = tree.xpath(
'/x:OAI-PMH/x:ListSets/x:set/x:setDescription/y:dc/'
'z:description/text()',
namespaces=NAMESPACES)
assert len(text) == 1
assert text[0] == 'test desc'
def test_listsets(app):
"""Test ListSets."""
with app.test_request_context():
with db.session.begin_nested():
a = OAISet(spec='test', name='Test', description="test desc")
db.session.add(a)
with app.test_client() as c:
result = c.get('/oai2d?verb=ListSets')
tree = etree.fromstring(result.data)
namespaces = {'x': NS_OAIPMH}
assert len(tree.xpath('/x:OAI-PMH', namespaces=namespaces)) == 1
assert len(tree.xpath('/x:OAI-PMH/x:ListSets',
namespaces=namespaces)) == 1
assert len(tree.xpath('/x:OAI-PMH/x:ListSets/x:set',
namespaces=namespaces)) == 1
assert len(tree.xpath('/x:OAI-PMH/x:ListSets/x:set/x:setSpec',
namespaces=namespaces)) == 1
assert len(tree.xpath('/x:OAI-PMH/x:ListSets/x:set/x:setName',
namespaces=namespaces)) == 1
assert len(tree.xpath(
'/x:OAI-PMH/x:ListSets/x:set/x:setDescription',
namespaces=namespaces
)) == 1
namespaces['y'] = NS_OAIDC
assert len(
tree.xpath('/x:OAI-PMH/x:ListSets/x:set/x:setDescription/y:dc',
namespaces=namespaces)
) == 1
namespaces['z'] = NS_DC
assert len(
tree.xpath('/x:OAI-PMH/x:ListSets/x:set/x:setDescription/y:dc/'
'z:description', namespaces=namespaces)
) == 1
text = tree.xpath(
'/x:OAI-PMH/x:ListSets/x:set/x:setDescription/y:dc/'
'z:description/text()', namespaces=namespaces)
assert len(text) == 1
assert text[0] == 'test desc'
def test_oai_with_for_cds_set(inspire_app, celery_app_with_context,
celery_session_worker):
data = {"_export_to": {"CDS": True}}
record_data = faker.record("lit", data)
record = LiteratureRecord.create(record_data)
record_marcxml = record2marcxml(record)
db.session.commit()
set_name = inspire_app.config["OAI_SET_CDS"]
oaiset = OAISet(spec=f"{set_name}", name="Test", description="Test")
db.session.add(oaiset)
db.session.commit()
sleep(2)
with inspire_app.test_client() as client:
response = client.get(
f"/api/oai2d?verb=ListRecords&metadataPrefix=marcxml&set={set_name}"
)
assert record_marcxml in response.data
def oaisets(db, communities):
"""Create custom OAISet objects.
Those should be custom OAISet objects which are not community based.
"""
oaisets_data = [
{
'spec': 'extra',
'search_pattern': 'title:extra'
},
{
'spec': 'user-extra', # Looks like a community-based OAISet
'search_pattern': 'title:foobar' # but has a search_pattern
},
]
for oai_data in oaisets_data:
obj = OAISet(**oai_data)
db.session.add(obj)
db.session.commit()
oai_data['id'] = obj.id
return oaisets_data
def oaiserver(number):
"""Initialize OAI-PMH server."""
from invenio_db import db
from invenio_oaiserver.models import OAISet
from invenio_records.api import Record
# create a OAI Set
with db.session.begin_nested():
for i in range(number):
db.session.add(
OAISet(
spec='test{0}'.format(i),
name='Test{0}'.format(i),
description='test desc {0}'.format(i),
search_pattern='title:Test{0}'.format(i),
))
# create a record
schema = {
'type': 'object',
'properties': {
'title': {
'type': 'string'
},
'field': {
'type': 'boolean'
},
},
'required': ['title'],
}
with db.session.begin_nested():
for i in range(number):
record_id = uuid.uuid4()
data = {'title': 'Test{0}'.format(i), '$schema': schema}
recid_minter(record_id, data)
oaiid_minter(record_id, data)
Record.create(data, id_=record_id)
db.session.commit()
def create(self, identity, data, uow=None):
"""Create a new OAI set."""
self.require_permission(identity, 'create')
valid_data, errors = self.schema.load(
data,
context={"identity": identity},
raise_errors=True,
)
new_set = OAISet(**valid_data)
existing_set, errors = self._get_one(spec=new_set.spec,
raise_error=False)
if existing_set:
raise OAIPMHSetSpecAlreadyExistsError(new_set.spec)
uow.register(OAISetCommitOp(new_set))
return self.result_item(
service=self,
identity=identity,
item=new_set,
links_tpl=self.links_item_tpl,
)
def test_oai_set_correspondence(db, users, oai_communities, oai_sets,
missing_oai_set, missing_community):
for c in oai_communities:
with db.session.begin_nested():
new_comm = Community.create(community_id=c, user_id=1)
db.session.commit()
# Delete the automatically created OAI Set id required by test case
if new_comm.oaiset_spec not in oai_sets:
with db.session.begin_nested():
db.session.delete(new_comm.oaiset)
db.session.commit()
for s in oai_sets:
if not OAISet.query.filter_by(spec=s).one_or_none():
with db.session.begin_nested():
db.session.add(OAISet(spec=s))
db.session.commit()
check = OAICorrespondenceCheck()
check.perform()
assert set(check.issues.get('missing_oai_set', [])) == set(missing_oai_set)
assert set(check.issues.get('missing_community', [])) == \
set(missing_community)
def _try_populate_oaisets():
"""Try to update collections."""
indexer = RecordIndexer()
schema = {
'type': 'object',
'properties': {
'title': {'type': 'string'},
'field': {'type': 'boolean'},
},
'required': ['title'],
}
def create_record(item_dict):
"""Create test record."""
record_id = uuid.uuid4()
recid_minter(record_id, item_dict)
oaiid_minter(record_id, item_dict)
record = Record.create(item_dict, id_=record_id)
indexer.index(record)
return record
a = OAISet(spec='a')
b = OAISet(spec='b')
e = OAISet(
spec="e", search_pattern="title:Test2 OR title:Test3")
c = OAISet(spec="c", search_pattern="title:Test0")
d = OAISet(spec="d", search_pattern="title:Test1")
f = OAISet(spec="f", search_pattern="title:Test2")
g = OAISet(spec="g")
h = OAISet(spec="h")
i = OAISet(spec="i", search_pattern="title:Test3")
j = OAISet(spec="j with space", search_pattern="title:Test4")
with db.session.begin_nested():
for oaiset in [a, b, c, d, e, f, g, h, i, j]:
db.session.add(oaiset)
db.session.commit()
a_id = a.id
i_id = i.id
# start tests
record0 = create_record({
'_oai': {'sets': ['a']}, 'title': 'Test0', '$schema': schema
})
assert 'a' in record0['_oai']['sets'], 'Keep manually managed set "a".'
assert 'c' in record0['_oai']['sets']
assert len(record0['_oai']['sets']) == 2
record_not_found = create_record(
{'title': 'TestNotFound', '$schema': schema}
)
assert record_not_found['_oai']['sets'] == []
record1 = create_record({'title': 'Test1', '$schema': schema})
assert 'd' in record1['_oai']['sets']
assert len(record1['_oai']['sets']) == 1
record2 = create_record({'title': 'Test2', '$schema': schema})
record2_id = record2.id
assert 'e' in record2['_oai']['sets']
assert 'f' in record2['_oai']['sets']
assert len(record2['_oai']['sets']) == 2
record3 = create_record({'title': 'Test3', '$schema': schema})
record3_id = record3.id
assert 'e' in record3['_oai']['sets']
assert 'i' in record3['_oai']['sets']
assert len(record3['_oai']['sets']) == 2
record4 = create_record({'title': 'Test4', '$schema': schema})
record4_id = record4.id
assert 'j with space' in record4['_oai']['sets']
assert len(record4['_oai']['sets']) == 1
# wait ElasticSearch end to index records
sleep(10)
# test delete
current_oaiserver.unregister_signals_oaiset()
with patch('invenio_oaiserver.receivers.after_delete_oai_set') as f:
current_oaiserver.register_signals_oaiset()
with db.session.begin_nested():
db.session.delete(j)
db.session.commit()
assert f.called
after_delete_oai_set(None, None, j)
record4_model = RecordMetadata.query.filter_by(
id=record4_id).first().json
assert 'j with space' not in record4_model['_oai']['sets']
assert len(record4_model['_oai']['sets']) == 0
current_oaiserver.unregister_signals_oaiset()
# test update search_pattern
with patch('invenio_oaiserver.receivers.after_update_oai_set') as f:
current_oaiserver.register_signals_oaiset()
with db.session.begin_nested():
i.search_pattern = None
assert current_oaiserver.sets is None, 'Cache should be empty.'
db.session.merge(i)
db.session.commit()
assert f.called
i = OAISet.query.get(i_id)
after_update_oai_set(None, None, i)
record3_model = RecordMetadata.query.filter_by(
id=record3_id).first().json
assert 'i' in record3['_oai']['sets'], 'Set "i" is manually managed.'
assert 'e' in record3_model['_oai']['sets']
assert len(record3_model['_oai']['sets']) == 2
current_oaiserver.unregister_signals_oaiset()
# test update search_pattern
with patch('invenio_oaiserver.receivers.after_update_oai_set') as f:
current_oaiserver.register_signals_oaiset()
with db.session.begin_nested():
i.search_pattern = 'title:Test3'
db.session.merge(i)
db.session.commit()
assert f.called
i = OAISet.query.get(i_id)
after_update_oai_set(None, None, i)
record3_model = RecordMetadata.query.filter_by(
id=record3_id).first().json
assert 'e' in record3_model['_oai']['sets']
assert 'i' in record3_model['_oai']['sets']
assert len(record3_model['_oai']['sets']) == 2
current_oaiserver.unregister_signals_oaiset()
# test update the spec
with pytest.raises(OAISetSpecUpdateError) as exc_info:
a = OAISet.query.get(a_id)
a.spec = 'new-a'
assert exc_info.type is OAISetSpecUpdateError
# test create new set
with patch('invenio_oaiserver.receivers.after_insert_oai_set') as f:
current_oaiserver.register_signals_oaiset()
with db.session.begin_nested():
k = OAISet(spec="k", search_pattern="title:Test2")
db.session.add(k)
db.session.commit()
assert f.called
after_insert_oai_set(None, None, k)
record2_model = RecordMetadata.query.filter_by(
id=record2_id).first().json
assert 'e' in record2_model['_oai']['sets']
assert 'f' in record2_model['_oai']['sets']
assert 'k' in record2_model['_oai']['sets']
assert len(record2_model['_oai']['sets']) == 3
current_oaiserver.register_signals_oaiset()
def test_populate_oaisets(app, with_record_signals):
"""Populate OAISets."""
indexer = RecordIndexer()
schema = {
'allOf': [{
'type': 'object',
'properties': {
'title': {'type': 'string'},
'genre': {'type': 'string'},
'field': {'type': 'boolean'},
},
'required': ['title'],
}, {
'$ref': 'http://inveniosoftware.org/schemas/'
'oaiserver/internal-v1.0.0.json',
}]
}
def create_record(item_dict, mint_oaiid=True):
"""Create test record."""
with app.test_request_context():
record_id = uuid.uuid4()
recid_minter(record_id, item_dict)
if mint_oaiid:
oaiid_minter(record_id, item_dict)
record = Record.create(item_dict, id_=record_id)
indexer.index(record)
return record
a = OAISet(spec='a')
b = OAISet(spec='b')
e = OAISet(
spec="e", search_pattern="title:Test2 OR title:Test3")
c = OAISet(spec="c", search_pattern="title:Test0")
d = OAISet(spec="d", search_pattern="title:Test1")
f = OAISet(spec="f", search_pattern="title:Test2")
g = OAISet(spec="g")
h = OAISet(spec="h")
i = OAISet(spec="i", search_pattern="title:Test3")
j = OAISet(spec="j with space", search_pattern="title:Test4")
# Note below: brackets around AND search query are required
l = OAISet(spec="math",
search_pattern="(title:foo AND genre:math)")
m = OAISet(spec="nonmath",
search_pattern="(title:foo AND -genre:math)")
with db.session.begin_nested():
for oaiset in [a, b, c, d, e, f, g, h, i, j, l, m]:
db.session.add(oaiset)
db.session.commit()
a_id = a.id
i_id = i.id
# start tests
record0 = create_record({
'_oai': {'sets': ['a']}, 'title': 'Test0', '$schema': schema
})
assert 'a' in record0['_oai']['sets'], 'Keep manually managed set "a".'
assert 'c' in record0['_oai']['sets']
assert len(record0['_oai']['sets']) == 2
record_not_found = create_record(
{'title': 'TestNotFound', '$schema': schema}
)
# Don't create empty sets list just because of commit
assert 'sets' not in record_not_found['_oai']
record1 = create_record({'title': 'Test1', '$schema': schema})
assert 'd' in record1['_oai']['sets']
assert len(record1['_oai']['sets']) == 1
record2 = create_record({'title': 'Test2', '$schema': schema})
record2_id = record2.id
assert 'e' in record2['_oai']['sets']
assert 'f' in record2['_oai']['sets']
assert len(record2['_oai']['sets']) == 2
record3 = create_record({'title': 'Test3', '$schema': schema})
record3_id = record3.id
assert 'e' in record3['_oai']['sets']
assert 'i' in record3['_oai']['sets']
assert len(record3['_oai']['sets']) == 2
record4 = create_record({'title': 'Test4', '$schema': schema})
record4_id = record4.id
assert 'j with space' in record4['_oai']['sets']
assert len(record4['_oai']['sets']) == 1
# If record does not have '_oai', don't add any sets,
# nor even the default '_oai' key
record5 = create_record({'title': 'Test1', '$schema': schema},
mint_oaiid=False)
assert '_oai' not in record5
# If 'sets' before and after record commit are equivalent
# don't bump up the '_oai.updated' timestamp...
record6 = create_record({'title': 'Test1', '$schema': schema})
assert record6['_oai']['sets'] == ['d']
prev_updated_r6 = record6['_oai']['updated']
record6.commit()
assert record6['_oai']['sets'] == ['d']
assert record6['_oai']['updated'] == prev_updated_r6 # date stays the same
# ...but do bump up '_oai.updated' if the sets are different
record7 = create_record({'title': 'Test1', '$schema': schema})
assert record7['_oai']['sets'] == ['d']
prev_updated_r7 = record7['_oai']['updated']
sleep(1) # 'updated' timestamp is accurate to a second, hence the wait
record7['_oai']['sets'] = ['d', 'f'] # 'f' should be removed after commit
record7.commit()
assert record7['_oai']['sets'] == ['d']
assert record7['_oai']['updated'] != prev_updated_r7 # date bumped
# Test 'AND' keyword for records
record8 = create_record(
{'title': 'foo', 'genre': 'math', '$schema': schema})
assert record8['_oai']['sets'] == ['math', ]
record9 = create_record(
{'title': 'foo', 'genre': 'physics', '$schema': schema})
assert record9['_oai']['sets'] == ['nonmath', ]
record10 = create_record(
{'title': 'bar', 'genre': 'math', '$schema': schema})
assert 'sets' not in record10['_oai'] # title is not 'foo'
# wait ElasticSearch end to index records
sleep(10)
# test delete
current_oaiserver.unregister_signals_oaiset()
with patch('invenio_oaiserver.receivers.after_delete_oai_set') as f:
current_oaiserver.register_signals_oaiset()
with db.session.begin_nested():
db.session.delete(j)
db.session.commit()
assert f.called
after_delete_oai_set(None, None, j)
record4_model = RecordMetadata.query.filter_by(
id=record4_id).first().json
assert 'j with space' not in record4_model['_oai']['sets']
assert len(record4_model['_oai']['sets']) == 0
current_oaiserver.unregister_signals_oaiset()
# test update search_pattern
with patch('invenio_oaiserver.receivers.after_update_oai_set') as f:
current_oaiserver.register_signals_oaiset()
with db.session.begin_nested():
i.search_pattern = None
assert current_oaiserver.sets is None, 'Cache should be empty.'
db.session.merge(i)
db.session.commit()
assert f.called
i = OAISet.query.get(i_id)
after_update_oai_set(None, None, i)
record3_model = RecordMetadata.query.filter_by(
id=record3_id).first().json
assert 'i' in record3['_oai']['sets'], 'Set "i" is manually managed.'
assert 'e' in record3_model['_oai']['sets']
assert len(record3_model['_oai']['sets']) == 2
current_oaiserver.unregister_signals_oaiset()
# test update search_pattern
with patch('invenio_oaiserver.receivers.after_update_oai_set') as f:
current_oaiserver.register_signals_oaiset()
with db.session.begin_nested():
i.search_pattern = 'title:Test3'
db.session.merge(i)
db.session.commit()
assert f.called
i = OAISet.query.get(i_id)
after_update_oai_set(None, None, i)
record3_model = RecordMetadata.query.filter_by(
id=record3_id).first().json
assert 'e' in record3_model['_oai']['sets']
assert 'i' in record3_model['_oai']['sets']
assert len(record3_model['_oai']['sets']) == 2
current_oaiserver.unregister_signals_oaiset()
# test update the spec
with pytest.raises(OAISetSpecUpdateError) as exc_info:
a = OAISet.query.get(a_id)
a.spec = 'new-a'
assert exc_info.type is OAISetSpecUpdateError
# test create new set
with patch('invenio_oaiserver.receivers.after_insert_oai_set') as f:
current_oaiserver.register_signals_oaiset()
with db.session.begin_nested():
k = OAISet(spec="k", search_pattern="title:Test2")
db.session.add(k)
db.session.commit()
assert f.called
after_insert_oai_set(None, None, k)
record2_model = RecordMetadata.query.filter_by(
id=record2_id).first().json
assert 'e' in record2_model['_oai']['sets']
assert 'f' in record2_model['_oai']['sets']
assert 'k' in record2_model['_oai']['sets']
assert len(record2_model['_oai']['sets']) == 3
current_oaiserver.register_signals_oaiset()
def _try_populate_oaisets():
"""Try to update collections."""
schema = {
'type': 'object',
'properties': {
'title': {
'type': 'string'
},
'field': {
'type': 'boolean'
},
},
'required': ['title'],
}
a = OAISet(name="a")
b = OAISet(name="b")
e = OAISet(name="e", search_pattern="title:Test2 OR title:Test3")
c = OAISet(name="c", search_pattern="title:Test0")
d = OAISet(name="d", search_pattern="title:Test1")
f = OAISet(name="f", search_pattern="title:Test2")
g = OAISet(name="g")
h = OAISet(name="h")
i = OAISet(name="i", search_pattern="title:Test3")
j = OAISet(name="j", search_pattern="title:Test4")
with db.session.begin_nested():
for coll in [a, b, c, d, e, f, g, h, i, j]:
db.session.add(coll)
db.session.commit()
# start tests
record0 = Record.create({'title': 'Test0', '$schema': schema})
assert 'c' in record0['_oaisets']
assert len(record0['_oaisets']) == 1
record = Record.create({'title': 'TestNotFound', '$schema': schema})
assert record['_oaisets'] == []
record = Record.create({'title': 'Test1', '$schema': schema})
assert 'd' in record['_oaisets']
assert len(record['_oaisets']) == 1
record = Record.create({'title': 'Test2', '$schema': schema})
assert 'e' in record['_oaisets']
assert 'f' in record['_oaisets']
assert len(record['_oaisets']) == 2
record3 = Record.create({'title': 'Test3', '$schema': schema})
assert 'e' in record3['_oaisets']
assert 'i' in record3['_oaisets']
assert len(record3['_oaisets']) == 2
record4 = Record.create({'title': 'Test4', '$schema': schema})
assert 'j' in record4['_oaisets']
assert len(record4['_oaisets']) == 1
# test delete
db.session.delete(j)
db.session.commit()
record4.commit()
assert 'h' not in record4['_oaisets']
assert 'j' not in record4['_oaisets']
assert len(record4['_oaisets']) == 0
# test update search_pattern
i.search_pattern = None
db.session.add(i)
db.session.commit()
record3.commit()
assert 'e' in record3['_oaisets']
assert len(record3['_oaisets']) == 1
# test update search_pattern
i.search_pattern = 'title:Test3'
db.session.add(i)
db.session.commit()
record3.commit()
assert 'e' in record3['_oaisets']
assert 'i' in record3['_oaisets']
assert len(record3['_oaisets']) == 2
# test update name
a.name = "new-a"
db.session.add(a)
db.session.commit()
record3.commit()
assert 'i' in record3['_oaisets']
assert 'e' in record3['_oaisets']
assert len(record3['_oaisets']) == 2
# test update name
c.name = "new-c"
db.session.add(c)
db.session.commit()
record0.commit()
assert 'new-c' in record0['_oaisets']
assert len(record0['_oaisets']) == 1
def _try_populate_oaisets():
"""Try to update collections."""
schema = {
'type': 'object',
'properties': {
'title': {'type': 'string'},
'field': {'type': 'boolean'},
},
'required': ['title'],
}
a = OAISet(spec="a")
b = OAISet(spec="b")
e = OAISet(
spec="e", search_pattern="title:Test2 OR title:Test3")
c = OAISet(spec="c", search_pattern="title:Test0")
d = OAISet(spec="d", search_pattern="title:Test1")
f = OAISet(spec="f", search_pattern="title:Test2")
g = OAISet(spec="g")
h = OAISet(spec="h")
i = OAISet(spec="i", search_pattern="title:Test3")
j = OAISet(spec="j", search_pattern="title:Test4")
with db.session.begin_nested():
for coll in [a, b, c, d, e, f, g, h, i, j]:
db.session.add(coll)
db.session.commit()
# start tests
record0 = Record.create({'title': 'Test0', '$schema': schema})
assert 'c' in record0['_oai']['sets']
assert len(record0['_oai']['sets']) == 1
record = Record.create({'title': 'TestNotFound', '$schema': schema})
assert record['_oai']['sets'] == []
record = Record.create({'title': 'Test1', '$schema': schema})
assert 'd' in record['_oai']['sets']
assert len(record['_oai']['sets']) == 1
record = Record.create({'title': 'Test2', '$schema': schema})
assert 'e' in record['_oai']['sets']
assert 'f' in record['_oai']['sets']
assert len(record['_oai']['sets']) == 2
record3 = Record.create({'title': 'Test3', '$schema': schema})
assert 'e' in record3['_oai']['sets']
assert 'i' in record3['_oai']['sets']
assert len(record3['_oai']['sets']) == 2
record4 = Record.create({'title': 'Test4', '$schema': schema})
assert 'j' in record4['_oai']['sets']
assert len(record4['_oai']['sets']) == 1
# test delete
db.session.delete(j)
db.session.commit()
record4.commit()
assert 'h' not in record4['_oai']['sets']
assert 'j' not in record4['_oai']['sets']
assert len(record4['_oai']['sets']) == 0
# test update search_pattern
i.search_pattern = None
db.session.add(i)
db.session.commit()
record3.commit()
assert 'e' in record3['_oai']['sets']
assert len(record3['_oai']['sets']) == 1
# test update search_pattern
i.search_pattern = 'title:Test3'
db.session.add(i)
db.session.commit()
record3.commit()
assert 'e' in record3['_oai']['sets']
assert 'i' in record3['_oai']['sets']
assert len(record3['_oai']['sets']) == 2
# test update the spec
a.spec = "new-a"
db.session.add(a)
db.session.commit()
record3.commit()
assert 'i' in record3['_oai']['sets']
assert 'e' in record3['_oai']['sets']
assert len(record3['_oai']['sets']) == 2
# test update name
c.spec = "new-c"
db.session.add(c)
db.session.commit()
record0.commit()
assert 'new-c' in record0['_oai']['sets']
assert len(record0['_oai']['sets']) == 1
