def test_search_pattern_change(app, without_oaiset_signals, schema):
"""Test search pattern change."""
record0 = create_record(app, {
'_oai': {'sets': ['a']}, 'title_statement': {'title': 'Test0'},
'$schema': schema
})
rec_uuid = record0.id
oaiset = OAISet(spec="a", search_pattern="title_statement.title:Test0")
db.session.add(oaiset)
db.session.commit()
run_after_insert_oai_set()
sleep(2)
record = Record.get_record(rec_uuid)
assert record['_oai']['sets'] == ['a']
# change search pattern: record0 will not inside it anymore
oaiset = OAISet.query.first()
oaiset.search_pattern = 'title_statement.title:Test1'
db.session.merge(oaiset)
db.session.commit()
after_update_oai_set(None, None, oaiset)
sleep(2)
record = Record.get_record(rec_uuid)
record.commit()
assert record['_oai']['sets'] == []
def test_response(db, schema, client):
oaiset = OAISet(spec='higgs', name='Higgs', description='...')
oaiset.search_pattern = 'title:higgs'
db.session.add(oaiset)
db.session.add(oaiset)
res = client.get('/oai2d?verb=ListSets')
print(res.data)
def oai_server(sender, app=None, **kwargs):
#app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite://'
app.config['CELERY_TASK_ALWAYS_EAGER'] = True
if not hasattr(app, 'cli'):
from flask_cli import FlaskCLI
ext_cli = FlaskCLI(app)
ext_db = InvenioDB(app)
ext_indexer = InvenioIndexer(app)
ext_pidstore = InvenioPIDStore(app)
ext_records = InvenioRecords(app)
ext_search = InvenioSearch(app)
ext_celery = FlaskCeleryExt(app)
app.config['OAISERVER_RECORD_INDEX'] = ['authors', 'records']
app.config['OAISERVER_ID_PREFIX'] = 'oai:example:'
app.config['OAISERVER_QUERY_PARSER_FIELDS'] = ["title"]
app.config['OAISERVER_METADATA_FORMATS'] = {
'oai_dc': {
'serializer': ('invenio_oaiserver.utils:dumps_etree', {
'xslt_filename':
pkg_resources.resource_filename(
'testinvenio.records', 'static/xsl/MARC21slim2OAIDC.xsl'),
'xslt_filename':
pathlib.Path(
"/home/alzbeta/testinvenio/testinvenio/records/static/xsl/MARC21slim2OAIDC.xsl"
)
}),
'schema':
'http://json-schema.org/draft-04/schema#',
'namespace':
'http://json-schema.org/draft-04/schema#',
}
}
ext_oaiserver = InvenioOAIServer(app)
app.register_blueprint(blueprint)
ctx = app.app_context()
ctx.push()
db.create_all()
ext_search.flush_and_refresh('_all')
oaiset = OAISet(spec='pattern', name='Pattern', description='...')
#nefunguje pokud search pattern ma pole ktere neni v obouch modelech...
oaiset.search_pattern = 'title:Some title1'
try:
print("1")
db.session.add(oaiset)
db.session.commit()
except:
pass
def add_set(spec, name, pattern, description='...'):
"""Add OAI set.
:param spec: set identifier
:param name: human readable name of the set
:param pattern: search pattern to get records
:param description: human readable description
"""
try:
oaiset = OAISet(spec=spec, name=name, description=description)
oaiset.search_pattern = pattern
db.session.add(oaiset)
db.session.commit()
msg = 'OAIset added: {name}'.format(name=name)
except Exception as err:
db.session.rollback()
msg = 'OAIset exist: {name}'.format(name=name)
return msg
def _try_populate_oaisets():
"""Try to update collections."""
schema = {
'type': 'object',
'properties': {
'title': {
'type': 'string'
},
'field': {
'type': 'boolean'
},
},
'required': ['title'],
}
a = OAISet(name="a")
b = OAISet(name="b")
e = OAISet(name="e", search_pattern="title:Test2 OR title:Test3")
c = OAISet(name="c", search_pattern="title:Test0")
d = OAISet(name="d", search_pattern="title:Test1")
f = OAISet(name="f", search_pattern="title:Test2")
g = OAISet(name="g")
h = OAISet(name="h")
i = OAISet(name="i", search_pattern="title:Test3")
j = OAISet(name="j", search_pattern="title:Test4")
with db.session.begin_nested():
for coll in [a, b, c, d, e, f, g, h, i, j]:
db.session.add(coll)
db.session.commit()
# start tests
record0 = Record.create({'title': 'Test0', '$schema': schema})
assert 'c' in record0['_oaisets']
assert len(record0['_oaisets']) == 1
record = Record.create({'title': 'TestNotFound', '$schema': schema})
assert record['_oaisets'] == []
record = Record.create({'title': 'Test1', '$schema': schema})
assert 'd' in record['_oaisets']
assert len(record['_oaisets']) == 1
record = Record.create({'title': 'Test2', '$schema': schema})
assert 'e' in record['_oaisets']
assert 'f' in record['_oaisets']
assert len(record['_oaisets']) == 2
record3 = Record.create({'title': 'Test3', '$schema': schema})
assert 'e' in record3['_oaisets']
assert 'i' in record3['_oaisets']
assert len(record3['_oaisets']) == 2
record4 = Record.create({'title': 'Test4', '$schema': schema})
assert 'j' in record4['_oaisets']
assert len(record4['_oaisets']) == 1
# test delete
db.session.delete(j)
db.session.commit()
record4.commit()
assert 'h' not in record4['_oaisets']
assert 'j' not in record4['_oaisets']
assert len(record4['_oaisets']) == 0
# test update search_pattern
i.search_pattern = None
db.session.add(i)
db.session.commit()
record3.commit()
assert 'e' in record3['_oaisets']
assert len(record3['_oaisets']) == 1
# test update search_pattern
i.search_pattern = 'title:Test3'
db.session.add(i)
db.session.commit()
record3.commit()
assert 'e' in record3['_oaisets']
assert 'i' in record3['_oaisets']
assert len(record3['_oaisets']) == 2
# test update name
a.name = "new-a"
db.session.add(a)
db.session.commit()
record3.commit()
assert 'i' in record3['_oaisets']
assert 'e' in record3['_oaisets']
assert len(record3['_oaisets']) == 2
# test update name
c.name = "new-c"
db.session.add(c)
db.session.commit()
record0.commit()
assert 'new-c' in record0['_oaisets']
assert len(record0['_oaisets']) == 1
def _try_populate_oaisets():
"""Try to update collections."""
schema = {
'type': 'object',
'properties': {
'title': {'type': 'string'},
'field': {'type': 'boolean'},
},
'required': ['title'],
}
a = OAISet(spec="a")
b = OAISet(spec="b")
e = OAISet(
spec="e", search_pattern="title:Test2 OR title:Test3")
c = OAISet(spec="c", search_pattern="title:Test0")
d = OAISet(spec="d", search_pattern="title:Test1")
f = OAISet(spec="f", search_pattern="title:Test2")
g = OAISet(spec="g")
h = OAISet(spec="h")
i = OAISet(spec="i", search_pattern="title:Test3")
j = OAISet(spec="j", search_pattern="title:Test4")
with db.session.begin_nested():
for coll in [a, b, c, d, e, f, g, h, i, j]:
db.session.add(coll)
db.session.commit()
# start tests
record0 = Record.create({'title': 'Test0', '$schema': schema})
assert 'c' in record0['_oai']['sets']
assert len(record0['_oai']['sets']) == 1
record = Record.create({'title': 'TestNotFound', '$schema': schema})
assert record['_oai']['sets'] == []
record = Record.create({'title': 'Test1', '$schema': schema})
assert 'd' in record['_oai']['sets']
assert len(record['_oai']['sets']) == 1
record = Record.create({'title': 'Test2', '$schema': schema})
assert 'e' in record['_oai']['sets']
assert 'f' in record['_oai']['sets']
assert len(record['_oai']['sets']) == 2
record3 = Record.create({'title': 'Test3', '$schema': schema})
assert 'e' in record3['_oai']['sets']
assert 'i' in record3['_oai']['sets']
assert len(record3['_oai']['sets']) == 2
record4 = Record.create({'title': 'Test4', '$schema': schema})
assert 'j' in record4['_oai']['sets']
assert len(record4['_oai']['sets']) == 1
# test delete
db.session.delete(j)
db.session.commit()
record4.commit()
assert 'h' not in record4['_oai']['sets']
assert 'j' not in record4['_oai']['sets']
assert len(record4['_oai']['sets']) == 0
# test update search_pattern
i.search_pattern = None
db.session.add(i)
db.session.commit()
record3.commit()
assert 'e' in record3['_oai']['sets']
assert len(record3['_oai']['sets']) == 1
# test update search_pattern
i.search_pattern = 'title:Test3'
db.session.add(i)
db.session.commit()
record3.commit()
assert 'e' in record3['_oai']['sets']
assert 'i' in record3['_oai']['sets']
assert len(record3['_oai']['sets']) == 2
# test update the spec
a.spec = "new-a"
db.session.add(a)
db.session.commit()
record3.commit()
assert 'i' in record3['_oai']['sets']
assert 'e' in record3['_oai']['sets']
assert len(record3['_oai']['sets']) == 2
# test update name
c.spec = "new-c"
db.session.add(c)
db.session.commit()
record0.commit()
assert 'new-c' in record0['_oai']['sets']
assert len(record0['_oai']['sets']) == 1
def test_populate_oaisets(app, with_record_signals):
"""Populate OAISets."""
indexer = RecordIndexer()
schema = {
'allOf': [{
'type': 'object',
'properties': {
'title': {'type': 'string'},
'genre': {'type': 'string'},
'field': {'type': 'boolean'},
},
'required': ['title'],
}, {
'$ref': 'http://inveniosoftware.org/schemas/'
'oaiserver/internal-v1.0.0.json',
}]
}
def create_record(item_dict, mint_oaiid=True):
"""Create test record."""
with app.test_request_context():
record_id = uuid.uuid4()
recid_minter(record_id, item_dict)
if mint_oaiid:
oaiid_minter(record_id, item_dict)
record = Record.create(item_dict, id_=record_id)
indexer.index(record)
return record
a = OAISet(spec='a')
b = OAISet(spec='b')
e = OAISet(
spec="e", search_pattern="title:Test2 OR title:Test3")
c = OAISet(spec="c", search_pattern="title:Test0")
d = OAISet(spec="d", search_pattern="title:Test1")
f = OAISet(spec="f", search_pattern="title:Test2")
g = OAISet(spec="g")
h = OAISet(spec="h")
i = OAISet(spec="i", search_pattern="title:Test3")
j = OAISet(spec="j with space", search_pattern="title:Test4")
# Note below: brackets around AND search query are required
l = OAISet(spec="math",
search_pattern="(title:foo AND genre:math)")
m = OAISet(spec="nonmath",
search_pattern="(title:foo AND -genre:math)")
with db.session.begin_nested():
for oaiset in [a, b, c, d, e, f, g, h, i, j, l, m]:
db.session.add(oaiset)
db.session.commit()
a_id = a.id
i_id = i.id
# start tests
record0 = create_record({
'_oai': {'sets': ['a']}, 'title': 'Test0', '$schema': schema
})
assert 'a' in record0['_oai']['sets'], 'Keep manually managed set "a".'
assert 'c' in record0['_oai']['sets']
assert len(record0['_oai']['sets']) == 2
record_not_found = create_record(
{'title': 'TestNotFound', '$schema': schema}
)
# Don't create empty sets list just because of commit
assert 'sets' not in record_not_found['_oai']
record1 = create_record({'title': 'Test1', '$schema': schema})
assert 'd' in record1['_oai']['sets']
assert len(record1['_oai']['sets']) == 1
record2 = create_record({'title': 'Test2', '$schema': schema})
record2_id = record2.id
assert 'e' in record2['_oai']['sets']
assert 'f' in record2['_oai']['sets']
assert len(record2['_oai']['sets']) == 2
record3 = create_record({'title': 'Test3', '$schema': schema})
record3_id = record3.id
assert 'e' in record3['_oai']['sets']
assert 'i' in record3['_oai']['sets']
assert len(record3['_oai']['sets']) == 2
record4 = create_record({'title': 'Test4', '$schema': schema})
record4_id = record4.id
assert 'j with space' in record4['_oai']['sets']
assert len(record4['_oai']['sets']) == 1
# If record does not have '_oai', don't add any sets,
# nor even the default '_oai' key
record5 = create_record({'title': 'Test1', '$schema': schema},
mint_oaiid=False)
assert '_oai' not in record5
# If 'sets' before and after record commit are equivalent
# don't bump up the '_oai.updated' timestamp...
record6 = create_record({'title': 'Test1', '$schema': schema})
assert record6['_oai']['sets'] == ['d']
prev_updated_r6 = record6['_oai']['updated']
record6.commit()
assert record6['_oai']['sets'] == ['d']
assert record6['_oai']['updated'] == prev_updated_r6 # date stays the same
# ...but do bump up '_oai.updated' if the sets are different
record7 = create_record({'title': 'Test1', '$schema': schema})
assert record7['_oai']['sets'] == ['d']
prev_updated_r7 = record7['_oai']['updated']
sleep(1) # 'updated' timestamp is accurate to a second, hence the wait
record7['_oai']['sets'] = ['d', 'f'] # 'f' should be removed after commit
record7.commit()
assert record7['_oai']['sets'] == ['d']
assert record7['_oai']['updated'] != prev_updated_r7 # date bumped
# Test 'AND' keyword for records
record8 = create_record(
{'title': 'foo', 'genre': 'math', '$schema': schema})
assert record8['_oai']['sets'] == ['math', ]
record9 = create_record(
{'title': 'foo', 'genre': 'physics', '$schema': schema})
assert record9['_oai']['sets'] == ['nonmath', ]
record10 = create_record(
{'title': 'bar', 'genre': 'math', '$schema': schema})
assert 'sets' not in record10['_oai'] # title is not 'foo'
# wait ElasticSearch end to index records
sleep(10)
# test delete
current_oaiserver.unregister_signals_oaiset()
with patch('invenio_oaiserver.receivers.after_delete_oai_set') as f:
current_oaiserver.register_signals_oaiset()
with db.session.begin_nested():
db.session.delete(j)
db.session.commit()
assert f.called
after_delete_oai_set(None, None, j)
record4_model = RecordMetadata.query.filter_by(
id=record4_id).first().json
assert 'j with space' not in record4_model['_oai']['sets']
assert len(record4_model['_oai']['sets']) == 0
current_oaiserver.unregister_signals_oaiset()
# test update search_pattern
with patch('invenio_oaiserver.receivers.after_update_oai_set') as f:
current_oaiserver.register_signals_oaiset()
with db.session.begin_nested():
i.search_pattern = None
assert current_oaiserver.sets is None, 'Cache should be empty.'
db.session.merge(i)
db.session.commit()
assert f.called
i = OAISet.query.get(i_id)
after_update_oai_set(None, None, i)
record3_model = RecordMetadata.query.filter_by(
id=record3_id).first().json
assert 'i' in record3['_oai']['sets'], 'Set "i" is manually managed.'
assert 'e' in record3_model['_oai']['sets']
assert len(record3_model['_oai']['sets']) == 2
current_oaiserver.unregister_signals_oaiset()
# test update search_pattern
with patch('invenio_oaiserver.receivers.after_update_oai_set') as f:
current_oaiserver.register_signals_oaiset()
with db.session.begin_nested():
i.search_pattern = 'title:Test3'
db.session.merge(i)
db.session.commit()
assert f.called
i = OAISet.query.get(i_id)
after_update_oai_set(None, None, i)
record3_model = RecordMetadata.query.filter_by(
id=record3_id).first().json
assert 'e' in record3_model['_oai']['sets']
assert 'i' in record3_model['_oai']['sets']
assert len(record3_model['_oai']['sets']) == 2
current_oaiserver.unregister_signals_oaiset()
# test update the spec
with pytest.raises(OAISetSpecUpdateError) as exc_info:
a = OAISet.query.get(a_id)
a.spec = 'new-a'
assert exc_info.type is OAISetSpecUpdateError
# test create new set
with patch('invenio_oaiserver.receivers.after_insert_oai_set') as f:
current_oaiserver.register_signals_oaiset()
with db.session.begin_nested():
k = OAISet(spec="k", search_pattern="title:Test2")
db.session.add(k)
db.session.commit()
assert f.called
after_insert_oai_set(None, None, k)
record2_model = RecordMetadata.query.filter_by(
id=record2_id).first().json
assert 'e' in record2_model['_oai']['sets']
assert 'f' in record2_model['_oai']['sets']
assert 'k' in record2_model['_oai']['sets']
assert len(record2_model['_oai']['sets']) == 3
current_oaiserver.register_signals_oaiset()
def _try_populate_oaisets():
"""Try to update collections."""
indexer = RecordIndexer()
schema = {
'type': 'object',
'properties': {
'title': {'type': 'string'},
'field': {'type': 'boolean'},
},
'required': ['title'],
}
def create_record(item_dict):
"""Create test record."""
record_id = uuid.uuid4()
recid_minter(record_id, item_dict)
oaiid_minter(record_id, item_dict)
record = Record.create(item_dict, id_=record_id)
indexer.index(record)
return record
a = OAISet(spec='a')
b = OAISet(spec='b')
e = OAISet(
spec="e", search_pattern="title:Test2 OR title:Test3")
c = OAISet(spec="c", search_pattern="title:Test0")
d = OAISet(spec="d", search_pattern="title:Test1")
f = OAISet(spec="f", search_pattern="title:Test2")
g = OAISet(spec="g")
h = OAISet(spec="h")
i = OAISet(spec="i", search_pattern="title:Test3")
j = OAISet(spec="j with space", search_pattern="title:Test4")
with db.session.begin_nested():
for oaiset in [a, b, c, d, e, f, g, h, i, j]:
db.session.add(oaiset)
db.session.commit()
a_id = a.id
i_id = i.id
# start tests
record0 = create_record({
'_oai': {'sets': ['a']}, 'title': 'Test0', '$schema': schema
})
assert 'a' in record0['_oai']['sets'], 'Keep manually managed set "a".'
assert 'c' in record0['_oai']['sets']
assert len(record0['_oai']['sets']) == 2
record_not_found = create_record(
{'title': 'TestNotFound', '$schema': schema}
)
assert record_not_found['_oai']['sets'] == []
record1 = create_record({'title': 'Test1', '$schema': schema})
assert 'd' in record1['_oai']['sets']
assert len(record1['_oai']['sets']) == 1
record2 = create_record({'title': 'Test2', '$schema': schema})
record2_id = record2.id
assert 'e' in record2['_oai']['sets']
assert 'f' in record2['_oai']['sets']
assert len(record2['_oai']['sets']) == 2
record3 = create_record({'title': 'Test3', '$schema': schema})
record3_id = record3.id
assert 'e' in record3['_oai']['sets']
assert 'i' in record3['_oai']['sets']
assert len(record3['_oai']['sets']) == 2
record4 = create_record({'title': 'Test4', '$schema': schema})
record4_id = record4.id
assert 'j with space' in record4['_oai']['sets']
assert len(record4['_oai']['sets']) == 1
# wait ElasticSearch end to index records
sleep(10)
# test delete
current_oaiserver.unregister_signals_oaiset()
with patch('invenio_oaiserver.receivers.after_delete_oai_set') as f:
current_oaiserver.register_signals_oaiset()
with db.session.begin_nested():
db.session.delete(j)
db.session.commit()
assert f.called
after_delete_oai_set(None, None, j)
record4_model = RecordMetadata.query.filter_by(
id=record4_id).first().json
assert 'j with space' not in record4_model['_oai']['sets']
assert len(record4_model['_oai']['sets']) == 0
current_oaiserver.unregister_signals_oaiset()
# test update search_pattern
with patch('invenio_oaiserver.receivers.after_update_oai_set') as f:
current_oaiserver.register_signals_oaiset()
with db.session.begin_nested():
i.search_pattern = None
assert current_oaiserver.sets is None, 'Cache should be empty.'
db.session.merge(i)
db.session.commit()
assert f.called
i = OAISet.query.get(i_id)
after_update_oai_set(None, None, i)
record3_model = RecordMetadata.query.filter_by(
id=record3_id).first().json
assert 'i' in record3['_oai']['sets'], 'Set "i" is manually managed.'
assert 'e' in record3_model['_oai']['sets']
assert len(record3_model['_oai']['sets']) == 2
current_oaiserver.unregister_signals_oaiset()
# test update search_pattern
with patch('invenio_oaiserver.receivers.after_update_oai_set') as f:
current_oaiserver.register_signals_oaiset()
with db.session.begin_nested():
i.search_pattern = 'title:Test3'
db.session.merge(i)
db.session.commit()
assert f.called
i = OAISet.query.get(i_id)
after_update_oai_set(None, None, i)
record3_model = RecordMetadata.query.filter_by(
id=record3_id).first().json
assert 'e' in record3_model['_oai']['sets']
assert 'i' in record3_model['_oai']['sets']
assert len(record3_model['_oai']['sets']) == 2
current_oaiserver.unregister_signals_oaiset()
# test update the spec
with pytest.raises(OAISetSpecUpdateError) as exc_info:
a = OAISet.query.get(a_id)
a.spec = 'new-a'
assert exc_info.type is OAISetSpecUpdateError
# test create new set
with patch('invenio_oaiserver.receivers.after_insert_oai_set') as f:
current_oaiserver.register_signals_oaiset()
with db.session.begin_nested():
k = OAISet(spec="k", search_pattern="title:Test2")
db.session.add(k)
db.session.commit()
assert f.called
after_insert_oai_set(None, None, k)
record2_model = RecordMetadata.query.filter_by(
id=record2_id).first().json
assert 'e' in record2_model['_oai']['sets']
assert 'f' in record2_model['_oai']['sets']
assert 'k' in record2_model['_oai']['sets']
assert len(record2_model['_oai']['sets']) == 3
current_oaiserver.register_signals_oaiset()
