def add_oaiset_spec(record_uuid, spec):
"""Add the OAI spec to the record and commit."""
rec = Record.get_record(record_uuid)
rec['_oai']['sets'] = sorted(rec['_oai'].get('sets', []) + [spec, ])
rec['_oai']['updated'] = datetime_to_datestamp(datetime.utcnow())
rec.commit()
db.session.commit()
RecordIndexer().bulk_index([str(rec.id), ])
def remove_oaiset_spec(record_uuid, spec):
"""Remove the OAI spec from the record and commit."""
rec = Record.get_record(record_uuid)
rec['_oai']['sets'] = sorted([s for s in rec['_oai'].get('sets', [])
if s != spec])
rec['_oai']['updated'] = datetime_to_datestamp(datetime.utcnow())
if not rec['_oai']['sets']:
del rec['_oai']['sets']
rec.commit()
db.session.commit()
RecordIndexer().bulk_index([str(rec.id), ])
def add_oaiset_spec(record_uuid, spec):
"""Add the OAI spec to the record and commit."""
rec = Record.get_record(record_uuid)
rec['_oai']['sets'] = sorted(rec['_oai'].get('sets', []) + [
spec,
])
rec['_oai']['updated'] = datetime_to_datestamp(datetime.utcnow())
rec.commit()
db.session.commit()
RecordIndexer().bulk_index([
str(rec.id),
])
def remove_oaiset_spec(record_uuid, spec):
"""Remove the OAI spec from the record and commit."""
rec = Record.get_record(record_uuid)
rec['_oai']['sets'] = sorted(
[s for s in rec['_oai'].get('sets', []) if s != spec])
rec['_oai']['updated'] = datetime_to_datestamp(datetime.utcnow())
if not rec['_oai']['sets']:
del rec['_oai']['sets']
rec.commit()
db.session.commit()
RecordIndexer().bulk_index([
str(rec.id),
])
def test_getrecord(app):
"""Test get record verb."""
with app.test_request_context():
pid_value = 'oai:legacy:1'
with db.session.begin_nested():
record_id = uuid.uuid4()
data = {
'_oai': {
'id': pid_value
},
'title_statement': {
'title': 'Test0'
},
}
pid = oaiid_minter(record_id, data)
record = Record.create(data, id_=record_id)
db.session.commit()
assert pid_value == pid.pid_value
record_updated = record.updated
with app.test_client() as c:
result = c.get(
'/oai2d?verb=GetRecord&identifier={0}&metadataPrefix=oai_dc'.
format(pid_value))
assert 200 == result.status_code
tree = etree.fromstring(result.data)
assert len(tree.xpath('/x:OAI-PMH', namespaces=NAMESPACES)) == 1
assert len(
tree.xpath('/x:OAI-PMH/x:GetRecord',
namespaces=NAMESPACES)) == 1
assert len(
tree.xpath('/x:OAI-PMH/x:GetRecord/x:record/x:header',
namespaces=NAMESPACES)) == 1
assert len(
tree.xpath(
'/x:OAI-PMH/x:GetRecord/x:record/x:header/x:identifier',
namespaces=NAMESPACES)) == 1
identifier = tree.xpath(
'/x:OAI-PMH/x:GetRecord/x:record/x:header/x:identifier/text()',
namespaces=NAMESPACES)
assert identifier == [pid_value]
datestamp = tree.xpath(
'/x:OAI-PMH/x:GetRecord/x:record/x:header/x:datestamp/text()',
namespaces=NAMESPACES)
assert datestamp == [datetime_to_datestamp(record_updated)]
assert len(
tree.xpath('/x:OAI-PMH/x:GetRecord/x:record/x:metadata',
namespaces=NAMESPACES)) == 1
def sync_record_oai(uuid, cache=None):
"""Mint OAI ID information for the record.
:type uuid: str
"""
rec = Record.get_record(uuid)
recid_s = str(rec['recid'])
# Try to get the already existing OAI PID for this record
oai_pid_q = PersistentIdentifier.query.filter_by(pid_type='oai',
object_uuid=rec.id)
if oai_pid_q.count() == 0:
pid = oaiid_minter(rec.id, rec)
synced_sets = get_synced_sets(rec, cache=cache)
rec['_oai']['sets'] = synced_sets
rec.commit()
db.session.commit()
RecordIndexer().bulk_index([
str(rec.id),
])
logger.info('Minted new OAI PID ({pid}) for record {id}'.format(
pid=pid, id=uuid))
elif oai_pid_q.count() == 1:
pid = oai_pid_q.one()
managed_prefixes = current_app.config['OAISERVER_MANAGED_ID_PREFIXES']
if not any(pid.pid_value.startswith(p) for p in managed_prefixes):
logger.exception('Unknown OAIID prefix: {0}'.format(pid.pid_value))
elif str(pid.get_assigned_object()) != uuid:
logger.exception('OAI PID ({pid}) for record {id} ({recid}) is '
'pointing to a different object ({id2})'.format(
pid=pid,
id=uuid,
id2=str(pid.get_assigned_object()),
recid=recid_s))
elif requires_sync(rec, cache=cache):
rec.setdefault('_oai', {})
rec['_oai']['id'] = pid.pid_value
rec['_oai']['updated'] = datetime_to_datestamp(datetime.utcnow())
synced_sets = get_synced_sets(rec, cache=cache)
rec['_oai']['sets'] = synced_sets
if not rec['_oai']['sets']:
del rec['_oai']['sets'] # Don't store empty list
rec.commit()
db.session.commit()
RecordIndexer().bulk_index([
str(rec.id),
])
logger.info('Matching OAI PID ({pid}) for record {id}'.format(
pid=pid, id=uuid))
def zenodo_oaiid_minter(record_uuid, data):
"""Mint OAI identifiers."""
pid_value = data.get('_oai', {}).get('id')
if pid_value is None:
assert 'recid' in data
pid_value = current_app.config.get('OAISERVER_ID_PREFIX', '') + str(
data['recid']
)
provider = OAIIDProvider.create(
object_type='rec', object_uuid=record_uuid,
pid_value=str(pid_value)
)
data.setdefault('_oai', {})
data['_oai']['id'] = provider.pid.pid_value
data['_oai']['updated'] = datetime_to_datestamp(datetime.utcnow())
return provider.pid
def b2share_oaiid_minter(rec_pid, data):
"""Mint record identifiers."""
oai_pid_value = data.get('_oai', {}).get('id')
if oai_pid_value is None:
oai_prefix = current_app.config.get('OAISERVER_ID_PREFIX', 'oai:')
oai_pid_value = str(oai_prefix) + str(rec_pid.pid_value)
provider = OAIIDProvider.create(object_type='rec',
object_uuid=rec_pid.object_uuid,
pid_value=oai_pid_value)
data.setdefault('_oai', {})
data['_oai'].update({
'id': oai_pid_value,
'sets': [data.get('community')], # community_id == setSpec
'updated': datetime_to_datestamp(datetime.utcnow()),
})
return provider.pid
def b2share_oaiid_minter(record_uuid, data):
"""Mint record identifiers."""
pid_value = data.get('_oai', {}).get('id')
if pid_value is None:
assert '_deposit' in data and 'id' in data['_deposit']
id_ = str(data['_deposit']['id'])
pid_value = current_app.config.get('OAISERVER_ID_PREFIX', '') + id_
provider = OAIIDProvider.create(object_type='rec',
object_uuid=record_uuid,
pid_value=str(pid_value))
data.setdefault('_oai', {})
data['_oai'].update({
'id': provider.pid.pid_value,
'sets': [],
'updated': datetime_to_datestamp(datetime.utcnow()),
})
return provider.pid
def b2share_oaiid_minter(rec_pid, data):
"""Mint record identifiers."""
oai_pid_value = data.get('_oai', {}).get('id')
if oai_pid_value is None:
oai_prefix = current_app.config.get('OAISERVER_ID_PREFIX', 'oai:')
oai_pid_value = str(oai_prefix) + str(rec_pid.pid_value)
provider = OAIIDProvider.create(
object_type='rec',
object_uuid=rec_pid.object_uuid,
pid_value=oai_pid_value
)
data.setdefault('_oai', {})
data['_oai'].update({
'id': oai_pid_value,
'sets': [data.get('community')], # community_id == setSpec
'updated': datetime_to_datestamp(datetime.utcnow()),
})
return provider.pid
def sync_record_oai(uuid, cache=None):
"""Mint OAI ID information for the record.
:type uuid: str
"""
rec = Record.get_record(uuid)
recid_s = str(rec['recid'])
# Try to get the already existing OAI PID for this record
oai_pid_q = PersistentIdentifier.query.filter_by(pid_type='oai',
object_uuid=rec.id)
if oai_pid_q.count() == 0:
pid = oaiid_minter(rec.id, rec)
synced_sets = get_synced_sets(rec, cache=cache)
rec['_oai']['sets'] = synced_sets
rec.commit()
db.session.commit()
RecordIndexer().bulk_index([str(rec.id), ])
logger.info('Minted new OAI PID ({pid}) for record {id}'.format(
pid=pid, id=uuid))
elif oai_pid_q.count() == 1:
pid = oai_pid_q.one()
managed_prefixes = current_app.config['OAISERVER_MANAGED_ID_PREFIXES']
if not any(pid.pid_value.startswith(p) for p in managed_prefixes):
logger.exception('Unknown OAIID prefix: {0}'.format(pid.pid_value))
elif str(pid.get_assigned_object()) != uuid:
logger.exception(
'OAI PID ({pid}) for record {id} ({recid}) is '
'pointing to a different object ({id2})'.format(
pid=pid, id=uuid, id2=str(pid.get_assigned_object()),
recid=recid_s))
elif requires_sync(rec, cache=cache):
rec.setdefault('_oai', {})
rec['_oai']['id'] = pid.pid_value
rec['_oai']['updated'] = datetime_to_datestamp(datetime.utcnow())
synced_sets = get_synced_sets(rec, cache=cache)
rec['_oai']['sets'] = synced_sets
if not rec['_oai']['sets']:
del rec['_oai']['sets'] # Don't store empty list
rec.commit()
db.session.commit()
RecordIndexer().bulk_index([str(rec.id), ])
logger.info('Matching OAI PID ({pid}) for record {id}'.format(
pid=pid, id=uuid))
def test_getrecord(app):
"""Test get record verb."""
with app.test_request_context():
pid_value = 'oai:legacy:1'
with db.session.begin_nested():
record_id = uuid.uuid4()
data = {
'_oai': {'id': pid_value},
'title_statement': {'title': 'Test0'},
}
pid = oaiid_minter(record_id, data)
record = Record.create(data, id_=record_id)
db.session.commit()
assert pid_value == pid.pid_value
record_updated = record.updated
with app.test_client() as c:
result = c.get(
'/oai2d?verb=GetRecord&identifier={0}&metadataPrefix=oai_dc'
.format(pid_value))
assert 200 == result.status_code
tree = etree.fromstring(result.data)
assert len(tree.xpath('/x:OAI-PMH', namespaces=NAMESPACES)) == 1
assert len(tree.xpath('/x:OAI-PMH/x:GetRecord',
namespaces=NAMESPACES)) == 1
assert len(tree.xpath('/x:OAI-PMH/x:GetRecord/x:record/x:header',
namespaces=NAMESPACES)) == 1
assert len(tree.xpath(
'/x:OAI-PMH/x:GetRecord/x:record/x:header/x:identifier',
namespaces=NAMESPACES)) == 1
identifier = tree.xpath(
'/x:OAI-PMH/x:GetRecord/x:record/x:header/x:identifier/text()',
namespaces=NAMESPACES)
assert identifier == [pid_value]
datestamp = tree.xpath(
'/x:OAI-PMH/x:GetRecord/x:record/x:header/x:datestamp/text()',
namespaces=NAMESPACES)
assert datestamp == [datetime_to_datestamp(record_updated)]
assert len(tree.xpath('/x:OAI-PMH/x:GetRecord/x:record/x:metadata',
namespaces=NAMESPACES)) == 1
def test_listidentifiers(app):
"""Test verb ListIdentifiers."""
from invenio_oaiserver.models import OAISet
with app.app_context():
current_oaiserver.unregister_signals_oaiset()
# create new OAI Set
with db.session.begin_nested():
oaiset = OAISet(
spec='test0',
name='Test0',
description='test desc 0',
search_pattern='title_statement.title:Test0',
)
db.session.add(oaiset)
db.session.commit()
run_after_insert_oai_set()
with app.test_request_context():
indexer = RecordIndexer()
# create a new record (inside the OAI Set)
with db.session.begin_nested():
record_id = uuid.uuid4()
data = {'title_statement': {'title': 'Test0'}}
recid_minter(record_id, data)
pid = oaiid_minter(record_id, data)
record = Record.create(data, id_=record_id)
db.session.commit()
indexer.index_by_id(record_id)
current_search.flush_and_refresh('_all')
pid_value = pid.pid_value
# get the list of identifiers
with app.test_client() as c:
result = c.get('/oai2d?verb=ListIdentifiers&metadataPrefix=oai_dc')
tree = etree.fromstring(result.data)
assert len(tree.xpath('/x:OAI-PMH', namespaces=NAMESPACES)) == 1
assert len(
tree.xpath('/x:OAI-PMH/x:ListIdentifiers',
namespaces=NAMESPACES)) == 1
assert len(
tree.xpath('/x:OAI-PMH/x:ListIdentifiers/x:header',
namespaces=NAMESPACES)) == 1
identifier = tree.xpath(
'/x:OAI-PMH/x:ListIdentifiers/x:header/x:identifier',
namespaces=NAMESPACES)
assert len(identifier) == 1
assert identifier[0].text == str(pid_value)
datestamp = tree.xpath(
'/x:OAI-PMH/x:ListIdentifiers/x:header/x:datestamp',
namespaces=NAMESPACES)
assert len(datestamp) == 1
assert datestamp[0].text == datetime_to_datestamp(record.updated)
# Check from:until range
with app.test_client() as c:
# Check date and datetime timestamps.
for granularity in (False, True):
result = c.get(
'/oai2d?verb=ListIdentifiers&metadataPrefix=oai_dc'
'&from={0}&until={1}&set=test0'.format(
datetime_to_datestamp(record.updated - timedelta(1),
day_granularity=granularity),
datetime_to_datestamp(record.updated + timedelta(1),
day_granularity=granularity),
))
assert result.status_code == 200
tree = etree.fromstring(result.data)
identifier = tree.xpath(
'/x:OAI-PMH/x:ListIdentifiers/x:header/x:identifier',
namespaces=NAMESPACES)
assert len(identifier) == 1
def test_listrecords(app):
"""Test ListRecords."""
total = 12
record_ids = []
with app.test_request_context():
indexer = RecordIndexer()
with db.session.begin_nested():
for idx in range(total):
record_id = uuid.uuid4()
data = {'title_statement': {'title': 'Test{0}'.format(idx)}}
recid_minter(record_id, data)
oaiid_minter(record_id, data)
record = Record.create(data, id_=record_id)
record_ids.append(record_id)
db.session.commit()
for record_id in record_ids:
indexer.index_by_id(record_id)
current_search.flush_and_refresh('_all')
with app.test_client() as c:
result = c.get('/oai2d?verb=ListRecords&metadataPrefix=oai_dc')
tree = etree.fromstring(result.data)
assert len(tree.xpath('/x:OAI-PMH', namespaces=NAMESPACES)) == 1
assert len(
tree.xpath('/x:OAI-PMH/x:ListRecords', namespaces=NAMESPACES)) == 1
assert len(
tree.xpath('/x:OAI-PMH/x:ListRecords/x:record',
namespaces=NAMESPACES)) == 10
assert len(
tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:header',
namespaces=NAMESPACES)) == 10
assert len(
tree.xpath(
'/x:OAI-PMH/x:ListRecords/x:record/x:header'
'/x:identifier',
namespaces=NAMESPACES)) == 10
assert len(
tree.xpath(
'/x:OAI-PMH/x:ListRecords/x:record/x:header'
'/x:datestamp',
namespaces=NAMESPACES)) == 10
assert len(
tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:metadata',
namespaces=NAMESPACES)) == 10
resumption_token = tree.xpath(
'/x:OAI-PMH/x:ListRecords/x:resumptionToken',
namespaces=NAMESPACES)[0]
assert resumption_token.text
with app.test_client() as c:
result = c.get(
'/oai2d?verb=ListRecords&resumptionToken={0}'.format(
resumption_token.text))
tree = etree.fromstring(result.data)
assert len(tree.xpath('/x:OAI-PMH', namespaces=NAMESPACES)) == 1
assert len(
tree.xpath('/x:OAI-PMH/x:ListRecords', namespaces=NAMESPACES)) == 1
assert len(
tree.xpath('/x:OAI-PMH/x:ListRecords/x:record',
namespaces=NAMESPACES)) == 2
assert len(
tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:header',
namespaces=NAMESPACES)) == 2
assert len(
tree.xpath(
'/x:OAI-PMH/x:ListRecords/x:record/x:header'
'/x:identifier',
namespaces=NAMESPACES)) == 2
assert len(
tree.xpath(
'/x:OAI-PMH/x:ListRecords/x:record/x:header'
'/x:datestamp',
namespaces=NAMESPACES)) == 2
assert len(
tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:metadata',
namespaces=NAMESPACES)) == 2
resumption_token = tree.xpath(
'/x:OAI-PMH/x:ListRecords/x:resumptionToken',
namespaces=NAMESPACES)[0]
assert not resumption_token.text
# Check from:until range
with app.test_client() as c:
# Check date and datetime timestamps.
for granularity in (False, True):
result = c.get('/oai2d?verb=ListRecords&metadataPrefix=oai_dc'
'&from={0}&until={1}'.format(
datetime_to_datestamp(
record.updated - timedelta(days=1),
day_granularity=granularity),
datetime_to_datestamp(
record.updated + timedelta(days=1),
day_granularity=granularity),
))
assert result.status_code == 200
tree = etree.fromstring(result.data)
assert len(
tree.xpath('/x:OAI-PMH/x:ListRecords/x:record',
namespaces=NAMESPACES)) == 10
def test_listidentifiers(app):
"""Test verb ListIdentifiers."""
from invenio_oaiserver.models import OAISet
with app.app_context():
current_oaiserver.unregister_signals_oaiset()
# create new OAI Set
with db.session.begin_nested():
oaiset = OAISet(
spec='test0',
name='Test0',
description='test desc 0',
search_pattern='title_statement.title:Test0',
)
db.session.add(oaiset)
db.session.commit()
run_after_insert_oai_set()
with app.test_request_context():
indexer = RecordIndexer()
# create a new record (inside the OAI Set)
with db.session.begin_nested():
record_id = uuid.uuid4()
data = {'title_statement': {'title': 'Test0'}}
recid_minter(record_id, data)
pid = oaiid_minter(record_id, data)
record = Record.create(data, id_=record_id)
db.session.commit()
indexer.index_by_id(record_id)
current_search.flush_and_refresh('_all')
pid_value = pid.pid_value
# get the list of identifiers
with app.test_client() as c:
result = c.get(
'/oai2d?verb=ListIdentifiers&metadataPrefix=oai_dc'
)
tree = etree.fromstring(result.data)
assert len(tree.xpath('/x:OAI-PMH', namespaces=NAMESPACES)) == 1
assert len(tree.xpath('/x:OAI-PMH/x:ListIdentifiers',
namespaces=NAMESPACES)) == 1
assert len(tree.xpath('/x:OAI-PMH/x:ListIdentifiers/x:header',
namespaces=NAMESPACES)) == 1
identifier = tree.xpath(
'/x:OAI-PMH/x:ListIdentifiers/x:header/x:identifier',
namespaces=NAMESPACES
)
assert len(identifier) == 1
assert identifier[0].text == str(pid_value)
datestamp = tree.xpath(
'/x:OAI-PMH/x:ListIdentifiers/x:header/x:datestamp',
namespaces=NAMESPACES
)
assert len(datestamp) == 1
assert datestamp[0].text == datetime_to_datestamp(record.updated)
# Check from:until range
with app.test_client() as c:
# Check date and datetime timestamps.
for granularity in (False, True):
result = c.get(
'/oai2d?verb=ListIdentifiers&metadataPrefix=oai_dc'
'&from={0}&until={1}&set=test0'.format(
datetime_to_datestamp(
record.updated - timedelta(1),
day_granularity=granularity),
datetime_to_datestamp(
record.updated + timedelta(1),
day_granularity=granularity),
)
)
assert result.status_code == 200
tree = etree.fromstring(result.data)
identifier = tree.xpath(
'/x:OAI-PMH/x:ListIdentifiers/x:header/x:identifier',
namespaces=NAMESPACES
)
assert len(identifier) == 1
def test_listrecords(app):
"""Test ListRecords."""
total = 12
record_ids = []
with app.test_request_context():
indexer = RecordIndexer()
with db.session.begin_nested():
for idx in range(total):
record_id = uuid.uuid4()
data = {'title_statement': {'title': 'Test{0}'.format(idx)}}
recid_minter(record_id, data)
oaiid_minter(record_id, data)
record = Record.create(data, id_=record_id)
record_ids.append(record_id)
db.session.commit()
for record_id in record_ids:
indexer.index_by_id(record_id)
current_search.flush_and_refresh('_all')
with app.test_client() as c:
result = c.get('/oai2d?verb=ListRecords&metadataPrefix=oai_dc')
tree = etree.fromstring(result.data)
assert len(tree.xpath('/x:OAI-PMH', namespaces=NAMESPACES)) == 1
assert len(tree.xpath('/x:OAI-PMH/x:ListRecords',
namespaces=NAMESPACES)) == 1
assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record',
namespaces=NAMESPACES)) == 10
assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:header',
namespaces=NAMESPACES)) == 10
assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:header'
'/x:identifier', namespaces=NAMESPACES)) == 10
assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:header'
'/x:datestamp', namespaces=NAMESPACES)) == 10
assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:metadata',
namespaces=NAMESPACES)) == 10
resumption_token = tree.xpath(
'/x:OAI-PMH/x:ListRecords/x:resumptionToken', namespaces=NAMESPACES
)[0]
assert resumption_token.text
with app.test_client() as c:
result = c.get(
'/oai2d?verb=ListRecords&resumptionToken={0}'.format(
resumption_token.text
)
)
tree = etree.fromstring(result.data)
assert len(tree.xpath('/x:OAI-PMH', namespaces=NAMESPACES)) == 1
assert len(tree.xpath('/x:OAI-PMH/x:ListRecords',
namespaces=NAMESPACES)) == 1
assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record',
namespaces=NAMESPACES)) == 2
assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:header',
namespaces=NAMESPACES)) == 2
assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:header'
'/x:identifier', namespaces=NAMESPACES)) == 2
assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:header'
'/x:datestamp', namespaces=NAMESPACES)) == 2
assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:metadata',
namespaces=NAMESPACES)) == 2
resumption_token = tree.xpath(
'/x:OAI-PMH/x:ListRecords/x:resumptionToken', namespaces=NAMESPACES
)[0]
assert not resumption_token.text
# Check from:until range
with app.test_client() as c:
# Check date and datetime timestamps.
for granularity in (False, True):
result = c.get(
'/oai2d?verb=ListRecords&metadataPrefix=oai_dc'
'&from={0}&until={1}'.format(
datetime_to_datestamp(
record.updated - timedelta(days=1),
day_granularity=granularity),
datetime_to_datestamp(
record.updated + timedelta(days=1),
day_granularity=granularity),
)
)
assert result.status_code == 200
tree = etree.fromstring(result.data)
assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record',
namespaces=NAMESPACES)) == 10
