def add_oaiset_spec(record_uuid, spec): """Add the OAI spec to the record and commit.""" rec = Record.get_record(record_uuid) rec['_oai']['sets'] = sorted(rec['_oai'].get('sets', []) + [spec, ]) rec['_oai']['updated'] = datetime_to_datestamp(datetime.utcnow()) rec.commit() db.session.commit() RecordIndexer().bulk_index([str(rec.id), ])
def remove_oaiset_spec(record_uuid, spec): """Remove the OAI spec from the record and commit.""" rec = Record.get_record(record_uuid) rec['_oai']['sets'] = sorted([s for s in rec['_oai'].get('sets', []) if s != spec]) rec['_oai']['updated'] = datetime_to_datestamp(datetime.utcnow()) if not rec['_oai']['sets']: del rec['_oai']['sets'] rec.commit() db.session.commit() RecordIndexer().bulk_index([str(rec.id), ])
def add_oaiset_spec(record_uuid, spec): """Add the OAI spec to the record and commit.""" rec = Record.get_record(record_uuid) rec['_oai']['sets'] = sorted(rec['_oai'].get('sets', []) + [ spec, ]) rec['_oai']['updated'] = datetime_to_datestamp(datetime.utcnow()) rec.commit() db.session.commit() RecordIndexer().bulk_index([ str(rec.id), ])
def remove_oaiset_spec(record_uuid, spec): """Remove the OAI spec from the record and commit.""" rec = Record.get_record(record_uuid) rec['_oai']['sets'] = sorted( [s for s in rec['_oai'].get('sets', []) if s != spec]) rec['_oai']['updated'] = datetime_to_datestamp(datetime.utcnow()) if not rec['_oai']['sets']: del rec['_oai']['sets'] rec.commit() db.session.commit() RecordIndexer().bulk_index([ str(rec.id), ])
def test_getrecord(app): """Test get record verb.""" with app.test_request_context(): pid_value = 'oai:legacy:1' with db.session.begin_nested(): record_id = uuid.uuid4() data = { '_oai': { 'id': pid_value }, 'title_statement': { 'title': 'Test0' }, } pid = oaiid_minter(record_id, data) record = Record.create(data, id_=record_id) db.session.commit() assert pid_value == pid.pid_value record_updated = record.updated with app.test_client() as c: result = c.get( '/oai2d?verb=GetRecord&identifier={0}&metadataPrefix=oai_dc'. format(pid_value)) assert 200 == result.status_code tree = etree.fromstring(result.data) assert len(tree.xpath('/x:OAI-PMH', namespaces=NAMESPACES)) == 1 assert len( tree.xpath('/x:OAI-PMH/x:GetRecord', namespaces=NAMESPACES)) == 1 assert len( tree.xpath('/x:OAI-PMH/x:GetRecord/x:record/x:header', namespaces=NAMESPACES)) == 1 assert len( tree.xpath( '/x:OAI-PMH/x:GetRecord/x:record/x:header/x:identifier', namespaces=NAMESPACES)) == 1 identifier = tree.xpath( '/x:OAI-PMH/x:GetRecord/x:record/x:header/x:identifier/text()', namespaces=NAMESPACES) assert identifier == [pid_value] datestamp = tree.xpath( '/x:OAI-PMH/x:GetRecord/x:record/x:header/x:datestamp/text()', namespaces=NAMESPACES) assert datestamp == [datetime_to_datestamp(record_updated)] assert len( tree.xpath('/x:OAI-PMH/x:GetRecord/x:record/x:metadata', namespaces=NAMESPACES)) == 1
def sync_record_oai(uuid, cache=None): """Mint OAI ID information for the record. :type uuid: str """ rec = Record.get_record(uuid) recid_s = str(rec['recid']) # Try to get the already existing OAI PID for this record oai_pid_q = PersistentIdentifier.query.filter_by(pid_type='oai', object_uuid=rec.id) if oai_pid_q.count() == 0: pid = oaiid_minter(rec.id, rec) synced_sets = get_synced_sets(rec, cache=cache) rec['_oai']['sets'] = synced_sets rec.commit() db.session.commit() RecordIndexer().bulk_index([ str(rec.id), ]) logger.info('Minted new OAI PID ({pid}) for record {id}'.format( pid=pid, id=uuid)) elif oai_pid_q.count() == 1: pid = oai_pid_q.one() managed_prefixes = current_app.config['OAISERVER_MANAGED_ID_PREFIXES'] if not any(pid.pid_value.startswith(p) for p in managed_prefixes): logger.exception('Unknown OAIID prefix: {0}'.format(pid.pid_value)) elif str(pid.get_assigned_object()) != uuid: logger.exception('OAI PID ({pid}) for record {id} ({recid}) is ' 'pointing to a different object ({id2})'.format( pid=pid, id=uuid, id2=str(pid.get_assigned_object()), recid=recid_s)) elif requires_sync(rec, cache=cache): rec.setdefault('_oai', {}) rec['_oai']['id'] = pid.pid_value rec['_oai']['updated'] = datetime_to_datestamp(datetime.utcnow()) synced_sets = get_synced_sets(rec, cache=cache) rec['_oai']['sets'] = synced_sets if not rec['_oai']['sets']: del rec['_oai']['sets'] # Don't store empty list rec.commit() db.session.commit() RecordIndexer().bulk_index([ str(rec.id), ]) logger.info('Matching OAI PID ({pid}) for record {id}'.format( pid=pid, id=uuid))
def zenodo_oaiid_minter(record_uuid, data): """Mint OAI identifiers.""" pid_value = data.get('_oai', {}).get('id') if pid_value is None: assert 'recid' in data pid_value = current_app.config.get('OAISERVER_ID_PREFIX', '') + str( data['recid'] ) provider = OAIIDProvider.create( object_type='rec', object_uuid=record_uuid, pid_value=str(pid_value) ) data.setdefault('_oai', {}) data['_oai']['id'] = provider.pid.pid_value data['_oai']['updated'] = datetime_to_datestamp(datetime.utcnow()) return provider.pid
def b2share_oaiid_minter(rec_pid, data): """Mint record identifiers.""" oai_pid_value = data.get('_oai', {}).get('id') if oai_pid_value is None: oai_prefix = current_app.config.get('OAISERVER_ID_PREFIX', 'oai:') oai_pid_value = str(oai_prefix) + str(rec_pid.pid_value) provider = OAIIDProvider.create(object_type='rec', object_uuid=rec_pid.object_uuid, pid_value=oai_pid_value) data.setdefault('_oai', {}) data['_oai'].update({ 'id': oai_pid_value, 'sets': [data.get('community')], # community_id == setSpec 'updated': datetime_to_datestamp(datetime.utcnow()), }) return provider.pid
def b2share_oaiid_minter(record_uuid, data): """Mint record identifiers.""" pid_value = data.get('_oai', {}).get('id') if pid_value is None: assert '_deposit' in data and 'id' in data['_deposit'] id_ = str(data['_deposit']['id']) pid_value = current_app.config.get('OAISERVER_ID_PREFIX', '') + id_ provider = OAIIDProvider.create(object_type='rec', object_uuid=record_uuid, pid_value=str(pid_value)) data.setdefault('_oai', {}) data['_oai'].update({ 'id': provider.pid.pid_value, 'sets': [], 'updated': datetime_to_datestamp(datetime.utcnow()), }) return provider.pid
def b2share_oaiid_minter(rec_pid, data): """Mint record identifiers.""" oai_pid_value = data.get('_oai', {}).get('id') if oai_pid_value is None: oai_prefix = current_app.config.get('OAISERVER_ID_PREFIX', 'oai:') oai_pid_value = str(oai_prefix) + str(rec_pid.pid_value) provider = OAIIDProvider.create( object_type='rec', object_uuid=rec_pid.object_uuid, pid_value=oai_pid_value ) data.setdefault('_oai', {}) data['_oai'].update({ 'id': oai_pid_value, 'sets': [data.get('community')], # community_id == setSpec 'updated': datetime_to_datestamp(datetime.utcnow()), }) return provider.pid
def sync_record_oai(uuid, cache=None): """Mint OAI ID information for the record. :type uuid: str """ rec = Record.get_record(uuid) recid_s = str(rec['recid']) # Try to get the already existing OAI PID for this record oai_pid_q = PersistentIdentifier.query.filter_by(pid_type='oai', object_uuid=rec.id) if oai_pid_q.count() == 0: pid = oaiid_minter(rec.id, rec) synced_sets = get_synced_sets(rec, cache=cache) rec['_oai']['sets'] = synced_sets rec.commit() db.session.commit() RecordIndexer().bulk_index([str(rec.id), ]) logger.info('Minted new OAI PID ({pid}) for record {id}'.format( pid=pid, id=uuid)) elif oai_pid_q.count() == 1: pid = oai_pid_q.one() managed_prefixes = current_app.config['OAISERVER_MANAGED_ID_PREFIXES'] if not any(pid.pid_value.startswith(p) for p in managed_prefixes): logger.exception('Unknown OAIID prefix: {0}'.format(pid.pid_value)) elif str(pid.get_assigned_object()) != uuid: logger.exception( 'OAI PID ({pid}) for record {id} ({recid}) is ' 'pointing to a different object ({id2})'.format( pid=pid, id=uuid, id2=str(pid.get_assigned_object()), recid=recid_s)) elif requires_sync(rec, cache=cache): rec.setdefault('_oai', {}) rec['_oai']['id'] = pid.pid_value rec['_oai']['updated'] = datetime_to_datestamp(datetime.utcnow()) synced_sets = get_synced_sets(rec, cache=cache) rec['_oai']['sets'] = synced_sets if not rec['_oai']['sets']: del rec['_oai']['sets'] # Don't store empty list rec.commit() db.session.commit() RecordIndexer().bulk_index([str(rec.id), ]) logger.info('Matching OAI PID ({pid}) for record {id}'.format( pid=pid, id=uuid))
def test_getrecord(app): """Test get record verb.""" with app.test_request_context(): pid_value = 'oai:legacy:1' with db.session.begin_nested(): record_id = uuid.uuid4() data = { '_oai': {'id': pid_value}, 'title_statement': {'title': 'Test0'}, } pid = oaiid_minter(record_id, data) record = Record.create(data, id_=record_id) db.session.commit() assert pid_value == pid.pid_value record_updated = record.updated with app.test_client() as c: result = c.get( '/oai2d?verb=GetRecord&identifier={0}&metadataPrefix=oai_dc' .format(pid_value)) assert 200 == result.status_code tree = etree.fromstring(result.data) assert len(tree.xpath('/x:OAI-PMH', namespaces=NAMESPACES)) == 1 assert len(tree.xpath('/x:OAI-PMH/x:GetRecord', namespaces=NAMESPACES)) == 1 assert len(tree.xpath('/x:OAI-PMH/x:GetRecord/x:record/x:header', namespaces=NAMESPACES)) == 1 assert len(tree.xpath( '/x:OAI-PMH/x:GetRecord/x:record/x:header/x:identifier', namespaces=NAMESPACES)) == 1 identifier = tree.xpath( '/x:OAI-PMH/x:GetRecord/x:record/x:header/x:identifier/text()', namespaces=NAMESPACES) assert identifier == [pid_value] datestamp = tree.xpath( '/x:OAI-PMH/x:GetRecord/x:record/x:header/x:datestamp/text()', namespaces=NAMESPACES) assert datestamp == [datetime_to_datestamp(record_updated)] assert len(tree.xpath('/x:OAI-PMH/x:GetRecord/x:record/x:metadata', namespaces=NAMESPACES)) == 1
def test_listidentifiers(app): """Test verb ListIdentifiers.""" from invenio_oaiserver.models import OAISet with app.app_context(): current_oaiserver.unregister_signals_oaiset() # create new OAI Set with db.session.begin_nested(): oaiset = OAISet( spec='test0', name='Test0', description='test desc 0', search_pattern='title_statement.title:Test0', ) db.session.add(oaiset) db.session.commit() run_after_insert_oai_set() with app.test_request_context(): indexer = RecordIndexer() # create a new record (inside the OAI Set) with db.session.begin_nested(): record_id = uuid.uuid4() data = {'title_statement': {'title': 'Test0'}} recid_minter(record_id, data) pid = oaiid_minter(record_id, data) record = Record.create(data, id_=record_id) db.session.commit() indexer.index_by_id(record_id) current_search.flush_and_refresh('_all') pid_value = pid.pid_value # get the list of identifiers with app.test_client() as c: result = c.get('/oai2d?verb=ListIdentifiers&metadataPrefix=oai_dc') tree = etree.fromstring(result.data) assert len(tree.xpath('/x:OAI-PMH', namespaces=NAMESPACES)) == 1 assert len( tree.xpath('/x:OAI-PMH/x:ListIdentifiers', namespaces=NAMESPACES)) == 1 assert len( tree.xpath('/x:OAI-PMH/x:ListIdentifiers/x:header', namespaces=NAMESPACES)) == 1 identifier = tree.xpath( '/x:OAI-PMH/x:ListIdentifiers/x:header/x:identifier', namespaces=NAMESPACES) assert len(identifier) == 1 assert identifier[0].text == str(pid_value) datestamp = tree.xpath( '/x:OAI-PMH/x:ListIdentifiers/x:header/x:datestamp', namespaces=NAMESPACES) assert len(datestamp) == 1 assert datestamp[0].text == datetime_to_datestamp(record.updated) # Check from:until range with app.test_client() as c: # Check date and datetime timestamps. for granularity in (False, True): result = c.get( '/oai2d?verb=ListIdentifiers&metadataPrefix=oai_dc' '&from={0}&until={1}&set=test0'.format( datetime_to_datestamp(record.updated - timedelta(1), day_granularity=granularity), datetime_to_datestamp(record.updated + timedelta(1), day_granularity=granularity), )) assert result.status_code == 200 tree = etree.fromstring(result.data) identifier = tree.xpath( '/x:OAI-PMH/x:ListIdentifiers/x:header/x:identifier', namespaces=NAMESPACES) assert len(identifier) == 1
def test_listrecords(app): """Test ListRecords.""" total = 12 record_ids = [] with app.test_request_context(): indexer = RecordIndexer() with db.session.begin_nested(): for idx in range(total): record_id = uuid.uuid4() data = {'title_statement': {'title': 'Test{0}'.format(idx)}} recid_minter(record_id, data) oaiid_minter(record_id, data) record = Record.create(data, id_=record_id) record_ids.append(record_id) db.session.commit() for record_id in record_ids: indexer.index_by_id(record_id) current_search.flush_and_refresh('_all') with app.test_client() as c: result = c.get('/oai2d?verb=ListRecords&metadataPrefix=oai_dc') tree = etree.fromstring(result.data) assert len(tree.xpath('/x:OAI-PMH', namespaces=NAMESPACES)) == 1 assert len( tree.xpath('/x:OAI-PMH/x:ListRecords', namespaces=NAMESPACES)) == 1 assert len( tree.xpath('/x:OAI-PMH/x:ListRecords/x:record', namespaces=NAMESPACES)) == 10 assert len( tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:header', namespaces=NAMESPACES)) == 10 assert len( tree.xpath( '/x:OAI-PMH/x:ListRecords/x:record/x:header' '/x:identifier', namespaces=NAMESPACES)) == 10 assert len( tree.xpath( '/x:OAI-PMH/x:ListRecords/x:record/x:header' '/x:datestamp', namespaces=NAMESPACES)) == 10 assert len( tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:metadata', namespaces=NAMESPACES)) == 10 resumption_token = tree.xpath( '/x:OAI-PMH/x:ListRecords/x:resumptionToken', namespaces=NAMESPACES)[0] assert resumption_token.text with app.test_client() as c: result = c.get( '/oai2d?verb=ListRecords&resumptionToken={0}'.format( resumption_token.text)) tree = etree.fromstring(result.data) assert len(tree.xpath('/x:OAI-PMH', namespaces=NAMESPACES)) == 1 assert len( tree.xpath('/x:OAI-PMH/x:ListRecords', namespaces=NAMESPACES)) == 1 assert len( tree.xpath('/x:OAI-PMH/x:ListRecords/x:record', namespaces=NAMESPACES)) == 2 assert len( tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:header', namespaces=NAMESPACES)) == 2 assert len( tree.xpath( '/x:OAI-PMH/x:ListRecords/x:record/x:header' '/x:identifier', namespaces=NAMESPACES)) == 2 assert len( tree.xpath( '/x:OAI-PMH/x:ListRecords/x:record/x:header' '/x:datestamp', namespaces=NAMESPACES)) == 2 assert len( tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:metadata', namespaces=NAMESPACES)) == 2 resumption_token = tree.xpath( '/x:OAI-PMH/x:ListRecords/x:resumptionToken', namespaces=NAMESPACES)[0] assert not resumption_token.text # Check from:until range with app.test_client() as c: # Check date and datetime timestamps. for granularity in (False, True): result = c.get('/oai2d?verb=ListRecords&metadataPrefix=oai_dc' '&from={0}&until={1}'.format( datetime_to_datestamp( record.updated - timedelta(days=1), day_granularity=granularity), datetime_to_datestamp( record.updated + timedelta(days=1), day_granularity=granularity), )) assert result.status_code == 200 tree = etree.fromstring(result.data) assert len( tree.xpath('/x:OAI-PMH/x:ListRecords/x:record', namespaces=NAMESPACES)) == 10
def test_listidentifiers(app): """Test verb ListIdentifiers.""" from invenio_oaiserver.models import OAISet with app.app_context(): current_oaiserver.unregister_signals_oaiset() # create new OAI Set with db.session.begin_nested(): oaiset = OAISet( spec='test0', name='Test0', description='test desc 0', search_pattern='title_statement.title:Test0', ) db.session.add(oaiset) db.session.commit() run_after_insert_oai_set() with app.test_request_context(): indexer = RecordIndexer() # create a new record (inside the OAI Set) with db.session.begin_nested(): record_id = uuid.uuid4() data = {'title_statement': {'title': 'Test0'}} recid_minter(record_id, data) pid = oaiid_minter(record_id, data) record = Record.create(data, id_=record_id) db.session.commit() indexer.index_by_id(record_id) current_search.flush_and_refresh('_all') pid_value = pid.pid_value # get the list of identifiers with app.test_client() as c: result = c.get( '/oai2d?verb=ListIdentifiers&metadataPrefix=oai_dc' ) tree = etree.fromstring(result.data) assert len(tree.xpath('/x:OAI-PMH', namespaces=NAMESPACES)) == 1 assert len(tree.xpath('/x:OAI-PMH/x:ListIdentifiers', namespaces=NAMESPACES)) == 1 assert len(tree.xpath('/x:OAI-PMH/x:ListIdentifiers/x:header', namespaces=NAMESPACES)) == 1 identifier = tree.xpath( '/x:OAI-PMH/x:ListIdentifiers/x:header/x:identifier', namespaces=NAMESPACES ) assert len(identifier) == 1 assert identifier[0].text == str(pid_value) datestamp = tree.xpath( '/x:OAI-PMH/x:ListIdentifiers/x:header/x:datestamp', namespaces=NAMESPACES ) assert len(datestamp) == 1 assert datestamp[0].text == datetime_to_datestamp(record.updated) # Check from:until range with app.test_client() as c: # Check date and datetime timestamps. for granularity in (False, True): result = c.get( '/oai2d?verb=ListIdentifiers&metadataPrefix=oai_dc' '&from={0}&until={1}&set=test0'.format( datetime_to_datestamp( record.updated - timedelta(1), day_granularity=granularity), datetime_to_datestamp( record.updated + timedelta(1), day_granularity=granularity), ) ) assert result.status_code == 200 tree = etree.fromstring(result.data) identifier = tree.xpath( '/x:OAI-PMH/x:ListIdentifiers/x:header/x:identifier', namespaces=NAMESPACES ) assert len(identifier) == 1
def test_listrecords(app): """Test ListRecords.""" total = 12 record_ids = [] with app.test_request_context(): indexer = RecordIndexer() with db.session.begin_nested(): for idx in range(total): record_id = uuid.uuid4() data = {'title_statement': {'title': 'Test{0}'.format(idx)}} recid_minter(record_id, data) oaiid_minter(record_id, data) record = Record.create(data, id_=record_id) record_ids.append(record_id) db.session.commit() for record_id in record_ids: indexer.index_by_id(record_id) current_search.flush_and_refresh('_all') with app.test_client() as c: result = c.get('/oai2d?verb=ListRecords&metadataPrefix=oai_dc') tree = etree.fromstring(result.data) assert len(tree.xpath('/x:OAI-PMH', namespaces=NAMESPACES)) == 1 assert len(tree.xpath('/x:OAI-PMH/x:ListRecords', namespaces=NAMESPACES)) == 1 assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record', namespaces=NAMESPACES)) == 10 assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:header', namespaces=NAMESPACES)) == 10 assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:header' '/x:identifier', namespaces=NAMESPACES)) == 10 assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:header' '/x:datestamp', namespaces=NAMESPACES)) == 10 assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:metadata', namespaces=NAMESPACES)) == 10 resumption_token = tree.xpath( '/x:OAI-PMH/x:ListRecords/x:resumptionToken', namespaces=NAMESPACES )[0] assert resumption_token.text with app.test_client() as c: result = c.get( '/oai2d?verb=ListRecords&resumptionToken={0}'.format( resumption_token.text ) ) tree = etree.fromstring(result.data) assert len(tree.xpath('/x:OAI-PMH', namespaces=NAMESPACES)) == 1 assert len(tree.xpath('/x:OAI-PMH/x:ListRecords', namespaces=NAMESPACES)) == 1 assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record', namespaces=NAMESPACES)) == 2 assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:header', namespaces=NAMESPACES)) == 2 assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:header' '/x:identifier', namespaces=NAMESPACES)) == 2 assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:header' '/x:datestamp', namespaces=NAMESPACES)) == 2 assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record/x:metadata', namespaces=NAMESPACES)) == 2 resumption_token = tree.xpath( '/x:OAI-PMH/x:ListRecords/x:resumptionToken', namespaces=NAMESPACES )[0] assert not resumption_token.text # Check from:until range with app.test_client() as c: # Check date and datetime timestamps. for granularity in (False, True): result = c.get( '/oai2d?verb=ListRecords&metadataPrefix=oai_dc' '&from={0}&until={1}'.format( datetime_to_datestamp( record.updated - timedelta(days=1), day_granularity=granularity), datetime_to_datestamp( record.updated + timedelta(days=1), day_granularity=granularity), ) ) assert result.status_code == 200 tree = etree.fromstring(result.data) assert len(tree.xpath('/x:OAI-PMH/x:ListRecords/x:record', namespaces=NAMESPACES)) == 10