def list_identifiers(self):
try:
#Template name
self.template_name = 'oai_pmh/xml/list_identifiers.xml'
query = dict()
items = []
#Handle FROM and UNTIL
query = self.check_dates()
try:
#Get the metadata format thanks to the prefix
myMetadataFormat = OaiMyMetadataFormat.objects.get(
metadataPrefix=self.metadataPrefix)
#Get all template using it (activated True)
templates = OaiTemplMfXslt.objects(
myMetadataFormat=myMetadataFormat,
activated=True).distinct(field="template")
#Ids
templatesID = [str(x.id) for x in templates]
#If myMetadataFormat is linked to a template, we add the template id
if myMetadataFormat.isTemplate:
templatesID.append(str(myMetadataFormat.template.id))
except:
#The metadata format doesn't exist
raise cannotDisseminateFormat(self.metadataPrefix)
if self.set:
try:
setsTemplates = OaiMySet.objects(
setSpec=self.set).only('templates').get()
templatesID = set(templatesID).intersection(
[str(x.id) for x in setsTemplates.templates])
except Exception, e:
raise noRecordsMatch
for template in templatesID:
#Retrieve sets for this template
sets = OaiMySet.objects(templates=template).all()
query['schema'] = template
#The record has to be published
query['ispublished'] = True
#Get all records for this template
data = XMLdata.executeQueryFullResult(query)
#IF no records, go to the next template
if len(data) == 0:
continue
for i in data:
#Fill the response
identifier = '%s:%s:id/%s' % (settings.OAI_SCHEME,
settings.OAI_REPO_IDENTIFIER,
str(i['_id']))
item_info = {
'identifier': identifier,
'last_modified': self.get_last_modified_date(i),
'sets': sets,
'deleted': i.get('status', '') == Status.DELETED
}
items.append(item_info)
#If there is no records
if len(items) == 0:
raise noRecordsMatch
return self.render_to_response({'items': items})
def list_identifiers(self):
try:
#Template name
self.template_name = 'oai_pmh/xml/list_identifiers.xml'
query = dict()
items=[]
#Handle FROM and UNTIL
query = self.check_dates()
try:
#Get the metadata format thanks to the prefix
myMetadataFormat = OaiMyMetadataFormat.objects.get(metadataPrefix=self.metadataPrefix)
#Get all template using it (activated True)
templates = OaiTemplMfXslt.objects(myMetadataFormat=myMetadataFormat, activated=True).distinct(field="template")
#Ids
templatesID = [str(x.id) for x in templates]
#If myMetadataFormat is linked to a template, we add the template id
if myMetadataFormat.isTemplate:
templatesID.append(str(myMetadataFormat.template.id))
except:
#The metadata format doesn't exist
raise cannotDisseminateFormat(self.metadataPrefix)
if self.set:
try:
setsTemplates = OaiMySet.objects(setSpec=self.set).only('templates').get()
templatesID = set(templatesID).intersection([str(x.id) for x in setsTemplates.templates])
except Exception, e:
raise noRecordsMatch
for template in templatesID:
#Retrieve sets for this template
sets = OaiMySet.objects(templates=template).all()
query['schema'] = template
#The record has to be published
query['ispublished'] = True
#Get all records for this template
data = XMLdata.executeQueryFullResult(query)
#IF no records, go to the next template
if len(data) == 0:
continue
for i in data:
#Fill the response
identifier = '%s:%s:id/%s' % (settings.OAI_SCHEME, settings.OAI_REPO_IDENTIFIER, str(i['_id']))
item_info = {
'identifier': identifier,
'last_modified': self.get_last_modified_date(i),
'sets': sets,
'deleted': i.get('status', '') == Status.DELETED
}
items.append(item_info)
#If there is no records
if len(items) == 0:
raise noRecordsMatch
return self.render_to_response({'items': items})
def list_records(self):
try:
items = []
#Template name
self.template_name = 'oai_pmh/xml/list_records.xml'
query = dict()
#Handle FROM and UNTIL
query = self.check_dates()
#Get the metadataformat for the provided prefix
try:
myMetadataFormat = OaiMyMetadataFormat.objects.get(metadataPrefix=self.metadataPrefix)
except Exception, e:
raise cannotDisseminateFormat(self.metadataPrefix)
if myMetadataFormat.isTemplate:
#GEt the corresponding template
templatesID = [str(myMetadataFormat.template.id)]
else:
#Get information about templates using this MF
objTempMfXslt = OaiTemplMfXslt.objects(myMetadataFormat=myMetadataFormat, activated=True).all()
#Ids
templatesID = [str(x.template.id) for x in objTempMfXslt]
#if a set was provided, we filter the templates
if self.set:
try:
setsTemplates = OaiMySet.objects(setSpec=self.set).only('templates').get()
templatesID = set(templatesID).intersection([str(x.id) for x in setsTemplates.templates])
except Exception, e:
raise noRecordsMatch
def test_listSets(self):
self.dump_oai_my_set()
data = {'verb': 'ListSets'}
r = self.doRequestServer(data=data)
self.isStatusOK(r.status_code)
self.checkTagExist(r.text, 'ListSets')
self.checkTagCount(r.text, 'set', len(OaiMySet.objects().all()))
def list_records(self):
try:
items = []
#Template name
self.template_name = 'oai_pmh/xml/list_records.xml'
query = dict()
#Handle FROM and UNTIL
query = self.check_dates()
#Get the metadataformat for the provided prefix
try:
myMetadataFormat = OaiMyMetadataFormat.objects.get(metadataPrefix=self.metadataPrefix)
except Exception, e:
raise cannotDisseminateFormat(self.metadataPrefix)
if myMetadataFormat.isTemplate:
#GEt the corresponding template
templatesID = [str(myMetadataFormat.template.id)]
else:
#Get information about templates using this MF
objTempMfXslt = OaiTemplMfXslt.objects(myMetadataFormat=myMetadataFormat, activated=True).all()
#Ids
templatesID = [str(x.template.id) for x in objTempMfXslt]
#if a set was provided, we filter the templates
if self.set:
try:
setsTemplates = OaiMySet.objects(setSpec=self.set).only('templates').get()
templatesID = set(templatesID).intersection([str(x.id) for x in setsTemplates.templates])
except Exception, e:
raise noRecordsMatch
def test_listSets(self):
self.dump_oai_my_set()
data = {'verb': 'ListSets'}
r = self.doRequestServer(data=data)
self.isStatusOK(r.status_code)
self.checkTagExist(r.text, 'ListSets')
self.checkTagCount(r.text, 'set', len(OaiMySet.objects().all()))
def get_record(self):
try:
#Bool if we need to transform the XML via XSLT
hasToBeTransformed = False
#Check if the identifier pattern is OK
id = self.check_identifier()
#Template name
self.template_name = 'oai_pmh/xml/get_record.xml'
query = dict()
#Convert id to ObjectId
try:
query['_id'] = ObjectId(id)
#The record has to be published
query['ispublished'] = True
except Exception:
raise idDoesNotExist(self.identifier)
data = XMLdata.executeQueryFullResult(query)
#This id doesn't exist
if len(data) == 0:
raise idDoesNotExist(self.identifier)
data = data[0]
#Get the template for the identifier
template = data['schema']
#Retrieve sets for this template
sets = OaiMySet.objects(templates=template).all()
#Retrieve the XSLT for the transformation
try:
#Get the metadataformat for the provided prefix
myMetadataFormat = OaiMyMetadataFormat.objects.get(metadataPrefix=self.metadataPrefix)
#If this metadata prefix is not associated to a template, we need to retrieve the XSLT to do the transformation
if not myMetadataFormat.isTemplate:
hasToBeTransformed = True
#Get information about the XSLT for the MF and the template
objTempMfXslt = OaiTemplMfXslt.objects(myMetadataFormat=myMetadataFormat, template=template, activated=True).get()
#If no information or desactivated
if not objTempMfXslt.xslt:
raise cannotDisseminateFormat(self.metadataPrefix)
else:
#Get the XSLT for the transformation
xslt = objTempMfXslt.xslt
except:
raise cannotDisseminateFormat(self.metadataPrefix)
#Transform XML data
dataToTransform = [{'title': data['_id'], 'content': self.cleanXML(xmltodict.unparse(data['content']))}]
if hasToBeTransformed:
dataXML = self.getXMLTranformXSLT(dataToTransform, xslt)
else:
dataXML = dataToTransform
#Fill the response
record_info = {
'identifier': self.identifier,
'last_modified': datestamp.datetime_to_datestamp(data['publicationdate']) if 'publicationdate' in data else datestamp.datetime_to_datestamp(datetime.datetime.min),
'sets': sets,
'XML': dataXML[0]['content']
}
return self.render_to_response(record_info)
except OAIExceptions, e:
return self.errors(e.errors)
def list_sets(self):
self.template_name = 'oai_pmh/xml/list_sets.xml'
items = []
#For the moment, we don't support sets
try:
#Retrieve sets
sets = OaiMySet.objects().all()
#If there is no sets, we raise noSetHierarchy
if len(sets) == 0:
raise noSetHierarchy
else:
#Fill the response
for set in sets:
item_info = {
'setSpec': set.setSpec,
'setName': set.setName,
'description': set.description
}
items.append(item_info)
return self.render_to_response({'items': items})
except OAIExceptions, e:
return self.errors(e.errors)
def list_sets(self):
self.template_name = 'oai_pmh/xml/list_sets.xml'
items = []
#For the moment, we don't support sets
try:
#Retrieve sets
sets = OaiMySet.objects().all()
#If there is no sets, we raise noSetHierarchy
if len(sets) == 0:
raise noSetHierarchy
else:
#Fill the response
for set in sets:
item_info = {
'setSpec': set.setSpec,
'setName': set.setName,
'description': set.description
}
items.append(item_info)
return self.render_to_response({'items': items})
except OAIExceptions, e:
return self.errors(e.errors)
def get_record(self):
try:
#Bool if we need to transform the XML via XSLT
hasToBeTransformed = False
#Check if the identifier pattern is OK
id = self.check_identifier()
#Template name
self.template_name = 'oai_pmh/xml/get_record.xml'
query = dict()
#Convert id to ObjectId
try:
query['_id'] = ObjectId(id)
#The record has to be published
query['ispublished'] = True
except Exception:
raise idDoesNotExist(self.identifier)
data = XMLdata.executeQueryFullResult(query, includeDeleted=True)
#This id doesn't exist
if len(data) == 0:
raise idDoesNotExist(self.identifier)
data = data[0]
#Get the template for the identifier
template = data['schema']
#Retrieve sets for this template
sets = OaiMySet.objects(templates=template).all()
#Retrieve the XSLT for the transformation
try:
#Get the metadataformat for the provided prefix
myMetadataFormat = OaiMyMetadataFormat.objects.get(metadataPrefix=self.metadataPrefix)
#If this metadata prefix is not associated to a template, we need to retrieve the XSLT to do the transformation
if not myMetadataFormat.isTemplate:
hasToBeTransformed = True
#Get information about the XSLT for the MF and the template
objTempMfXslt = OaiTemplMfXslt.objects(myMetadataFormat=myMetadataFormat, template=template, activated=True).get()
#If no information or desactivated
if not objTempMfXslt.xslt:
raise cannotDisseminateFormat(self.metadataPrefix)
else:
#Get the XSLT for the transformation
xslt = objTempMfXslt.xslt
except:
raise cannotDisseminateFormat(self.metadataPrefix)
#Transform XML data
dataToTransform = [{'title': data['title'], 'content': self.cleanXML(data['xml_file'])}]
if hasToBeTransformed:
dataXML = self.getXMLTranformXSLT(dataToTransform, xslt)
else:
dataXML = dataToTransform
#Fill the response
record_info = {
'identifier': self.identifier,
'last_modified': self.get_last_modified_date(data),
'sets': sets,
'XML': dataXML[0]['content'],
'deleted': data.get('status', '') == Status.DELETED
}
return self.render_to_response(record_info)
except OAIExceptions, e:
return self.errors(e.errors)
else:
#Get information about templates using this MF
objTempMfXslt = OaiTemplMfXslt.objects(myMetadataFormat=myMetadataFormat, activated=True).all()
#Ids
templatesID = [str(x.template.id) for x in objTempMfXslt]
#if a set was provided, we filter the templates
if self.set:
try:
setsTemplates = OaiMySet.objects(setSpec=self.set).only('templates').get()
templatesID = set(templatesID).intersection([str(x.id) for x in setsTemplates.templates])
except Exception, e:
raise noRecordsMatch
#For each template found
for template in templatesID:
#Retrieve sets for this template
sets = OaiMySet.objects(templates=template).all()
query['schema'] = template
#The record has to be published
query['ispublished'] = True
#Get all records for this template
data = XMLdata.executeQueryFullResult(query, includeDeleted=True)
#IF no records, go to the next template
if len(data) == 0:
continue
if not myMetadataFormat.isTemplate:
xslt = objTempMfXslt(template=template).get().xslt
#Add each record
for elt in data:
dataToTransform = [{'title': elt['title'], 'content': self.cleanXML(elt['xml_file'])}]
def dump_oai_my_set(self):
self.assertEquals(len(OaiMySet.objects()), 0)
self.restoreDump(join(DUMP_OAI_PMH_TEST_PATH, 'oai_my_set.bson'),
'oai_my_set')
self.assertTrue(len(OaiMySet.objects()) > 0)
def dump_oai_my_set_bad(self):
self.assertEquals(len(OaiMySet.objects()), 0)
self.restoreDump(join(DUMP_OAI_PMH_TEST_PATH, 'oai_my_set_bad.bson'), 'oai_my_set')
self.assertTrue(len(OaiMySet.objects()) > 0)
else:
#Get information about templates using this MF
objTempMfXslt = OaiTemplMfXslt.objects(myMetadataFormat=myMetadataFormat, activated=True).all()
#Ids
templatesID = [str(x.template.id) for x in objTempMfXslt]
#if a set was provided, we filter the templates
if self.set:
try:
setsTemplates = OaiMySet.objects(setSpec=self.set).only('templates').get()
templatesID = set(templatesID).intersection([str(x.id) for x in setsTemplates.templates])
except Exception, e:
raise noRecordsMatch
#For each template found
for template in templatesID:
#Retrieve sets for this template
sets = OaiMySet.objects(templates=template).all()
query['schema'] = template
#The record has to be published
query['ispublished'] = True
#Get all records for this template
data = XMLdata.executeQueryFullResult(query)
#IF no records, go to the next template
if len(data) == 0:
continue
dataToTransform = [{'title': x['_id'], 'content': self.cleanXML(xmltodict.unparse(x['content']))} for x in data]
if myMetadataFormat.isTemplate:
#No transformation needed
dataXML = dataToTransform
else:
#Get the XSLT file
xslt = objTempMfXslt(template=template).get().xslt
def load_sets():
"""
Load default metadata prefixes for OAI-PMH
"""
sets = OaiMySet.objects.all()
if len(sets) == 0:
#Add NMRR templates as sets
templates = {
'all': {
'path': 'res-md.xsd',
'setSpec': 'all',
'setName': 'all',
'description': 'Get all records'
},
'organization': {
'path': 'res-md.xsd',
'setSpec': 'org',
'setName': 'organization',
'description': 'Get organization records'
},
'datacollection': {
'path': 'res-md.xsd',
'setSpec': 'datacol',
'setName': 'datacollection',
'description': 'Get datacollection records'
},
'repository': {
'path': 'res-md.xsd',
'setSpec': 'repo',
'setName': 'repository',
'description': 'Get repository records'
},
'projectarchive': {
'path': 'res-md.xsd',
'setSpec': 'proj',
'setName': 'projectarchive',
'description': 'Get projectarchive records'
},
'database': {
'path': 'res-md.xsd',
'setSpec': 'database',
'setName': 'database',
'description': 'Get database records'
},
'dataset': {
'path': 'res-md.xsd',
'setSpec': 'dataset',
'setName': 'dataset',
'description': 'Get dataset records'
},
'service': {
'path': 'res-md.xsd',
'setSpec': 'serv',
'setName': 'service',
'description': 'Get service records'
},
'informational': {
'path': 'res-md.xsd',
'setSpec': 'info',
'setName': 'informational',
'description': 'Get informational records'
},
'software': {
'path': 'res-md.xsd',
'setSpec': 'soft',
'setName': 'software',
'description': 'Get software records'
},
}
for template_name, info in templates.iteritems():
try:
template = Template.objects.get(title=template_name,
filename=info['path'])
#Add in database
OaiMySet(setSpec=info['setSpec'],
setName=info['setName'],
description=info['description'],
templates=[template]).save()
except Exception, e:
print(
'ERROR : Impossible to set the template "{!s}" as a set. {!s}'
.format(template_name, e.message))