def identify():
data = self.get_base_response(verb, body)
serviceDoc = helpers.getServiceDocument(BASIC_HARVEST_SERVICE_DOC)
data['identify'] = {
"node_id":
sourceLRNode.nodeDescription.node_name,
"repositoryName":
sourceLRNode.communityDescription.community_name,
"baseURL":
serviceDoc['service_endpoint'],
"protocolVersion":
"2.0",
"service_version":
serviceDoc['service_version'],
"earliestDatestamp":
h.earliestDate(),
"deletedRecord":
sourceLRNode.nodeDescription.
node_policy['deleted_data_policy'],
"granularity":
helpers.getDatetimePrecision(serviceDoc),
"adminEmail":
sourceLRNode.nodeDescription.node_admin_identity
}
return json.dumps(data)
def get_service_doc():
global __service_doc
if not __service_doc or recache:
__service_doc = h.getServiceDocument(
m.base_model.appConfig["lr.sword.docid"])
return __service_doc
def mapResource(config_key, member_name, collection_name):
try:
service_doc_id = config[config_key]
service_doc = h.getServiceDocument(service_doc_id)
if service_doc is not None and service_doc["active"]:
map.resource(member_name, collection_name)
map.connect("/" + collection_name,
controller=collection_name,
action='options',
conditions=dict(method=['OPTIONS']))
if member_name == 'swordservice':
map.connect("/swordpub",
controller='swordservice',
action='create')
if member_name == 'distribute':
map.connect("/destination",
controller='distribute',
action='destination',
conditions=dict(method='GET'))
log.info(
"Enabling service route for: {0} member: {1} collection: {2}"
.format(service_doc_id, member_name, collection_name))
else:
log.info(
"Service route for {0} is disabled".format(service_doc_id))
except:
log.exception(
"Exception caught: Not enabling service route for config: {0} member: {1} collection: {2}"
.format(config_key, member_name, collection_name))
def test_resumption(self):
slice_doc = helpers.getServiceDocument("access:slice")
page_size = slice_doc["service_data"]["doc_limit"]
##add test to assert that flow control is enabled, check that flow_control in service_data is true
parameters = {}
parameters[IDENTITY] = self.identities[1] + "test_resumption"
parameters[IDS_ONLY] = False
response = self._slice(parameters)
data = json.loads(response.body)
docs = data["documents"]
if len(docs) != 100:
print "resumption assert will fail. doc count is: " + str(len(docs))
assert len(docs) == 100
for doc in docs:
assert self._checkIdentity(doc["resource_data_description"], self.identities[1] + "test_resumption")
resumption_token = data["resumption_token"]
parameters[RESUMPTION] = resumption_token
response = self._slice(parameters)
data = json.loads(response.body)
docs = data["documents"]
assert len(docs) == 50
for doc in docs:
assert self._checkIdentity(doc["resource_data_description"], self.identities[1] + "test_resumption")
def mapResource(config_key, member_name, collection_name):
try:
service_doc_id = config[config_key]
service_doc = h.getServiceDocument(service_doc_id)
if service_doc is not None and service_doc["active"]:
map.resource(member_name, collection_name)
map.connect(
"/" + collection_name,
controller=collection_name,
action="options",
conditions=dict(method=["OPTIONS"]),
)
if member_name == "swordservice":
map.connect("/swordpub", controller="swordservice", action="create")
if member_name == "distribute":
map.connect(
"/destination", controller="distribute", action="destination", conditions=dict(method="GET")
)
log.info(
"Enabling service route for: {0} member: {1} collection: {2}".format(
service_doc_id, member_name, collection_name
)
)
else:
log.info("Service route for {0} is disabled".format(service_doc_id))
except:
log.exception(
"Exception caught: Not enabling service route for config: {0} member: {1} collection: {2}".format(
config_key, member_name, collection_name
)
)
def test_resumption(self):
slice_doc = helpers.getServiceDocument(config["lr.slice.docid"])
page_size = slice_doc["service_data"]["doc_limit"]
##add test to assert that flow control is enabled, check that flow_control in service_data is true
date_int = helpers.convertDateTime(self.test_start_date_string)
parameters = {}
parameters[START_DATE] = self.test_start_date_string
parameters[IDS_ONLY] = False
parameters[ANY_TAGS] = self.testKeys[0] + 'test_resumption'
response = self._slice(parameters)
result = json.loads(response.body)
assert "resumption_token" in result
test_key = self.testKeys[0] + 'test_resumption'
def validate_page(docs):
assert len(
docs
) <= page_size, "resumption assert will fail. doc count is: " + str(
len(docs))
assert len([
x for x in docs
if helpers.convertDateTime(x['resource_data_description']
["node_timestamp"]) >= date_int
]) == len(docs)
assert all(test_key in doc['resource_data_description']["keys"]
for doc in docs)
self._validate_page(parameters, response, validate_page, 10)
def test_resumption(self):
# response = urlopen(obj.couch_url+"/resource_data/_design/learningregistry-slice/_view/docs?reduce=false&key=\""+obj.testDataKey+"\"")
# body = response.read()
# data = json.loads(body)
# page_size = data["rows"]
slice_doc = helpers.getServiceDocument("access:slice")
page_size = slice_doc["service_data"]["doc_limit"]
##add test to assert that flow control is enabled, check that flow_control in service_data is true
parameters = {}
parameters[IDENTITY] = self.identities[1]+"test_resumption"
parameters[IDS_ONLY] = False
response = self._slice(parameters)
data = json.loads(response.body)
docs = data["documents"]
if len(docs)!=100 :
print "resumption assert will fail. doc count is: " + str(len(docs))
assert len(docs)==100
for doc in docs:
assert self._checkIdentity(doc['resource_data_description'], self.identities[1]+"test_resumption")
resumption_token = data["resumption_token"]
parameters[RESUMPTION] = resumption_token
response = self._slice(parameters)
data = json.loads(response.body)
docs = data["documents"]
assert len(docs)==50
for doc in docs:
assert self._checkIdentity(doc['resource_data_description'], self.identities[1]+"test_resumption")
def create(self):
results = {self.__OK:True}
error_message = None
try:
data = json.loads(request.body)
doc_limit = h.getServiceDocument(config['lr.publish.docid'])['service_data']['doc_limit']
if not self.__DOCUMENTS in data.keys():
# Comply with LR-RQST-009 'Missing documents in POST'
results[self.__ERROR] = "Missing field 'documents' in post body"
elif len(data[self.__DOCUMENTS]) < 1:
# Comply with LR-API-PUBLISH-001 'List of documents is empty'
results[self.__ERROR] = "List of documents is empty"
elif len(data[self.__DOCUMENTS]) > doc_limit:
error_message = "number of posted docs {0} exceeds doc limit: {1}".format(
len(data['documents']), str(doc_limit))
log.debug(error_message)
results[self.__ERROR] = error_message
else:
results[self.__DOCUMENT_RESULTS ] = map(self._publish, data[self.__DOCUMENTS])
except Exception as ex:
log.exception(ex)
results[self.__ERROR] = str(ex)
if results.has_key(self.__ERROR):
results[self.__OK] = False
return json.dumps(results)
def __init__(self, server=appConfig["couchdb.url"], database="resource_data"):
"""
Constructor
"""
harvest.__init__(self, server, database)
self.server = couchdb.Server(server)
self.db = self.server[database]
self.res_data_url = "/".join([appConfig["couchdb.url"], appConfig["couchdb.db.resourcedata"]])
self.service_doc = helpers.getServiceDocument(appConfig["lr.oaipmh.docid"])
def mapResource(config_key, member_name, collection_name):
try:
service_doc_id = config[config_key]
service_doc = h.getServiceDocument(service_doc_id)
if service_doc is not None and service_doc["active"]:
map.resource(member_name, collection_name)
if member_name == 'swordservice':
map.connect("/swordpub",controller='swordservice',action='create')
log.info("Enabling service route for: {0} member: {1} collection: {2}".format(service_doc_id, member_name, collection_name))
else:
log.info("Service route for {0} is disabled".format(service_doc_id))
except:
log.exception("Exception caught: Not enabling service route for config: {0} member: {1} collection: {2}".format(config_key, member_name, collection_name))
def __init__(self, server=appConfig['couchdb.url'], database="resource_data"):
'''
Constructor
'''
harvest.__init__(self, server, database)
self.server = couchdb.Server(server)
self.db = self.server[database]
self.res_data_url = '/'.join([
appConfig['couchdb.url'],
appConfig['couchdb.db.resourcedata']
])
self.service_doc = helpers.getServiceDocument(appConfig["lr.oaipmh.docid"])
def identify():
data = self.get_base_response(verb,body)
serviceDoc = helpers.getServiceDocument(BASIC_HARVEST_SERVICE_DOC)
data['identify']={
"node_id":sourceLRNode.nodeDescription.node_name,
"repositoryName":sourceLRNode.communityDescription.community_name,
"baseURL":serviceDoc['service_endpoint'],
"protocolVersion":"2.0",
"service_version":serviceDoc['service_version'],
"earliestDatestamp":h.earliestDate(),
"deletedRecord":sourceLRNode.nodeDescription.node_policy['deleted_data_policy'],
"granularity":helpers.getDatetimePrecision(serviceDoc),
"adminEmail":sourceLRNode.nodeDescription.node_admin_identity
}
return json.dumps(data)
def _getServiceDocment(self,full_docs):
self.enable_flow_control = False
self.limit = None
self.service_id = None
serviceDoc = h.getServiceDocument(appConfig['lr.obtain.docid'])
if serviceDoc != None:
if 'service_id' in serviceDoc:
self.service_id = serviceDoc['service_id']
if 'service_data' in serviceDoc:
serviceData = serviceDoc['service_data']
if 'flow_control' in serviceData:
self.enable_flow_control = serviceData['flow_control']
limit_type = 'id_limit'
if full_docs:
limit_type = "doc_limit"
if self.enable_flow_control and limit_type in serviceData:
self.limit = serviceData['id_limit']
elif self.enable_flow_control:
self.limit = 100
def mapResource(config_key, member_name, collection_name):
try:
service_doc_id = config[config_key]
service_doc = h.getServiceDocument(service_doc_id)
if service_doc is not None and service_doc["active"]:
map.resource(member_name, collection_name)
if member_name == 'swordservice':
map.connect("/swordpub",
controller='swordservice',
action='create')
log.info(
"Enabling service route for: {0} member: {1} collection: {2}"
.format(service_doc_id, member_name, collection_name))
else:
log.info(
"Service route for {0} is disabled".format(service_doc_id))
except:
log.exception(
"Exception caught: Not enabling service route for config: {0} member: {1} collection: {2}"
.format(config_key, member_name, collection_name))
def _getServiceDocment(self, full_docs):
self.enable_flow_control = False
self.limit = None
self.service_id = None
serviceDoc = h.getServiceDocument(appConfig['lr.obtain.docid'])
if serviceDoc != None:
if 'service_id' in serviceDoc:
self.service_id = serviceDoc['service_id']
if 'service_data' in serviceDoc:
serviceData = serviceDoc['service_data']
if 'flow_control' in serviceData:
self.enable_flow_control = serviceData['flow_control']
limit_type = 'id_limit'
if full_docs:
limit_type = "doc_limit"
if self.enable_flow_control and limit_type in serviceData:
self.limit = serviceData['id_limit']
elif self.enable_flow_control:
self.limit = 100
def test_resumption(self):
slice_doc = helpers.getServiceDocument(config["lr.slice.docid"])
page_size = slice_doc["service_data"]["doc_limit"]
##add test to assert that flow control is enabled, check that flow_control in service_data is true
date_int = helpers.convertDateTime(self.test_start_date_string)
parameters = {}
parameters[START_DATE] = self.test_start_date_string
parameters[IDS_ONLY] = False
parameters[ANY_TAGS] = self.testKeys[0] + 'test_resumption'
response = self._slice(parameters)
result = json.loads(response.body)
assert "resumption_token" in result
test_key = self.testKeys[0] + 'test_resumption'
def validate_page(docs):
assert len(docs) <= page_size, "resumption assert will fail. doc count is: " + str(len(docs))
assert len([x for x in docs if helpers.convertDateTime(x['resource_data_description']["node_timestamp"]) >= date_int]) == len(docs)
assert all(test_key in doc['resource_data_description']["keys"] for doc in docs)
self._validate_page(parameters, response, validate_page, 10)
def create(self):
results = {self.__OK:True}
error_message = None
try:
data = json.loads(request.body)
doc_limit = h.getServiceDocument(config['lr.publish.docid'])['service_data']['doc_limit']
if len(data[self.__DOCUMENTS]) > doc_limit:
error_message = "number of posted docs {0} exceeds doc limit: {1}".format(
len(data['documents']), str(doc_limit))
log.debug(error_message)
results[self.__ERROR] = error_message
else:
results[self.__DOCUMENT_RESULTS ] = map(self._publish, data[self.__DOCUMENTS])
except Exception as ex:
log.exception(ex)
results[self.__ERROR] = "internal error"
if results.has_key(self.__ERROR):
results[self.__OK] = False
return json.dumps(results)
def _getServiceDocment(self,full_docs):
self.enable_flow_control = False
self.limit = None
self.service_id = None
serviceDoc = helpers.getServiceDocument(BASIC_HARVEST_SERVICE_DOC)
if serviceDoc != None:
if 'service_id' in serviceDoc:
self.service_id = serviceDoc['service_id']
if 'service_data' in serviceDoc:
serviceData = serviceDoc['service_data']
if 'flow_control' in serviceData:
self.enable_flow_control = serviceData['flow_control']
limit_type = 'id_limit'
if full_docs:
limit_type = "doc_limit"
if self.enable_flow_control and limit_type in serviceData:
self.limit = serviceData[limit_type]
elif self.enable_flow_control:
self.limit = 100
if 'metadataformats' in serviceData:
self.metadataFormats = serviceData['metadataformats']
def _getServiceDocment(self, full_docs):
self.enable_flow_control = False
self.limit = None
self.service_id = None
serviceDoc = helpers.getServiceDocument(BASIC_HARVEST_SERVICE_DOC)
if serviceDoc != None:
if 'service_id' in serviceDoc:
self.service_id = serviceDoc['service_id']
if 'service_data' in serviceDoc:
serviceData = serviceDoc['service_data']
if 'flow_control' in serviceData:
self.enable_flow_control = serviceData['flow_control']
limit_type = 'id_limit'
if full_docs:
limit_type = "doc_limit"
if self.enable_flow_control and limit_type in serviceData:
self.limit = serviceData[limit_type]
elif self.enable_flow_control:
self.limit = 100
if 'metadataformats' in serviceData:
self.metadataFormats = serviceData['metadataformats']
def __before__(self):
self.enable_flow_control = False
self.fc_id_limit = None
self.fc_doc_limit = None
self.serviceDoc = h.getServiceDocument("access:slice")
if self.serviceDoc != None:
if 'service_id' in self.serviceDoc:
self.service_id = self.serviceDoc['service_id']
if 'service_data' in self.serviceDoc:
serviceData = self.serviceDoc['service_data']
if 'flow_control' in serviceData:
self.enable_flow_control = serviceData['flow_control']
if self.enable_flow_control and 'id_limit' in serviceData:
self.fc_id_limit = serviceData['id_limit']
elif self.enable_flow_control:
self.fc_id_limit = 100
if self.enable_flow_control and 'doc_limit' in serviceData:
self.fc_doc_limit = serviceData['doc_limit']
elif self.enable_flow_control:
self.fc_doc_limit = 100
def __before__(self):
self.enable_flow_control = False
self.fc_id_limit = None
self.fc_doc_limit = None
self.serviceDoc = h.getServiceDocument("access:slice")
if self.serviceDoc != None:
if 'service_id' in self.serviceDoc:
self.service_id = self.serviceDoc['service_id']
if 'service_data' in self.serviceDoc:
serviceData = self.serviceDoc['service_data']
if 'flow_control' in serviceData:
self.enable_flow_control = serviceData['flow_control']
if self.enable_flow_control and 'id_limit' in serviceData:
self.fc_id_limit = serviceData['id_limit']
elif self.enable_flow_control:
self.fc_id_limit = 100
if self.enable_flow_control and 'doc_limit' in serviceData:
self.fc_doc_limit = serviceData['doc_limit']
elif self.enable_flow_control:
self.fc_doc_limit = 100
def create(self):
results = {self.__OK: True}
error_message = None
try:
data = json.loads(request.body)
doc_limit = h.getServiceDocument(
config['lr.publish.docid'])['service_data']['doc_limit']
if len(data[self.__DOCUMENTS]) > doc_limit:
error_message = "number of posted docs {0} exceeds doc limit: {1}".format(
len(data['documents']), str(doc_limit))
log.debug(error_message)
results[self.__ERROR] = error_message
else:
results[self.__DOCUMENT_RESULTS] = map(self._publish,
data[self.__DOCUMENTS])
except Exception as ex:
log.exception(ex)
results[self.__ERROR] = str(ex)
if results.has_key(self.__ERROR):
results[self.__OK] = False
return json.dumps(results)
def get_service_doc():
global __service_doc
if not __service_doc or recache:
__service_doc = h.getServiceDocument(m.base_model.appConfig["lr.sword.docid"])
return __service_doc
def _handleOAIRequest(self, format='html'):
t_req = request._current_obj()
t_res = response._current_obj()
enable_flow_control = False
fc_id_limit = None
fc_doc_limit = None
service_id = None
serviceDoc = h.getServiceDocument(appConfig['lr.oaipmh.docid'])
if serviceDoc != None:
if 'service_id' in serviceDoc:
service_id = serviceDoc['service_id']
if 'service_data' in serviceDoc:
serviceData = serviceDoc['service_data']
if 'flow_control' in serviceData:
enable_flow_control = serviceData['flow_control']
if enable_flow_control and 'id_limit' in serviceData:
fc_id_limit = serviceData['id_limit']
elif enable_flow_control:
fc_id_limit = 100
if enable_flow_control and 'doc_limit' in serviceData:
fc_doc_limit = serviceData['doc_limit']
elif enable_flow_control:
fc_doc_limit = 100
o = oaipmh()
def GetRecord(params):
try:
from lr.mustache.oaipmh import GetRecord as must_GetRecord
identifier = params["identifier"]
if params["by_doc_ID"] == True:
resolver = OAIPMHDocumentResolver()
single_doc = o.get_record(params["identifier"])
if single_doc is not None:
docList = [resolver.process({"doc": single_doc})]
else:
docList = []
else:
docList = o.get_records_by_resource(params["identifier"])
doc_idx = 0
valid_docs = 0
mustache = must_GetRecord()
for doc in docList:
if doc is not None:
doc_idx += 1
if "payload_schema" in doc and params[
"metadataPrefix"] in map(
lambda x: o_mod.getMetadataPrefix(x),
doc["payload_schema"]
) and OAIPMHDocumentResolver.PAYLOAD_ERROR not in doc:
valid_docs += 1
if valid_docs == 1:
part = mustache.prefix(**self._initMustache(
args=params, req=t_req))
yield h.fixUtf8(
self._returnResponse(part, res=t_res))
part = mustache.doc(doc)
yield h.fixUtf8(
self._returnResponse(part, res=t_res))
if doc_idx == 0:
raise IdDoesNotExistError(params['verb'], req=t_req)
elif valid_docs == 0:
raise CannotDisseminateFormatError(params['verb'],
req=t_req)
else:
yield h.fixUtf8(
self._returnResponse(mustache.suffix(), res=t_res))
except oaipmherrors.Error as e:
from lr.mustache.oaipmh import Error as err_stache
err = err_stache()
yield h.fixUtf8(self._returnResponse(err.xml(e), res=t_res))
def ListGeneric(params, showDocs=False, record_limit=None):
if not showDocs:
from lr.mustache.oaipmh import ListIdentifiers as must_ListID
mustache = must_ListID()
else:
from lr.mustache.oaipmh import ListRecords as must_ListRec
mustache = must_ListRec()
try:
doc_index = 0
err_count = 0
metadataPrefix = params["metadataPrefix"]
from_date = params["from"]
until_date = params["until"]
doc_err = None
rendered_init = False
resumptionToken = None if "resumptionToken" not in params else params[
'resumptionToken']
records = o.list_identifiers_or_records(metadataPrefix,
from_date=from_date,
until_date=until_date,
rt=resumptionToken,
fc_limit=record_limit,
include_docs=showDocs)
for ident in records:
doc_index += 1
doc_err = False
if OAIPMHDocumentResolver.PAYLOAD_ERROR in ident:
err_count += 1
doc_err = True
log.debug(
"Payload Error detected, doc_index: {0}, err_count: {1}"
.format(doc_index, err_count))
if doc_index - err_count == 1:
rendered_init = True
part = mustache.prefix(
**self._initMustache(args=params, req=t_req))
yield h.fixUtf8(self._returnResponse(part, res=t_res))
if doc_err is False and (record_limit is None
or doc_index <= record_limit):
part = mustache.doc(ident)
yield h.fixUtf8(part)
elif enable_flow_control:
from lr.lib import resumption_token
if doc_index - err_count > 0 and doc_index > record_limit:
opts = o.list_opts(
metadataPrefix,
h.convertToISO8601UTC(ident["node_timestamp"]),
until_date)
opts["startkey_docid"] = ident["doc_ID"]
token = resumption_token.get_token(
serviceid=service_id,
from_date=from_date,
until_date=until_date,
**opts)
part = mustache.resumptionToken(token)
yield h.fixUtf8(part)
break
elif doc_index - err_count == 0 and doc_index > record_limit:
opts = o.list_opts(
metadataPrefix,
h.convertToISO8601UTC(ident["node_timestamp"]),
until_date)
opts["startkey_docid"] = ident["doc_ID"]
payload = resumption_token.get_payload(
from_date=from_date,
until_date=until_date,
**opts)
records = o.list_identifiers_or_records(
metadataPrefix,
from_date=from_date,
until_date=until_date,
rt=payload,
fc_limit=record_limit,
include_docs=showDocs)
doc_index = 0
err_count = 0
if doc_index == 0 and err_count == 0:
raise NoRecordsMatchError(params['verb'], req=t_req)
elif (doc_index - err_count) == 0:
raise CannotDisseminateFormatError(params['verb'],
req=t_req)
else:
if enable_flow_control and doc_index <= record_limit:
yield h.fixUtf8(mustache.resumptionToken())
yield h.fixUtf8(mustache.suffix())
except oaipmherrors.Error as e:
if not rendered_init:
from lr.mustache.oaipmh import Error as err_stache
err = err_stache()
yield h.fixUtf8(self._returnResponse(err.xml(e),
res=t_res))
else:
from lr.mustache.oaipmh import ErrorOnly as err_stache
err = err_stache()
yield h.fixUtf8(
self._returnResponse(err.xml(e) + mustache.suffix(),
res=t_res))
except:
log.exception("Unknown Error Occurred")
def ListIdentifiers(params):
return ListGeneric(params, False, fc_id_limit)
def ListRecords(params):
return ListGeneric(params, True, fc_doc_limit)
# def ListRecords(params):
# try:
# from lr.mustache.oaipmh import ListRecords as must_ListRec
#
# doc_index = 0
# mustache = must_ListRec()
# for record in o.list_records(params["metadataPrefix"],from_date=params["from"], until_date=params["until"] ):
# doc_index += 1
# log.debug(json.dumps(record))
# if doc_index == 1:
# part = mustache.prefix(**self._initMustache(args=params, req=t_req))
# yield self._returnResponse(part, res=t_res)
#
# part = mustache.doc(record)
# yield self._returnResponse(part, res=t_res)
#
#
# if doc_index == 0:
# raise NoRecordsMatchError(params['verb'], req=t_req)
# else:
# yield mustache.suffix()
#
# except oaipmherrors.Error as e:
# from lr.mustache.oaipmh import Error as err_stache
# err = err_stache()
# yield self._returnResponse(err.xml(e), res=t_res)
# except:
# log.exception("Unable to render template")
def Identify(params=None):
body = ""
try:
self._initRender(params, ctx=c, req=t_req)
c.identify = o.identify()
body = render("/oaipmh-Identify.mako")
except Exception as e:
raise BadVerbError()
return self._returnResponse(body, res=t_res)
def ListMetadataFormats(params):
body = ""
try:
self._initRender(params, ctx=c, req=t_req)
fmts = o.list_metadata_formats(identity=params["identifier"],
by_doc_ID=params["by_doc_ID"])
if len(fmts) == 0:
raise NoMetadataFormats(params["verb"])
c.formats = fmts
body = render("/oaipmh-ListMetadataFormats.mako")
return self._returnResponse(body, res=t_res)
except Error as e:
raise e
def ListSets(params=None):
raise NoSetHierarchyError(verb)
def NotYetSupported(params=None):
raise BadVerbError()
switch = {
'GetRecord': GetRecord,
'ListRecords': ListRecords,
'ListIdentifiers': ListIdentifiers,
'Identify': Identify,
'ListMetadataFormats': ListMetadataFormats,
'ListSets': ListSets
}
try:
params = self._parseParams(flow_control=enable_flow_control,
serviceid=service_id)
# If this is a special case where we are actually using OAI interface to serve basic harvest
if params.has_key("metadataPrefix") and params[
"metadataPrefix"] == "LR_JSON_0.10.0":
if params.has_key("identifier") == True:
params[self.REQUESTID] = params["identifier"]
if params.has_key("from") and isinstance(
params["from"], datetime):
params["from"] = h.convertToISO8601Zformat(params["from"])
if params.has_key("until") and isinstance(
params["until"], datetime):
params["until"] = h.convertToISO8601Zformat(
params["until"])
return HarvestController.harvest(self, params, request.body,
params['verb'].lower())
verb = params['verb']
response.headers['Content-Type'] = "text/xml; charset=utf-8"
return switch[verb](params)
except Error as e:
from lr.mustache.oaipmh import Error as err_stache
err = err_stache()
return self._returnResponse(err.xml(e), res=t_res)
def get_service_doc():
global __service_doc
if not __service_doc or recache:
__service_doc = h.getServiceDocument(config["lr.publish.docid"])
return __service_doc
def _handleOAIRequest(self, format='html'):
t_req = request._current_obj()
t_res = response._current_obj()
enable_flow_control = False
fc_id_limit = None
fc_doc_limit = None
service_id = None
serviceDoc = h.getServiceDocument(appConfig['lr.oaipmh.docid'])
if serviceDoc != None:
if 'service_id' in serviceDoc:
service_id = serviceDoc['service_id']
if 'service_data' in serviceDoc:
serviceData = serviceDoc['service_data']
if 'flow_control' in serviceData:
enable_flow_control = serviceData['flow_control']
if enable_flow_control and 'id_limit' in serviceData:
fc_id_limit = serviceData['id_limit']
elif enable_flow_control:
fc_id_limit = 100
if enable_flow_control and 'doc_limit' in serviceData:
fc_doc_limit = serviceData['doc_limit']
elif enable_flow_control:
fc_doc_limit = 100
o = oaipmh()
def GetRecord(params):
try:
from lr.mustache.oaipmh import GetRecord as must_GetRecord
identifier = params["identifier"]
if params["by_doc_ID"] == True:
resolver = OAIPMHDocumentResolver()
single_doc = o.get_record(params["identifier"])
if single_doc is not None:
docList = [resolver.process({ "doc": single_doc })]
else:
docList = []
else:
docList = o.get_records_by_resource(params["identifier"])
doc_idx = 0
valid_docs = 0
mustache = must_GetRecord()
for doc in docList:
if doc is not None:
doc_idx += 1
if "payload_schema" in doc and params["metadataPrefix"] in map(lambda x: o_mod.getMetadataPrefix(x), doc["payload_schema"]) and OAIPMHDocumentResolver.PAYLOAD_ERROR not in doc:
valid_docs += 1
if valid_docs == 1:
part = mustache.prefix(**self._initMustache(args=params, req=t_req))
yield h.fixUtf8(self._returnResponse(part, res=t_res))
part = mustache.doc(doc)
yield h.fixUtf8(self._returnResponse(part, res=t_res))
if doc_idx == 0:
raise IdDoesNotExistError(params['verb'], req=t_req)
elif valid_docs == 0:
raise CannotDisseminateFormatError(params['verb'], req=t_req)
else:
yield h.fixUtf8(self._returnResponse(mustache.suffix(), res=t_res))
except oaipmherrors.Error as e:
from lr.mustache.oaipmh import Error as err_stache
err = err_stache()
yield h.fixUtf8(self._returnResponse(err.xml(e), res=t_res))
def ListGeneric(params, showDocs=False, record_limit=None):
if not showDocs:
from lr.mustache.oaipmh import ListIdentifiers as must_ListID
mustache = must_ListID()
else:
from lr.mustache.oaipmh import ListRecords as must_ListRec
mustache = must_ListRec()
try:
doc_index = 0
err_count = 0
metadataPrefix=params["metadataPrefix"]
from_date=params["from"]
until_date=params["until"]
doc_err = None
rendered_init = False
resumptionToken = None if "resumptionToken" not in params else params['resumptionToken']
records = o.list_identifiers_or_records(metadataPrefix,
from_date=from_date,
until_date=until_date,
rt=resumptionToken,
fc_limit=record_limit,
include_docs=showDocs )
for ident in records:
doc_index += 1
doc_err = False
if OAIPMHDocumentResolver.PAYLOAD_ERROR in ident:
err_count += 1
doc_err = True
log.debug("Payload Error detected, doc_index: {0}, err_count: {1}".format(doc_index, err_count))
if doc_index - err_count == 1:
rendered_init = True
part = mustache.prefix(**self._initMustache(args=params, req=t_req))
yield h.fixUtf8(self._returnResponse(part, res=t_res))
if doc_err is False and (record_limit is None or doc_index <= record_limit):
part = mustache.doc(ident)
yield h.fixUtf8(part)
elif enable_flow_control:
from lr.lib import resumption_token
if doc_index - err_count > 0 and doc_index > record_limit:
opts = o.list_opts(metadataPrefix, h.convertToISO8601UTC(ident["node_timestamp"]), until_date)
opts["startkey_docid"] = ident["doc_ID"]
token = resumption_token.get_token(serviceid=service_id, from_date=from_date, until_date=until_date, **opts)
part = mustache.resumptionToken(token)
yield h.fixUtf8(part)
break
elif doc_index - err_count == 0 and doc_index > record_limit:
opts = o.list_opts(metadataPrefix, h.convertToISO8601UTC(ident["node_timestamp"]), until_date)
opts["startkey_docid"] = ident["doc_ID"]
payload = resumption_token.get_payload(from_date=from_date, until_date=until_date, **opts)
records = o.list_identifiers_or_records(metadataPrefix,
from_date=from_date,
until_date=until_date,
rt=payload,
fc_limit=record_limit,
include_docs=showDocs )
doc_index = 0
err_count = 0
if doc_index == 0 and err_count == 0:
raise NoRecordsMatchError(params['verb'], req=t_req)
elif (doc_index - err_count) == 0:
raise CannotDisseminateFormatError(params['verb'], req=t_req)
else:
if enable_flow_control and doc_index <= record_limit:
yield h.fixUtf8(mustache.resumptionToken())
yield h.fixUtf8(mustache.suffix())
except oaipmherrors.Error as e:
if not rendered_init:
from lr.mustache.oaipmh import Error as err_stache
err = err_stache()
yield h.fixUtf8(self._returnResponse(err.xml(e), res=t_res))
else:
from lr.mustache.oaipmh import ErrorOnly as err_stache
err = err_stache()
yield h.fixUtf8(self._returnResponse(err.xml(e)+mustache.suffix(), res=t_res))
except:
log.exception("Unknown Error Occurred")
def ListIdentifiers(params):
return ListGeneric(params, False, fc_id_limit)
def ListRecords(params):
return ListGeneric(params, True, fc_doc_limit)
# def ListRecords(params):
# try:
# from lr.mustache.oaipmh import ListRecords as must_ListRec
#
# doc_index = 0
# mustache = must_ListRec()
# for record in o.list_records(params["metadataPrefix"],from_date=params["from"], until_date=params["until"] ):
# doc_index += 1
# log.debug(json.dumps(record))
# if doc_index == 1:
# part = mustache.prefix(**self._initMustache(args=params, req=t_req))
# yield self._returnResponse(part, res=t_res)
#
# part = mustache.doc(record)
# yield self._returnResponse(part, res=t_res)
#
#
# if doc_index == 0:
# raise NoRecordsMatchError(params['verb'], req=t_req)
# else:
# yield mustache.suffix()
#
# except oaipmherrors.Error as e:
# from lr.mustache.oaipmh import Error as err_stache
# err = err_stache()
# yield self._returnResponse(err.xml(e), res=t_res)
# except:
# log.exception("Unable to render template")
def Identify(params=None):
body = ""
try:
self._initRender(params, ctx=c, req=t_req)
c.identify = o.identify()
body = render("/oaipmh-Identify.mako")
except Exception as e:
raise BadVerbError()
return self._returnResponse(body, res=t_res)
def ListMetadataFormats(params):
body = ""
try:
self._initRender(params, ctx=c, req=t_req)
fmts = o.list_metadata_formats(identity=params["identifier"], by_doc_ID=params["by_doc_ID"])
if len(fmts) == 0:
raise NoMetadataFormats(params["verb"])
c.formats = fmts
body = render("/oaipmh-ListMetadataFormats.mako")
return self._returnResponse(body, res=t_res)
except Error as e:
raise e
def ListSets(params=None):
raise NoSetHierarchyError(verb)
def NotYetSupported(params=None):
raise BadVerbError()
switch = {
'GetRecord': GetRecord,
'ListRecords': ListRecords,
'ListIdentifiers': ListIdentifiers,
'Identify': Identify,
'ListMetadataFormats': ListMetadataFormats,
'ListSets': ListSets
}
try:
params = self._parseParams(flow_control=enable_flow_control, serviceid=service_id)
# If this is a special case where we are actually using OAI interface to serve basic harvest
if params.has_key("metadataPrefix") and params["metadataPrefix"] == "LR_JSON_0.10.0":
if params.has_key("identifier") == True:
params[self.REQUESTID] = params["identifier"]
if params.has_key("from") and isinstance(params["from"], datetime):
params["from"] = h.convertToISO8601Zformat(params["from"])
if params.has_key("until") and isinstance(params["until"], datetime):
params["until"] = h.convertToISO8601Zformat(params["until"])
return HarvestController.harvest(self, params, request.body, params['verb'].lower())
verb = params['verb']
response.headers['Content-Type'] = "text/xml; charset=utf-8"
return switch[verb](params)
except Error as e:
from lr.mustache.oaipmh import Error as err_stache
err = err_stache()
return self._returnResponse(err.xml(e), res=t_res)
def get_service_doc():
global __service_doc
if not __service_doc or recache:
__service_doc = h.getServiceDocument(config["lr.publish.docid"])
return __service_doc