def identify(): data = self.get_base_response(verb, body) serviceDoc = helpers.getServiceDocument(BASIC_HARVEST_SERVICE_DOC) data['identify'] = { "node_id": sourceLRNode.nodeDescription.node_name, "repositoryName": sourceLRNode.communityDescription.community_name, "baseURL": serviceDoc['service_endpoint'], "protocolVersion": "2.0", "service_version": serviceDoc['service_version'], "earliestDatestamp": h.earliestDate(), "deletedRecord": sourceLRNode.nodeDescription. node_policy['deleted_data_policy'], "granularity": helpers.getDatetimePrecision(serviceDoc), "adminEmail": sourceLRNode.nodeDescription.node_admin_identity } return json.dumps(data)
def get_service_doc(): global __service_doc if not __service_doc or recache: __service_doc = h.getServiceDocument( m.base_model.appConfig["lr.sword.docid"]) return __service_doc
def mapResource(config_key, member_name, collection_name): try: service_doc_id = config[config_key] service_doc = h.getServiceDocument(service_doc_id) if service_doc is not None and service_doc["active"]: map.resource(member_name, collection_name) map.connect("/" + collection_name, controller=collection_name, action='options', conditions=dict(method=['OPTIONS'])) if member_name == 'swordservice': map.connect("/swordpub", controller='swordservice', action='create') if member_name == 'distribute': map.connect("/destination", controller='distribute', action='destination', conditions=dict(method='GET')) log.info( "Enabling service route for: {0} member: {1} collection: {2}" .format(service_doc_id, member_name, collection_name)) else: log.info( "Service route for {0} is disabled".format(service_doc_id)) except: log.exception( "Exception caught: Not enabling service route for config: {0} member: {1} collection: {2}" .format(config_key, member_name, collection_name))
def test_resumption(self): slice_doc = helpers.getServiceDocument("access:slice") page_size = slice_doc["service_data"]["doc_limit"] ##add test to assert that flow control is enabled, check that flow_control in service_data is true parameters = {} parameters[IDENTITY] = self.identities[1] + "test_resumption" parameters[IDS_ONLY] = False response = self._slice(parameters) data = json.loads(response.body) docs = data["documents"] if len(docs) != 100: print "resumption assert will fail. doc count is: " + str(len(docs)) assert len(docs) == 100 for doc in docs: assert self._checkIdentity(doc["resource_data_description"], self.identities[1] + "test_resumption") resumption_token = data["resumption_token"] parameters[RESUMPTION] = resumption_token response = self._slice(parameters) data = json.loads(response.body) docs = data["documents"] assert len(docs) == 50 for doc in docs: assert self._checkIdentity(doc["resource_data_description"], self.identities[1] + "test_resumption")
def mapResource(config_key, member_name, collection_name): try: service_doc_id = config[config_key] service_doc = h.getServiceDocument(service_doc_id) if service_doc is not None and service_doc["active"]: map.resource(member_name, collection_name) map.connect( "/" + collection_name, controller=collection_name, action="options", conditions=dict(method=["OPTIONS"]), ) if member_name == "swordservice": map.connect("/swordpub", controller="swordservice", action="create") if member_name == "distribute": map.connect( "/destination", controller="distribute", action="destination", conditions=dict(method="GET") ) log.info( "Enabling service route for: {0} member: {1} collection: {2}".format( service_doc_id, member_name, collection_name ) ) else: log.info("Service route for {0} is disabled".format(service_doc_id)) except: log.exception( "Exception caught: Not enabling service route for config: {0} member: {1} collection: {2}".format( config_key, member_name, collection_name ) )
def test_resumption(self): slice_doc = helpers.getServiceDocument(config["lr.slice.docid"]) page_size = slice_doc["service_data"]["doc_limit"] ##add test to assert that flow control is enabled, check that flow_control in service_data is true date_int = helpers.convertDateTime(self.test_start_date_string) parameters = {} parameters[START_DATE] = self.test_start_date_string parameters[IDS_ONLY] = False parameters[ANY_TAGS] = self.testKeys[0] + 'test_resumption' response = self._slice(parameters) result = json.loads(response.body) assert "resumption_token" in result test_key = self.testKeys[0] + 'test_resumption' def validate_page(docs): assert len( docs ) <= page_size, "resumption assert will fail. doc count is: " + str( len(docs)) assert len([ x for x in docs if helpers.convertDateTime(x['resource_data_description'] ["node_timestamp"]) >= date_int ]) == len(docs) assert all(test_key in doc['resource_data_description']["keys"] for doc in docs) self._validate_page(parameters, response, validate_page, 10)
def test_resumption(self): # response = urlopen(obj.couch_url+"/resource_data/_design/learningregistry-slice/_view/docs?reduce=false&key=\""+obj.testDataKey+"\"") # body = response.read() # data = json.loads(body) # page_size = data["rows"] slice_doc = helpers.getServiceDocument("access:slice") page_size = slice_doc["service_data"]["doc_limit"] ##add test to assert that flow control is enabled, check that flow_control in service_data is true parameters = {} parameters[IDENTITY] = self.identities[1]+"test_resumption" parameters[IDS_ONLY] = False response = self._slice(parameters) data = json.loads(response.body) docs = data["documents"] if len(docs)!=100 : print "resumption assert will fail. doc count is: " + str(len(docs)) assert len(docs)==100 for doc in docs: assert self._checkIdentity(doc['resource_data_description'], self.identities[1]+"test_resumption") resumption_token = data["resumption_token"] parameters[RESUMPTION] = resumption_token response = self._slice(parameters) data = json.loads(response.body) docs = data["documents"] assert len(docs)==50 for doc in docs: assert self._checkIdentity(doc['resource_data_description'], self.identities[1]+"test_resumption")
def create(self): results = {self.__OK:True} error_message = None try: data = json.loads(request.body) doc_limit = h.getServiceDocument(config['lr.publish.docid'])['service_data']['doc_limit'] if not self.__DOCUMENTS in data.keys(): # Comply with LR-RQST-009 'Missing documents in POST' results[self.__ERROR] = "Missing field 'documents' in post body" elif len(data[self.__DOCUMENTS]) < 1: # Comply with LR-API-PUBLISH-001 'List of documents is empty' results[self.__ERROR] = "List of documents is empty" elif len(data[self.__DOCUMENTS]) > doc_limit: error_message = "number of posted docs {0} exceeds doc limit: {1}".format( len(data['documents']), str(doc_limit)) log.debug(error_message) results[self.__ERROR] = error_message else: results[self.__DOCUMENT_RESULTS ] = map(self._publish, data[self.__DOCUMENTS]) except Exception as ex: log.exception(ex) results[self.__ERROR] = str(ex) if results.has_key(self.__ERROR): results[self.__OK] = False return json.dumps(results)
def __init__(self, server=appConfig["couchdb.url"], database="resource_data"): """ Constructor """ harvest.__init__(self, server, database) self.server = couchdb.Server(server) self.db = self.server[database] self.res_data_url = "/".join([appConfig["couchdb.url"], appConfig["couchdb.db.resourcedata"]]) self.service_doc = helpers.getServiceDocument(appConfig["lr.oaipmh.docid"])
def mapResource(config_key, member_name, collection_name): try: service_doc_id = config[config_key] service_doc = h.getServiceDocument(service_doc_id) if service_doc is not None and service_doc["active"]: map.resource(member_name, collection_name) if member_name == 'swordservice': map.connect("/swordpub",controller='swordservice',action='create') log.info("Enabling service route for: {0} member: {1} collection: {2}".format(service_doc_id, member_name, collection_name)) else: log.info("Service route for {0} is disabled".format(service_doc_id)) except: log.exception("Exception caught: Not enabling service route for config: {0} member: {1} collection: {2}".format(config_key, member_name, collection_name))
def __init__(self, server=appConfig['couchdb.url'], database="resource_data"): ''' Constructor ''' harvest.__init__(self, server, database) self.server = couchdb.Server(server) self.db = self.server[database] self.res_data_url = '/'.join([ appConfig['couchdb.url'], appConfig['couchdb.db.resourcedata'] ]) self.service_doc = helpers.getServiceDocument(appConfig["lr.oaipmh.docid"])
def identify(): data = self.get_base_response(verb,body) serviceDoc = helpers.getServiceDocument(BASIC_HARVEST_SERVICE_DOC) data['identify']={ "node_id":sourceLRNode.nodeDescription.node_name, "repositoryName":sourceLRNode.communityDescription.community_name, "baseURL":serviceDoc['service_endpoint'], "protocolVersion":"2.0", "service_version":serviceDoc['service_version'], "earliestDatestamp":h.earliestDate(), "deletedRecord":sourceLRNode.nodeDescription.node_policy['deleted_data_policy'], "granularity":helpers.getDatetimePrecision(serviceDoc), "adminEmail":sourceLRNode.nodeDescription.node_admin_identity } return json.dumps(data)
def _getServiceDocment(self,full_docs): self.enable_flow_control = False self.limit = None self.service_id = None serviceDoc = h.getServiceDocument(appConfig['lr.obtain.docid']) if serviceDoc != None: if 'service_id' in serviceDoc: self.service_id = serviceDoc['service_id'] if 'service_data' in serviceDoc: serviceData = serviceDoc['service_data'] if 'flow_control' in serviceData: self.enable_flow_control = serviceData['flow_control'] limit_type = 'id_limit' if full_docs: limit_type = "doc_limit" if self.enable_flow_control and limit_type in serviceData: self.limit = serviceData['id_limit'] elif self.enable_flow_control: self.limit = 100
def mapResource(config_key, member_name, collection_name): try: service_doc_id = config[config_key] service_doc = h.getServiceDocument(service_doc_id) if service_doc is not None and service_doc["active"]: map.resource(member_name, collection_name) if member_name == 'swordservice': map.connect("/swordpub", controller='swordservice', action='create') log.info( "Enabling service route for: {0} member: {1} collection: {2}" .format(service_doc_id, member_name, collection_name)) else: log.info( "Service route for {0} is disabled".format(service_doc_id)) except: log.exception( "Exception caught: Not enabling service route for config: {0} member: {1} collection: {2}" .format(config_key, member_name, collection_name))
def _getServiceDocment(self, full_docs): self.enable_flow_control = False self.limit = None self.service_id = None serviceDoc = h.getServiceDocument(appConfig['lr.obtain.docid']) if serviceDoc != None: if 'service_id' in serviceDoc: self.service_id = serviceDoc['service_id'] if 'service_data' in serviceDoc: serviceData = serviceDoc['service_data'] if 'flow_control' in serviceData: self.enable_flow_control = serviceData['flow_control'] limit_type = 'id_limit' if full_docs: limit_type = "doc_limit" if self.enable_flow_control and limit_type in serviceData: self.limit = serviceData['id_limit'] elif self.enable_flow_control: self.limit = 100
def test_resumption(self): slice_doc = helpers.getServiceDocument(config["lr.slice.docid"]) page_size = slice_doc["service_data"]["doc_limit"] ##add test to assert that flow control is enabled, check that flow_control in service_data is true date_int = helpers.convertDateTime(self.test_start_date_string) parameters = {} parameters[START_DATE] = self.test_start_date_string parameters[IDS_ONLY] = False parameters[ANY_TAGS] = self.testKeys[0] + 'test_resumption' response = self._slice(parameters) result = json.loads(response.body) assert "resumption_token" in result test_key = self.testKeys[0] + 'test_resumption' def validate_page(docs): assert len(docs) <= page_size, "resumption assert will fail. doc count is: " + str(len(docs)) assert len([x for x in docs if helpers.convertDateTime(x['resource_data_description']["node_timestamp"]) >= date_int]) == len(docs) assert all(test_key in doc['resource_data_description']["keys"] for doc in docs) self._validate_page(parameters, response, validate_page, 10)
def create(self): results = {self.__OK:True} error_message = None try: data = json.loads(request.body) doc_limit = h.getServiceDocument(config['lr.publish.docid'])['service_data']['doc_limit'] if len(data[self.__DOCUMENTS]) > doc_limit: error_message = "number of posted docs {0} exceeds doc limit: {1}".format( len(data['documents']), str(doc_limit)) log.debug(error_message) results[self.__ERROR] = error_message else: results[self.__DOCUMENT_RESULTS ] = map(self._publish, data[self.__DOCUMENTS]) except Exception as ex: log.exception(ex) results[self.__ERROR] = "internal error" if results.has_key(self.__ERROR): results[self.__OK] = False return json.dumps(results)
def _getServiceDocment(self,full_docs): self.enable_flow_control = False self.limit = None self.service_id = None serviceDoc = helpers.getServiceDocument(BASIC_HARVEST_SERVICE_DOC) if serviceDoc != None: if 'service_id' in serviceDoc: self.service_id = serviceDoc['service_id'] if 'service_data' in serviceDoc: serviceData = serviceDoc['service_data'] if 'flow_control' in serviceData: self.enable_flow_control = serviceData['flow_control'] limit_type = 'id_limit' if full_docs: limit_type = "doc_limit" if self.enable_flow_control and limit_type in serviceData: self.limit = serviceData[limit_type] elif self.enable_flow_control: self.limit = 100 if 'metadataformats' in serviceData: self.metadataFormats = serviceData['metadataformats']
def _getServiceDocment(self, full_docs): self.enable_flow_control = False self.limit = None self.service_id = None serviceDoc = helpers.getServiceDocument(BASIC_HARVEST_SERVICE_DOC) if serviceDoc != None: if 'service_id' in serviceDoc: self.service_id = serviceDoc['service_id'] if 'service_data' in serviceDoc: serviceData = serviceDoc['service_data'] if 'flow_control' in serviceData: self.enable_flow_control = serviceData['flow_control'] limit_type = 'id_limit' if full_docs: limit_type = "doc_limit" if self.enable_flow_control and limit_type in serviceData: self.limit = serviceData[limit_type] elif self.enable_flow_control: self.limit = 100 if 'metadataformats' in serviceData: self.metadataFormats = serviceData['metadataformats']
def __before__(self): self.enable_flow_control = False self.fc_id_limit = None self.fc_doc_limit = None self.serviceDoc = h.getServiceDocument("access:slice") if self.serviceDoc != None: if 'service_id' in self.serviceDoc: self.service_id = self.serviceDoc['service_id'] if 'service_data' in self.serviceDoc: serviceData = self.serviceDoc['service_data'] if 'flow_control' in serviceData: self.enable_flow_control = serviceData['flow_control'] if self.enable_flow_control and 'id_limit' in serviceData: self.fc_id_limit = serviceData['id_limit'] elif self.enable_flow_control: self.fc_id_limit = 100 if self.enable_flow_control and 'doc_limit' in serviceData: self.fc_doc_limit = serviceData['doc_limit'] elif self.enable_flow_control: self.fc_doc_limit = 100
def create(self): results = {self.__OK: True} error_message = None try: data = json.loads(request.body) doc_limit = h.getServiceDocument( config['lr.publish.docid'])['service_data']['doc_limit'] if len(data[self.__DOCUMENTS]) > doc_limit: error_message = "number of posted docs {0} exceeds doc limit: {1}".format( len(data['documents']), str(doc_limit)) log.debug(error_message) results[self.__ERROR] = error_message else: results[self.__DOCUMENT_RESULTS] = map(self._publish, data[self.__DOCUMENTS]) except Exception as ex: log.exception(ex) results[self.__ERROR] = str(ex) if results.has_key(self.__ERROR): results[self.__OK] = False return json.dumps(results)
def get_service_doc(): global __service_doc if not __service_doc or recache: __service_doc = h.getServiceDocument(m.base_model.appConfig["lr.sword.docid"]) return __service_doc
def _handleOAIRequest(self, format='html'): t_req = request._current_obj() t_res = response._current_obj() enable_flow_control = False fc_id_limit = None fc_doc_limit = None service_id = None serviceDoc = h.getServiceDocument(appConfig['lr.oaipmh.docid']) if serviceDoc != None: if 'service_id' in serviceDoc: service_id = serviceDoc['service_id'] if 'service_data' in serviceDoc: serviceData = serviceDoc['service_data'] if 'flow_control' in serviceData: enable_flow_control = serviceData['flow_control'] if enable_flow_control and 'id_limit' in serviceData: fc_id_limit = serviceData['id_limit'] elif enable_flow_control: fc_id_limit = 100 if enable_flow_control and 'doc_limit' in serviceData: fc_doc_limit = serviceData['doc_limit'] elif enable_flow_control: fc_doc_limit = 100 o = oaipmh() def GetRecord(params): try: from lr.mustache.oaipmh import GetRecord as must_GetRecord identifier = params["identifier"] if params["by_doc_ID"] == True: resolver = OAIPMHDocumentResolver() single_doc = o.get_record(params["identifier"]) if single_doc is not None: docList = [resolver.process({"doc": single_doc})] else: docList = [] else: docList = o.get_records_by_resource(params["identifier"]) doc_idx = 0 valid_docs = 0 mustache = must_GetRecord() for doc in docList: if doc is not None: doc_idx += 1 if "payload_schema" in doc and params[ "metadataPrefix"] in map( lambda x: o_mod.getMetadataPrefix(x), doc["payload_schema"] ) and OAIPMHDocumentResolver.PAYLOAD_ERROR not in doc: valid_docs += 1 if valid_docs == 1: part = mustache.prefix(**self._initMustache( args=params, req=t_req)) yield h.fixUtf8( self._returnResponse(part, res=t_res)) part = mustache.doc(doc) yield h.fixUtf8( self._returnResponse(part, res=t_res)) if doc_idx == 0: raise IdDoesNotExistError(params['verb'], req=t_req) elif valid_docs == 0: raise CannotDisseminateFormatError(params['verb'], req=t_req) else: yield h.fixUtf8( self._returnResponse(mustache.suffix(), res=t_res)) except oaipmherrors.Error as e: from lr.mustache.oaipmh import Error as err_stache err = err_stache() yield h.fixUtf8(self._returnResponse(err.xml(e), res=t_res)) def ListGeneric(params, showDocs=False, record_limit=None): if not showDocs: from lr.mustache.oaipmh import ListIdentifiers as must_ListID mustache = must_ListID() else: from lr.mustache.oaipmh import ListRecords as must_ListRec mustache = must_ListRec() try: doc_index = 0 err_count = 0 metadataPrefix = params["metadataPrefix"] from_date = params["from"] until_date = params["until"] doc_err = None rendered_init = False resumptionToken = None if "resumptionToken" not in params else params[ 'resumptionToken'] records = o.list_identifiers_or_records(metadataPrefix, from_date=from_date, until_date=until_date, rt=resumptionToken, fc_limit=record_limit, include_docs=showDocs) for ident in records: doc_index += 1 doc_err = False if OAIPMHDocumentResolver.PAYLOAD_ERROR in ident: err_count += 1 doc_err = True log.debug( "Payload Error detected, doc_index: {0}, err_count: {1}" .format(doc_index, err_count)) if doc_index - err_count == 1: rendered_init = True part = mustache.prefix( **self._initMustache(args=params, req=t_req)) yield h.fixUtf8(self._returnResponse(part, res=t_res)) if doc_err is False and (record_limit is None or doc_index <= record_limit): part = mustache.doc(ident) yield h.fixUtf8(part) elif enable_flow_control: from lr.lib import resumption_token if doc_index - err_count > 0 and doc_index > record_limit: opts = o.list_opts( metadataPrefix, h.convertToISO8601UTC(ident["node_timestamp"]), until_date) opts["startkey_docid"] = ident["doc_ID"] token = resumption_token.get_token( serviceid=service_id, from_date=from_date, until_date=until_date, **opts) part = mustache.resumptionToken(token) yield h.fixUtf8(part) break elif doc_index - err_count == 0 and doc_index > record_limit: opts = o.list_opts( metadataPrefix, h.convertToISO8601UTC(ident["node_timestamp"]), until_date) opts["startkey_docid"] = ident["doc_ID"] payload = resumption_token.get_payload( from_date=from_date, until_date=until_date, **opts) records = o.list_identifiers_or_records( metadataPrefix, from_date=from_date, until_date=until_date, rt=payload, fc_limit=record_limit, include_docs=showDocs) doc_index = 0 err_count = 0 if doc_index == 0 and err_count == 0: raise NoRecordsMatchError(params['verb'], req=t_req) elif (doc_index - err_count) == 0: raise CannotDisseminateFormatError(params['verb'], req=t_req) else: if enable_flow_control and doc_index <= record_limit: yield h.fixUtf8(mustache.resumptionToken()) yield h.fixUtf8(mustache.suffix()) except oaipmherrors.Error as e: if not rendered_init: from lr.mustache.oaipmh import Error as err_stache err = err_stache() yield h.fixUtf8(self._returnResponse(err.xml(e), res=t_res)) else: from lr.mustache.oaipmh import ErrorOnly as err_stache err = err_stache() yield h.fixUtf8( self._returnResponse(err.xml(e) + mustache.suffix(), res=t_res)) except: log.exception("Unknown Error Occurred") def ListIdentifiers(params): return ListGeneric(params, False, fc_id_limit) def ListRecords(params): return ListGeneric(params, True, fc_doc_limit) # def ListRecords(params): # try: # from lr.mustache.oaipmh import ListRecords as must_ListRec # # doc_index = 0 # mustache = must_ListRec() # for record in o.list_records(params["metadataPrefix"],from_date=params["from"], until_date=params["until"] ): # doc_index += 1 # log.debug(json.dumps(record)) # if doc_index == 1: # part = mustache.prefix(**self._initMustache(args=params, req=t_req)) # yield self._returnResponse(part, res=t_res) # # part = mustache.doc(record) # yield self._returnResponse(part, res=t_res) # # # if doc_index == 0: # raise NoRecordsMatchError(params['verb'], req=t_req) # else: # yield mustache.suffix() # # except oaipmherrors.Error as e: # from lr.mustache.oaipmh import Error as err_stache # err = err_stache() # yield self._returnResponse(err.xml(e), res=t_res) # except: # log.exception("Unable to render template") def Identify(params=None): body = "" try: self._initRender(params, ctx=c, req=t_req) c.identify = o.identify() body = render("/oaipmh-Identify.mako") except Exception as e: raise BadVerbError() return self._returnResponse(body, res=t_res) def ListMetadataFormats(params): body = "" try: self._initRender(params, ctx=c, req=t_req) fmts = o.list_metadata_formats(identity=params["identifier"], by_doc_ID=params["by_doc_ID"]) if len(fmts) == 0: raise NoMetadataFormats(params["verb"]) c.formats = fmts body = render("/oaipmh-ListMetadataFormats.mako") return self._returnResponse(body, res=t_res) except Error as e: raise e def ListSets(params=None): raise NoSetHierarchyError(verb) def NotYetSupported(params=None): raise BadVerbError() switch = { 'GetRecord': GetRecord, 'ListRecords': ListRecords, 'ListIdentifiers': ListIdentifiers, 'Identify': Identify, 'ListMetadataFormats': ListMetadataFormats, 'ListSets': ListSets } try: params = self._parseParams(flow_control=enable_flow_control, serviceid=service_id) # If this is a special case where we are actually using OAI interface to serve basic harvest if params.has_key("metadataPrefix") and params[ "metadataPrefix"] == "LR_JSON_0.10.0": if params.has_key("identifier") == True: params[self.REQUESTID] = params["identifier"] if params.has_key("from") and isinstance( params["from"], datetime): params["from"] = h.convertToISO8601Zformat(params["from"]) if params.has_key("until") and isinstance( params["until"], datetime): params["until"] = h.convertToISO8601Zformat( params["until"]) return HarvestController.harvest(self, params, request.body, params['verb'].lower()) verb = params['verb'] response.headers['Content-Type'] = "text/xml; charset=utf-8" return switch[verb](params) except Error as e: from lr.mustache.oaipmh import Error as err_stache err = err_stache() return self._returnResponse(err.xml(e), res=t_res)
def get_service_doc(): global __service_doc if not __service_doc or recache: __service_doc = h.getServiceDocument(config["lr.publish.docid"]) return __service_doc
def _handleOAIRequest(self, format='html'): t_req = request._current_obj() t_res = response._current_obj() enable_flow_control = False fc_id_limit = None fc_doc_limit = None service_id = None serviceDoc = h.getServiceDocument(appConfig['lr.oaipmh.docid']) if serviceDoc != None: if 'service_id' in serviceDoc: service_id = serviceDoc['service_id'] if 'service_data' in serviceDoc: serviceData = serviceDoc['service_data'] if 'flow_control' in serviceData: enable_flow_control = serviceData['flow_control'] if enable_flow_control and 'id_limit' in serviceData: fc_id_limit = serviceData['id_limit'] elif enable_flow_control: fc_id_limit = 100 if enable_flow_control and 'doc_limit' in serviceData: fc_doc_limit = serviceData['doc_limit'] elif enable_flow_control: fc_doc_limit = 100 o = oaipmh() def GetRecord(params): try: from lr.mustache.oaipmh import GetRecord as must_GetRecord identifier = params["identifier"] if params["by_doc_ID"] == True: resolver = OAIPMHDocumentResolver() single_doc = o.get_record(params["identifier"]) if single_doc is not None: docList = [resolver.process({ "doc": single_doc })] else: docList = [] else: docList = o.get_records_by_resource(params["identifier"]) doc_idx = 0 valid_docs = 0 mustache = must_GetRecord() for doc in docList: if doc is not None: doc_idx += 1 if "payload_schema" in doc and params["metadataPrefix"] in map(lambda x: o_mod.getMetadataPrefix(x), doc["payload_schema"]) and OAIPMHDocumentResolver.PAYLOAD_ERROR not in doc: valid_docs += 1 if valid_docs == 1: part = mustache.prefix(**self._initMustache(args=params, req=t_req)) yield h.fixUtf8(self._returnResponse(part, res=t_res)) part = mustache.doc(doc) yield h.fixUtf8(self._returnResponse(part, res=t_res)) if doc_idx == 0: raise IdDoesNotExistError(params['verb'], req=t_req) elif valid_docs == 0: raise CannotDisseminateFormatError(params['verb'], req=t_req) else: yield h.fixUtf8(self._returnResponse(mustache.suffix(), res=t_res)) except oaipmherrors.Error as e: from lr.mustache.oaipmh import Error as err_stache err = err_stache() yield h.fixUtf8(self._returnResponse(err.xml(e), res=t_res)) def ListGeneric(params, showDocs=False, record_limit=None): if not showDocs: from lr.mustache.oaipmh import ListIdentifiers as must_ListID mustache = must_ListID() else: from lr.mustache.oaipmh import ListRecords as must_ListRec mustache = must_ListRec() try: doc_index = 0 err_count = 0 metadataPrefix=params["metadataPrefix"] from_date=params["from"] until_date=params["until"] doc_err = None rendered_init = False resumptionToken = None if "resumptionToken" not in params else params['resumptionToken'] records = o.list_identifiers_or_records(metadataPrefix, from_date=from_date, until_date=until_date, rt=resumptionToken, fc_limit=record_limit, include_docs=showDocs ) for ident in records: doc_index += 1 doc_err = False if OAIPMHDocumentResolver.PAYLOAD_ERROR in ident: err_count += 1 doc_err = True log.debug("Payload Error detected, doc_index: {0}, err_count: {1}".format(doc_index, err_count)) if doc_index - err_count == 1: rendered_init = True part = mustache.prefix(**self._initMustache(args=params, req=t_req)) yield h.fixUtf8(self._returnResponse(part, res=t_res)) if doc_err is False and (record_limit is None or doc_index <= record_limit): part = mustache.doc(ident) yield h.fixUtf8(part) elif enable_flow_control: from lr.lib import resumption_token if doc_index - err_count > 0 and doc_index > record_limit: opts = o.list_opts(metadataPrefix, h.convertToISO8601UTC(ident["node_timestamp"]), until_date) opts["startkey_docid"] = ident["doc_ID"] token = resumption_token.get_token(serviceid=service_id, from_date=from_date, until_date=until_date, **opts) part = mustache.resumptionToken(token) yield h.fixUtf8(part) break elif doc_index - err_count == 0 and doc_index > record_limit: opts = o.list_opts(metadataPrefix, h.convertToISO8601UTC(ident["node_timestamp"]), until_date) opts["startkey_docid"] = ident["doc_ID"] payload = resumption_token.get_payload(from_date=from_date, until_date=until_date, **opts) records = o.list_identifiers_or_records(metadataPrefix, from_date=from_date, until_date=until_date, rt=payload, fc_limit=record_limit, include_docs=showDocs ) doc_index = 0 err_count = 0 if doc_index == 0 and err_count == 0: raise NoRecordsMatchError(params['verb'], req=t_req) elif (doc_index - err_count) == 0: raise CannotDisseminateFormatError(params['verb'], req=t_req) else: if enable_flow_control and doc_index <= record_limit: yield h.fixUtf8(mustache.resumptionToken()) yield h.fixUtf8(mustache.suffix()) except oaipmherrors.Error as e: if not rendered_init: from lr.mustache.oaipmh import Error as err_stache err = err_stache() yield h.fixUtf8(self._returnResponse(err.xml(e), res=t_res)) else: from lr.mustache.oaipmh import ErrorOnly as err_stache err = err_stache() yield h.fixUtf8(self._returnResponse(err.xml(e)+mustache.suffix(), res=t_res)) except: log.exception("Unknown Error Occurred") def ListIdentifiers(params): return ListGeneric(params, False, fc_id_limit) def ListRecords(params): return ListGeneric(params, True, fc_doc_limit) # def ListRecords(params): # try: # from lr.mustache.oaipmh import ListRecords as must_ListRec # # doc_index = 0 # mustache = must_ListRec() # for record in o.list_records(params["metadataPrefix"],from_date=params["from"], until_date=params["until"] ): # doc_index += 1 # log.debug(json.dumps(record)) # if doc_index == 1: # part = mustache.prefix(**self._initMustache(args=params, req=t_req)) # yield self._returnResponse(part, res=t_res) # # part = mustache.doc(record) # yield self._returnResponse(part, res=t_res) # # # if doc_index == 0: # raise NoRecordsMatchError(params['verb'], req=t_req) # else: # yield mustache.suffix() # # except oaipmherrors.Error as e: # from lr.mustache.oaipmh import Error as err_stache # err = err_stache() # yield self._returnResponse(err.xml(e), res=t_res) # except: # log.exception("Unable to render template") def Identify(params=None): body = "" try: self._initRender(params, ctx=c, req=t_req) c.identify = o.identify() body = render("/oaipmh-Identify.mako") except Exception as e: raise BadVerbError() return self._returnResponse(body, res=t_res) def ListMetadataFormats(params): body = "" try: self._initRender(params, ctx=c, req=t_req) fmts = o.list_metadata_formats(identity=params["identifier"], by_doc_ID=params["by_doc_ID"]) if len(fmts) == 0: raise NoMetadataFormats(params["verb"]) c.formats = fmts body = render("/oaipmh-ListMetadataFormats.mako") return self._returnResponse(body, res=t_res) except Error as e: raise e def ListSets(params=None): raise NoSetHierarchyError(verb) def NotYetSupported(params=None): raise BadVerbError() switch = { 'GetRecord': GetRecord, 'ListRecords': ListRecords, 'ListIdentifiers': ListIdentifiers, 'Identify': Identify, 'ListMetadataFormats': ListMetadataFormats, 'ListSets': ListSets } try: params = self._parseParams(flow_control=enable_flow_control, serviceid=service_id) # If this is a special case where we are actually using OAI interface to serve basic harvest if params.has_key("metadataPrefix") and params["metadataPrefix"] == "LR_JSON_0.10.0": if params.has_key("identifier") == True: params[self.REQUESTID] = params["identifier"] if params.has_key("from") and isinstance(params["from"], datetime): params["from"] = h.convertToISO8601Zformat(params["from"]) if params.has_key("until") and isinstance(params["until"], datetime): params["until"] = h.convertToISO8601Zformat(params["until"]) return HarvestController.harvest(self, params, request.body, params['verb'].lower()) verb = params['verb'] response.headers['Content-Type'] = "text/xml; charset=utf-8" return switch[verb](params) except Error as e: from lr.mustache.oaipmh import Error as err_stache err = err_stache() return self._returnResponse(err.xml(e), res=t_res)