def get_payload(startkey=None, endkey={}, startkey_docid=None, from_date=None, until_date=None, key=None, keys=None, any_tags=None, identity=None):
    payload = {}

    payload["startkey"] = startkey
    payload["endkey"] = endkey

    if key:
        payload["key"] = key

    if keys:
        payload["keys"] = keys

    if startkey_docid:
        payload["startkey_docid"] = startkey_docid

    if any_tags:
        payload['any_tags'] = any_tags

    if identity:
        payload['identity'] = identity

    if from_date and isinstance(from_date, datetime.datetime):
        from lr.lib import helpers as h
        payload["from_date"] = h.convertToISO8601Zformat(from_date)
    if until_date and isinstance(until_date, datetime.datetime):
        payload["until_date"] = h.convertToISO8601Zformat(until_date)

    return payload
Esempio n. 2
0
    def list_identifiers(self,h, body ,params, verb = 'GET'):        

        data = self.get_base_response(verb,body)
        if params.has_key('from'):
            data['request']['from'] = params['from']
        if params.has_key('until'):
            data['request']['until'] = params['until']
        from_date, until_date = self._test_time_params(params)
        data['listidentifiers'] =  []
        base_response =  json.dumps(data).split('[')
        self._getServiceDocment(False)        
        resumption_token = None
        if self.enable_flow_control and params.has_key('resumption_token'):
            resumption_token = rt.parse_token(self.service_id,params['resumption_token'])                
        yield base_response[0] +'['
        first = True;
        count = 0
        lastID = None
        lastKey = None
        for d in h.list_identifiers(from_date,until_date,resumption_token=resumption_token, limit=self.limit):
            count += 1
            lastID = d['id']
            lastKey = d['key']
            if not first:
                yield ',\n'
            first = False            
            return_value = {"header":{'identifier':d['id'], 'datestamp':helpers.convertToISO8601Zformat(datetime.today()) ,'status':'active'}}
            yield json.dumps(return_value)
        if self.enable_flow_control and self.limit <= count:
            token = rt.get_token(serviceid=self.service_id,startkey=lastKey,endkey=helpers.convertToISO8601Zformat(until_date),startkey_docid=lastID,from_date=helpers.convertToISO8601Zformat(from_date),until_date=helpers.convertToISO8601Zformat(until_date))
            resp = base_response[1]
            yield resp[:-1] +(',"resumption_token":"%s"' %token) +resp[-1:]
        else:
            yield base_response[1]
Esempio n. 3
0
 def getrecord():
     data = self.get_base_response(verb, body)
     by_doc_ID = self._check_bool_param(params, 'by_doc_ID')
     by_resource_ID = self._check_bool_param(params, 'by_resource_ID')
     if not params.has_key(self.REQUESTID) and not params.has_key(
             self.REQUESTID.lower()):
         data['OK'] = False
         data['error'] = 'badArgument'
         return json.dumps(data)
     if by_doc_ID and by_resource_ID:
         data['OK'] = False
         data['error'] = 'badArgument'
         return json.dumps(data)
     if params.has_key(self.REQUESTID):
         request_id = params[self.REQUESTID]
     elif params.has_key(self.REQUESTID.lower()):
         request_id = params[self.REQUESTID.lower()]
     if by_doc_ID:
         document = h.get_record(request_id)
         if document is not None:
             records = map(
                 lambda doc: {
                     "header": {
                         'identifier':
                         doc['_id'],
                         'datestamp':
                         helpers.convertToISO8601Zformat(datetime.today(
                         )),
                         'status':
                         'active'
                     },
                     'resource_data': doc
                 }, [document])
         else:
             records = []
     else:
         request_id = unquote_plus(request_id)
         records = map(
             lambda doc: {
                 "header": {
                     'identifier':
                     doc['_id'],
                     'datestamp':
                     helpers.convertToISO8601Zformat(datetime.today()),
                     'status':
                     'active'
                 },
                 'resource_data': doc
             }, h.get_records_by_resource(request_id))
     if len(records) == 0:
         data['OK'] = False
         data['error'] = 'idDoesNotExist'
         return json.dumps(data)
     data['getrecord'] = {'record': records}
     data['request']['identifier'] = request_id
     data['request']['by_doc_ID'] = by_doc_ID
     data['request']['by_resource_ID'] = by_resource_ID
     return json.dumps(data)
Esempio n. 4
0
        def getrecord():
            data = self.get_base_response(verb, body)
            by_doc_ID = params.has_key('by_doc_ID') and ast.literal_eval(
                str(params['by_doc_ID']))
            by_resource_ID = params.has_key(
                'by_resource_ID') and ast.literal_eval(
                    str(params['by_resource_ID']))
            if not params.has_key('request_id'):
                data['OK'] = False
                data['error'] = 'badArgument'
                return json.dumps(data)
            if by_doc_ID and by_resource_ID:
                data['OK'] = False
                data['error'] = 'badArgument'
                return json.dumps(data)

            request_id = params['request_id']
            if by_doc_ID:
                records = map(
                    lambda doc: {
                        'record': {
                            "header": {
                                'identifier':
                                doc.id,
                                'datestamp':
                                helpers.convertToISO8601Zformat(datetime.today(
                                )),
                                'status':
                                'active'
                            }
                        },
                        'resource_data': doc
                    }, [h.get_record(request_id)])
            else:
                records = map(
                    lambda doc: {
                        'record': {
                            "header": {
                                'identifier':
                                doc.id,
                                'datestamp':
                                helpers.convertToISO8601Zformat(datetime.today(
                                )),
                                'status':
                                'active'
                            }
                        },
                        'resource_data': doc
                    }, h.get_records_by_resource(request_id))
            data['getrecord'] = {'record': records}
            return json.dumps(data)
Esempio n. 5
0
 def list_identifiers(self, h, body, params, verb='GET'):
     data = self.get_base_response(verb, body)
     if params.has_key('from'):
         data['request']['from'] = params['from']
     if params.has_key('until'):
         data['request']['until'] = params['until']
     from_date, until_date = self._test_time_params(params)
     data['listidentifiers'] = []
     base_response = json.dumps(data).split('[')
     yield base_response[0] + '['
     first = True
     for id in h.list_identifiers(from_date, until_date):
         if not first:
             yield ','
         first = False
         return_value = {
             "header": {
                 'identifier': id,
                 'datestamp':
                 helpers.convertToISO8601Zformat(datetime.today()),
                 'status': 'active'
             }
         }
         yield json.dumps(return_value)
     yield base_response[1]
def get_payload(startkey=None, endkey={}, startkey_docid=None, from_date=None, until_date=None):
    payload = {}

    payload["startkey"] = startkey
    payload["endkey"] = endkey
    
    if startkey_docid:
        payload["startkey_docid"] = startkey_docid
        
    if from_date and isinstance(from_date, datetime.datetime):
        from lr.lib import helpers as h
        payload["from_date"] = h.convertToISO8601Zformat(from_date)
    if until_date and isinstance(until_date, datetime.datetime):
        payload["until_date"] = h.convertToISO8601Zformat(until_date)
    
    return payload
Esempio n. 7
0
 def get_base_response(self, verb, body):
     return {
         'OK': True,
         'error': '',
         'responseDate': helpers.convertToISO8601Zformat(datetime.today()),
         'request': {
             'verb': verb,
             'HTTP_request': body
         }
     }
Esempio n. 8
0
 def get_base_response(self, verb, body):
   return {
            'OK':True,
            'error':'',
            'responseDate':helpers.convertToISO8601Zformat(datetime.today()),
            'request':{
              'verb':verb,
              'HTTP_request': body
              }    
           }
def get_payload(startkey=None,
                endkey={},
                startkey_docid=None,
                from_date=None,
                until_date=None):
    payload = {}

    payload["startkey"] = startkey
    payload["endkey"] = endkey

    if startkey_docid:
        payload["startkey_docid"] = startkey_docid

    if from_date and isinstance(from_date, datetime.datetime):
        from lr.lib import helpers as h
        payload["from_date"] = h.convertToISO8601Zformat(from_date)
    if until_date and isinstance(until_date, datetime.datetime):
        payload["until_date"] = h.convertToISO8601Zformat(until_date)

    return payload
Esempio n. 10
0
 def debug_map(doc):
     data = {
         'record': {
             "header": {
                 'identifier':
                 doc.id,
                 'datestamp':
                 helpers.convertToISO8601Zformat(datetime.today()),
                 'status':
                 'active'
             },
             'resource_data': doc
         }
     }
     return data
Esempio n. 11
0
def get_payload(startkey=None,
                endkey={},
                startkey_docid=None,
                from_date=None,
                until_date=None,
                key=None,
                keys=None,
                any_tags=None,
                identity=None):
    payload = {}

    payload["startkey"] = startkey
    payload["endkey"] = endkey

    if key:
        payload["key"] = key

    if keys:
        payload["keys"] = keys

    if startkey_docid:
        payload["startkey_docid"] = startkey_docid

    if any_tags:
        payload['any_tags'] = any_tags

    if identity:
        payload['identity'] = identity

    if from_date and isinstance(from_date, datetime.datetime):
        from lr.lib import helpers as h
        payload["from_date"] = h.convertToISO8601Zformat(from_date)
    if until_date and isinstance(until_date, datetime.datetime):
        payload["until_date"] = h.convertToISO8601Zformat(until_date)

    return payload
Esempio n. 12
0
 def list_identifiers(self,h,body ,params, verb = 'GET'):        
     data = self.get_base_response(verb,body)
     if params.has_key('from'):
         data['request']['from'] = params['from']
     if params.has_key('until'):
         data['request']['until'] = params['until']
     from_date, until_date = self._test_time_params(params)
     data['listidentifiers'] =  []
     base_response =  json.dumps(data).split('[')
     yield base_response[0] +'['
     first = True;
     for id in h.list_identifiers(from_date,until_date):
         if not first:
             yield ','
         first = False            
         return_value = {"header":{'identifier':id, 'datestamp':helpers.convertToISO8601Zformat(datetime.today()) ,'status':'active'}}
         yield json.dumps(return_value)
     yield base_response[1]
Esempio n. 13
0
 def list_records(self, h , body , params, verb = 'GET' ):                
     data = self.get_base_response(verb,body)
     if params.has_key('from'):
         data['request']['from'] = params['from']
     if params.has_key('until'):
         data['request']['until'] = params['until']
     from_date, until_date = self._test_time_params(params)
     data['listrecords'] =  []
     self._getServiceDocment(False)        
     resumption_token = None
     count = 0
     lastID = None
     lastKey = None
     if self.enable_flow_control and params.has_key('resumption_token'):
         resumption_token = rt.parse_token(self.service_id,params['resumption_token'])                        
     base_response =  json.dumps(data).split('[')
     yield base_response[0] +'['
     def debug_map(doc):
         data ={'record':{"header":{'identifier':doc['_id'], 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'},'resource_data':doc}}
         return data
     if from_date > until_date:
       data['OK'] = False
       data['error'] = 'badArgument'
     else:
         first = True
         for data in h.list_records(from_date,until_date,resumption_token=resumption_token, limit=self.limit):                        
             lastID = data['id']
             lastKey = data['key']
             doc = data['doc']
             count += 1
             if not first:
                 yield ',\n'
             first = False
             yield json.dumps(debug_map(doc))
     if self.enable_flow_control and self.limit <= count:
         token = rt.get_token(serviceid=self.service_id,startkey=lastKey,endkey=helpers.convertToISO8601Zformat(until_date),startkey_docid=lastID,from_date=helpers.convertToISO8601Zformat(from_date),until_date=helpers.convertToISO8601Zformat(until_date))
         resp = base_response[1]
         yield resp[:-1] +(',"resumption_token":"%s"' %token) +resp[-1:]
     else:
         yield base_response[1]
Esempio n. 14
0
 def getrecord():
   data = self.get_base_response(verb,body)
   by_doc_ID = self._check_bool_param(params,'by_doc_ID')
   by_resource_ID = self._check_bool_param(params,'by_resource_ID') 
   if not params.has_key(self.REQUESTID):
     data['OK'] = False
     data['error'] = 'badArgument'
     return json.dumps(data)
   if by_doc_ID and by_resource_ID:
     data['OK'] = False
     data['error'] = 'badArgument'
     return json.dumps(data)          
   request_id = params[self.REQUESTID]
   if by_doc_ID:
     document = h.get_record(request_id)
     if document is not None:
         records = map(lambda doc: {"header":{'identifier':doc['_id'], 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'},'resource_data':doc},[document])
     else:
         records = []
   else:
     records = map(lambda doc: {"header":{'identifier':doc['_id'], 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'},'resource_data':doc},h.get_records_by_resource(request_id))
   if len(records) == 0:
     abort(500,'idDoesNotExist')
   data['getrecord'] ={
     'record': records
   }
   data['request']['identifier']  = request_id
   data['request']['by_doc_ID'] = by_doc_ID
   data['request']['by_resource_ID'] = by_resource_ID
   return json.dumps(data)
Esempio n. 15
0
 def debug_map(doc):
     data ={'record':{"header":{'identifier':doc['_id'], 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'},'resource_data':doc}}
     return data
Esempio n. 16
0
    def _handleOAIRequest(self, format='html'):
        t_req = request._current_obj()
        t_res = response._current_obj()

        enable_flow_control = False
        fc_id_limit = None
        fc_doc_limit = None
        service_id = None
        serviceDoc = h.getServiceDocument(appConfig['lr.oaipmh.docid'])
        if serviceDoc != None:
            if 'service_id' in serviceDoc:
                service_id = serviceDoc['service_id']

            if 'service_data' in serviceDoc:
                serviceData = serviceDoc['service_data']
                if 'flow_control' in serviceData:
                    enable_flow_control = serviceData['flow_control']

                if enable_flow_control and 'id_limit' in serviceData:
                    fc_id_limit = serviceData['id_limit']
                elif enable_flow_control:
                    fc_id_limit = 100

                if enable_flow_control and 'doc_limit' in serviceData:
                    fc_doc_limit = serviceData['doc_limit']
                elif enable_flow_control:
                    fc_doc_limit = 100

        o = oaipmh()

        def GetRecord(params):
            try:
                from lr.mustache.oaipmh import GetRecord as must_GetRecord
                identifier = params["identifier"]
                if params["by_doc_ID"] == True:
                    resolver = OAIPMHDocumentResolver()
                    single_doc = o.get_record(params["identifier"])
                    if single_doc is not None:
                        docList = [resolver.process({"doc": single_doc})]
                    else:
                        docList = []
                else:
                    docList = o.get_records_by_resource(params["identifier"])

                doc_idx = 0
                valid_docs = 0
                mustache = must_GetRecord()
                for doc in docList:
                    if doc is not None:
                        doc_idx += 1

                        if "payload_schema" in doc and params[
                                "metadataPrefix"] in map(
                                    lambda x: o_mod.getMetadataPrefix(x),
                                    doc["payload_schema"]
                                ) and OAIPMHDocumentResolver.PAYLOAD_ERROR not in doc:
                            valid_docs += 1

                            if valid_docs == 1:
                                part = mustache.prefix(**self._initMustache(
                                    args=params, req=t_req))
                                yield h.fixUtf8(
                                    self._returnResponse(part, res=t_res))

                            part = mustache.doc(doc)
                            yield h.fixUtf8(
                                self._returnResponse(part, res=t_res))

                if doc_idx == 0:
                    raise IdDoesNotExistError(params['verb'], req=t_req)
                elif valid_docs == 0:
                    raise CannotDisseminateFormatError(params['verb'],
                                                       req=t_req)
                else:
                    yield h.fixUtf8(
                        self._returnResponse(mustache.suffix(), res=t_res))

            except oaipmherrors.Error as e:
                from lr.mustache.oaipmh import Error as err_stache
                err = err_stache()
                yield h.fixUtf8(self._returnResponse(err.xml(e), res=t_res))

        def ListGeneric(params, showDocs=False, record_limit=None):
            if not showDocs:
                from lr.mustache.oaipmh import ListIdentifiers as must_ListID
                mustache = must_ListID()
            else:
                from lr.mustache.oaipmh import ListRecords as must_ListRec
                mustache = must_ListRec()

            try:

                doc_index = 0
                err_count = 0
                metadataPrefix = params["metadataPrefix"]
                from_date = params["from"]
                until_date = params["until"]
                doc_err = None
                rendered_init = False
                resumptionToken = None if "resumptionToken" not in params else params[
                    'resumptionToken']
                records = o.list_identifiers_or_records(metadataPrefix,
                                                        from_date=from_date,
                                                        until_date=until_date,
                                                        rt=resumptionToken,
                                                        fc_limit=record_limit,
                                                        include_docs=showDocs)
                for ident in records:
                    doc_index += 1
                    doc_err = False

                    if OAIPMHDocumentResolver.PAYLOAD_ERROR in ident:
                        err_count += 1
                        doc_err = True
                        log.debug(
                            "Payload Error detected, doc_index: {0}, err_count: {1}"
                            .format(doc_index, err_count))

                    if doc_index - err_count == 1:
                        rendered_init = True
                        part = mustache.prefix(
                            **self._initMustache(args=params, req=t_req))
                        yield h.fixUtf8(self._returnResponse(part, res=t_res))

                    if doc_err is False and (record_limit is None
                                             or doc_index <= record_limit):
                        part = mustache.doc(ident)
                        yield h.fixUtf8(part)
                    elif enable_flow_control:
                        from lr.lib import resumption_token
                        if doc_index - err_count > 0 and doc_index > record_limit:
                            opts = o.list_opts(
                                metadataPrefix,
                                h.convertToISO8601UTC(ident["node_timestamp"]),
                                until_date)
                            opts["startkey_docid"] = ident["doc_ID"]
                            token = resumption_token.get_token(
                                serviceid=service_id,
                                from_date=from_date,
                                until_date=until_date,
                                **opts)
                            part = mustache.resumptionToken(token)
                            yield h.fixUtf8(part)
                            break
                        elif doc_index - err_count == 0 and doc_index > record_limit:
                            opts = o.list_opts(
                                metadataPrefix,
                                h.convertToISO8601UTC(ident["node_timestamp"]),
                                until_date)
                            opts["startkey_docid"] = ident["doc_ID"]
                            payload = resumption_token.get_payload(
                                from_date=from_date,
                                until_date=until_date,
                                **opts)
                            records = o.list_identifiers_or_records(
                                metadataPrefix,
                                from_date=from_date,
                                until_date=until_date,
                                rt=payload,
                                fc_limit=record_limit,
                                include_docs=showDocs)
                            doc_index = 0
                            err_count = 0

                if doc_index == 0 and err_count == 0:
                    raise NoRecordsMatchError(params['verb'], req=t_req)
                elif (doc_index - err_count) == 0:
                    raise CannotDisseminateFormatError(params['verb'],
                                                       req=t_req)
                else:
                    if enable_flow_control and doc_index <= record_limit:
                        yield h.fixUtf8(mustache.resumptionToken())
                    yield h.fixUtf8(mustache.suffix())

            except oaipmherrors.Error as e:
                if not rendered_init:
                    from lr.mustache.oaipmh import Error as err_stache
                    err = err_stache()
                    yield h.fixUtf8(self._returnResponse(err.xml(e),
                                                         res=t_res))
                else:
                    from lr.mustache.oaipmh import ErrorOnly as err_stache
                    err = err_stache()
                    yield h.fixUtf8(
                        self._returnResponse(err.xml(e) + mustache.suffix(),
                                             res=t_res))
            except:
                log.exception("Unknown Error Occurred")

        def ListIdentifiers(params):
            return ListGeneric(params, False, fc_id_limit)

        def ListRecords(params):
            return ListGeneric(params, True, fc_doc_limit)
#        def ListRecords(params):
#            try:
#                from lr.mustache.oaipmh import ListRecords as must_ListRec
#
#                doc_index = 0
#                mustache = must_ListRec()
#                for record in o.list_records(params["metadataPrefix"],from_date=params["from"], until_date=params["until"] ):
#                    doc_index += 1
#                    log.debug(json.dumps(record))
#                    if doc_index == 1:
#                        part = mustache.prefix(**self._initMustache(args=params, req=t_req))
#                        yield self._returnResponse(part, res=t_res)
#
#                    part = mustache.doc(record)
#                    yield self._returnResponse(part, res=t_res)
#
#
#                if doc_index == 0:
#                    raise NoRecordsMatchError(params['verb'], req=t_req)
#                else:
#                    yield mustache.suffix()
#
#            except oaipmherrors.Error as e:
#                from lr.mustache.oaipmh import Error as err_stache
#                err = err_stache()
#                yield self._returnResponse(err.xml(e), res=t_res)
#            except:
#                log.exception("Unable to render template")

        def Identify(params=None):
            body = ""
            try:
                self._initRender(params, ctx=c, req=t_req)
                c.identify = o.identify()
                body = render("/oaipmh-Identify.mako")
            except Exception as e:
                raise BadVerbError()
            return self._returnResponse(body, res=t_res)

        def ListMetadataFormats(params):
            body = ""
            try:
                self._initRender(params, ctx=c, req=t_req)

                fmts = o.list_metadata_formats(identity=params["identifier"],
                                               by_doc_ID=params["by_doc_ID"])
                if len(fmts) == 0:
                    raise NoMetadataFormats(params["verb"])
                c.formats = fmts
                body = render("/oaipmh-ListMetadataFormats.mako")
                return self._returnResponse(body, res=t_res)
            except Error as e:
                raise e

        def ListSets(params=None):
            raise NoSetHierarchyError(verb)

        def NotYetSupported(params=None):
            raise BadVerbError()

        switch = {
            'GetRecord': GetRecord,
            'ListRecords': ListRecords,
            'ListIdentifiers': ListIdentifiers,
            'Identify': Identify,
            'ListMetadataFormats': ListMetadataFormats,
            'ListSets': ListSets
        }
        try:
            params = self._parseParams(flow_control=enable_flow_control,
                                       serviceid=service_id)

            # If this is a special case where we are actually using OAI interface to serve basic harvest
            if params.has_key("metadataPrefix") and params[
                    "metadataPrefix"] == "LR_JSON_0.10.0":
                if params.has_key("identifier") == True:
                    params[self.REQUESTID] = params["identifier"]
                if params.has_key("from") and isinstance(
                        params["from"], datetime):
                    params["from"] = h.convertToISO8601Zformat(params["from"])
                if params.has_key("until") and isinstance(
                        params["until"], datetime):
                    params["until"] = h.convertToISO8601Zformat(
                        params["until"])

                return HarvestController.harvest(self, params, request.body,
                                                 params['verb'].lower())

            verb = params['verb']
            response.headers['Content-Type'] = "text/xml; charset=utf-8"

            return switch[verb](params)
        except Error as e:
            from lr.mustache.oaipmh import Error as err_stache
            err = err_stache()
            return self._returnResponse(err.xml(e), res=t_res)
Esempio n. 17
0
 def listGeneral(self, h , body , params, includeDocs,verb = 'GET'):                        
     data = self.get_base_response(verb,body)
     try:
         from_date, until_date = self._test_time_params(params)
     except Exception as ex:
         log.error(ex)
         data['OK'] = False
         data['error'] = 'badArgument'
         yield json.dumps(data)
         return
     data['request']['from'] = from_date
     data['request']['until'] = until_date         
     if from_date > until_date:
       data['OK'] = False
       data['error'] = 'badArgument'
       yield json.dumps(data)
     else:
         self._getServiceDocment(includeDocs)        
         resumption_token = None
         count = 0
         lastID = None
         lastKey = None
         if self.enable_flow_control and params.has_key('resumption_token'):
             resumption_token = rt.parse_token(self.service_id,params['resumption_token'])                                    
         if includeDocs:       
             data['listrecords'] =  []   
             viewResults = h.list_records(from_date,until_date,resumption_token=resumption_token, limit=self.limit)                
             debug_map = lambda doc: {'record':{"header":{'identifier':doc['id'], 'datestamp':doc['key']+"Z",'status':'active'},'resource_data':doc['doc']}}                
         else:
             data['listidentifiers'] =  []   
             viewResults = h.list_identifiers(from_date,until_date,resumption_token=resumption_token, limit=self.limit)
             debug_map = lambda doc:{"header":{'identifier':doc['id'], 'datestamp':doc['key']+"Z",'status':'active'}}                
         base_response =  json.dumps(data).split('[')            
         yield base_response[0] +'['
         first = True
         for data in viewResults:                        
             lastID = data['id']
             lastKey = data['key']
             count += 1
             if not first:
                 yield ',\n'
             first = False
             yield json.dumps(debug_map(data))
         if self.enable_flow_control and self.limit <= count:
             token = rt.get_token(serviceid=self.service_id,startkey=lastKey,endkey=helpers.convertToISO8601Zformat(until_date),startkey_docid=lastID,from_date=helpers.convertToISO8601Zformat(from_date),until_date=helpers.convertToISO8601Zformat(until_date))
             resp = base_response[1]
             yield resp[:-1] +(',"resumption_token":"%s"' %token) +resp[-1:]
         elif self.limit > count:
             resp = base_response[1]
             yield resp[:-1] +(',"resumption_token":"%s"' %'null') +resp[-1:]    
         else:
             yield base_response[1]
Esempio n. 18
0
    def _handleOAIRequest(self, format='html'):
        t_req = request._current_obj()
        t_res = response._current_obj()
        
        enable_flow_control = False
        fc_id_limit = None
        fc_doc_limit = None
        service_id = None
        serviceDoc = h.getServiceDocument(appConfig['lr.oaipmh.docid'])
        if serviceDoc != None:
            if 'service_id' in serviceDoc:
                service_id = serviceDoc['service_id']
                
            if 'service_data' in serviceDoc:
                serviceData = serviceDoc['service_data']
                if 'flow_control' in serviceData:
                    enable_flow_control = serviceData['flow_control']
                
                if enable_flow_control and 'id_limit' in serviceData:
                    fc_id_limit = serviceData['id_limit']
                elif enable_flow_control:
                    fc_id_limit = 100
                
                if enable_flow_control and 'doc_limit' in serviceData:
                    fc_doc_limit = serviceData['doc_limit']
                elif enable_flow_control:
                    fc_doc_limit = 100
        
        o = oaipmh()
        
        def GetRecord(params):
            try:
                from lr.mustache.oaipmh import GetRecord as must_GetRecord
                identifier = params["identifier"]
                if params["by_doc_ID"] == True:
                    resolver = OAIPMHDocumentResolver()
                    single_doc = o.get_record(params["identifier"])
                    if single_doc is not None: 
                        docList = [resolver.process({ "doc": single_doc })]
                    else:
                        docList = []
                else:
                    docList = o.get_records_by_resource(params["identifier"])

                doc_idx = 0
                valid_docs = 0
                mustache = must_GetRecord()
                for doc in docList:
                    if doc is not None:
                        doc_idx += 1
                        
                        if "payload_schema" in doc and params["metadataPrefix"] in map(lambda x: o_mod.getMetadataPrefix(x), doc["payload_schema"]) and OAIPMHDocumentResolver.PAYLOAD_ERROR not in doc:
                            valid_docs += 1
                        
                            if valid_docs == 1:
                                part = mustache.prefix(**self._initMustache(args=params, req=t_req))
                                yield h.fixUtf8(self._returnResponse(part, res=t_res))
                                
                            part = mustache.doc(doc)
                            yield h.fixUtf8(self._returnResponse(part, res=t_res))
                        
                if doc_idx == 0:
                    raise IdDoesNotExistError(params['verb'], req=t_req)
                elif valid_docs == 0:
                    raise CannotDisseminateFormatError(params['verb'], req=t_req)
                else:
                    yield h.fixUtf8(self._returnResponse(mustache.suffix(), res=t_res))
                
            except oaipmherrors.Error as e:
                from lr.mustache.oaipmh import Error as err_stache
                err = err_stache()
                yield h.fixUtf8(self._returnResponse(err.xml(e), res=t_res))

        def ListGeneric(params, showDocs=False, record_limit=None):
            if not showDocs:
                from lr.mustache.oaipmh import ListIdentifiers as must_ListID
                mustache = must_ListID()
            else:
                from lr.mustache.oaipmh import ListRecords as must_ListRec
                mustache = must_ListRec()
                
            try:
                
                doc_index = 0
                err_count = 0
                metadataPrefix=params["metadataPrefix"]
                from_date=params["from"]
                until_date=params["until"]
                doc_err = None
                rendered_init = False
                resumptionToken = None if "resumptionToken" not in params else params['resumptionToken']
                records = o.list_identifiers_or_records(metadataPrefix,
                                                from_date=from_date, 
                                                until_date=until_date, 
                                                rt=resumptionToken, 
                                                fc_limit=record_limit, 
                                                include_docs=showDocs )
                for ident in records:
                    doc_index += 1
                    doc_err = False
                    
                    if OAIPMHDocumentResolver.PAYLOAD_ERROR in ident:
                        err_count += 1
                        doc_err = True
                        log.debug("Payload Error detected, doc_index: {0}, err_count: {1}".format(doc_index, err_count))
                    
                    if doc_index - err_count == 1:
                        rendered_init = True
                        part = mustache.prefix(**self._initMustache(args=params, req=t_req))
                        yield h.fixUtf8(self._returnResponse(part, res=t_res))

                    if doc_err is False and (record_limit is None or doc_index <= record_limit):
                        part = mustache.doc(ident)
                        yield h.fixUtf8(part)
                    elif enable_flow_control:
                        from lr.lib import resumption_token
                        if doc_index - err_count > 0 and doc_index > record_limit:
                            opts = o.list_opts(metadataPrefix, h.convertToISO8601UTC(ident["node_timestamp"]), until_date)
                            opts["startkey_docid"] = ident["doc_ID"]
                            token = resumption_token.get_token(serviceid=service_id, from_date=from_date, until_date=until_date, **opts)
                            part = mustache.resumptionToken(token)
                            yield h.fixUtf8(part)
                            break
                        elif doc_index - err_count == 0 and doc_index > record_limit:
                            opts = o.list_opts(metadataPrefix, h.convertToISO8601UTC(ident["node_timestamp"]), until_date)
                            opts["startkey_docid"] = ident["doc_ID"]
                            payload = resumption_token.get_payload(from_date=from_date, until_date=until_date, **opts)
                            records = o.list_identifiers_or_records(metadataPrefix,
                                                from_date=from_date, 
                                                until_date=until_date, 
                                                rt=payload, 
                                                fc_limit=record_limit, 
                                                include_docs=showDocs )
                            doc_index = 0
                            err_count = 0
                
                if doc_index == 0 and err_count == 0:
                    raise NoRecordsMatchError(params['verb'], req=t_req)
                elif (doc_index - err_count) == 0:
                    raise CannotDisseminateFormatError(params['verb'], req=t_req)
                else:
                    if enable_flow_control and doc_index <= record_limit:
                        yield h.fixUtf8(mustache.resumptionToken())
                    yield h.fixUtf8(mustache.suffix())
                    
            except oaipmherrors.Error as e:
                if not rendered_init:
                    from lr.mustache.oaipmh import Error as err_stache
                    err = err_stache()
                    yield h.fixUtf8(self._returnResponse(err.xml(e), res=t_res))
                else:
                    from lr.mustache.oaipmh import ErrorOnly as err_stache
                    err = err_stache()
                    yield h.fixUtf8(self._returnResponse(err.xml(e)+mustache.suffix(), res=t_res))
            except:
                log.exception("Unknown Error Occurred")

        def ListIdentifiers(params):
            return ListGeneric(params, False, fc_id_limit)
        
        def ListRecords(params):
            return ListGeneric(params, True, fc_doc_limit)
#        def ListRecords(params):
#            try:
#                from lr.mustache.oaipmh import ListRecords as must_ListRec
#                
#                doc_index = 0
#                mustache = must_ListRec()
#                for record in o.list_records(params["metadataPrefix"],from_date=params["from"], until_date=params["until"] ):
#                    doc_index += 1
#                    log.debug(json.dumps(record))
#                    if doc_index == 1:
#                        part = mustache.prefix(**self._initMustache(args=params, req=t_req))
#                        yield self._returnResponse(part, res=t_res)
#                    
#                    part = mustache.doc(record)
#                    yield self._returnResponse(part, res=t_res)
#                    
#                
#                if doc_index == 0:
#                    raise NoRecordsMatchError(params['verb'], req=t_req)
#                else:
#                    yield mustache.suffix()
#                
#            except oaipmherrors.Error as e:
#                from lr.mustache.oaipmh import Error as err_stache
#                err = err_stache()
#                yield self._returnResponse(err.xml(e), res=t_res)
#            except:
#                log.exception("Unable to render template")
        
        def Identify(params=None):
            body = ""
            try:
                self._initRender(params, ctx=c, req=t_req)
                c.identify = o.identify()
                body = render("/oaipmh-Identify.mako")
            except Exception as e:
                raise BadVerbError()
            return self._returnResponse(body, res=t_res)
        
        def ListMetadataFormats(params):
            body = ""
            try:
                self._initRender(params, ctx=c, req=t_req)
                
                fmts = o.list_metadata_formats(identity=params["identifier"], by_doc_ID=params["by_doc_ID"])
                if len(fmts) == 0:
                    raise NoMetadataFormats(params["verb"])
                c.formats = fmts
                body = render("/oaipmh-ListMetadataFormats.mako")
                return self._returnResponse(body, res=t_res)
            except Error as e:
                raise e
        
        def ListSets(params=None):
            raise NoSetHierarchyError(verb)
            
        def NotYetSupported(params=None):
            raise BadVerbError()
        
        
            
        switch = {
                  'GetRecord': GetRecord,
                  'ListRecords': ListRecords,
                  'ListIdentifiers': ListIdentifiers,
                  'Identify': Identify,
                  'ListMetadataFormats': ListMetadataFormats,
                  'ListSets': ListSets
                  }
        try:
            params = self._parseParams(flow_control=enable_flow_control, serviceid=service_id)
            
            # If this is a special case where we are actually using OAI interface to serve basic harvest
            if params.has_key("metadataPrefix") and params["metadataPrefix"] == "LR_JSON_0.10.0":
                if params.has_key("identifier") == True:
                    params[self.REQUESTID] = params["identifier"]
                if params.has_key("from") and isinstance(params["from"], datetime):
                    params["from"] = h.convertToISO8601Zformat(params["from"])
                if params.has_key("until") and isinstance(params["until"], datetime):
                    params["until"] = h.convertToISO8601Zformat(params["until"])
                
                return HarvestController.harvest(self, params, request.body, params['verb'].lower())
        
            verb = params['verb']
            response.headers['Content-Type'] = "text/xml; charset=utf-8"
            
            return switch[verb](params)
        except Error as e:
            from lr.mustache.oaipmh import Error as err_stache
            err = err_stache()
            return self._returnResponse(err.xml(e), res=t_res)
Esempio n. 19
0
        def getrecord():
          data = self.get_base_response(verb,body)
          by_doc_ID = params.has_key('by_doc_ID') and ast.literal_eval(str(params['by_doc_ID']))
          by_resource_ID = params.has_key('by_resource_ID') and ast.literal_eval(str(params['by_resource_ID']))
          if not params.has_key('request_id'):
            data['OK'] = False
            data['error'] = 'badArgument'
            return json.dumps(data)
          if by_doc_ID and by_resource_ID:
            data['OK'] = False
            data['error'] = 'badArgument'
            return json.dumps(data)

          request_id = params['request_id']
          if by_doc_ID:
            records = map(lambda doc: {'record':{"header":{'identifier':doc.id, 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'}},'resource_data':doc},[h.get_record(request_id)])
          else:
            records = map(lambda doc: {'record':{"header":{'identifier':doc.id, 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'}},'resource_data':doc},h.get_records_by_resource(request_id))
          data['getrecord'] ={
            'record': records
            }
          return json.dumps(data)
Esempio n. 20
0
 def listGeneral(self, h, body, params, includeDocs, verb='GET'):
     data = self.get_base_response(verb, body)
     try:
         from_date, until_date = self._test_time_params(params)
     except:
         data['OK'] = False
         data['error'] = 'badArgument'
         yield json.dumps(data)
         return
     data['request']['from'] = from_date
     data['request']['until'] = until_date
     if from_date > until_date:
         data['OK'] = False
         data['error'] = 'badArgument'
         yield json.dumps(data)
     else:
         self._getServiceDocment(includeDocs)
         resumption_token = None
         count = 0
         lastID = None
         lastKey = None
         if self.enable_flow_control and params.has_key('resumption_token'):
             resumption_token = rt.parse_token(self.service_id,
                                               params['resumption_token'])
         if includeDocs:
             data['listrecords'] = []
             viewResults = h.list_records(from_date,
                                          until_date,
                                          resumption_token=resumption_token,
                                          limit=self.limit)
             debug_map = lambda doc: {
                 'record': {
                     "header": {
                         'identifier': doc['id'],
                         'datestamp': doc['key'] + "Z",
                         'status': 'active'
                     },
                     'resource_data': doc['doc']
                 }
             }
         else:
             data['listidentifiers'] = []
             viewResults = h.list_identifiers(
                 from_date,
                 until_date,
                 resumption_token=resumption_token,
                 limit=self.limit)
             debug_map = lambda doc: {
                 "header": {
                     'identifier': doc['id'],
                     'datestamp': doc['key'] + "Z",
                     'status': 'active'
                 }
             }
         base_response = json.dumps(data).split('[')
         yield base_response[0] + '['
         first = True
         for data in viewResults:
             lastID = data['id']
             lastKey = data['key']
             count += 1
             if not first:
                 yield ',\n'
             first = False
             yield json.dumps(debug_map(data))
         if self.enable_flow_control and self.limit <= count:
             token = rt.get_token(
                 serviceid=self.service_id,
                 startkey=lastKey,
                 endkey=helpers.convertToISO8601Zformat(until_date),
                 startkey_docid=lastID,
                 from_date=helpers.convertToISO8601Zformat(from_date),
                 until_date=helpers.convertToISO8601Zformat(until_date))
             resp = base_response[1]
             yield resp[:-1] + (',"resumption_token":"%s"' %
                                token) + resp[-1:]
         elif self.limit > count:
             resp = base_response[1]
             yield resp[:-1] + (',"resumption_token":"%s"' %
                                'null') + resp[-1:]
         else:
             yield base_response[1]