def setupClass(self): self.setup = True with open("lr/tests/data/nsdl_dc/data-000000000.json",'r') as f: data = json.load(f) if hasattr(self, "attr"): app = self.app else: controller = TestObtainController(methodName="test_empty") app = controller.app h = harvest() self.db = h.db self.server = h.server result = app.post('/publish', params=json.dumps(data), headers=headers) result = json.loads(result.body) self.ids = [] self.ids.extend(map(lambda doc: doc['doc_ID'],result['document_results'])) result = app.post('/publish', params=json.dumps(data), headers=headers) result = json.loads(result.body) self.ids.extend(map(lambda doc: doc['doc_ID'],result['document_results'])) self.resourceLocators = map(lambda doc: doc['resource_locator'],data['documents']) done = False distributableIds = map(lambda id: id+'-distributable',self.ids) while not done: view = self.db.view('_all_docs',keys=distributableIds) done = len(distributableIds) == len(view.rows) time.sleep(0.5) len(self.db.view('_design/learningregistry-resources/_view/docs')) len(self.db.view('_design/learningregistry-resource-location/_view/docs'))
def show(self, id, format='html'): """GET /harvest/id: Show a specific item""" h = harvest() def getrecord(): by_doc_ID = request.params.has_key( 'by_doc_ID') and request.params['by_doc_ID'] request_id = request.params['request_id'] if by_doc_ID: return json.dumps(h.get_record(request_id)) else: return json.dumps(h.get_records_by_resource(request_id)) def listidentifiers(): from_date = datetime.strptime(request.params['from'], time_format) until_date = datetime.strptime(request.params['until'], time_format) return self.list_identifiers(from_date, until_date, h) def listrecords(): from_date = datetime.strptime(request.params['from'], time_format) until_date = datetime.strptime(request.params['until'], time_format) return self.list_records(from_date, until_date, h) def identify(): data = self.get_base_response("GET") data['identify'] = { 'node_id': 'string', 'repositoryName': 'string', 'baseURL': 'string', 'protocolVersion': '2.0', 'service_version': 'string', 'earliestDatestamp': 'string', 'deletedRecord': 'string', 'granularity': 'string', 'adminEmail': 'string' } return json.dumps(data) def listmetadataformats(): data = self.get_base_response("GET") data['metadataFormat'] = h.list_metadata_formats() return json.dumps(data) def listsets(): data = self.get_base_response("GET") data['OK'] = False data['error'] = 'noSetHierarchy' return json.dumps(data) switch = { 'getrecord': getrecord, 'listrecords': listrecords, 'listidentifiers': listidentifiers, 'identify': identify, 'listmetadataformats': listmetadataformats, 'listsets': listsets } return switch[id]()
def setupClass(self): self.setup = True with open("lr/tests/data/nsdl_dc/data-000000000.json", "r") as f: data = json.load(f) if hasattr(self, "attr"): app = self.app else: controller = TestHarvestController(methodName="test_empty") app = controller.app h = harvest() self.db = h.db self.server = h.server global db db = self.db result = app.post("/publish", params=json.dumps(data), headers=headers) result = json.loads(result.body) self.ids = [] self.ids.extend(map(lambda doc: doc["doc_ID"], result["document_results"])) result = app.post("/publish", params=json.dumps(data), headers=headers) result = json.loads(result.body) self.ids.extend(map(lambda doc: doc["doc_ID"], result["document_results"])) self.resourceLocators = map(lambda doc: doc["resource_locator"], data["documents"]) done = False distributableIds = map(lambda id: id + "-distributable", self.ids) while not done: view = self.db.view("_all_docs", keys=distributableIds) done = len(distributableIds) == len(view.rows) time.sleep(0.5)
def setupClass(self): self.setup = True with open("lr/tests/data/nsdl_dc/data-000000000.json", 'r') as f: data = json.load(f) if hasattr(self, "attr"): app = self.app else: controller = TestObtainController(methodName="test_empty") app = controller.app h = harvest() self.db = h.db self.server = h.server result = app.post('/publish', params=json.dumps(data), headers=headers) result = json.loads(result.body) self.ids = [] self.ids.extend( map(lambda doc: doc['doc_ID'], result['document_results'])) result = app.post('/publish', params=json.dumps(data), headers=headers) result = json.loads(result.body) self.ids.extend( map(lambda doc: doc['doc_ID'], result['document_results'])) self.resourceLocators = map(lambda doc: doc['resource_locator'], data['documents']) done = False distributableIds = map(lambda id: id + '-distributable', self.ids) while not done: view = self.db.view('_all_docs', keys=distributableIds) done = len(distributableIds) == len(view.rows) time.sleep(0.5)
def setupClass(self): self.setup = True with open("lr/tests/data/nsdl_dc/data-000000000.json",'r') as f: data = json.load(f) if hasattr(self, "attr"): app = self.app else: controller = TestExtractController(methodName="test_empty") app = controller.app h = harvest() self.db = h.db self.server = h.server global db db = self.db result = app.post('/publish', params=json.dumps(data), headers=headers) result = json.loads(result.body) self.ids = [] self.ids.extend(map(lambda doc: doc['doc_ID'],result['document_results'])) result = app.post('/publish', params=json.dumps(data), headers=headers) result = json.loads(result.body) self.ids.extend(map(lambda doc: doc['doc_ID'],result['document_results'])) self.resourceLocators = map(lambda doc: doc['resource_locator'],data['documents']) done = False distributableIds = map(lambda id: id+'-distributable',self.ids) while not done: view = self.db.view('_all_docs',keys=distributableIds) done = len(distributableIds) == len(view.rows) time.sleep(0.5) #install data_service view couchdb_url = config['couchdb.url.dbadmin'] resource_data_db = config['couchdb.db.resourcedata'] _pushCouchApp("../data_services/standards-alignment-dct-conformsTo", "{0}/{1}".format(couchdb_url,resource_data_db))
def harvest(self, params, body, verb): h = harvest() def getrecord(): data = self.get_base_response(verb,body) by_doc_ID = params.has_key('by_doc_ID') and ast.literal_eval(str(params['by_doc_ID'])) by_resource_ID = params.has_key('by_resource_ID') and ast.literal_eval(str(params['by_resource_ID'])) if not params.has_key('request_id'): data['OK'] = False data['error'] = 'badArgument' return json.dumps(data) if by_doc_ID and by_resource_ID: data['OK'] = False data['error'] = 'badArgument' return json.dumps(data) request_id = params['request_id'] if by_doc_ID: records = map(lambda doc: {'record':{"header":{'identifier':doc.id, 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'}},'resource_data':doc},[h.get_record(request_id)]) else: records = map(lambda doc: {'record':{"header":{'identifier':doc.id, 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'}},'resource_data':doc},h.get_records_by_resource(request_id)) data['getrecord'] ={ 'record': records } return json.dumps(data) def listidentifiers(): return self.list_identifiers(h,body,params,verb) def listrecords(): return self.list_records(h,body,params,verb) def identify(): data = self.get_base_response(verb,body) data['identify']={ 'node_id': 'string', 'repositoryName': 'string', 'baseURL': 'string', 'protocolVersion': '2.0', 'service_version': 'string', 'earliestDatestamp': 'string', 'deletedRecord': 'string', 'granularity': 'string', 'adminEmail': 'string' } return json.dumps(data) def listmetadataformats(): data = self.get_base_response(verb,body) data['listmetadataformats']=h.list_metadata_formats() return json.dumps(data) def listsets(): data = self.get_base_response(verb,body) data['OK']=False data['error']='noSetHierarchy' return json.dumps(data) switch = { 'getrecord':getrecord, 'listrecords':listrecords, 'listidentifiers':listidentifiers, 'identify': identify, 'listmetadataformats': listmetadataformats, 'listsets': listsets } return switch[verb]()
def show(self, id, format="html"): """GET /harvest/id: Show a specific item""" h = harvest() def getrecord(): by_doc_ID = request.params.has_key("by_doc_ID") and request.params["by_doc_ID"] request_id = request.params["request_id"] if by_doc_ID: return json.dumps(h.get_record(request_id)) else: return json.dumps(h.get_records_by_resource(request_id)) def listidentifiers(): from_date = datetime.strptime(request.params["from"], time_format) until_date = datetime.strptime(request.params["until"], time_format) return self.list_identifiers(from_date, until_date, h) def listrecords(): from_date = datetime.strptime(request.params["from"], time_format) until_date = datetime.strptime(request.params["until"], time_format) return self.list_records(from_date, until_date, h) def identify(): data = self.get_base_response("GET") data["identify"] = { "node_id": "string", "repositoryName": "string", "baseURL": "string", "protocolVersion": "2.0", "service_version": "string", "earliestDatestamp": "string", "deletedRecord": "string", "granularity": "string", "adminEmail": "string", } return json.dumps(data) def listmetadataformats(): data = self.get_base_response("GET") data["metadataFormat"] = h.list_metadata_formats() return json.dumps(data) def listsets(): data = self.get_base_response("GET") data["OK"] = False data["error"] = "noSetHierarchy" return json.dumps(data) switch = { "getrecord": getrecord, "listrecords": listrecords, "listidentifiers": listidentifiers, "identify": identify, "listmetadataformats": listmetadataformats, "listsets": listsets, } return switch[id]()
def index(self, format='html'): """GET /status: All items in the collection""" #if sourceLRNode.isServiceAvailable(NodeServiceModel.ADMINISTRATIVE) == False: # return "Administrative service not is available" har = harvest() s = sourceLRNode.status earliestDate = har.earliestDate() #print earliestDate s['earliestDatestamp'] =earliestDate #print "here" return json.dumps(s )
def index(self, format='html'): """GET /status: All items in the collection""" #if sourceLRNode.isServiceAvailable(NodeServiceModel.ADMINISTRATIVE) == False: # return "Administrative service not is available" har = harvest() s = sourceLRNode.status earliestDate = har.earliestDate() #print earliestDate s['earliestDatestamp'] = earliestDate #print "here" return json.dumps(s)
def harvest(self, params, body, verb): h = harvest() def getrecord(): data = self.get_base_response(verb,body) by_doc_ID = self._check_bool_param(params,'by_doc_ID') by_resource_ID = self._check_bool_param(params,'by_resource_ID') if not params.has_key(self.REQUESTID): data['OK'] = False data['error'] = 'badArgument' return json.dumps(data) if by_doc_ID and by_resource_ID: data['OK'] = False data['error'] = 'badArgument' return json.dumps(data) request_id = params[self.REQUESTID] if by_doc_ID: document = h.get_record(request_id) if document is not None: records = map(lambda doc: {"header":{'identifier':doc['_id'], 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'},'resource_data':doc},[document]) else: records = [] else: records = map(lambda doc: {"header":{'identifier':doc['_id'], 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'},'resource_data':doc},h.get_records_by_resource(request_id)) if len(records) == 0: abort(500,'idDoesNotExist') data['getrecord'] ={ 'record': records } data['request']['identifier'] = request_id data['request']['by_doc_ID'] = by_doc_ID data['request']['by_resource_ID'] = by_resource_ID return json.dumps(data) def listidentifiers(): return self.list_identifiers(h,body,params,verb) def listrecords(): return self.list_records(h,body,params,verb) def identify(): data = self.get_base_response(verb,body) serviceDoc = helpers.getServiceDocument(BASIC_HARVEST_SERVICE_DOC) data['identify']={ "node_id":sourceLRNode.nodeDescription.node_name, "repositoryName":sourceLRNode.communityDescription.community_name, "baseURL":serviceDoc['service_endpoint'], "protocolVersion":"2.0", "service_version":serviceDoc['service_version'], "earliestDatestamp":h.earliestDate(), "deletedRecord":sourceLRNode.nodeDescription.node_policy['deleted_data_policy'], "granularity":helpers.getDatetimePrecision(serviceDoc), "adminEmail":sourceLRNode.nodeDescription.node_admin_identity } return json.dumps(data) def listmetadataformats(): self._getServiceDocment(False) data = self.get_base_response(verb,body) data['listmetadataformats']=map(lambda format: {'metadataformat':{'metadataPrefix':format['metadataPrefix']}},self.metadataFormats) return json.dumps(data) def listsets(): data = self.get_base_response(verb,body) data['OK']=False data['error']='noSetHierarchy' return json.dumps(data) switch = { 'getrecord':getrecord, 'listrecords':listrecords, 'listidentifiers':listidentifiers, 'identify': identify, 'listmetadataformats': listmetadataformats, 'listsets': listsets } if switch.has_key(verb): return switch[verb]() else: abort(500,"Invalid Verb")
def __init__(self): self.h = harvest()
def harvest(self, params, body, verb): h = harvest() def getrecord(): data = self.get_base_response(verb, body) by_doc_ID = self._check_bool_param(params, 'by_doc_ID') by_resource_ID = self._check_bool_param(params, 'by_resource_ID') if not params.has_key(self.REQUESTID) and not params.has_key( self.REQUESTID.lower()): data['OK'] = False data['error'] = 'badArgument' return json.dumps(data) if by_doc_ID and by_resource_ID: data['OK'] = False data['error'] = 'badArgument' return json.dumps(data) if params.has_key(self.REQUESTID): request_id = params[self.REQUESTID] elif params.has_key(self.REQUESTID.lower()): request_id = params[self.REQUESTID.lower()] if by_doc_ID: document = h.get_record(request_id) if document is not None: records = map( lambda doc: { "header": { 'identifier': doc['_id'], 'datestamp': helpers.convertToISO8601Zformat(datetime.today( )), 'status': 'active' }, 'resource_data': doc }, [document]) else: records = [] else: request_id = unquote_plus(request_id) records = map( lambda doc: { "header": { 'identifier': doc['_id'], 'datestamp': helpers.convertToISO8601Zformat(datetime.today()), 'status': 'active' }, 'resource_data': doc }, h.get_records_by_resource(request_id)) if len(records) == 0: data['OK'] = False data['error'] = 'idDoesNotExist' return json.dumps(data) data['getrecord'] = {'record': records} data['request']['identifier'] = request_id data['request']['by_doc_ID'] = by_doc_ID data['request']['by_resource_ID'] = by_resource_ID return json.dumps(data) def listidentifiers(): return self.listGeneral(h, body, params, False, verb) def listrecords(): return self.listGeneral(h, body, params, True, verb) def identify(): data = self.get_base_response(verb, body) serviceDoc = helpers.getServiceDocument(BASIC_HARVEST_SERVICE_DOC) data['identify'] = { "node_id": sourceLRNode.nodeDescription.node_name, "repositoryName": sourceLRNode.communityDescription.community_name, "baseURL": serviceDoc['service_endpoint'], "protocolVersion": "2.0", "service_version": serviceDoc['service_version'], "earliestDatestamp": h.earliestDate(), "deletedRecord": sourceLRNode.nodeDescription. node_policy['deleted_data_policy'], "granularity": helpers.getDatetimePrecision(serviceDoc), "adminEmail": sourceLRNode.nodeDescription.node_admin_identity } return json.dumps(data) def listmetadataformats(): self._getServiceDocment(False) data = self.get_base_response(verb, body) data['listmetadataformats'] = map( lambda format: { 'metadataformat': { "metadataPrefix": format['metadataFormat'] } }, self.metadataFormats) return json.dumps(data) def listsets(): data = self.get_base_response(verb, body) data['OK'] = False data['error'] = 'noSetHierarchy' return json.dumps(data) switch = { 'getrecord': getrecord, 'listrecords': listrecords, 'listidentifiers': listidentifiers, 'identify': identify, 'listmetadataformats': listmetadataformats, 'listsets': listsets } if switch.has_key(verb): return switch[verb]() else: abort(500, "Invalid Verb")
def harvest(self, params, body, verb): h = harvest() def getrecord(): data = self.get_base_response(verb, body) by_doc_ID = params.has_key('by_doc_ID') and ast.literal_eval( str(params['by_doc_ID'])) by_resource_ID = params.has_key( 'by_resource_ID') and ast.literal_eval( str(params['by_resource_ID'])) if not params.has_key('request_id'): data['OK'] = False data['error'] = 'badArgument' return json.dumps(data) if by_doc_ID and by_resource_ID: data['OK'] = False data['error'] = 'badArgument' return json.dumps(data) request_id = params['request_id'] if by_doc_ID: records = map( lambda doc: { 'record': { "header": { 'identifier': doc.id, 'datestamp': helpers.convertToISO8601Zformat(datetime.today( )), 'status': 'active' } }, 'resource_data': doc }, [h.get_record(request_id)]) else: records = map( lambda doc: { 'record': { "header": { 'identifier': doc.id, 'datestamp': helpers.convertToISO8601Zformat(datetime.today( )), 'status': 'active' } }, 'resource_data': doc }, h.get_records_by_resource(request_id)) data['getrecord'] = {'record': records} return json.dumps(data) def listidentifiers(): return self.list_identifiers(h, body, params, verb) def listrecords(): return self.list_records(h, body, params, verb) def identify(): data = self.get_base_response(verb, body) data['identify'] = { 'node_id': 'string', 'repositoryName': 'string', 'baseURL': 'string', 'protocolVersion': '2.0', 'service_version': 'string', 'earliestDatestamp': 'string', 'deletedRecord': 'string', 'granularity': 'string', 'adminEmail': 'string' } return json.dumps(data) def listmetadataformats(): data = self.get_base_response(verb, body) data['listmetadataformats'] = h.list_metadata_formats() return json.dumps(data) def listsets(): data = self.get_base_response(verb, body) data['OK'] = False data['error'] = 'noSetHierarchy' return json.dumps(data) switch = { 'getrecord': getrecord, 'listrecords': listrecords, 'listidentifiers': listidentifiers, 'identify': identify, 'listmetadataformats': listmetadataformats, 'listsets': listsets } return switch[verb]()