def setupClass(self):
self.setup = True
with open("lr/tests/data/nsdl_dc/data-000000000.json",'r') as f:
data = json.load(f)
if hasattr(self, "attr"):
app = self.app
else:
controller = TestObtainController(methodName="test_empty")
app = controller.app
h = harvest()
self.db = h.db
self.server = h.server
result = app.post('/publish', params=json.dumps(data), headers=headers)
result = json.loads(result.body)
self.ids = []
self.ids.extend(map(lambda doc: doc['doc_ID'],result['document_results']))
result = app.post('/publish', params=json.dumps(data), headers=headers)
result = json.loads(result.body)
self.ids.extend(map(lambda doc: doc['doc_ID'],result['document_results']))
self.resourceLocators = map(lambda doc: doc['resource_locator'],data['documents'])
done = False
distributableIds = map(lambda id: id+'-distributable',self.ids)
while not done:
view = self.db.view('_all_docs',keys=distributableIds)
done = len(distributableIds) == len(view.rows)
time.sleep(0.5)
len(self.db.view('_design/learningregistry-resources/_view/docs'))
len(self.db.view('_design/learningregistry-resource-location/_view/docs'))
def show(self, id, format='html'):
"""GET /harvest/id: Show a specific item"""
h = harvest()
def getrecord():
by_doc_ID = request.params.has_key(
'by_doc_ID') and request.params['by_doc_ID']
request_id = request.params['request_id']
if by_doc_ID:
return json.dumps(h.get_record(request_id))
else:
return json.dumps(h.get_records_by_resource(request_id))
def listidentifiers():
from_date = datetime.strptime(request.params['from'], time_format)
until_date = datetime.strptime(request.params['until'],
time_format)
return self.list_identifiers(from_date, until_date, h)
def listrecords():
from_date = datetime.strptime(request.params['from'], time_format)
until_date = datetime.strptime(request.params['until'],
time_format)
return self.list_records(from_date, until_date, h)
def identify():
data = self.get_base_response("GET")
data['identify'] = {
'node_id': 'string',
'repositoryName': 'string',
'baseURL': 'string',
'protocolVersion': '2.0',
'service_version': 'string',
'earliestDatestamp': 'string',
'deletedRecord': 'string',
'granularity': 'string',
'adminEmail': 'string'
}
return json.dumps(data)
def listmetadataformats():
data = self.get_base_response("GET")
data['metadataFormat'] = h.list_metadata_formats()
return json.dumps(data)
def listsets():
data = self.get_base_response("GET")
data['OK'] = False
data['error'] = 'noSetHierarchy'
return json.dumps(data)
switch = {
'getrecord': getrecord,
'listrecords': listrecords,
'listidentifiers': listidentifiers,
'identify': identify,
'listmetadataformats': listmetadataformats,
'listsets': listsets
}
return switch[id]()
def setupClass(self):
self.setup = True
with open("lr/tests/data/nsdl_dc/data-000000000.json", "r") as f:
data = json.load(f)
if hasattr(self, "attr"):
app = self.app
else:
controller = TestHarvestController(methodName="test_empty")
app = controller.app
h = harvest()
self.db = h.db
self.server = h.server
global db
db = self.db
result = app.post("/publish", params=json.dumps(data), headers=headers)
result = json.loads(result.body)
self.ids = []
self.ids.extend(map(lambda doc: doc["doc_ID"], result["document_results"]))
result = app.post("/publish", params=json.dumps(data), headers=headers)
result = json.loads(result.body)
self.ids.extend(map(lambda doc: doc["doc_ID"], result["document_results"]))
self.resourceLocators = map(lambda doc: doc["resource_locator"], data["documents"])
done = False
distributableIds = map(lambda id: id + "-distributable", self.ids)
while not done:
view = self.db.view("_all_docs", keys=distributableIds)
done = len(distributableIds) == len(view.rows)
time.sleep(0.5)
def setupClass(self):
self.setup = True
with open("lr/tests/data/nsdl_dc/data-000000000.json", 'r') as f:
data = json.load(f)
if hasattr(self, "attr"):
app = self.app
else:
controller = TestObtainController(methodName="test_empty")
app = controller.app
h = harvest()
self.db = h.db
self.server = h.server
result = app.post('/publish', params=json.dumps(data), headers=headers)
result = json.loads(result.body)
self.ids = []
self.ids.extend(
map(lambda doc: doc['doc_ID'], result['document_results']))
result = app.post('/publish', params=json.dumps(data), headers=headers)
result = json.loads(result.body)
self.ids.extend(
map(lambda doc: doc['doc_ID'], result['document_results']))
self.resourceLocators = map(lambda doc: doc['resource_locator'],
data['documents'])
done = False
distributableIds = map(lambda id: id + '-distributable', self.ids)
while not done:
view = self.db.view('_all_docs', keys=distributableIds)
done = len(distributableIds) == len(view.rows)
time.sleep(0.5)
def setupClass(self):
self.setup = True
with open("lr/tests/data/nsdl_dc/data-000000000.json",'r') as f:
data = json.load(f)
if hasattr(self, "attr"):
app = self.app
else:
controller = TestExtractController(methodName="test_empty")
app = controller.app
h = harvest()
self.db = h.db
self.server = h.server
global db
db = self.db
result = app.post('/publish', params=json.dumps(data), headers=headers)
result = json.loads(result.body)
self.ids = []
self.ids.extend(map(lambda doc: doc['doc_ID'],result['document_results']))
result = app.post('/publish', params=json.dumps(data), headers=headers)
result = json.loads(result.body)
self.ids.extend(map(lambda doc: doc['doc_ID'],result['document_results']))
self.resourceLocators = map(lambda doc: doc['resource_locator'],data['documents'])
done = False
distributableIds = map(lambda id: id+'-distributable',self.ids)
while not done:
view = self.db.view('_all_docs',keys=distributableIds)
done = len(distributableIds) == len(view.rows)
time.sleep(0.5)
#install data_service view
couchdb_url = config['couchdb.url.dbadmin']
resource_data_db = config['couchdb.db.resourcedata']
_pushCouchApp("../data_services/standards-alignment-dct-conformsTo", "{0}/{1}".format(couchdb_url,resource_data_db))
def harvest(self, params, body, verb):
h = harvest()
def getrecord():
data = self.get_base_response(verb,body)
by_doc_ID = params.has_key('by_doc_ID') and ast.literal_eval(str(params['by_doc_ID']))
by_resource_ID = params.has_key('by_resource_ID') and ast.literal_eval(str(params['by_resource_ID']))
if not params.has_key('request_id'):
data['OK'] = False
data['error'] = 'badArgument'
return json.dumps(data)
if by_doc_ID and by_resource_ID:
data['OK'] = False
data['error'] = 'badArgument'
return json.dumps(data)
request_id = params['request_id']
if by_doc_ID:
records = map(lambda doc: {'record':{"header":{'identifier':doc.id, 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'}},'resource_data':doc},[h.get_record(request_id)])
else:
records = map(lambda doc: {'record':{"header":{'identifier':doc.id, 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'}},'resource_data':doc},h.get_records_by_resource(request_id))
data['getrecord'] ={
'record': records
}
return json.dumps(data)
def listidentifiers():
return self.list_identifiers(h,body,params,verb)
def listrecords():
return self.list_records(h,body,params,verb)
def identify():
data = self.get_base_response(verb,body)
data['identify']={
'node_id': 'string',
'repositoryName': 'string',
'baseURL': 'string',
'protocolVersion': '2.0',
'service_version': 'string',
'earliestDatestamp': 'string',
'deletedRecord': 'string',
'granularity': 'string',
'adminEmail': 'string'
}
return json.dumps(data)
def listmetadataformats():
data = self.get_base_response(verb,body)
data['listmetadataformats']=h.list_metadata_formats()
return json.dumps(data)
def listsets():
data = self.get_base_response(verb,body)
data['OK']=False
data['error']='noSetHierarchy'
return json.dumps(data)
switch = {
'getrecord':getrecord,
'listrecords':listrecords,
'listidentifiers':listidentifiers,
'identify': identify,
'listmetadataformats': listmetadataformats,
'listsets': listsets
}
return switch[verb]()
def show(self, id, format="html"):
"""GET /harvest/id: Show a specific item"""
h = harvest()
def getrecord():
by_doc_ID = request.params.has_key("by_doc_ID") and request.params["by_doc_ID"]
request_id = request.params["request_id"]
if by_doc_ID:
return json.dumps(h.get_record(request_id))
else:
return json.dumps(h.get_records_by_resource(request_id))
def listidentifiers():
from_date = datetime.strptime(request.params["from"], time_format)
until_date = datetime.strptime(request.params["until"], time_format)
return self.list_identifiers(from_date, until_date, h)
def listrecords():
from_date = datetime.strptime(request.params["from"], time_format)
until_date = datetime.strptime(request.params["until"], time_format)
return self.list_records(from_date, until_date, h)
def identify():
data = self.get_base_response("GET")
data["identify"] = {
"node_id": "string",
"repositoryName": "string",
"baseURL": "string",
"protocolVersion": "2.0",
"service_version": "string",
"earliestDatestamp": "string",
"deletedRecord": "string",
"granularity": "string",
"adminEmail": "string",
}
return json.dumps(data)
def listmetadataformats():
data = self.get_base_response("GET")
data["metadataFormat"] = h.list_metadata_formats()
return json.dumps(data)
def listsets():
data = self.get_base_response("GET")
data["OK"] = False
data["error"] = "noSetHierarchy"
return json.dumps(data)
switch = {
"getrecord": getrecord,
"listrecords": listrecords,
"listidentifiers": listidentifiers,
"identify": identify,
"listmetadataformats": listmetadataformats,
"listsets": listsets,
}
return switch[id]()
def index(self, format='html'):
"""GET /status: All items in the collection"""
#if sourceLRNode.isServiceAvailable(NodeServiceModel.ADMINISTRATIVE) == False:
# return "Administrative service not is available"
har = harvest()
s = sourceLRNode.status
earliestDate = har.earliestDate()
#print earliestDate
s['earliestDatestamp'] =earliestDate
#print "here"
return json.dumps(s )
def index(self, format='html'):
"""GET /status: All items in the collection"""
#if sourceLRNode.isServiceAvailable(NodeServiceModel.ADMINISTRATIVE) == False:
# return "Administrative service not is available"
har = harvest()
s = sourceLRNode.status
earliestDate = har.earliestDate()
#print earliestDate
s['earliestDatestamp'] = earliestDate
#print "here"
return json.dumps(s)
def harvest(self, params, body, verb):
h = harvest()
def getrecord():
data = self.get_base_response(verb,body)
by_doc_ID = self._check_bool_param(params,'by_doc_ID')
by_resource_ID = self._check_bool_param(params,'by_resource_ID')
if not params.has_key(self.REQUESTID):
data['OK'] = False
data['error'] = 'badArgument'
return json.dumps(data)
if by_doc_ID and by_resource_ID:
data['OK'] = False
data['error'] = 'badArgument'
return json.dumps(data)
request_id = params[self.REQUESTID]
if by_doc_ID:
document = h.get_record(request_id)
if document is not None:
records = map(lambda doc: {"header":{'identifier':doc['_id'], 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'},'resource_data':doc},[document])
else:
records = []
else:
records = map(lambda doc: {"header":{'identifier':doc['_id'], 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'},'resource_data':doc},h.get_records_by_resource(request_id))
if len(records) == 0:
abort(500,'idDoesNotExist')
data['getrecord'] ={
'record': records
}
data['request']['identifier'] = request_id
data['request']['by_doc_ID'] = by_doc_ID
data['request']['by_resource_ID'] = by_resource_ID
return json.dumps(data)
def listidentifiers():
return self.list_identifiers(h,body,params,verb)
def listrecords():
return self.list_records(h,body,params,verb)
def identify():
data = self.get_base_response(verb,body)
serviceDoc = helpers.getServiceDocument(BASIC_HARVEST_SERVICE_DOC)
data['identify']={
"node_id":sourceLRNode.nodeDescription.node_name,
"repositoryName":sourceLRNode.communityDescription.community_name,
"baseURL":serviceDoc['service_endpoint'],
"protocolVersion":"2.0",
"service_version":serviceDoc['service_version'],
"earliestDatestamp":h.earliestDate(),
"deletedRecord":sourceLRNode.nodeDescription.node_policy['deleted_data_policy'],
"granularity":helpers.getDatetimePrecision(serviceDoc),
"adminEmail":sourceLRNode.nodeDescription.node_admin_identity
}
return json.dumps(data)
def listmetadataformats():
self._getServiceDocment(False)
data = self.get_base_response(verb,body)
data['listmetadataformats']=map(lambda format: {'metadataformat':{'metadataPrefix':format['metadataPrefix']}},self.metadataFormats)
return json.dumps(data)
def listsets():
data = self.get_base_response(verb,body)
data['OK']=False
data['error']='noSetHierarchy'
return json.dumps(data)
switch = {
'getrecord':getrecord,
'listrecords':listrecords,
'listidentifiers':listidentifiers,
'identify': identify,
'listmetadataformats': listmetadataformats,
'listsets': listsets
}
if switch.has_key(verb):
return switch[verb]()
else:
abort(500,"Invalid Verb")
def __init__(self):
self.h = harvest()
def harvest(self, params, body, verb):
h = harvest()
def getrecord():
data = self.get_base_response(verb, body)
by_doc_ID = self._check_bool_param(params, 'by_doc_ID')
by_resource_ID = self._check_bool_param(params, 'by_resource_ID')
if not params.has_key(self.REQUESTID) and not params.has_key(
self.REQUESTID.lower()):
data['OK'] = False
data['error'] = 'badArgument'
return json.dumps(data)
if by_doc_ID and by_resource_ID:
data['OK'] = False
data['error'] = 'badArgument'
return json.dumps(data)
if params.has_key(self.REQUESTID):
request_id = params[self.REQUESTID]
elif params.has_key(self.REQUESTID.lower()):
request_id = params[self.REQUESTID.lower()]
if by_doc_ID:
document = h.get_record(request_id)
if document is not None:
records = map(
lambda doc: {
"header": {
'identifier':
doc['_id'],
'datestamp':
helpers.convertToISO8601Zformat(datetime.today(
)),
'status':
'active'
},
'resource_data': doc
}, [document])
else:
records = []
else:
request_id = unquote_plus(request_id)
records = map(
lambda doc: {
"header": {
'identifier':
doc['_id'],
'datestamp':
helpers.convertToISO8601Zformat(datetime.today()),
'status':
'active'
},
'resource_data': doc
}, h.get_records_by_resource(request_id))
if len(records) == 0:
data['OK'] = False
data['error'] = 'idDoesNotExist'
return json.dumps(data)
data['getrecord'] = {'record': records}
data['request']['identifier'] = request_id
data['request']['by_doc_ID'] = by_doc_ID
data['request']['by_resource_ID'] = by_resource_ID
return json.dumps(data)
def listidentifiers():
return self.listGeneral(h, body, params, False, verb)
def listrecords():
return self.listGeneral(h, body, params, True, verb)
def identify():
data = self.get_base_response(verb, body)
serviceDoc = helpers.getServiceDocument(BASIC_HARVEST_SERVICE_DOC)
data['identify'] = {
"node_id":
sourceLRNode.nodeDescription.node_name,
"repositoryName":
sourceLRNode.communityDescription.community_name,
"baseURL":
serviceDoc['service_endpoint'],
"protocolVersion":
"2.0",
"service_version":
serviceDoc['service_version'],
"earliestDatestamp":
h.earliestDate(),
"deletedRecord":
sourceLRNode.nodeDescription.
node_policy['deleted_data_policy'],
"granularity":
helpers.getDatetimePrecision(serviceDoc),
"adminEmail":
sourceLRNode.nodeDescription.node_admin_identity
}
return json.dumps(data)
def listmetadataformats():
self._getServiceDocment(False)
data = self.get_base_response(verb, body)
data['listmetadataformats'] = map(
lambda format: {
'metadataformat': {
"metadataPrefix": format['metadataFormat']
}
}, self.metadataFormats)
return json.dumps(data)
def listsets():
data = self.get_base_response(verb, body)
data['OK'] = False
data['error'] = 'noSetHierarchy'
return json.dumps(data)
switch = {
'getrecord': getrecord,
'listrecords': listrecords,
'listidentifiers': listidentifiers,
'identify': identify,
'listmetadataformats': listmetadataformats,
'listsets': listsets
}
if switch.has_key(verb):
return switch[verb]()
else:
abort(500, "Invalid Verb")
def harvest(self, params, body, verb):
h = harvest()
def getrecord():
data = self.get_base_response(verb, body)
by_doc_ID = params.has_key('by_doc_ID') and ast.literal_eval(
str(params['by_doc_ID']))
by_resource_ID = params.has_key(
'by_resource_ID') and ast.literal_eval(
str(params['by_resource_ID']))
if not params.has_key('request_id'):
data['OK'] = False
data['error'] = 'badArgument'
return json.dumps(data)
if by_doc_ID and by_resource_ID:
data['OK'] = False
data['error'] = 'badArgument'
return json.dumps(data)
request_id = params['request_id']
if by_doc_ID:
records = map(
lambda doc: {
'record': {
"header": {
'identifier':
doc.id,
'datestamp':
helpers.convertToISO8601Zformat(datetime.today(
)),
'status':
'active'
}
},
'resource_data': doc
}, [h.get_record(request_id)])
else:
records = map(
lambda doc: {
'record': {
"header": {
'identifier':
doc.id,
'datestamp':
helpers.convertToISO8601Zformat(datetime.today(
)),
'status':
'active'
}
},
'resource_data': doc
}, h.get_records_by_resource(request_id))
data['getrecord'] = {'record': records}
return json.dumps(data)
def listidentifiers():
return self.list_identifiers(h, body, params, verb)
def listrecords():
return self.list_records(h, body, params, verb)
def identify():
data = self.get_base_response(verb, body)
data['identify'] = {
'node_id': 'string',
'repositoryName': 'string',
'baseURL': 'string',
'protocolVersion': '2.0',
'service_version': 'string',
'earliestDatestamp': 'string',
'deletedRecord': 'string',
'granularity': 'string',
'adminEmail': 'string'
}
return json.dumps(data)
def listmetadataformats():
data = self.get_base_response(verb, body)
data['listmetadataformats'] = h.list_metadata_formats()
return json.dumps(data)
def listsets():
data = self.get_base_response(verb, body)
data['OK'] = False
data['error'] = 'noSetHierarchy'
return json.dumps(data)
switch = {
'getrecord': getrecord,
'listrecords': listrecords,
'listidentifiers': listidentifiers,
'identify': identify,
'listmetadataformats': listmetadataformats,
'listsets': listsets
}
return switch[verb]()
