def explore_all_versions_results(request):
template_id = request.GET['id']
template = Template.objects().get(pk=template_id)
version_id = template.templateVersion
template_version = TemplateVersion.objects().get(pk=version_id)
if len(template_version.versions) == 1:
query = {"schema": template_id}
else:
list_query = []
for version in template_version.versions:
list_query.append({'schema': version})
query = {"$or": list_query}
request.session['queryExplore'] = query
if 'HTTPS' in request.META['SERVER_PROTOCOL']:
protocol = "https"
else:
protocol = "http"
json_instances = [Instance(name="Local", protocol=protocol, address=request.META['REMOTE_ADDR'], port=request.META['SERVER_PORT'], access_token="token", refresh_token="token").to_json()]
request.session['instancesExplore'] = json_instances
template = loader.get_template('explore/explore_results.html')
context = RequestContext(request, {
'': '',
})
if 'exploreCurrentTemplateID' not in request.session:
return redirect('/explore/select-template')
else:
return HttpResponse(template.render(context))
def get_token(self, username, password, client_id, client_secret,
application):
try:
url = URL_TEST + "/o/token/"
headers = {'content-type': 'application/x-www-form-urlencoded'}
data = {
'grant_type': 'password',
'username': username,
'password': password,
'client_id': client_id,
'client_secret': client_secret
}
r = requests.post(url=url,
data=data,
headers=headers,
timeout=int(310000))
if r.status_code == 200:
now = datetime.now()
delta = timedelta(seconds=int(eval(r.content)["expires_in"]))
expires = now + delta
token = Instance(name=application,
protocol='http',
address='127.0.0.1',
port='8082',
access_token=eval(r.content)["access_token"],
refresh_token=eval(
r.content)["refresh_token"],
expires=expires).save()
return token
else:
return ''
except Exception, e:
return ''
def explore_perform_search(request):
try:
template = loader.get_template('explore/explore_perform_search.html')
instances = Instance.objects()
if 'HTTPS' in request.META['SERVER_PROTOCOL']:
protocol = "https"
else:
protocol = "http"
local = Instance(name="Local", protocol=protocol, address=request.META['REMOTE_ADDR'], port=request.META['SERVER_PORT'])
listInstances = [local]
for instance in instances:
listInstances.append(instance)
template_hash = Template.objects.get(pk=request.session['exploreCurrentTemplateID']).hash
queries = SavedQuery.objects(user=str(request.user.id), template=str(request.session['exploreCurrentTemplateID']))
context = RequestContext(request, {
'instances': listInstances,
'template_hash': template_hash,
'queries':queries,
'template_id': request.session['exploreCurrentTemplateID']
})
if 'exploreCurrentTemplateID' not in request.session:
return redirect('/explore/select-template')
else:
return HttpResponse(template.render(context))
except:
return redirect("/explore")
def explore_all_results(request):
template_id = request.GET['id']
if 'HTTPS' in request.META['SERVER_PROTOCOL']:
protocol = "https"
else:
protocol = "http"
request.session['queryExplore'] = {"schema": template_id}
json_instances = [
Instance(name="Local",
protocol=protocol,
address=request.META['REMOTE_ADDR'],
port=request.META['SERVER_PORT'],
access_token="token",
refresh_token="token").to_json()
]
request.session['instancesExplore'] = json_instances
template = loader.get_template('explore/explore_results.html')
context = RequestContext(request, {
'': '',
})
if 'exploreCurrentTemplateID' not in request.session:
return redirect('/explore/select-template')
else:
return HttpResponse(template.render(context))
def create_instance(content, request):
now = datetime.now()
delta = timedelta(seconds=int(json.loads(content)["expires_in"]))
expires = now + delta
Instance(name=request["name"], protocol=request["protocol"], address=request["ip_address"],
port=request["port"], access_token=json.loads(content)["access_token"],
refresh_token=json.loads(content)["refresh_token"], expires=expires).save()
def test_update_instance(self):
instance = Instance(name='testtest',
protocol='h',
address='a',
port=12,
access_token='a',
refresh_token='a',
expires=datetime.now()).save()
response = '{"access_token": "token", "token_type": "Bearer", "expires_in": 31536000, "refresh_token": "refresh_token", "scope": "read write"}'
update_instance(instance=instance, content=response)
instance_updated = Instance.objects.get(pk=ObjectId(instance.id))
self.assertIsNotNone(instance_updated)
self.assertEquals(instance_updated.access_token, "token")
self.assertEquals(instance_updated.refresh_token, "refresh_token")
self.assertNotEqual(instance_updated.expires, instance.expires)
def get_results_by_instance_keyword(request):
print 'BEGIN def getResultsKeyword(request)'
resultsByKeyword = []
results = []
resultString = ""
#Instance
json_instances = []
if 'HTTPS' in request.META['SERVER_PROTOCOL']:
protocol = "https"
else:
protocol = "http"
instance = Instance(name="Local",
protocol=protocol,
address=request.META['REMOTE_ADDR'],
port=request.META['SERVER_PORT'],
access_token="token",
refresh_token="token")
json_instances.append(instance.to_json())
request.session['instancesExplore'] = json_instances
sessionName = "resultsExploreOaiPMh" + instance['name']
keyword = request.POST.get('keyword', '')
schemas = request.POST.getlist('schemas[]', [])
user_schemas = request.POST.getlist('userSchemas[]', [])
refinements = refinements_to_mongo(
json.loads(request.POST.get('refinements', '{}')))
registries = request.POST.getlist('registries[]', [])
if 'onlySuggestions' in request.POST:
onlySuggestions = json.loads(request.POST['onlySuggestions'])
else:
onlySuggestions = False
metadata_format_ids = _get_metadata_formats_id(schemas=schemas,
user_schemas=user_schemas,
registries=registries)
instanceResults = OaiRecord.executeFullTextQuery(keyword,
metadata_format_ids,
refinements)
if len(instanceResults) > 0:
if not onlySuggestions:
xsltPath = os.path.join(settings.SITE_ROOT,
'static/resources/xsl/xml2html.xsl')
xslt = etree.parse(xsltPath)
transform = etree.XSLT(xslt)
template = loader.get_template(
'oai_pmh/explore/explore_result_keyword.html')
#Retrieve schema and registries. Avoid to retrieve the information for each result
registriesName = {}
objMetadataFormats = {}
listRegistriesID = set([x['registry'] for x in instanceResults])
registriesURL = {}
for registryId in listRegistriesID:
obj = OaiRegistry.objects(pk=registryId).get()
registriesName[str(registryId)] = obj.name
registriesURL[str(registryId)] = obj.url
listSchemaId = set([x['metadataformat'] for x in instanceResults])
for schemaId in listSchemaId:
obj = OaiMetadataFormat.objects(pk=schemaId).get()
objMetadataFormats[str(schemaId)] = obj
listItems = []
xmltodictunparse = XMLdata.unparse
appendResult = results.append
toXML = etree.XML
parse = etree.parse
XSLT = etree.XSLT
if not onlySuggestions:
for instanceResult in instanceResults:
custom_xslt = False
appendResult({
'title':
instanceResult['identifier'],
'content':
xmltodictunparse(instanceResult['metadata']),
'id':
str(instanceResult['_id'])
})
dom = toXML(
str(
xmltodictunparse(
instanceResult['metadata']).encode('utf-8')))
#Check if a custom list result XSLT has to be used
try:
metadataFormat = objMetadataFormats[str(
instanceResult['metadataformat'])]
if metadataFormat.template.ResultXsltList:
listXslt = parse(
BytesIO(
metadataFormat.template.ResultXsltList.content.
encode('utf-8')))
listTransform = XSLT(listXslt)
newdom = listTransform(dom)
custom_xslt = True
else:
newdom = transform(dom)
except Exception, e:
#We use the default one
newdom = transform(dom)
custom_xslt = False
registry_name = registriesName[instanceResult['registry']]
if len(registry_name) > 30:
registry_name = "{0}...".format(registry_name[:30])
url = urlparse(registriesURL[instanceResult['registry']])
context = RequestContext(
request, {
'id': str(instanceResult['_id']),
'xml': str(newdom),
'title': instanceResult['identifier'],
'custom_xslt': custom_xslt,
'template_name': metadataFormat.template.title,
'registry_name': registry_name,
'registry_url': "{0}://{1}".format(
url.scheme, url.netloc),
'oai_pmh': True
})
resultString += template.render(context)
else:
for instanceResult in instanceResults[:20]:
wordList = re.sub("[^\w]", " ", keyword).split()
wordList = [x + "|" + x + "\w+" for x in wordList]
wordList = '|'.join(wordList)
listWholeKeywords = re.findall(
"\\b(" + wordList + ")\\b",
XMLdata.unparse(
instanceResult['metadata']).encode('utf-8'),
flags=re.IGNORECASE)
labels = list(set(listWholeKeywords))
for label in labels:
label = label.lower()
result_json = {}
result_json['label'] = label
result_json['value'] = label
if not result_json in resultsByKeyword:
resultsByKeyword.append(result_json)
def get_results_by_instance_keyword(request):
print 'BEGIN def getResultsKeyword(request)'
resultsByKeyword = []
results = []
resultString = ""
#Instance
json_instances = []
if 'HTTPS' in request.META['SERVER_PROTOCOL']:
protocol = "https"
else:
protocol = "http"
instance = Instance(name="Local",
protocol=protocol,
address=request.META['REMOTE_ADDR'],
port=request.META['SERVER_PORT'],
access_token="token",
refresh_token="token")
json_instances.append(instance.to_json())
request.session['instancesExplore'] = json_instances
sessionName = "resultsExploreOaiPMh" + instance['name']
try:
keyword = request.GET['keyword']
schemas = request.GET.getlist('schemas[]')
mergedSchemas = []
for schema in schemas:
t = json.loads(schema)
mergedSchemas += t['oai-pmh']
if 'onlySuggestions' in request.GET:
onlySuggestions = json.loads(request.GET['onlySuggestions'])
else:
onlySuggestions = False
except:
keyword = ''
schemas = []
onlySuggestions = True
mergedSchemas = []
instanceResults = OaiRecord.executeFullTextQuery(keyword, mergedSchemas)
if len(instanceResults) > 0:
if not onlySuggestions:
xsltPath = os.path.join(settings.SITE_ROOT,
'static/resources/xsl/xml2html.xsl')
xslt = etree.parse(xsltPath)
transform = etree.XSLT(xslt)
#Retrieve schema and registries. Avoid to retrieve the information for each result
registriesName = {}
schemasName = {}
listRegistriesID = set([x['registry'] for x in instanceResults])
for registryId in listRegistriesID:
obj = OaiRegistry.objects(pk=registryId).get()
registriesName[str(registryId)] = obj.name
listSchemaId = set([x['metadataformat'] for x in instanceResults])
for schemaId in listSchemaId:
obj = OaiMetadataFormat.objects(pk=schemaId).get()
schemasName[str(schemaId)] = obj
listItems = []
xmltodictunparse = unparse
appendResult = results.append
toXML = etree.XML
parse = etree.parse
XSLT = etree.XSLT
if not onlySuggestions:
for instanceResult in instanceResults:
custom_xslt = False
appendResult({
'title':
instanceResult['identifier'],
'content':
xmltodictunparse(instanceResult['metadata']),
'id':
str(instanceResult['_id'])
})
dom = toXML(
str(
xmltodictunparse(
instanceResult['metadata']).encode('utf-8')))
#Check if a custom list result XSLT has to be used
try:
schema = schemasName[str(instanceResult['metadataformat'])]
if schema.ResultXsltList:
listXslt = parse(
BytesIO(
schema.ResultXsltList.content.encode('utf-8')))
listTransform = XSLT(listXslt)
newdom = listTransform(dom)
custom_xslt = True
else:
newdom = transform(dom)
except Exception, e:
#We use the default one
newdom = transform(dom)
custom_xslt = False
item = {
'id': str(instanceResult['_id']),
'xml': str(newdom),
'title': instanceResult['identifier'],
'custom_xslt': custom_xslt,
'schema_name': schema.metadataPrefix,
'registry_name': registriesName[instanceResult['registry']]
}
listItems.append(item)
context = RequestContext(request, {'list_results': listItems})
else:
for instanceResult in instanceResults[:20]:
wordList = re.sub("[^\w]", " ", keyword).split()
wordList = [x + "|" + x + "\w+" for x in wordList]
wordList = '|'.join(wordList)
listWholeKeywords = re.findall(
"\\b(" + wordList + ")\\b",
unparse(instanceResult['metadata']).encode('utf-8'),
flags=re.IGNORECASE)
labels = list(set(listWholeKeywords))
for label in labels:
label = label.lower()
result_json = {}
result_json['label'] = label
result_json['value'] = label
if not result_json in resultsByKeyword:
resultsByKeyword.append(result_json)
if not onlySuggestions:
template = loader.get_template(
'oai_pmh/explore/explore_result_keyword.html')
resultString += template.render(context)
def add_repository(request):
if request.method == 'POST':
form = RepositoryForm(request.POST)
if form.is_valid():
if request.POST["action"] == "Register":
errors = ""
# test if the name is "Local"
if (request.POST["name"].upper() == "LOCAL"):
errors += "By default, the instance named Local is the instance currently running."
else:
# test if an instance with the same name exists
instance = Instance.objects(name=request.POST["name"])
if len(instance) != 0:
errors += "An instance with the same name already exists.<br/>"
# test if new instance is not the same as the local instance
if request.POST["ip_address"] == request.META[
'REMOTE_ADDR'] and request.POST[
"port"] == request.META['SERVER_PORT']:
errors += "The address and port you entered refer to the instance currently running."
else:
# test if an instance with the same address/port exists
instance = Instance.objects(
address=request.POST["ip_address"],
port=request.POST["port"])
if len(instance) != 0:
errors += "An instance with the address/port already exists.<br/>"
# If some errors display them, otherwise insert the instance
if (errors == ""):
try:
url = request.POST["protocol"] + "://" + request.POST[
"ip_address"] + ":" + request.POST[
"port"] + "/o/token/"
# data="grant_type=password&username="******"username"] + "&password="******"password"]
headers = {
'content-type': 'application/x-www-form-urlencoded'
}
data = {
'grant_type': 'password',
'username': request.POST["username"],
'password': request.POST["password"],
'client_id': request.POST["client_id"],
'client_secret': request.POST["client_secret"]
}
r = requests.post(url=url,
data=data,
headers=headers,
timeout=int(request.POST["timeout"]))
if r.status_code == 200:
now = datetime.now()
delta = timedelta(
seconds=int(eval(r.content)["expires_in"]))
expires = now + delta
Instance(
name=request.POST["name"],
protocol=request.POST["protocol"],
address=request.POST["ip_address"],
port=request.POST["port"],
access_token=eval(r.content)["access_token"],
refresh_token=eval(r.content)["refresh_token"],
expires=expires).save()
messages.add_message(
request, messages.INFO,
'Repository registered with success.')
return redirect('/admin/repositories')
else:
message = "Unable to get access to the remote instance using these parameters."
return render(request, 'admin/add_repository.html',
{
'form': form,
'action_result': message
})
except Exception, e:
message = "Unable to get access to the remote instance using these parameters."
return render(request, 'admin/add_repository.html', {
'form': form,
'action_result': message
})
else:
return render(request, 'admin/add_repository.html', {
'form': form,
'action_result': errors
})
elif request.POST["action"] == "Ping":
try:
url = request.POST["protocol"] + "://" + request.POST[
"ip_address"] + ":" + request.POST[
"port"] + "/rest/ping"
r = requests.get(url,
auth=(request.POST["username"],
request.POST["password"]),
timeout=int(request.POST["timeout"]))
if r.status_code == 200:
message = "Remote API reached with success."
else:
if 'detail' in eval(r.content):
message = "Error: " + eval(r.content)['detail']
else:
message = "Error: Unable to reach the remote API."
except Exception, e:
message = "Error: Unable to reach the remote API."
return render(request, 'admin/add_repository.html', {
'form': form,
'action_result': message
})
def get_results_by_instance_keyword(request):
print 'BEGIN def getResultsKeyword(request)'
resultsByKeyword = []
results = []
resultString = ""
#Instance
json_instances = []
if 'HTTPS' in request.META['SERVER_PROTOCOL']:
protocol = "https"
else:
protocol = "http"
instance = Instance(name="Local",
protocol=protocol,
address=request.META['REMOTE_ADDR'],
port=request.META['SERVER_PORT'],
access_token="token",
refresh_token="token")
json_instances.append(instance.to_json())
request.session['instancesExplore'] = json_instances
sessionName = "resultsExploreOaiPMh" + instance['name']
try:
keyword = request.GET['keyword']
schemas = request.GET.getlist('schemas[]')
userSchemas = request.GET.getlist('userSchemas[]')
refinements = refinements_to_mongo(
request.GET.getlist('refinements[]'))
if 'onlySuggestions' in request.GET:
onlySuggestions = json.loads(request.GET['onlySuggestions'])
else:
onlySuggestions = False
except:
keyword = ''
schemas = []
userSchemas = []
refinements = {}
onlySuggestions = True
#We get all template versions for the given schemas
#First, we take care of user defined schema
templatesIDUser = Template.objects(title__in=userSchemas).distinct(
field="id")
templatesIDUser = [str(x) for x in templatesIDUser]
#Take care of the rest, with versions
templatesVersions = Template.objects(title__in=schemas).distinct(
field="templateVersion")
#We get all templates ID, for all versions
allTemplatesIDCommon = TemplateVersion.objects(
pk__in=templatesVersions, isDeleted=False).distinct(field="versions")
#We remove the removed version
allTemplatesIDCommonRemoved = TemplateVersion.objects(
pk__in=templatesVersions,
isDeleted=False).distinct(field="deletedVersions")
templatesIDCommon = list(
set(allTemplatesIDCommon) - set(allTemplatesIDCommonRemoved))
templatesID = templatesIDUser + templatesIDCommon
#We retrieve deactivated registries so as not to get their metadata formats
deactivatedRegistries = [
str(x.id)
for x in OaiRegistry.objects(isDeactivated=True).order_by('id')
]
metadataFormatsID = OaiMetadataFormat.objects(
template__in=templatesID,
registry__not__in=deactivatedRegistries).distinct(field="id")
instanceResults = OaiRecord.executeFullTextQuery(keyword,
metadataFormatsID,
refinements)
if len(instanceResults) > 0:
if not onlySuggestions:
xsltPath = os.path.join(settings.SITE_ROOT,
'static/resources/xsl/xml2html.xsl')
xslt = etree.parse(xsltPath)
transform = etree.XSLT(xslt)
template = loader.get_template(
'oai_pmh/explore/explore_result_keyword.html')
#Retrieve schema and registries. Avoid to retrieve the information for each result
registriesName = {}
objMetadataFormats = {}
listRegistriesID = set([x['registry'] for x in instanceResults])
for registryId in listRegistriesID:
obj = OaiRegistry.objects(pk=registryId).get()
registriesName[str(registryId)] = obj.name
listSchemaId = set([x['metadataformat'] for x in instanceResults])
for schemaId in listSchemaId:
obj = OaiMetadataFormat.objects(pk=schemaId).get()
objMetadataFormats[str(schemaId)] = obj
listItems = []
xmltodictunparse = xmltodict.unparse
appendResult = results.append
toXML = etree.XML
parse = etree.parse
XSLT = etree.XSLT
if not onlySuggestions:
for instanceResult in instanceResults:
custom_xslt = False
appendResult({
'title':
instanceResult['identifier'],
'content':
xmltodictunparse(instanceResult['metadata']),
'id':
str(instanceResult['_id'])
})
dom = toXML(
str(
xmltodictunparse(
instanceResult['metadata']).encode('utf-8')))
#Check if a custom list result XSLT has to be used
try:
metadataFormat = objMetadataFormats[str(
instanceResult['metadataformat'])]
if metadataFormat.template.ResultXsltList:
listXslt = parse(
BytesIO(
metadataFormat.template.ResultXsltList.content.
encode('utf-8')))
listTransform = XSLT(listXslt)
newdom = listTransform(dom)
custom_xslt = True
else:
newdom = transform(dom)
except Exception, e:
#We use the default one
newdom = transform(dom)
custom_xslt = False
context = RequestContext(
request, {
'id': str(instanceResult['_id']),
'xml': str(newdom),
'title': instanceResult['identifier'],
'custom_xslt': custom_xslt,
'schema_name': metadataFormat.metadataPrefix,
'registry_name':
registriesName[instanceResult['registry']],
'oai_pmh': True
})
resultString += template.render(context)
else:
for instanceResult in instanceResults[:20]:
wordList = re.sub("[^\w]", " ", keyword).split()
wordList = [x + "|" + x + "\w+" for x in wordList]
wordList = '|'.join(wordList)
listWholeKeywords = re.findall(
"\\b(" + wordList + ")\\b",
xmltodict.unparse(
instanceResult['metadata']).encode('utf-8'),
flags=re.IGNORECASE)
labels = list(set(listWholeKeywords))
for label in labels:
label = label.lower()
result_json = {}
result_json['label'] = label
result_json['value'] = label
if not result_json in resultsByKeyword:
resultsByKeyword.append(result_json)