def stream(index='record',key='register.metadata.record.subject.term',size=1000,raw=False,q=False,counts=False,order='term'):
indices = []
for idx in index.split(','):
if idx not in app.config['NO_QUERY']:
indices.append(idx)
if not isinstance(key,list):
keys = key.split(',')
if not counts: counts = request.values.get('counts',False)
if request.values.get('order',False): order = request.values['order']
if not q: q = request.values.get('q','*')
if not q.endswith("*") and "~" not in q: q += "*"
if not q.startswith("*") and "~" not in q: q = "*" + q
qry = {
'query':{'match_all':{}},
'size': 0,
'facets':{}
}
if q != "*": qry['query'] = {"query_string": {"query": q}}
for ky in keys:
ks = ky.replace('.exact','')
qry['facets'][ks] = {"terms":{"field":ks+app.config['FACET_FIELD'],"order":order, "size":request.values.get('size',size)}}
base = app.config.get("OARR_API_BASE_URL")
if base is None:
abort(500)
client = OARRClient(base)
try:
r = client.query(qry).json()
except:
r = {}
res = []
try:
if counts:
for k in keys:
ks = k.replace('.exact','')
res = res + [[i['term'],i['count']] for i in r.get('facets',{}).get(ks,{}).get("terms",[])]
else:
for k in keys:
ks = k.replace('.exact','')
res = res + [i['term'] for i in r.get('facets',{}).get(ks,{}).get("terms",[])]
except:
pass
if raw:
return res
else:
resp = make_response( json.dumps(res) )
resp.mimetype = "application/json"
return resp
def repository(repo_id):
base = app.config.get("OARR_API_BASE_URL")
if base is None:
abort(500)
client = OARRClient(base)
if util.request_wants_json():
repo_id = repo_id.replace('.json','')
try:
record = client.get_record(repo_id)
resp = make_response( record.json )
resp.mimetype = "application/json"
return resp
except:
abort(404)
else:
record = client.get_record(repo_id)
if record is None:
abort(404)
return render_template("repository.html", repo=record, searchurl=searchurl, search_similar=True)
def organisation(org):
base = app.config.get("OARR_API_BASE_URL")
if base is None:
abort(500)
client = OARRClient(base)
if org.endswith('.json'):
org = org.replace('.json','')
try:
record = client.get_org(org)
resp = make_response( record.json )
resp.mimetype = "application/json"
return resp
except:
abort(404)
else:
try:
record = client.get_org(org)
return render_template("organisation.html", org=record.record)
except:
abort(404)
def duplicate(url=False,raw=False):
if 'url' in request.values: url = request.values['url']
if url:
turl = '*' + url.replace('http://','').replace('https://','').replace('www','').strip('/') + '*'
base = app.config.get("OARR_API_BASE_URL")
if base is None: abort(500)
client = OARRClient(base)
r = client.query(qry={'query':{'query_string':{"query":turl}}})
if r.get('hits',{}).get('total',0) != 0:
res = r['hits']['hits'][0]['_id']
else:
res = False
else:
res = False
if raw:
return res
else:
resp = make_response( json.dumps(res) )
resp.mimetype = "application/json"
return resp
def contribute():
detectdone = False
dup = False
if request.method == 'GET':
# check to see if this is an update
if 'updaterequest' in request.values:
# get record from OARR
try:
base = app.config.get("OARR_API_BASE_URL")
if base is None: abort(500)
client = OARRClient(base)
record = client.get_record(request.values["updaterequest"]).raw
if "opendoar" not in record["admin"]: record["admin"]["opendoar"] = {}
record["admin"]["opendoar"]["updaterequest"] = request.values["updaterequest"]
detectdone = True
except:
abort(404)
# check for a url request param
elif 'url' in request.values:
# if there is one, then try to set the initial object
if len(request.values['url']) != 0:
try:
register = autodiscovery.discover(request.values['url'])
record = register.raw
for k, v in util.defaultrecord['register']['metadata'][0]['record'].iteritems():
if k not in record.get('register',{}).get('metadata',[{"record":{}}])[0]['record']:
record['register']['metadata'][0]['record'][k] = v
except:
record = util.defaultrecord
# check if there is already a record with this url
dup = rawduplicate(request.values['url'],raw=True)
else:
record = util.defaultrecord
detectdone = True
else:
# otherwise set a default initial object
record = util.defaultrecord
if util.request_wants_json():
resp = make_response( json.dumps({"record":record}) )
resp.mimetype = "application/json"
return resp
else:
return render_template("contribute.html", record=record, detectdone=detectdone, duplicate=dup)
elif request.method == 'POST':
base = app.config.get("OARR_API_BASE_URL")
apikey = app.config.get("OARR_API_KEY")
if base is None or apikey is None: abort(500)
client = OARRClient(base, apikey)
record = client.prep_record(util.defaultrecord,request)
if 'updaterequest' not in record['admin']['opendoar']: record['admin']['opendoar']['newcontribution'] = True
saved = client.save_record(record)
if saved:
flash('Thank you very much for your submission. Your request will be processed as soon as possible, and usually within three working days.', 'success')
return redirect('/')
else:
flash('Sorry, there was a problem saving your submission. Please try again.', 'error')
return redirect('/contribute')
def stream(index='record',key='register.metadata.record.subject.term',size=1000,raw=False,q=False,counts=False,order='term'):
indices = []
for idx in index.split(','):
if idx not in app.config['NO_QUERY']:
indices.append(idx)
if not isinstance(key,list):
keys = key.split(',')
try:
if not counts: counts = request.values.get('counts',False)
if request.values.get('order',False): order = request.values['order']
if not q: q = request.values.get('q','*')
size = request.values.get('size',size)
except:
q = '*'
if not q.endswith("*") and "~" not in q: q += "*"
if not q.startswith("*") and "~" not in q: q = "*" + q
qry = {
'query':{'match_all':{}},
'size': 0,
'facets':{}
}
if q != "*":
qry['query'] = {
"query_string": {
"query": q,
'fields': keys,
'use_dis_max': True
}
}
for ky in keys:
ks = ky.replace('.exact','')
qry['facets'][ks] = {"terms":{"field":ks+app.config['FACET_FIELD'],"order":order, "size":size}}
base = app.config.get("OARR_API_BASE_URL")
if base is None:
abort(500)
client = OARRClient(base)
try:
r = client.query(qry)
except:
r = {}
res = []
try:
if counts:
for k in keys:
ks = k.replace('.exact','')
res = res + [[i['term'],i['count']] for i in r.get('facets',{}).get(ks,{}).get("terms",[])]
else:
for k in keys:
ks = k.replace('.exact','')
res = res + [i['term'] for i in r.get('facets',{}).get(ks,{}).get("terms",[])]
except:
pass
qc = q.replace('*','').replace('~','')
if "register.metadata.record.country" in keys or "register.organisation.details.country" in keys:
additional = [i.name for i in pycountry.countries]
elif "register.metadata.record.language" in keys:
additional = [i.name for i in pycountry.languages]
elif "register.metadata.lang" in keys:
additional = []
for i in pycountry.languages:
try:
additional.append(i.alpha2)
except:
pass
else:
additional = []
for val in additional:
if (qc.lower() in val.lower() or len(qc) == 0) and val not in res:
res.append(val)
if raw:
return res
else:
resp = make_response( json.dumps(res) )
resp.mimetype = "application/json"
return resp
def record(uuid=None):
base = app.config.get("OARR_API_BASE_URL")
apikey = app.config.get("OARR_API_KEY")
if base is None or apikey is None:
abort(500)
client = OARRClient(base, apikey)
detectdone = False
dup = False
if request.method == "GET":
if uuid is None or uuid == "new":
# check for a url request param
if "url" in request.values:
# if there is one, then try to set the initial object
if len(request.values["url"]) != 0:
try:
register = autodiscovery.discover(request.values["url"])
record = register.raw
for k, v in util.defaultrecord["register"]["metadata"][0]["record"].iteritems():
if k not in record.get("register", {}).get("metadata", [{"record": {}}])[0]["record"]:
record["register"]["metadata"][0]["record"][k] = v
except:
record = util.defaultrecord
else:
record = util.defaultrecord
detectdone = True
# check if there is already a record with this url
dup = duplicate(request.values["url"], raw=True)
else:
# otherwise set a default initial object
record = util.defaultrecord
else:
# get record from OARR
try:
record = client.get_record(uuid.replace(".json", "")).raw
detectdone = True
# check if there is already a record with this url
dup = duplicate(record["register"]["metadata"][0]["record"]["url"], raw=True)
except:
abort(404)
if util.request_wants_json():
resp = make_response(json.dumps({"record": record}))
resp.mimetype = "application/json"
return resp
else:
return render_template("contribute.html", record=record, detectdone=detectdone, duplicate=dup)
elif (request.method == "POST" and request.values.get("submit", "") == "Delete") or request.method == "DELETE":
record = client.get_record(uuid)
if record is None:
abort(404)
client.delete_record(uuid)
time.sleep(1)
flash("Record delete")
return redirect(url_for(".index"))
elif request.method == "POST":
if uuid == "new":
# save the new record to OARR
record = client.prep_record(util.defaultrecord, request)
saved = client.save_record(record)
if saved:
flash("New record created", "success")
return redirect(url_for(".index"))
else:
flash("Sorry, the attempt to create a new record was unsuccessful", "error")
return redirect("/admin/record/new")
else:
rec = client.get_record(uuid)
if rec is None:
abort(404)
# remove any newcontribution tag on first save via admin
if "newcontribution" in rec.raw.get("admin", {}).get("opendoar", {}):
del rec.raw["admin"]["opendoar"]["newcontribution"]
# if this is an update acceptance, do the update
if "updaterequest" in rec.raw.get("admin", {}).get("opendoar", {}):
# get the original record, prep it with the update, delete the update request record
forupdate = client.get_record(rec.raw["admin"]["opendoar"]["updaterequest"])
if forupdate is None:
flash("Sorry, an original record cannot be found to correspond with this update request.", "error")
return redirect("/admin/record/" + uuid)
else:
record = client.prep_record(forupdate.raw, request)
try:
del record["admin"]["opendoar"]["updaterequest"]
saved = client.save_record(record)
if saved:
client.delete_record(uuid)
time.sleep(1)
flash(
"This original record has been successfully updated, and the update request record has been deleted.",
"success",
)
return redirect("/admin/record/" + str(record["id"]))
else:
flash(
"Sorry, there was an error. Your changes have not been saved. Please try again.",
"error",
)
return redirect("/admin/record/" + uuid)
except:
flash("Sorry, there was an error. Your changes have not been saved. Please try again.", "error")
return redirect("/admin/record/" + uuid)
# otherwise save the record changes to OARR
else:
# do whatever needs done here to update the record from the form input
record = client.prep_record(rec.raw, request)
saved = client.save_record(record)
if saved:
detectdone = True
time.sleep(1)
flash("Record has been updated", "success")
return render_template("contribute.html", record=record, detectdone=detectdone, duplicate=dup)
else:
flash("Sorry, there was an error. Your changes have not been saved. Please try again.", "error")
return redirect("/admin/record/" + uuid)
def index():
base = app.config.get("OARR_API_BASE_URL")
if base is None:
abort(500)
client = OARRClient(base)
# run some queries to build up some stats for the page
qr = {
"query": {
"bool": {
"should": [
{"term": {"admin.opendoar.in_opendoar": False}},
{"constant_score": {"filter": {"missing": {"field": "admin.opendoar.in_opendoar"}}}},
]
}
},
"sort": [{"last_updated": {"order": "desc"}}],
"size": 0,
}
stats = {}
# get number not in_opendoar
res = client.query(qr)
stats["out"] = res.get("hits", {}).get("total", 0)
# get recently edited in opendoar and not since saved
qr["query"]["bool"]["must"] = [{"term": {"admin.opendoar.in_opendoar": True}}]
qr["query"]["bool"]["must"] = [{"constant_score": {"filter": {"exists": {"field": "admin.opendoar.last_saved"}}}}]
del qr["query"]["bool"]["should"]
qr["filter"] = {
"script": {
"script": "unviewed"
# "script" : "doc['last_updated'].value > doc['admin.opendoar.last_saved'].value"
}
}
res = client.query(qr)
try:
stats["unviewed"] = res["hits"]["total"]
except:
stats["unviewed"] = "MISSING SCRIPT?"
# get update requests
qr = {
"query": {
"bool": {"must": [{"constant_score": {"filter": {"exists": {"field": "admin.opendoar.updaterequest"}}}}]}
},
"sort": [{"last_updated": {"order": "desc"}}],
"size": 0,
}
res = client.query(qr)
stats["updaterequest"] = res.get("hits", {}).get("total", 0)
# get new contributions
qr = {
"query": {
"bool": {"must": [{"constant_score": {"filter": {"exists": {"field": "admin.opendoar.newcontribution"}}}}]}
},
"sort": [{"last_updated": {"order": "desc"}}],
"size": 0,
}
res = client.query(qr)
stats["newcontribution"] = res.get("hits", {}).get("total", 0)
return render_template("admin/index.html", stats=stats)