def stream(index='record',key='register.metadata.record.subject.term',size=1000,raw=False,q=False,counts=False,order='term'):
indices = []
for idx in index.split(','):
if idx not in app.config['NO_QUERY']:
indices.append(idx)
if not isinstance(key,list):
keys = key.split(',')
if not counts: counts = request.values.get('counts',False)
if request.values.get('order',False): order = request.values['order']
if not q: q = request.values.get('q','*')
if not q.endswith("*") and "~" not in q: q += "*"
if not q.startswith("*") and "~" not in q: q = "*" + q
qry = {
'query':{'match_all':{}},
'size': 0,
'facets':{}
}
if q != "*": qry['query'] = {"query_string": {"query": q}}
for ky in keys:
ks = ky.replace('.exact','')
qry['facets'][ks] = {"terms":{"field":ks+app.config['FACET_FIELD'],"order":order, "size":request.values.get('size',size)}}
base = app.config.get("OARR_API_BASE_URL")
if base is None:
abort(500)
client = OARRClient(base)
try:
r = client.query(qry).json()
except:
r = {}
res = []
try:
if counts:
for k in keys:
ks = k.replace('.exact','')
res = res + [[i['term'],i['count']] for i in r.get('facets',{}).get(ks,{}).get("terms",[])]
else:
for k in keys:
ks = k.replace('.exact','')
res = res + [i['term'] for i in r.get('facets',{}).get(ks,{}).get("terms",[])]
except:
pass
if raw:
return res
else:
resp = make_response( json.dumps(res) )
resp.mimetype = "application/json"
return resp
def duplicate(url=False,raw=False):
if 'url' in request.values: url = request.values['url']
if url:
turl = '*' + url.replace('http://','').replace('https://','').replace('www','').strip('/') + '*'
base = app.config.get("OARR_API_BASE_URL")
if base is None: abort(500)
client = OARRClient(base)
r = client.query(qry={'query':{'query_string':{"query":turl}}})
if r.get('hits',{}).get('total',0) != 0:
res = r['hits']['hits'][0]['_id']
else:
res = False
else:
res = False
if raw:
return res
else:
resp = make_response( json.dumps(res) )
resp.mimetype = "application/json"
return resp
def stream(index='record',key='register.metadata.record.subject.term',size=1000,raw=False,q=False,counts=False,order='term'):
indices = []
for idx in index.split(','):
if idx not in app.config['NO_QUERY']:
indices.append(idx)
if not isinstance(key,list):
keys = key.split(',')
try:
if not counts: counts = request.values.get('counts',False)
if request.values.get('order',False): order = request.values['order']
if not q: q = request.values.get('q','*')
size = request.values.get('size',size)
except:
q = '*'
if not q.endswith("*") and "~" not in q: q += "*"
if not q.startswith("*") and "~" not in q: q = "*" + q
qry = {
'query':{'match_all':{}},
'size': 0,
'facets':{}
}
if q != "*":
qry['query'] = {
"query_string": {
"query": q,
'fields': keys,
'use_dis_max': True
}
}
for ky in keys:
ks = ky.replace('.exact','')
qry['facets'][ks] = {"terms":{"field":ks+app.config['FACET_FIELD'],"order":order, "size":size}}
base = app.config.get("OARR_API_BASE_URL")
if base is None:
abort(500)
client = OARRClient(base)
try:
r = client.query(qry)
except:
r = {}
res = []
try:
if counts:
for k in keys:
ks = k.replace('.exact','')
res = res + [[i['term'],i['count']] for i in r.get('facets',{}).get(ks,{}).get("terms",[])]
else:
for k in keys:
ks = k.replace('.exact','')
res = res + [i['term'] for i in r.get('facets',{}).get(ks,{}).get("terms",[])]
except:
pass
qc = q.replace('*','').replace('~','')
if "register.metadata.record.country" in keys or "register.organisation.details.country" in keys:
additional = [i.name for i in pycountry.countries]
elif "register.metadata.record.language" in keys:
additional = [i.name for i in pycountry.languages]
elif "register.metadata.lang" in keys:
additional = []
for i in pycountry.languages:
try:
additional.append(i.alpha2)
except:
pass
else:
additional = []
for val in additional:
if (qc.lower() in val.lower() or len(qc) == 0) and val not in res:
res.append(val)
if raw:
return res
else:
resp = make_response( json.dumps(res) )
resp.mimetype = "application/json"
return resp
def index():
base = app.config.get("OARR_API_BASE_URL")
if base is None:
abort(500)
client = OARRClient(base)
# run some queries to build up some stats for the page
qr = {
"query": {
"bool": {
"should": [
{"term": {"admin.opendoar.in_opendoar": False}},
{"constant_score": {"filter": {"missing": {"field": "admin.opendoar.in_opendoar"}}}},
]
}
},
"sort": [{"last_updated": {"order": "desc"}}],
"size": 0,
}
stats = {}
# get number not in_opendoar
res = client.query(qr)
stats["out"] = res.get("hits", {}).get("total", 0)
# get recently edited in opendoar and not since saved
qr["query"]["bool"]["must"] = [{"term": {"admin.opendoar.in_opendoar": True}}]
qr["query"]["bool"]["must"] = [{"constant_score": {"filter": {"exists": {"field": "admin.opendoar.last_saved"}}}}]
del qr["query"]["bool"]["should"]
qr["filter"] = {
"script": {
"script": "unviewed"
# "script" : "doc['last_updated'].value > doc['admin.opendoar.last_saved'].value"
}
}
res = client.query(qr)
try:
stats["unviewed"] = res["hits"]["total"]
except:
stats["unviewed"] = "MISSING SCRIPT?"
# get update requests
qr = {
"query": {
"bool": {"must": [{"constant_score": {"filter": {"exists": {"field": "admin.opendoar.updaterequest"}}}}]}
},
"sort": [{"last_updated": {"order": "desc"}}],
"size": 0,
}
res = client.query(qr)
stats["updaterequest"] = res.get("hits", {}).get("total", 0)
# get new contributions
qr = {
"query": {
"bool": {"must": [{"constant_score": {"filter": {"exists": {"field": "admin.opendoar.newcontribution"}}}}]}
},
"sort": [{"last_updated": {"order": "desc"}}],
"size": 0,
}
res = client.query(qr)
stats["newcontribution"] = res.get("hits", {}).get("total", 0)
return render_template("admin/index.html", stats=stats)
