def stream(index='record',key='register.metadata.record.subject.term',size=1000,raw=False,q=False,counts=False,order='term'): indices = [] for idx in index.split(','): if idx not in app.config['NO_QUERY']: indices.append(idx) if not isinstance(key,list): keys = key.split(',') if not counts: counts = request.values.get('counts',False) if request.values.get('order',False): order = request.values['order'] if not q: q = request.values.get('q','*') if not q.endswith("*") and "~" not in q: q += "*" if not q.startswith("*") and "~" not in q: q = "*" + q qry = { 'query':{'match_all':{}}, 'size': 0, 'facets':{} } if q != "*": qry['query'] = {"query_string": {"query": q}} for ky in keys: ks = ky.replace('.exact','') qry['facets'][ks] = {"terms":{"field":ks+app.config['FACET_FIELD'],"order":order, "size":request.values.get('size',size)}} base = app.config.get("OARR_API_BASE_URL") if base is None: abort(500) client = OARRClient(base) try: r = client.query(qry).json() except: r = {} res = [] try: if counts: for k in keys: ks = k.replace('.exact','') res = res + [[i['term'],i['count']] for i in r.get('facets',{}).get(ks,{}).get("terms",[])] else: for k in keys: ks = k.replace('.exact','') res = res + [i['term'] for i in r.get('facets',{}).get(ks,{}).get("terms",[])] except: pass if raw: return res else: resp = make_response( json.dumps(res) ) resp.mimetype = "application/json" return resp
def duplicate(url=False,raw=False): if 'url' in request.values: url = request.values['url'] if url: turl = '*' + url.replace('http://','').replace('https://','').replace('www','').strip('/') + '*' base = app.config.get("OARR_API_BASE_URL") if base is None: abort(500) client = OARRClient(base) r = client.query(qry={'query':{'query_string':{"query":turl}}}) if r.get('hits',{}).get('total',0) != 0: res = r['hits']['hits'][0]['_id'] else: res = False else: res = False if raw: return res else: resp = make_response( json.dumps(res) ) resp.mimetype = "application/json" return resp
def stream(index='record',key='register.metadata.record.subject.term',size=1000,raw=False,q=False,counts=False,order='term'): indices = [] for idx in index.split(','): if idx not in app.config['NO_QUERY']: indices.append(idx) if not isinstance(key,list): keys = key.split(',') try: if not counts: counts = request.values.get('counts',False) if request.values.get('order',False): order = request.values['order'] if not q: q = request.values.get('q','*') size = request.values.get('size',size) except: q = '*' if not q.endswith("*") and "~" not in q: q += "*" if not q.startswith("*") and "~" not in q: q = "*" + q qry = { 'query':{'match_all':{}}, 'size': 0, 'facets':{} } if q != "*": qry['query'] = { "query_string": { "query": q, 'fields': keys, 'use_dis_max': True } } for ky in keys: ks = ky.replace('.exact','') qry['facets'][ks] = {"terms":{"field":ks+app.config['FACET_FIELD'],"order":order, "size":size}} base = app.config.get("OARR_API_BASE_URL") if base is None: abort(500) client = OARRClient(base) try: r = client.query(qry) except: r = {} res = [] try: if counts: for k in keys: ks = k.replace('.exact','') res = res + [[i['term'],i['count']] for i in r.get('facets',{}).get(ks,{}).get("terms",[])] else: for k in keys: ks = k.replace('.exact','') res = res + [i['term'] for i in r.get('facets',{}).get(ks,{}).get("terms",[])] except: pass qc = q.replace('*','').replace('~','') if "register.metadata.record.country" in keys or "register.organisation.details.country" in keys: additional = [i.name for i in pycountry.countries] elif "register.metadata.record.language" in keys: additional = [i.name for i in pycountry.languages] elif "register.metadata.lang" in keys: additional = [] for i in pycountry.languages: try: additional.append(i.alpha2) except: pass else: additional = [] for val in additional: if (qc.lower() in val.lower() or len(qc) == 0) and val not in res: res.append(val) if raw: return res else: resp = make_response( json.dumps(res) ) resp.mimetype = "application/json" return resp
def index(): base = app.config.get("OARR_API_BASE_URL") if base is None: abort(500) client = OARRClient(base) # run some queries to build up some stats for the page qr = { "query": { "bool": { "should": [ {"term": {"admin.opendoar.in_opendoar": False}}, {"constant_score": {"filter": {"missing": {"field": "admin.opendoar.in_opendoar"}}}}, ] } }, "sort": [{"last_updated": {"order": "desc"}}], "size": 0, } stats = {} # get number not in_opendoar res = client.query(qr) stats["out"] = res.get("hits", {}).get("total", 0) # get recently edited in opendoar and not since saved qr["query"]["bool"]["must"] = [{"term": {"admin.opendoar.in_opendoar": True}}] qr["query"]["bool"]["must"] = [{"constant_score": {"filter": {"exists": {"field": "admin.opendoar.last_saved"}}}}] del qr["query"]["bool"]["should"] qr["filter"] = { "script": { "script": "unviewed" # "script" : "doc['last_updated'].value > doc['admin.opendoar.last_saved'].value" } } res = client.query(qr) try: stats["unviewed"] = res["hits"]["total"] except: stats["unviewed"] = "MISSING SCRIPT?" # get update requests qr = { "query": { "bool": {"must": [{"constant_score": {"filter": {"exists": {"field": "admin.opendoar.updaterequest"}}}}]} }, "sort": [{"last_updated": {"order": "desc"}}], "size": 0, } res = client.query(qr) stats["updaterequest"] = res.get("hits", {}).get("total", 0) # get new contributions qr = { "query": { "bool": {"must": [{"constant_score": {"filter": {"exists": {"field": "admin.opendoar.newcontribution"}}}}]} }, "sort": [{"last_updated": {"order": "desc"}}], "size": 0, } res = client.query(qr) stats["newcontribution"] = res.get("hits", {}).get("total", 0) return render_template("admin/index.html", stats=stats)