def index(): if current_user.is_anonymous(): abort(401) if not current_user.is_super: return redirect('/account/' + current_user.id) users = models.Account.query(q={"query":{"match_all":{}},"sort":{'id.exact':{'order':'asc'}}, "size":100000}) userstats = { "super_user": 0, "do_admin": 0, "view_admin": 0 } if users['hits']['total'] != 0: accs = [models.Account.pull(i['_source']['id']) for i in users['hits']['hits']] # explicitly mapped to ensure no leakage of sensitive data. augment as necessary users = [] for acc in accs: if acc.id in app.config['SUPER_USER']: userstats['super_user'] += 1 elif acc.data.get('do_admin',"") != "": userstats["do_admin"] += 1 elif acc.data.get('view_admin',"") != "": userstats["view_admin"] += 1 user = {'id':acc.id} if 'created_date' in acc.data: user['created_date'] = acc.data['created_date'] users.append(user) if util.request_wants_json(): resp = make_response( json.dumps(users, sort_keys=True, indent=4) ) resp.mimetype = "application/json" return resp else: return render_template('account/all.html', users=users, userstats=userstats)
def story(sid): story = models.Blocked.pull(sid.replace('.json','')) #story = models.Record.pull(sid.replace('.json','')) turl = None cat = None user = None if story is not None: about = story.about(sid.replace('.json',''), exclude=story.id) turl = story.data['url'] user = models.Account.pull(story.data['author']) else: about = models.Record.about(sid.replace('.json','')) if about.get('hits',{}).get('total',0) == 0: flash('Sorry, there was no story found about ' + sid, 'warning') return redirect('/story') else: turl = about['hits']['hits'][0]['_source']['url'] if turl is not None: c = models.Catalogue.pull_by_url(turl) if c is not None: cat = c.data if util.request_wants_json() or sid.endswith('.json'): resp = make_response( story.json ) resp.mimetype = "application/json" return resp else: # TODO this should not pass all user data, some of that should be stored with the story return render_template("story.html", story=story, about=about, catalogue=cat, user=user.data)
def username(username): acc = models.Account.pull(username) if acc is None: abort(404) elif ( request.method == 'DELETE' or ( request.method == 'POST' and request.values.get('submit','').lower() == 'delete' ) ): acc.delete() flash('Account ' + acc.id + ' deleted') return redirect(url_for('.index')) elif request.method == 'POST': newdata = request.json if request.json else request.values if newdata.get('id',False): if newdata['id'] != username: acc = models.Account.pull(newdata['id']) else: newdata['api_key'] = acc.data['api_key'] for k, v in newdata.items(): if k not in ['submit','password']: acc.data[k] = v if 'password' in newdata and not newdata['password'].startswith('sha1'): acc.set_password(newdata['password']) acc.save() flash("Record updated") return render_template('account/view.html', account=acc) else: if util.request_wants_json(): resp = make_response( json.dumps(acc.data, sort_keys=True, indent=4) ) resp.mimetype = "application/json" return resp else: return render_template('account/view.html', account=acc)
def story(sid): story = models.Record.pull(sid.replace('.json','')) if story is None: abort(404) if util.request_wants_json() or sid.endswith('.json'): resp = make_response( story.json ) resp.mimetype = "application/json" return resp else: return render_template("story.html", story=story.json)
def username(username): acc = models.Account.pull(username) if acc is None: abort(404) elif (request.method == 'DELETE' or (request.method == 'POST' and request.values.get('submit', False) == 'Delete')): if current_user.id != acc.id and not current_user.is_super: abort(401) else: conf = request.values.get("confirm") if conf is None or conf != "confirm": flash('check the box to confirm you really mean it!', "error") return render_template('account/view.html', account=acc) acc.delete() flash('Account ' + acc.id + ' deleted') return redirect(url_for('.index')) elif request.method == 'POST': if current_user.id != acc.id and not current_user.is_super: abort(401) newdata = request.json if request.json else request.values if newdata.get('id', False): if newdata['id'] != username: acc = models.Account.pull(newdata['id']) else: newdata['api_key'] = acc.data['api_key'] for k, v in newdata.items(): if k not in [ 'submit', 'password', 'role', 'confirm', 'reset_token', 'reset_expires', 'last_updated', 'created_date', 'id' ]: acc.data[k] = v if 'password' in newdata and not newdata['password'].startswith( 'sha1'): acc.set_password(newdata['password']) # only super users can re-write roles if "role" in newdata and current_user.is_super: new_roles = [r.strip() for r in newdata.get("role").split(",")] acc.set_role(new_roles) acc.save() flash("Record updated") return render_template('account/view.html', account=acc) else: if util.request_wants_json(): resp = make_response(json.dumps(acc.data, sort_keys=True, indent=4)) resp.mimetype = "application/json" return resp else: # do an mget on the journals, so that we can present them to the user return render_template('account/view.html', account=acc)
def username(username): acc = models.Account.pull(username) if acc is None: acc = models.Account.pull_by_email(username) if acc is not None: return redirect('/account/' + acc.id) if acc is None: acc = models.Account.pull_by_api_key(username) if acc is not None: return redirect('/account/' + acc.id) if acc is None: abort(404) elif (request.method == 'DELETE' or (request.method == 'POST' and request.values.get('submit', False) == 'Delete')): if current_user.id != acc.id and not current_user.is_super: abort(401) else: acc.delete() flash('Account ' + acc.id + ' deleted') return redirect(url_for('.index')) elif request.method == 'POST': if current_user.id != acc.id and not current_user.is_super: abort(401) newdata = request.json if request.json else request.values if newdata.get('id', False): if newdata['id'] != username: acc = models.Account.pull(newdata['id']) else: newdata['api_key'] = acc.data['api_key'] for k, v in newdata.items(): if k not in ['submit', 'password']: acc.data[k] = v if 'password' in newdata and not newdata['password'].startswith( 'sha1') and len(newdata["password"]) > 4: acc.set_password(newdata['password']) acc.save() time.sleep(1) flash("Your account has been updated", 'info') return render_template('account/view.html', account=acc) else: if util.request_wants_json(): resp = make_response(json.dumps(acc.data, sort_keys=True, indent=4)) resp.mimetype = "application/json" return resp else: if current_user.is_super and acc.id != current_user.id: flash( 'Hi ' + current_user.id + ' - as a super user you can view and edit this account page. Be careful...', 'warning') return render_template('account/view.html', account=acc)
def record(rid=False): if rid: record = models.Catalogue.pull(rid.replace('.json','')) if record is None: abort(404) elif util.request_wants_json(): resp = make_response( record.json ) resp.mimetype = "application/json" return resp else: return render_template('catalogue.html',record=record) else: return render_template('catalogue.html')
def record(rid=False): if rid: record = models.Catalogue.pull(rid.replace('.json', '')) if record is None: abort(404) elif util.request_wants_json(): resp = make_response(record.json) resp.mimetype = "application/json" return resp else: return render_template('catalogue.html', record=record) else: return render_template('catalogue.html')
def username(username): acc = models.Account.pull(username) if acc is None: abort(404) elif ( request.method == 'DELETE' or ( request.method == 'POST' and request.values.get('submit', False) == 'Delete' ) ): if current_user.id != acc.id and not current_user.is_super: abort(401) else: conf = request.values.get("confirm") if conf is None or conf != "confirm": flash('check the box to confirm you really mean it!', "error") return render_template('account/view.html', account=acc) acc.delete() flash('Account ' + acc.id + ' deleted') return redirect(url_for('.index')) elif request.method == 'POST': if current_user.id != acc.id and not current_user.is_super: abort(401) newdata = request.json if request.json else request.values if newdata.get('id', False): if newdata['id'] != username: acc = models.Account.pull(newdata['id']) if request.values.get('submit', False) == 'Generate': acc.generate_api_key() for k, v in newdata.items(): if k not in ['marketing_consent', 'submit','password', 'role', 'confirm', 'reset_token', 'reset_expires', 'last_updated', 'created_date', 'id']: acc.data[k] = v if 'password' in newdata and not newdata['password'].startswith('sha1'): acc.set_password(newdata['password']) # only super users can re-write roles if "role" in newdata and current_user.is_super: new_roles = [r.strip() for r in newdata.get("role").split(",")] acc.set_role(new_roles) if "marketing_consent" in newdata: acc.set_marketing_consent(newdata["marketing_consent"] == "true") acc.save() flash("Record updated") return render_template('account/view.html', account=acc) else: if util.request_wants_json(): resp = make_response( json.dumps(acc.data, sort_keys=True, indent=4) ) resp.mimetype = "application/json" return resp else: # do an mget on the journals, so that we can present them to the user return render_template('account/view.html', account=acc)
def username(username): acc = models.Account.pull(username) if acc is None: abort(404) elif request.method == "DELETE" or (request.method == "POST" and request.values.get("submit", False) == "Delete"): if current_user.id != acc.id and not current_user.is_super: abort(401) else: conf = request.values.get("confirm") if conf is None or conf != "confirm": flash("check the box to confirm you really mean it!", "error") return render_template("account/view.html", account=acc) acc.delete() flash("Account " + acc.id + " deleted") return redirect(url_for(".index")) elif request.method == "POST": if current_user.id != acc.id and not current_user.is_super: abort(401) newdata = request.json if request.json else request.values if newdata.get("id", False): if newdata["id"] != username: acc = models.Account.pull(newdata["id"]) else: newdata["api_key"] = acc.data["api_key"] for k, v in newdata.items(): if k not in ["submit", "password", "role", "confirm"]: acc.data[k] = v if "password" in newdata and not newdata["password"].startswith("sha1"): if "confirm" in newdata and newdata["password"] == newdata["confirm"]: acc.set_password(newdata["password"]) else: flash("Passwords do not match", "error") return render_template("account/view.html", account=acc) # only super users can re-write roles if "role" in newdata and current_user.is_super: new_roles = [r.strip() for r in newdata.get("role").split(",")] acc.set_role(new_roles) acc.save() flash("Account updated", "success") return render_template("account/view.html", account=acc) else: if util.request_wants_json(): resp = make_response(json.dumps(acc.data, sort_keys=True, indent=4)) resp.mimetype = "application/json" return resp else: # do an mget on the journals, so that we can present them to the user return render_template("account/view.html", account=acc)
def detect(): if request.method == "GET": return render_template("detect.html") if request.method == "POST": url = request.values.get("url") if url is None: return render_template("detect.html") register = autodiscovery.discover(url) if util.request_wants_json(): resp = make_response( json.dumps(register) ) resp.mimetype = "application/json" return resp else: return render_template("repository.html", repo=register, searchurl=searchurl)
def pagemanager(path=''): url = '/' + path.lstrip('/').rstrip('/').replace('../',"") if url == '/': url = '/index' if url.endswith('.json'): url = url.replace('.json','') rec = models.Pages.pull_by_url(url) if rec is not None and rec.data.get('editable',False): if current_user.is_anonymous() and not rec.data.get('accessible',True): abort(401) else: return redirect(url_for('.edit',path=url)) elif ( ( request.method == 'DELETE' or ( request.method == 'POST' and request.form['submit'] == 'Delete' ) ) and not current_user.is_anonymous() ): if rec is None: abort(404) else: _sync_delete(rec) return "" elif request.method == 'POST' and not current_user.is_anonymous(): if rec is None: rec = models.Pages() rec.save_from_form(request) if app.config.get('CONTENT_FOLDER',False): _sync_fs_from_es(rec) return redirect(rec.data.get('url','/')) elif rec is None: if current_user.is_anonymous(): abort(404) else: return redirect(url_for('.settings', path=url)) elif request.method == 'GET': if current_user.is_anonymous() and not rec.data.get('accessible',True): abort(401) elif util.request_wants_json(): resp = make_response( rec.json ) resp.mimetype = "application/json" return resp else: try: content = render_template( 'pagemanager/content' + url, record=rec ) except: content = rec.data.get('content',"") return _render_page(rec,content) else: abort(401)
def username(username): acc = models.Account.pull(username) if acc is None: acc = models.Account.pull_by_email(username) if acc is not None: return redirect('/account/' + acc.id) if acc is None: acc = models.Account.pull_by_api_key(username) if acc is not None: return redirect('/account/' + acc.id) if acc is None: abort(404) elif ( request.method == 'DELETE' or ( request.method == 'POST' and request.values.get('submit',False) == 'Delete' ) ): if current_user.id != acc.id and not current_user.is_super: abort(401) else: acc.delete() flash('Account ' + acc.id + ' deleted') return redirect(url_for('.index')) elif request.method == 'POST': if current_user.id != acc.id and not current_user.is_super: abort(401) newdata = request.json if request.json else request.values if newdata.get('id',False): if newdata['id'] != username: acc = models.Account.pull(newdata['id']) else: newdata['api_key'] = acc.data['api_key'] for k, v in newdata.items(): if k not in ['submit','password']: acc.data[k] = v if 'password' in newdata and not newdata['password'].startswith('sha1') and len(newdata["password"]) > 4: acc.set_password(newdata['password']) acc.save() time.sleep(1) flash("Your account has been updated",'info') return render_template('account/view.html', account=acc) else: if util.request_wants_json(): resp = make_response( json.dumps(acc.data, sort_keys=True, indent=4) ) resp.mimetype = "application/json" return resp else: if current_user.is_super and acc.id != current_user.id: flash('Hi ' + current_user.id + ' - as a super user you can view and edit this account page. Be careful...', 'warning') return render_template('account/view.html', account=acc)
def adminitem(itype, iid=False): try: klass = getattr(models, itype[0].capitalize() + itype[1:]) if iid: if iid.endswith('.json'): iid = iid.replace('.json', '') if iid == "new": rec = None else: rec = klass.pull(iid) if request.method == 'GET': if not util.request_wants_json(): return render_template('admin/' + itype + '.html', record=rec) elif rec is not None: resp = make_response(rec.json) resp.mimetype = "application/json" return resp else: abort(404) elif ( rec and (request.method == 'DELETE' or (request.method == "POST" and request.form.get('submit', False).lower() == "delete"))): rec.delete() time.sleep(300) return redirect(url_for('.index')) elif request.method == 'POST': if rec is None: rec = klass() try: rec.save_from_form(request) except: newdata = request.json if request.json else request.values for k, v in newdata: if k not in ['submit']: rec.data[k] = v rec.save() return render_template('admin/' + itype + '.html', record=rec) else: return render_template('admin/' + itype + '.html') except: abort(404)
def adminitem(itype,iid=False): try: klass = getattr(models, itype[0].capitalize() + itype[1:] ) if iid: if iid.endswith('.json'): iid = iid.replace('.json','') if iid == "new": rec = None else: rec = klass.pull(iid) if request.method == 'GET': if not util.request_wants_json(): return render_template( 'admin/' + itype + '.html', record=rec ) elif rec is not None: resp = make_response( rec.json ) resp.mimetype = "application/json" return resp else: abort(404) elif ( rec and (request.method == 'DELETE' or ( request.method == "POST" and request.form.get('submit',False).lower() == "delete")) ): rec.delete() time.sleep(300) return redirect(url_for('.index')) elif request.method == 'POST': if rec is None: rec = klass() try: rec.save_from_form(request) except: newdata = request.json if request.json else request.values for k, v in newdata: if k not in ['submit']: rec.data[k] = v rec.save() return render_template('admin/' + itype + '.html', record=rec) else: return render_template('admin/' + itype + '.html') except: abort(404)
def media(): listing = os.listdir( mediadir ) listing = sorted(listing, key=str.lower) if util.request_wants_json(): response = make_response( json.dumps(listing,""," ") ) response.headers["Content-type"] = "application/json" return response else: usedin = {} for f in listing: # see if it is used in any records #try: r = models.Pages().query(q='*' + f + '*') usedin[f] = [i['_source']['url'] for i in r.get('hits',{}).get('hits',[])] #except: # usedin[f] = [] return render_template('media/media.html', files=listing, usedin=usedin)
def repository(repo_id): base = app.config.get("OARR_API_BASE_URL") if base is None: abort(500) client = OARRClient(base) if util.request_wants_json(): repo_id = repo_id.replace('.json','') try: record = client.get_record(repo_id) resp = make_response( record.json ) resp.mimetype = "application/json" return resp except: abort(404) else: record = client.get_record(repo_id) if record is None: abort(404) return render_template("repository.html", repo=record, searchurl=searchurl, search_similar=True)
def index(): if current_user.is_anonymous(): abort(401) users = models.Account.query() #{"sort":{'id':{'order':'asc'}}},size=1000000 if users['hits']['total'] != 0: accs = [models.Account.pull(i['_source']['id']) for i in users['hits']['hits']] # explicitly mapped to ensure no leakage of sensitive data. augment as necessary users = [] for acc in accs: user = {'id':acc.id} if 'created_date' in acc.data: user['created_date'] = acc.data['created_date'] users.append(user) if util.request_wants_json(): resp = make_response( json.dumps(users, sort_keys=True, indent=4) ) resp.mimetype = "application/json" return resp else: return render_template('account/users.html', users=users)
def index(): if not current_user.has_role("list_users"): abort(401) # users = models.Account.query() #{"sort":{'id':{'order':'asc'}}},size=1000000 accs = models.Account.iterall() # NOTE: this is only suitable if there is a small number of users - we will iterate through all of them here users = [] for acc in accs: # explicitly mapped to ensure no leakage of sensitive data. augment as necessary user = {'id': acc.id, "email": acc.email, "role": acc.role} if 'created_date' in acc.data: user['created_date'] = acc.data['created_date'] users.append(user) if util.request_wants_json(): resp = make_response(json.dumps(users, sort_keys=True, indent=4)) resp.mimetype = "application/json" return resp else: return render_template('account/users.html', users=users)
def organisation(org): base = app.config.get("OARR_API_BASE_URL") if base is None: abort(500) client = OARRClient(base) if util.request_wants_json(): org = org.replace('.json','') try: record = client.get_org(org) resp = make_response( record.json ) resp.mimetype = "application/json" return resp except: abort(404) else: try: record = client.get_org(org) return render_template("organisation.html", org=record.record) except: abort(404)
def media(): listing = os.listdir(mediadir) listing = sorted(listing, key=str.lower) if util.request_wants_json(): response = make_response(json.dumps(listing, "", " ")) response.headers["Content-type"] = "application/json" return response else: usedin = {} for f in listing: # see if it is used in any records #try: r = models.Pages().query(q='*' + f + '*') usedin[f] = [ i['_source']['url'] for i in r.get('hits', {}).get('hits', []) ] #except: # usedin[f] = [] return render_template('media/media.html', files=listing, usedin=usedin)
def username(username): acc = models.Account.pull(username) if acc is None: abort(404) elif (request.method == 'DELETE' or (request.method == 'POST' and request.values.get('submit', False) == 'Delete')): if current_user.id != acc.id and not current_user.is_super: abort(401) else: acc.delete() flash('Account ' + acc.id + ' deleted') return redirect(url_for('.index')) elif request.method == 'POST': if current_user.id != acc.id and not current_user.is_super: abort(401) newdata = request.json if request.json else request.values if newdata.get('id', False): if newdata['id'] != username: acc = models.Account.pull(newdata['id']) else: newdata['api_key'] = acc.data['api_key'] for k, v in newdata.items(): if k not in ['submit', 'password']: acc.data[k] = v if 'password' in newdata and not newdata['password'].startswith( 'sha1'): acc.set_password(newdata['password']) acc.save() flash("Record updated") return render_template('account/view.html', account=acc) else: if util.request_wants_json(): resp = make_response(json.dumps(acc.data, sort_keys=True, indent=4)) resp.mimetype = "application/json" return resp else: return render_template('account/view.html', account=acc)
def username(username): acc = models.Account.pull(username) if ( request.method == 'POST' and request.values.get('submit','') == 'delete' ) or request.method == 'DELETE': if not auth.user.update(acc,current_user): abort(401) if acc: acc.delete() flash('Account deleted') time.sleep(1) return redirect('/account') elif request.method == 'POST': if not auth.user.update(acc,current_user): abort(401) info = request.json if request.json else request.values if info.get('id',False): if info['id'] != username: acc = models.Account.pull(info['id']) else: info['api_key'] = acc.data['api_key'] for k, v in info.items(): if k not in ['submit']: acc.data[k] = v if 'password' in info and not info['password'].startswith('sha1'): acc.set_password(info['password']) acc.save() time.sleep(1) flash('Account updated') return redirect('/account/' + username) else: if not acc: abort(404) if util.request_wants_json(): if not auth.user.update(acc,current_user): abort(401) resp = make_response( json.dumps(acc.data, sort_keys=True, indent=4) ) resp.mimetype = "application/json" return resp else: return render_template('account/view.html', account=acc)
def index(): if current_user.is_anonymous(): abort(401) users = models.Account.query( ) #{"sort":{'id':{'order':'asc'}}},size=1000000 if users['hits']['total'] != 0: accs = [ models.Account.pull(i['_source']['id']) for i in users['hits']['hits'] ] # explicitly mapped to ensure no leakage of sensitive data. augment as necessary users = [] for acc in accs: user = {'id': acc.id} if 'created_date' in acc.data: user['created_date'] = acc.data['created_date'] users.append(user) if util.request_wants_json(): resp = make_response(json.dumps(users, sort_keys=True, indent=4)) resp.mimetype = "application/json" return resp else: return render_template('account/users.html', users=users)
def username(username): acc = models.Account.pull(username) if request.method == 'DELETE': if not auth.user.update(acc,current_user): abort(401) if acc: acc.delete() return '' elif request.method == 'POST': if not auth.user.update(acc,current_user): abort(401) info = request.json if request.json else request.values if info.get('id',False): if info['id'] != username: acc = models.Account.pull(info['id']) else: info['api_key'] = acc.data['api_key'] for k, v in info.items(): if k == 'password': acc.set_password(info['password']) elif k not in ['submit']: acc.data[k] = info[k] acc.save() flash('Account updated', "success") return render_template('account/view.html', account=acc) else: if not acc: abort(404) if util.request_wants_json(): if not auth.user.update(acc,current_user): abort(401) resp = make_response( json.dumps(acc.data, sort_keys=True, indent=4) ) resp.mimetype = "application/json" return resp else: return render_template('account/view.html', account=acc)
def pagemanager(path=''): url = '/' + path.lstrip('/').rstrip('/').replace('../',"") if url == '/': url = '/index' if url.endswith('.json'): url = url.replace('.json','') rec = models.Pages.pull_by_url(url) if rec is not None and rec.data.get('editable',False): return redirect(url_for('.edit',path=url)) elif ( ( request.method == 'DELETE' or ( request.method == 'POST' and request.form['submit'] == 'Delete' ) ) and not current_user.is_anonymous() ): if rec is None: abort(404) else: _sync_delete(rec) return "" elif request.method == 'POST' and not current_user.is_anonymous(): if rec is None: rec = models.Pages() rec.save_from_form(request) if app.config.get('CONTENT_FOLDER',False): _sync_fs_from_es(rec) return redirect(rec.data.get('url','/')) elif rec is None: if current_user.is_anonymous(): abort(404) else: return redirect(url_for('.settings', path=url)) elif request.method == 'GET': if current_user.is_anonymous() and not rec.data.get('accessible',True): abort(401) elif util.request_wants_json(): resp = make_response( rec.json ) resp.mimetype = "application/json" return resp else: try: content = render_template( 'pagemanager/content/contentMine.wiki/' + urlend + '.md', record=rec ) except: try: content = render_template( 'pagemanager/content' + url, record=rec ) except: content = rec.data.get('content',"") content = markdown.markdown(content) content = re.sub(r'\[\[(.*?)\]\]',r'<a href="/\1">\1</a>',content) # if an embedded file url has been provided, embed it in content if rec.data.get('embed', False): if ( rec.data['embed'].find('/pub?') != -1 or rec.data['embed'].find('docs.google.com') != -1 ): content += '<iframe id="embedded" src="' + rec.data['embed'] content += '" width="100%" height="1000" ' content += 'style="border:none;"></iframe>' else: content += '<iframe id="embedded" ' content += 'src="http://docs.google.com/viewer?url=' content += urllib.quote_plus(rec.data['embed']) content += '&embedded=true" width="100%" height="1000" ' content += 'style="border:none;"></iframe>' # TODO: try adding js dynamic includes server-side? return render_template( 'pagemanager/index.html', content=content, record=rec ) else: abort(401)
def pagemanager(path=''): url = '/' + path.lstrip('/').rstrip('/').replace('../', "") if url == '/': url = '/index' if url.endswith('.json'): url = url.replace('.json', '') rec = models.Pages.pull_by_url(url) # check if a wiki page exists for the current end path, even though no record exists if rec is None: try: if url == '/wiki': urlend = 'Home' else: urlend = url.replace('/wiki', '').split('/')[-1].replace(' ', '-') fl = open(contentdir + "/contentMine.wiki/" + urlend + '.md', 'r') p = models.Pages() p.data = { "url": url, "title": urlend, "content": fl.read(), "author": "system" } fl.close() p.save() time.sleep(1) return redirect(url) except: pass if rec is not None and rec.data.get('editable', False): return redirect(url_for('.edit', path=url)) elif ((request.method == 'DELETE' or (request.method == 'POST' and request.form['submit'] == 'Delete')) and not current_user.is_anonymous()): if rec is None: abort(404) else: _sync_delete(rec) return "" elif request.method == 'POST' and not current_user.is_anonymous(): if rec is None: rec = models.Pages() rec.save_from_form(request) if app.config.get('CONTENT_FOLDER', False): _sync_fs_from_es(rec) return redirect(rec.data.get('url', '/')) elif rec is None: if current_user.is_anonymous(): abort(404) else: return redirect(url_for('.settings', path=url)) elif request.method == 'GET': if current_user.is_anonymous() and not rec.data.get( 'accessible', True): abort(401) elif util.request_wants_json(): resp = make_response(rec.json) resp.mimetype = "application/json" return resp else: try: content = render_template( 'pagemanager/content/contentMine.wiki/' + urlend + '.md', record=rec) except: try: content = render_template('pagemanager/content' + url, record=rec) except: content = rec.data.get('content', "") content = markdown.markdown(content) content = re.sub(r'\[\[(.*?)\]\]', r'<a href="/\1">\1</a>', content) # if an embedded file url has been provided, embed it in content if rec.data.get('embed', False): if (rec.data['embed'].find('/pub?') != -1 or rec.data['embed'].find('docs.google.com') != -1): content += '<iframe id="embedded" src="' + rec.data['embed'] content += '" width="100%" height="1000" ' content += 'style="border:none;"></iframe>' else: content += '<iframe id="embedded" ' content += 'src="http://docs.google.com/viewer?url=' content += urllib.quote_plus(rec.data['embed']) content += '&embedded=true" width="100%" height="1000" ' content += 'style="border:none;"></iframe>' # TODO: try adding js dynamic includes server-side? return render_template('pagemanager/index.html', content=content, record=rec) else: abort(401)
def matching(mainorg, suffix=None): # get any match parameters provided in the request if request.json: project = request.json.get('project',[]) person = request.json.get('person',[]) keyword = request.json.get('keyword',[]) url = request.json.get('url',[]) else: project = request.values.get('project',"").split(',') person = request.values.get('person',"").split(',') keyword = request.values.get('keyword',"").split(',') url = request.values.get('url',"").split(',') # process the match parameters into search parameters # TODO: should make this a dict by weightings and include them from below # and also enable end users to prioritise certain keywords or types # TODO: perhaps also add maximums from each source, or maximum length of params overall? params = [] for k in keyword: if k not in params and len(k) > 0: params.append(k) for pr in project: # match project title to a project if len(pr) > 0: r = models.Record.pull_by_key('project.title',pr) if r is not None: extract = mine.mine_project_record(r) count = 0 # TODO: proper control of how many url-extracted keywords to include for e in extract: for i in extract[e]: count += 1 if count <= 10: if i not in params: params.append(i.replace('(','')) # TODO: regex out anything non-az09 for u in url: if len(u) > 0: if not u.startswith('http://') and not u.startswith('https://'): u = 'http://' + u #try: r = requests.get(u) clean = mine.html_text(r.text) extract = mine.full_extract(web_text=clean, web_weight=5) count = 0 # TODO: proper control of how many url-extracted keywords to include for e in extract: for i in extract[e]: count += 1 if count <= 10: if i not in params: params.append(i) #except: # pass potential = [] nodes = [{"type":"param","color":"#666","value":1,"label":i} for i in params] links = [] linked = [] # perform the search with the defined params and build a list of matching orgs if len(params) > 0 or len(person) > 0: qry = { 'query': { 'bool': { 'must': [ { 'query_string': { 'query': " OR ".join(params) } } ], 'must_not': [ { 'term': { 'collaboratorOrganisation.canonical.exact': mainorg } } ] } }, "size": 1000, "facets" : { "collaborators" : { "terms" : { "field" : "collaboratorOrganisation.canonical.exact", "size" : 100 } } } } # add people, if any , to the search if len(person) > 0: qry['query']['bool']['should'] = [] qry['query']['bool']['minimum_should_match'] = 1 for p in person: if len(p) > 0: qry['query']['bool']['should'].append({ 'term': { 'collaboratorPerson.canonical.exact': p } }) # get the collaborator orgs found in the search results # strip the ones who already collaborate with the mainorg # then find out some info about the remaining ones r = models.Record.query(q=qry) collabs = [i['term'] for i in r.get("facets", {}).get("collaborators", {}).get("terms", [])] collaboration_definition = ["leadro", "principal_investigator", "co_investigator", "fellow"] # FIXME: collaboration definition is going to need to pervade the code cs = [i['term'] for i in models.Record().ordered_collaborators(mainorg,10000,collaboration_definition)] for collab in collabs: if collab not in cs and len(potential) < 10: p = organisation(collab, raw=True) p['related'] = [] for i in r.get("hits", {}).get("hits",[]): canonicals = [l.get('canonical','') for l in i['_source'].get('collaboratorOrganisation',[])] title = i['_source']['project']['title'] if collab in canonicals and mainorg not in canonicals and title not in p['related']: p['related'].append({"title":title,"id":i['_id'],"abstract":i['_source']['project'].get('abstractText',"")}) for k, param in enumerate(params): if param in i['_source']['project']['title'] + " " + i['_source']['project'].get('abstractText',""): if param + collab not in linked: links.append({'source':k,'target':len(nodes)}) linked.append(param + collab) potential.append(p) nodes.append({"type":"org","label":collab,"color":"#f9ce50","value":len(p['related'])+2}) matchinfo = { "new_potential": potential, "nodes": nodes, "links": links, "params": params, "person": person, "project": project, "keyword": keyword, "url": url } if util.request_wants_json(): resp = make_response(json.dumps(matchinfo)) resp.mimetype = "application/json" return resp elif suffix == "csv": output = StringIO.StringIO() writer = csv.writer(output) if len(potential) > 0: headers = ["name","projects","collaborators","funding","related"] writer.writerow(headers) for p in potential: r = [ p['name'], p['projects'], p['collaborators'], p['totalfunding'], len(p['related']) ] writer.writerow(r) resp = make_response(output.getvalue()) resp.mimetype = "text/csv" resp.headers['Content-Disposition'] = 'attachment; filename="' + mainorg + '_new_potential_report.csv"' return resp else: return render_template('organisation/match.html', mainorg=mainorg, matchinfo=matchinfo)
def organisation(mainorg, raw=False): """ Any request to a specific organisation's home page """ # TODO: # list all this orgs projects # list a blurb and website about this org # list the main contact of this org (and perhaps other users) # offer ability to update the page about this org # show this org snapshot data, top projects, recent funding by years, pubs # offer a download report overview of this org logo = mainorg.lower().replace(' ','_').replace("'",'').replace('university_','').replace('_university','').replace('_of','').replace('of_','').replace('_the','').replace('the_','').replace('_and','').replace('and_','').replace('_for','').replace('for_','').replace('_.','.') + '.png'; logofolder = os.path.dirname(os.path.abspath( __file__ )).replace('/view','/static/logos') logos=os.listdir(logofolder) if logo not in logos: logo = '' else: logo = '/static/logos/' + logo # logo = "" # FIXME: should be in the models layer qry = { "query": { "term": { "collaboratorOrganisation.canonical.exact": mainorg } }, "size": 1, "facets": { "collaborators":{ "terms_stats" : { "key_field" : "collaboratorOrganisation.canonical.exact", "value_field" : "project.fund.valuePounds", "size" : 0 } }, "value_stats" : { "statistical" : { "field" : "project.fund.valuePounds" } } } } r = models.Record.query(q=qry) org = { 'name': mainorg, 'logo': logo, 'projects': r.get('hits',{}).get('total',0), 'collaborators': len(r.get('facets',{}).get('collaborators',{}).get('terms',[])) - 1, 'totalfunding': "{:,.0f}".format(r.get('facets',{}).get('value_stats',{}).get('total',0)) } # TODO: post codes should perhaps be processed into the index data, to save # processing them here # get post code - lat long lookup table pcll = json.load(open("postcodes.json")) try: orgs = r.get('hits',{}).get('hits',[])[0]['_source']['collaboratorOrganisation'] for o in orgs: if o['canonical'] == mainorg: outcode = o['organisation']['address']['postCode'].replace(' ','') if len(outcode) == 7: outcode = outcode[0:4] elif len(outcode) == 6: outcode = outcode[0:3] pc = pcll[outcode] org['lat'] = pc['lat'] org['lng'] = pc['lng'] except: org['lat'] = 0 org['lng'] = 0 # get similar names for user fixing # TODO: this should be in model too similar = dropdowns('record','collaboratorOrganisation.canonical') n = mainorg.lower().replace('university','').replace('college','').replace('of','').replace('the','').replace(' ','').replace('.','') checklist = [] for s in similar: if mainorg != s and n in s.lower(): checklist.append(s) # TODO: should really have an org object with the above info in it and it # should be passed to the page instead of the mainorg string if raw: return org elif util.request_wants_json(): resp = make_response(json.dumps(org)) resp.mimetype = "application/json" return resp else: return render_template('organisation/org.html', org=org, checklist=checklist)
def adminitem(itype, iid=False): #try: klass = getattr(models, itype[0].capitalize() + itype[1:]) if iid: if iid.endswith('.json'): iid = iid.replace('.json', '') if iid == "new": rec = None else: rec = klass.pull(iid) if request.method == 'GET': if not util.request_wants_json(): return render_template( 'admin/' + itype + '.html', record=rec, dropdowns={ "clpeople": dropdowns('account', 'id'), "partners": [ i['_source']['id'] for i in models.Account.query( terms={ "partner": "yes" }, size=1000).get('hits', {}).get('hits', []) ], "seniors": [ i['_source']['id'] for i in models.Account.query( terms={ "senior": "yes" }, size=1000).get('hits', {}).get('hits', []) ], "contacts": dropdowns('project', ['externals', 'customer', 'funder']), "tags": dropdowns('project', ['tags']) }) elif rec is not None: resp = make_response(rec.json) resp.mimetype = "application/json" return resp else: abort(404) elif (rec and (request.method == 'DELETE' or (request.method == "POST" and request.form.get('submit', False).lower() == "delete"))): rec.delete() flash('Deleted') return redirect(url_for('.index')) elif request.method == 'POST': if rec is None: rec = klass() rec.save_from_form(request) if iid == "new": flash('Created') return redirect(url_for('.index') + itype + '/' + rec.id) else: flash('Updated') return redirect(url_for('.index') + itype + '/' + iid) else: return redirect(url_for('.index') + itype + '/new')
def contribute(): detectdone = False dup = False if request.method == 'GET': # check to see if this is an update if 'updaterequest' in request.values: # get record from OARR try: base = app.config.get("OARR_API_BASE_URL") if base is None: abort(500) client = OARRClient(base) record = client.get_record(request.values["updaterequest"]).raw if "opendoar" not in record["admin"]: record["admin"]["opendoar"] = {} record["admin"]["opendoar"]["updaterequest"] = request.values["updaterequest"] detectdone = True except: abort(404) # check for a url request param elif 'url' in request.values: # if there is one, then try to set the initial object if len(request.values['url']) != 0: try: register = autodiscovery.discover(request.values['url']) record = register.raw for k, v in util.defaultrecord['register']['metadata'][0]['record'].iteritems(): if k not in record.get('register',{}).get('metadata',[{"record":{}}])[0]['record']: record['register']['metadata'][0]['record'][k] = v except: record = util.defaultrecord # check if there is already a record with this url dup = rawduplicate(request.values['url'],raw=True) else: record = util.defaultrecord detectdone = True else: # otherwise set a default initial object record = util.defaultrecord if util.request_wants_json(): resp = make_response( json.dumps({"record":record}) ) resp.mimetype = "application/json" return resp else: return render_template("contribute.html", record=record, detectdone=detectdone, duplicate=dup) elif request.method == 'POST': base = app.config.get("OARR_API_BASE_URL") apikey = app.config.get("OARR_API_KEY") if base is None or apikey is None: abort(500) client = OARRClient(base, apikey) record = client.prep_record(util.defaultrecord,request) if 'updaterequest' not in record['admin']['opendoar']: record['admin']['opendoar']['newcontribution'] = True saved = client.save_record(record) if saved: flash('Thank you very much for your submission. Your request will be processed as soon as possible, and usually within three working days.', 'success') return redirect('/') else: flash('Sorry, there was a problem saving your submission. Please try again.', 'error') return redirect('/contribute')
def pagemanager(path=""): url = "/" + path.lstrip("/").rstrip("/").replace("../", "") if url == "/": url = "/index" if url.endswith(".json"): url = url.replace(".json", "") rec = models.Pages.pull_by_url(url) # check if a wiki page exists for the current end path, even though no record exists if rec is None: try: if url == "/wiki": urlend = "Home" else: urlend = url.replace("/wiki", "").split("/")[-1].replace(" ", "-") fl = open(contentdir + "/contentMine.wiki/" + urlend + ".md", "r") p = models.Pages() p.data = {"url": url, "title": urlend, "content": fl.read(), "author": "system"} fl.close() p.save() time.sleep(1) return redirect(url) except: pass if rec is not None and rec.data.get("editable", False): return redirect(url_for(".edit", path=url)) elif ( request.method == "DELETE" or (request.method == "POST" and request.form["submit"] == "Delete") ) and not current_user.is_anonymous(): if rec is None: abort(404) else: _sync_delete(rec) return "" elif request.method == "POST" and not current_user.is_anonymous(): if rec is None: rec = models.Pages() rec.save_from_form(request) if app.config.get("CONTENT_FOLDER", False): _sync_fs_from_es(rec) return redirect(rec.data.get("url", "/")) elif rec is None: if current_user.is_anonymous(): abort(404) else: return redirect(url_for(".settings", path=url)) elif request.method == "GET": if current_user.is_anonymous() and not rec.data.get("accessible", True): abort(401) elif util.request_wants_json(): resp = make_response(rec.json) resp.mimetype = "application/json" return resp else: try: content = render_template("pagemanager/content/contentMine.wiki/" + urlend + ".md", record=rec) except: try: content = render_template("pagemanager/content" + url, record=rec) except: content = rec.data.get("content", "") content = markdown.markdown(content) content = re.sub(r"\[\[(.*?)\]\]", r'<a href="/\1">\1</a>', content) # if an embedded file url has been provided, embed it in content if rec.data.get("embed", False): if rec.data["embed"].find("/pub?") != -1 or rec.data["embed"].find("docs.google.com") != -1: content += '<iframe id="embedded" src="' + rec.data["embed"] content += '" width="100%" height="1000" ' content += 'style="border:none;"></iframe>' else: content += '<iframe id="embedded" ' content += 'src="http://docs.google.com/viewer?url=' content += urllib.quote_plus(rec.data["embed"]) content += '&embedded=true" width="100%" height="1000" ' content += 'style="border:none;"></iframe>' # TODO: try adding js dynamic includes server-side? return render_template("pagemanager/index.html", content=content, record=rec) else: abort(401)
def record(uuid=None): base = app.config.get("OARR_API_BASE_URL") apikey = app.config.get("OARR_API_KEY") if base is None or apikey is None: abort(500) client = OARRClient(base, apikey) detectdone = False dup = False if request.method == "GET": if uuid is None or uuid == "new": # check for a url request param if "url" in request.values: # if there is one, then try to set the initial object if len(request.values["url"]) != 0: try: register = autodiscovery.discover(request.values["url"]) record = register.raw for k, v in util.defaultrecord["register"]["metadata"][0]["record"].iteritems(): if k not in record.get("register", {}).get("metadata", [{"record": {}}])[0]["record"]: record["register"]["metadata"][0]["record"][k] = v except: record = util.defaultrecord else: record = util.defaultrecord detectdone = True # check if there is already a record with this url dup = duplicate(request.values["url"], raw=True) else: # otherwise set a default initial object record = util.defaultrecord else: # get record from OARR try: record = client.get_record(uuid.replace(".json", "")).raw detectdone = True # check if there is already a record with this url dup = duplicate(record["register"]["metadata"][0]["record"]["url"], raw=True) except: abort(404) if util.request_wants_json(): resp = make_response(json.dumps({"record": record})) resp.mimetype = "application/json" return resp else: return render_template("contribute.html", record=record, detectdone=detectdone, duplicate=dup) elif (request.method == "POST" and request.values.get("submit", "") == "Delete") or request.method == "DELETE": record = client.get_record(uuid) if record is None: abort(404) client.delete_record(uuid) time.sleep(1) flash("Record delete") return redirect(url_for(".index")) elif request.method == "POST": if uuid == "new": # save the new record to OARR record = client.prep_record(util.defaultrecord, request) saved = client.save_record(record) if saved: flash("New record created", "success") return redirect(url_for(".index")) else: flash("Sorry, the attempt to create a new record was unsuccessful", "error") return redirect("/admin/record/new") else: rec = client.get_record(uuid) if rec is None: abort(404) # remove any newcontribution tag on first save via admin if "newcontribution" in rec.raw.get("admin", {}).get("opendoar", {}): del rec.raw["admin"]["opendoar"]["newcontribution"] # if this is an update acceptance, do the update if "updaterequest" in rec.raw.get("admin", {}).get("opendoar", {}): # get the original record, prep it with the update, delete the update request record forupdate = client.get_record(rec.raw["admin"]["opendoar"]["updaterequest"]) if forupdate is None: flash("Sorry, an original record cannot be found to correspond with this update request.", "error") return redirect("/admin/record/" + uuid) else: record = client.prep_record(forupdate.raw, request) try: del record["admin"]["opendoar"]["updaterequest"] saved = client.save_record(record) if saved: client.delete_record(uuid) time.sleep(1) flash( "This original record has been successfully updated, and the update request record has been deleted.", "success", ) return redirect("/admin/record/" + str(record["id"])) else: flash( "Sorry, there was an error. Your changes have not been saved. Please try again.", "error", ) return redirect("/admin/record/" + uuid) except: flash("Sorry, there was an error. Your changes have not been saved. Please try again.", "error") return redirect("/admin/record/" + uuid) # otherwise save the record changes to OARR else: # do whatever needs done here to update the record from the form input record = client.prep_record(rec.raw, request) saved = client.save_record(record) if saved: detectdone = True time.sleep(1) flash("Record has been updated", "success") return render_template("contribute.html", record=record, detectdone=detectdone, duplicate=dup) else: flash("Sorry, there was an error. Your changes have not been saved. Please try again.", "error") return redirect("/admin/record/" + uuid)
def pagemanager(path=''): url = '/' + path.lstrip('/').rstrip('/').replace('../', "") if url == '/': url = '/index' if url.endswith('.json'): url = url.replace('.json', '') rec = models.Pages.pull_by_url(url) if rec is not None and rec.data.get('editable', False): return redirect(url_for('.edit', path=url)) elif ((request.method == 'DELETE' or (request.method == 'POST' and request.form['submit'] == 'Delete')) and not current_user.is_anonymous()): if rec is None: abort(404) else: _sync_delete(rec) return "" elif request.method == 'POST' and not current_user.is_anonymous(): if rec is None: rec = models.Pages() rec.save_from_form(request) if app.config.get('CONTENT_FOLDER', False): _sync_fs_from_es(rec) return redirect(rec.data.get('url', '/')) elif rec is None: if current_user.is_anonymous(): abort(404) else: return redirect(url_for('.settings', path=url)) elif request.method == 'GET': if current_user.is_anonymous() and not rec.data.get( 'accessible', True): abort(401) elif util.request_wants_json(): resp = make_response(rec.json) resp.mimetype = "application/json" return resp else: try: content = render_template('pagemanager/content' + url, record=rec) except: content = rec.data.get('content', "") content = markdown.markdown(content) # if an embedded file url has been provided, embed it in content if rec.data.get('embed', False): if (rec.data['embed'].find('/pub?') != -1 or rec.data['embed'].find('docs.google.com') != -1): content += '<iframe id="embedded" src="' + rec.data['embed'] content += '" width="100%" height="1000" ' content += 'style="border:none;"></iframe>' else: content += '<iframe id="embedded" ' content += 'src="http://docs.google.com/viewer?url=' content += urllib.quote_plus(rec.data['embed']) content += '&embedded=true" width="100%" height="1000" ' content += 'style="border:none;"></iframe>' # TODO: try adding js dynamic includes server-side? return render_template('pagemanager/index.html', content=content, record=rec) else: abort(401)