def index():
if current_user.is_anonymous():
abort(401)
if not current_user.is_super:
return redirect('/account/' + current_user.id)
users = models.Account.query(q={"query":{"match_all":{}},"sort":{'id.exact':{'order':'asc'}}, "size":100000})
userstats = {
"super_user": 0,
"do_admin": 0,
"view_admin": 0
}
if users['hits']['total'] != 0:
accs = [models.Account.pull(i['_source']['id']) for i in users['hits']['hits']]
# explicitly mapped to ensure no leakage of sensitive data. augment as necessary
users = []
for acc in accs:
if acc.id in app.config['SUPER_USER']: userstats['super_user'] += 1
elif acc.data.get('do_admin',"") != "": userstats["do_admin"] += 1
elif acc.data.get('view_admin',"") != "": userstats["view_admin"] += 1
user = {'id':acc.id}
if 'created_date' in acc.data:
user['created_date'] = acc.data['created_date']
users.append(user)
if util.request_wants_json():
resp = make_response( json.dumps(users, sort_keys=True, indent=4) )
resp.mimetype = "application/json"
return resp
else:
return render_template('account/all.html', users=users, userstats=userstats)
def story(sid):
story = models.Blocked.pull(sid.replace('.json',''))
#story = models.Record.pull(sid.replace('.json',''))
turl = None
cat = None
user = None
if story is not None:
about = story.about(sid.replace('.json',''), exclude=story.id)
turl = story.data['url']
user = models.Account.pull(story.data['author'])
else:
about = models.Record.about(sid.replace('.json',''))
if about.get('hits',{}).get('total',0) == 0:
flash('Sorry, there was no story found about ' + sid, 'warning')
return redirect('/story')
else:
turl = about['hits']['hits'][0]['_source']['url']
if turl is not None:
c = models.Catalogue.pull_by_url(turl)
if c is not None:
cat = c.data
if util.request_wants_json() or sid.endswith('.json'):
resp = make_response( story.json )
resp.mimetype = "application/json"
return resp
else:
# TODO this should not pass all user data, some of that should be stored with the story
return render_template("story.html", story=story, about=about, catalogue=cat, user=user.data)
def username(username):
acc = models.Account.pull(username)
if acc is None:
abort(404)
elif ( request.method == 'DELETE' or
( request.method == 'POST' and
request.values.get('submit','').lower() == 'delete' ) ):
acc.delete()
flash('Account ' + acc.id + ' deleted')
return redirect(url_for('.index'))
elif request.method == 'POST':
newdata = request.json if request.json else request.values
if newdata.get('id',False):
if newdata['id'] != username:
acc = models.Account.pull(newdata['id'])
else:
newdata['api_key'] = acc.data['api_key']
for k, v in newdata.items():
if k not in ['submit','password']:
acc.data[k] = v
if 'password' in newdata and not newdata['password'].startswith('sha1'):
acc.set_password(newdata['password'])
acc.save()
flash("Record updated")
return render_template('account/view.html', account=acc)
else:
if util.request_wants_json():
resp = make_response(
json.dumps(acc.data, sort_keys=True, indent=4) )
resp.mimetype = "application/json"
return resp
else:
return render_template('account/view.html', account=acc)
def story(sid):
story = models.Blocked.pull(sid.replace('.json',''))
#story = models.Record.pull(sid.replace('.json',''))
turl = None
cat = None
user = None
if story is not None:
about = story.about(sid.replace('.json',''), exclude=story.id)
turl = story.data['url']
user = models.Account.pull(story.data['author'])
else:
about = models.Record.about(sid.replace('.json',''))
if about.get('hits',{}).get('total',0) == 0:
flash('Sorry, there was no story found about ' + sid, 'warning')
return redirect('/story')
else:
turl = about['hits']['hits'][0]['_source']['url']
if turl is not None:
c = models.Catalogue.pull_by_url(turl)
if c is not None:
cat = c.data
if util.request_wants_json() or sid.endswith('.json'):
resp = make_response( story.json )
resp.mimetype = "application/json"
return resp
else:
# TODO this should not pass all user data, some of that should be stored with the story
return render_template("story.html", story=story, about=about, catalogue=cat, user=user.data)
def story(sid):
story = models.Record.pull(sid.replace('.json',''))
if story is None: abort(404)
if util.request_wants_json() or sid.endswith('.json'):
resp = make_response( story.json )
resp.mimetype = "application/json"
return resp
else:
return render_template("story.html", story=story.json)
def username(username):
acc = models.Account.pull(username)
if acc is None:
abort(404)
elif (request.method == 'DELETE'
or (request.method == 'POST'
and request.values.get('submit', False) == 'Delete')):
if current_user.id != acc.id and not current_user.is_super:
abort(401)
else:
conf = request.values.get("confirm")
if conf is None or conf != "confirm":
flash('check the box to confirm you really mean it!', "error")
return render_template('account/view.html', account=acc)
acc.delete()
flash('Account ' + acc.id + ' deleted')
return redirect(url_for('.index'))
elif request.method == 'POST':
if current_user.id != acc.id and not current_user.is_super:
abort(401)
newdata = request.json if request.json else request.values
if newdata.get('id', False):
if newdata['id'] != username:
acc = models.Account.pull(newdata['id'])
else:
newdata['api_key'] = acc.data['api_key']
for k, v in newdata.items():
if k not in [
'submit', 'password', 'role', 'confirm', 'reset_token',
'reset_expires', 'last_updated', 'created_date', 'id'
]:
acc.data[k] = v
if 'password' in newdata and not newdata['password'].startswith(
'sha1'):
acc.set_password(newdata['password'])
# only super users can re-write roles
if "role" in newdata and current_user.is_super:
new_roles = [r.strip() for r in newdata.get("role").split(",")]
acc.set_role(new_roles)
acc.save()
flash("Record updated")
return render_template('account/view.html', account=acc)
else:
if util.request_wants_json():
resp = make_response(json.dumps(acc.data, sort_keys=True,
indent=4))
resp.mimetype = "application/json"
return resp
else:
# do an mget on the journals, so that we can present them to the user
return render_template('account/view.html', account=acc)
def username(username):
acc = models.Account.pull(username)
if acc is None:
acc = models.Account.pull_by_email(username)
if acc is not None: return redirect('/account/' + acc.id)
if acc is None:
acc = models.Account.pull_by_api_key(username)
if acc is not None: return redirect('/account/' + acc.id)
if acc is None:
abort(404)
elif (request.method == 'DELETE'
or (request.method == 'POST'
and request.values.get('submit', False) == 'Delete')):
if current_user.id != acc.id and not current_user.is_super:
abort(401)
else:
acc.delete()
flash('Account ' + acc.id + ' deleted')
return redirect(url_for('.index'))
elif request.method == 'POST':
if current_user.id != acc.id and not current_user.is_super:
abort(401)
newdata = request.json if request.json else request.values
if newdata.get('id', False):
if newdata['id'] != username:
acc = models.Account.pull(newdata['id'])
else:
newdata['api_key'] = acc.data['api_key']
for k, v in newdata.items():
if k not in ['submit', 'password']:
acc.data[k] = v
if 'password' in newdata and not newdata['password'].startswith(
'sha1') and len(newdata["password"]) > 4:
acc.set_password(newdata['password'])
acc.save()
time.sleep(1)
flash("Your account has been updated", 'info')
return render_template('account/view.html', account=acc)
else:
if util.request_wants_json():
resp = make_response(json.dumps(acc.data, sort_keys=True,
indent=4))
resp.mimetype = "application/json"
return resp
else:
if current_user.is_super and acc.id != current_user.id:
flash(
'Hi ' + current_user.id +
' - as a super user you can view and edit this account page. Be careful...',
'warning')
return render_template('account/view.html', account=acc)
def record(rid=False):
if rid:
record = models.Catalogue.pull(rid.replace('.json',''))
if record is None:
abort(404)
elif util.request_wants_json():
resp = make_response( record.json )
resp.mimetype = "application/json"
return resp
else:
return render_template('catalogue.html',record=record)
else:
return render_template('catalogue.html')
def record(rid=False):
if rid:
record = models.Catalogue.pull(rid.replace('.json', ''))
if record is None:
abort(404)
elif util.request_wants_json():
resp = make_response(record.json)
resp.mimetype = "application/json"
return resp
else:
return render_template('catalogue.html', record=record)
else:
return render_template('catalogue.html')
def username(username):
acc = models.Account.pull(username)
if acc is None:
abort(404)
elif ( request.method == 'DELETE' or
( request.method == 'POST' and request.values.get('submit', False) == 'Delete' ) ):
if current_user.id != acc.id and not current_user.is_super:
abort(401)
else:
conf = request.values.get("confirm")
if conf is None or conf != "confirm":
flash('check the box to confirm you really mean it!', "error")
return render_template('account/view.html', account=acc)
acc.delete()
flash('Account ' + acc.id + ' deleted')
return redirect(url_for('.index'))
elif request.method == 'POST':
if current_user.id != acc.id and not current_user.is_super:
abort(401)
newdata = request.json if request.json else request.values
if newdata.get('id', False):
if newdata['id'] != username:
acc = models.Account.pull(newdata['id'])
if request.values.get('submit', False) == 'Generate':
acc.generate_api_key()
for k, v in newdata.items():
if k not in ['marketing_consent', 'submit','password', 'role', 'confirm', 'reset_token', 'reset_expires', 'last_updated', 'created_date', 'id']:
acc.data[k] = v
if 'password' in newdata and not newdata['password'].startswith('sha1'):
acc.set_password(newdata['password'])
# only super users can re-write roles
if "role" in newdata and current_user.is_super:
new_roles = [r.strip() for r in newdata.get("role").split(",")]
acc.set_role(new_roles)
if "marketing_consent" in newdata:
acc.set_marketing_consent(newdata["marketing_consent"] == "true")
acc.save()
flash("Record updated")
return render_template('account/view.html', account=acc)
else:
if util.request_wants_json():
resp = make_response(
json.dumps(acc.data, sort_keys=True, indent=4) )
resp.mimetype = "application/json"
return resp
else:
# do an mget on the journals, so that we can present them to the user
return render_template('account/view.html', account=acc)
def username(username):
acc = models.Account.pull(username)
if acc is None:
abort(404)
elif request.method == "DELETE" or (request.method == "POST" and request.values.get("submit", False) == "Delete"):
if current_user.id != acc.id and not current_user.is_super:
abort(401)
else:
conf = request.values.get("confirm")
if conf is None or conf != "confirm":
flash("check the box to confirm you really mean it!", "error")
return render_template("account/view.html", account=acc)
acc.delete()
flash("Account " + acc.id + " deleted")
return redirect(url_for(".index"))
elif request.method == "POST":
if current_user.id != acc.id and not current_user.is_super:
abort(401)
newdata = request.json if request.json else request.values
if newdata.get("id", False):
if newdata["id"] != username:
acc = models.Account.pull(newdata["id"])
else:
newdata["api_key"] = acc.data["api_key"]
for k, v in newdata.items():
if k not in ["submit", "password", "role", "confirm"]:
acc.data[k] = v
if "password" in newdata and not newdata["password"].startswith("sha1"):
if "confirm" in newdata and newdata["password"] == newdata["confirm"]:
acc.set_password(newdata["password"])
else:
flash("Passwords do not match", "error")
return render_template("account/view.html", account=acc)
# only super users can re-write roles
if "role" in newdata and current_user.is_super:
new_roles = [r.strip() for r in newdata.get("role").split(",")]
acc.set_role(new_roles)
acc.save()
flash("Account updated", "success")
return render_template("account/view.html", account=acc)
else:
if util.request_wants_json():
resp = make_response(json.dumps(acc.data, sort_keys=True, indent=4))
resp.mimetype = "application/json"
return resp
else:
# do an mget on the journals, so that we can present them to the user
return render_template("account/view.html", account=acc)
def detect():
if request.method == "GET":
return render_template("detect.html")
if request.method == "POST":
url = request.values.get("url")
if url is None:
return render_template("detect.html")
register = autodiscovery.discover(url)
if util.request_wants_json():
resp = make_response( json.dumps(register) )
resp.mimetype = "application/json"
return resp
else:
return render_template("repository.html", repo=register, searchurl=searchurl)
def pagemanager(path=''):
url = '/' + path.lstrip('/').rstrip('/').replace('../',"")
if url == '/': url = '/index'
if url.endswith('.json'): url = url.replace('.json','')
rec = models.Pages.pull_by_url(url)
if rec is not None and rec.data.get('editable',False):
if current_user.is_anonymous() and not rec.data.get('accessible',True):
abort(401)
else:
return redirect(url_for('.edit',path=url))
elif ( ( request.method == 'DELETE' or ( request.method == 'POST' and request.form['submit'] == 'Delete' ) ) and not current_user.is_anonymous() ):
if rec is None:
abort(404)
else:
_sync_delete(rec)
return ""
elif request.method == 'POST' and not current_user.is_anonymous():
if rec is None: rec = models.Pages()
rec.save_from_form(request)
if app.config.get('CONTENT_FOLDER',False): _sync_fs_from_es(rec)
return redirect(rec.data.get('url','/'))
elif rec is None:
if current_user.is_anonymous():
abort(404)
else:
return redirect(url_for('.settings', path=url))
elif request.method == 'GET':
if current_user.is_anonymous() and not rec.data.get('accessible',True):
abort(401)
elif util.request_wants_json():
resp = make_response( rec.json )
resp.mimetype = "application/json"
return resp
else:
try:
content = render_template(
'pagemanager/content' + url,
record=rec
)
except:
content = rec.data.get('content',"")
return _render_page(rec,content)
else:
abort(401)
def username(username):
acc = models.Account.pull(username)
if acc is None:
acc = models.Account.pull_by_email(username)
if acc is not None: return redirect('/account/' + acc.id)
if acc is None:
acc = models.Account.pull_by_api_key(username)
if acc is not None: return redirect('/account/' + acc.id)
if acc is None:
abort(404)
elif ( request.method == 'DELETE' or
( request.method == 'POST' and
request.values.get('submit',False) == 'Delete' ) ):
if current_user.id != acc.id and not current_user.is_super:
abort(401)
else:
acc.delete()
flash('Account ' + acc.id + ' deleted')
return redirect(url_for('.index'))
elif request.method == 'POST':
if current_user.id != acc.id and not current_user.is_super:
abort(401)
newdata = request.json if request.json else request.values
if newdata.get('id',False):
if newdata['id'] != username:
acc = models.Account.pull(newdata['id'])
else:
newdata['api_key'] = acc.data['api_key']
for k, v in newdata.items():
if k not in ['submit','password']:
acc.data[k] = v
if 'password' in newdata and not newdata['password'].startswith('sha1') and len(newdata["password"]) > 4:
acc.set_password(newdata['password'])
acc.save()
time.sleep(1)
flash("Your account has been updated",'info')
return render_template('account/view.html', account=acc)
else:
if util.request_wants_json():
resp = make_response(
json.dumps(acc.data, sort_keys=True, indent=4) )
resp.mimetype = "application/json"
return resp
else:
if current_user.is_super and acc.id != current_user.id:
flash('Hi ' + current_user.id + ' - as a super user you can view and edit this account page. Be careful...', 'warning')
return render_template('account/view.html', account=acc)
def adminitem(itype, iid=False):
try:
klass = getattr(models, itype[0].capitalize() + itype[1:])
if iid:
if iid.endswith('.json'): iid = iid.replace('.json', '')
if iid == "new":
rec = None
else:
rec = klass.pull(iid)
if request.method == 'GET':
if not util.request_wants_json():
return render_template('admin/' + itype + '.html',
record=rec)
elif rec is not None:
resp = make_response(rec.json)
resp.mimetype = "application/json"
return resp
else:
abort(404)
elif (
rec and
(request.method == 'DELETE' or
(request.method == "POST"
and request.form.get('submit', False).lower() == "delete"))):
rec.delete()
time.sleep(300)
return redirect(url_for('.index'))
elif request.method == 'POST':
if rec is None:
rec = klass()
try:
rec.save_from_form(request)
except:
newdata = request.json if request.json else request.values
for k, v in newdata:
if k not in ['submit']:
rec.data[k] = v
rec.save()
return render_template('admin/' + itype + '.html', record=rec)
else:
return render_template('admin/' + itype + '.html')
except:
abort(404)
def adminitem(itype,iid=False):
try:
klass = getattr(models, itype[0].capitalize() + itype[1:] )
if iid:
if iid.endswith('.json'): iid = iid.replace('.json','')
if iid == "new":
rec = None
else:
rec = klass.pull(iid)
if request.method == 'GET':
if not util.request_wants_json():
return render_template(
'admin/' + itype + '.html',
record=rec
)
elif rec is not None:
resp = make_response( rec.json )
resp.mimetype = "application/json"
return resp
else:
abort(404)
elif ( rec and (request.method == 'DELETE' or
( request.method == "POST" and
request.form.get('submit',False).lower() == "delete")) ):
rec.delete()
time.sleep(300)
return redirect(url_for('.index'))
elif request.method == 'POST':
if rec is None:
rec = klass()
try:
rec.save_from_form(request)
except:
newdata = request.json if request.json else request.values
for k, v in newdata:
if k not in ['submit']:
rec.data[k] = v
rec.save()
return render_template('admin/' + itype + '.html', record=rec)
else:
return render_template('admin/' + itype + '.html')
except:
abort(404)
def media():
listing = os.listdir( mediadir )
listing = sorted(listing, key=str.lower)
if util.request_wants_json():
response = make_response( json.dumps(listing,""," ") )
response.headers["Content-type"] = "application/json"
return response
else:
usedin = {}
for f in listing:
# see if it is used in any records
#try:
r = models.Pages().query(q='*' + f + '*')
usedin[f] = [i['_source']['url'] for i in r.get('hits',{}).get('hits',[])]
#except:
# usedin[f] = []
return render_template('media/media.html', files=listing, usedin=usedin)
def repository(repo_id):
base = app.config.get("OARR_API_BASE_URL")
if base is None:
abort(500)
client = OARRClient(base)
if util.request_wants_json():
repo_id = repo_id.replace('.json','')
try:
record = client.get_record(repo_id)
resp = make_response( record.json )
resp.mimetype = "application/json"
return resp
except:
abort(404)
else:
record = client.get_record(repo_id)
if record is None:
abort(404)
return render_template("repository.html", repo=record, searchurl=searchurl, search_similar=True)
def index():
if current_user.is_anonymous():
abort(401)
users = models.Account.query() #{"sort":{'id':{'order':'asc'}}},size=1000000
if users['hits']['total'] != 0:
accs = [models.Account.pull(i['_source']['id']) for i in users['hits']['hits']]
# explicitly mapped to ensure no leakage of sensitive data. augment as necessary
users = []
for acc in accs:
user = {'id':acc.id}
if 'created_date' in acc.data:
user['created_date'] = acc.data['created_date']
users.append(user)
if util.request_wants_json():
resp = make_response( json.dumps(users, sort_keys=True, indent=4) )
resp.mimetype = "application/json"
return resp
else:
return render_template('account/users.html', users=users)
def index():
if not current_user.has_role("list_users"):
abort(401)
# users = models.Account.query() #{"sort":{'id':{'order':'asc'}}},size=1000000
accs = models.Account.iterall() # NOTE: this is only suitable if there is a small number of users - we will iterate through all of them here
users = []
for acc in accs:
# explicitly mapped to ensure no leakage of sensitive data. augment as necessary
user = {'id': acc.id, "email": acc.email, "role": acc.role}
if 'created_date' in acc.data:
user['created_date'] = acc.data['created_date']
users.append(user)
if util.request_wants_json():
resp = make_response(json.dumps(users, sort_keys=True, indent=4))
resp.mimetype = "application/json"
return resp
else:
return render_template('account/users.html', users=users)
def organisation(org):
base = app.config.get("OARR_API_BASE_URL")
if base is None:
abort(500)
client = OARRClient(base)
if util.request_wants_json():
org = org.replace('.json','')
try:
record = client.get_org(org)
resp = make_response( record.json )
resp.mimetype = "application/json"
return resp
except:
abort(404)
else:
try:
record = client.get_org(org)
return render_template("organisation.html", org=record.record)
except:
abort(404)
def media():
listing = os.listdir(mediadir)
listing = sorted(listing, key=str.lower)
if util.request_wants_json():
response = make_response(json.dumps(listing, "", " "))
response.headers["Content-type"] = "application/json"
return response
else:
usedin = {}
for f in listing:
# see if it is used in any records
#try:
r = models.Pages().query(q='*' + f + '*')
usedin[f] = [
i['_source']['url'] for i in r.get('hits', {}).get('hits', [])
]
#except:
# usedin[f] = []
return render_template('media/media.html',
files=listing,
usedin=usedin)
def username(username):
acc = models.Account.pull(username)
if acc is None:
abort(404)
elif (request.method == 'DELETE'
or (request.method == 'POST'
and request.values.get('submit', False) == 'Delete')):
if current_user.id != acc.id and not current_user.is_super:
abort(401)
else:
acc.delete()
flash('Account ' + acc.id + ' deleted')
return redirect(url_for('.index'))
elif request.method == 'POST':
if current_user.id != acc.id and not current_user.is_super:
abort(401)
newdata = request.json if request.json else request.values
if newdata.get('id', False):
if newdata['id'] != username:
acc = models.Account.pull(newdata['id'])
else:
newdata['api_key'] = acc.data['api_key']
for k, v in newdata.items():
if k not in ['submit', 'password']:
acc.data[k] = v
if 'password' in newdata and not newdata['password'].startswith(
'sha1'):
acc.set_password(newdata['password'])
acc.save()
flash("Record updated")
return render_template('account/view.html', account=acc)
else:
if util.request_wants_json():
resp = make_response(json.dumps(acc.data, sort_keys=True,
indent=4))
resp.mimetype = "application/json"
return resp
else:
return render_template('account/view.html', account=acc)
def username(username):
acc = models.Account.pull(username)
if ( request.method == 'POST' and request.values.get('submit','') == 'delete' ) or request.method == 'DELETE':
if not auth.user.update(acc,current_user):
abort(401)
if acc: acc.delete()
flash('Account deleted')
time.sleep(1)
return redirect('/account')
elif request.method == 'POST':
if not auth.user.update(acc,current_user):
abort(401)
info = request.json if request.json else request.values
if info.get('id',False):
if info['id'] != username:
acc = models.Account.pull(info['id'])
else:
info['api_key'] = acc.data['api_key']
for k, v in info.items():
if k not in ['submit']:
acc.data[k] = v
if 'password' in info and not info['password'].startswith('sha1'):
acc.set_password(info['password'])
acc.save()
time.sleep(1)
flash('Account updated')
return redirect('/account/' + username)
else:
if not acc:
abort(404)
if util.request_wants_json():
if not auth.user.update(acc,current_user):
abort(401)
resp = make_response( json.dumps(acc.data, sort_keys=True, indent=4) )
resp.mimetype = "application/json"
return resp
else:
return render_template('account/view.html', account=acc)
def index():
if current_user.is_anonymous():
abort(401)
users = models.Account.query(
) #{"sort":{'id':{'order':'asc'}}},size=1000000
if users['hits']['total'] != 0:
accs = [
models.Account.pull(i['_source']['id'])
for i in users['hits']['hits']
]
# explicitly mapped to ensure no leakage of sensitive data. augment as necessary
users = []
for acc in accs:
user = {'id': acc.id}
if 'created_date' in acc.data:
user['created_date'] = acc.data['created_date']
users.append(user)
if util.request_wants_json():
resp = make_response(json.dumps(users, sort_keys=True, indent=4))
resp.mimetype = "application/json"
return resp
else:
return render_template('account/users.html', users=users)
def username(username):
acc = models.Account.pull(username)
if request.method == 'DELETE':
if not auth.user.update(acc,current_user):
abort(401)
if acc: acc.delete()
return ''
elif request.method == 'POST':
if not auth.user.update(acc,current_user):
abort(401)
info = request.json if request.json else request.values
if info.get('id',False):
if info['id'] != username:
acc = models.Account.pull(info['id'])
else:
info['api_key'] = acc.data['api_key']
for k, v in info.items():
if k == 'password':
acc.set_password(info['password'])
elif k not in ['submit']:
acc.data[k] = info[k]
acc.save()
flash('Account updated', "success")
return render_template('account/view.html', account=acc)
else:
if not acc:
abort(404)
if util.request_wants_json():
if not auth.user.update(acc,current_user):
abort(401)
resp = make_response( json.dumps(acc.data, sort_keys=True, indent=4) )
resp.mimetype = "application/json"
return resp
else:
return render_template('account/view.html', account=acc)
def pagemanager(path=''):
url = '/' + path.lstrip('/').rstrip('/').replace('../',"")
if url == '/': url = '/index'
if url.endswith('.json'): url = url.replace('.json','')
rec = models.Pages.pull_by_url(url)
if rec is not None and rec.data.get('editable',False):
return redirect(url_for('.edit',path=url))
elif ( ( request.method == 'DELETE' or ( request.method == 'POST' and request.form['submit'] == 'Delete' ) ) and not current_user.is_anonymous() ):
if rec is None:
abort(404)
else:
_sync_delete(rec)
return ""
elif request.method == 'POST' and not current_user.is_anonymous():
if rec is None: rec = models.Pages()
rec.save_from_form(request)
if app.config.get('CONTENT_FOLDER',False): _sync_fs_from_es(rec)
return redirect(rec.data.get('url','/'))
elif rec is None:
if current_user.is_anonymous():
abort(404)
else:
return redirect(url_for('.settings', path=url))
elif request.method == 'GET':
if current_user.is_anonymous() and not rec.data.get('accessible',True):
abort(401)
elif util.request_wants_json():
resp = make_response( rec.json )
resp.mimetype = "application/json"
return resp
else:
try:
content = render_template(
'pagemanager/content/contentMine.wiki/' + urlend + '.md',
record=rec
)
except:
try:
content = render_template(
'pagemanager/content' + url,
record=rec
)
except:
content = rec.data.get('content',"")
content = markdown.markdown(content)
content = re.sub(r'\[\[(.*?)\]\]',r'<a href="/\1">\1</a>',content)
# if an embedded file url has been provided, embed it in content
if rec.data.get('embed', False):
if ( rec.data['embed'].find('/pub?') != -1 or
rec.data['embed'].find('docs.google.com') != -1 ):
content += '<iframe id="embedded" src="' + rec.data['embed']
content += '" width="100%" height="1000" '
content += 'style="border:none;"></iframe>'
else:
content += '<iframe id="embedded" '
content += 'src="http://docs.google.com/viewer?url='
content += urllib.quote_plus(rec.data['embed'])
content += '&embedded=true" width="100%" height="1000" '
content += 'style="border:none;"></iframe>'
# TODO: try adding js dynamic includes server-side?
return render_template(
'pagemanager/index.html',
content=content,
record=rec
)
else:
abort(401)
def pagemanager(path=''):
url = '/' + path.lstrip('/').rstrip('/').replace('../', "")
if url == '/': url = '/index'
if url.endswith('.json'): url = url.replace('.json', '')
rec = models.Pages.pull_by_url(url)
# check if a wiki page exists for the current end path, even though no record exists
if rec is None:
try:
if url == '/wiki':
urlend = 'Home'
else:
urlend = url.replace('/wiki',
'').split('/')[-1].replace(' ', '-')
fl = open(contentdir + "/contentMine.wiki/" + urlend + '.md', 'r')
p = models.Pages()
p.data = {
"url": url,
"title": urlend,
"content": fl.read(),
"author": "system"
}
fl.close()
p.save()
time.sleep(1)
return redirect(url)
except:
pass
if rec is not None and rec.data.get('editable', False):
return redirect(url_for('.edit', path=url))
elif ((request.method == 'DELETE' or
(request.method == 'POST' and request.form['submit'] == 'Delete'))
and not current_user.is_anonymous()):
if rec is None:
abort(404)
else:
_sync_delete(rec)
return ""
elif request.method == 'POST' and not current_user.is_anonymous():
if rec is None: rec = models.Pages()
rec.save_from_form(request)
if app.config.get('CONTENT_FOLDER', False): _sync_fs_from_es(rec)
return redirect(rec.data.get('url', '/'))
elif rec is None:
if current_user.is_anonymous():
abort(404)
else:
return redirect(url_for('.settings', path=url))
elif request.method == 'GET':
if current_user.is_anonymous() and not rec.data.get(
'accessible', True):
abort(401)
elif util.request_wants_json():
resp = make_response(rec.json)
resp.mimetype = "application/json"
return resp
else:
try:
content = render_template(
'pagemanager/content/contentMine.wiki/' + urlend + '.md',
record=rec)
except:
try:
content = render_template('pagemanager/content' + url,
record=rec)
except:
content = rec.data.get('content', "")
content = markdown.markdown(content)
content = re.sub(r'\[\[(.*?)\]\]', r'<a href="/\1">\1</a>',
content)
# if an embedded file url has been provided, embed it in content
if rec.data.get('embed', False):
if (rec.data['embed'].find('/pub?') != -1
or rec.data['embed'].find('docs.google.com') != -1):
content += '<iframe id="embedded" src="' + rec.data['embed']
content += '" width="100%" height="1000" '
content += 'style="border:none;"></iframe>'
else:
content += '<iframe id="embedded" '
content += 'src="http://docs.google.com/viewer?url='
content += urllib.quote_plus(rec.data['embed'])
content += '&embedded=true" width="100%" height="1000" '
content += 'style="border:none;"></iframe>'
# TODO: try adding js dynamic includes server-side?
return render_template('pagemanager/index.html',
content=content,
record=rec)
else:
abort(401)
def matching(mainorg, suffix=None):
# get any match parameters provided in the request
if request.json:
project = request.json.get('project',[])
person = request.json.get('person',[])
keyword = request.json.get('keyword',[])
url = request.json.get('url',[])
else:
project = request.values.get('project',"").split(',')
person = request.values.get('person',"").split(',')
keyword = request.values.get('keyword',"").split(',')
url = request.values.get('url',"").split(',')
# process the match parameters into search parameters
# TODO: should make this a dict by weightings and include them from below
# and also enable end users to prioritise certain keywords or types
# TODO: perhaps also add maximums from each source, or maximum length of params overall?
params = []
for k in keyword:
if k not in params and len(k) > 0: params.append(k)
for pr in project:
# match project title to a project
if len(pr) > 0:
r = models.Record.pull_by_key('project.title',pr)
if r is not None:
extract = mine.mine_project_record(r)
count = 0 # TODO: proper control of how many url-extracted keywords to include
for e in extract:
for i in extract[e]:
count += 1
if count <= 10:
if i not in params: params.append(i.replace('(','')) # TODO: regex out anything non-az09
for u in url:
if len(u) > 0:
if not u.startswith('http://') and not u.startswith('https://'):
u = 'http://' + u
#try:
r = requests.get(u)
clean = mine.html_text(r.text)
extract = mine.full_extract(web_text=clean, web_weight=5)
count = 0 # TODO: proper control of how many url-extracted keywords to include
for e in extract:
for i in extract[e]:
count += 1
if count <= 10:
if i not in params: params.append(i)
#except:
# pass
potential = []
nodes = [{"type":"param","color":"#666","value":1,"label":i} for i in params]
links = []
linked = []
# perform the search with the defined params and build a list of matching orgs
if len(params) > 0 or len(person) > 0:
qry = {
'query': {
'bool': {
'must': [
{
'query_string': {
'query': " OR ".join(params)
}
}
],
'must_not': [
{
'term': {
'collaboratorOrganisation.canonical.exact': mainorg
}
}
]
}
},
"size": 1000,
"facets" : {
"collaborators" : {
"terms" : {
"field" : "collaboratorOrganisation.canonical.exact",
"size" : 100
}
}
}
}
# add people, if any , to the search
if len(person) > 0:
qry['query']['bool']['should'] = []
qry['query']['bool']['minimum_should_match'] = 1
for p in person:
if len(p) > 0:
qry['query']['bool']['should'].append({
'term': {
'collaboratorPerson.canonical.exact': p
}
})
# get the collaborator orgs found in the search results
# strip the ones who already collaborate with the mainorg
# then find out some info about the remaining ones
r = models.Record.query(q=qry)
collabs = [i['term'] for i in r.get("facets", {}).get("collaborators", {}).get("terms", [])]
collaboration_definition = ["leadro", "principal_investigator", "co_investigator", "fellow"] # FIXME: collaboration definition is going to need to pervade the code
cs = [i['term'] for i in models.Record().ordered_collaborators(mainorg,10000,collaboration_definition)]
for collab in collabs:
if collab not in cs and len(potential) < 10:
p = organisation(collab, raw=True)
p['related'] = []
for i in r.get("hits", {}).get("hits",[]):
canonicals = [l.get('canonical','') for l in i['_source'].get('collaboratorOrganisation',[])]
title = i['_source']['project']['title']
if collab in canonicals and mainorg not in canonicals and title not in p['related']:
p['related'].append({"title":title,"id":i['_id'],"abstract":i['_source']['project'].get('abstractText',"")})
for k, param in enumerate(params):
if param in i['_source']['project']['title'] + " " + i['_source']['project'].get('abstractText',""):
if param + collab not in linked:
links.append({'source':k,'target':len(nodes)})
linked.append(param + collab)
potential.append(p)
nodes.append({"type":"org","label":collab,"color":"#f9ce50","value":len(p['related'])+2})
matchinfo = {
"new_potential": potential,
"nodes": nodes,
"links": links,
"params": params,
"person": person,
"project": project,
"keyword": keyword,
"url": url
}
if util.request_wants_json():
resp = make_response(json.dumps(matchinfo))
resp.mimetype = "application/json"
return resp
elif suffix == "csv":
output = StringIO.StringIO()
writer = csv.writer(output)
if len(potential) > 0:
headers = ["name","projects","collaborators","funding","related"]
writer.writerow(headers)
for p in potential:
r = [
p['name'],
p['projects'],
p['collaborators'],
p['totalfunding'],
len(p['related'])
]
writer.writerow(r)
resp = make_response(output.getvalue())
resp.mimetype = "text/csv"
resp.headers['Content-Disposition'] = 'attachment; filename="' + mainorg + '_new_potential_report.csv"'
return resp
else:
return render_template('organisation/match.html', mainorg=mainorg, matchinfo=matchinfo)
def organisation(mainorg, raw=False):
"""
Any request to a specific organisation's home page
"""
# TODO:
# list all this orgs projects
# list a blurb and website about this org
# list the main contact of this org (and perhaps other users)
# offer ability to update the page about this org
# show this org snapshot data, top projects, recent funding by years, pubs
# offer a download report overview of this org
logo = mainorg.lower().replace(' ','_').replace("'",'').replace('university_','').replace('_university','').replace('_of','').replace('of_','').replace('_the','').replace('the_','').replace('_and','').replace('and_','').replace('_for','').replace('for_','').replace('_.','.') + '.png';
logofolder = os.path.dirname(os.path.abspath( __file__ )).replace('/view','/static/logos')
logos=os.listdir(logofolder)
if logo not in logos:
logo = ''
else:
logo = '/static/logos/' + logo
# logo = ""
# FIXME: should be in the models layer
qry = {
"query": {
"term": {
"collaboratorOrganisation.canonical.exact": mainorg
}
},
"size": 1,
"facets": {
"collaborators":{
"terms_stats" : {
"key_field" : "collaboratorOrganisation.canonical.exact",
"value_field" : "project.fund.valuePounds",
"size" : 0
}
},
"value_stats" : {
"statistical" : {
"field" : "project.fund.valuePounds"
}
}
}
}
r = models.Record.query(q=qry)
org = {
'name': mainorg,
'logo': logo,
'projects': r.get('hits',{}).get('total',0),
'collaborators': len(r.get('facets',{}).get('collaborators',{}).get('terms',[])) - 1,
'totalfunding': "{:,.0f}".format(r.get('facets',{}).get('value_stats',{}).get('total',0))
}
# TODO: post codes should perhaps be processed into the index data, to save
# processing them here
# get post code - lat long lookup table
pcll = json.load(open("postcodes.json"))
try:
orgs = r.get('hits',{}).get('hits',[])[0]['_source']['collaboratorOrganisation']
for o in orgs:
if o['canonical'] == mainorg:
outcode = o['organisation']['address']['postCode'].replace(' ','')
if len(outcode) == 7: outcode = outcode[0:4]
elif len(outcode) == 6: outcode = outcode[0:3]
pc = pcll[outcode]
org['lat'] = pc['lat']
org['lng'] = pc['lng']
except:
org['lat'] = 0
org['lng'] = 0
# get similar names for user fixing
# TODO: this should be in model too
similar = dropdowns('record','collaboratorOrganisation.canonical')
n = mainorg.lower().replace('university','').replace('college','').replace('of','').replace('the','').replace(' ','').replace('.','')
checklist = []
for s in similar:
if mainorg != s and n in s.lower():
checklist.append(s)
# TODO: should really have an org object with the above info in it and it
# should be passed to the page instead of the mainorg string
if raw:
return org
elif util.request_wants_json():
resp = make_response(json.dumps(org))
resp.mimetype = "application/json"
return resp
else:
return render_template('organisation/org.html', org=org, checklist=checklist)
def adminitem(itype, iid=False):
#try:
klass = getattr(models, itype[0].capitalize() + itype[1:])
if iid:
if iid.endswith('.json'): iid = iid.replace('.json', '')
if iid == "new":
rec = None
else:
rec = klass.pull(iid)
if request.method == 'GET':
if not util.request_wants_json():
return render_template(
'admin/' + itype + '.html',
record=rec,
dropdowns={
"clpeople":
dropdowns('account', 'id'),
"partners": [
i['_source']['id'] for i in models.Account.query(
terms={
"partner": "yes"
}, size=1000).get('hits', {}).get('hits', [])
],
"seniors": [
i['_source']['id'] for i in models.Account.query(
terms={
"senior": "yes"
}, size=1000).get('hits', {}).get('hits', [])
],
"contacts":
dropdowns('project',
['externals', 'customer', 'funder']),
"tags":
dropdowns('project', ['tags'])
})
elif rec is not None:
resp = make_response(rec.json)
resp.mimetype = "application/json"
return resp
else:
abort(404)
elif (rec and
(request.method == 'DELETE' or
(request.method == "POST"
and request.form.get('submit', False).lower() == "delete"))):
rec.delete()
flash('Deleted')
return redirect(url_for('.index'))
elif request.method == 'POST':
if rec is None:
rec = klass()
rec.save_from_form(request)
if iid == "new":
flash('Created')
return redirect(url_for('.index') + itype + '/' + rec.id)
else:
flash('Updated')
return redirect(url_for('.index') + itype + '/' + iid)
else:
return redirect(url_for('.index') + itype + '/new')
def contribute():
detectdone = False
dup = False
if request.method == 'GET':
# check to see if this is an update
if 'updaterequest' in request.values:
# get record from OARR
try:
base = app.config.get("OARR_API_BASE_URL")
if base is None: abort(500)
client = OARRClient(base)
record = client.get_record(request.values["updaterequest"]).raw
if "opendoar" not in record["admin"]: record["admin"]["opendoar"] = {}
record["admin"]["opendoar"]["updaterequest"] = request.values["updaterequest"]
detectdone = True
except:
abort(404)
# check for a url request param
elif 'url' in request.values:
# if there is one, then try to set the initial object
if len(request.values['url']) != 0:
try:
register = autodiscovery.discover(request.values['url'])
record = register.raw
for k, v in util.defaultrecord['register']['metadata'][0]['record'].iteritems():
if k not in record.get('register',{}).get('metadata',[{"record":{}}])[0]['record']:
record['register']['metadata'][0]['record'][k] = v
except:
record = util.defaultrecord
# check if there is already a record with this url
dup = rawduplicate(request.values['url'],raw=True)
else:
record = util.defaultrecord
detectdone = True
else:
# otherwise set a default initial object
record = util.defaultrecord
if util.request_wants_json():
resp = make_response( json.dumps({"record":record}) )
resp.mimetype = "application/json"
return resp
else:
return render_template("contribute.html", record=record, detectdone=detectdone, duplicate=dup)
elif request.method == 'POST':
base = app.config.get("OARR_API_BASE_URL")
apikey = app.config.get("OARR_API_KEY")
if base is None or apikey is None: abort(500)
client = OARRClient(base, apikey)
record = client.prep_record(util.defaultrecord,request)
if 'updaterequest' not in record['admin']['opendoar']: record['admin']['opendoar']['newcontribution'] = True
saved = client.save_record(record)
if saved:
flash('Thank you very much for your submission. Your request will be processed as soon as possible, and usually within three working days.', 'success')
return redirect('/')
else:
flash('Sorry, there was a problem saving your submission. Please try again.', 'error')
return redirect('/contribute')
def pagemanager(path=""):
url = "/" + path.lstrip("/").rstrip("/").replace("../", "")
if url == "/":
url = "/index"
if url.endswith(".json"):
url = url.replace(".json", "")
rec = models.Pages.pull_by_url(url)
# check if a wiki page exists for the current end path, even though no record exists
if rec is None:
try:
if url == "/wiki":
urlend = "Home"
else:
urlend = url.replace("/wiki", "").split("/")[-1].replace(" ", "-")
fl = open(contentdir + "/contentMine.wiki/" + urlend + ".md", "r")
p = models.Pages()
p.data = {"url": url, "title": urlend, "content": fl.read(), "author": "system"}
fl.close()
p.save()
time.sleep(1)
return redirect(url)
except:
pass
if rec is not None and rec.data.get("editable", False):
return redirect(url_for(".edit", path=url))
elif (
request.method == "DELETE" or (request.method == "POST" and request.form["submit"] == "Delete")
) and not current_user.is_anonymous():
if rec is None:
abort(404)
else:
_sync_delete(rec)
return ""
elif request.method == "POST" and not current_user.is_anonymous():
if rec is None:
rec = models.Pages()
rec.save_from_form(request)
if app.config.get("CONTENT_FOLDER", False):
_sync_fs_from_es(rec)
return redirect(rec.data.get("url", "/"))
elif rec is None:
if current_user.is_anonymous():
abort(404)
else:
return redirect(url_for(".settings", path=url))
elif request.method == "GET":
if current_user.is_anonymous() and not rec.data.get("accessible", True):
abort(401)
elif util.request_wants_json():
resp = make_response(rec.json)
resp.mimetype = "application/json"
return resp
else:
try:
content = render_template("pagemanager/content/contentMine.wiki/" + urlend + ".md", record=rec)
except:
try:
content = render_template("pagemanager/content" + url, record=rec)
except:
content = rec.data.get("content", "")
content = markdown.markdown(content)
content = re.sub(r"\[\[(.*?)\]\]", r'<a href="/\1">\1</a>', content)
# if an embedded file url has been provided, embed it in content
if rec.data.get("embed", False):
if rec.data["embed"].find("/pub?") != -1 or rec.data["embed"].find("docs.google.com") != -1:
content += '<iframe id="embedded" src="' + rec.data["embed"]
content += '" width="100%" height="1000" '
content += 'style="border:none;"></iframe>'
else:
content += '<iframe id="embedded" '
content += 'src="http://docs.google.com/viewer?url='
content += urllib.quote_plus(rec.data["embed"])
content += '&embedded=true" width="100%" height="1000" '
content += 'style="border:none;"></iframe>'
# TODO: try adding js dynamic includes server-side?
return render_template("pagemanager/index.html", content=content, record=rec)
else:
abort(401)
def record(uuid=None):
base = app.config.get("OARR_API_BASE_URL")
apikey = app.config.get("OARR_API_KEY")
if base is None or apikey is None:
abort(500)
client = OARRClient(base, apikey)
detectdone = False
dup = False
if request.method == "GET":
if uuid is None or uuid == "new":
# check for a url request param
if "url" in request.values:
# if there is one, then try to set the initial object
if len(request.values["url"]) != 0:
try:
register = autodiscovery.discover(request.values["url"])
record = register.raw
for k, v in util.defaultrecord["register"]["metadata"][0]["record"].iteritems():
if k not in record.get("register", {}).get("metadata", [{"record": {}}])[0]["record"]:
record["register"]["metadata"][0]["record"][k] = v
except:
record = util.defaultrecord
else:
record = util.defaultrecord
detectdone = True
# check if there is already a record with this url
dup = duplicate(request.values["url"], raw=True)
else:
# otherwise set a default initial object
record = util.defaultrecord
else:
# get record from OARR
try:
record = client.get_record(uuid.replace(".json", "")).raw
detectdone = True
# check if there is already a record with this url
dup = duplicate(record["register"]["metadata"][0]["record"]["url"], raw=True)
except:
abort(404)
if util.request_wants_json():
resp = make_response(json.dumps({"record": record}))
resp.mimetype = "application/json"
return resp
else:
return render_template("contribute.html", record=record, detectdone=detectdone, duplicate=dup)
elif (request.method == "POST" and request.values.get("submit", "") == "Delete") or request.method == "DELETE":
record = client.get_record(uuid)
if record is None:
abort(404)
client.delete_record(uuid)
time.sleep(1)
flash("Record delete")
return redirect(url_for(".index"))
elif request.method == "POST":
if uuid == "new":
# save the new record to OARR
record = client.prep_record(util.defaultrecord, request)
saved = client.save_record(record)
if saved:
flash("New record created", "success")
return redirect(url_for(".index"))
else:
flash("Sorry, the attempt to create a new record was unsuccessful", "error")
return redirect("/admin/record/new")
else:
rec = client.get_record(uuid)
if rec is None:
abort(404)
# remove any newcontribution tag on first save via admin
if "newcontribution" in rec.raw.get("admin", {}).get("opendoar", {}):
del rec.raw["admin"]["opendoar"]["newcontribution"]
# if this is an update acceptance, do the update
if "updaterequest" in rec.raw.get("admin", {}).get("opendoar", {}):
# get the original record, prep it with the update, delete the update request record
forupdate = client.get_record(rec.raw["admin"]["opendoar"]["updaterequest"])
if forupdate is None:
flash("Sorry, an original record cannot be found to correspond with this update request.", "error")
return redirect("/admin/record/" + uuid)
else:
record = client.prep_record(forupdate.raw, request)
try:
del record["admin"]["opendoar"]["updaterequest"]
saved = client.save_record(record)
if saved:
client.delete_record(uuid)
time.sleep(1)
flash(
"This original record has been successfully updated, and the update request record has been deleted.",
"success",
)
return redirect("/admin/record/" + str(record["id"]))
else:
flash(
"Sorry, there was an error. Your changes have not been saved. Please try again.",
"error",
)
return redirect("/admin/record/" + uuid)
except:
flash("Sorry, there was an error. Your changes have not been saved. Please try again.", "error")
return redirect("/admin/record/" + uuid)
# otherwise save the record changes to OARR
else:
# do whatever needs done here to update the record from the form input
record = client.prep_record(rec.raw, request)
saved = client.save_record(record)
if saved:
detectdone = True
time.sleep(1)
flash("Record has been updated", "success")
return render_template("contribute.html", record=record, detectdone=detectdone, duplicate=dup)
else:
flash("Sorry, there was an error. Your changes have not been saved. Please try again.", "error")
return redirect("/admin/record/" + uuid)
def pagemanager(path=''):
url = '/' + path.lstrip('/').rstrip('/').replace('../', "")
if url == '/': url = '/index'
if url.endswith('.json'): url = url.replace('.json', '')
rec = models.Pages.pull_by_url(url)
if rec is not None and rec.data.get('editable', False):
return redirect(url_for('.edit', path=url))
elif ((request.method == 'DELETE' or
(request.method == 'POST' and request.form['submit'] == 'Delete'))
and not current_user.is_anonymous()):
if rec is None:
abort(404)
else:
_sync_delete(rec)
return ""
elif request.method == 'POST' and not current_user.is_anonymous():
if rec is None: rec = models.Pages()
rec.save_from_form(request)
if app.config.get('CONTENT_FOLDER', False): _sync_fs_from_es(rec)
return redirect(rec.data.get('url', '/'))
elif rec is None:
if current_user.is_anonymous():
abort(404)
else:
return redirect(url_for('.settings', path=url))
elif request.method == 'GET':
if current_user.is_anonymous() and not rec.data.get(
'accessible', True):
abort(401)
elif util.request_wants_json():
resp = make_response(rec.json)
resp.mimetype = "application/json"
return resp
else:
try:
content = render_template('pagemanager/content' + url,
record=rec)
except:
content = rec.data.get('content', "")
content = markdown.markdown(content)
# if an embedded file url has been provided, embed it in content
if rec.data.get('embed', False):
if (rec.data['embed'].find('/pub?') != -1
or rec.data['embed'].find('docs.google.com') != -1):
content += '<iframe id="embedded" src="' + rec.data['embed']
content += '" width="100%" height="1000" '
content += 'style="border:none;"></iframe>'
else:
content += '<iframe id="embedded" '
content += 'src="http://docs.google.com/viewer?url='
content += urllib.quote_plus(rec.data['embed'])
content += '&embedded=true" width="100%" height="1000" '
content += 'style="border:none;"></iframe>'
# TODO: try adding js dynamic includes server-side?
return render_template('pagemanager/index.html',
content=content,
record=rec)
else:
abort(401)
