def hooks():
message = request.json.get('payload', {})
which = message.get('repository', {}).get('name', "")
# now define the actions that should be performed when a particular
# repo is updated (identified by which)
# this example just triggers a pull when the message is received
# then restarts a service via supervisor
if which == "portality":
_pull(message)
call("sudo supervisorctl restart portality", shell=True)
# this example does some index updating based on content of the message
# this is for use with a pagemanager that writes file content to disk
if which == "content":
_pull(message)
import portality.models as models
for commit in message.get('commits', []):
for add in commit.get('added', []):
try:
content = open(
app.config['REPOS'][which]['path'] + '/' + add,
'r').read()
path, fn = add.rstrip('/').split('/')
p = models.Pages({
"url": '/' + add,
"title": fn,
"content": content
})
p.save()
content.close()
except:
pass
for remove in commit.get('removed', []):
try:
p = models.Pages().pull_by_url('/' + remove)
if p is not None:
pagemanager._sync_delete(p)
except:
pass
for modify in commit.get('modified', []):
try:
p = models.Pages().pull_by_url('/' + modify)
if p is not None:
content = open(
app.config['REPOS'][which]['path'] + '/' + modify,
'r').read()
p.data['content'] = content
p.save()
content.close()
except:
pass
def settings(path='/'):
if current_user.is_anonymous():
abort(401)
if path != '/': path = '/' + path
next = util.is_safe_url(request.values.get('next', path))
record = models.Pages.pull_by_url(next)
if record is None:
rec = {'url': next, 'author': current_user.id}
else:
rec = record.data
if request.method == 'GET':
if record is None:
flash('There is no page here yet - create one.')
return render_template('pagemanager/settings.html',
tags=json.dumps(dropdowns(
'pagemanager', 'tags')),
urls=json.dumps(dropdowns('pagemanager',
'url')),
record=rec)
elif (request.method == 'DELETE'
or (request.method == 'POST'
and request.values.get('submit', False) == 'Delete')):
if record is None:
abort(404)
else:
_sync_delete(record)
flash('Page deleted')
return redirect("/")
elif request.method == 'POST':
if record is None:
record = models.Pages()
if request.values.get('template', False):
tmpl = models.Pages.pull_by_url(request.values['template'])
if tmpl is not None and 'content' in tmpl.data:
record.data['content'] = tmpl.data['content']
newdata = request.json if request.json else request.values
# check the new url does not overwrite an already present url
if 'url' not in newdata:
flash("A URL is required.")
elif (record.data.get('url', '') != newdata['url']
and models.Pages.pull_by_url(newdata['url']) is not None):
# update record data so form is up to date but don't save it
record.update_from_form(request)
flash(
"There is already a page present at the URL you specifed. You must delete that page first."
)
else:
record.save_from_form(request)
flash('Settings updated')
time.sleep(1)
return render_template('pagemanager/settings.html',
tags=json.dumps(dropdowns('pages', 'tags')),
urls=json.dumps(dropdowns('pages', 'url')),
record=record.data)
def pagemanager(path=''):
url = '/' + path.lstrip('/').rstrip('/').replace('../',"")
if url == '/': url = '/index'
if url.endswith('.json'): url = url.replace('.json','')
rec = models.Pages.pull_by_url(url)
if rec is not None and rec.data.get('editable',False):
if current_user.is_anonymous() and not rec.data.get('accessible',True):
abort(401)
else:
return redirect(url_for('.edit',path=url))
elif ( ( request.method == 'DELETE' or ( request.method == 'POST' and request.form['submit'] == 'Delete' ) ) and not current_user.is_anonymous() ):
if rec is None:
abort(404)
else:
_sync_delete(rec)
return ""
elif request.method == 'POST' and not current_user.is_anonymous():
if rec is None: rec = models.Pages()
rec.save_from_form(request)
if app.config.get('CONTENT_FOLDER',False): _sync_fs_from_es(rec)
return redirect(rec.data.get('url','/'))
elif rec is None:
if current_user.is_anonymous():
abort(404)
else:
return redirect(url_for('.settings', path=url))
elif request.method == 'GET':
if current_user.is_anonymous() and not rec.data.get('accessible',True):
abort(401)
elif util.request_wants_json():
resp = make_response( rec.json )
resp.mimetype = "application/json"
return resp
else:
try:
content = render_template(
'pagemanager/content' + url,
record=rec
)
except:
content = rec.data.get('content',"")
return _render_page(rec,content)
else:
abort(401)
def media():
listing = os.listdir(mediadir)
listing = sorted(listing, key=str.lower)
if util.request_wants_json():
response = make_response(json.dumps(listing, "", " "))
response.headers["Content-type"] = "application/json"
return response
else:
usedin = {}
for f in listing:
# see if it is used in any records
#try:
r = models.Pages().query(q='*' + f + '*')
usedin[f] = [
i['_source']['url'] for i in r.get('hits', {}).get('hits', [])
]
#except:
# usedin[f] = []
return render_template('media/media.html',
files=listing,
usedin=usedin)
def pagemanager(path=''):
url = '/' + path.lstrip('/').rstrip('/').replace('../', "")
if url == '/': url = '/index'
if url.endswith('.json'): url = url.replace('.json', '')
rec = models.Pages.pull_by_url(url)
if rec is not None and rec.data.get('editable', False):
return redirect(url_for('.edit', path=url))
elif ((request.method == 'DELETE' or
(request.method == 'POST' and request.form['submit'] == 'Delete'))
and not current_user.is_anonymous()):
if rec is None:
abort(404)
else:
_sync_delete(rec)
return ""
elif request.method == 'POST' and not current_user.is_anonymous():
if rec is None: rec = models.Pages()
rec.save_from_form(request)
if app.config.get('CONTENT_FOLDER', False): _sync_fs_from_es(rec)
return redirect(rec.data.get('url', '/'))
elif rec is None:
if current_user.is_anonymous():
abort(404)
else:
return redirect(url_for('.settings', path=url))
elif request.method == 'GET':
if current_user.is_anonymous() and not rec.data.get(
'accessible', True):
abort(401)
elif util.request_wants_json():
resp = make_response(rec.json)
resp.mimetype = "application/json"
return resp
else:
try:
content = render_template('pagemanager/content' + url,
record=rec)
except:
content = rec.data.get('content', "")
content = markdown.markdown(content)
# if an embedded file url has been provided, embed it in content
if rec.data.get('embed', False):
if (rec.data['embed'].find('/pub?') != -1
or rec.data['embed'].find('docs.google.com') != -1):
content += '<iframe id="embedded" src="' + rec.data['embed']
content += '" width="100%" height="1000" '
content += 'style="border:none;"></iframe>'
else:
content += '<iframe id="embedded" '
content += 'src="http://docs.google.com/viewer?url='
content += urllib.quote_plus(rec.data['embed'])
content += '&embedded=true" width="100%" height="1000" '
content += 'style="border:none;"></iframe>'
# TODO: try adding js dynamic includes server-side?
return render_template('pagemanager/index.html',
content=content,
record=rec)
else:
abort(401)
def parser(path='',
blurb='',
scale=3,
minoccur=2,
omitscores=False,
boostphrases=False,
size=0,
raw=False):
size = int(request.values.get('size', size))
minoccur = int(request.values.get('minoccur', minoccur))
scale = int(request.values.get('scale', scale))
if 'omitscores' in request.values and request.values['omitscores'].lower(
) not in ['false', '0', 'no']:
omitscores = True
if 'boostphrases' in request.values and request.values[
'boostphrases'].lower() not in ['false', '0', 'no']:
boostphrases = True
extractor.filter = extract.DefaultFilter(singleStrengthMinOccur=minoccur)
if path:
url = path
else:
url = request.values.get('url', '')
if url and not url.startswith('http://') and not url.startswith(
'https://'):
url = 'http://' + url
if url:
#try:
# NOTE: GETTING THIS TO WORK PIP INSTALL SPYNNER WHICH REQUIRES
# A BUNCH OF OTHER STUFF (I AM NOT SURE EXACTLY WHICH YET):
# sudo apt-get install libx11 libx11-dev xvfb libxtst-dev libpng-dev
# https://github.com/makinacorpus/spynner/#dependencies
# still fails on importing PyQt4 which is on my system but can't
# get it into my virtual env. Rely on non-js page content for now.
#import spynner
#browser = spynner.Browser()
#browser.create_webview(True)
#browser.load(url, load_timeout=60)
#c = browser._get_html()
#browser.close()
#except:
loc = app.config.get('BASE_URL', 'http://cottagelabs.com')
if url.startswith(loc):
lloc = url.replace(loc, '')
if lloc == '': lloc = '/index'
rec = models.Pages().pull_by_url(lloc)
if rec is not None:
c = rec.data.get('content', '')
else:
c = ''
else:
g = requests.get(url)
c = g.text
else:
c = request.values.get('blurb', blurb)
if c:
content = _html_text(c)
terms = extractor(content)
result = {}
for t, o, l in terms:
if boostphrases:
ct = o + l
else:
ct = o
result[t.lower()] = ct * scale
res = [{
"term": i[0],
"score": i[1]
} for i in sorted(result.items(), key=lambda x: x[1], reverse=True)
if len(i[0].replace(' ', '')) > 2 and i[0].replace(
' ', '') not in url and i[0][0] in string.lowercase +
string.uppercase and i[0][len(i[0]) - 1] in string.lowercase +
string.uppercase]
if omitscores: res = [i[0] for i in res]
if size is not 0: res = res[:size]
else:
res = []
if raw:
return res
else:
resp = make_response(json.dumps(res))
resp.mimetype = "application/json"
return resp
def pagemanager(path=''):
url = '/' + path.lstrip('/').rstrip('/').replace('../', "")
if url == '/': url = '/index'
if url.endswith('.json'): url = url.replace('.json', '')
rec = models.Pages.pull_by_url(url)
# check if a wiki page exists for the current end path, even though no record exists
if rec is None:
try:
if url == '/wiki':
urlend = 'Home'
else:
urlend = url.replace('/wiki',
'').split('/')[-1].replace(' ', '-')
fl = open(contentdir + "/contentMine.wiki/" + urlend + '.md', 'r')
p = models.Pages()
p.data = {
"url": url,
"title": urlend,
"content": fl.read(),
"author": "system"
}
fl.close()
p.save()
time.sleep(1)
return redirect(url)
except:
pass
if rec is not None and rec.data.get('editable', False):
return redirect(url_for('.edit', path=url))
elif ((request.method == 'DELETE' or
(request.method == 'POST' and request.form['submit'] == 'Delete'))
and not current_user.is_anonymous()):
if rec is None:
abort(404)
else:
_sync_delete(rec)
return ""
elif request.method == 'POST' and not current_user.is_anonymous():
if rec is None: rec = models.Pages()
rec.save_from_form(request)
if app.config.get('CONTENT_FOLDER', False): _sync_fs_from_es(rec)
return redirect(rec.data.get('url', '/'))
elif rec is None:
if current_user.is_anonymous():
abort(404)
else:
return redirect(url_for('.settings', path=url))
elif request.method == 'GET':
if current_user.is_anonymous() and not rec.data.get(
'accessible', True):
abort(401)
elif util.request_wants_json():
resp = make_response(rec.json)
resp.mimetype = "application/json"
return resp
else:
try:
content = render_template(
'pagemanager/content/contentMine.wiki/' + urlend + '.md',
record=rec)
except:
try:
content = render_template('pagemanager/content' + url,
record=rec)
except:
content = rec.data.get('content', "")
content = markdown.markdown(content)
content = re.sub(r'\[\[(.*?)\]\]', r'<a href="/\1">\1</a>',
content)
# if an embedded file url has been provided, embed it in content
if rec.data.get('embed', False):
if (rec.data['embed'].find('/pub?') != -1
or rec.data['embed'].find('docs.google.com') != -1):
content += '<iframe id="embedded" src="' + rec.data['embed']
content += '" width="100%" height="1000" '
content += 'style="border:none;"></iframe>'
else:
content += '<iframe id="embedded" '
content += 'src="http://docs.google.com/viewer?url='
content += urllib.quote_plus(rec.data['embed'])
content += '&embedded=true" width="100%" height="1000" '
content += 'style="border:none;"></iframe>'
# TODO: try adding js dynamic includes server-side?
return render_template('pagemanager/index.html',
content=content,
record=rec)
else:
abort(401)