def page_path(slug, **kwargs):
p = Node.all().filter('abs_path = ', slug).get()
if p is None:
p = Node.all().filter('slug = ', slug).get()
if p is None:
return u''
return p.get_absolute_url(**kwargs)
def menu(abs_path=''):
base = Node.all().filter('abs_path = ', abs_path).get()
qs = Node.all().filter('active = ', True).filter('state = ', PUBLISHED)
l = len(base.abs_path)
nodes = dict([(n.get_key(), n) for n in qs if n.abs_path.startswith(base.abs_path)])
node = simple_rec(base, nodes)
return node
def wrapper(self, project_id):
project = Project.get_by_id(long(project_id))
if project:
user = users.get_current_user()
if project.owner == user.user_id():
nodes = Node.all()
nodes.ancestor(project)
nodes.order('title')
nodes = tuple(nodes) # prevent re-execution when iterating
request_node, current_node = self.request.get('node'), None
if request_node: # self.request.get always return a string
try:
current_node_id = long(request_node)
except ValueError:
pass
else:
for node in nodes:
if node.key().id() == current_node_id:
current_node = node
break
if current_node is None:
self.redirect(project.permalink)
return
f(self, user, project, nodes, current_node)
else:
self.error(403)
else:
self.redirect('/')
def wrapper(self, project_id):
project = Project.get_by_id(long(project_id))
if project:
user = users.get_current_user()
if project.owner == user.user_id():
nodes = Node.all()
nodes.ancestor(project)
nodes.order('title')
nodes = tuple(nodes) # prevent re-execution when iterating
request_node, current_node = self.request.get('node'), None
if request_node: # self.request.get always return a string
try:
current_node_id = long(request_node)
except ValueError:
pass
else:
for node in nodes:
if node.key().id() == current_node_id:
current_node = node
break
if current_node is None:
self.redirect(project.permalink)
return
f(self, user, project, nodes, current_node)
else:
self.error(403)
else:
self.redirect('/')
def post(self):
self.response.headers['Content-Type'] = 'text/plain'
self.write('Started section regeneration task:')
key = self.request.get('key')
self.write('- key = ' + key)
# ok, so get the section first
section = Section.get( self.request.get('key') )
if section is None:
self.write('No section found')
logging.warn( 'No section found for key: ' + key )
return
# keep a count of all these things
labels = {}
archives = {}
# get all the nodes
# FixMe: why doesn't "nodes = section.nodes()" work ???
nodes = Node.all().filter('section =', section.key())
self.write('Nodes:')
for node in nodes:
self.write('- node=' + node.kind()) # FixMe: why doesn't "node.name()" work :(
# count all of the labels
for label in node.label:
self.write(' - ' + label)
if label in labels:
labels[label] += 1
else:
labels[label] = 1
# the archive counts
for archive in node.archive:
self.write(' - ' + archive)
if archive in archives:
archives[archive] += 1
else:
archives[archive] = 1
# after all that, make them into a list (for ease of use in Django Templates)
archives = [
{ 'archive' : x, 'count' : archives[x] }
for x in sorted(archives)
if re.search(r'^\d\d\d\d(-\d\d)?$', x, re.DOTALL | re.VERBOSE) # just get years and months
]
labels = [
{ 'label' : x, 'count' : labels[x] }
for x in sorted(labels)
]
# now that we have our counts, save it as JSON
section.archive_json = archives
section.label_json = labels
section.put()
self.write('Finished')
def list_pages(request):
nodes = dict([(n.get_key(), n) for n in Node.all()])
root = None
for node in nodes:
node = nodes[node]
if node.path == '0':
root = node
break
if root is not None:
root = rec(root, nodes)
return render_template('app:pages/list.html', root=root)
def transaction():
q = Node.all()
q.ancestor(project)
q.filter('title =', title)
node = q.get()
if node is None:
node = Node(
parent=project,
title=title,
)
node.put()
return node
def transaction():
q = Node.all()
q.ancestor(project)
q.filter('title =', title)
node = q.get()
if node is None:
node = Node(
parent=project,
title=title,
)
node.put()
return node
def app_balance_containers():
strategy = None
try:
strategy = request.form["strategy"]
except ValueError:
pass
opts = None
try:
opts = request.form["opts"]
except:
pass
nodes = Node.all()
problem = Problem(strategy, opts)
containers = []
for node in nodes:
containers += node.containers()
problem.bins.append(node.to_bin())
for container in containers:
try:
problem.items.append(container.to_item())
except ContainerNotFound:
pass
problem.normalize()
result = problem.solve()
mapping = result['mapping']
migrations = []
for i in range(len(mapping)):
# If the node didn't change, next elem of the results
if nodes[mapping[i]].host == containers[i].host:
continue
# If it has changed, migrate the container to the giving node
new_container = containers[i].migrate(nodes[mapping[i]])
migrations.append({
"Service": containers[i].service(),
"Started": { "Node": new_container.host, "Id": new_container.info["Id"]},
"Stopped": { "Node": containers[i].host, "Id": containers[i].info["Id"]}
})
try:
result['datetime'] = str(result['datetime'])
except:
pass
return json.dumps({"items": problem.items, "bins": problem.bins, "result": result, "migrations": migrations}, cls=BinJSONEncoder)
def show(request, key):
# eventually the key should be the "abs_path"
page = Node.all().filter('type =', PAGE).filter('abs_path = ', key).get()
if page is None:
raise Exception("Page not found")
if page.layout is None:
raise Exception("invalid layout")
string = [
"{%% extends 'nut:layout/%s' %%}" % page.layout.abs_path,
"{%% block body %%} %s {%% endblock %%}" % page.body ]
info(string)
for block in page.blocks:
string.append("{%% block %s %%} %s {%% endblock %%}" % (block.name, block.body))
resp = layout_response_from_string('\n'.join(string), page.content_type, title=page.name, this=page)
resp.expires = datetime.now() + timedelta(7)
return resp
def app_migrate_container(host, cid):
try:
node = Node.find(host)
except ValueError:
return Response("{} is not in the cluster".format(host), status=422)
container = Container.find(node, cid)
nodes = Node.all()
strategy = AllocationStrategy.from_name(current_app.config['strategy'])
selected_node = strategy.select_node(nodes, container.service())
new_container = container.migrate(selected_node)
return Response(json.dumps(
{"Started": new_container,
"Stopped": container}, cls=ContainerJSONEncoder), status=201)
def app_new_container():
try:
service = request.form['service']
except KeyError:
return Response("service field should be provided", status=422)
nodes = Node.all()
strategy = AllocationStrategy.from_name(current_app.config['strategy'])
selected_node = strategy.select_node(nodes, service)
try:
image = request.form['image']
except KeyError:
image = "soulou/msc-thesis-fibo-http-service"
started_container = Container.create(selected_node, service, image)
return Response(json.dumps(started_container, cls=ContainerJSONEncoder), status=201)
def post(self):
self.response.headers['Content-Type'] = 'text/plain'
self.write('Started section duplicate node check:')
section_key = self.request.get('section_key')
name = self.request.get('name')
self.write('- key = ' + section_key)
self.write('- name = ' + name)
# get the section first (and if the section_key is crap just finish)
section = None
try:
section = Section.get( self.request.get('section_key') )
except:
# by just returning here, it ends up a 200, so all is good
return
if section is None:
self.write('No section found')
logging.warn( 'No section found for key: ' + section_key )
return
nodes = Node.all().filter('section =', section).filter('name =', name)
if nodes.count() <= 1:
msg = 'Only [%d] nodes of this name in this section' % nodes.count()
self.write(msg)
return
msg = 'More than one node named [%s] in section [%s]' % (name, section.path)
self.write(msg)
logging.warn(msg)
admin_email = util.config_value('Admin Email')
if not mail.is_email_valid(admin_email):
return
url_edit = util.construct_url() + '/admin/node/'
body = 'Section %s has two nodes named %s ' % (section.path, name)
mail.send_mail(admin_email, admin_email, 'Duplicate node name in section ' + section.path, body)
def delete(request, key):
form = ConfirmDeleteForm(request.form)
if request.method == 'POST' and form.validate():
if form.drop.data is True:
node = Node.get(key)
db.delete(node.blocks)
Node.drop(key)
return redirect('/admin/pages/', 301)
if form.cascade.data is True:
# cascade callback
# inject a cascade callback that does
# # try to delete the key
# for i in range(2):
# if memcache.delete(node.get_absolute_url()) > 0:
# break
Node.drop(key, cascade=True)
# FIXME: remove blocks from dropped Nodes
return redirect(url_for('nut:pages/list_pages'), 301)
node = Node.get(key)
nodes = dict([(n.get_key(), n) for n in Node.all().filter("ancestors = ", key)])
node = rec(node, nodes)
return render_template('app:pages/confirm_delete.html', form=form,
node=node)
def get(self):
path = urllib.unquote(self.request.path)
m = parts.search(path)
if m is None:
self.error(404)
return
this_path = m.group(1)
this_page = m.group(3) or 'index'
this_ext = m.group(4) or 'html'
if this_page is None:
this_page = 'index'
this_ext = 'html'
section = Section.all().filter('path =', this_path).get()
if section is None:
self.error(404)
return
# if this is an index, call a different template
if this_page == 'index' and this_ext == 'html':
# index.html
vals = {
'page' : 'index.html',
'section' : section,
}
self.template( section.layout + '-index.html', vals, util.config_value('Theme') );
elif this_page == 'rss20' and this_ext == 'xml':
# rss20.xml
nodes = self.latest_nodes(section, 'index-entry', 10)
vals = {
'page' : 'rss20.xml',
'section' : section,
'nodes' : nodes,
}
self.response.headers['Content-Type'] = 'application/rss+xml'
self.template( 'rss20.xml', vals, 'rss' );
elif this_page == 'sitefeed' and this_ext == 'xml' and section.has('sitefeed'):
# sitefeed.xml
nodes = Node.all().filter('attribute =', 'index-entry').order('-inserted').fetch(10)
vals = {
'page' : 'sitefeed.xml',
'section' : section,
'nodes' : nodes,
}
self.response.headers['Content-Type'] = 'application/rss+xml'
self.template( 'rss20.xml', vals, 'rss' );
elif this_page == 'sitemapindex' and this_ext == 'xml':
# sitemapindex.xml
vals = {
'page' : 'sitemapindex.xml',
'sections' : Section.all().filter('attribute =', 'sitemap-entry').order('inserted'),
}
self.response.headers['Content-Type'] = 'text/xml'
self.template( 'sitemapindex.xml', vals, 'sitemaps' );
elif this_page == 'urlset' and this_ext == 'xml':
# urlset.xml
vals = {
'page' : 'urlset.xml',
'section' : section,
'nodes' : Node.all().filter('section =', section.key()).filter('attribute =', 'index-entry').order('inserted')
}
self.response.headers['Content-Type'] = 'text/xml'
self.template( 'urlset.xml', vals, 'sitemaps' );
elif label_page.search(this_page) and this_ext == 'html':
# path =~ 'label:something.html'
m = label_page.search(this_page)
label = m.group(1)
vals = {
'page' : 'label:' + label + '.html',
'section' : section,
'nodes' : Node.all().filter('section =', section.key()).filter('label =', label).order('-inserted'),
'label' : label
}
self.template( 'label-index.html', vals, util.config_value('Theme') );
elif archive_page.search(this_page) and this_ext == 'html':
# path =~ 'archive:2009.html'
m = archive_page.search(this_page)
archive = m.group(1)
vals = {
'page' : 'archive:' + archive + '.html',
'section' : section,
'nodes' : Node.all().filter('section =', section.key()).filter('archive =', archive).order('-inserted'),
'archive' : archive
}
self.template( 'archive-index.html', vals, util.config_value('Theme') );
elif this_page == 'comment' and this_ext == 'html':
# get the comment if it exists
try:
comment = Comment.get( self.request.get('key') )
except db.BadKeyError:
self.error(404)
return
if comment is None:
self.error(404)
return
vals = {
'page' : 'comment.html',
'section' : section,
'node' : comment.node,
'comment' : comment,
}
self.template( 'comment.html', vals, util.config_value('Theme') );
elif this_ext == 'html':
# get the node itself
node_query = Node.all().filter('section =', section.key()).filter('name =', this_page)
if node_query.count() == 0:
self.error(404)
return
node = node_query.fetch(1)[0]
# get the approved comments (but only if we know some are there, save a trip to the datastore)
comments = None
if node.comment_count:
comments = Comment.all().filter('node =', node.key()).filter('status =', 'approved').order('inserted')
vals = {
'page' : this_page + '.html',
'section' : section,
'node' : node,
'comments' : comments,
}
self.template( 'node.html', vals, util.config_value('Theme') );
else:
# 404
self.error(404)
return
def latest_nodes(self, section, attribute, limit):
nodes = Node.all().filter('section =', section.key()).filter('attribute =', attribute).order('-inserted').fetch(limit)
return nodes
def post(self):
path = urllib.unquote(self.request.path)
m = parts.search(path)
if m is None:
self.error(404)
return
this_path = m.group(1)
this_page = m.group(3) or 'index'
this_ext = m.group(4) or 'html'
# get section and node
section = Section.all().filter('path =', this_path).get()
node = Node.all().filter('section =', section).get()
if section is None or node is None:
self.error(404)
return
self.request.charset = 'utf8'
# remove the horribleness from comment
if this_page == 'comment' and this_ext == 'html':
# firstly, check the 'faux' field and if something is in there, redirect
faux = self.request.get('faux')
if len(faux) > 0:
logging.info('COMMENT: Spam comment detected (Faux field not empty)')
self.redirect('/')
return
# comment submission for each section
node = Node.get( self.request.get('node') )
name = self.request.get('name')
email = self.request.get('email')
website = self.request.get('website')
comment_text = re.sub('\r', '', self.request.get('comment'));
# if there are more than 4 links (https?://) in the comment, we consider it spam
if spammy_links(comment_text):
logging.info('COMMENT: Spam comment detected (Too many links)')
self.redirect('/')
return
# now create the comment
comment = Comment(
node = node,
name = name,
email = email,
website = website,
comment = comment_text,
)
comment.set_derivatives()
comment.put()
# send a mail to the admin
admin_email = util.config_value('Admin Email')
if mail.is_email_valid(admin_email):
url_post = util.construct_url() + node.section.path + node.name + '.html'
url_mod = util.construct_url() + '/admin/comment/?key=' + str(comment.key()) + ';status='
url_del = util.construct_url() + '/admin/comment/del.html?key='+ str(comment.key())
body = 'From: ' + name + ' <' + email + '>\n'
body = body + 'Site: ' + website + '\n\n'
body = body + comment_text + '\n\n'
body = body + '*** Actions ***\n\n'
body = body + 'ViewPost = ' + url_post + '\n\n'
body = body + 'Approve = ' + url_mod + 'approve\n'
body = body + 'Reject = ' + url_mod + 'reject\n'
body = body + 'Delete = ' + url_del + '\n'
mail.send_mail(admin_email, admin_email, 'New comment on ' + section.path + node.name + '.html', body)
else:
# don't do anything
logging.info('No valid email set, skipping sending admin an email for new comment')
# redirect to the comment page
self.redirect('comment.html?key=' + str(comment.key()))
return
elif this_page == 'message' and this_ext == 'html':
# firstly, check the 'faux' field and if something is in there, redirect
faux = self.request.get('faux')
if len(faux) > 0:
logging.info('MESSAGE: Spam detected, not saving')
self.redirect('/')
return
# message submission for each section
type = self.request.get('type')
subject = self.request.get('subject')
message = self.request.POST.items()
redirect = self.request.get('redirect')
# create the full URL we should be redirecting to
full_redirect = util.construct_redirect( redirect )
# now create the message
msg = Message(
type = type,
subject = subject,
message = message,
)
msg.put()
# send a mail to the admin
admin_email = util.config_value('Admin Email')
if mail.is_email_valid(admin_email):
body = 'type : ' + type + '\n'
body = body + 'subject : ' + subject + '\n'
for k, v in self.request.POST.items():
body = body + k + ' : ' + v + '\n'
mail.send_mail(admin_email, admin_email, '[' + type + '] ' + subject, body)
else:
# don't do anything
logging.info('No valid email set, skipping sending admin an email for new message')
self.redirect(full_redirect)
return
else:
# not found
self.error(404)
return
def page_list(abs_path=''):
base = Node.all().filter('abs_path = ', abs_path).get()
if base:
return Node.get(base.children)
else:
return None
def app_nodes_status():
nodes = Node.all()
statuses = {}
for node in nodes:
statuses[node.host] = node.status()
return json.dumps(statuses)
def app_status():
nodes = Node.all()
containers = {}
for node in nodes:
containers[node.host] = node.containers()
return Response(json.dumps(containers, cls=ContainerJSONEncoder), status=200)
