def view_y(self):
year = int(self.request.matchdict['year'])
return dict(
posts=Post.by_year(year),
pages=Page.all(),
logged_in=authenticated_userid(self.request),
)
def index(request):
params = {}
if settings.SHOWPAGES:
pages = Page.all()
pages.order("-modified")
posts = pages.fetch(settings.SHOWPAGES)
params['posts'] = posts
return respond(request, 'base.html', params)
else:
home_category = 'Main'
home_page = 'Home'
page = db.GqlQuery("SELECT * FROM Page WHERE name = :1", home_page).get()
if page == None:
if users.get_current_user():
category = db.GqlQuery("SELECT * FROM Category WHERE name = :1", home_category).get()
if category == None:
category = Category(name=home_category, sequence=1)
category.put()
page = Page(name=home_page, category=home_category, sequence=1, content='<h1>Home Page</h1><h5>Change me!</h5>')
page.put()
else:
return HttpResponseRedirect(users.create_login_url(request.get_full_path()))
return viewPage(request, 'Main', 'Home')
def view(self):
year, month, day, route = self._get_route_args()
post = Post.by_route(route)
if post is None:
return HTTPNotFound("No such post")
form = BlogView.CommentForm(self.request.POST)
if self.request.method == 'POST' and form.validate():
if form.parent.data == '':
post_comment = PostComment()
post_comment.comment = Comment(form.name.data, form.email.data, form.body.data)
post.comments.append(post_comment)
else:
parent_id = int(form.parent.data)
parent_comment = Comment.by_id(parent_id)
parent_comment.childs.append(
Comment(
form.name.data,
form.email.data,
form.body.data,
parent_id,
)
)
return HTTPFound(
location=self.request.route_url('post_view',
year=year,
month=month,
day=day,
route=RouteAbleMixin.url_quote(post.title))
)
return dict(
form=form,
post=post,
pages=Page.all(),
logged_in=authenticated_userid(self.request),
)
def main(request):
#visitor = Visitor()
#visitor.ip = request.META["REMOTE_ADDR"]
#visitor.put()
#pages = ['InÃcio', 'Perfil', 'Serviços', 'Contato']
pages = Page.all()
return render_to_response('main.html', {'pages':pages})
def add(self):
form = BlogView.Form(self.request.POST)
if self.request.method == 'POST' and form.validate():
# create post
post = Post.add(
form.title.data,
form.body.data,
form.created.data,
form.is_published.data
)
# add tags
for tag in Tag.from_list(form.tags.data):
nt = NodeTag()
nt.tag = tag
post.tags.append(nt)
return HTTPFound(location=self.request.route_url('post_view',
year=post.created.strftime('%Y'),
month=post.created.strftime('%m'),
day=post.created.strftime('%d'),
route=RouteAbleMixin.url_quote(post.title)
)
)
return dict(
form=form,
url=self.request.route_url('post_add'),
pages=Page.all(),
logged_in=authenticated_userid(self.request),
)
def add(self):
request = self.request
form = ImageView.UploadForm(request.POST)
if request.method == 'POST' and form.validate():
# looks like we will us form here only for validation an html gen
# name = form.image.name doesn't work this way there is always 'image' as name
name = request.POST['image'].filename
path = os.path.join(os.path.dirname(__file__),'files', name)
in_file = request.POST['image'].file
with open(path, "w") as fd:
in_file.seek(0)
while 1:
data = in_file.read(2<<16)
if not data:
break
fd.write(data)
image = Image.add(name)
return HTTPFound(location=request.route_url(
'image_view',
id=image.id,
)
)
return dict(
form=form,
url=self.request.route_url('image_add'),
pages=Page.all(),
logged_in=authenticated_userid(self.request),
)
def edit(self):
year, month, day, route = self._get_route_args()
post = Post.by_route(route)
if post is None:
return HTTPNotFound('No such post')
form = BlogView.Form(self.request.POST, post)
if self.request.method == 'POST' and form.validate():
post.title = form.title.data
post.created = form.created.data
post.body = form.body.data
post.is_published = form.is_published.data
# edit tags
del post.tags
for tag in Tag.from_list(form.tags.data):
nt = NodeTag()
nt.tag = tag
post.tags.append(nt)
return HTTPFound(
location=self.request.route_url('post_view',
year=year,
month=month,
day=day,
route=RouteAbleMixin.url_quote(post.title))
)
return dict(
post=post,
form=form,
pages=Page.all(),
logged_in=authenticated_userid(self.request),
)
def view_ym(self):
year = int(self.request.matchdict['year'])
month = int(self.request.matchdict['month'])
return dict(
posts=Post.by_month(year, month),
pages=Page.all(),
logged_in=authenticated_userid(self.request),
)
def view(self):
route = self.request.matchdict['route']
page = Page.by_route(route)
if page is None:
return HTTPNotFound("No such page")
return dict(
page=page,
pages=Page.all(),
logged_in=authenticated_userid(self.request),
)
def get(self, page_title):
page = Page.all().filter('title =', page_title).get()
data = {
'page': page,
'uri_for': uri_for,
}
template = jinja.get_template('page-edit.html')
return self.response.out.write(template.render(data))
def update_pages():
logging.info('-> updateing pages')
from importing import get_docs_data
docs = get_docs_data(Var.get_value('admin'),Var.get_value('password'))
docs_by_keys = dict((doc['res_id'],doc) for doc in docs)
updated_or_deleted = set()
updated_cnt = deleted_cnt = created_cnt = 0
# updateing
for page in Page.all():
# delete
if not page.res_id in docs_by_keys:
page.delete()
deleted_cnt+=1
logging.info('page %s deleted'%page.res_id)
updated_or_deleted.add( page.res_id )
else:
doc = docs_by_keys[page.res_id]
# update existing
page.slug = doc['slug']
page.lang = doc['lang']
page.title = doc['title']
page.etag = doc['etag']
page.updated = doc['updated']
page.content = get_doc_content(doc['src'])
page.src = doc['src']
page.edit_url = doc['edit_url']
page.put()
logging.info('page %s updated'%doc['res_id'])
updated_cnt+=1
updated_or_deleted.add( page.res_id )
# new pages
new_pages_ids = set(docs_by_keys) - updated_or_deleted
for new_page_id in new_pages_ids:
doc = docs_by_keys[new_page_id]
# create new page
page = Page(key_name=doc['res_id'],
slug = doc['slug'],
lang = doc['lang'],
title = doc['title'],
etag = doc['etag'],
updated = doc['updated'],
src = doc['src'],
edit_url = doc['edit_url'],
content = get_doc_content(doc['src']),
)
page.put()
logging.info('page %s created'%doc['res_id'])
created_cnt+=1
logging.info('<- updateing pages updated=%s created=%s deleted=%s'%(updated_cnt, created_cnt, deleted_cnt))
def view(self):
request = self.request
id = request.matchdict['id']
image = Image.by_id(id)
if image is None:
return HTTPNotFound("No such image")
return dict(
image=image,
pages=Page.all(),
logged_in=authenticated_userid(self.request),
)
def update_cache(self):
""" odswiezenie calego cachu stron """
cnt = 0
for page in Page.all():
ok, _ = page.update_cache()
ok2 = page.update_cache_content()
if ok and ok2:
cnt+=1
self.set_flash(u'Odświeżono w cache %s stron.'%cnt)
return self.redirect_to('admin')
def get(self, page_title='Main Page'):
page = Page.all().filter('title =', page_title).get()
if not page:
page = create_default_page(page_title)
data = {
'page': page,
'uri_for': uri_for,
}
template = jinja.get_template('page.html')
return self.response.out.write(template.render(data))
def get(self):
output = get_cache("home")
if output is None:
items = Page.all().order('-publish_date').fetch(50)
context = {
"items": items,
"title": "Hacker Posts",
}
# calculate the template path
output = self.render("index.html", context)
memcache.add("home", output, 3600)
self.response.out.write(output)
def search(request):
hits = 0
params = {}
s = request.GET.get('s', '')
results = Page.all()
results.search(s)
for result in results:
hits += 1
params['results'] = results
return respond(request, 'wiki/list', params)
def respond(request, template, params=None):
if params is None:
params = {}
if request.LANGUAGE_CODE != 'en':
params['is_foreign'] = request.LANGUAGE_CODE
if not template.endswith('.html'):
template += '.html'
params['skin'] = settings.SKIN
params['app_id'] = settings.APP_ID
session_token = None
user = users.get_current_user()
if user:
params['user'] = user
params['author'] = db.Query(Author).filter('email = ', user.email()).get()
if users.IsCurrentUserAdmin():
params['authsub_url'], session_token, client = check_auth(request, user)
params['is_admin'] = True # users.IsCurrentUserAdmin() # global
params['developer'] = ('Dev' in os.getenv('SERVER_SOFTWARE'))
params['sign_out'] = users.CreateLogoutURL('/')
else:
params['sign_in'] = users.CreateLoginURL(request.path)
params['session_token'] = session_token
# Events
startTime = datetime.datetime.now() + datetime.timedelta(weeks=-2)
endTime = datetime.datetime.now() + datetime.timedelta(weeks=2)
urlparams = 'start-min=%s&start-max=%s' % (startTime.strftime("%Y-%m-%dT%H:%M:%S"), endTime.strftime("%Y-%m-%dT%H:%M:%S"))
url = 'http://www.google.com/calendar/feeds/%s/public/basic?' % settings.GOOGLE_ACCOUNT
url = url + urlparams
feed = urlfetch.fetch(url)
calendar = feedparser.parse(feed.content)
events = calendar['entries']
params['events'] = events
# sidebar categories & page links
categories = Category.all()
categories.order("sequence")
params['categories'] = categories
pages = Page.all()
pages.order("sequence")
params['pages'] = pages
return shortcuts.render_to_response(template, params)
def add(self):
form = PageView.Form(self.request.POST)
if self.request.method == 'POST' and form.validate():
page = Page.add(form.title.data, form.body.data, form.is_published.data)
return HTTPFound(
location=self.request.route_url('page_view',
route=page.route)
)
return dict(
url=self.request.route_url('page_add'),
form=form,
pages=Page.all(),
logged_in=authenticated_userid(self.request),
)
def edit(self):
request = self.request
user = request.user
if user is None:
return HTTPNotFound('User doesn\'t exist')
form = UserView.SettingsForm(request.POST, user)
if request.method == 'POST' and form.validate():
user.password = form.password1.data
return dict(
url=request.route_url('user_edit', id=user.id),
form=form,
pages=Page.all(),
logged_in=authenticated_userid(self.request),
)
def set_on_cdn():
cursor = request.form.get('cursor', None)
pages = Page.all()
if cursor:
pages.with_cursor(start_cursor=cursor)
requeue = False
for page in pages.fetch(100):
requeue = True
page.on_cdn = False
page.put()
page.queue_upload()
if requeue:
taskqueue.add(url=url_for('tasks.set_on_cdn'), params={'cursor': pages.cursor()})
return 'OK'
def get(self):
section = None
if self.request.get('section'):
try:
section = Section.get( self.request.get('section') )
except BadKeyError:
# invalid key
self.redirect('.')
pages = Page.all().filter('section =', section).order('-inserted').fetch(page_count+1)
else:
pages = Page.all().order('-inserted').fetch(page_count+1)
more = True if len(pages) > page_count else False
vals = {
'title' : 'Page List',
'sections' : Section.all(),
'section' : section,
'pages' : pages,
'page_count' : page_count if more else len(pages),
'more' : more,
}
self.template( 'page-list.html', vals, 'admin' );
def post(self, **kwargs):
try:
privatekey = '792ec9274bc52418e995f355e3a8a4d1'
if hashlib.md5(self.request.get("security_data") + privatekey).hexdigest() != self.request.get("security_hash"):
self.abort(500)
referrer = urllib.unquote(self.request.get("referrer"))
referrer = referrer.split('|')
uid = referrer[0]
pages = referrer[1]
subscriber_info = {}
subscriber_info['details'] = self.request.get("details")
subscriber_info['event'] = self.request.get("event")
subscriber_info['productname'] = self.request.get("productname")
subscriber_info['quantity'] = self.request.get("quantity")
subscriber_info['reference'] = self.request.get("reference")
subscriber_info['uid'] = uid
subscriber_info['pages'] = pages
subscriber_info['status'] = self.request.get("status")
subscriber_info['type'] = self.request.get("type")
subscriber_info['enddate'] = self.request.get("enddate")
subscriber_info['nextperioddate'] = self.request.get("nextperioddate")
subscriber_info = simplejson.dumps(subscriber_info)
user = User.get_by_key_name(uid)
user.subscriber_info = subscriber_info
db.put(user)
if self.request.get("event") == 'Active':
pages = pages.split(',');
batch = []
for p in pages:
page = Page.get_by_key_name(p)
if page:
page.upgraded = '1'
page.upgraded_by = user
batch.append(page)
db.put(batch)
if self.request.get("event") == 'Inactive':
pages = Page.all().filter('upgraded_by =', user)
batch = []
for p in pages:
p.upgraded = '0'
p.upgraded_by = None
batch.append(p)
db.put(batch)
self.response.set_status(200)
except:
logging.error('Listen Error')
self.abort(500)
def get(self):
query = self.request.get("q")
if not query:
items = []
title = ""
else:
items = Page.all().search(query).order("-publish_date")
title = " for %s" % query
context = {
"items": items,
"query": self.request.get("q"),
"title": "Search%s" % title
}
output = self.render("search.html", context)
self.response.out.write(output)
def get(self, year, month=None, day=None):
# we want a canonical url with the trailing slash
# so if it's missing we need to throw a 301, adding the slash in
# the process
if self.request.path[-1] != "/":
self.redirect("%s/" % self.request.path, True)
return
output = get_cache("%s%s%s" % (year, month, day))
if output is None:
if not month:
# we're dealing with a year only
# check how many days in this year
if isleap(int(year)):
days = 366
else:
days = 365
lower_limit = date(int(year), 1, 1)
upper_limit = lower_limit + timedelta(days)
title = year
elif not day:
# we have a month and year
lower_limit = date(int(year), int(month), 1)
upper_limit = lower_limit + timedelta(monthrange(int(year), int(month))[1])
title = "%s %s" % (lower_limit.strftime("%B"), year)
else:
# we have everything
lower_limit = date(int(year), int(month), int(day))
upper_limit = lower_limit + timedelta(1)
title = "%s %s %s" % (lower_limit.strftime("%d"), lower_limit.strftime("%B"), year)
items = Page.all()
items.filter('publish_date >', lower_limit)
items.filter('publish_date <', upper_limit)
items.order('-publish_date')
context = {
"items": items,
"title": "News from %s" % title
}
output = self.render("list.html", context)
memcache.add("%s%s%s" % (year, month, day), output, 3600)
self.response.out.write(output)
def edit(self):
route = self.request.matchdict['route']
page = Page.by_route(route)
if page is None:
return HTTPNotFound("No such page")
form = PageView.Form(self.request.POST, page)
if self.request.method == 'POST' and form.validate():
page.title = form.title.data
page.body = form.body.data
page.is_published = form.is_published.data
return HTTPFound(
location=self.request.route_url('page_view',
route=page.route)
)
return dict(
form=form,
page=page,
pages=Page.all(),
logged_in=authenticated_userid(self.request),
)
def get(self, tag):
# we want a canonical url with the trailing slash
# so if it's missing we need to throw a 301, adding the slash in
# the process
if self.request.path[-1] != "/":
self.redirect("%s/" % self.request.path, True)
return
output = get_cache("%s" % tag)
if output is None:
items = Page.all()
items.order('-publish_date')
items.filter('tags = ', tag.replace("-", " "))
context = {
"items": items,
"title": "Content tagged <strong>%s</strong>" % tag.replace("-", " "),
}
output = self.render("list.html", context)
memcache.add("%s" % tag, output, 3600)
self.response.out.write(output)
def post(self, page_title):
page = Page.all().filter('title =', page_title).get()
text = self.request.get('text')
page.text = text
page.put()
self.redirect(uri_for('page', page_title=page_title))
def index():
letters = {}
for page in Page.all().order('-name').fetch(-1):
letters.setdefault(page.name.capitalize()[0], []).append(page)
return web.render_template('index.html', letters=sorted(letters.items()))
print "importing from %s" % feed
feed_obj = feedparser.parse(feed)
# then loop over each entry
for entry in feed_obj['entries']:
title = entry.title
link = entry.links[0].href
# use content if available, otherwise get the summary
try:
content = entry.content[0].value
except AttributeError:
content = entry.summary
# create the object in the datastore only if one with the
# same external link doesn't already exist
if Page.all().filter('external_url =', link).count() == 0:
print " creating %s" % entry.title
try:
# use the Yahoo term extraction API to get tags for the content
srch = TermExtraction(app_id=settings.TERM_EXTRACTION_APP_ID)
srch.context = content
tags = []
for tag in srch.parse_results():
tags.append(tag)
# create out new Page object
item = Page(
title=title,
content=content,
def process_search():
search_query = request.GET.get('search_query', '').strip()
query = search_query.lower()
show_daverank = False
results = False
number_pages = 10
number_videos = 5
#Move this stuff to its own procedure tomorrow!
if query.find('--') == 0:
if query.find('--forum') == 0:
redirect_url = 'http://www.udacity-forums.com/cs101/search/?q=' + urllib.quote(
query[8:])
return redirect(redirect_url)
if query.find('--cs373') == 0:
redirect_url = 'http://www.udacity-forums.com/cs373/search/?q=' + urllib.quote(
query[8:])
return redirect(redirect_url)
if query.find('--python') == 0:
redirect_url = 'http://docs.python.org/search.html?q=' + urllib.quote(
query[9:])
return redirect(redirect_url)
if query.find('--searchwithpeterdotinfo') == 0:
redirect_url = 'http://searchwithpeter.info/secretplans.html?q=' + urllib.quote(
query[25:])
return redirect(redirect_url)
if query.find('--showmore') == 0:
query = query[11:]
search_query = query
number_pages = 20
number_videos = 10
if query.find('--daverank') == 0:
query = query[11:]
search_query = query
show_daverank = True
if query.find('python') == 0:
pyquery = query[7:]
else:
pyquery = query
ddgurl_root = 'http://duckduckgo.com/?q=python+'
ddgurl_suffix = urllib.quote(pyquery) + '&format=json'
response = urllib.urlopen(ddgurl_root + ddgurl_suffix)
response_json = response.read()
pythonterm = json.loads(response_json)
if pythonterm:
pyterm_info = {}
if pythonterm['AbstractSource'] == 'Python Documentation':
pyterm = BeautifulSoup(pythonterm['AbstractText'])
try:
pyterm_code = pyterm.find('code').string
pyterm.pre.decompose()
pyterm_info['code'] = pyterm_code
except:
pyterm_info['code'] = None
pyterm_desc = pyterm.get_text()
pyterm_info['desc'] = pyterm_desc
pyterm_info['url'] = pythonterm['AbstractURL']
results = True
else:
pyterm_info = None
query_words = query.split()
for word in query_words:
if word in stopwords:
query_words.remove(word)
query_urls = []
for term in query_words:
# Get all SearchTerm objects that match the search_query.
q = SearchTerm.all().filter('term =', term).get()
if q:
query_urls.append(set(q.urls))
if query_urls:
query_url_set = set.intersection(*query_urls)
query_url_list = list(query_url_set)
if len(query_url_list) > 0:
results = True
if len(query_url_list) > 30:
query_url_list = query_url_list[0:30]
page_results = Page.all().filter(
'url IN', query_url_list).order('-dave_rank').fetch(number_pages)
page_dicts = []
for page in page_results:
page_info = {}
query_index = page.text.find(query)
if query_index != -1:
i = page.text.find(' ', query_index - 25)
excerpt_words = page.text[i:].split(' ')
page_info['exact_match'] = True
else:
excerpt_words = page.text.split(' ')
page_info['exact_match'] = False
excerpt = ' '.join(excerpt_words[:50])
page_info['text'] = excerpt
page_info['title'] = page.title
page_info['url'] = page.url
page_info['daverank'] = page.dave_rank
page_info['doc'] = page.doc
page_dicts.append(page_info)
page_dicts.sort(key=itemgetter('exact_match'), reverse=True)
video_results = Video.all().filter(
'url IN', query_url_list).order('-views').fetch(number_videos)
video_dicts = []
for video in video_results:
video_info = {}
subtitles = video.text.lower()
query_index = subtitles.find(query)
time_string = ''
if query_index != -1:
subtitle_list = subtitles.splitlines()
for phrase in subtitle_list:
if phrase.find(query) != -1:
timestamp_index = subtitle_list.index(phrase) - 1
timestamp = subtitle_list[timestamp_index]
if len(timestamp) > 1:
minutes = timestamp[3:5]
seconds = timestamp[6:8]
time_string = '#t=' + minutes + 'm' + seconds + 's'
start = 60 * int(minutes) + int(seconds)
if time_string:
url = video.url + time_string
video_info['exact_match'] = True
else:
url = video.url
start = 0
video_info['exact_match'] = False
video_info['title'] = video.title
video_info['url'] = url
video_info['subtitle'] = video.text[-20:query_index:20]
video_info['id'] = video.id
video_info['start'] = start
video_dicts.append(video_info)
video_dicts.sort(key=itemgetter('exact_match'), reverse=True)
else:
page_dicts = None
video_dicts = None
query_string_words = query.split()
return template('templates/results',
search_query=search_query,
query_string_words=query_string_words,
page_dicts=page_dicts,
video_dicts=video_dicts,
pyterm_info=pyterm_info,
show_daverank=show_daverank,
results=results)
def queue_preview_imgs():
for page in Page.all():
page.queue_preview()
def add_channels_to_pages():
for page in Page.all():
page.channels = []
page.put()
def get(self):
pages = Page.all()
self.render('index.jinja', pages=pages)
if len(sys.argv) < 2:
print "Usage: %s [host]" % sys.argv[0]
exit(1)
def auth_func():
"Callable which asks for a username and password"
return raw_input('Username:'******'Password:'******'/remote_api', auth_func, host)
from models import Page
item = Page.all().filter('internal_url =', '/about/')[0]
item.delete()
item = Page(
title="About",
content="""
<p>This software behind this site is a work in progress. As such it doesn't really have a name yet
and the current design is very much just a placeholder.</p>
<p>The eventual plan is something like a cross
between a planet style agregator and an online magazine for a specific context;
say an event or community.</p>
<p>This site is currently seeded with the blog feeds of some of
the top 100 posters on <a href="news.ycombinator.com/">Hacker News</a> taken from their bios.
This seemed to provide an interesting data set with which to experiment but if you want to be
remove then just let me know. If you want to know more please email
<a href="mailto:[email protected]">Gareth</a>.</p>
def latest(request): entries = Page.all() return HttpResponse(entries, content_type='application/atom+xml')