def run():
blog = bf.config.controllers.blog
#Parse the posts
blog.posts = post.parse_posts("_posts")
blog.pages = post.parse_posts("_pages")
blog.dir = bf.util.path_join(bf.writer.output_dir, blog.path)
blog.comments = comments.get_comments()
# Find all the categories and archives before we write any pages
blog.archived_posts = {} ## "/archive/Year/Month" -> [post, post, ... ]
blog.archive_links = [] ## [("/archive/2009/12", name, num_in_archive1), ...] (sorted in reverse by date)
blog.categorized_posts = {} ## "Category Name" -> [post, post, ... ]
blog.all_categories = [] ## [("Category 1",num_in_category_1), ...] (sorted alphabetically)
categories.sort_into_categories()
blog.logger = logging.getLogger(config['name'])
thexnews_legacy.run()
chronological.run()
permapage.run()
page.run()
categories.run()
feed.run()
shortlinks.run()
def post(channel_name, post_id):
if posts.is_deleted(post_id):
return redirect(url_for("channel", channel_name=channel_name))
list = comments.get_comments(post_id)
post = posts.get_post(post_id)
if request.method == "GET":
return render_template("post.html",
comments=list,
post=post,
channel_name=channel_name,
post_id=post_id)
if request.method == "POST":
if session["csrf_token"] != request.form["csrf_token"]:
abort(403)
comment = request.form["comment"]
if len(comment) < 1:
flash("Viesti oli tyhjä")
return redirect(
url_for("post", channel_name=channel_name, post_id=post_id))
elif len(comment) > 5000:
flash("Viestin koko ylitti maksimin! (5000 merkkiä)")
return redirect(
url_for("post", channel_name=channel_name, post_id=post_id))
elif comments.send(comment, post_id) == False:
flash("Viestin lähetys epäonnistui")
return redirect(
url_for("post", channel_name=channel_name, post_id=post_id))
else:
flash("Viesti lähetetty")
return redirect(request.url)
def main():
if not os.path.exists("logs/"):
os.makedirs("logs/")
if not os.path.exists("logs/" + str(int(torrent_id) / 100000) + "xxxxx/"):
os.makedirs("logs/" + str(int(torrent_id) / 100000) + "xxxxx/")
time_log = open("logs/" + str(int(torrent_id) / 100000) + "xxxxx/" + "download.log", 'a')
error_file = open("logs/" + str(int(torrent_id) / 100000) + "xxxxx/" + "errors.log", 'a')
while True:
try:
tp_status_code = torrent_page.get_torrent_page(torrent_id, protocol)
if (tp_status_code == 200):
filelist.get_filelist(torrent_id, protocol)
comments.get_comments(torrent_id, protocol)
elif (tp_status_code == 404):
print "Skipping filelist..."
print "Skipping comments..."
else:
print "ERROR: HTTP " + str(tp_status_code)
error_file.write(datetime.datetime.utcnow().strftime("[%Y-%m-%dT%H:%M:%SZ]") + ' ' + str(torrent_id) + ": ERROR: HTTP " + str(tp_status_code) + '\n')
error_file.flush()
sys.exit(1)
time_log.write(datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + ' ' + str(torrent_id) + " " + str(tp_status_code) + '\n')
time_log.flush()
break # Success! Break out of the while loop
except requests.exceptions.ConnectionError:
print "Connection error. Retrying..."
error_file.write(datetime.datetime.utcnow().strftime("[%Y-%m-%dT%H:%M:%SZ]") + ' ' + str(torrent_id) + ": Connection error" + '\n')
error_file.flush()
tp_status_code = torrent_page.get_torrent_page(torrent_id, protocol)
if (tp_status_code == 200):
filelist.get_filelist(torrent_id, protocol)
comments.get_comments(torrent_id, protocol)
else:
print "Skipping filelist..."
print "Skipping comments..."
time_log.write(datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + ' ' + str(torrent_id) + " " + str(tp_status_code) + '\n')
time_log.flush()
break # Success! Break out of the while loop
except AttributeError:
print "Malformed download. Retrying..."
error_file.write(datetime.datetime.utcnow().strftime("[%Y-%m-%dT%H:%M:%SZ]") + ' ' + str(torrent_id) + ": Malformed download" + '\n')
error_file.flush()
tp_status_code = torrent_page.get_torrent_page(torrent_id, protocol)
if (tp_status_code == 200):
filelist.get_filelist(torrent_id, protocol)
comments.get_comments(torrent_id, protocol)
else:
print "Skipping filelist..."
print "Skipping comments..."
time_log.write(datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + ' ' + str(torrent_id) + " " + str(tp_status_code) + '\n')
time_log.flush()
break # Success! Break out of the while loop
except HTMLParser.HTMLParseError:
time_log.write(datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + ' ' + str(torrent_id) + " BAD" + '\n')
time_log.flush()
break
def browse_specific_challenge(challenge_name):
supported_languages = {'C', 'Python', 'Java', 'C++'}
data = {'C':"// enter code here", 'Python':"# enter code here", 'Java':"// enter code here", 'C++':"// enter code here"}
cur = query_db('select * from challenges where name = ?', [challenge_name], one=True)
if cur is not None:
challenge_id = cur[0]
name = cur[1]
description = cur[2]
sample_tests = cur[5]
input_desc = cur[6]
output_desc = cur[7]
com_flag = cur[8]
else:
abort(404)
challenge_info = {'challenge_id': challenge_id, 'name': name, 'description': description, 'sample_tests': sample_tests,
'input_desc': input_desc, 'output_desc': output_desc, 'languages':supported_languages, 'com_flag':com_flag}
#check for posted comment
if request.method == 'POST':
comment = request.form['comment']
if comment:
post_comment(session['user'], challenge_id, comment)
#Check if it's a redirect from submission and the program
#has produced output
#Stored in session cookie
if 'output' in session:
info = session['output']
session.pop('output', None)
else:
info = None
#check for submitted code from user
if 'code' in session:
code = session['code']
session.pop('code', None)
else:
code = None
if 'language' in session:
language = session['language']
session.pop('language', None)
else:
language = 'C'
# Variables for playlist add
exists = False
playlist_name = request.args.get('playlist_name')
# Prepare playlist information for the dropdown if user logged in
playlists = {}
if 'user' in session:
username = session['user']
# Convert user session to user ID
cur = query_db('select id from users where username = ?', [username], one=True)
if cur is not None:
user_id = cur[0]
# Retrieve the playlists available to this user
cur = query_db('select * from playlists where owner_id = ?', [cur[0]])
# Build a dictionary to pass to the page later
playlists = [dict(id=row[0],name=row[1]) for row in cur]
# Selected to add to playlist
if playlist_name:
add_list = query_db('select * from playlists where owner_id = ? and name = ?',
[user_id, playlist_name], one=True)
# Check if playlist name exists
if add_list is not None:
challenge_ids = add_list[3]
# Check that challenge IDs is not an empty string
if challenge_ids:
id_list = [int(s) for s in challenge_ids.split('|')]
if challenge_id not in id_list:
challenge_ids += "|" + str(challenge_id)
# if challenge id is in list, set the exists flag for the alert
else:
exists = True
# Add id to playlist
else:
challenge_ids = str(challenge_id)
add_str = "update playlists set challenges=? where name=? and owner_id=?;"
update_db(add_str, [challenge_ids,playlist_name,user_id])
# Playlist name not found
else:
playlist_name = ""
else:
abort(401)
#insert the comments section
question_comments = get_comments(challenge_id)
return render_template('challenge.html', challenge_info=challenge_info, output=info, code=code, playlists=playlists,
language=language, comments=question_comments, data=json.dumps(data), playlist_name=playlist_name, exists=exists)
def browse_specific_challenge(challenge_name):
supported_languages = {'C', 'Python', 'Java', 'C++'}
data = {
'C': "// enter code here",
'Python': "# enter code here",
'Java': "// enter code here",
'C++': "// enter code here"
}
cur = query_db('select * from challenges where name = ?', [challenge_name],
one=True)
if cur is not None:
challenge_id = cur[0]
name = cur[1]
description = cur[2]
sample_tests = cur[5]
input_desc = cur[6]
output_desc = cur[7]
com_flag = cur[8]
else:
abort(404)
challenge_info = {
'challenge_id': challenge_id,
'name': name,
'description': description,
'sample_tests': sample_tests,
'input_desc': input_desc,
'output_desc': output_desc,
'languages': supported_languages,
'com_flag': com_flag
}
#check for posted comment
if request.method == 'POST':
comment = request.form['comment']
if comment:
post_comment(session['user'], challenge_id, comment)
#Check if it's a redirect from submission and the program
#has produced output
#Stored in session cookie
if 'output' in session:
info = session['output']
session.pop('output', None)
else:
info = None
#check for submitted code from user
if 'code' in session:
code = session['code']
session.pop('code', None)
else:
code = None
if 'language' in session:
language = session['language']
session.pop('language', None)
else:
language = 'C'
# Variables for playlist add
exists = False
playlist_name = request.args.get('playlist_name')
# Prepare playlist information for the dropdown if user logged in
playlists = {}
if 'user' in session:
username = session['user']
# Convert user session to user ID
cur = query_db('select id from users where username = ?', [username],
one=True)
if cur is not None:
user_id = cur[0]
# Retrieve the playlists available to this user
cur = query_db('select * from playlists where owner_id = ?',
[cur[0]])
# Build a dictionary to pass to the page later
playlists = [dict(id=row[0], name=row[1]) for row in cur]
# Selected to add to playlist
if playlist_name:
add_list = query_db(
'select * from playlists where owner_id = ? and name = ?',
[user_id, playlist_name],
one=True)
# Check if playlist name exists
if add_list is not None:
challenge_ids = add_list[3]
# Check that challenge IDs is not an empty string
if challenge_ids:
id_list = [int(s) for s in challenge_ids.split('|')]
if challenge_id not in id_list:
challenge_ids += "|" + str(challenge_id)
# if challenge id is in list, set the exists flag for the alert
else:
exists = True
# Add id to playlist
else:
challenge_ids = str(challenge_id)
add_str = "update playlists set challenges=? where name=? and owner_id=?;"
update_db(add_str, [challenge_ids, playlist_name, user_id])
# Playlist name not found
else:
playlist_name = ""
else:
abort(401)
#insert the comments section
question_comments = get_comments(challenge_id)
return render_template('challenge.html',
challenge_info=challenge_info,
output=info,
code=code,
playlists=playlists,
language=language,
comments=question_comments,
data=json.dumps(data),
playlist_name=playlist_name,
exists=exists)
# -*- coding: utf-8 -*-
"""
Created on Fri Apr 29 12:00:39 2016
@author: jmzhao
"""
import comments
import gensim as g
with open('../../dataset_python3/scikit.txt') as src_file :
l_comments = comments.get_comments(src_file.read())
documents = [t[1].split('\n\n')[0] for t in l_comments]
# remove common words and tokenize
stoplist = set(open('../../RAKE-tutorial/SmartStoplist.txt').read().split())
stemmer = g.parsing.PorterStemmer()
texts = [[stemmer.stem(word) for word in g.utils.tokenize(document, to_lower=True) if word not in stoplist]
for document in documents]
# remove words that appear only once
from collections import defaultdict
frequency = defaultdict(int)
for text in texts:
for token in text:
frequency[token] += 1
texts = [[token for token in text if frequency[token] > 1]
for text in texts]
from pprint import pprint # pretty-printer
def main():
if not os.path.exists("logs/"):
os.makedirs("logs/")
if not os.path.exists("logs/" + str(int(torrent_id) / 100000) + "xxxxx/"):
os.makedirs("logs/" + str(int(torrent_id) / 100000) + "xxxxx/")
time_log = open(
"logs/" + str(int(torrent_id) / 100000) + "xxxxx/" + "download.log",
'a')
error_file = open(
"logs/" + str(int(torrent_id) / 100000) + "xxxxx/" + "errors.log", 'a')
while True:
try:
tp_status_code = torrent_page.get_torrent_page(
torrent_id, protocol)
if (tp_status_code == 200):
filelist.get_filelist(torrent_id, protocol)
comments.get_comments(torrent_id, protocol)
elif (tp_status_code == 404):
print "Skipping filelist..."
print "Skipping comments..."
else:
print "ERROR: HTTP " + str(tp_status_code)
error_file.write(datetime.datetime.utcnow().strftime(
"[%Y-%m-%dT%H:%M:%SZ]") + ' ' + str(torrent_id) +
": ERROR: HTTP " + str(tp_status_code) + '\n')
error_file.flush()
sys.exit(1)
time_log.write(
datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") +
' ' + str(torrent_id) + " " + str(tp_status_code) + '\n')
time_log.flush()
break # Success! Break out of the while loop
except requests.exceptions.ConnectionError:
print "Connection error. Retrying..."
error_file.write(
datetime.datetime.utcnow().strftime("[%Y-%m-%dT%H:%M:%SZ]") +
' ' + str(torrent_id) + ": Connection error" + '\n')
error_file.flush()
tp_status_code = torrent_page.get_torrent_page(
torrent_id, protocol)
if (tp_status_code == 200):
filelist.get_filelist(torrent_id, protocol)
comments.get_comments(torrent_id, protocol)
else:
print "Skipping filelist..."
print "Skipping comments..."
time_log.write(
datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") +
' ' + str(torrent_id) + " " + str(tp_status_code) + '\n')
time_log.flush()
break # Success! Break out of the while loop
except AttributeError:
print "Malformed download. Retrying..."
error_file.write(
datetime.datetime.utcnow().strftime("[%Y-%m-%dT%H:%M:%SZ]") +
' ' + str(torrent_id) + ": Malformed download" + '\n')
error_file.flush()
tp_status_code = torrent_page.get_torrent_page(
torrent_id, protocol)
if (tp_status_code == 200):
filelist.get_filelist(torrent_id, protocol)
comments.get_comments(torrent_id, protocol)
else:
print "Skipping filelist..."
print "Skipping comments..."
time_log.write(
datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") +
' ' + str(torrent_id) + " " + str(tp_status_code) + '\n')
time_log.flush()
break # Success! Break out of the while loop
except HTMLParser.HTMLParseError:
time_log.write(
datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") +
' ' + str(torrent_id) + " BAD" + '\n')
time_log.flush()
break
