def monitor(request):
if request.method == 'POST':
key = request.form.get('keyword')
if not key or len(key) < 5 or len(key) > 20:
raise NotFound()
keyword = session.query(Keyword).filter(
Keyword.keyword.ilike(key)).first()
if not keyword:
chars = '1234567890qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM'
hash = ''.join([random.choice(chars) for c in range(5)])
keyword = Keyword(key, hash)
session.commit()
if request.logged_in:
user = session.query(User).filter(
User.name == request.user).first()
monitored = session.query(Monitoring).filter(
Monitoring.user_uid == user.uid).filter(
Monitoring.keyword_uid == keyword.uid).first()
if not monitored:
m = Monitoring(user.uid, keyword.uid)
session.commit()
return redirect(
url_for('monitor', hash=keyword.hash, slug=keyword.keyword))
cachekey = _hash('views_content_monitor')
cached = cache.get(cachekey)
if cached:
return cached
keys = ['bozarking', 'reddit search', 'jailbait']
keywords = session.query(Keyword).filter(Keyword.keyword.in_(keys)).all()
response = serve_response('monitor.html', keywords=keywords)
cache.set(cachekey, response, 10 * 60)
return response
def register(username, passwd):
if re.search(disallowed_chars,
username) or len(username) > 20 or get_user(username):
raise Forbidden
user = User(username, passwd)
session.commit()
return user
def cachedproxy(request, name):
cached = cache.get('submissions_' + name)
if cached:
return cached
subreddit = session.query(Subreddit).filter(
Subreddit.url.ilike(name)).first()
if not subreddit:
raise NotFound
http = httplib2.Http()
uri = 'http://www.reddit.com/r/%s/.json?limit=5' % name
response, content = http.request(uri, 'GET')
if response['status'] == '200':
out = json.loads(content)
if out:
out = out['data']['children']
if not out:
r = serve_json('there doesn\'t seem to be anything here.')
else:
subreddit.all_age_latest = unix_days_ago(
out[0]['data']['created_utc'])
session.commit()
r = serve_response('api/submissions.html', submissions=out)
cache.set('submissions_' + name, r, timeout=60 * 60)
else:
r = serve_json('fetching submissions failed.')
return r
def get(self):
http = httplib2.Http()
headers = self._login()
print headers
images = SoupStrainer('img')
subreddits = session.query(Subreddit).filter(
Subreddit.logo == None).order_by(
Subreddit.subscribers.desc()).all()
for subreddit in subreddits:
url = 'http://www.reddit.com/r/%s' % subreddit.url
response, content = http.request(url, headers=headers)
if response['status'] >= '500':
self.delay.more_exp()
print response['status'], subreddit.url
elif response['status'] >= '400':
subreddit.logo = False
session.commit()
else:
self.delay.less()
soup = BeautifulSoup(content, parseOnlyThese=images)
img_link = soup.findAll(id='header-img')[0]['src']
if img_link == 'http://static.reddit.com/reddit.com.header.png':
subreddit.logo = False
else:
try:
resp, img = http.request(img_link)
f = open(paths.logos + '/' + subreddit.url + '.png',
"w")
f.write(img)
f.close()
subreddit.logo = True
except:
print 'Saving image failed for %s.' % subreddit.url
session.commit()
self.delay.sleep()
def get(self):
http = httplib2.Http()
headers = self._login()
print headers
images = SoupStrainer('img')
subreddits = session.query(Subreddit).filter(Subreddit.logo==None).order_by(Subreddit.subscribers.desc()).all()
for subreddit in subreddits:
url = 'http://www.reddit.com/r/%s' % subreddit.url
response, content = http.request(url, headers=headers)
if response['status'] >= '500':
self.delay.more_exp()
print response['status'], subreddit.url
elif response['status'] >= '400':
subreddit.logo = False
session.commit()
else:
self.delay.less()
soup = BeautifulSoup(content, parseOnlyThese=images)
img_link = soup.findAll(id='header-img')[0]['src']
if img_link == 'http://static.reddit.com/reddit.com.header.png':
subreddit.logo = False
else:
try:
resp, img = http.request(img_link)
f = open(paths.logos + '/' + subreddit.url + '.png', "w")
f.write(img)
f.close()
subreddit.logo = True
except:
print 'Saving image failed for %s.' % subreddit.url
session.commit()
self.delay.sleep()
def monitor(request):
if request.method == "POST":
key = request.form.get("keyword")
if not key or len(key) < 5 or len(key) > 20:
raise NotFound()
keyword = session.query(Keyword).filter(Keyword.keyword.ilike(key)).first()
if not keyword:
chars = "1234567890qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM"
hash = "".join([random.choice(chars) for c in range(5)])
keyword = Keyword(key, hash)
session.commit()
if request.logged_in:
user = session.query(User).filter(User.name == request.user).first()
monitored = (
session.query(Monitoring)
.filter(Monitoring.user_uid == user.uid)
.filter(Monitoring.keyword_uid == keyword.uid)
.first()
)
if not monitored:
m = Monitoring(user.uid, keyword.uid)
session.commit()
return redirect(url_for("monitor", hash=keyword.hash, slug=keyword.keyword))
cachekey = _hash("views_content_monitor")
cached = cache.get(cachekey)
if cached:
return cached
keys = ["bozarking", "reddit search", "jailbait"]
keywords = session.query(Keyword).filter(Keyword.keyword.in_(keys)).all()
response = serve_response("monitor.html", keywords=keywords)
cache.set(cachekey, response, 10 * 60)
return response
def remove_monitoring(request, hash):
user = session.query(User).filter(User.name==request.user).first()
keyword = session.query(Keyword).filter(Keyword.hash==hash).first()
if not user or not keyword:
raise Forbidden()
session.query(Monitoring).filter(Monitoring.user_uid==user.uid).filter(Monitoring.keyword_uid==keyword.uid).delete(synchronize_session=False)
session.commit()
return serve_json('')
def exec_todo(request, user, todo):
r = todo.split('_')
type = r[0]
id = r[1]
action = '_'.join(r[2:])
if type == 'vote':
subreddit_tag = session.query(SubredditTag).filter(
SubredditTag.uid == id).first()
if not subreddit_tag:
raise NotFound
vote = session.query(Vote).filter(Vote.subreddit_tag_id == id).filter(
Vote.user_id == user.uid).first()
if not vote:
vote = Vote(subreddit_tag.uid, user.uid)
if action == 'up':
vote.up()
elif action == 'down':
vote.down()
else:
raise NotFound
session.commit()
return serve_response('api/bigtag.html', tag=subreddit_tag)
elif type == 'tag':
subreddit = session.query(Subreddit).filter_by(id=id).first()
tag = session.query(Tag).filter_by(name=action).first()
if not tag:
tag = Tag()
tag.user_id = user.uid
tag.name = action
session.commit()
#see if this reddit is tagged already
subreddit_tag = session.query(SubredditTag)\
.filter(SubredditTag.tag_id==tag.uid)\
.filter(SubredditTag.subreddit_id==subreddit.uid).first()
if subreddit_tag:
#upvote
vote = session.query(Vote)\
.filter(Vote.subreddit_tag_id==subreddit_tag.uid)\
.filter(Vote.user_id==user.uid).first()
if not vote:
vote = Vote(subreddit_tag.uid, user.uid)
vote.up()
session.commit()
return serve_response('api/bigtag.html', tag=subreddit_tag)
subreddit_tag = SubredditTag()
subreddit_tag.tag_id = tag.uid
subreddit_tag.subreddit_id = subreddit.uid
subreddit_tag.user_id = user.uid
subreddit.tags.append(subreddit_tag)
session.commit()
vote = Vote(subreddit_tag.uid, user.uid)
vote.up()
session.commit()
return serve_response('api/bigtag.html', tag=subreddit_tag)
else:
raise NotFound
def remove_monitoring(request, hash):
user = session.query(User).filter(User.name == request.user).first()
keyword = session.query(Keyword).filter(Keyword.hash == hash).first()
if not user or not keyword:
raise Forbidden()
session.query(Monitoring).filter(Monitoring.user_uid == user.uid).filter(
Monitoring.keyword_uid == keyword.uid).delete(
synchronize_session=False)
session.commit()
return serve_json('')
def _set_new(self, reddit, new):
reddit.updated = datetime.utcnow()
reddit.fp_ups = new['ups']
reddit.fp_downs = new['downs']
reddit.fp_comments = new['comments']
reddit.fp_media = new['media']
reddit.fp_submissions = new['submissions']
reddit.selfposts = new['selfposts']
reddit.fp_oldest = new['oldest']
reddit.all_age_latest = new['latest']
session.commit()
def fix_relative_links():
import re
from myapp.env import session
from myapp.models import Subreddit
make_app()
reddits = session.query(Subreddit).filter(
Subreddit.description.ilike('%]\(%')).all()
for r in reddits:
print r.url
r.description = re.sub(r'\]\\\(', '](', r.description)
session.commit()
def _set_new(self, reddit, new):
reddit.updated = datetime.utcnow()
reddit.fp_ups = new['ups']
reddit.fp_downs = new['downs']
reddit.fp_comments = new['comments']
reddit.fp_media = new['media']
reddit.fp_submissions = new['submissions']
reddit.selfposts = new['selfposts']
reddit.fp_oldest = new['oldest']
reddit.all_age_latest = new['latest']
session.commit()
def exec_todo(request, user, todo):
r = todo.split('_');
type = r[0]
id = r[1]
action = '_'.join(r[2:])
if type == 'vote':
subreddit_tag = session.query(SubredditTag).filter(SubredditTag.uid==id).first()
if not subreddit_tag:
raise NotFound
vote = session.query(Vote).filter(Vote.subreddit_tag_id==id).filter(Vote.user_id==user.uid).first()
if not vote:
vote = Vote(subreddit_tag.uid, user.uid)
if action == 'up':
vote.up()
elif action == 'down':
vote.down()
else:
raise NotFound
session.commit()
return serve_response('api/bigtag.html', tag=subreddit_tag)
elif type == 'tag':
subreddit = session.query(Subreddit).filter_by(id=id).first()
tag = session.query(Tag).filter_by(name=action).first()
if not tag:
tag = Tag()
tag.user_id = user.uid
tag.name = action
session.commit()
#see if this reddit is tagged already
subreddit_tag = session.query(SubredditTag)\
.filter(SubredditTag.tag_id==tag.uid)\
.filter(SubredditTag.subreddit_id==subreddit.uid).first()
if subreddit_tag:
#upvote
vote = session.query(Vote)\
.filter(Vote.subreddit_tag_id==subreddit_tag.uid)\
.filter(Vote.user_id==user.uid).first()
if not vote:
vote = Vote(subreddit_tag.uid, user.uid)
vote.up()
session.commit()
return serve_response('api/bigtag.html', tag=subreddit_tag)
subreddit_tag = SubredditTag()
subreddit_tag.tag_id = tag.uid
subreddit_tag.subreddit_id = subreddit.uid
subreddit_tag.user_id = user.uid
subreddit.tags.append(subreddit_tag)
session.commit()
vote = Vote(subreddit_tag.uid, user.uid)
vote.up()
session.commit()
return serve_response('api/bigtag.html', tag=subreddit_tag)
else:
raise NotFound
def fix_relative_links():
import re
from myapp.env import session
from myapp.models import Subreddit
make_app()
reddits = session.query(Subreddit).filter(Subreddit.description.ilike("%]\(%")).all()
for r in reddits:
print r.url
r.description = re.sub(r"\]\\\(", "](", r.description)
session.commit()
def register_form(request):
# getting and checking input
username = request.form.get('username', None)
pw1 = request.form.get('passwd', None)
pw2 = request.form.get('passwd2', None)
if not (username and pw1 and pw2):
return serve_response('login.html', flashMsg='All fields are required.')
if ' ' in username:
return serve_response('login.html', flashMsg='Username cannot contain spaces.')
if get_user(username):
return serve_response('login.html', flashMsg='Username exists! Please choose another username.')
if pw1 != pw2:
return serve_response('login.html', flashMsg='Passwords don\'t match.')
# create user
user = User(username, pw1)
session.commit()
return login_form(request)
def register_form(request):
# getting and checking input
username = request.form.get('username', None)
pw1 = request.form.get('passwd', None)
pw2 = request.form.get('passwd2', None)
if not (username and pw1 and pw2):
return serve_response('login.html',
flashMsg='All fields are required.')
if ' ' in username:
return serve_response('login.html',
flashMsg='Username cannot contain spaces.')
if get_user(username):
return serve_response(
'login.html',
flashMsg='Username exists! Please choose another username.')
if pw1 != pw2:
return serve_response('login.html', flashMsg='Passwords don\'t match.')
# create user
user = User(username, pw1)
session.commit()
return login_form(request)
def cleandb(self):
for k in self._keywords():
if k.accessed < (datetime.utcnow() - timedelta(30)):
session.query(Mention)\
.filter(Mention.keyword_uid==k.uid)\
.delete(synchronize_session=False)
session.query(Monitoring)\
.filter(Monitoring.keyword_uid==k.uid)\
.delete(synchronize_session=False)
session.query(Keyword)\
.filter(Keyword.uid==k.uid)\
.delete(synchronize_session=False)
elif len(k.mentions) > 50:
m = session.query(Mention)\
.filter(Mention.keyword_uid==k.uid)\
.order_by(Mention.created.desc())\
.offset(49).first()
session.query(Mention)\
.filter(Mention.keyword_uid==k.uid)\
.filter(Mention.uid<m.uid)\
.delete(synchronize_session=False)
session.commit()
def cleandb(self):
for k in self._keywords():
if k.accessed < (datetime.utcnow() - timedelta(30)):
session.query(Mention)\
.filter(Mention.keyword_uid==k.uid)\
.delete(synchronize_session=False)
session.query(Monitoring)\
.filter(Monitoring.keyword_uid==k.uid)\
.delete(synchronize_session=False)
session.query(Keyword)\
.filter(Keyword.uid==k.uid)\
.delete(synchronize_session=False)
elif len(k.mentions) > 50:
m = session.query(Mention)\
.filter(Mention.keyword_uid==k.uid)\
.order_by(Mention.created.desc())\
.offset(49).first()
session.query(Mention)\
.filter(Mention.keyword_uid==k.uid)\
.filter(Mention.uid<m.uid)\
.delete(synchronize_session=False)
session.commit()
def get_new(self):
new_reddits = 0
first_uri = 'http://www.reddit.com/reddits/new/.json'
current_uri = first_uri
counter = 0
while True:
page = self._get_json(current_uri)
if not page:
self.logger.error(
'ERROR retrieving page %s. Spidering aborted.\n'
'%s reddits scanned.\n%s new reddits found.' %
(current_uri, counter, new_reddits))
return
reddits = page['data']['children']
for reddit in reddits:
reddit = reddit['data']
id = reddit['id']
s = session.query(Subreddit).filter_by(id=id).first()
if not s:
self.logger.info('new subreddit: %s' % reddit['url'])
new_reddits += 1
s = Subreddit()
s.name = reddit['name']
s.created = unix_string(int(reddit['created']))
s.url = reddit['url'][3:-1]
s.title = reddit['title']
s.over18 = reddit['over18']
s.subscribers = reddit['subscribers']
s.id = reddit['id']
s.description = reddit['description']
session.commit()
counter += len(reddits)
after = page['data']['after']
current_uri = '%s?count=%s&after=%s' % (first_uri, counter, after)
if not after:
self.logger.info('Finished spidering.\n'
'%s reddits scanned.\n%s new reddits found.' %
(counter, new_reddits))
return
def get_new(self):
new_reddits = 0
first_uri = 'http://www.reddit.com/reddits/new/.json'
current_uri = first_uri
counter = 0
while True:
page = self._get_json(current_uri)
if not page:
self.logger.error('ERROR retrieving page %s. Spidering aborted.\n'
'%s reddits scanned.\n%s new reddits found.'
% (current_uri, counter, new_reddits))
return
reddits = page['data']['children']
for reddit in reddits:
reddit = reddit['data']
id = reddit['id']
s = session.query(Subreddit).filter_by(id=id).first()
if not s:
self.logger.info('new subreddit: %s' % reddit['url'])
new_reddits += 1
s = Subreddit()
s.name = reddit['name']
s.created = unix_string(int(reddit['created']))
s.url = reddit['url'][3:-1]
s.title = reddit['title']
s.over18 = reddit['over18']
s.subscribers = reddit['subscribers']
s.id = reddit['id']
s.description = reddit['description']
session.commit()
counter += len(reddits)
after = page['data']['after']
current_uri = '%s?count=%s&after=%s' % (first_uri, counter, after)
if not after:
self.logger.info('Finished spidering.\n'
'%s reddits scanned.\n%s new reddits found.'
% (counter, new_reddits))
return
def _scan_posts(self, url, newest):
seen = 0
data = self.spider._get_json(url)
posts = data['data']['children']
after = data['data']['after']
for i, c in enumerate(posts):
post = c['data']
if i == 0:
next_newest = post['id']
if post['id'] <= newest:
seen = len(posts) - i
break
title = post['title'].lower()
selftext = post['selftext'].lower()
for k in self._mentioned_keywords(title, text2=selftext):
mention = Mention()
mention.keyword_uid = k.uid
mention.thread_id = post['id']
mention.author = post['author']
mention.subreddit = post['subreddit']
mention.created = unix_string(int(post['created_utc']))
session.commit()
return (seen, after, next_newest)
def _scan_comments(self, url, newest):
seen = 0
data = self.spider._get_json(url)
comments = data['data']['children']
after = data['data']['after']
for i, c in enumerate(comments):
comment = c['data']
if i == 0:
next_newest = comment['id']
if comment['id'] <= newest:
seen = len(comments) - i - 1
break
body = comment['body'].lower()
for k in self._mentioned_keywords(body):
mention = Mention()
mention.keyword_uid = k.uid
mention.thread_id = comment['link_id'][3:]
mention.comment_id = comment['id']
mention.author = comment['author']
mention.subreddit = comment['subreddit']
mention.created = unix_string(int(comment['created_utc']))
session.commit()
return (seen, after, next_newest)
def _scan_posts(self, url, newest):
seen = 0
data = self.spider._get_json(url)
posts = data['data']['children']
after = data['data']['after']
for i, c in enumerate(posts):
post = c['data']
if i == 0:
next_newest = post['id']
if post['id'] <= newest:
seen = len(posts) - i
break
title = post['title'].lower()
selftext = post['selftext'].lower()
for k in self._mentioned_keywords(title, text2=selftext):
mention = Mention()
mention.keyword_uid = k.uid
mention.thread_id = post['id']
mention.author = post['author']
mention.subreddit = post['subreddit']
mention.created = unix_string(int(post['created_utc']))
session.commit()
return (seen, after, next_newest)
def _scan_comments(self, url, newest):
seen = 0
data = self.spider._get_json(url)
comments = data['data']['children']
after = data['data']['after']
for i, c in enumerate(comments):
comment = c['data']
if i == 0:
next_newest = comment['id']
if comment['id'] <= newest:
seen = len(comments) - i - 1
break
body = comment['body'].lower()
for k in self._mentioned_keywords(body):
mention = Mention()
mention.keyword_uid = k.uid
mention.thread_id = comment['link_id'][3:]
mention.comment_id = comment['id']
mention.author = comment['author']
mention.subreddit = comment['subreddit']
mention.created = unix_string(int(comment['created_utc']))
session.commit()
return (seen, after, next_newest)
def cachedproxy(request, name):
cached = cache.get('submissions_' + name)
if cached:
return cached
subreddit = session.query(Subreddit).filter(Subreddit.url.ilike(name)).first()
if not subreddit:
raise NotFound
http = httplib2.Http()
uri = 'http://www.reddit.com/r/%s/.json?limit=5' % name
response, content = http.request(uri, 'GET')
if response['status'] == '200':
out = json.loads(content)
if out:
out = out['data']['children']
if not out:
r = serve_json('there doesn\'t seem to be anything here.')
else:
subreddit.all_age_latest = unix_days_ago(out[0]['data']['created_utc'])
session.commit()
r = serve_response('api/submissions.html', submissions=out)
cache.set('submissions_' + name, r, timeout=60*60)
else:
r = serve_json('fetching submissions failed.')
return r
def register(username, passwd):
if re.search(disallowed_chars, username) or len(username)>20 or get_user(username):
raise Forbidden
user = User(username, passwd)
session.commit()
return user
