def monitor(request): if request.method == 'POST': key = request.form.get('keyword') if not key or len(key) < 5 or len(key) > 20: raise NotFound() keyword = session.query(Keyword).filter( Keyword.keyword.ilike(key)).first() if not keyword: chars = '1234567890qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM' hash = ''.join([random.choice(chars) for c in range(5)]) keyword = Keyword(key, hash) session.commit() if request.logged_in: user = session.query(User).filter( User.name == request.user).first() monitored = session.query(Monitoring).filter( Monitoring.user_uid == user.uid).filter( Monitoring.keyword_uid == keyword.uid).first() if not monitored: m = Monitoring(user.uid, keyword.uid) session.commit() return redirect( url_for('monitor', hash=keyword.hash, slug=keyword.keyword)) cachekey = _hash('views_content_monitor') cached = cache.get(cachekey) if cached: return cached keys = ['bozarking', 'reddit search', 'jailbait'] keywords = session.query(Keyword).filter(Keyword.keyword.in_(keys)).all() response = serve_response('monitor.html', keywords=keywords) cache.set(cachekey, response, 10 * 60) return response
def register(username, passwd): if re.search(disallowed_chars, username) or len(username) > 20 or get_user(username): raise Forbidden user = User(username, passwd) session.commit() return user
def cachedproxy(request, name): cached = cache.get('submissions_' + name) if cached: return cached subreddit = session.query(Subreddit).filter( Subreddit.url.ilike(name)).first() if not subreddit: raise NotFound http = httplib2.Http() uri = 'http://www.reddit.com/r/%s/.json?limit=5' % name response, content = http.request(uri, 'GET') if response['status'] == '200': out = json.loads(content) if out: out = out['data']['children'] if not out: r = serve_json('there doesn\'t seem to be anything here.') else: subreddit.all_age_latest = unix_days_ago( out[0]['data']['created_utc']) session.commit() r = serve_response('api/submissions.html', submissions=out) cache.set('submissions_' + name, r, timeout=60 * 60) else: r = serve_json('fetching submissions failed.') return r
def get(self): http = httplib2.Http() headers = self._login() print headers images = SoupStrainer('img') subreddits = session.query(Subreddit).filter( Subreddit.logo == None).order_by( Subreddit.subscribers.desc()).all() for subreddit in subreddits: url = 'http://www.reddit.com/r/%s' % subreddit.url response, content = http.request(url, headers=headers) if response['status'] >= '500': self.delay.more_exp() print response['status'], subreddit.url elif response['status'] >= '400': subreddit.logo = False session.commit() else: self.delay.less() soup = BeautifulSoup(content, parseOnlyThese=images) img_link = soup.findAll(id='header-img')[0]['src'] if img_link == 'http://static.reddit.com/reddit.com.header.png': subreddit.logo = False else: try: resp, img = http.request(img_link) f = open(paths.logos + '/' + subreddit.url + '.png', "w") f.write(img) f.close() subreddit.logo = True except: print 'Saving image failed for %s.' % subreddit.url session.commit() self.delay.sleep()
def get(self): http = httplib2.Http() headers = self._login() print headers images = SoupStrainer('img') subreddits = session.query(Subreddit).filter(Subreddit.logo==None).order_by(Subreddit.subscribers.desc()).all() for subreddit in subreddits: url = 'http://www.reddit.com/r/%s' % subreddit.url response, content = http.request(url, headers=headers) if response['status'] >= '500': self.delay.more_exp() print response['status'], subreddit.url elif response['status'] >= '400': subreddit.logo = False session.commit() else: self.delay.less() soup = BeautifulSoup(content, parseOnlyThese=images) img_link = soup.findAll(id='header-img')[0]['src'] if img_link == 'http://static.reddit.com/reddit.com.header.png': subreddit.logo = False else: try: resp, img = http.request(img_link) f = open(paths.logos + '/' + subreddit.url + '.png', "w") f.write(img) f.close() subreddit.logo = True except: print 'Saving image failed for %s.' % subreddit.url session.commit() self.delay.sleep()
def monitor(request): if request.method == "POST": key = request.form.get("keyword") if not key or len(key) < 5 or len(key) > 20: raise NotFound() keyword = session.query(Keyword).filter(Keyword.keyword.ilike(key)).first() if not keyword: chars = "1234567890qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM" hash = "".join([random.choice(chars) for c in range(5)]) keyword = Keyword(key, hash) session.commit() if request.logged_in: user = session.query(User).filter(User.name == request.user).first() monitored = ( session.query(Monitoring) .filter(Monitoring.user_uid == user.uid) .filter(Monitoring.keyword_uid == keyword.uid) .first() ) if not monitored: m = Monitoring(user.uid, keyword.uid) session.commit() return redirect(url_for("monitor", hash=keyword.hash, slug=keyword.keyword)) cachekey = _hash("views_content_monitor") cached = cache.get(cachekey) if cached: return cached keys = ["bozarking", "reddit search", "jailbait"] keywords = session.query(Keyword).filter(Keyword.keyword.in_(keys)).all() response = serve_response("monitor.html", keywords=keywords) cache.set(cachekey, response, 10 * 60) return response
def remove_monitoring(request, hash): user = session.query(User).filter(User.name==request.user).first() keyword = session.query(Keyword).filter(Keyword.hash==hash).first() if not user or not keyword: raise Forbidden() session.query(Monitoring).filter(Monitoring.user_uid==user.uid).filter(Monitoring.keyword_uid==keyword.uid).delete(synchronize_session=False) session.commit() return serve_json('')
def exec_todo(request, user, todo): r = todo.split('_') type = r[0] id = r[1] action = '_'.join(r[2:]) if type == 'vote': subreddit_tag = session.query(SubredditTag).filter( SubredditTag.uid == id).first() if not subreddit_tag: raise NotFound vote = session.query(Vote).filter(Vote.subreddit_tag_id == id).filter( Vote.user_id == user.uid).first() if not vote: vote = Vote(subreddit_tag.uid, user.uid) if action == 'up': vote.up() elif action == 'down': vote.down() else: raise NotFound session.commit() return serve_response('api/bigtag.html', tag=subreddit_tag) elif type == 'tag': subreddit = session.query(Subreddit).filter_by(id=id).first() tag = session.query(Tag).filter_by(name=action).first() if not tag: tag = Tag() tag.user_id = user.uid tag.name = action session.commit() #see if this reddit is tagged already subreddit_tag = session.query(SubredditTag)\ .filter(SubredditTag.tag_id==tag.uid)\ .filter(SubredditTag.subreddit_id==subreddit.uid).first() if subreddit_tag: #upvote vote = session.query(Vote)\ .filter(Vote.subreddit_tag_id==subreddit_tag.uid)\ .filter(Vote.user_id==user.uid).first() if not vote: vote = Vote(subreddit_tag.uid, user.uid) vote.up() session.commit() return serve_response('api/bigtag.html', tag=subreddit_tag) subreddit_tag = SubredditTag() subreddit_tag.tag_id = tag.uid subreddit_tag.subreddit_id = subreddit.uid subreddit_tag.user_id = user.uid subreddit.tags.append(subreddit_tag) session.commit() vote = Vote(subreddit_tag.uid, user.uid) vote.up() session.commit() return serve_response('api/bigtag.html', tag=subreddit_tag) else: raise NotFound
def remove_monitoring(request, hash): user = session.query(User).filter(User.name == request.user).first() keyword = session.query(Keyword).filter(Keyword.hash == hash).first() if not user or not keyword: raise Forbidden() session.query(Monitoring).filter(Monitoring.user_uid == user.uid).filter( Monitoring.keyword_uid == keyword.uid).delete( synchronize_session=False) session.commit() return serve_json('')
def _set_new(self, reddit, new): reddit.updated = datetime.utcnow() reddit.fp_ups = new['ups'] reddit.fp_downs = new['downs'] reddit.fp_comments = new['comments'] reddit.fp_media = new['media'] reddit.fp_submissions = new['submissions'] reddit.selfposts = new['selfposts'] reddit.fp_oldest = new['oldest'] reddit.all_age_latest = new['latest'] session.commit()
def fix_relative_links(): import re from myapp.env import session from myapp.models import Subreddit make_app() reddits = session.query(Subreddit).filter( Subreddit.description.ilike('%]\(%')).all() for r in reddits: print r.url r.description = re.sub(r'\]\\\(', '](', r.description) session.commit()
def exec_todo(request, user, todo): r = todo.split('_'); type = r[0] id = r[1] action = '_'.join(r[2:]) if type == 'vote': subreddit_tag = session.query(SubredditTag).filter(SubredditTag.uid==id).first() if not subreddit_tag: raise NotFound vote = session.query(Vote).filter(Vote.subreddit_tag_id==id).filter(Vote.user_id==user.uid).first() if not vote: vote = Vote(subreddit_tag.uid, user.uid) if action == 'up': vote.up() elif action == 'down': vote.down() else: raise NotFound session.commit() return serve_response('api/bigtag.html', tag=subreddit_tag) elif type == 'tag': subreddit = session.query(Subreddit).filter_by(id=id).first() tag = session.query(Tag).filter_by(name=action).first() if not tag: tag = Tag() tag.user_id = user.uid tag.name = action session.commit() #see if this reddit is tagged already subreddit_tag = session.query(SubredditTag)\ .filter(SubredditTag.tag_id==tag.uid)\ .filter(SubredditTag.subreddit_id==subreddit.uid).first() if subreddit_tag: #upvote vote = session.query(Vote)\ .filter(Vote.subreddit_tag_id==subreddit_tag.uid)\ .filter(Vote.user_id==user.uid).first() if not vote: vote = Vote(subreddit_tag.uid, user.uid) vote.up() session.commit() return serve_response('api/bigtag.html', tag=subreddit_tag) subreddit_tag = SubredditTag() subreddit_tag.tag_id = tag.uid subreddit_tag.subreddit_id = subreddit.uid subreddit_tag.user_id = user.uid subreddit.tags.append(subreddit_tag) session.commit() vote = Vote(subreddit_tag.uid, user.uid) vote.up() session.commit() return serve_response('api/bigtag.html', tag=subreddit_tag) else: raise NotFound
def fix_relative_links(): import re from myapp.env import session from myapp.models import Subreddit make_app() reddits = session.query(Subreddit).filter(Subreddit.description.ilike("%]\(%")).all() for r in reddits: print r.url r.description = re.sub(r"\]\\\(", "](", r.description) session.commit()
def register_form(request): # getting and checking input username = request.form.get('username', None) pw1 = request.form.get('passwd', None) pw2 = request.form.get('passwd2', None) if not (username and pw1 and pw2): return serve_response('login.html', flashMsg='All fields are required.') if ' ' in username: return serve_response('login.html', flashMsg='Username cannot contain spaces.') if get_user(username): return serve_response('login.html', flashMsg='Username exists! Please choose another username.') if pw1 != pw2: return serve_response('login.html', flashMsg='Passwords don\'t match.') # create user user = User(username, pw1) session.commit() return login_form(request)
def register_form(request): # getting and checking input username = request.form.get('username', None) pw1 = request.form.get('passwd', None) pw2 = request.form.get('passwd2', None) if not (username and pw1 and pw2): return serve_response('login.html', flashMsg='All fields are required.') if ' ' in username: return serve_response('login.html', flashMsg='Username cannot contain spaces.') if get_user(username): return serve_response( 'login.html', flashMsg='Username exists! Please choose another username.') if pw1 != pw2: return serve_response('login.html', flashMsg='Passwords don\'t match.') # create user user = User(username, pw1) session.commit() return login_form(request)
def cleandb(self): for k in self._keywords(): if k.accessed < (datetime.utcnow() - timedelta(30)): session.query(Mention)\ .filter(Mention.keyword_uid==k.uid)\ .delete(synchronize_session=False) session.query(Monitoring)\ .filter(Monitoring.keyword_uid==k.uid)\ .delete(synchronize_session=False) session.query(Keyword)\ .filter(Keyword.uid==k.uid)\ .delete(synchronize_session=False) elif len(k.mentions) > 50: m = session.query(Mention)\ .filter(Mention.keyword_uid==k.uid)\ .order_by(Mention.created.desc())\ .offset(49).first() session.query(Mention)\ .filter(Mention.keyword_uid==k.uid)\ .filter(Mention.uid<m.uid)\ .delete(synchronize_session=False) session.commit()
def get_new(self): new_reddits = 0 first_uri = 'http://www.reddit.com/reddits/new/.json' current_uri = first_uri counter = 0 while True: page = self._get_json(current_uri) if not page: self.logger.error( 'ERROR retrieving page %s. Spidering aborted.\n' '%s reddits scanned.\n%s new reddits found.' % (current_uri, counter, new_reddits)) return reddits = page['data']['children'] for reddit in reddits: reddit = reddit['data'] id = reddit['id'] s = session.query(Subreddit).filter_by(id=id).first() if not s: self.logger.info('new subreddit: %s' % reddit['url']) new_reddits += 1 s = Subreddit() s.name = reddit['name'] s.created = unix_string(int(reddit['created'])) s.url = reddit['url'][3:-1] s.title = reddit['title'] s.over18 = reddit['over18'] s.subscribers = reddit['subscribers'] s.id = reddit['id'] s.description = reddit['description'] session.commit() counter += len(reddits) after = page['data']['after'] current_uri = '%s?count=%s&after=%s' % (first_uri, counter, after) if not after: self.logger.info('Finished spidering.\n' '%s reddits scanned.\n%s new reddits found.' % (counter, new_reddits)) return
def get_new(self): new_reddits = 0 first_uri = 'http://www.reddit.com/reddits/new/.json' current_uri = first_uri counter = 0 while True: page = self._get_json(current_uri) if not page: self.logger.error('ERROR retrieving page %s. Spidering aborted.\n' '%s reddits scanned.\n%s new reddits found.' % (current_uri, counter, new_reddits)) return reddits = page['data']['children'] for reddit in reddits: reddit = reddit['data'] id = reddit['id'] s = session.query(Subreddit).filter_by(id=id).first() if not s: self.logger.info('new subreddit: %s' % reddit['url']) new_reddits += 1 s = Subreddit() s.name = reddit['name'] s.created = unix_string(int(reddit['created'])) s.url = reddit['url'][3:-1] s.title = reddit['title'] s.over18 = reddit['over18'] s.subscribers = reddit['subscribers'] s.id = reddit['id'] s.description = reddit['description'] session.commit() counter += len(reddits) after = page['data']['after'] current_uri = '%s?count=%s&after=%s' % (first_uri, counter, after) if not after: self.logger.info('Finished spidering.\n' '%s reddits scanned.\n%s new reddits found.' % (counter, new_reddits)) return
def _scan_posts(self, url, newest): seen = 0 data = self.spider._get_json(url) posts = data['data']['children'] after = data['data']['after'] for i, c in enumerate(posts): post = c['data'] if i == 0: next_newest = post['id'] if post['id'] <= newest: seen = len(posts) - i break title = post['title'].lower() selftext = post['selftext'].lower() for k in self._mentioned_keywords(title, text2=selftext): mention = Mention() mention.keyword_uid = k.uid mention.thread_id = post['id'] mention.author = post['author'] mention.subreddit = post['subreddit'] mention.created = unix_string(int(post['created_utc'])) session.commit() return (seen, after, next_newest)
def _scan_comments(self, url, newest): seen = 0 data = self.spider._get_json(url) comments = data['data']['children'] after = data['data']['after'] for i, c in enumerate(comments): comment = c['data'] if i == 0: next_newest = comment['id'] if comment['id'] <= newest: seen = len(comments) - i - 1 break body = comment['body'].lower() for k in self._mentioned_keywords(body): mention = Mention() mention.keyword_uid = k.uid mention.thread_id = comment['link_id'][3:] mention.comment_id = comment['id'] mention.author = comment['author'] mention.subreddit = comment['subreddit'] mention.created = unix_string(int(comment['created_utc'])) session.commit() return (seen, after, next_newest)
def cachedproxy(request, name): cached = cache.get('submissions_' + name) if cached: return cached subreddit = session.query(Subreddit).filter(Subreddit.url.ilike(name)).first() if not subreddit: raise NotFound http = httplib2.Http() uri = 'http://www.reddit.com/r/%s/.json?limit=5' % name response, content = http.request(uri, 'GET') if response['status'] == '200': out = json.loads(content) if out: out = out['data']['children'] if not out: r = serve_json('there doesn\'t seem to be anything here.') else: subreddit.all_age_latest = unix_days_ago(out[0]['data']['created_utc']) session.commit() r = serve_response('api/submissions.html', submissions=out) cache.set('submissions_' + name, r, timeout=60*60) else: r = serve_json('fetching submissions failed.') return r
def register(username, passwd): if re.search(disallowed_chars, username) or len(username)>20 or get_user(username): raise Forbidden user = User(username, passwd) session.commit() return user