def main(notify): r = praw.Reddit(user_agent='subreddit info grabber 1.0 by u/lungfungus') r.config.log_requests = 2 session = Session() count = session.query(Subreddit).count() for sub_stub in session.query(DiscoveredSub).filter(DiscoveredSub.status == 'NONE'): sub_name = sub_stub.url.split('/')[2].strip() count += 1 try: if sub_name == 'random' or sub_name == None or sub_name == '': raise praw.errors.InvalidSubreddit("random or None") s = r.get_subreddit(sub_name) sub = Subreddit() sub.update_from_praw(s) sub_stub.status = "OK" session.add(sub) session.add(sub_stub) except (praw.errors.RedirectException, praw.errors.InvalidSubreddit) as e: print "ERROR", str(e) sub_stub.status = "E_NO_RETRY" session.add(sub_stub) except praw.requests.exceptions.HTTPError as e: print "ERROR", str(e) if "client error" in e.message.lower(): sub_stub.status = "E_NO_RETRY" else: sub_stub.status = "E_RETRY" session.add(sub_stub) except ValueError as e: print "ERROR", str(e) sub_stub.status = "E_RETRY" session.add(sub_stub) if count % 25 == 0: print "FINISHED %d subreddits" % count session.commit() if count % 2000 == 0: notify("is at %d" % count) if len(session.dirty) > 0: session.commit()
def post_links(): session = Session() data = json.loads(request.data) for link in data['links']: post = Post.get_or_create(session, link['reddit_id']) subreddit = Subreddit.get_or_create(session, link['sub_name']) user = User.get_or_create(session, link['authorname']) post.subreddit_id = subreddit.id post.author_id = user.id post.domain = link['domain'] post.title = link['title'] post.url = link['url'] post.score = link['score'] post.downs = link['downs'] post.ups = link['ups'] post.is_self = link['is_self'] post.over_18 = link['over_18'] post.thumbnail = link['thumbnail'] post.created = float(link['created']) post.scraped_time = Util.now() session.add(post) session.commit() session.close() return jsonify({'received':True})
def main(notify): session = Session() gen = r.get_popular_subreddits(limit=30000) count = session.query(Subreddit).count() notify("Getting subs, initial count: %d" % count) for praw_subreddit in gen: count += 1 subreddit = Subreddit.get_or_create(session, praw_subreddit.display_name) subreddit.update_from_praw(praw_subreddit) session.add(subreddit) session.commit() notify("Now have %d" % count)