def main(notify):
r = praw.Reddit(user_agent='subreddit info grabber 1.0 by u/lungfungus')
r.config.log_requests = 2
session = Session()
count = session.query(Subreddit).count()
for sub_stub in session.query(DiscoveredSub).filter(DiscoveredSub.status == 'NONE'):
sub_name = sub_stub.url.split('/')[2].strip()
count += 1
try:
if sub_name == 'random' or sub_name == None or sub_name == '':
raise praw.errors.InvalidSubreddit("random or None")
s = r.get_subreddit(sub_name)
sub = Subreddit()
sub.update_from_praw(s)
sub_stub.status = "OK"
session.add(sub)
session.add(sub_stub)
except (praw.errors.RedirectException, praw.errors.InvalidSubreddit) as e:
print "ERROR", str(e)
sub_stub.status = "E_NO_RETRY"
session.add(sub_stub)
except praw.requests.exceptions.HTTPError as e:
print "ERROR", str(e)
if "client error" in e.message.lower():
sub_stub.status = "E_NO_RETRY"
else:
sub_stub.status = "E_RETRY"
session.add(sub_stub)
except ValueError as e:
print "ERROR", str(e)
sub_stub.status = "E_RETRY"
session.add(sub_stub)
if count % 25 == 0:
print "FINISHED %d subreddits" % count
session.commit()
if count % 2000 == 0:
notify("is at %d" % count)
if len(session.dirty) > 0:
session.commit()
def post_links():
session = Session()
data = json.loads(request.data)
for link in data['links']:
post = Post.get_or_create(session, link['reddit_id'])
subreddit = Subreddit.get_or_create(session, link['sub_name'])
user = User.get_or_create(session, link['authorname'])
post.subreddit_id = subreddit.id
post.author_id = user.id
post.domain = link['domain']
post.title = link['title']
post.url = link['url']
post.score = link['score']
post.downs = link['downs']
post.ups = link['ups']
post.is_self = link['is_self']
post.over_18 = link['over_18']
post.thumbnail = link['thumbnail']
post.created = float(link['created'])
post.scraped_time = Util.now()
session.add(post)
session.commit()
session.close()
return jsonify({'received':True})
def main(notify):
session = Session()
gen = r.get_popular_subreddits(limit=30000)
count = session.query(Subreddit).count()
notify("Getting subs, initial count: %d" % count)
for praw_subreddit in gen:
count += 1
subreddit = Subreddit.get_or_create(session, praw_subreddit.display_name)
subreddit.update_from_praw(praw_subreddit)
session.add(subreddit)
session.commit()
notify("Now have %d" % count)