def backfill_posts(legacy=True):
(username, password) = db.get_credentials('reddit')
reddit.login(username, password)
where = ''
if legacy:
where = 'where legacy = 1'
cur = db.conn.cursor()
query = '''
select id, userid, title, selftext, url, subreddit, over_18, created, legacy, permalink, ups, downs
from posts
%s
order by id
''' % where
total = 0
ids_to_fetch = []
# Store existing values in dict
for (postid, userid, title, selftext, url, subreddit, over_18, created,
legacy, permalink, ups, downs) in cur.execute(query):
ids_to_fetch.append(str(postid))
if len(ids_to_fetch) >= 99:
total += len(ids_to_fetch)
ids_to_fetch.append('1234')
url = 'http://www.reddit.com/by_id/t3_%s.json' % ',t3_'.join(
ids_to_fetch)
try:
posts = reddit.get(url)
except HTTPError, e:
print 'HTTPError: %s' % str(e)
posts = []
for post in posts:
oldpost = {}
oldpost['title'] = post.title
oldpost['url'] = post.url
oldpost['selftext'] = post.selftext
oldpost['subreddit'] = post.subreddit
oldpost['created'] = int(post.created)
oldpost['permalink'] = post.permalink()
oldpost['over_18'] = int(post.over_18)
oldpost['legacy'] = 0
oldpost['id'] = post.id.rjust(6, '0')
oldpost['ups'] = post.ups
oldpost['downs'] = post.downs
Reddit.debug('updating post %s by %s' % (post.id, post.author))
update_post(oldpost)
db.conn.commit()
ids_to_fetch = list()
print 'running total: %d' % total
def backfill_posts(legacy=True):
(username, password) = db.get_credentials('reddit')
reddit.login(username, password)
where = ''
if legacy:
where = 'where legacy = 1'
cur = db.conn.cursor()
query = '''
select id, userid, title, selftext, url, subreddit, over_18, created, legacy, permalink, ups, downs
from posts
%s
order by id
''' % where
total = 0
ids_to_fetch = []
# Store existing values in dict
for (postid, userid, title, selftext, url, subreddit, over_18, created, legacy, permalink, ups, downs) in cur.execute(query):
ids_to_fetch.append(str(postid))
if len(ids_to_fetch) >= 99:
total += len(ids_to_fetch)
ids_to_fetch.append('1234')
url = 'http://www.reddit.com/by_id/t3_%s.json' % ',t3_'.join(ids_to_fetch)
try:
posts = reddit.get(url)
except HTTPError, e:
print 'HTTPError: %s' % str(e)
posts = []
for post in posts:
oldpost = {}
oldpost['title'] = post.title
oldpost['url'] = post.url
oldpost['selftext'] = post.selftext
oldpost['subreddit'] = post.subreddit
oldpost['created'] = int(post.created)
oldpost['permalink'] = post.permalink()
oldpost['over_18'] = int(post.over_18)
oldpost['legacy'] = 0
oldpost['id'] = post.id.rjust(6, '0')
oldpost['ups'] = post.ups
oldpost['downs'] = post.downs
Reddit.debug('updating post %s by %s' % (post.id, post.author))
update_post(oldpost)
db.conn.commit()
ids_to_fetch = list()
print 'running total: %d' % total
print 'HTTPError: %s' % str(e)
posts = []
for post in posts:
oldpost = {}
oldpost['title'] = post.title
oldpost['url'] = post.url
oldpost['selftext'] = post.selftext
oldpost['subreddit'] = post.subreddit
oldpost['created'] = int(post.created)
oldpost['permalink'] = post.permalink()
oldpost['over_18'] = int(post.over_18)
oldpost['legacy'] = 0
oldpost['id'] = post.id.rjust(6, '0')
oldpost['ups'] = post.ups
oldpost['downs'] = post.downs
Reddit.debug('updating post %s by %s' % (post.id, post.author))
update_post(oldpost)
db.conn.commit()
print 'total posts updated: %d' % total
def update_post(post):
query = '''
update posts
set
title = ?,
url = ?,
selftext = ?,
subreddit = ?,
over_18 = ?,
created = ?,
print 'HTTPError: %s' % str(e)
posts = []
for post in posts:
oldpost = {}
oldpost['title'] = post.title
oldpost['url'] = post.url
oldpost['selftext'] = post.selftext
oldpost['subreddit'] = post.subreddit
oldpost['created'] = int(post.created)
oldpost['permalink'] = post.permalink()
oldpost['over_18'] = int(post.over_18)
oldpost['legacy'] = 0
oldpost['id'] = post.id.rjust(6, '0')
oldpost['ups'] = post.ups
oldpost['downs'] = post.downs
Reddit.debug('updating post %s by %s' % (post.id, post.author))
update_post(oldpost)
db.conn.commit()
print 'total posts updated: %d' % total
def update_post(post):
query = '''
update posts
set
title = ?,
url = ?,
selftext = ?,
subreddit = ?,
over_18 = ?,
created = ?,
permalink = ?,
