def vote_poller(url):
print "Starting Vote poller for ",url
components = get_url_components(url)
if not components:
print "Invalid URL : ",url
return
subreddit, uid, postname = components
json_url = "http://www.reddit.com/r/%s/comments/%s.json" % (subreddit,uid)
sr_dir,srinfo_dir = get_subreddit_dirs(subreddit)
record_file = sr_dir + uid
info_file = srinfo_dir + uid
if not record_info_success(json_url,info_file):
print "Could not Record Info! url : %s",json_url
return
JOBS.append(url)
while True:
r = reddit.make_get_request(json_url)
rjson = json.loads(r.content)
ups = get_ups(rjson)
downs = get_downs(rjson)
save_record_entry(record_file, time.time(), ups, downs)
job_ctime = get_file_ctime(record_file)
time_now = time.time()
# if job created over 24 hours ago, kill it
if time_now - job_ctime > ONEDAY:
break
if debug:
print "Successfully polled ",json_url
gevent.sleep(FIVEMINUTES) # sleep for 5 minutes
def record_info_success(url, info_file):
r = reddit.make_get_request(url)
content = r.content
rjson = json.loads(content)
if isinstance(rjson,dict) and "error" in rjson:
print "Request Error : ",rjson["error"]
return False
f = open(info_file,'w')
f.write(content)
f.close()
return True
def start_collection():
jobs = []
jcount = 0
while True:
r = reddit.make_get_request(RINDIA_NEW)
rj = json.loads(r.content)
for post in rj["data"]["children"]:
permalink = "http://www.reddit.com" + post["data"]["permalink"]
if not rrj.job_running(permalink):
jobs += [gevent.spawn(rrj.start,permalink)]
jcount += 1
print "SPAWNED %d JOBS!!" % jcount
gevent.sleep(15*60)
