def get_stories(request, offset):
# We do this stupid offset thing because web requests to heroku timeout after 30s...
# So essentially we just hit /getstories 5 times, grabbing 100 stories each time
offset = int(offset) * 100
top_stories = requests.get("https://hacker-news.firebaseio.com/v0/topstories.json").json()[offset: offset + 100]
start_time = time.time()
# Figure out which stories we don't have saved in our database yet
new_story_ids = []
for story_id in top_stories:
if not Story.objects.filter(story_id=story_id).exists():
new_story_ids.append(story_id)
# The fun part: asynchronously hit HN api for all the info we need using grequests
# This is ~5x faster than synchronous requests for big updates
urls = ["https://hacker-news.firebaseio.com/v0/item/{0}.json".format(story_id) for story_id in new_story_ids]
new_rs = (grequests.get(u, timeout=10) for u in urls)
new_resps = grequests.map(new_rs)
# Store all of our new stories in the database
for resp in new_resps:
Story.create_from_story_resp(resp)
# Go through every organization and figure out which stories they haven't seen yet
for posting_settings in Organization.objects.filter(uninstalled=False):
posted = post_stories(top_stories, posting_settings)
if posted and offset == 400:
settings_link = request.build_absolute_uri(reverse('change_settings', args=[posting_settings.webhook_url_end]))
resp = requests.post(posting_settings.webhook_url, data=json.dumps(
{'text': "_<{0}|You can change your article keywords here.>_".format(settings_link)}))
return HttpResponse("Found new stories in {0} seconds!".format(str(time.time() - start_time)))
