def task(stories):
def check_story(rpc):
try:
result = rpc.get_result()
story = json.loads(result.content)
if story and story.get('score') >= SCORE_THRESHOLD:
StoryPost.add(story)
elif story:
logging.info(
'STOP: {id} has low score ({score})'.format(**story))
else:
logging.info('STOP: story was probably deleted/flagged')
except urlfetch.DownloadError as ex:
logging.exception(ex)
except ValueError as ex:
logging.info(result.content)
logging.exception(ex)
# stringify ids for use in memcache and convert to set for later
ids = set(str(story_id) for story_id in stories)
logging.info('checking stories: {}'.format(ids))
# get stories that we already posted to reduce the number of requests
cached_stories = set(memcache.get_multi(ids).keys())
logging.info('cached stories: {}'.format(cached_stories))
# remove stories we know about from stories that we need to check
stories_to_check = ids.difference(cached_stories)
rpcs = map(lambda id: item_async(id, check_story), stories_to_check)
for rpc in rpcs:
rpc.wait()
def task(stories):
def check_story(rpc):
try:
result = rpc.get_result()
story = json.loads(result.content)
if story and story.get('score') >= 100:
StoryPost.add(story)
elif story:
logging.info('STOP: {id} has low score ({score})'.format(**story))
else:
logging.info('STOP: story was probably deleted/flagged')
except urlfetch.DownloadError as ex:
logging.exception(ex)
except ValueError as ex:
logging.info(result.content)
logging.exception(ex)
# stringify ids for use in memcache and convert to set for later
ids = set(str(story_id) for story_id in stories)
logging.info('checking stories: {}'.format(ids))
# get stories that we already posted to reduce the number of requests
cached_stories = set(memcache.get_multi(ids).keys())
logging.info('cached stories: {}'.format(cached_stories))
# remove stories we know about from stories that we need to check
stories_to_check = ids.difference(cached_stories)
rpcs = map(lambda id: item_async(id, check_story), stories_to_check)
for rpc in rpcs:
rpc.wait()