def _process_message():
if not g.sitemap_sqs_queue:
return
sqs = SQSConnection()
sqs_q = sqs.get_queue(g.sitemap_sqs_queue)
messages = sqs.receive_message(sqs_q, number_messages=1)
if not messages:
return
message, = messages
js = json.loads(message.get_body())
s3path = parse_s3_path(js['location'])
# There are some error cases that allow us to get messages
# for sitemap creation that are now out of date.
timestamp = js.get('timestamp')
if timestamp is not None and _before_last_sitemap(timestamp):
sqs_q.delete_message(message)
return
g.log.info("Got import job %r", js)
subreddits = find_all_subreddits(s3path)
store_sitemaps_in_s3(subreddits)
sqs_q.delete_message(message)
def _process_subreddit_sitemaps(s3path, **kw):
# There are some error cases that allow us to get messages
# for sitemap creation that are now out of date.
if 'timestamp' in kw and _is_before_last_sitemap(kw['timestamp']):
return
subreddits = find_all_subreddits(s3path)
store.generate_and_upload_subreddit_sitemaps(subreddits)
def _process_subreddit_sitemaps(s3path, **kw):
# There are some error cases that allow us to get messages
# for sitemap creation that are now out of date.
if 'timestamp' in kw and _is_before_last_sitemap(kw['timestamp']):
return
subreddits = find_all_subreddits(s3path)
store.generate_and_upload_subreddit_sitemaps(subreddits)
def store_sitemaps():
sitemap_updater = SitemapUpdater()
for subreddit_sitemap in subreddit_sitemaps(find_all_subreddits()):
sitemap_updater.add_subreddit_sitemap(subreddit_sitemap)
sitemap_updater.add_sitemap_index(sitemap_index(sitemap_updater.count))
def store_sitemaps():
sitemap_updater = SitemapUpdater()
for subreddit_sitemap in subreddit_sitemaps(find_all_subreddits()):
sitemap_updater.add_subreddit_sitemap(subreddit_sitemap)
sitemap_updater.add_sitemap_index(sitemap_index(sitemap_updater.count))