def generate_feed(uid):
subscriptions = Subscription.query(Subscription.uid == uid).fetch(200)
subscription_urls = [sub.url for sub in subscriptions if sub.url]
if len(subscription_urls) > 0:
sources = Source.query(Source.url.IN(subscription_urls)).order(-Source.most_recent_article_added_date).fetch(len(subscription_urls))
source_jsons = {}
for source_json in memcache.get_multi([source.feed_cache_key() for source in sources]).itervalues():
source_jsons[source_json['id']] = source_json
to_fetch = [source for source in sources if source.key.id() not in source_jsons]
print 'HITS {0} TO_FETCH {1}'.format(len(source_jsons), len(to_fetch))
if len(to_fetch):
source_promises = [src.json(include_articles=True, article_limit=FEED_ARTICLE_LIMIT, return_promise=True) for src in to_fetch]
for promise in source_promises:
data = promise()
source_jsons[data['id']] = data
# put the cache keys:
if len(to_fetch):
memcache.set_multi({source.feed_cache_key(): source_jsons[source.key.id()] for source in to_fetch if (source.key.id() in source_jsons)})
source_json = [source_jsons[source.key.id()] for source in sources if source.key.id() in source_jsons]
else:
source_json = []
return {
"sources": source_json
}
def featured_sources_by_category(category=None):
q = Source.query(Source.featured_priority < 1)
if category: q = q.filter(Source.categories == category)
q = q.order(Source.featured_priority)
sources = q.fetch(400)
categories = util.unique_ordered_list(util.flatten(s.categories for s in sources))
if category and category not in categories: categories.append(category)
category_order = {category: i for i, category in enumerate(["Newspapers", "Culture", "Politics", "Tech", "Humor", "Local", "Longform"])}
categories.sort(key=lambda x: category_order.get(x, 99999))
sources_by_category = defaultdict(list)
for source in sources:
for category in source.categories:
sources_by_category[category].append(source)
max_items_per_category = 60 if category else 15
for category, items in sources_by_category.items():
sources_by_category[category] = items[:min(len(items), max_items_per_category)]
category_jsons = []
for category in categories:
category_jsons.append({"id": category, "name": category, "sources": [s.json() for s in sources_by_category[category]]})
return category_jsons
def post(self):
taskqueue.Queue('sources').purge()
time.sleep(1) # TODO: anything but this; we need to wait 1 seconds, but how?
for source in Source.query():
source.most_recent_article_added_date = None
source.enqueue_fetch(rand=True)
self.response.write('done')
def get(self):
vars = {
"featured_sources": Source.query().filter(Source.featured_priority >= 0).order(-Source.featured_priority).fetch()
}
self.response.write(template('sources.html', vars))