def handle(self, *args, **options):
settings.LOG_TO_STREAM = True
now = datetime.datetime.utcnow()
feeds = Feed.objects.filter(
next_scheduled_update__lte=now,
active=True
).exclude(
active_subscribers=0
).order_by('?')
if options['force']:
feeds = Feed.objects.all().order_by('pk')
print " ---> Tasking %s feeds..." % feeds.count()
publisher = Task.get_publisher()
feed_queue = []
size = 12
for f in feeds:
f.queued_date = datetime.datetime.utcnow()
f.set_next_scheduled_update()
for feed_queue in (feeds[pos:pos + size] for pos in xrange(0, len(feeds), size)):
print feed_queue
feed_ids = [feed.pk for feed in feed_queue]
print feed_ids
UpdateFeeds.apply_async(args=(feed_ids,), queue='update_feeds', publisher=publisher)
publisher.connection.close()
def queue_new_feeds(self):
new_feeds = UserSubscription.objects.filter(user=self.user, feed__fetched_once=False).values("feed_id")
new_feeds = list(set([f["feed_id"] for f in new_feeds]))
logging.info(" ---> [%s] Queueing NewFeeds: (%s) %s" % (self.user, len(new_feeds), new_feeds))
size = 4
publisher = Task.get_publisher(exchange="new_feeds")
for t in (new_feeds[pos : pos + size] for pos in xrange(0, len(new_feeds), size)):
NewFeeds.apply_async(args=(t,), queue="new_feeds", publisher=publisher)
publisher.connection.close()
def queue_new_feeds(self, new_feeds=None):
if not new_feeds:
new_feeds = UserSubscription.objects.filter(user=self.user,
feed__fetched_once=False,
active=True).values('feed_id')
new_feeds = list(set([f['feed_id'] for f in new_feeds]))
logging.user(self.user, "~BB~FW~SBQueueing NewFeeds: ~FC(%s) %s" % (len(new_feeds), new_feeds))
size = 4
publisher = Task.get_publisher(exchange="new_feeds")
for t in (new_feeds[pos:pos + size] for pos in xrange(0, len(new_feeds), size)):
NewFeeds.apply_async(args=(t,), queue="new_feeds", publisher=publisher)
publisher.connection.close()
def queue_new_feeds(self, new_feeds=None):
if not new_feeds:
new_feeds = UserSubscription.objects.filter(user=self.user,
feed__fetched_once=False,
active=True).values('feed_id')
new_feeds = list(set([f['feed_id'] for f in new_feeds]))
logging.user(self.user, "~BB~FW~SBQueueing NewFeeds: ~FC(%s) %s" % (len(new_feeds), new_feeds))
size = 4
publisher = Task.get_publisher(exchange="new_feeds")
for t in (new_feeds[pos:pos + size] for pos in xrange(0, len(new_feeds), size)):
NewFeeds.apply_async(args=(t,), queue="new_feeds", publisher=publisher)
publisher.connection.close()
def task_feeds(cls, feeds, queue_size=12):
print " ---> Tasking %s feeds..." % feeds.count()
publisher = Task.get_publisher()
feed_queue = []
for f in feeds:
f.queued_date = datetime.datetime.utcnow()
f.set_next_scheduled_update()
for feed_queue in (feeds[pos : pos + queue_size] for pos in xrange(0, len(feeds), queue_size)):
feed_ids = [feed.pk for feed in feed_queue]
UpdateFeeds.apply_async(args=(feed_ids,), queue="update_feeds", publisher=publisher)
publisher.connection.close()
def task_feeds(cls, feeds, queue_size=12):
print " ---> Tasking %s feeds..." % feeds.count()
publisher = Task.get_publisher()
feed_queue = []
for f in feeds:
f.queued_date = datetime.datetime.utcnow()
f.set_next_scheduled_update()
for feed_queue in (feeds[pos:pos + queue_size]
for pos in xrange(0, len(feeds), queue_size)):
feed_ids = [feed.pk for feed in feed_queue]
UpdateFeeds.apply_async(args=(feed_ids, ),
queue='update_feeds',
publisher=publisher)
publisher.connection.close()