def handle(self, *args, **options):
if options['daemonize']:
daemonize()
settings.LOG_TO_STREAM = True
now = datetime.datetime.utcnow()
if options['skip']:
feeds = Feed.objects.filter(
next_scheduled_update__lte=now,
average_stories_per_month__lt=options['skip'],
active=True)
print " ---> Skipping %s feeds" % feeds.count()
for feed in feeds:
feed.set_next_scheduled_update()
print '.',
return
socket.setdefaulttimeout(options['timeout'])
if options['force']:
feeds = Feed.objects.all()
elif options['username']:
feeds = Feed.objects.filter(subscribers__user=User.objects.get(
username=options['username']))
elif options['feed']:
feeds = Feed.objects.filter(pk=options['feed'])
else:
feeds = Feed.objects.filter(next_scheduled_update__lte=now,
active=True)
feeds = feeds.order_by('?')
for f in feeds:
f.set_next_scheduled_update()
num_workers = min(len(feeds), options['workerthreads'])
if options['single_threaded']:
num_workers = 1
options['compute_scores'] = True
options['quick'] = float(MStatistics.get('quick_fetch', 0))
options['updates_off'] = MStatistics.get('updates_off', False)
disp = feed_fetcher.Dispatcher(options, num_workers)
feeds_queue = []
for _ in range(num_workers):
feeds_queue.append([])
i = 0
for feed in feeds:
feeds_queue[i % num_workers].append(feed.pk)
i += 1
disp.add_jobs(feeds_queue, i)
django.db.connection.close()
print " ---> Fetching %s feeds..." % feeds.count()
disp.run_jobs()
def update(self, force=False, single_threaded=True, compute_scores=True):
from utils import feed_fetcher
try:
self.feed_address = self.feed_address % {
'NEWSBLUR_DIR': settings.NEWSBLUR_DIR
}
except:
pass
self.set_next_scheduled_update()
options = {
'verbose': 1 if not force else 2,
'timeout': 10,
'single_threaded': single_threaded,
'force': force,
'compute_scores': compute_scores,
}
disp = feed_fetcher.Dispatcher(options, 1)
disp.add_jobs([[self.pk]])
disp.run_jobs()
def handle(self, *args, **options):
if options['daemonize']:
daemonize()
settings.LOG_TO_STREAM = True
now = datetime.datetime.utcnow()
if options['skip']:
feeds = Feed.objects.filter(
next_scheduled_update__lte=now,
average_stories_per_month__lt=options['skip'],
active=True)
print " ---> Skipping %s feeds" % feeds.count()
for feed in feeds:
feed.set_next_scheduled_update()
print '.',
return
socket.setdefaulttimeout(options['timeout'])
if options['force']:
feeds = Feed.objects.all()
elif options['username']:
feeds = Feed.objects.filter(subscribers__user=User.objects.get(
username=options['username']))
# Added by Xinyan Lu: domain based feed refresh
elif options['domain']:
feeds = Feed.objects.filter(
feed_address__contains=options['domain'])
elif options['id']:
feeds = Feed.objects.filter(id=options['id'])
elif options['all']:
# feeds = Feed.objects.all()
feeds = Feed.objects.filter(num_subscribers__gte=2)
else:
# feeds = Feed.objects.filter(next_scheduled_update__lte=now, active=True)
feeds = Feed.objects.filter(next_scheduled_update__lte=now,
num_subscribers__gte=2)
if options['has_feed_exception']:
feeds = feeds.filter(has_feed_exception=True)
feeds = feeds.order_by('?')
if options['verbose']:
print 'num of feeds:', len(feeds)
for f in feeds:
f.set_next_scheduled_update()
if options['verbose']:
print 'set_next_scheduled_update done'
num_workers = min(len(feeds), options['workerthreads'])
if options['single_threaded']:
num_workers = 1
if options['verbose']:
print 'num_workers', num_workers
options['compute_scores'] = True
# Modified by Xinyan Lu ([email protected]): No MStatistics data available
# options['quick'] = float(MStatistics.get('quick_fetch', 0))
options['quick'] = 0
print options
disp = feed_fetcher.Dispatcher(options, num_workers)
feeds_queue = []
for _ in range(num_workers):
feeds_queue.append([])
i = 0
for feed in feeds:
feeds_queue[i % num_workers].append(feed.pk)
i += 1
disp.add_jobs(feeds_queue, i)
django.db.connection.close()
print " ---> Fetching %s feeds..." % feeds.count()
disp.run_jobs()