def calculate_metrics(self):
from apps.statistics.models import MStatistics
return {
'feed_loadtimes_avg_hour': MStatistics.get('latest_avg_time_taken'),
'feeds_loaded_hour': MStatistics.get('latest_sites_loaded'),
}
def run(self, feed_pks, **kwargs):
from apps.rss_feeds.models import Feed
from apps.statistics.models import MStatistics
r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL)
mongodb_replication_lag = int(MStatistics.get('mongodb_replication_lag', 0))
compute_scores = bool(mongodb_replication_lag < 10)
profiler = DBProfilerMiddleware()
profiler_activated = profiler.process_celery()
if profiler_activated:
mongo_middleware = MongoDumpMiddleware()
mongo_middleware.process_celery(profiler)
redis_middleware = RedisDumpMiddleware()
redis_middleware.process_celery(profiler)
options = {
'quick': float(MStatistics.get('quick_fetch', 0)),
'updates_off': MStatistics.get('updates_off', False),
'compute_scores': compute_scores,
'mongodb_replication_lag': mongodb_replication_lag,
}
if not isinstance(feed_pks, list):
feed_pks = [feed_pks]
for feed_pk in feed_pks:
feed = Feed.get_by_id(feed_pk)
if not feed or feed.pk != int(feed_pk):
logging.info(" ---> ~FRRemoving feed_id %s from tasked_feeds queue, points to %s..." % (feed_pk, feed and feed.pk))
r.zrem('tasked_feeds', feed_pk)
if feed:
feed.update(**options)
if profiler_activated: profiler.process_celery_finished()
def run(self, feed_pks, **kwargs):
from apps.rss_feeds.models import Feed
from apps.statistics.models import MStatistics
r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL)
mongodb_replication_lag = int(
MStatistics.get('mongodb_replication_lag', 0))
compute_scores = bool(mongodb_replication_lag < 10)
options = {
'quick': float(MStatistics.get('quick_fetch', 0)),
'compute_scores': compute_scores,
'mongodb_replication_lag': mongodb_replication_lag,
}
if not isinstance(feed_pks, list):
feed_pks = [feed_pks]
for feed_pk in feed_pks:
feed = Feed.get_by_id(feed_pk)
if not feed or feed.pk != int(feed_pk):
logging.info(
" ---> ~FRRemoving feed_id %s from tasked_feeds queue, points to %s..."
% (feed_pk, feed and feed.pk))
r.zrem('tasked_feeds', feed_pk)
if feed:
feed.update(**options)
def run(self, feed_pks, **kwargs):
from apps.rss_feeds.models import Feed
from apps.statistics.models import MStatistics
mongodb_replication_lag = int(MStatistics.get('mongodb_replication_lag', 0))
compute_scores = bool(mongodb_replication_lag < 10)
options = {
'fake': bool(MStatistics.get('fake_fetch')),
'quick': float(MStatistics.get('quick_fetch', 0)),
'compute_scores': compute_scores,
'mongodb_replication_lag': mongodb_replication_lag,
}
if not isinstance(feed_pks, list):
feed_pks = [feed_pks]
for feed_pk in feed_pks:
try:
feed = Feed.get_by_id(feed_pk)
if not feed:
raise Feed.DoesNotExist
feed.update(**options)
except Feed.DoesNotExist:
logging.info(" ---> Feed doesn't exist: [%s]" % feed_pk)
def run(self, feed_pks, **kwargs):
from apps.rss_feeds.models import Feed
from apps.statistics.models import MStatistics
r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL)
mongodb_replication_lag = int(MStatistics.get('mongodb_replication_lag', 0))
compute_scores = bool(mongodb_replication_lag < 10)
options = {
'quick': float(MStatistics.get('quick_fetch', 0)),
'updates_off': MStatistics.get('updates_off', False),
'compute_scores': compute_scores,
'mongodb_replication_lag': mongodb_replication_lag,
}
if not isinstance(feed_pks, list):
feed_pks = [feed_pks]
for feed_pk in feed_pks:
feed = Feed.get_by_id(feed_pk)
if not feed or feed.pk != int(feed_pk):
logging.info(" ---> ~FRRemoving feed_id %s from tasked_feeds queue, points to %s..." % (feed_pk, feed and feed.pk))
r.zrem('tasked_feeds', feed_pk)
if feed:
feed.update(**options)
def handle(self, *args, **options):
if options['daemonize']:
daemonize()
settings.LOG_TO_STREAM = True
now = datetime.datetime.utcnow()
if options['skip']:
feeds = Feed.objects.filter(
next_scheduled_update__lte=now,
average_stories_per_month__lt=options['skip'],
active=True)
print " ---> Skipping %s feeds" % feeds.count()
for feed in feeds:
feed.set_next_scheduled_update()
print '.',
return
socket.setdefaulttimeout(options['timeout'])
if options['force']:
feeds = Feed.objects.all()
elif options['username']:
feeds = Feed.objects.filter(subscribers__user=User.objects.get(
username=options['username']))
elif options['feed']:
feeds = Feed.objects.filter(pk=options['feed'])
else:
feeds = Feed.objects.filter(next_scheduled_update__lte=now,
active=True)
feeds = feeds.order_by('?')
for f in feeds:
f.set_next_scheduled_update()
num_workers = min(len(feeds), options['workerthreads'])
if options['single_threaded']:
num_workers = 1
options['compute_scores'] = True
options['quick'] = float(MStatistics.get('quick_fetch', 0))
options['updates_off'] = MStatistics.get('updates_off', False)
disp = feed_fetcher.Dispatcher(options, num_workers)
feeds_queue = []
for _ in range(num_workers):
feeds_queue.append([])
i = 0
for feed in feeds:
feeds_queue[i % num_workers].append(feed.pk)
i += 1
disp.add_jobs(feeds_queue, i)
django.db.connection.close()
print " ---> Fetching %s feeds..." % feeds.count()
disp.run_jobs()
def calculate_metrics(self):
from apps.statistics.models import MStatistics
return {
'feed_loadtimes_avg_hour':
MStatistics.get('latest_avg_time_taken'),
'feeds_loaded_hour': MStatistics.get('latest_sites_loaded'),
}
def calculate_metrics(self):
from apps.statistics.models import MStatistics
return {
'sql_avg': MStatistics.get('latest_sql_avg'),
'mongo_avg': MStatistics.get('latest_mongo_avg'),
'redis_avg': MStatistics.get('latest_redis_avg'),
}
def calculate_metrics(self):
from apps.statistics.models import MStatistics
return {
'sql_avg': MStatistics.get('latest_sql_avg'),
'mongo_avg': MStatistics.get('latest_mongo_avg'),
'redis_avg': MStatistics.get('latest_redis_avg'),
}
def handle(self, *args, **options):
if options['daemonize']:
daemonize()
settings.LOG_TO_STREAM = True
now = datetime.datetime.utcnow()
if options['skip']:
feeds = Feed.objects.filter(next_scheduled_update__lte=now,
average_stories_per_month__lt=options['skip'],
active=True)
print " ---> Skipping %s feeds" % feeds.count()
for feed in feeds:
feed.set_next_scheduled_update()
print '.',
return
socket.setdefaulttimeout(options['timeout'])
if options['force']:
feeds = Feed.objects.all()
elif options['username']:
feeds = Feed.objects.filter(subscribers__user=User.objects.get(username=options['username']))
elif options['feed']:
feeds = Feed.objects.filter(pk=options['feed'])
else:
feeds = Feed.objects.filter(next_scheduled_update__lte=now, active=True)
feeds = feeds.order_by('?')
for f in feeds:
f.set_next_scheduled_update()
num_workers = min(len(feeds), options['workerthreads'])
if options['single_threaded']:
num_workers = 1
options['compute_scores'] = True
options['quick'] = float(MStatistics.get('quick_fetch', 0))
options['updates_off'] = MStatistics.get('updates_off', False)
disp = feed_fetcher.Dispatcher(options, num_workers)
feeds_queue = []
for _ in range(num_workers):
feeds_queue.append([])
i = 0
for feed in feeds:
feeds_queue[i%num_workers].append(feed.pk)
i += 1
disp.add_jobs(feeds_queue, i)
django.db.connection.close()
print " ---> Fetching %s feeds..." % feeds.count()
disp.run_jobs()
def run(self, feed_pks, **kwargs):
from apps.rss_feeds.models import Feed
from apps.statistics.models import MStatistics
options = {"fake": bool(MStatistics.get("fake_fetch")), "quick": float(MStatistics.get("quick_fetch", 0))}
if not isinstance(feed_pks, list):
feed_pks = [feed_pks]
for feed_pk in feed_pks:
try:
feed = Feed.objects.get(pk=feed_pk)
feed.update(options=options)
except Feed.DoesNotExist:
logging.info(" ---> Feed doesn't exist: [%s]" % feed_pk)
def calculate_metrics(self):
from apps.statistics.models import MStatistics
statistics = MStatistics.all()
return {
'feed_success': statistics['feeds_fetched'],
}
def dashboard_graphs(request):
statistics = MStatistics.all()
return render_to_response(
"statistics/render_statistics_graphs.xhtml",
{"statistics": statistics},
context_instance=RequestContext(request),
)
def calculate_metrics(self):
from apps.statistics.models import MStatistics
statistics = MStatistics.all()
return {
'feed_success': statistics['feeds_fetched'],
}
def UpdateFeeds(feed_pks):
from apps.rss_feeds.models import Feed
from apps.statistics.models import MStatistics
r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL)
mongodb_replication_lag = int(MStatistics.get('mongodb_replication_lag',
0))
compute_scores = bool(mongodb_replication_lag < 10)
profiler = DBProfilerMiddleware()
profiler_activated = profiler.process_celery()
if profiler_activated:
mongo_middleware = MongoDumpMiddleware()
mongo_middleware.process_celery(profiler)
redis_middleware = RedisDumpMiddleware()
redis_middleware.process_celery(profiler)
options = {
'quick': float(MStatistics.get('quick_fetch', 0)),
'updates_off': MStatistics.get('updates_off', False),
'compute_scores': compute_scores,
'mongodb_replication_lag': mongodb_replication_lag,
}
if not isinstance(feed_pks, list):
feed_pks = [feed_pks]
for feed_pk in feed_pks:
feed = Feed.get_by_id(feed_pk)
if not feed or feed.pk != int(feed_pk):
logging.info(
" ---> ~FRRemoving feed_id %s from tasked_feeds queue, points to %s..."
% (feed_pk, feed and feed.pk))
r.zrem('tasked_feeds', feed_pk)
if not feed:
continue
try:
feed.update(**options)
except SoftTimeLimitExceeded, e:
feed.save_feed_history(505, 'Timeout', e)
logging.info(
" ---> [%-30s] ~BR~FWTime limit hit!~SB~FR Moving on to next feed..."
% feed)
if profiler_activated: profiler.process_celery_finished()
def index(request):
if request.method == "POST":
if request.POST['submit'] == 'login':
login_form = LoginForm(request.POST, prefix='login')
signup_form = SignupForm(prefix='signup')
else:
login_form = LoginForm(prefix='login')
signup_form = SignupForm(request.POST, prefix='signup')
else:
login_form = LoginForm(prefix='login')
signup_form = SignupForm(prefix='signup')
user = get_user(request)
authed = request.user.is_authenticated()
features = Feature.objects.all()[:3]
feature_form = FeatureForm() if request.user.is_staff else None
feed_count = UserSubscription.objects.filter(
user=request.user).count() if authed else 0
active_count = UserSubscription.objects.filter(
user=request.user, active=True).count() if authed else 0
train_count = UserSubscription.objects.filter(
user=request.user,
active=True,
is_trained=False,
feed__stories_last_month__gte=1).count() if authed else 0
recommended_feeds = RecommendedFeed.objects.filter(
is_public=True,
approved_date__lte=datetime.datetime.now()).select_related('feed')[:2]
unmoderated_feeds = RecommendedFeed.objects.filter(
is_public=False, declined_date__isnull=True).select_related('feed')[:2]
statistics = MStatistics.all()
feedbacks = MFeedback.all()
start_import_from_google_reader = request.session.get(
'import_from_google_reader', False)
if start_import_from_google_reader:
del request.session['import_from_google_reader']
return render_to_response(
'reader/feeds.xhtml', {
'user_profile': hasattr(user, 'profile') and user.profile,
'login_form': login_form,
'signup_form': signup_form,
'feature_form': feature_form,
'features': features,
'feed_count': feed_count,
'active_count': active_count,
'train_count': active_count - train_count,
'account_images': range(1, 4),
'recommended_feeds': recommended_feeds,
'unmoderated_feeds': unmoderated_feeds,
'statistics': statistics,
'feedbacks': feedbacks,
'start_import_from_google_reader': start_import_from_google_reader,
},
context_instance=RequestContext(request))
def get(self, request):
from apps.statistics.models import MStatistics
data = {
'feed_loadtimes_avg_hour': MStatistics.get('latest_avg_time_taken'),
'feeds_loaded_hour': MStatistics.get('latest_sites_loaded'),
}
chart_name = "load_times"
chart_type = "counter"
formatted_data = {}
for k, v in data.items():
formatted_data[k] = f'{chart_name}{{category="{k}"}} {v}'
context = {
"data": formatted_data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request, 'monitor/prometheus_data.html', context, content_type="text/plain")
def run(self, feed_id, xml, **kwargs):
from apps.rss_feeds.models import Feed
from apps.statistics.models import MStatistics
mongodb_replication_lag = int(MStatistics.get('mongodb_replication_lag', 0))
compute_scores = bool(mongodb_replication_lag < 60)
options = {
'feed_xml': xml,
'compute_scores': compute_scores,
'mongodb_replication_lag': mongodb_replication_lag,
}
feed = Feed.get_by_id(feed_id)
feed.update(options=options)
def run(self, feed_pks, **kwargs):
from apps.rss_feeds.models import Feed
from apps.statistics.models import MStatistics
mongodb_replication_lag = int(MStatistics.get('mongodb_replication_lag', 0))
compute_scores = bool(mongodb_replication_lag < 250)
options = {
'fake': bool(MStatistics.get('fake_fetch')),
'quick': float(MStatistics.get('quick_fetch', 0)),
'compute_scores': compute_scores,
'mongodb_replication_lag': mongodb_replication_lag,
}
if not isinstance(feed_pks, list):
feed_pks = [feed_pks]
for feed_pk in feed_pks:
try:
feed = Feed.objects.get(pk=feed_pk)
feed.update(**options)
except Feed.DoesNotExist:
logging.info(" ---> Feed doesn't exist: [%s]" % feed_pk)
def run(self, feed_id, xml, **kwargs):
from apps.rss_feeds.models import Feed
from apps.statistics.models import MStatistics
mongodb_replication_lag = int(MStatistics.get('mongodb_replication_lag', 0))
compute_scores = bool(mongodb_replication_lag < 250)
options = {
'feed_xml': xml,
'compute_scores': compute_scores,
'mongodb_replication_lag': mongodb_replication_lag,
}
feed = Feed.objects.get(pk=feed_id)
feed.update(options=options)
def run(self, feed_pks, **kwargs):
from apps.rss_feeds.models import Feed
from apps.statistics.models import MStatistics
mongodb_replication_lag = int(MStatistics.get("mongodb_replication_lag", 0))
compute_scores = bool(mongodb_replication_lag < 10)
options = {
"fake": bool(MStatistics.get("fake_fetch")),
"quick": float(MStatistics.get("quick_fetch", 0)),
"compute_scores": compute_scores,
"mongodb_replication_lag": mongodb_replication_lag,
}
if not isinstance(feed_pks, list):
feed_pks = [feed_pks]
for feed_pk in feed_pks:
try:
feed = Feed.get_by_id(feed_pk)
feed.update(**options)
except Feed.DoesNotExist:
logging.info(" ---> Feed doesn't exist: [%s]" % feed_pk)
def run(self, feed_pks, **kwargs):
from apps.rss_feeds.models import Feed
from apps.statistics.models import MStatistics
r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL)
mongodb_replication_lag = int(
MStatistics.get('mongodb_replication_lag', 0))
compute_scores = bool(mongodb_replication_lag < 10)
profiler = DBProfilerMiddleware()
profiler_activated = profiler.process_celery()
if profiler_activated:
mongo_middleware = MongoDumpMiddleware()
mongo_middleware.process_celery(profiler)
redis_middleware = RedisDumpMiddleware()
redis_middleware.process_celery(profiler)
options = {
'quick': float(MStatistics.get('quick_fetch', 0)),
'updates_off': MStatistics.get('updates_off', False),
'compute_scores': compute_scores,
'mongodb_replication_lag': mongodb_replication_lag,
}
if not isinstance(feed_pks, list):
feed_pks = [feed_pks]
for feed_pk in feed_pks:
feed = Feed.get_by_id(feed_pk)
if not feed or feed.pk != int(feed_pk):
logging.info(
" ---> ~FRRemoving feed_id %s from tasked_feeds queue, points to %s..."
% (feed_pk, feed and feed.pk))
r.zrem('tasked_feeds', feed_pk)
if feed:
feed.update(**options)
if profiler_activated: profiler.process_celery_finished()
def run(self, feed_pks, **kwargs):
from apps.rss_feeds.models import Feed
from apps.statistics.models import MStatistics
fake = bool(MStatistics.get('fake_fetch'))
if not isinstance(feed_pks, list):
feed_pks = [feed_pks]
for feed_pk in feed_pks:
try:
feed = Feed.objects.get(pk=feed_pk)
feed.update(fake=fake)
except Feed.DoesNotExist:
logging.info(" ---> Feed doesn't exist: [%s]" % feed_pk)
def index(request):
if request.method == "POST":
if request.POST['submit'] == 'login':
login_form = LoginForm(request.POST, prefix='login')
signup_form = SignupForm(prefix='signup')
else:
login_form = LoginForm(prefix='login')
signup_form = SignupForm(request.POST, prefix='signup')
else:
login_form = LoginForm(prefix='login')
signup_form = SignupForm(prefix='signup')
user = get_user(request)
features = Feature.objects.all()[:3]
feature_form = None
if request.user.is_staff:
feature_form = FeatureForm()
feed_count = 0
active_count = 0
train_count = 0
if request.user.is_authenticated():
feed_count = UserSubscription.objects.filter(user=request.user).count()
active_count = UserSubscription.objects.filter(user=request.user, active=True).count()
train_count = UserSubscription.objects.filter(user=request.user, active=True, is_trained=False, feed__stories_last_month__gte=1).count()
now = datetime.datetime.now()
recommended_feeds = RecommendedFeed.objects.filter(is_public=True, approved_date__lte=now).select_related('feed')[:2]
# recommended_feed_feedback = RecommendedFeedUserFeedback.objects.filter(recommendation=recommended_feed)
statistics = MStatistics.all()
howitworks_page = 0 # random.randint(0, 5)
return render_to_response('reader/feeds.xhtml', {
'user_profile': user.profile,
'login_form': login_form,
'signup_form': signup_form,
'feature_form': feature_form,
'features': features,
'start_import_from_google_reader': request.session.get('import_from_google_reader', False),
'howitworks_page': howitworks_page,
'feed_count': feed_count,
'active_count': active_count,
'train_count': active_count - train_count,
'account_images': range(1, 4),
'recommended_feeds': recommended_feeds,
'statistics': statistics,
# 'recommended_feed_feedback': recommended_feed_feedback,
}, context_instance=RequestContext(request))
def PushFeeds(feed_id, xml):
from apps.rss_feeds.models import Feed
from apps.statistics.models import MStatistics
mongodb_replication_lag = int(MStatistics.get('mongodb_replication_lag',
0))
compute_scores = bool(mongodb_replication_lag < 60)
options = {
'feed_xml': xml,
'compute_scores': compute_scores,
'mongodb_replication_lag': mongodb_replication_lag,
}
feed = Feed.get_by_id(feed_id)
if feed:
feed.update(options=options)
def welcome(request, **kwargs):
user = get_user(request)
statistics = MStatistics.all()
social_profile = MSocialProfile.get_user(user.pk)
if request.method == "POST":
pass
else:
login_form = LoginForm(prefix='login')
signup_form = SignupForm(prefix='signup')
return {
'user_profile': hasattr(user, 'profile') and user.profile,
'login_form': login_form,
'signup_form': signup_form,
'statistics': statistics,
'social_profile': social_profile,
'post_request': request.method == 'POST'
}, "reader/welcome.xhtml"
def index(request):
if request.method == "POST":
if request.POST['submit'] == 'login':
login_form = LoginForm(request.POST, prefix='login')
signup_form = SignupForm(prefix='signup')
else:
login_form = LoginForm(prefix='login')
signup_form = SignupForm(request.POST, prefix='signup')
else:
login_form = LoginForm(prefix='login')
signup_form = SignupForm(prefix='signup')
user = get_user(request)
authed = request.user.is_authenticated()
features = Feature.objects.all()[:3]
feature_form = FeatureForm() if request.user.is_staff else None
feed_count = UserSubscription.objects.filter(user=request.user).count() if authed else 0
active_count = UserSubscription.objects.filter(user=request.user, active=True).count() if authed else 0
train_count = UserSubscription.objects.filter(user=request.user, active=True, is_trained=False, feed__stories_last_month__gte=1).count() if authed else 0
recommended_feeds = RecommendedFeed.objects.filter(is_public=True, approved_date__lte=datetime.datetime.now()).select_related('feed')[:2]
unmoderated_feeds = RecommendedFeed.objects.filter(is_public=False, declined_date__isnull=True).select_related('feed')[:2]
statistics = MStatistics.all()
feedbacks = MFeedback.all()
start_import_from_google_reader = request.session.get('import_from_google_reader', False)
if start_import_from_google_reader:
del request.session['import_from_google_reader']
return render_to_response('reader/feeds.xhtml', {
'user_profile' : hasattr(user, 'profile') and user.profile,
'login_form' : login_form,
'signup_form' : signup_form,
'feature_form' : feature_form,
'features' : features,
'feed_count' : feed_count,
'active_count' : active_count,
'train_count' : active_count - train_count,
'account_images' : range(1, 4),
'recommended_feeds' : recommended_feeds,
'unmoderated_feeds' : unmoderated_feeds,
'statistics' : statistics,
'feedbacks' : feedbacks,
'start_import_from_google_reader': start_import_from_google_reader,
}, context_instance=RequestContext(request))
def get(self, request):
statistics = MStatistics.all()
data = {
'feed_success': statistics['feeds_fetched'],
}
chart_name = "errors"
chart_type = "counter"
formatted_data = {}
for k, v in data.items():
formatted_data[k] = f'feed_success {v}'
context = {
"data": formatted_data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request,
'monitor/prometheus_data.html',
context,
content_type="text/plain")
def dashboard(request, **kwargs):
user = request.user
feed_count = UserSubscription.objects.filter(user=user).count()
recommended_feeds = RecommendedFeed.objects.filter(is_public=True,
approved_date__lte=datetime.datetime.now()
).select_related('feed')[:2]
unmoderated_feeds = []
if user.is_staff:
unmoderated_feeds = RecommendedFeed.objects.filter(is_public=False,
declined_date__isnull=True
).select_related('feed')[:2]
statistics = MStatistics.all()
social_profile = MSocialProfile.get_user(user.pk)
start_import_from_google_reader = request.session.get('import_from_google_reader', False)
if start_import_from_google_reader:
del request.session['import_from_google_reader']
if not user.is_active:
url = "https://%s%s" % (Site.objects.get_current().domain,
reverse('stripe-form'))
return HttpResponseRedirect(url)
logging.user(request, "~FBLoading dashboard")
return {
'user_profile': user.profile,
'feed_count': feed_count,
'account_images': range(1,4),
'recommended_feeds': recommended_feeds,
'unmoderated_feeds': unmoderated_feeds,
'statistics': statistics,
'social_profile': social_profile,
'start_import_from_google_reader': start_import_from_google_reader,
'debug': settings.DEBUG
}, "reader/dashboard.xhtml"
def get(self, request):
feeds_count = MStatistics.get('munin:feeds_count')
if not feeds_count:
feeds_count = Feed.objects.all().count()
MStatistics.set('munin:feeds_count', feeds_count, 60 * 60 * 12)
subscriptions_count = MStatistics.get('munin:subscriptions_count')
if not subscriptions_count:
subscriptions_count = UserSubscription.objects.all().count()
MStatistics.set('munin:subscriptions_count', subscriptions_count,
60 * 60 * 12)
data = {
'feeds':
feeds_count,
'subscriptions':
subscriptions_count,
'profiles':
MSocialProfile.objects._collection.count(),
'social_subscriptions':
MSocialSubscription.objects._collection.count(),
}
chart_name = "feeds"
chart_type = "counter"
formatted_data = {}
for k, v in data.items():
formatted_data[k] = f'{chart_name}{{category="{k}"}} {v}'
context = {
"data": formatted_data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request,
'monitor/prometheus_data.html',
context,
content_type="text/plain")
def calculate_metrics(self):
from apps.rss_feeds.models import Feed
from apps.reader.models import UserSubscription
from apps.social.models import MSocialProfile, MSocialSubscription
from apps.statistics.models import MStatistics
feeds_count = MStatistics.get('munin:feeds_count')
if not feeds_count:
feeds_count = Feed.objects.all().count()
MStatistics.set('munin:feeds_count', feeds_count, 60*60*12)
subscriptions_count = MStatistics.get('munin:subscriptions_count')
if not subscriptions_count:
subscriptions_count = UserSubscription.objects.all().count()
MStatistics.set('munin:subscriptions_count', subscriptions_count, 60*60*12)
return {
'feeds': feeds_count,
'subscriptions': subscriptions_count,
'profiles': MSocialProfile.objects.count(),
'social_subscriptions': MSocialSubscription.objects.count(),
}
def calculate_metrics(self):
from apps.rss_feeds.models import Feed
from apps.reader.models import UserSubscription
from apps.social.models import MSocialProfile, MSocialSubscription
from apps.statistics.models import MStatistics
feeds_count = MStatistics.get('munin:feeds_count')
if not feeds_count:
feeds_count = Feed.objects.all().count()
MStatistics.set('munin:feeds_count', feeds_count, 60*60*12)
subscriptions_count = MStatistics.get('munin:subscriptions_count')
if not subscriptions_count:
subscriptions_count = UserSubscription.objects.all().count()
MStatistics.set('munin:subscriptions_count', subscriptions_count, 60*60*12)
return {
'feeds': feeds_count,
'subscriptions': subscriptions_count,
'profiles': MSocialProfile.objects.count(),
'social_subscriptions': MSocialSubscription.objects.count(),
}
def get(self, request):
fs_size_bytes = MStatistics.get('munin:fs_size_bytes')
if not fs_size_bytes:
fs_size_bytes = Feed.objects.aggregate(
Sum('fs_size_bytes'))['fs_size_bytes__sum']
MStatistics.set('munin:fs_size_bytes', fs_size_bytes, 60 * 60 * 12)
archive_users_size_bytes = MStatistics.get(
'munin:archive_users_size_bytes')
if not archive_users_size_bytes:
archive_users_size_bytes = Feed.objects.filter(
archive_subscribers__gte=1).aggregate(
Sum('fs_size_bytes'))['fs_size_bytes__sum']
MStatistics.set('munin:archive_users_size_bytes',
archive_users_size_bytes, 60 * 60 * 12)
data = {
'fs_size_bytes': fs_size_bytes,
'archive_users_size_bytes': archive_users_size_bytes,
}
chart_name = "feed_sizes"
chart_type = "counter"
formatted_data = {}
for k, v in data.items():
formatted_data[k] = f'{chart_name}{{category="{k}"}} {v}'
context = {
"data": formatted_data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request,
'monitor/prometheus_data.html',
context,
content_type="text/plain")
def handle(self, *args, **options):
MStatistics.collect_statistics()
def CollectStats():
logging.debug(" ---> ~FBCollecting stats...")
MStatistics.collect_statistics()
def index(request):
if request.method == "POST":
if request.POST['submit'] == 'login':
login_form = LoginForm(request.POST, prefix='login')
signup_form = SignupForm(prefix='signup')
else:
login_form = LoginForm(prefix='login')
signup_form = SignupForm(request.POST, prefix='signup')
else:
login_form = LoginForm(prefix='login')
signup_form = SignupForm(prefix='signup')
user = get_user(request)
features = Feature.objects.all()[:3]
feature_form = None
if request.user.is_staff:
feature_form = FeatureForm()
feed_count = 0
active_count = 0
train_count = 0
if request.user.is_authenticated():
feed_count = UserSubscription.objects.filter(user=request.user).count()
active_count = UserSubscription.objects.filter(user=request.user,
active=True).count()
train_count = UserSubscription.objects.filter(
user=request.user,
active=True,
is_trained=False,
feed__stories_last_month__gte=1).count()
now = datetime.datetime.now()
recommended_feeds = RecommendedFeed.objects.filter(
is_public=True, approved_date__lte=now).select_related('feed')[:2]
# recommended_feed_feedback = RecommendedFeedUserFeedback.objects.filter(recommendation=recommended_feed)
statistics = MStatistics.all()
howitworks_page = 0 # random.randint(0, 5)
return render_to_response(
'reader/feeds.xhtml',
{
'user_profile':
user.profile,
'login_form':
login_form,
'signup_form':
signup_form,
'feature_form':
feature_form,
'features':
features,
'start_import_from_google_reader':
request.session.get('import_from_google_reader', False),
'howitworks_page':
howitworks_page,
'feed_count':
feed_count,
'active_count':
active_count,
'train_count':
active_count - train_count,
'account_images':
range(1, 4),
'recommended_feeds':
recommended_feeds,
'statistics':
statistics,
# 'recommended_feed_feedback': recommended_feed_feedback,
},
context_instance=RequestContext(request))
def run(self, **kwargs):
# logging.debug(" ---> ~FBCollecting stats...")
MStatistics.collect_statistics()
def dashboard_graphs(request):
statistics = MStatistics.all()
return render_to_response('statistics/render_statistics_graphs.xhtml', {
'statistics': statistics,
}, context_instance=RequestContext(request))
def calculate_metrics(self):
from apps.rss_feeds.models import Feed, DuplicateFeed
from apps.push.models import PushSubscription
from django.conf import settings
from apps.statistics.models import MStatistics
exception_feeds = MStatistics.get('munin:exception_feeds')
if not exception_feeds:
exception_feeds = Feed.objects.filter(has_feed_exception=True).count()
MStatistics.set('munin:exception_feeds', exception_feeds, 60*60*12)
exception_pages = MStatistics.get('munin:exception_pages')
if not exception_pages:
exception_pages = Feed.objects.filter(has_page_exception=True).count()
MStatistics.set('munin:exception_pages', exception_pages, 60*60*12)
duplicate_feeds = MStatistics.get('munin:duplicate_feeds')
if not duplicate_feeds:
duplicate_feeds = DuplicateFeed.objects.count()
MStatistics.set('munin:duplicate_feeds', duplicate_feeds, 60*60*12)
active_feeds = MStatistics.get('munin:active_feeds')
if not active_feeds:
active_feeds = Feed.objects.filter(active_subscribers__gt=0).count()
MStatistics.set('munin:active_feeds', active_feeds, 60*60*12)
push_feeds = MStatistics.get('munin:push_feeds')
if not push_feeds:
push_feeds = PushSubscription.objects.filter(verified=True).count()
MStatistics.set('munin:push_feeds', push_feeds, 60*60*12)
r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL)
return {
'scheduled_feeds': r.zcard('scheduled_updates'),
'exception_feeds': exception_feeds,
'exception_pages': exception_pages,
'duplicate_feeds': duplicate_feeds,
'active_feeds': active_feeds,
'push_feeds': push_feeds,
}
def get(self, request):
exception_feeds = MStatistics.get('munin:exception_feeds')
if not exception_feeds:
exception_feeds = Feed.objects.filter(
has_feed_exception=True).count()
MStatistics.set('munin:exception_feeds', exception_feeds,
60 * 60 * 12)
exception_pages = MStatistics.get('munin:exception_pages')
if not exception_pages:
exception_pages = Feed.objects.filter(
has_page_exception=True).count()
MStatistics.set('munin:exception_pages', exception_pages,
60 * 60 * 12)
duplicate_feeds = MStatistics.get('munin:duplicate_feeds')
if not duplicate_feeds:
duplicate_feeds = DuplicateFeed.objects.count()
MStatistics.set('munin:duplicate_feeds', duplicate_feeds,
60 * 60 * 12)
active_feeds = MStatistics.get('munin:active_feeds')
if not active_feeds:
active_feeds = Feed.objects.filter(
active_subscribers__gt=0).count()
MStatistics.set('munin:active_feeds', active_feeds, 60 * 60 * 12)
push_feeds = MStatistics.get('munin:push_feeds')
if not push_feeds:
push_feeds = PushSubscription.objects.filter(verified=True).count()
MStatistics.set('munin:push_feeds', push_feeds, 60 * 60 * 12)
r = redis.Redis(connection_pool=settings.REDIS_FEED_UPDATE_POOL)
data = {
'scheduled_feeds': r.zcard('scheduled_updates'),
'exception_feeds': exception_feeds,
'exception_pages': exception_pages,
'duplicate_feeds': duplicate_feeds,
'active_feeds': active_feeds,
'push_feeds': push_feeds,
}
chart_name = "feed_counts"
chart_type = "counter"
formatted_data = {}
for k, v in data.items():
formatted_data[k] = f'{chart_name}{{category="{k}"}} {v}'
context = {
"data": formatted_data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request,
'monitor/prometheus_data.html',
context,
content_type="text/plain")
def handle(self, *args, **options):
MStatistics.collect_statistics()
def calculate_metrics(self):
from apps.rss_feeds.models import Feed, DuplicateFeed
from apps.push.models import PushSubscription
from django.conf import settings
from apps.statistics.models import MStatistics
exception_feeds = MStatistics.get('munin:exception_feeds')
if not exception_feeds:
exception_feeds = Feed.objects.filter(
has_feed_exception=True).count()
MStatistics.set('munin:exception_feeds', exception_feeds,
60 * 60 * 12)
exception_pages = MStatistics.get('munin:exception_pages')
if not exception_pages:
exception_pages = Feed.objects.filter(
has_page_exception=True).count()
MStatistics.set('munin:exception_pages', exception_pages,
60 * 60 * 12)
duplicate_feeds = MStatistics.get('munin:duplicate_feeds')
if not duplicate_feeds:
duplicate_feeds = DuplicateFeed.objects.count()
MStatistics.set('munin:duplicate_feeds', duplicate_feeds,
60 * 60 * 12)
active_feeds = MStatistics.get('munin:active_feeds')
if not active_feeds:
active_feeds = Feed.objects.filter(
active_subscribers__gt=0).count()
MStatistics.set('munin:active_feeds', active_feeds, 60 * 60 * 12)
push_feeds = MStatistics.get('munin:push_feeds')
if not push_feeds:
push_feeds = PushSubscription.objects.filter(verified=True).count()
MStatistics.set('munin:push_feeds', push_feeds, 60 * 60 * 12)
r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL)
return {
'scheduled_feeds': r.zcard('scheduled_updates'),
'exception_feeds': exception_feeds,
'exception_pages': exception_pages,
'duplicate_feeds': duplicate_feeds,
'active_feeds': active_feeds,
'push_feeds': push_feeds,
}
def handle(self, *args, **options):
MStatistics.collect_statistics()
MStatistics.delete_old_stats()
def dashboard_graphs(request):
statistics = MStatistics.all()
return render_to_response('statistics/render_statistics_graphs.xhtml', {
'statistics': statistics,
},
context_instance=RequestContext(request))
def get(self, request):
last_year = datetime.datetime.utcnow() - datetime.timedelta(days=365)
last_month = datetime.datetime.utcnow() - datetime.timedelta(days=30)
last_day = datetime.datetime.utcnow() - datetime.timedelta(minutes=60 *
24)
expiration_sec = 60 * 60 # 1 hour
data = {
'all':
MStatistics.get('munin:users_count',
lambda: User.objects.count(),
set_default=True,
expiration_sec=expiration_sec),
'yearly':
MStatistics.get('munin:users_yearly',
lambda: Profile.objects.filter(last_seen_on__gte=
last_year).count(),
set_default=True,
expiration_sec=expiration_sec),
'monthly':
MStatistics.get('munin:users_monthly',
lambda: Profile.objects.filter(last_seen_on__gte=
last_month).count(),
set_default=True,
expiration_sec=expiration_sec),
'daily':
MStatistics.get('munin:users_daily',
lambda: Profile.objects.filter(last_seen_on__gte=
last_day).count(),
set_default=True,
expiration_sec=expiration_sec),
'premium':
MStatistics.get(
'munin:users_premium',
lambda: Profile.objects.filter(is_premium=True).count(),
set_default=True,
expiration_sec=expiration_sec),
'archive':
MStatistics.get(
'munin:users_archive',
lambda: Profile.objects.filter(is_archive=True).count(),
set_default=True,
expiration_sec=expiration_sec),
'pro':
MStatistics.get(
'munin:users_pro',
lambda: Profile.objects.filter(is_pro=True).count(),
set_default=True,
expiration_sec=expiration_sec),
'queued':
MStatistics.get('munin:users_queued',
lambda: RNewUserQueue.user_count(),
set_default=True,
expiration_sec=expiration_sec),
}
chart_name = "users"
chart_type = "counter"
formatted_data = {}
for k, v in data.items():
formatted_data[k] = f'{chart_name}{{category="{k}"}} {v}'
context = {
"data": formatted_data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request,
'monitor/prometheus_data.html',
context,
content_type="text/plain")
def get(self, request):
data = {
'sql_avg':
MStatistics.get('latest_sql_avg'),
'mongo_avg':
MStatistics.get('latest_mongo_avg'),
'redis_avg':
MStatistics.get('latest_redis_avg'),
'redis_user_avg':
MStatistics.get('latest_redis_user_avg'),
'redis_story_avg':
MStatistics.get('latest_redis_story_avg'),
'redis_session_avg':
MStatistics.get('latest_redis_session_avg'),
'redis_pubsub_avg':
MStatistics.get('latest_redis_pubsub_avg'),
'task_sql_avg':
MStatistics.get('latest_task_sql_avg'),
'task_mongo_avg':
MStatistics.get('latest_task_mongo_avg'),
'task_redis_avg':
MStatistics.get('latest_task_redis_avg'),
'task_redis_user_avg':
MStatistics.get('latest_task_redis_user_avg'),
'task_redis_story_avg':
MStatistics.get('latest_task_redis_story_avg'),
'task_redis_session_avg':
MStatistics.get('latest_task_redis_session_avg'),
'task_redis_pubsub_avg':
MStatistics.get('latest_task_redis_pubsub_avg'),
}
chart_name = "db_times"
chart_type = "counter"
formatted_data = {}
for k, v in data.items():
formatted_data[k] = f'{chart_name}{{db="{k}"}} {v}'
context = {
"data": formatted_data,
"chart_name": chart_name,
"chart_type": chart_type,
}
return render(request,
'monitor/prometheus_data.html',
context,
content_type="text/plain")
def dashboard_graphs(request):
statistics = MStatistics.all()
return render(request, 'statistics/render_statistics_graphs.xhtml',
{'statistics': statistics})
def run(self, **kwargs):
logging.debug(" ---> Collecting stats...")
MStatistics.collect_statistics()
MStatistics.delete_old_stats()
class ProcessFeed:
def __init__(self, feed_id, fpf, options, raw_feed=None):
self.feed_id = feed_id
self.options = options
self.fpf = fpf
self.raw_feed = raw_feed
def refresh_feed(self):
self.feed = Feed.get_by_id(self.feed_id)
if self.feed_id != self.feed.pk:
logging.debug(" ***> Feed has changed: from %s to %s" %
(self.feed_id, self.feed.pk))
self.feed_id = self.feed.pk
def process(self):
""" Downloads and parses a feed.
"""
start = time.time()
self.refresh_feed()
ret_values = dict(new=0, updated=0, same=0, error=0)
if hasattr(self.fpf, 'status'):
if self.options['verbose']:
if self.fpf.bozo and self.fpf.status != 304:
logging.debug(
u' ---> [%-30s] ~FRBOZO exception: %s ~SB(%s entries)'
% (self.feed.log_title[:30], self.fpf.bozo_exception,
len(self.fpf.entries)))
if self.fpf.status == 304:
self.feed = self.feed.save()
self.feed.save_feed_history(304, "Not modified")
return FEED_SAME, ret_values
# 302 and 307: Temporary redirect: ignore
# 301 and 308: Permanent redirect: save it (after 10 tries)
if self.fpf.status == 301 or self.fpf.status == 308:
if self.fpf.href.endswith('feedburner.com/atom.xml'):
return FEED_ERRHTTP, ret_values
redirects, non_redirects = self.feed.count_redirects_in_history(
'feed')
self.feed.save_feed_history(
self.fpf.status,
"HTTP Redirect (%d to go)" % (10 - len(redirects)))
if len(redirects) >= 10 or len(non_redirects) == 0:
address = self.fpf.href
if self.options['force'] and address:
address = qurl(address, remove=['_'])
self.feed.feed_address = address
if not self.feed.known_good:
self.feed.fetched_once = True
logging.debug(
" ---> [%-30s] ~SB~SK~FRFeed is %s'ing. Refetching..."
% (self.feed.log_title[:30], self.fpf.status))
self.feed = self.feed.schedule_feed_fetch_immediately()
if not self.fpf.entries:
self.feed = self.feed.save()
self.feed.save_feed_history(self.fpf.status,
"HTTP Redirect")
return FEED_ERRHTTP, ret_values
if self.fpf.status >= 400:
logging.debug(
" ---> [%-30s] ~SB~FRHTTP Status code: %s. Checking address..."
% (self.feed.log_title[:30], self.fpf.status))
fixed_feed = None
if not self.feed.known_good:
fixed_feed, feed = self.feed.check_feed_link_for_feed_address(
)
if not fixed_feed:
self.feed.save_feed_history(self.fpf.status, "HTTP Error")
else:
self.feed = feed
self.feed = self.feed.save()
return FEED_ERRHTTP, ret_values
if not self.fpf:
logging.debug(
" ---> [%-30s] ~SB~FRFeed is Non-XML. No feedparser feed either!"
% (self.feed.log_title[:30]))
self.feed.save_feed_history(551, "Broken feed")
return FEED_ERRHTTP, ret_values
if self.fpf and not self.fpf.entries:
if self.fpf.bozo and isinstance(self.fpf.bozo_exception,
feedparser.NonXMLContentType):
logging.debug(
" ---> [%-30s] ~SB~FRFeed is Non-XML. %s entries. Checking address..."
% (self.feed.log_title[:30], len(self.fpf.entries)))
fixed_feed = None
if not self.feed.known_good:
fixed_feed, feed = self.feed.check_feed_link_for_feed_address(
)
if not fixed_feed:
self.feed.save_feed_history(552, 'Non-xml feed',
self.fpf.bozo_exception)
else:
self.feed = feed
self.feed = self.feed.save()
return FEED_ERRPARSE, ret_values
elif self.fpf.bozo and isinstance(
self.fpf.bozo_exception, xml.sax._exceptions.SAXException):
logging.debug(
" ---> [%-30s] ~SB~FRFeed has SAX/XML parsing issues. %s entries. Checking address..."
% (self.feed.log_title[:30], len(self.fpf.entries)))
fixed_feed = None
if not self.feed.known_good:
fixed_feed, feed = self.feed.check_feed_link_for_feed_address(
)
if not fixed_feed:
self.feed.save_feed_history(553, 'Not an RSS feed',
self.fpf.bozo_exception)
else:
self.feed = feed
self.feed = self.feed.save()
return FEED_ERRPARSE, ret_values
# the feed has changed (or it is the first time we parse it)
# saving the etag and last_modified fields
original_etag = self.feed.etag
self.feed.etag = self.fpf.get('etag')
if self.feed.etag:
self.feed.etag = self.feed.etag[:255]
# some times this is None (it never should) *sigh*
if self.feed.etag is None:
self.feed.etag = ''
if self.feed.etag != original_etag:
self.feed.save(update_fields=['etag'])
original_last_modified = self.feed.last_modified
if hasattr(self.fpf, 'modified') and self.fpf.modified:
try:
self.feed.last_modified = datetime.datetime.strptime(
self.fpf.modified, '%a, %d %b %Y %H:%M:%S %Z')
except Exception, e:
self.feed.last_modified = None
logging.debug("Broken mtime %s: %s" %
(self.feed.last_modified, e))
pass
if self.feed.last_modified != original_last_modified:
self.feed.save(update_fields=['last_modified'])
self.fpf.entries = self.fpf.entries[:100]
original_title = self.feed.feed_title
if self.fpf.feed.get('title'):
self.feed.feed_title = strip_tags(self.fpf.feed.get('title'))
if self.feed.feed_title != original_title:
self.feed.save(update_fields=['feed_title'])
tagline = self.fpf.feed.get('tagline', self.feed.data.feed_tagline)
if tagline:
original_tagline = self.feed.data.feed_tagline
self.feed.data.feed_tagline = smart_unicode(tagline)
if self.feed.data.feed_tagline != original_tagline:
self.feed.data.save(update_fields=['feed_tagline'])
if not self.feed.feed_link_locked:
new_feed_link = self.fpf.feed.get('link') or self.fpf.feed.get(
'id') or self.feed.feed_link
if self.options['force'] and new_feed_link:
new_feed_link = qurl(new_feed_link, remove=['_'])
if new_feed_link != self.feed.feed_link:
logging.debug(
" ---> [%-30s] ~SB~FRFeed's page is different: %s to %s"
% (self.feed.log_title[:30], self.feed.feed_link,
new_feed_link))
redirects, non_redirects = self.feed.count_redirects_in_history(
'page')
self.feed.save_page_history(
301, "HTTP Redirect (%s to go)" % (10 - len(redirects)))
if len(redirects) >= 10 or len(non_redirects) == 0:
self.feed.feed_link = new_feed_link
self.feed.save(update_fields=['feed_link'])
# Determine if stories aren't valid and replace broken guids
guids_seen = set()
permalinks_seen = set()
for entry in self.fpf.entries:
guids_seen.add(entry.get('guid'))
permalinks_seen.add(Feed.get_permalink(entry))
guid_difference = len(guids_seen) != len(self.fpf.entries)
single_guid = len(guids_seen) == 1
replace_guids = single_guid and guid_difference
permalink_difference = len(permalinks_seen) != len(self.fpf.entries)
single_permalink = len(permalinks_seen) == 1
replace_permalinks = single_permalink and permalink_difference
# Compare new stories to existing stories, adding and updating
start_date = datetime.datetime.utcnow()
story_hashes = []
stories = []
for entry in self.fpf.entries:
story = pre_process_story(entry, self.fpf.encoding)
if not story['title'] and not story['story_content']: continue
if story.get('published') < start_date:
start_date = story.get('published')
if replace_guids:
if replace_permalinks:
new_story_guid = unicode(story.get('published'))
if self.options['verbose']:
logging.debug(
u' ---> [%-30s] ~FBReplacing guid (%s) with timestamp: %s'
% (self.feed.log_title[:30], story.get('guid'),
new_story_guid))
story['guid'] = new_story_guid
else:
new_story_guid = Feed.get_permalink(story)
if self.options['verbose']:
logging.debug(
u' ---> [%-30s] ~FBReplacing guid (%s) with permalink: %s'
% (self.feed.log_title[:30], story.get('guid'),
new_story_guid))
story['guid'] = new_story_guid
story['story_hash'] = MStory.feed_guid_hash_unsaved(
self.feed.pk, story.get('guid'))
stories.append(story)
story_hashes.append(story.get('story_hash'))
original_story_hash_count = len(story_hashes)
story_hashes_in_unread_cutoff = self.feed.story_hashes_in_unread_cutoff[:
original_story_hash_count]
story_hashes.extend(story_hashes_in_unread_cutoff)
story_hashes = list(set(story_hashes))
if self.options['verbose'] or settings.DEBUG:
logging.debug(
u' ---> [%-30s] ~FBFound ~SB%s~SN guids, adding ~SB%s~SN/%s guids from db'
% (self.feed.log_title[:30], original_story_hash_count,
len(story_hashes) - original_story_hash_count,
len(story_hashes_in_unread_cutoff)))
existing_stories = dict((s.story_hash, s) for s in MStory.objects(
story_hash__in=story_hashes,
# story_date__gte=start_date,
# story_feed_id=self.feed.pk
))
# if len(existing_stories) == 0:
# existing_stories = dict((s.story_hash, s) for s in MStory.objects(
# story_date__gte=start_date,
# story_feed_id=self.feed.pk
# ))
ret_values = self.feed.add_update_stories(
stories,
existing_stories,
verbose=self.options['verbose'],
updates_off=self.options['updates_off'])
# PubSubHubbub
if (hasattr(self.fpf, 'feed') and hasattr(self.fpf.feed, 'links')
and self.fpf.feed.links):
hub_url = None
self_url = self.feed.feed_address
for link in self.fpf.feed.links:
if link['rel'] == 'hub' and not hub_url:
hub_url = link['href']
elif link['rel'] == 'self':
self_url = link['href']
push_expired = False
if self.feed.is_push:
try:
push_expired = self.feed.push.lease_expires < datetime.datetime.now(
)
except PushSubscription.DoesNotExist:
self.feed.is_push = False
if (hub_url and self_url and not settings.DEBUG
and self.feed.active_subscribers > 0
and (push_expired or not self.feed.is_push
or self.options.get('force'))):
logging.debug(
u' ---> [%-30s] ~BB~FW%sSubscribing to PuSH hub: %s' %
(self.feed.log_title[:30],
"~SKRe-~SN" if push_expired else "", hub_url))
try:
PushSubscription.objects.subscribe(self_url,
feed=self.feed,
hub=hub_url)
except TimeoutError:
logging.debug(
u' ---> [%-30s] ~BB~FW~FRTimed out~FW subscribing to PuSH hub: %s'
% (self.feed.log_title[:30], hub_url))
elif (self.feed.is_push
and (self.feed.active_subscribers <= 0 or not hub_url)):
logging.debug(
u' ---> [%-30s] ~BB~FWTurning off PuSH, no hub found' %
(self.feed.log_title[:30]))
self.feed.is_push = False
self.feed = self.feed.save()
# Push notifications
if ret_values['new'] > 0 and MUserFeedNotification.feed_has_users(
self.feed.pk) > 0:
QueueNotifications.delay(self.feed.pk, ret_values['new'])
# All Done
logging.debug(
u' ---> [%-30s] ~FYParsed Feed: %snew=%s~SN~FY %sup=%s~SN same=%s%s~SN %serr=%s~SN~FY total=~SB%s'
% (self.feed.log_title[:30], '~FG~SB' if ret_values['new'] else '',
ret_values['new'], '~FY~SB' if ret_values['updated'] else '',
ret_values['updated'], '~SB' if ret_values['same'] else '',
ret_values['same'], '~FR~SB' if ret_values['error'] else '',
ret_values['error'], len(self.fpf.entries)))
self.feed.update_all_statistics(has_new_stories=bool(
ret_values['new']),
force=self.options['force'])
fetch_date = datetime.datetime.now()
if ret_values['new']:
if not getattr(settings, 'TEST_DEBUG', False):
self.feed.trim_feed()
self.feed.expire_redis()
if MStatistics.get('raw_feed', None) == self.feed.pk:
self.feed.save_raw_feed(self.raw_feed, fetch_date)
self.feed.save_feed_history(200, "OK", date=fetch_date)
if self.options['verbose']:
logging.debug(u' ---> [%-30s] ~FBTIME: feed parse in ~FM%.4ss' %
(self.feed.log_title[:30], time.time() - start))
return FEED_OK, ret_values
