def collect_statistics_for_db(cls, debug=False): lag = db_functions.mongo_max_replication_lag(settings.MONGODB) cls.set('mongodb_replication_lag', lag) now = round_time(datetime.datetime.now(), round_to=60) r = redis.Redis(connection_pool=settings.REDIS_STATISTICS_POOL) db_times = {} latest_db_times = {} for db in [ 'sql', 'mongo', 'redis', 'task_sql', 'task_mongo', 'task_redis' ]: db_times[db] = [] for hour in range(24): start_hours_ago = now - datetime.timedelta(hours=hour + 1) pipe = r.pipeline() for m in range(60): minute = start_hours_ago + datetime.timedelta(minutes=m) key = "DB:%s:%s" % (db, minute.strftime('%s')) if debug: print(" -> %s:c" % key) pipe.get("%s:c" % key) pipe.get("%s:t" % key) times = pipe.execute() counts = [int(c or 0) for c in times[::2]] avgs = [float(a or 0) for a in times[1::2]] if counts and avgs: count = sum(counts) avg = round(sum(avgs) / count, 3) if count else 0 else: count = 0 avg = 0 if hour == 0: latest_count = float(counts[-1]) if len(counts) else 0 latest_avg = float(avgs[-1]) if len(avgs) else 0 latest_db_times[ db] = latest_avg / latest_count if latest_count else 0 db_times[db].append(avg) db_times[db].reverse() values = ( ('avg_sql_times', json.encode(db_times['sql'])), ('avg_mongo_times', json.encode(db_times['mongo'])), ('avg_redis_times', json.encode(db_times['redis'])), ('latest_sql_avg', latest_db_times['sql']), ('latest_mongo_avg', latest_db_times['mongo']), ('latest_redis_avg', latest_db_times['redis']), ('latest_task_sql_avg', latest_db_times['task_sql']), ('latest_task_mongo_avg', latest_db_times['task_mongo']), ('latest_task_redis_avg', latest_db_times['task_redis']), ) for key, value in values: cls.objects(key=key).update_one(upsert=True, set__key=key, set__value=value)
def process_request(self, request): user_agent = request.environ.get('HTTP_USER_AGENT', 'missing').lower() if 'profile' in request.path: return if 'haproxy' in request.path: return if 'dbcheck' in request.path: return if 'account' in request.path: return if 'push' in request.path: return if getattr(settings, 'TEST_DEBUG'): return if any(ua in user_agent for ua in BANNED_USER_AGENTS): data = {'error': 'User agent banned: %s' % user_agent, 'code': -1} logging.user( request, "~FB~SN~BBBanned UA: ~SB%s / %s (%s)" % (user_agent, request.path, request.META)) return HttpResponse(json.encode(data), status=403, mimetype='text/json') if request.user.is_authenticated() and any( username == request.user.username for username in BANNED_USERNAMES): data = { 'error': 'User banned: %s' % request.user.username, 'code': -1 } logging.user( request, "~FB~SN~BBBanned Username: ~SB%s / %s (%s)" % (request.user, request.path, request.META)) return HttpResponse(json.encode(data), status=403, mimetype='text/json')
def process_request(self, request): user_agent = request.environ.get('HTTP_USER_AGENT', 'missing').lower() if 'profile' in request.path: return if 'haproxy' in request.path: return if 'dbcheck' in request.path: return if 'account' in request.path: return if 'push' in request.path: return if getattr(settings, 'TEST_DEBUG'): return if any(ua in user_agent for ua in BANNED_USER_AGENTS): data = { 'error': 'User agent banned: %s' % user_agent, 'code': -1 } logging.user(request, "~FB~SN~BBBanned UA: ~SB%s / %s (%s)" % (user_agent, request.path, request.META)) return HttpResponse(json.encode(data), status=403, mimetype='text/json') if request.user.is_authenticated() and any(username == request.user.username for username in BANNED_USERNAMES): data = { 'error': 'User banned: %s' % request.user.username, 'code': -1 } logging.user(request, "~FB~SN~BBBanned Username: ~SB%s / %s (%s)" % (request.user, request.path, request.META)) return HttpResponse(json.encode(data), status=403, mimetype='text/json')
def collect_statistics_sites_loaded(cls, last_day=None): if not last_day: last_day = datetime.datetime.now() - datetime.timedelta(hours=24) now = datetime.datetime.now() sites_loaded = [] avg_time_taken = [] for hour in range(24): start_hours_ago = now - datetime.timedelta(hours=hour) end_hours_ago = now - datetime.timedelta(hours=hour+1) aggregates = dict(count=Count('loadtime'), avg=Avg('loadtime')) load_times = FeedLoadtime.objects.filter( date_accessed__lte=start_hours_ago, date_accessed__gte=end_hours_ago ).aggregate(**aggregates) sites_loaded.append(load_times['count'] or 0) avg_time_taken.append(load_times['avg'] or 0) sites_loaded.reverse() avg_time_taken.reverse() values = ( ('sites_loaded', json.encode(sites_loaded)), ('avg_time_taken', json.encode(avg_time_taken)), ('latest_sites_loaded', sites_loaded[-1]), ('latest_avg_time_taken', avg_time_taken[-1]), ('max_sites_loaded', max(sites_loaded)), ('max_avg_time_taken', max(1, max(avg_time_taken))), ) for key, value in values: cls.objects(key=key).update_one(upsert=True, set__key=key, set__value=value)
def collect_statistics_sites_loaded(cls, last_day=None): if not last_day: last_day = datetime.datetime.now() - datetime.timedelta(hours=24) now = datetime.datetime.now() sites_loaded = [] avg_time_taken = [] for hour in range(24): start_hours_ago = now - datetime.timedelta(hours=hour) end_hours_ago = now - datetime.timedelta(hours=hour + 1) aggregates = dict(count=Count('loadtime'), avg=Avg('loadtime')) load_times = FeedLoadtime.objects.filter( date_accessed__lte=start_hours_ago, date_accessed__gte=end_hours_ago).aggregate(**aggregates) sites_loaded.append(load_times['count'] or 0) avg_time_taken.append(load_times['avg'] or 0) sites_loaded.reverse() avg_time_taken.reverse() values = ( ('sites_loaded', json.encode(sites_loaded)), ('avg_time_taken', json.encode(avg_time_taken)), ('latest_sites_loaded', sites_loaded[-1]), ('latest_avg_time_taken', avg_time_taken[-1]), ('max_sites_loaded', max(sites_loaded)), ('max_avg_time_taken', max(1, max(avg_time_taken))), ) for key, value in values: cls.objects(key=key).update_one(upsert=True, set__key=key, set__value=value)
def _parse_user_info(user): return { "user_info": { "is_anonymous": json.encode(user.is_anonymous()), "is_authenticated": json.encode(user.is_authenticated()), "username": json.encode(user.username if user.is_authenticated() else "Anonymous"), } }
def _parse_user_info(user): return { 'user_info': { 'is_anonymous': json.encode(user.is_anonymous()), 'is_authenticated': json.encode(user.is_authenticated()), 'username': json.encode(user.username if user.is_authenticated() else 'Anonymous') } }
def collect_statistics_sites_loaded(cls): now = datetime.datetime.now() sites_loaded = [] avg_time_taken = [] for hour in range(24): start_hours_ago = now - datetime.timedelta(hours=hour) end_hours_ago = now - datetime.timedelta(hours=hour+1) load_times = settings.MONGOANALYTICSDB.nbanalytics.page_loads.aggregate([{ "$match": { "date": { "$gte": end_hours_ago, "$lte": start_hours_ago, }, "path": { "$in": [ "/reader/feed/", "/social/stories/", "/reader/river_stories/", "/social/river_stories/", ] } }, }, { "$group": { "_id" : 1, "count" : {"$sum": 1}, "avg" : {"$avg": "$duration"}, }, }]) count = 0 avg = 0 if load_times['result']: count = load_times['result'][0]['count'] avg = load_times['result'][0]['avg'] sites_loaded.append(count) avg_time_taken.append(avg) sites_loaded.reverse() avg_time_taken.reverse() values = ( ('sites_loaded', json.encode(sites_loaded)), ('avg_time_taken', json.encode(avg_time_taken)), ('latest_sites_loaded', sites_loaded[-1]), ('latest_avg_time_taken', avg_time_taken[-1]), ('max_sites_loaded', max(sites_loaded)), ('max_avg_time_taken', max(1, max(avg_time_taken))), ) for key, value in values: cls.objects(key=key).update_one(upsert=True, set__key=key, set__value=value)
def collect_statistics_for_db(cls): lag = db_functions.mongo_max_replication_lag(settings.MONGODB) cls.set('mongodb_replication_lag', lag) now = round_time(datetime.datetime.now(), round_to=60) r = redis.Redis(connection_pool=settings.REDIS_STATISTICS_POOL) db_times = {} latest_db_times = {} for db in ['sql', 'mongo', 'redis', 'task_sql', 'task_mongo', 'task_redis']: db_times[db] = [] for hour in range(24): start_hours_ago = now - datetime.timedelta(hours=hour+1) pipe = r.pipeline() for m in range(60): minute = start_hours_ago + datetime.timedelta(minutes=m) key = "DB:%s:%s" % (db, minute.strftime('%s')) pipe.get("%s:c" % key) pipe.get("%s:t" % key) times = pipe.execute() counts = [int(c or 0) for c in times[::2]] avgs = [float(a or 0) for a in times[1::2]] if counts and avgs: count = sum(counts) avg = round(sum(avgs) / count, 3) if count else 0 else: count = 0 avg = 0 if hour == 0: latest_count = float(counts[-1]) if len(counts) else 0 latest_avg = float(avgs[-1]) if len(avgs) else 0 latest_db_times[db] = latest_avg / latest_count if latest_count else 0 db_times[db].append(avg) db_times[db].reverse() values = ( ('avg_sql_times', json.encode(db_times['sql'])), ('avg_mongo_times', json.encode(db_times['mongo'])), ('avg_redis_times', json.encode(db_times['redis'])), ('latest_sql_avg', latest_db_times['sql']), ('latest_mongo_avg', latest_db_times['mongo']), ('latest_redis_avg', latest_db_times['redis']), ('latest_task_sql_avg', latest_db_times['task_sql']), ('latest_task_mongo_avg', latest_db_times['task_mongo']), ('latest_task_redis_avg', latest_db_times['task_redis']), ) for key, value in values: cls.objects(key=key).update_one(upsert=True, set__key=key, set__value=value)
def collect_statistics_sites_loaded(cls): now = round_time(datetime.datetime.now(), round_to=60) sites_loaded = [] avg_time_taken = [] last_5_min_time_taken = 0 r = redis.Redis(connection_pool=settings.REDIS_STATISTICS_POOL) for hour in range(24): start_hours_ago = now - datetime.timedelta(hours=hour + 1) pipe = r.pipeline() for m in range(60): minute = start_hours_ago + datetime.timedelta(minutes=m) key = "%s:%s" % (RStats.stats_type('page_load'), minute.strftime('%s')) pipe.get("%s:s" % key) pipe.get("%s:a" % key) times = pipe.execute() counts = [int(c) for c in times[::2] if c] avgs = [float(a) for a in times[1::2] if a] if hour == 0: last_5_min_time_taken = round( sum(avgs[:1]) / max(1, sum(counts[:1])), 2) if counts and avgs: count = max(1, sum(counts)) avg = round(sum(avgs) / count, 3) else: count = 0 avg = 0 sites_loaded.append(count) avg_time_taken.append(avg) sites_loaded.reverse() avg_time_taken.reverse() values = ( ('sites_loaded', json.encode(sites_loaded)), ('avg_time_taken', json.encode(avg_time_taken)), ('latest_sites_loaded', sites_loaded[-1]), ('latest_avg_time_taken', avg_time_taken[-1]), ('max_sites_loaded', max(sites_loaded)), ('max_avg_time_taken', max(1, max(avg_time_taken))), ('last_5_min_time_taken', last_5_min_time_taken), ) for key, value in values: cls.objects(key=key).update_one(upsert=True, set__key=key, set__value=value)
def _view(request, *args, **kwargs): if request.user.is_anonymous(): return HttpResponse(content=json.encode({ "message": "You must have a valid OAuth token.", }), status=401) else: try: setattr(request, 'body_json', json.decode(request.body)) except JSONDecodeError: return HttpResponse(content=json.encode({ "message": "Your JSON body is malformed.", }), status=400) return view_func(request, *args, **kwargs)
def collect_statistics_sites_loaded(cls): now = round_time(datetime.datetime.now(), round_to=60) sites_loaded = [] avg_time_taken = [] last_5_min_time_taken = 0 r = redis.Redis(connection_pool=settings.REDIS_STATISTICS_POOL) for hour in range(24): start_hours_ago = now - datetime.timedelta(hours=hour+1) pipe = r.pipeline() for m in range(60): minute = start_hours_ago + datetime.timedelta(minutes=m) key = "%s:%s" % (RStats.stats_type('page_load'), minute.strftime('%s')) pipe.get("%s:s" % key) pipe.get("%s:a" % key) times = pipe.execute() counts = [int(c) for c in times[::2] if c] avgs = [float(a) for a in times[1::2] if a] if hour == 0: last_5_min_time_taken = round(sum(avgs[:1]) / max(1, sum(counts[:1])), 2) if counts and avgs: count = max(1, sum(counts)) avg = round(sum(avgs) / count, 3) else: count = 0 avg = 0 sites_loaded.append(count) avg_time_taken.append(avg) sites_loaded.reverse() avg_time_taken.reverse() values = ( ('sites_loaded', json.encode(sites_loaded)), ('avg_time_taken', json.encode(avg_time_taken)), ('latest_sites_loaded', sites_loaded[-1]), ('latest_avg_time_taken', avg_time_taken[-1]), ('max_sites_loaded', max(sites_loaded)), ('max_avg_time_taken', max(1, max(avg_time_taken))), ('last_5_min_time_taken', last_5_min_time_taken), ) for key, value in values: cls.objects(key=key).update_one(upsert=True, set__key=key, set__value=value)
def add_site_load_script(request, token): code = 0 usf = None profile = None user_profile = None starred_counts = {} def image_base64(image_name, path='icons/circular/'): image_file = open( os.path.join(settings.MEDIA_ROOT, 'img/%s%s' % (path, image_name)), 'rb') return base64.b64encode(image_file.read()).decode('utf-8') accept_image = image_base64('newuser_icn_setup.png') error_image = image_base64('newuser_icn_sharewith_active.png') new_folder_image = image_base64('g_icn_arrow_right.png') add_image = image_base64('g_icn_expand_hover.png') try: profiles = Profile.objects.filter(secret_token=token) if profiles: profile = profiles[0] usf = UserSubscriptionFolders.objects.get(user=profile.user) user_profile = MSocialProfile.get_user(user_id=profile.user.pk) starred_counts = MStarredStoryCounts.user_counts(profile.user.pk) else: code = -1 except Profile.DoesNotExist: code = -1 except UserSubscriptionFolders.DoesNotExist: code = -1 return render( request, 'api/share_bookmarklet.js', { 'code': code, 'token': token, 'folders': (usf and usf.folders) or [], 'user': profile and profile.user or {}, 'user_profile': user_profile and json.encode(user_profile.canonical()) or {}, 'starred_counts': json.encode(starred_counts), 'accept_image': accept_image, 'error_image': error_image, 'add_image': add_image, 'new_folder_image': new_folder_image, }, content_type='application/javascript')
def delete_folder(self, folder_to_delete, in_folder, feed_ids_in_folder, commit_delete=True): def _find_folder_in_folders(old_folders, folder_name, feeds_to_delete, deleted_folder=None): new_folders = [] for k, folder in enumerate(old_folders): if isinstance(folder, int): new_folders.append(folder) if folder in feeds_to_delete: feeds_to_delete.remove(folder) elif isinstance(folder, dict): for f_k, f_v in folder.items(): if f_k == folder_to_delete and (folder_name == in_folder or in_folder is None): logging.user(self.user, "~FBDeleting folder '~SB%s~SN' in '%s': %s" % (f_k, folder_name, folder)) deleted_folder = folder else: nf, feeds_to_delete, deleted_folder = _find_folder_in_folders(f_v, f_k, feeds_to_delete, deleted_folder) new_folders.append({f_k: nf}) return new_folders, feeds_to_delete, deleted_folder user_sub_folders = json.decode(self.folders) user_sub_folders, feeds_to_delete, deleted_folder = _find_folder_in_folders(user_sub_folders, '', feed_ids_in_folder) self.folders = json.encode(user_sub_folders) self.save() if commit_delete: UserSubscription.objects.filter(user=self.user, feed__in=feeds_to_delete).delete() return deleted_folder
def set_view_setting(request): print 'KHong profile_views set_view_setting()' code = 1 feed_id = request.POST['feed_id'] feed_view_setting = request.POST.get('feed_view_setting') feed_order_setting = request.POST.get('feed_order_setting') feed_read_filter_setting = request.POST.get('feed_read_filter_setting') feed_layout_setting = request.POST.get('feed_layout_setting') view_settings = json.decode(request.user.profile.view_settings) setting = view_settings.get(feed_id, {}) if isinstance(setting, basestring): setting = {'v': setting} if feed_view_setting: setting['v'] = feed_view_setting if feed_order_setting: setting['o'] = feed_order_setting if feed_read_filter_setting: setting['r'] = feed_read_filter_setting if feed_layout_setting: setting['l'] = feed_layout_setting view_settings[feed_id] = setting request.user.profile.view_settings = json.encode(view_settings) request.user.profile.save() logging.user(request, "~FMView settings: %s/%s/%s/%s" % (feed_view_setting, feed_order_setting, feed_read_filter_setting, feed_layout_setting)) response = dict(code=code) return response
def add_site_authed(request): code = 0 url = request.GET['url'] folder = request.GET['folder'] new_folder = request.GET.get('new_folder') callback = request.GET['callback'] user = get_user(request) if not url: code = -1 else: if new_folder: usf, _ = UserSubscriptionFolders.objects.get_or_create(user=user) usf.add_folder(folder, new_folder) folder = new_folder code, message, us = UserSubscription.add_subscription( user=user, feed_address=url, folder=folder, bookmarklet=True ) if code > 0: message = 'OK' logging.user(user, "~FRAdding authed URL from site: ~SB%s (in %s)" % (url, folder), request=request) return HttpResponse(callback + '(' + json.encode({ 'code': code, 'message': message, 'usersub': us and us.feed_id, }) + ')', content_type='text/plain')
def delete_feed(self, feed_id, in_folder): def _find_feed_in_folders(old_folders, folder_name='', multiples_found=False, deleted=False): new_folders = [] for k, folder in enumerate(old_folders): if isinstance(folder, int): if (folder == feed_id and ( (folder_name != in_folder) or (folder_name == in_folder and deleted))): multiples_found = True logging.info(" ---> [%s] Deleting feed, and a multiple has been found in '%s'" % (self.user, folder_name)) if folder == feed_id and folder_name == in_folder and not deleted: logging.info(" ---> [%s] Delete feed: %s'th item: %s folders/feeds" % ( self.user, k, len(old_folders) )) deleted = True else: new_folders.append(folder) elif isinstance(folder, dict): for f_k, f_v in folder.items(): nf, multiples_found, deleted = _find_feed_in_folders(f_v, f_k, multiples_found, deleted) new_folders.append({f_k: nf}) return new_folders, multiples_found, deleted user_sub_folders = json.decode(self.folders) user_sub_folders, multiples_found, deleted = _find_feed_in_folders(user_sub_folders) self.folders = json.encode(user_sub_folders) self.save() if not multiples_found and deleted: user_sub = UserSubscription.objects.get(user=self.user, feed=feed_id) user_sub.delete() MUserStory.objects(user_id=self.user.pk, feed_id=feed_id).delete()
def add_site(request, token): code = 0 url = request.GET['url'] folder = request.GET['folder'] callback = request.GET['callback'] if not url: code = -1 else: try: profile = Profile.objects.get(secret_token=token) code, message, us = UserSubscription.add_subscription( user=profile.user, feed_address=url, folder=folder, bookmarklet=True ) except Profile.DoesNotExist: code = -1 if code > 0: message = 'OK' return HttpResponse(callback + '(' + json.encode({ 'code': code, 'message': message, 'usersub': us and us.feed.pk, }) + ')', mimetype='text/plain')
def add_subscription(cls, user, feed_address, folder=None, bookmarklet=False, auto_active=True, skip_fetch=False): feed = None us = None logging.user( user, "~FRAdding URL: ~SB%s (in %s) %s" % (feed_address, folder, "~FCAUTO-ADD" if not auto_active else "")) feed = Feed.get_feed_from_url(feed_address) if not feed: code = -1 if bookmarklet: message = "This site does not have an RSS feed. Nothing is linked to from this page." else: message = "This address does not point to an RSS feed or a website with an RSS feed." else: us, subscription_created = cls.objects.get_or_create( feed=feed, user=user, defaults={ 'needs_unread_recalc': True, 'active': auto_active, }) code = 1 message = "" if us: user_sub_folders_object, created = UserSubscriptionFolders.objects.get_or_create( user=user, defaults={'folders': '[]'}) if created: user_sub_folders = [] else: user_sub_folders = json.decode(user_sub_folders_object.folders) user_sub_folders = add_object_to_folder(feed.pk, folder, user_sub_folders) user_sub_folders_object.folders = json.encode(user_sub_folders) user_sub_folders_object.save() if auto_active or user.profile.is_premium: us.active = True us.save() if not skip_fetch and feed.last_update < datetime.datetime.utcnow( ) - datetime.timedelta(days=1): feed = feed.update() from apps.social.models import MActivity MActivity.new_feed_subscription(user_id=user.pk, feed_id=feed.pk, feed_title=feed.title) feed.setup_feed_for_premium_subscribers() return code, message, us
def process(self): outline = opml.from_string(self.opml_xml) self.clear_feeds() folders = self.process_outline(outline) UserSubscriptionFolders.objects.create(user=self.user, folders=json.encode(folders)) self.queue_new_feeds() return folders
def opml_upload(request): xml_opml = None message = "OK" code = 1 payload = {} if request.method == 'POST': if 'file' in request.FILES: logging.user(request, "~FR~SBOPML upload starting...") file = request.FILES['file'] xml_opml = file.read() opml_importer = OPMLImporter(xml_opml, request.user) folders = opml_importer.process() feeds = UserSubscription.objects.filter(user=request.user).values() payload = dict(folders=folders, feeds=feeds) logging.user( request, "~FR~SBOPML Upload: ~SK%s~SN~SB~FR feeds" % (len(feeds))) request.session['import_from_google_reader'] = False else: message = "Attach an .opml file." code = -1 data = json.encode(dict(message=message, code=code, payload=payload)) return HttpResponse(data, mimetype='text/plain')
def set_view_setting(request): code = 1 feed_id = request.POST["feed_id"] feed_view_setting = request.POST.get("feed_view_setting") feed_order_setting = request.POST.get("feed_order_setting") feed_read_filter_setting = request.POST.get("feed_read_filter_setting") feed_layout_setting = request.POST.get("feed_layout_setting") view_settings = json.decode(request.user.profile.view_settings) setting = view_settings.get(feed_id, {}) if isinstance(setting, basestring): setting = {"v": setting} if feed_view_setting: setting["v"] = feed_view_setting if feed_order_setting: setting["o"] = feed_order_setting if feed_read_filter_setting: setting["r"] = feed_read_filter_setting if feed_layout_setting: setting["l"] = feed_layout_setting view_settings[feed_id] = setting request.user.profile.view_settings = json.encode(view_settings) request.user.profile.save() logging.user( request, "~FMView settings: %s/%s/%s/%s" % (feed_view_setting, feed_order_setting, feed_read_filter_setting, feed_layout_setting), ) response = dict(code=code) return response
def opml_upload(request): xml_opml = None message = "OK" code = 1 payload = {} if request.method == 'POST': if 'file' in request.FILES: logging.user(request, "~FR~SBOPML upload starting...") file = request.FILES['file'] xml_opml = file.read() opml_importer = OPMLImporter(xml_opml, request.user) folders = opml_importer.process() feeds = UserSubscription.objects.filter(user=request.user).values() payload = dict(folders=folders, feeds=feeds) logging.user(request, "~FR~SBOPML Upload: ~SK%s~SN~SB~FR feeds" % (len(feeds))) request.session['import_from_google_reader'] = False else: message = "Attach an .opml file." code = -1 data = json.encode(dict(message=message, code=code, payload=payload)) return HttpResponse(data, mimetype='text/plain')
def collect_orphan_feeds(cls, user): us = cls.objects.filter(user=user) try: usf = UserSubscriptionFolders.objects.get(user=user) except UserSubscriptionFolders.DoesNotExist: return us_feed_ids = set([sub.feed_id for sub in us]) folders = json.decode(usf.folders) def collect_ids(folders, found_ids): for item in folders: # print ' --> %s' % item if isinstance(item, int): # print ' --> Adding feed: %s' % item found_ids.add(item) elif isinstance(item, dict): # print ' --> Descending folder dict: %s' % item.values() found_ids.update(collect_ids(item.values(), found_ids)) elif isinstance(item, list): # print ' --> Descending folder list: %s' % len(item) found_ids.update(collect_ids(item, found_ids)) # print ' --> Returning: %s' % found_ids return found_ids found_ids = collect_ids(folders, set()) diff = len(us_feed_ids) - len(found_ids) if diff > 0: logging.info( " ---> Collecting orphans on %s. %s feeds with %s orphans" % (user.username, len(us_feed_ids), diff)) orphan_ids = us_feed_ids - found_ids folders.extend(list(orphan_ids)) usf.folders = json.encode(folders) usf.save()
def set_view_setting(request): code = 1 feed_id = request.POST['feed_id'] feed_view_setting = request.POST.get('feed_view_setting') feed_order_setting = request.POST.get('feed_order_setting') feed_read_filter_setting = request.POST.get('feed_read_filter_setting') feed_layout_setting = request.POST.get('feed_layout_setting') view_settings = json.decode(request.user.profile.view_settings) setting = view_settings.get(feed_id, {}) if isinstance(setting, str): setting = {'v': setting} if feed_view_setting: setting['v'] = feed_view_setting if feed_order_setting: setting['o'] = feed_order_setting if feed_read_filter_setting: setting['r'] = feed_read_filter_setting if feed_layout_setting: setting['l'] = feed_layout_setting view_settings[feed_id] = setting request.user.profile.view_settings = json.encode(view_settings) request.user.profile.save() logging.user( request, "~FMView settings: %s/%s/%s/%s" % (feed_view_setting, feed_order_setting, feed_read_filter_setting, feed_layout_setting)) response = dict(code=code) return response
def add_site_authed(request): code = 0 url = request.GET['url'] folder = request.GET['folder'] new_folder = request.GET.get('new_folder') callback = request.GET['callback'] user = get_user(request) if not url: code = -1 else: if new_folder: usf, _ = UserSubscriptionFolders.objects.get_or_create(user=user) usf.add_folder(folder, new_folder) folder = new_folder code, message, us = UserSubscription.add_subscription( user=user, feed_address=url, folder=folder, bookmarklet=True ) if code > 0: message = 'OK' logging.user(user, "~FRAdding authed URL from site: ~SB%s (in %s)" % (url, folder), request=request) return HttpResponse(callback + '(' + json.encode({ 'code': code, 'message': message, 'usersub': us and us.feed_id, }) + ')', mimetype='text/plain')
def delete_folder(self, folder_to_delete, in_folder, feed_ids_in_folder): def _find_folder_in_folders(old_folders, folder_name, feeds_to_delete): new_folders = [] for k, folder in enumerate(old_folders): if isinstance(folder, int): new_folders.append(folder) if folder in feeds_to_delete: feeds_to_delete.remove(folder) elif isinstance(folder, dict): for f_k, f_v in folder.items(): if f_k == folder_to_delete and folder_name == in_folder: logging.info( " ---> [%s] ~FBDeleting folder '~SB%s~SN' in '%s': %s" % (self.user, f_k, folder_name, folder)) else: nf, feeds_to_delete = _find_folder_in_folders( f_v, f_k, feeds_to_delete) new_folders.append({f_k: nf}) return new_folders, feeds_to_delete user_sub_folders = json.decode(self.folders) user_sub_folders, feeds_to_delete = _find_folder_in_folders( user_sub_folders, '', feed_ids_in_folder) self.folders = json.encode(user_sub_folders) self.save() UserSubscription.objects.filter(user=self.user, feed__in=feeds_to_delete).delete()
def collect_statistics_stories_shared(cls, last_day=None): if not last_day: last_day = datetime.datetime.now() - datetime.timedelta(hours=24) now = datetime.datetime.now() stories_shared = [] for hour in range(24): start_hours_ago = now - datetime.timedelta(hours=hour) end_hours_ago = now - datetime.timedelta(hours=hour + 1) shares = MSharedStory.objects.filter( shared_date__lte=start_hours_ago, shared_date__gte=end_hours_ago).count() stories_shared.append(shares) stories_shared.reverse() values = ( ('stories_shared', json.encode(stories_shared)), ('latest_stories_shared', stories_shared[-1]), ('max_stories_shared', max(stories_shared)), ) for key, value in values: cls.objects(key=key).update_one(upsert=True, set__key=key, set__value=value)
def add_site(request, token): code = 0 url = request.GET["url"] folder = request.GET["folder"] new_folder = request.GET.get("new_folder") callback = request.GET["callback"] if not url: code = -1 else: try: profile = Profile.objects.get(secret_token=token) if new_folder: usf, _ = UserSubscriptionFolders.objects.get_or_create(user=profile.user) usf.add_folder(folder, new_folder) folder = new_folder code, message, us = UserSubscription.add_subscription( user=profile.user, feed_address=url, folder=folder, bookmarklet=True ) except Profile.DoesNotExist: code = -1 if code > 0: message = "OK" logging.user(profile.user, "~FRAdding URL from site: ~SB%s (in %s)" % (url, folder), request=request) return HttpResponse( callback + "(" + json.encode({"code": code, "message": message, "usersub": us and us.feed_id}) + ")", mimetype="text/plain", )
def add_missing_feeds(self): all_feeds = self.flat() subs = [us.feed_id for us in UserSubscription.objects.filter(user=self.user).only('feed')] missing_subs = set(all_feeds) - set(subs) if missing_subs: logging.debug(" ---> %s is missing %s subs. Adding %s..." % ( self.user, len(missing_subs), missing_subs)) for feed_id in missing_subs: feed = Feed.get_by_id(feed_id) if feed: us, _ = UserSubscription.objects.get_or_create(user=self.user, feed=feed, defaults={ 'needs_unread_recalc': True }) if not us.needs_unread_recalc: us.needs_unread_recalc = True us.save() missing_folder_feeds = set(subs) - set(all_feeds) if missing_folder_feeds: user_sub_folders = json.decode(self.folders) logging.debug(" ---> %s is missing %s folder feeds. Adding %s..." % ( self.user, len(missing_folder_feeds), missing_folder_feeds)) for feed_id in missing_folder_feeds: feed = Feed.get_by_id(feed_id) if feed and feed.pk == feed_id: user_sub_folders = add_object_to_folder(feed_id, "", user_sub_folders) self.folders = json.encode(user_sub_folders) self.save()
def set_preference(request): code = 1 message = '' new_preferences = request.POST preferences = json.decode(request.user.profile.preferences) for preference_name, preference_value in new_preferences.items(): if preference_value in ['true','false']: preference_value = True if preference_value == 'true' else False if preference_name in SINGLE_FIELD_PREFS: setattr(request.user.profile, preference_name, preference_value) elif preference_name in SPECIAL_PREFERENCES: if preference_name == 'autofollow_friends': social_services, _ = MSocialServices.objects.get_or_create(user_id=request.user.pk) social_services.autofollow = preference_value social_services.save() elif preference_name == 'dashboard_date': request.user.profile.dashboard_date = datetime.datetime.utcnow() else: if preference_value in ["true", "false"]: preference_value = True if preference_value == "true" else False preferences[preference_name] = preference_value if preference_name == 'intro_page': logging.user(request, "~FBAdvancing intro to page ~FM~SB%s" % preference_value) request.user.profile.preferences = json.encode(preferences) request.user.profile.save() logging.user(request, "~FMSaving preference: %s" % new_preferences) response = dict(code=code, message=message, new_preferences=new_preferences) return response
def collect_orphan_feeds(cls, user): us = cls.objects.filter(user=user) try: usf = UserSubscriptionFolders.objects.get(user=user) except UserSubscriptionFolders.DoesNotExist: return us_feed_ids = set([sub.feed_id for sub in us]) folders = json.decode(usf.folders) def collect_ids(folders, found_ids): for item in folders: # print ' --> %s' % item if isinstance(item, int): # print ' --> Adding feed: %s' % item found_ids.add(item) elif isinstance(item, dict): # print ' --> Descending folder dict: %s' % item.values() found_ids.update(collect_ids(item.values(), found_ids)) elif isinstance(item, list): # print ' --> Descending folder list: %s' % len(item) found_ids.update(collect_ids(item, found_ids)) # print ' --> Returning: %s' % found_ids return found_ids found_ids = collect_ids(folders, set()) diff = len(us_feed_ids) - len(found_ids) if diff > 0: logging.info(" ---> Collecting orphans on %s. %s feeds with %s orphans" % (user.username, len(us_feed_ids), diff)) orphan_ids = us_feed_ids - found_ids folders.extend(list(orphan_ids)) usf.folders = json.encode(folders) usf.save()
def set_preference(request): code = 1 message = '' new_preferences = request.POST preferences = json.decode(request.user.profile.preferences) for preference_name, preference_value in new_preferences.items(): if preference_name in SINGLE_FIELD_PREFS: setattr(request.user.profile, preference_name, preference_value) elif preference_name in SPECIAL_PREFERENCES: if (preference_name == 'old_password' and (new_preferences['old_password'] or new_preferences['new_password'])): code = change_password(request.user, new_preferences['old_password'], new_preferences['new_password']) if code == -1: message = "Your old password is incorrect." else: if preference_value in ["true", "false"]: preference_value = True if preference_value == "true" else False preferences[preference_name] = preference_value request.user.profile.preferences = json.encode(preferences) request.user.profile.save() response = dict(code=code, message=message, new_preferences=new_preferences) return response
def clear_view_setting(request): code = 1 view_setting_type = request.POST.get('view_setting_type') view_settings = json.decode(request.user.profile.view_settings) new_view_settings = {} removed = 0 for feed_id, view_setting in list(view_settings.items()): if view_setting_type == 'layout' and 'l' in view_setting: del view_setting['l'] removed += 1 if view_setting_type == 'view' and 'v' in view_setting: del view_setting['v'] removed += 1 if view_setting_type == 'order' and 'o' in view_setting: del view_setting['o'] removed += 1 if view_setting_type == 'order' and 'r' in view_setting: del view_setting['r'] removed += 1 new_view_settings[feed_id] = view_setting request.user.profile.view_settings = json.encode(new_view_settings) request.user.profile.save() logging.user( request, "~FMClearing view settings: %s (found %s)" % (view_setting_type, removed)) response = dict(code=code, view_settings=view_settings, removed=removed) return response
def add_site(request, token): code = 0 url = request.GET['url'] folder = request.GET['folder'] new_folder = request.GET.get('new_folder') callback = request.GET['callback'] if not url: code = -1 else: try: profile = Profile.objects.get(secret_token=token) if new_folder: usf, _ = UserSubscriptionFolders.objects.get_or_create(user=profile.user) usf.add_folder(folder, new_folder) folder = new_folder code, message, us = UserSubscription.add_subscription( user=profile.user, feed_address=url, folder=folder, bookmarklet=True ) except Profile.DoesNotExist: code = -1 if code > 0: message = 'OK' logging.user(profile.user, "~FRAdding URL from site: ~SB%s (in %s)" % (url, folder)) return HttpResponse(callback + '(' + json.encode({ 'code': code, 'message': message, 'usersub': us and us.feed.pk, }) + ')', mimetype='text/plain')
def clear_view_setting(request): code = 1 view_setting_type = request.POST.get('view_setting_type') view_settings = json.decode(request.user.profile.view_settings) new_view_settings = {} removed = 0 for feed_id, view_setting in view_settings.items(): if view_setting_type == 'layout' and 'l' in view_setting: del view_setting['l'] removed += 1 if view_setting_type == 'view' and 'v' in view_setting: del view_setting['v'] removed += 1 if view_setting_type == 'order' and 'o' in view_setting: del view_setting['o'] removed += 1 if view_setting_type == 'order' and 'r' in view_setting: del view_setting['r'] removed += 1 new_view_settings[feed_id] = view_setting request.user.profile.view_settings = json.encode(new_view_settings) request.user.profile.save() logging.user(request, "~FMClearing view settings: %s (found %s)" % (view_setting_type, removed)) response = dict(code=code, view_settings=view_settings, removed=removed) return response
def process(self): outline = opml.from_string(self.opml_xml) self.clear_feeds() folders = self.process_outline(outline) UserSubscriptionFolders.objects.create(user=self.user, folders=json.encode(folders)) return folders
def logout(request): logging.user(request, "~FG~BBLogout~FW") logout_user(request) if request.GET.get('api'): return HttpResponse(json.encode(dict(code=1)), mimetype='application/json') else: return HttpResponseRedirect(reverse('index'))
def logout(request): logging.user(request.user, "~FG~BBLogout~FW") logout_user(request) if request.GET.get('api'): return HttpResponse(json.encode(dict(code=1)), mimetype='application/json') else: return HttpResponseRedirect(reverse('index'))
def logout(request): logging.info(" ---> [%s] ~FG~BBLogout~FW" % request.user) logout_user(request) if request.GET.get("api"): return HttpResponse(json.encode(dict(code=1)), mimetype="application/json") else: return HttpResponseRedirect(reverse("index"))
def opml_upload(request): xml_opml = None message = "OK" code = 1 payload = {} if request.method == 'POST': if 'file' in request.FILES: logging.user(request, "~FR~SBOPML upload starting...") file = request.FILES['file'] xml_opml = file.read() try: UploadedOPML.objects.create(user_id=request.user.pk, opml_file=xml_opml) except (UnicodeDecodeError, ValidationError, InvalidStringData): folders = None code = -1 message = "There was a Unicode decode error when reading your OPML file. Ensure it's a text file with a .opml or .xml extension. Is it a zip file?" opml_importer = OPMLImporter(xml_opml, request.user) try: folders = opml_importer.try_processing() except TimeoutError: folders = None ProcessOPML.delay(request.user.pk) feed_count = opml_importer.count_feeds_in_opml() logging.user( request, "~FR~SBOPML upload took too long, found %s feeds. Tasking..." % feed_count) payload = dict(folders=folders, delayed=True, feed_count=feed_count) code = 2 message = "" except AttributeError: code = -1 message = "OPML import failed. Couldn't parse XML file." folders = None if folders: code = 1 feeds = UserSubscription.objects.filter( user=request.user).values() payload = dict(folders=folders, feeds=feeds) logging.user( request, "~FR~SBOPML Upload: ~SK%s~SN~SB~FR feeds" % (len(feeds))) UserSubscription.queue_new_feeds(request.user) UserSubscription.refresh_stale_feeds(request.user, exclude_new=True) else: message = "Attach an .opml file." code = -1 return HttpResponse(json.encode( dict(message=message, code=code, payload=payload)), content_type='text/html')
def opml_upload(request): xml_opml = None message = "OK" code = 1 payload = {} if request.method == 'POST': if 'file' in request.FILES: logging.user(request, "~FR~SBOPML upload starting...") file = request.FILES['file'] xml_opml = file.read() try: uploaded_opml = UploadedOPML.objects.create( user_id=request.user.pk, opml_file=xml_opml) except (UnicodeDecodeError, InvalidStringData): uploaded_opml = None folders = None code = -1 message = "There was a Unicode decode error when reading your OPML file." if uploaded_opml: opml_importer = OPMLImporter(xml_opml, request.user) try: folders = opml_importer.try_processing() except TimeoutError: folders = None ProcessOPML.delay(request.user.pk) feed_count = opml_importer.count_feeds_in_opml() logging.user( request, "~FR~SBOPML pload took too long, found %s feeds. Tasking..." % feed_count) payload = dict(folders=folders, delayed=True, feed_count=feed_count) code = 2 message = "" except AttributeError: code = -1 message = "OPML import failed. Couldn't parse XML file." folders = None if folders: feeds = UserSubscription.objects.filter( user=request.user).values() payload = dict(folders=folders, feeds=feeds) logging.user( request, "~FR~SBOPML Upload: ~SK%s~SN~SB~FR feeds" % (len(feeds))) request.session['import_from_google_reader'] = False else: message = "Attach an .opml file." code = -1 return HttpResponse(json.encode( dict(message=message, code=code, payload=payload)), mimetype='text/html')
def move_feed_to_folder(self, feed_id, in_folder=None, to_folder=None): user_sub_folders = json.decode(self.folders) self.delete_feed(feed_id, in_folder, commit_delete=False) user_sub_folders = json.decode(self.folders) user_sub_folders = add_object_to_folder(int(feed_id), to_folder, user_sub_folders) self.folders = json.encode(user_sub_folders) self.save() return self
def move_folder_to_folder(self, folder_name, in_folder=None, to_folder=None): user_sub_folders = json.decode(self.folders) deleted_folder = self.delete_folder(folder_name, in_folder, [], commit_delete=False) user_sub_folders = json.decode(self.folders) user_sub_folders = add_object_to_folder(deleted_folder, to_folder, user_sub_folders) self.folders = json.encode(user_sub_folders) self.save() return self
def add_folder(self, parent_folder, folder): if self.folders: user_sub_folders = json.decode(self.folders) else: user_sub_folders = [] obj = {folder: []} user_sub_folders = add_object_to_folder(obj, parent_folder, user_sub_folders) self.folders = json.encode(user_sub_folders) self.save()
def move_folder_to_folder(self, folder_name, in_folder=None, to_folder=None): logging.user(self.user, "~FBMoving folder '~SB%s~SN' in '%s' to: ~SB%s" % (folder_name, in_folder, to_folder)) user_sub_folders = json.decode(self.folders) deleted_folder = self.delete_folder(folder_name, in_folder, [], commit_delete=False) user_sub_folders = json.decode(self.folders) user_sub_folders = add_object_to_folder(deleted_folder, to_folder, user_sub_folders) self.folders = json.encode(user_sub_folders) self.save() return self
def move_feed_to_folder(self, feed_id, in_folder=None, to_folder=None): logging.user(self.user, "~FBMoving feed '~SB%s~SN' in '%s' to: ~SB%s" % (feed_id, in_folder, to_folder)) user_sub_folders = json.decode(self.folders) self.delete_feed(feed_id, in_folder, commit_delete=False) user_sub_folders = json.decode(self.folders) user_sub_folders = add_object_to_folder(int(feed_id), to_folder, user_sub_folders) self.folders = json.encode(user_sub_folders) self.save() return self
def process(self): # self.clear_feeds() outline = opml.from_string(str(self.opml_xml)) folders = self.get_folders() folders = self.process_outline(outline, folders) # self.clear_folders() self.usf.folders = json.encode(folders) self.usf.save() return folders
def delete_feed(self, feed_id, in_folder, commit_delete=True): def _find_feed_in_folders(old_folders, folder_name='', multiples_found=False, deleted=False): new_folders = [] for k, folder in enumerate(old_folders): if isinstance(folder, int): if (folder == feed_id and ((folder_name != in_folder) or (folder_name == in_folder and deleted))): multiples_found = True logging.user( self.user, "~FB~SBDeleting feed, and a multiple has been found in '%s'" % (folder_name)) if folder == feed_id and (folder_name == in_folder) and not deleted: logging.user( self.user, "~FBDelete feed: %s'th item: %s folders/feeds" % (k, len(old_folders))) deleted = True else: new_folders.append(folder) elif isinstance(folder, dict): for f_k, f_v in folder.items(): nf, multiples_found, deleted = _find_feed_in_folders( f_v, f_k, multiples_found, deleted) new_folders.append({f_k: nf}) return new_folders, multiples_found, deleted user_sub_folders = json.decode(self.folders) user_sub_folders, multiples_found, deleted = _find_feed_in_folders( user_sub_folders) self.folders = json.encode(user_sub_folders) self.save() if not multiples_found and deleted and commit_delete: try: user_sub = UserSubscription.objects.get(user=self.user, feed=feed_id) except Feed.DoesNotExist: duplicate_feed = DuplicateFeed.objects.filter( duplicate_feed_id=feed_id) if duplicate_feed: try: user_sub = UserSubscription.objects.get( user=self.user, feed=duplicate_feed[0].feed) except Feed.DoesNotExist: return if user_sub: user_sub.delete() MUserStory.objects(user_id=self.user_id, feed_id=feed_id).delete()
def move_feed_to_folder(self, feed_id, in_folder=None, to_folder=None): logging.user(self.user, "~FBMoving feed '~SB%s~SN' in '%s' to: ~SB%s" % ( feed_id, in_folder, to_folder)) user_sub_folders = json.decode(self.folders) self.delete_feed(feed_id, in_folder, commit_delete=False) user_sub_folders = json.decode(self.folders) user_sub_folders = add_object_to_folder(int(feed_id), to_folder, user_sub_folders) self.folders = json.encode(user_sub_folders) self.save() return self
def move_folder_to_folder(self, folder_name, in_folder=None, to_folder=None): logging.user(self.user, "~FBMoving folder '~SB%s~SN' in '%s' to: ~SB%s" % ( folder_name, in_folder, to_folder)) user_sub_folders = json.decode(self.folders) deleted_folder = self.delete_folder(folder_name, in_folder, [], commit_delete=False) user_sub_folders = json.decode(self.folders) user_sub_folders = add_object_to_folder(deleted_folder, to_folder, user_sub_folders) self.folders = json.encode(user_sub_folders) self.save() return self
def forwards(self, orm): for profile in orm.Profile.objects.all(): if 'feed_pane_size' in profile.preferences: preferences = json.decode(profile.preferences) feed_pane_size = int(preferences.get('feed_pane_size', 240)) del preferences['feed_pane_size'] print " --> User %s: %s" % (profile.user.username, feed_pane_size) profile.feed_pane_size = feed_pane_size profile.preferences = json.encode(preferences) profile.save()
def set_view_setting(request): code = 1 feed_id = request.POST['feed_id'] feed_view_setting = request.POST['feed_view_setting'] view_settings = json.decode(request.user.profile.view_settings) view_settings[feed_id] = feed_view_setting request.user.profile.view_settings = json.encode(view_settings) request.user.profile.save() response = dict(code=code) return response
def process_feeds(self, feeds_xml): # self.clear_feeds() # self.clear_folders() folders = self.get_folders() self.feeds = self.parse(feeds_xml) for item in self.feeds: folders = self.process_item(item, folders) logging.user(self.user, "~BB~FW~SBGoogle Reader import: ~BT~FW%s" % (folders)) self.usf.folders = json.encode(folders) self.usf.save()
def process_request(self, request): user_agent = request.environ.get('HTTP_USER_AGENT', 'missing').lower() if 'profile' in request.path: return if 'haproxy' in request.path: return if getattr(settings, 'TEST_DEBUG'): return if any(ua in user_agent for ua in BANNED_USER_AGENTS): data = {'error': 'User agent banned: %s' % user_agent, 'code': -1} logging.user(request, "~FB~SN~BBBanned UA: ~SB%s" % (user_agent)) return HttpResponse(json.encode(data), status=403, mimetype='text/json')