def original_text(request): story_id = request.REQUEST.get('story_id') feed_id = request.REQUEST.get('feed_id') story_hash = request.REQUEST.get('story_hash', None) force = request.REQUEST.get('force', False) debug = request.REQUEST.get('debug', False) if story_hash: story, _ = MStory.find_story(story_hash=story_hash) else: story, _ = MStory.find_story(story_id=story_id, story_feed_id=feed_id) if not story: logging.user(request, "~FYFetching ~FGoriginal~FY story text: ~FRstory not found") return {'code': -1, 'message': 'Story not found.', 'original_text': None, 'failed': True} original_text = story.fetch_original_text(force=force, request=request, debug=debug) return { 'feed_id': story.story_feed_id, 'story_hash': story.story_hash, 'story_id': story.story_guid, 'image_urls': story.image_urls, 'secure_image_urls': Feed.secure_image_urls(story.image_urls), 'original_text': original_text, 'failed': not original_text or len(original_text) < 100, }
def original_text(request): story_id = request.REQUEST.get('story_id') feed_id = request.REQUEST.get('feed_id') story_hash = request.REQUEST.get('story_hash', None) force = request.REQUEST.get('force', False) debug = request.REQUEST.get('debug', False) if story_hash: story, _ = MStory.find_story(story_hash=story_hash) else: story, _ = MStory.find_story(story_id=story_id, story_feed_id=feed_id) if not story: logging.user( request, "~FYFetching ~FGoriginal~FY story text: ~FRstory not found") return { 'code': -1, 'message': 'Story not found.', 'original_text': None, 'failed': True } original_text = story.fetch_original_text(force=force, request=request, debug=debug) return { 'feed_id': feed_id, 'story_id': story_id, 'original_text': original_text, 'failed': not original_text or len(original_text) < 100, }
def original_text(request): # iOS sends a POST, web sends a GET GET_POST = getattr(request, request.method) story_id = GET_POST.get('story_id') feed_id = GET_POST.get('feed_id') story_hash = GET_POST.get('story_hash', None) force = GET_POST.get('force', False) debug = GET_POST.get('debug', False) if not story_hash and not story_id: return {'code': -1, 'message': 'Missing story_hash.', 'original_text': None, 'failed': True} if story_hash: story, _ = MStory.find_story(story_hash=story_hash) else: story, _ = MStory.find_story(story_id=story_id, story_feed_id=feed_id) if not story: logging.user(request, "~FYFetching ~FGoriginal~FY story text: ~FRstory not found") return {'code': -1, 'message': 'Story not found.', 'original_text': None, 'failed': True} original_text = story.fetch_original_text(force=force, request=request, debug=debug) return { 'feed_id': story.story_feed_id, 'story_hash': story.story_hash, 'story_id': story.story_guid, 'image_urls': story.image_urls, 'secure_image_urls': Feed.secure_image_urls(story.image_urls), 'original_text': original_text, 'failed': not original_text or len(original_text) < 100, }
def mark_story_ids_as_read(self, story_ids, request=None): data = dict(code=0, payload=story_ids) if not request: request = self.user if not self.needs_unread_recalc: self.needs_unread_recalc = True self.save() if len(story_ids) > 1: logging.user(request, "~FYRead %s stories in feed: %s" % (len(story_ids), self.feed)) else: logging.user(request, "~FYRead story in feed: %s" % (self.feed)) for story_id in set(story_ids): story, _ = MStory.find_story(story_feed_id=self.feed_id, story_id=story_id) if not story: continue now = datetime.datetime.utcnow() date = now if now > story.story_date else story.story_date # For handling future stories m, _ = MUserStory.objects.get_or_create(story_id=story_id, user_id=self.user_id, feed_id=self.feed_id, defaults={ 'read_date': date, 'story': story, 'story_date': story.story_date, }) return data
def story_hash(cls, story_id, story_feed_id): if not cls.RE_STORY_HASH.match(story_id): story, _ = MStory.find_story(story_feed_id=story_feed_id, story_id=story_id) if not story: return story_id = story.story_hash return story_id
def story_hash(cls, story_id, story_feed_id): if not cls.RE_STORY_HASH.match(story_id): story, _ = MStory.find_story(story_feed_id=story_feed_id, story_id=story_id) if story: story_id = story.story_hash else: story_id = "%s:%s" % (story_feed_id, hashlib.sha1(story_id).hexdigest()[:6]) return story_id
def story_hash(cls, story_id, story_feed_id): if not cls.RE_STORY_HASH.match(story_id): story, _ = MStory.find_story(story_feed_id=story_feed_id, story_id=story_id) if story: story_id = story.story_hash else: story_id = "%s:%s" % (story_feed_id, hashlib.sha1(story_id).hexdigest()[:6]) return story_id
def story_changes(request): story_hash = request.REQUEST.get('story_hash', None) show_changes = is_true(request.REQUEST.get('show_changes', True)) story, _ = MStory.find_story(story_hash=story_hash) if not story: logging.user(request, "~FYFetching ~FGoriginal~FY story page: ~FRstory not found") return {'code': -1, 'message': 'Story not found.', 'original_page': None, 'failed': True} return { 'story': Feed.format_story(story, show_changes=show_changes) }
def story_changes(request): story_hash = request.REQUEST.get('story_hash', None) show_changes = is_true(request.REQUEST.get('show_changes', True)) story, _ = MStory.find_story(story_hash=story_hash) if not story: logging.user(request, "~FYFetching ~FGoriginal~FY story page: ~FRstory not found") return {'code': -1, 'message': 'Story not found.', 'original_page': None, 'failed': True} return { 'story': Feed.format_story(story, show_changes=show_changes) }
def original_story(request): story_hash = request.REQUEST.get('story_hash') force = request.REQUEST.get('force', False) debug = request.REQUEST.get('debug', False) story, _ = MStory.find_story(story_hash=story_hash) if not story: logging.user(request, "~FYFetching ~FGoriginal~FY story page: ~FRstory not found") return {'code': -1, 'message': 'Story not found.', 'original_page': None, 'failed': True} original_page = story.fetch_original_page(force=force, request=request, debug=debug) return HttpResponse(original_page or "")
def original_text(request): story_id = request.GET["story_id"] feed_id = request.GET["feed_id"] force = request.GET.get("force", False) story, _ = MStory.find_story(story_id=story_id, story_feed_id=feed_id) if not story: logging.user(request, "~FYFetching ~FGoriginal~FY story text: ~FRstory not found") return {"code": -1, "message": "Story not found."} original_text = story.fetch_original_text(force=force, request=request) return {"feed_id": feed_id, "story_id": story_id, "original_text": original_text}
def original_story(request): story_hash = request.REQUEST.get("story_hash") force = request.REQUEST.get("force", False) debug = request.REQUEST.get("debug", False) story, _ = MStory.find_story(story_hash=story_hash) if not story: logging.user(request, "~FYFetching ~FGoriginal~FY story page: ~FRstory not found") return {"code": -1, "message": "Story not found.", "original_page": None, "failed": True} original_page = story.fetch_original_page(force=force, request=request, debug=debug) return HttpResponse(original_page or "")
def original_story(request): story_hash = request.REQUEST.get('story_hash') force = request.REQUEST.get('force', False) debug = request.REQUEST.get('debug', False) story, _ = MStory.find_story(story_hash=story_hash) if not story: logging.user(request, "~FYFetching ~FGoriginal~FY story page: ~FRstory not found") return {'code': -1, 'message': 'Story not found.', 'original_page': None, 'failed': True} original_page = story.fetch_original_page(force=force, request=request, debug=debug) return HttpResponse(original_page or "")
def original_text(request): story_id = request.REQUEST.get("story_id") feed_id = request.REQUEST.get("feed_id") story_hash = request.REQUEST.get("story_hash", None) force = request.REQUEST.get("force", False) debug = request.REQUEST.get("debug", False) if story_hash: story, _ = MStory.find_story(story_hash=story_hash) else: story, _ = MStory.find_story(story_id=story_id, story_feed_id=feed_id) if not story: logging.user(request, "~FYFetching ~FGoriginal~FY story text: ~FRstory not found") return {"code": -1, "message": "Story not found.", "original_text": None, "failed": True} original_text = story.fetch_original_text(force=force, request=request, debug=debug) return { "feed_id": feed_id, "story_id": story_id, "original_text": original_text, "failed": not original_text or len(original_text) < 100, }
def original_text(request): story_id = request.GET['story_id'] feed_id = request.GET['feed_id'] force = request.GET.get('force', False) story, _ = MStory.find_story(story_id=story_id, story_feed_id=feed_id) if not story: logging.user(request, "~FYFetching ~FGoriginal~FY story text: ~FRstory not found") return {'code': -1, 'message': 'Story not found.'} original_text = story.fetch_original_text(force=force, request=request) return { 'feed_id': feed_id, 'story_id': story_id, 'original_text': original_text, }
def story_db_id(self): if self.story: return self.story.id elif self.found_story: if "_ref" in self.found_story: return self.found_story["_ref"].id elif hasattr(self.found_story, "id"): return self.found_story.id story, found_original = MStory.find_story(self.feed_id, self.story_id) if story: if found_original: self.story = story else: self.found_story = story self.save() return story.id
def original_text(request): story_id = request.GET['story_id'] feed_id = request.GET['feed_id'] force = request.GET.get('force', False) story, _ = MStory.find_story(story_id=story_id, story_feed_id=feed_id) if not story: logging.user(request, "~FYFetching ~FGoriginal~FY story text: ~FRstory not found") return {'code': -1, 'message': 'Story not found.'} original_text = story.fetch_original_text(force=force, request=request) return { 'feed_id': feed_id, 'story_id': story_id, 'original_text': original_text, }
def story_db_id(self): if self.story: return self.story.id elif self.found_story: if '_ref' in self.found_story: return self.found_story['_ref'].id elif hasattr(self.found_story, 'id'): return self.found_story.id story, found_original = MStory.find_story(self.feed_id, self.story_id) if story: if found_original: self.story = story else: self.found_story = story self.save() return story.id
def story_db_id(self): if self.story: return self.story.id elif self.found_story: if '_ref' in self.found_story: return self.found_story['_ref'].id elif hasattr(self.found_story, 'id'): return self.found_story.id story, found_original = MStory.find_story(self.feed_id, self.story_id) if story: if found_original: self.story = story else: self.found_story = story self.save() return story.id
def original_text(request): story_id = request.REQUEST.get('story_id') feed_id = request.REQUEST.get('feed_id') force = request.REQUEST.get('force', False) story, _ = MStory.find_story(story_id=story_id, story_feed_id=feed_id) if not story: logging.user(request, "~FYFetching ~FGoriginal~FY story text: ~FRstory not found") return {'code': -1, 'message': 'Story not found.', 'original_text': None, 'failed': True} original_text = story.fetch_original_text(force=force, request=request) return { 'feed_id': feed_id, 'story_id': story_id, 'original_text': original_text, 'failed': not original_text or len(original_text) < 100, }
def mark_story_ids_as_read(self, story_ids, request=None): data = dict(code=0, payload=story_ids) if not request: request = self.user if not self.needs_unread_recalc: self.needs_unread_recalc = True self.save() if len(story_ids) > 1: logging.user( request, "~FYRead %s stories in feed: %s" % (len(story_ids), self.feed)) else: logging.user(request, "~FYRead story in feed: %s" % (self.feed)) for story_id in set(story_ids): story, _ = MStory.find_story(story_feed_id=self.feed_id, story_id=story_id) if not story: continue now = datetime.datetime.utcnow() date = now if now > story.story_date else story.story_date # For handling future stories m, _ = MUserStory.objects.get_or_create(story_id=story_id, user_id=self.user_id, feed_id=self.feed_id, defaults={ 'read_date': date, 'story': story, 'story_date': story.story_date, }) return data
def mark_story_as_shared(request): code = 1 feed_id = int(request.POST["feed_id"]) story_id = request.POST["story_id"] comments = request.POST.get("comments", "") source_user_id = request.POST.get("source_user_id") relative_user_id = request.POST.get("relative_user_id") or request.user.pk post_to_services = request.POST.getlist("post_to_services") format = request.REQUEST.get("format", "json") MSocialProfile.get_user(request.user.pk) story, original_story_found = MStory.find_story(feed_id, story_id) if not story: return json.json_response( request, {"code": -1, "message": "Could not find the original story and no copies could be found."} ) shared_story = ( MSharedStory.objects.filter(user_id=request.user.pk, story_feed_id=feed_id, story_guid=story_id) .limit(1) .first() ) if not shared_story: story_db = dict([(k, v) for k, v in story._data.items() if k is not None and v is not None]) story_values = dict( user_id=request.user.pk, comments=comments, has_comments=bool(comments), story_db_id=story.id ) story_db.update(story_values) shared_story = MSharedStory.objects.create(**story_db) if source_user_id: shared_story.set_source_user_id(int(source_user_id)) socialsubs = MSocialSubscription.objects.filter(subscription_user_id=request.user.pk) for socialsub in socialsubs: socialsub.needs_unread_recalc = True socialsub.save() logging.user(request, "~FCSharing ~FM%s: ~SB~FB%s" % (story.story_title[:20], comments[:30])) else: shared_story.comments = comments shared_story.has_comments = bool(comments) shared_story.save() logging.user(request, "~FCUpdating shared story ~FM%s: ~SB~FB%s" % (story.story_title[:20], comments[:30])) if original_story_found: story.count_comments() shared_story.publish_update_to_subscribers() story = Feed.format_story(story) check_all = not original_story_found stories, profiles = MSharedStory.stories_with_comments_and_profiles([story], relative_user_id, check_all=check_all) story = stories[0] story["shared_comments"] = strip_tags(shared_story["comments"] or "") story["shared_by_user"] = True if post_to_services: for service in post_to_services: if service not in shared_story.posted_to_services: PostToService.delay(shared_story_id=shared_story.id, service=service) if shared_story.source_user_id and shared_story.comments: EmailStoryReshares.apply_async( kwargs=dict(shared_story_id=shared_story.id), countdown=settings.SECONDS_TO_DELAY_CELERY_EMAILS ) if format == "html": stories = MSharedStory.attach_users_to_stories(stories, profiles) return render_to_response( "social/social_story.xhtml", {"story": story}, context_instance=RequestContext(request) ) else: return json.json_response(request, {"code": code, "story": story, "user_profiles": profiles})
def mark_story_as_shared(request): code = 1 feed_id = int(request.POST['feed_id']) story_id = request.POST['story_id'] comments = request.POST.get('comments', '') source_user_id = request.POST.get('source_user_id') post_to_services = request.POST.getlist('post_to_services') format = request.REQUEST.get('format', 'json') MSocialProfile.get_user(request.user.pk) story, original_story_found = MStory.find_story(feed_id, story_id) if not story: return json.json_response(request, { 'code': -1, 'message': 'Could not find the original story and no copies could be found.' }) shared_story = MSharedStory.objects.filter(user_id=request.user.pk, story_feed_id=feed_id, story_guid=story_id).limit(1).first() if not shared_story: story_db = dict([(k, v) for k, v in story._data.items() if k is not None and v is not None]) story_values = dict(user_id=request.user.pk, comments=comments, has_comments=bool(comments), story_db_id=story.id) story_db.update(story_values) shared_story = MSharedStory.objects.create(**story_db) if source_user_id: shared_story.set_source_user_id(int(source_user_id)) socialsubs = MSocialSubscription.objects.filter(subscription_user_id=request.user.pk) for socialsub in socialsubs: socialsub.needs_unread_recalc = True socialsub.save() logging.user(request, "~FCSharing ~FM%s: ~SB~FB%s" % (story.story_title[:20], comments[:30])) else: shared_story.comments = comments shared_story.has_comments = bool(comments) shared_story.save() logging.user(request, "~FCUpdating shared story ~FM%s: ~SB~FB%s" % ( story.story_title[:20], comments[:30])) if original_story_found: story.count_comments() shared_story.publish_update_to_subscribers() story = Feed.format_story(story) check_all = not original_story_found stories, profiles = MSharedStory.stories_with_comments_and_profiles([story], request.user.pk, check_all=check_all) story = stories[0] story['shared_comments'] = strip_tags(shared_story['comments'] or "") if post_to_services: for service in post_to_services: if service not in shared_story.posted_to_services: PostToService.delay(shared_story_id=shared_story.id, service=service) if shared_story.source_user_id and shared_story.comments: EmailStoryReshares.apply_async(kwargs=dict(shared_story_id=shared_story.id), countdown=60) if format == 'html': stories = MSharedStory.attach_users_to_stories(stories, profiles) return render_to_response('social/story_share.xhtml', { 'story': story, }, context_instance=RequestContext(request)) else: return json.json_response(request, { 'code': code, 'story': story, 'user_profiles': profiles, })
class UserSubscription(models.Model): """ A feed which a user has subscrubed to. Carries all of the cached information about the subscription, including unread counts of the three primary scores. Also has a dirty flag (needs_unread_recalc) which means that the unread counts are not accurate and need to be calculated with `self.calculate_feed_scores()`. """ UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta( days=settings.DAYS_OF_UNREAD) user = models.ForeignKey(User, related_name='subscriptions') feed = models.ForeignKey(Feed, related_name='subscribers') user_title = models.CharField(max_length=255, null=True, blank=True) active = models.BooleanField(default=False) last_read_date = models.DateTimeField(default=UNREAD_CUTOFF) mark_read_date = models.DateTimeField(default=UNREAD_CUTOFF) unread_count_neutral = models.IntegerField(default=0) unread_count_positive = models.IntegerField(default=0) unread_count_negative = models.IntegerField(default=0) unread_count_updated = models.DateTimeField(default=datetime.datetime.now) oldest_unread_story_date = models.DateTimeField( default=datetime.datetime.now) needs_unread_recalc = models.BooleanField(default=False) feed_opens = models.IntegerField(default=0) is_trained = models.BooleanField(default=False) objects = UserSubscriptionManager() def __unicode__(self): return '[%s (%s): %s (%s)] ' % (self.user.username, self.user.pk, self.feed.feed_title, self.feed.pk) class Meta: unique_together = ("user", "feed") def canonical(self, full=False, include_favicon=True, classifiers=None): feed = self.feed.canonical(full=full, include_favicon=include_favicon) feed['feed_title'] = self.user_title or feed['feed_title'] feed['ps'] = self.unread_count_positive feed['nt'] = self.unread_count_neutral feed['ng'] = self.unread_count_negative feed['active'] = self.active feed['feed_opens'] = self.feed_opens feed['subscribed'] = True if classifiers: feed['classifiers'] = classifiers if not self.active and self.user.profile.is_premium: feed['active'] = True self.active = True self.save() return feed def save(self, *args, **kwargs): user_title_max = self._meta.get_field('user_title').max_length if self.user_title and len(self.user_title) > user_title_max: self.user_title = self.user_title[:user_title_max] if not self.active and self.user.profile.is_premium: self.active = True try: super(UserSubscription, self).save(*args, **kwargs) except IntegrityError: duplicate_feeds = DuplicateFeed.objects.filter( duplicate_feed_id=self.feed_id) for duplicate_feed in duplicate_feeds: already_subscribed = UserSubscription.objects.filter( user=self.user, feed=duplicate_feed.feed) if not already_subscribed: self.feed = duplicate_feed.feed super(UserSubscription, self).save(*args, **kwargs) break else: self.delete() @classmethod def sync_all_redis(cls, user_id, skip_feed=False): us = cls.objects.filter(user=user_id) for sub in us: print " ---> Syncing usersub: %s" % sub sub.sync_redis(skip_feed=skip_feed) def sync_redis(self, skip_feed=False): r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL) h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL) UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta( days=settings.DAYS_OF_UNREAD + 1) userstories = MUserStory.objects.filter(feed_id=self.feed_id, user_id=self.user_id, read_date__gte=UNREAD_CUTOFF) total = userstories.count() logging.debug(" ---> ~SN~FMSyncing ~SB%s~SN stories (%s)" % (total, self)) pipeline = r.pipeline() hashpipe = h.pipeline() for userstory in userstories: userstory.sync_redis(pipeline=pipeline, hashpipe=hashpipe) pipeline.execute() hashpipe.execute() def get_stories(self, offset=0, limit=6, order='newest', read_filter='all', withscores=False): r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL) ignore_user_stories = False stories_key = 'F:%s' % (self.feed_id) read_stories_key = 'RS:%s:%s' % (self.user_id, self.feed_id) unread_stories_key = 'U:%s:%s' % (self.user_id, self.feed_id) unread_ranked_stories_key = 'zU:%s:%s' % (self.user_id, self.feed_id) if offset and not withscores and r.exists(unread_ranked_stories_key): pass else: r.delete(unread_ranked_stories_key) if not r.exists(stories_key): print " ---> No stories on feed: %s" % self return [] elif read_filter != 'unread' or not r.exists(read_stories_key): ignore_user_stories = True unread_stories_key = stories_key else: r.sdiffstore(unread_stories_key, stories_key, read_stories_key) sorted_stories_key = 'zF:%s' % (self.feed_id) unread_ranked_stories_key = 'zU:%s:%s' % (self.user_id, self.feed_id) r.zinterstore(unread_ranked_stories_key, [sorted_stories_key, unread_stories_key]) current_time = int(time.time() + 60 * 60 * 24) if order == 'oldest': byscorefunc = r.zrangebyscore if read_filter == 'unread': min_score = int(time.mktime( self.mark_read_date.timetuple())) + 1 else: now = datetime.datetime.now() two_weeks_ago = now - datetime.timedelta( days=settings.DAYS_OF_UNREAD) min_score = int(time.mktime(two_weeks_ago.timetuple())) - 1000 max_score = current_time else: byscorefunc = r.zrevrangebyscore min_score = current_time if read_filter == 'unread': # +1 for the intersection b/w zF and F, which carries an implicit score of 1. max_score = int(time.mktime( self.mark_read_date.timetuple())) + 1 else: max_score = 0 if settings.DEBUG: debug_stories = r.zrevrange(unread_ranked_stories_key, 0, -1, withscores=True) print " ---> Unread all stories (%s - %s) %s stories: %s" % ( min_score, max_score, len(debug_stories), debug_stories) story_ids = byscorefunc(unread_ranked_stories_key, min_score, max_score, start=offset, num=500, withscores=withscores)[:limit] r.expire(unread_ranked_stories_key, 24 * 60 * 60) if not ignore_user_stories: r.delete(unread_stories_key) # XXX TODO: Remove below line after combing redis for these None's. story_ids = [s for s in story_ids if s and s != 'None'] # ugh, hack if withscores: return story_ids elif story_ids: story_date_order = "%sstory_date" % ('' if order == 'oldest' else '-') mstories = MStory.objects( id__in=story_ids).order_by(story_date_order) stories = Feed.format_stories(mstories) return stories else: return [] @classmethod def feed_stories(cls, user_id, feed_ids, offset=0, limit=6, order='newest', read_filter='all'): r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL) if order == 'oldest': range_func = r.zrange else: range_func = r.zrevrange if not isinstance(feed_ids, list): feed_ids = [feed_ids] unread_ranked_stories_keys = 'zU:%s:feeds' % (user_id) if offset and r.exists(unread_ranked_stories_keys): story_guids = range_func(unread_ranked_stories_keys, offset, limit) return story_guids else: r.delete(unread_ranked_stories_keys) for feed_id in feed_ids: try: us = cls.objects.get(user=user_id, feed=feed_id) except cls.DoesNotExist: continue story_guids = us.get_stories(offset=0, limit=200, order=order, read_filter=read_filter, withscores=True) if story_guids: r.zadd(unread_ranked_stories_keys, **dict(story_guids)) story_guids = range_func(unread_ranked_stories_keys, offset, limit) r.expire(unread_ranked_stories_keys, 24 * 60 * 60) return story_guids @classmethod def add_subscription(cls, user, feed_address, folder=None, bookmarklet=False, auto_active=True, skip_fetch=False): feed = None us = None logging.user( user, "~FRAdding URL: ~SB%s (in %s) %s" % (feed_address, folder, "~FCAUTO-ADD" if not auto_active else "")) feed = Feed.get_feed_from_url(feed_address) if not feed: code = -1 if bookmarklet: message = "This site does not have an RSS feed. Nothing is linked to from this page." else: message = "This address does not point to an RSS feed or a website with an RSS feed." else: us, subscription_created = cls.objects.get_or_create( feed=feed, user=user, defaults={ 'needs_unread_recalc': True, 'active': auto_active, }) code = 1 message = "" if us: user_sub_folders_object, created = UserSubscriptionFolders.objects.get_or_create( user=user, defaults={'folders': '[]'}) if created: user_sub_folders = [] else: user_sub_folders = json.decode(user_sub_folders_object.folders) user_sub_folders = add_object_to_folder(feed.pk, folder, user_sub_folders) user_sub_folders_object.folders = json.encode(user_sub_folders) user_sub_folders_object.save() if auto_active or user.profile.is_premium: us.active = True us.save() if not skip_fetch and feed.last_update < datetime.datetime.utcnow( ) - datetime.timedelta(days=1): feed = feed.update() from apps.social.models import MActivity MActivity.new_feed_subscription(user_id=user.pk, feed_id=feed.pk, feed_title=feed.title) feed.setup_feed_for_premium_subscribers() return code, message, us @classmethod def feeds_with_updated_counts(cls, user, feed_ids=None, check_fetch_status=False): feeds = {} # Get subscriptions for user user_subs = cls.objects.select_related('feed').filter(user=user, active=True) feed_ids = [f for f in feed_ids if f and not f.startswith('river')] if feed_ids: user_subs = user_subs.filter(feed__in=feed_ids) UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta( days=settings.DAYS_OF_UNREAD) for i, sub in enumerate(user_subs): # Count unreads if subscription is stale. if (sub.needs_unread_recalc or sub.unread_count_updated < UNREAD_CUTOFF or sub.oldest_unread_story_date < UNREAD_CUTOFF): sub = sub.calculate_feed_scores(silent=True) if not sub: continue # TODO: Figure out the correct sub and give it a new feed_id feed_id = sub.feed_id feeds[feed_id] = { 'ps': sub.unread_count_positive, 'nt': sub.unread_count_neutral, 'ng': sub.unread_count_negative, 'id': feed_id, } if not sub.feed.fetched_once or check_fetch_status: feeds[feed_id]['fetched_once'] = sub.feed.fetched_once feeds[feed_id][ 'not_yet_fetched'] = not sub.feed.fetched_once # Legacy. Dammit. if sub.feed.favicon_fetching: feeds[feed_id]['favicon_fetching'] = True if sub.feed.has_feed_exception or sub.feed.has_page_exception: feeds[feed_id]['has_exception'] = True feeds[feed_id][ 'exception_type'] = 'feed' if sub.feed.has_feed_exception else 'page' feeds[feed_id]['feed_address'] = sub.feed.feed_address feeds[feed_id]['exception_code'] = sub.feed.exception_code return feeds def mark_feed_read(self): if (self.unread_count_negative == 0 and self.unread_count_neutral == 0 and self.unread_count_positive == 0 and not self.needs_unread_recalc): return now = datetime.datetime.utcnow() # Use the latest story to get last read time. latest_story = MStory.objects(story_feed_id=self.feed.pk).order_by( '-story_date').only('story_date').limit(1) if latest_story and len(latest_story) >= 1: latest_story_date = latest_story[0]['story_date']\ + datetime.timedelta(seconds=1) else: latest_story_date = now self.last_read_date = latest_story_date self.mark_read_date = latest_story_date self.unread_count_negative = 0 self.unread_count_positive = 0 self.unread_count_neutral = 0 self.unread_count_updated = now self.oldest_unread_story_date = now self.needs_unread_recalc = False # No longer removing old user read stories, since they're needed for social, # and they get cleaned up automatically when new stories come in. # MUserStory.delete_old_stories(self.user_id, self.feed_id) self.save() def mark_story_ids_as_read(self, story_ids, request=None): data = dict(code=0, payload=story_ids) if not request: request = self.user if not self.needs_unread_recalc: self.needs_unread_recalc = True self.save() if len(story_ids) > 1: logging.user( request, "~FYRead %s stories in feed: %s" % (len(story_ids), self.feed)) else: logging.user(request, "~FYRead story in feed: %s" % (self.feed)) for story_id in set(story_ids): story, _ = MStory.find_story(story_feed_id=self.feed_id, story_id=story_id) if not story: continue now = datetime.datetime.utcnow() date = now if now > story.story_date else story.story_date # For handling future stories m, _ = MUserStory.objects.get_or_create(story_id=story_id, user_id=self.user_id, feed_id=self.feed_id, defaults={ 'read_date': date, 'story': story, 'story_date': story.story_date, }) return data def calculate_feed_scores(self, silent=False, stories=None): # now = datetime.datetime.strptime("2009-07-06 22:30:03", "%Y-%m-%d %H:%M:%S") now = datetime.datetime.now() UNREAD_CUTOFF = now - datetime.timedelta(days=settings.DAYS_OF_UNREAD) if self.user.profile.last_seen_on < UNREAD_CUTOFF: # if not silent: # logging.info(' ---> [%s] SKIPPING Computing scores: %s (1 week+)' % (self.user, self.feed)) return if not self.feed.fetched_once: if not silent: logging.info(' ---> [%s] NOT Computing scores: %s' % (self.user, self.feed)) self.needs_unread_recalc = False self.save() return feed_scores = dict(negative=0, neutral=0, positive=0) # Two weeks in age. If mark_read_date is older, mark old stories as read. date_delta = UNREAD_CUTOFF if date_delta < self.mark_read_date: date_delta = self.mark_read_date else: self.mark_read_date = date_delta if not stories: stories_db = MStory.objects(story_feed_id=self.feed_id, story_date__gte=date_delta) stories = Feed.format_stories(stories_db, self.feed_id) story_ids = [s['id'] for s in stories] read_stories = MUserStory.objects(user_id=self.user_id, feed_id=self.feed_id, story_id__in=story_ids) read_stories_ids = [us.story_id for us in read_stories] oldest_unread_story_date = now unread_stories = [] for story in stories: if story['story_date'] < date_delta: continue if story['id'] not in read_stories_ids: unread_stories.append(story) if story['story_date'] < oldest_unread_story_date: oldest_unread_story_date = story['story_date'] # if not silent: # logging.info(' ---> [%s] Format stories: %s' % (self.user, datetime.datetime.now() - now)) classifier_feeds = list( MClassifierFeed.objects(user_id=self.user_id, feed_id=self.feed_id, social_user_id=0)) classifier_authors = list( MClassifierAuthor.objects(user_id=self.user_id, feed_id=self.feed_id)) classifier_titles = list( MClassifierTitle.objects(user_id=self.user_id, feed_id=self.feed_id)) classifier_tags = list( MClassifierTag.objects(user_id=self.user_id, feed_id=self.feed_id)) # if not silent: # logging.info(' ---> [%s] Classifiers: %s (%s)' % (self.user, datetime.datetime.now() - now, classifier_feeds.count() + classifier_authors.count() + classifier_tags.count() + classifier_titles.count())) scores = { 'feed': apply_classifier_feeds(classifier_feeds, self.feed), } for story in unread_stories: scores.update({ 'author': apply_classifier_authors(classifier_authors, story), 'tags': apply_classifier_tags(classifier_tags, story), 'title': apply_classifier_titles(classifier_titles, story), }) max_score = max(scores['author'], scores['tags'], scores['title']) min_score = min(scores['author'], scores['tags'], scores['title']) if max_score > 0: feed_scores['positive'] += 1 elif min_score < 0: feed_scores['negative'] += 1 else: if scores['feed'] > 0: feed_scores['positive'] += 1 elif scores['feed'] < 0: feed_scores['negative'] += 1 else: feed_scores['neutral'] += 1 # if not silent: # logging.info(' ---> [%s] End classifiers: %s' % (self.user, datetime.datetime.now() - now)) self.unread_count_positive = feed_scores['positive'] self.unread_count_neutral = feed_scores['neutral'] self.unread_count_negative = feed_scores['negative'] self.unread_count_updated = datetime.datetime.now() self.oldest_unread_story_date = oldest_unread_story_date self.needs_unread_recalc = False self.save() if (self.unread_count_positive == 0 and self.unread_count_neutral == 0 and self.unread_count_negative == 0): self.mark_feed_read() if not silent: logging.user( self.user, '~FC~SNComputing scores: %s (~SB%s~SN/~SB%s~SN/~SB%s~SN)' % (self.feed, feed_scores['negative'], feed_scores['neutral'], feed_scores['positive'])) return self def switch_feed(self, new_feed, old_feed): # Rewrite feed in subscription folders try: user_sub_folders = UserSubscriptionFolders.objects.get( user=self.user) except Exception, e: logging.info(" *** ---> UserSubscriptionFolders error: %s" % e) return # Switch to original feed for the user subscription logging.info(" ===> %s " % self.user) self.feed = new_feed self.needs_unread_recalc = True try: self.save() user_sub_folders.rewrite_feed(new_feed, old_feed) except (IntegrityError, OperationError): logging.info(" !!!!> %s already subscribed" % self.user) self.delete() return # Switch read stories user_stories = MUserStory.objects(user_id=self.user_id, feed_id=old_feed.pk) if user_stories.count() > 0: logging.info(" ---> %s read stories" % user_stories.count()) for user_story in user_stories: user_story.feed_id = new_feed.pk duplicate_story = user_story.story if duplicate_story: story_guid = duplicate_story.story_guid if hasattr( duplicate_story, 'story_guid') else duplicate_story.id original_story, _ = MStory.find_story( story_feed_id=new_feed.pk, story_id=story_guid, original_only=True) if original_story: user_story.story = original_story try: user_story.save() except OperationError: # User read the story in the original feed, too. Ugh, just ignore it. pass else: user_story.delete() else: user_story.delete() def switch_feed_for_classifier(model): duplicates = model.objects(feed_id=old_feed.pk, user_id=self.user_id) if duplicates.count(): logging.info(" ---> Switching %s %s" % (duplicates.count(), model)) for duplicate in duplicates: duplicate.feed_id = new_feed.pk if duplicate.social_user_id is None: duplicate.social_user_id = 0 try: duplicate.save() pass except (IntegrityError, OperationError): logging.info(" !!!!> %s already exists" % duplicate) duplicate.delete() switch_feed_for_classifier(MClassifierTitle) switch_feed_for_classifier(MClassifierAuthor) switch_feed_for_classifier(MClassifierFeed) switch_feed_for_classifier(MClassifierTag)