def original_text(request):
story_id = request.REQUEST.get('story_id')
feed_id = request.REQUEST.get('feed_id')
story_hash = request.REQUEST.get('story_hash', None)
force = request.REQUEST.get('force', False)
debug = request.REQUEST.get('debug', False)
if story_hash:
story, _ = MStory.find_story(story_hash=story_hash)
else:
story, _ = MStory.find_story(story_id=story_id, story_feed_id=feed_id)
if not story:
logging.user(request, "~FYFetching ~FGoriginal~FY story text: ~FRstory not found")
return {'code': -1, 'message': 'Story not found.', 'original_text': None, 'failed': True}
original_text = story.fetch_original_text(force=force, request=request, debug=debug)
return {
'feed_id': story.story_feed_id,
'story_hash': story.story_hash,
'story_id': story.story_guid,
'image_urls': story.image_urls,
'secure_image_urls': Feed.secure_image_urls(story.image_urls),
'original_text': original_text,
'failed': not original_text or len(original_text) < 100,
}
def original_text(request):
story_id = request.REQUEST.get('story_id')
feed_id = request.REQUEST.get('feed_id')
story_hash = request.REQUEST.get('story_hash', None)
force = request.REQUEST.get('force', False)
debug = request.REQUEST.get('debug', False)
if story_hash:
story, _ = MStory.find_story(story_hash=story_hash)
else:
story, _ = MStory.find_story(story_id=story_id, story_feed_id=feed_id)
if not story:
logging.user(
request,
"~FYFetching ~FGoriginal~FY story text: ~FRstory not found")
return {
'code': -1,
'message': 'Story not found.',
'original_text': None,
'failed': True
}
original_text = story.fetch_original_text(force=force,
request=request,
debug=debug)
return {
'feed_id': feed_id,
'story_id': story_id,
'original_text': original_text,
'failed': not original_text or len(original_text) < 100,
}
def original_text(request):
# iOS sends a POST, web sends a GET
GET_POST = getattr(request, request.method)
story_id = GET_POST.get('story_id')
feed_id = GET_POST.get('feed_id')
story_hash = GET_POST.get('story_hash', None)
force = GET_POST.get('force', False)
debug = GET_POST.get('debug', False)
if not story_hash and not story_id:
return {'code': -1, 'message': 'Missing story_hash.', 'original_text': None, 'failed': True}
if story_hash:
story, _ = MStory.find_story(story_hash=story_hash)
else:
story, _ = MStory.find_story(story_id=story_id, story_feed_id=feed_id)
if not story:
logging.user(request, "~FYFetching ~FGoriginal~FY story text: ~FRstory not found")
return {'code': -1, 'message': 'Story not found.', 'original_text': None, 'failed': True}
original_text = story.fetch_original_text(force=force, request=request, debug=debug)
return {
'feed_id': story.story_feed_id,
'story_hash': story.story_hash,
'story_id': story.story_guid,
'image_urls': story.image_urls,
'secure_image_urls': Feed.secure_image_urls(story.image_urls),
'original_text': original_text,
'failed': not original_text or len(original_text) < 100,
}
def mark_story_ids_as_read(self, story_ids, request=None):
data = dict(code=0, payload=story_ids)
if not request:
request = self.user
if not self.needs_unread_recalc:
self.needs_unread_recalc = True
self.save()
if len(story_ids) > 1:
logging.user(request, "~FYRead %s stories in feed: %s" % (len(story_ids), self.feed))
else:
logging.user(request, "~FYRead story in feed: %s" % (self.feed))
for story_id in set(story_ids):
story, _ = MStory.find_story(story_feed_id=self.feed_id, story_id=story_id)
if not story: continue
now = datetime.datetime.utcnow()
date = now if now > story.story_date else story.story_date # For handling future stories
m, _ = MUserStory.objects.get_or_create(story_id=story_id, user_id=self.user_id,
feed_id=self.feed_id, defaults={
'read_date': date,
'story': story,
'story_date': story.story_date,
})
return data
def story_hash(cls, story_id, story_feed_id):
if not cls.RE_STORY_HASH.match(story_id):
story, _ = MStory.find_story(story_feed_id=story_feed_id, story_id=story_id)
if not story: return
story_id = story.story_hash
return story_id
def story_hash(cls, story_id, story_feed_id):
if not cls.RE_STORY_HASH.match(story_id):
story, _ = MStory.find_story(story_feed_id=story_feed_id, story_id=story_id)
if story:
story_id = story.story_hash
else:
story_id = "%s:%s" % (story_feed_id, hashlib.sha1(story_id).hexdigest()[:6])
return story_id
def story_hash(cls, story_id, story_feed_id):
if not cls.RE_STORY_HASH.match(story_id):
story, _ = MStory.find_story(story_feed_id=story_feed_id, story_id=story_id)
if story:
story_id = story.story_hash
else:
story_id = "%s:%s" % (story_feed_id, hashlib.sha1(story_id).hexdigest()[:6])
return story_id
def story_changes(request):
story_hash = request.REQUEST.get('story_hash', None)
show_changes = is_true(request.REQUEST.get('show_changes', True))
story, _ = MStory.find_story(story_hash=story_hash)
if not story:
logging.user(request, "~FYFetching ~FGoriginal~FY story page: ~FRstory not found")
return {'code': -1, 'message': 'Story not found.', 'original_page': None, 'failed': True}
return {
'story': Feed.format_story(story, show_changes=show_changes)
}
def story_changes(request):
story_hash = request.REQUEST.get('story_hash', None)
show_changes = is_true(request.REQUEST.get('show_changes', True))
story, _ = MStory.find_story(story_hash=story_hash)
if not story:
logging.user(request, "~FYFetching ~FGoriginal~FY story page: ~FRstory not found")
return {'code': -1, 'message': 'Story not found.', 'original_page': None, 'failed': True}
return {
'story': Feed.format_story(story, show_changes=show_changes)
}
def original_story(request):
story_hash = request.REQUEST.get('story_hash')
force = request.REQUEST.get('force', False)
debug = request.REQUEST.get('debug', False)
story, _ = MStory.find_story(story_hash=story_hash)
if not story:
logging.user(request, "~FYFetching ~FGoriginal~FY story page: ~FRstory not found")
return {'code': -1, 'message': 'Story not found.', 'original_page': None, 'failed': True}
original_page = story.fetch_original_page(force=force, request=request, debug=debug)
return HttpResponse(original_page or "")
def original_text(request):
story_id = request.GET["story_id"]
feed_id = request.GET["feed_id"]
force = request.GET.get("force", False)
story, _ = MStory.find_story(story_id=story_id, story_feed_id=feed_id)
if not story:
logging.user(request, "~FYFetching ~FGoriginal~FY story text: ~FRstory not found")
return {"code": -1, "message": "Story not found."}
original_text = story.fetch_original_text(force=force, request=request)
return {"feed_id": feed_id, "story_id": story_id, "original_text": original_text}
def original_story(request):
story_hash = request.REQUEST.get("story_hash")
force = request.REQUEST.get("force", False)
debug = request.REQUEST.get("debug", False)
story, _ = MStory.find_story(story_hash=story_hash)
if not story:
logging.user(request, "~FYFetching ~FGoriginal~FY story page: ~FRstory not found")
return {"code": -1, "message": "Story not found.", "original_page": None, "failed": True}
original_page = story.fetch_original_page(force=force, request=request, debug=debug)
return HttpResponse(original_page or "")
def original_story(request):
story_hash = request.REQUEST.get('story_hash')
force = request.REQUEST.get('force', False)
debug = request.REQUEST.get('debug', False)
story, _ = MStory.find_story(story_hash=story_hash)
if not story:
logging.user(request, "~FYFetching ~FGoriginal~FY story page: ~FRstory not found")
return {'code': -1, 'message': 'Story not found.', 'original_page': None, 'failed': True}
original_page = story.fetch_original_page(force=force, request=request, debug=debug)
return HttpResponse(original_page or "")
def original_text(request):
story_id = request.REQUEST.get("story_id")
feed_id = request.REQUEST.get("feed_id")
story_hash = request.REQUEST.get("story_hash", None)
force = request.REQUEST.get("force", False)
debug = request.REQUEST.get("debug", False)
if story_hash:
story, _ = MStory.find_story(story_hash=story_hash)
else:
story, _ = MStory.find_story(story_id=story_id, story_feed_id=feed_id)
if not story:
logging.user(request, "~FYFetching ~FGoriginal~FY story text: ~FRstory not found")
return {"code": -1, "message": "Story not found.", "original_text": None, "failed": True}
original_text = story.fetch_original_text(force=force, request=request, debug=debug)
return {
"feed_id": feed_id,
"story_id": story_id,
"original_text": original_text,
"failed": not original_text or len(original_text) < 100,
}
def original_text(request):
story_id = request.GET['story_id']
feed_id = request.GET['feed_id']
force = request.GET.get('force', False)
story, _ = MStory.find_story(story_id=story_id, story_feed_id=feed_id)
if not story:
logging.user(request, "~FYFetching ~FGoriginal~FY story text: ~FRstory not found")
return {'code': -1, 'message': 'Story not found.'}
original_text = story.fetch_original_text(force=force, request=request)
return {
'feed_id': feed_id,
'story_id': story_id,
'original_text': original_text,
}
def story_db_id(self):
if self.story:
return self.story.id
elif self.found_story:
if "_ref" in self.found_story:
return self.found_story["_ref"].id
elif hasattr(self.found_story, "id"):
return self.found_story.id
story, found_original = MStory.find_story(self.feed_id, self.story_id)
if story:
if found_original:
self.story = story
else:
self.found_story = story
self.save()
return story.id
def original_text(request):
story_id = request.GET['story_id']
feed_id = request.GET['feed_id']
force = request.GET.get('force', False)
story, _ = MStory.find_story(story_id=story_id, story_feed_id=feed_id)
if not story:
logging.user(request, "~FYFetching ~FGoriginal~FY story text: ~FRstory not found")
return {'code': -1, 'message': 'Story not found.'}
original_text = story.fetch_original_text(force=force, request=request)
return {
'feed_id': feed_id,
'story_id': story_id,
'original_text': original_text,
}
def story_db_id(self):
if self.story:
return self.story.id
elif self.found_story:
if '_ref' in self.found_story:
return self.found_story['_ref'].id
elif hasattr(self.found_story, 'id'):
return self.found_story.id
story, found_original = MStory.find_story(self.feed_id, self.story_id)
if story:
if found_original:
self.story = story
else:
self.found_story = story
self.save()
return story.id
def story_db_id(self):
if self.story:
return self.story.id
elif self.found_story:
if '_ref' in self.found_story:
return self.found_story['_ref'].id
elif hasattr(self.found_story, 'id'):
return self.found_story.id
story, found_original = MStory.find_story(self.feed_id, self.story_id)
if story:
if found_original:
self.story = story
else:
self.found_story = story
self.save()
return story.id
def original_text(request):
story_id = request.REQUEST.get('story_id')
feed_id = request.REQUEST.get('feed_id')
force = request.REQUEST.get('force', False)
story, _ = MStory.find_story(story_id=story_id, story_feed_id=feed_id)
if not story:
logging.user(request, "~FYFetching ~FGoriginal~FY story text: ~FRstory not found")
return {'code': -1, 'message': 'Story not found.', 'original_text': None, 'failed': True}
original_text = story.fetch_original_text(force=force, request=request)
return {
'feed_id': feed_id,
'story_id': story_id,
'original_text': original_text,
'failed': not original_text or len(original_text) < 100,
}
def mark_story_ids_as_read(self, story_ids, request=None):
data = dict(code=0, payload=story_ids)
if not request:
request = self.user
if not self.needs_unread_recalc:
self.needs_unread_recalc = True
self.save()
if len(story_ids) > 1:
logging.user(
request,
"~FYRead %s stories in feed: %s" % (len(story_ids), self.feed))
else:
logging.user(request, "~FYRead story in feed: %s" % (self.feed))
for story_id in set(story_ids):
story, _ = MStory.find_story(story_feed_id=self.feed_id,
story_id=story_id)
if not story: continue
now = datetime.datetime.utcnow()
date = now if now > story.story_date else story.story_date # For handling future stories
m, _ = MUserStory.objects.get_or_create(story_id=story_id,
user_id=self.user_id,
feed_id=self.feed_id,
defaults={
'read_date':
date,
'story':
story,
'story_date':
story.story_date,
})
return data
def mark_story_as_shared(request):
code = 1
feed_id = int(request.POST["feed_id"])
story_id = request.POST["story_id"]
comments = request.POST.get("comments", "")
source_user_id = request.POST.get("source_user_id")
relative_user_id = request.POST.get("relative_user_id") or request.user.pk
post_to_services = request.POST.getlist("post_to_services")
format = request.REQUEST.get("format", "json")
MSocialProfile.get_user(request.user.pk)
story, original_story_found = MStory.find_story(feed_id, story_id)
if not story:
return json.json_response(
request, {"code": -1, "message": "Could not find the original story and no copies could be found."}
)
shared_story = (
MSharedStory.objects.filter(user_id=request.user.pk, story_feed_id=feed_id, story_guid=story_id)
.limit(1)
.first()
)
if not shared_story:
story_db = dict([(k, v) for k, v in story._data.items() if k is not None and v is not None])
story_values = dict(
user_id=request.user.pk, comments=comments, has_comments=bool(comments), story_db_id=story.id
)
story_db.update(story_values)
shared_story = MSharedStory.objects.create(**story_db)
if source_user_id:
shared_story.set_source_user_id(int(source_user_id))
socialsubs = MSocialSubscription.objects.filter(subscription_user_id=request.user.pk)
for socialsub in socialsubs:
socialsub.needs_unread_recalc = True
socialsub.save()
logging.user(request, "~FCSharing ~FM%s: ~SB~FB%s" % (story.story_title[:20], comments[:30]))
else:
shared_story.comments = comments
shared_story.has_comments = bool(comments)
shared_story.save()
logging.user(request, "~FCUpdating shared story ~FM%s: ~SB~FB%s" % (story.story_title[:20], comments[:30]))
if original_story_found:
story.count_comments()
shared_story.publish_update_to_subscribers()
story = Feed.format_story(story)
check_all = not original_story_found
stories, profiles = MSharedStory.stories_with_comments_and_profiles([story], relative_user_id, check_all=check_all)
story = stories[0]
story["shared_comments"] = strip_tags(shared_story["comments"] or "")
story["shared_by_user"] = True
if post_to_services:
for service in post_to_services:
if service not in shared_story.posted_to_services:
PostToService.delay(shared_story_id=shared_story.id, service=service)
if shared_story.source_user_id and shared_story.comments:
EmailStoryReshares.apply_async(
kwargs=dict(shared_story_id=shared_story.id), countdown=settings.SECONDS_TO_DELAY_CELERY_EMAILS
)
if format == "html":
stories = MSharedStory.attach_users_to_stories(stories, profiles)
return render_to_response(
"social/social_story.xhtml", {"story": story}, context_instance=RequestContext(request)
)
else:
return json.json_response(request, {"code": code, "story": story, "user_profiles": profiles})
def mark_story_as_shared(request):
code = 1
feed_id = int(request.POST['feed_id'])
story_id = request.POST['story_id']
comments = request.POST.get('comments', '')
source_user_id = request.POST.get('source_user_id')
post_to_services = request.POST.getlist('post_to_services')
format = request.REQUEST.get('format', 'json')
MSocialProfile.get_user(request.user.pk)
story, original_story_found = MStory.find_story(feed_id, story_id)
if not story:
return json.json_response(request, {
'code': -1,
'message': 'Could not find the original story and no copies could be found.'
})
shared_story = MSharedStory.objects.filter(user_id=request.user.pk,
story_feed_id=feed_id,
story_guid=story_id).limit(1).first()
if not shared_story:
story_db = dict([(k, v) for k, v in story._data.items()
if k is not None and v is not None])
story_values = dict(user_id=request.user.pk, comments=comments,
has_comments=bool(comments), story_db_id=story.id)
story_db.update(story_values)
shared_story = MSharedStory.objects.create(**story_db)
if source_user_id:
shared_story.set_source_user_id(int(source_user_id))
socialsubs = MSocialSubscription.objects.filter(subscription_user_id=request.user.pk)
for socialsub in socialsubs:
socialsub.needs_unread_recalc = True
socialsub.save()
logging.user(request, "~FCSharing ~FM%s: ~SB~FB%s" % (story.story_title[:20], comments[:30]))
else:
shared_story.comments = comments
shared_story.has_comments = bool(comments)
shared_story.save()
logging.user(request, "~FCUpdating shared story ~FM%s: ~SB~FB%s" % (
story.story_title[:20], comments[:30]))
if original_story_found:
story.count_comments()
shared_story.publish_update_to_subscribers()
story = Feed.format_story(story)
check_all = not original_story_found
stories, profiles = MSharedStory.stories_with_comments_and_profiles([story], request.user.pk,
check_all=check_all)
story = stories[0]
story['shared_comments'] = strip_tags(shared_story['comments'] or "")
if post_to_services:
for service in post_to_services:
if service not in shared_story.posted_to_services:
PostToService.delay(shared_story_id=shared_story.id, service=service)
if shared_story.source_user_id and shared_story.comments:
EmailStoryReshares.apply_async(kwargs=dict(shared_story_id=shared_story.id), countdown=60)
if format == 'html':
stories = MSharedStory.attach_users_to_stories(stories, profiles)
return render_to_response('social/story_share.xhtml', {
'story': story,
}, context_instance=RequestContext(request))
else:
return json.json_response(request, {
'code': code,
'story': story,
'user_profiles': profiles,
})
class UserSubscription(models.Model):
"""
A feed which a user has subscrubed to. Carries all of the cached information
about the subscription, including unread counts of the three primary scores.
Also has a dirty flag (needs_unread_recalc) which means that the unread counts
are not accurate and need to be calculated with `self.calculate_feed_scores()`.
"""
UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(
days=settings.DAYS_OF_UNREAD)
user = models.ForeignKey(User, related_name='subscriptions')
feed = models.ForeignKey(Feed, related_name='subscribers')
user_title = models.CharField(max_length=255, null=True, blank=True)
active = models.BooleanField(default=False)
last_read_date = models.DateTimeField(default=UNREAD_CUTOFF)
mark_read_date = models.DateTimeField(default=UNREAD_CUTOFF)
unread_count_neutral = models.IntegerField(default=0)
unread_count_positive = models.IntegerField(default=0)
unread_count_negative = models.IntegerField(default=0)
unread_count_updated = models.DateTimeField(default=datetime.datetime.now)
oldest_unread_story_date = models.DateTimeField(
default=datetime.datetime.now)
needs_unread_recalc = models.BooleanField(default=False)
feed_opens = models.IntegerField(default=0)
is_trained = models.BooleanField(default=False)
objects = UserSubscriptionManager()
def __unicode__(self):
return '[%s (%s): %s (%s)] ' % (self.user.username, self.user.pk,
self.feed.feed_title, self.feed.pk)
class Meta:
unique_together = ("user", "feed")
def canonical(self, full=False, include_favicon=True, classifiers=None):
feed = self.feed.canonical(full=full, include_favicon=include_favicon)
feed['feed_title'] = self.user_title or feed['feed_title']
feed['ps'] = self.unread_count_positive
feed['nt'] = self.unread_count_neutral
feed['ng'] = self.unread_count_negative
feed['active'] = self.active
feed['feed_opens'] = self.feed_opens
feed['subscribed'] = True
if classifiers:
feed['classifiers'] = classifiers
if not self.active and self.user.profile.is_premium:
feed['active'] = True
self.active = True
self.save()
return feed
def save(self, *args, **kwargs):
user_title_max = self._meta.get_field('user_title').max_length
if self.user_title and len(self.user_title) > user_title_max:
self.user_title = self.user_title[:user_title_max]
if not self.active and self.user.profile.is_premium:
self.active = True
try:
super(UserSubscription, self).save(*args, **kwargs)
except IntegrityError:
duplicate_feeds = DuplicateFeed.objects.filter(
duplicate_feed_id=self.feed_id)
for duplicate_feed in duplicate_feeds:
already_subscribed = UserSubscription.objects.filter(
user=self.user, feed=duplicate_feed.feed)
if not already_subscribed:
self.feed = duplicate_feed.feed
super(UserSubscription, self).save(*args, **kwargs)
break
else:
self.delete()
@classmethod
def sync_all_redis(cls, user_id, skip_feed=False):
us = cls.objects.filter(user=user_id)
for sub in us:
print " ---> Syncing usersub: %s" % sub
sub.sync_redis(skip_feed=skip_feed)
def sync_redis(self, skip_feed=False):
r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
h = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(
days=settings.DAYS_OF_UNREAD + 1)
userstories = MUserStory.objects.filter(feed_id=self.feed_id,
user_id=self.user_id,
read_date__gte=UNREAD_CUTOFF)
total = userstories.count()
logging.debug(" ---> ~SN~FMSyncing ~SB%s~SN stories (%s)" %
(total, self))
pipeline = r.pipeline()
hashpipe = h.pipeline()
for userstory in userstories:
userstory.sync_redis(pipeline=pipeline, hashpipe=hashpipe)
pipeline.execute()
hashpipe.execute()
def get_stories(self,
offset=0,
limit=6,
order='newest',
read_filter='all',
withscores=False):
r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
ignore_user_stories = False
stories_key = 'F:%s' % (self.feed_id)
read_stories_key = 'RS:%s:%s' % (self.user_id, self.feed_id)
unread_stories_key = 'U:%s:%s' % (self.user_id, self.feed_id)
unread_ranked_stories_key = 'zU:%s:%s' % (self.user_id, self.feed_id)
if offset and not withscores and r.exists(unread_ranked_stories_key):
pass
else:
r.delete(unread_ranked_stories_key)
if not r.exists(stories_key):
print " ---> No stories on feed: %s" % self
return []
elif read_filter != 'unread' or not r.exists(read_stories_key):
ignore_user_stories = True
unread_stories_key = stories_key
else:
r.sdiffstore(unread_stories_key, stories_key, read_stories_key)
sorted_stories_key = 'zF:%s' % (self.feed_id)
unread_ranked_stories_key = 'zU:%s:%s' % (self.user_id,
self.feed_id)
r.zinterstore(unread_ranked_stories_key,
[sorted_stories_key, unread_stories_key])
current_time = int(time.time() + 60 * 60 * 24)
if order == 'oldest':
byscorefunc = r.zrangebyscore
if read_filter == 'unread':
min_score = int(time.mktime(
self.mark_read_date.timetuple())) + 1
else:
now = datetime.datetime.now()
two_weeks_ago = now - datetime.timedelta(
days=settings.DAYS_OF_UNREAD)
min_score = int(time.mktime(two_weeks_ago.timetuple())) - 1000
max_score = current_time
else:
byscorefunc = r.zrevrangebyscore
min_score = current_time
if read_filter == 'unread':
# +1 for the intersection b/w zF and F, which carries an implicit score of 1.
max_score = int(time.mktime(
self.mark_read_date.timetuple())) + 1
else:
max_score = 0
if settings.DEBUG:
debug_stories = r.zrevrange(unread_ranked_stories_key,
0,
-1,
withscores=True)
print " ---> Unread all stories (%s - %s) %s stories: %s" % (
min_score, max_score, len(debug_stories), debug_stories)
story_ids = byscorefunc(unread_ranked_stories_key,
min_score,
max_score,
start=offset,
num=500,
withscores=withscores)[:limit]
r.expire(unread_ranked_stories_key, 24 * 60 * 60)
if not ignore_user_stories:
r.delete(unread_stories_key)
# XXX TODO: Remove below line after combing redis for these None's.
story_ids = [s for s in story_ids if s and s != 'None'] # ugh, hack
if withscores:
return story_ids
elif story_ids:
story_date_order = "%sstory_date" % (''
if order == 'oldest' else '-')
mstories = MStory.objects(
id__in=story_ids).order_by(story_date_order)
stories = Feed.format_stories(mstories)
return stories
else:
return []
@classmethod
def feed_stories(cls,
user_id,
feed_ids,
offset=0,
limit=6,
order='newest',
read_filter='all'):
r = redis.Redis(connection_pool=settings.REDIS_STORY_POOL)
if order == 'oldest':
range_func = r.zrange
else:
range_func = r.zrevrange
if not isinstance(feed_ids, list):
feed_ids = [feed_ids]
unread_ranked_stories_keys = 'zU:%s:feeds' % (user_id)
if offset and r.exists(unread_ranked_stories_keys):
story_guids = range_func(unread_ranked_stories_keys, offset, limit)
return story_guids
else:
r.delete(unread_ranked_stories_keys)
for feed_id in feed_ids:
try:
us = cls.objects.get(user=user_id, feed=feed_id)
except cls.DoesNotExist:
continue
story_guids = us.get_stories(offset=0,
limit=200,
order=order,
read_filter=read_filter,
withscores=True)
if story_guids:
r.zadd(unread_ranked_stories_keys, **dict(story_guids))
story_guids = range_func(unread_ranked_stories_keys, offset, limit)
r.expire(unread_ranked_stories_keys, 24 * 60 * 60)
return story_guids
@classmethod
def add_subscription(cls,
user,
feed_address,
folder=None,
bookmarklet=False,
auto_active=True,
skip_fetch=False):
feed = None
us = None
logging.user(
user, "~FRAdding URL: ~SB%s (in %s) %s" %
(feed_address, folder, "~FCAUTO-ADD" if not auto_active else ""))
feed = Feed.get_feed_from_url(feed_address)
if not feed:
code = -1
if bookmarklet:
message = "This site does not have an RSS feed. Nothing is linked to from this page."
else:
message = "This address does not point to an RSS feed or a website with an RSS feed."
else:
us, subscription_created = cls.objects.get_or_create(
feed=feed,
user=user,
defaults={
'needs_unread_recalc': True,
'active': auto_active,
})
code = 1
message = ""
if us:
user_sub_folders_object, created = UserSubscriptionFolders.objects.get_or_create(
user=user, defaults={'folders': '[]'})
if created:
user_sub_folders = []
else:
user_sub_folders = json.decode(user_sub_folders_object.folders)
user_sub_folders = add_object_to_folder(feed.pk, folder,
user_sub_folders)
user_sub_folders_object.folders = json.encode(user_sub_folders)
user_sub_folders_object.save()
if auto_active or user.profile.is_premium:
us.active = True
us.save()
if not skip_fetch and feed.last_update < datetime.datetime.utcnow(
) - datetime.timedelta(days=1):
feed = feed.update()
from apps.social.models import MActivity
MActivity.new_feed_subscription(user_id=user.pk,
feed_id=feed.pk,
feed_title=feed.title)
feed.setup_feed_for_premium_subscribers()
return code, message, us
@classmethod
def feeds_with_updated_counts(cls,
user,
feed_ids=None,
check_fetch_status=False):
feeds = {}
# Get subscriptions for user
user_subs = cls.objects.select_related('feed').filter(user=user,
active=True)
feed_ids = [f for f in feed_ids if f and not f.startswith('river')]
if feed_ids:
user_subs = user_subs.filter(feed__in=feed_ids)
UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(
days=settings.DAYS_OF_UNREAD)
for i, sub in enumerate(user_subs):
# Count unreads if subscription is stale.
if (sub.needs_unread_recalc
or sub.unread_count_updated < UNREAD_CUTOFF
or sub.oldest_unread_story_date < UNREAD_CUTOFF):
sub = sub.calculate_feed_scores(silent=True)
if not sub:
continue # TODO: Figure out the correct sub and give it a new feed_id
feed_id = sub.feed_id
feeds[feed_id] = {
'ps': sub.unread_count_positive,
'nt': sub.unread_count_neutral,
'ng': sub.unread_count_negative,
'id': feed_id,
}
if not sub.feed.fetched_once or check_fetch_status:
feeds[feed_id]['fetched_once'] = sub.feed.fetched_once
feeds[feed_id][
'not_yet_fetched'] = not sub.feed.fetched_once # Legacy. Dammit.
if sub.feed.favicon_fetching:
feeds[feed_id]['favicon_fetching'] = True
if sub.feed.has_feed_exception or sub.feed.has_page_exception:
feeds[feed_id]['has_exception'] = True
feeds[feed_id][
'exception_type'] = 'feed' if sub.feed.has_feed_exception else 'page'
feeds[feed_id]['feed_address'] = sub.feed.feed_address
feeds[feed_id]['exception_code'] = sub.feed.exception_code
return feeds
def mark_feed_read(self):
if (self.unread_count_negative == 0 and self.unread_count_neutral == 0
and self.unread_count_positive == 0
and not self.needs_unread_recalc):
return
now = datetime.datetime.utcnow()
# Use the latest story to get last read time.
latest_story = MStory.objects(story_feed_id=self.feed.pk).order_by(
'-story_date').only('story_date').limit(1)
if latest_story and len(latest_story) >= 1:
latest_story_date = latest_story[0]['story_date']\
+ datetime.timedelta(seconds=1)
else:
latest_story_date = now
self.last_read_date = latest_story_date
self.mark_read_date = latest_story_date
self.unread_count_negative = 0
self.unread_count_positive = 0
self.unread_count_neutral = 0
self.unread_count_updated = now
self.oldest_unread_story_date = now
self.needs_unread_recalc = False
# No longer removing old user read stories, since they're needed for social,
# and they get cleaned up automatically when new stories come in.
# MUserStory.delete_old_stories(self.user_id, self.feed_id)
self.save()
def mark_story_ids_as_read(self, story_ids, request=None):
data = dict(code=0, payload=story_ids)
if not request:
request = self.user
if not self.needs_unread_recalc:
self.needs_unread_recalc = True
self.save()
if len(story_ids) > 1:
logging.user(
request,
"~FYRead %s stories in feed: %s" % (len(story_ids), self.feed))
else:
logging.user(request, "~FYRead story in feed: %s" % (self.feed))
for story_id in set(story_ids):
story, _ = MStory.find_story(story_feed_id=self.feed_id,
story_id=story_id)
if not story: continue
now = datetime.datetime.utcnow()
date = now if now > story.story_date else story.story_date # For handling future stories
m, _ = MUserStory.objects.get_or_create(story_id=story_id,
user_id=self.user_id,
feed_id=self.feed_id,
defaults={
'read_date':
date,
'story':
story,
'story_date':
story.story_date,
})
return data
def calculate_feed_scores(self, silent=False, stories=None):
# now = datetime.datetime.strptime("2009-07-06 22:30:03", "%Y-%m-%d %H:%M:%S")
now = datetime.datetime.now()
UNREAD_CUTOFF = now - datetime.timedelta(days=settings.DAYS_OF_UNREAD)
if self.user.profile.last_seen_on < UNREAD_CUTOFF:
# if not silent:
# logging.info(' ---> [%s] SKIPPING Computing scores: %s (1 week+)' % (self.user, self.feed))
return
if not self.feed.fetched_once:
if not silent:
logging.info(' ---> [%s] NOT Computing scores: %s' %
(self.user, self.feed))
self.needs_unread_recalc = False
self.save()
return
feed_scores = dict(negative=0, neutral=0, positive=0)
# Two weeks in age. If mark_read_date is older, mark old stories as read.
date_delta = UNREAD_CUTOFF
if date_delta < self.mark_read_date:
date_delta = self.mark_read_date
else:
self.mark_read_date = date_delta
if not stories:
stories_db = MStory.objects(story_feed_id=self.feed_id,
story_date__gte=date_delta)
stories = Feed.format_stories(stories_db, self.feed_id)
story_ids = [s['id'] for s in stories]
read_stories = MUserStory.objects(user_id=self.user_id,
feed_id=self.feed_id,
story_id__in=story_ids)
read_stories_ids = [us.story_id for us in read_stories]
oldest_unread_story_date = now
unread_stories = []
for story in stories:
if story['story_date'] < date_delta:
continue
if story['id'] not in read_stories_ids:
unread_stories.append(story)
if story['story_date'] < oldest_unread_story_date:
oldest_unread_story_date = story['story_date']
# if not silent:
# logging.info(' ---> [%s] Format stories: %s' % (self.user, datetime.datetime.now() - now))
classifier_feeds = list(
MClassifierFeed.objects(user_id=self.user_id,
feed_id=self.feed_id,
social_user_id=0))
classifier_authors = list(
MClassifierAuthor.objects(user_id=self.user_id,
feed_id=self.feed_id))
classifier_titles = list(
MClassifierTitle.objects(user_id=self.user_id,
feed_id=self.feed_id))
classifier_tags = list(
MClassifierTag.objects(user_id=self.user_id, feed_id=self.feed_id))
# if not silent:
# logging.info(' ---> [%s] Classifiers: %s (%s)' % (self.user, datetime.datetime.now() - now, classifier_feeds.count() + classifier_authors.count() + classifier_tags.count() + classifier_titles.count()))
scores = {
'feed': apply_classifier_feeds(classifier_feeds, self.feed),
}
for story in unread_stories:
scores.update({
'author':
apply_classifier_authors(classifier_authors, story),
'tags':
apply_classifier_tags(classifier_tags, story),
'title':
apply_classifier_titles(classifier_titles, story),
})
max_score = max(scores['author'], scores['tags'], scores['title'])
min_score = min(scores['author'], scores['tags'], scores['title'])
if max_score > 0:
feed_scores['positive'] += 1
elif min_score < 0:
feed_scores['negative'] += 1
else:
if scores['feed'] > 0:
feed_scores['positive'] += 1
elif scores['feed'] < 0:
feed_scores['negative'] += 1
else:
feed_scores['neutral'] += 1
# if not silent:
# logging.info(' ---> [%s] End classifiers: %s' % (self.user, datetime.datetime.now() - now))
self.unread_count_positive = feed_scores['positive']
self.unread_count_neutral = feed_scores['neutral']
self.unread_count_negative = feed_scores['negative']
self.unread_count_updated = datetime.datetime.now()
self.oldest_unread_story_date = oldest_unread_story_date
self.needs_unread_recalc = False
self.save()
if (self.unread_count_positive == 0 and self.unread_count_neutral == 0
and self.unread_count_negative == 0):
self.mark_feed_read()
if not silent:
logging.user(
self.user,
'~FC~SNComputing scores: %s (~SB%s~SN/~SB%s~SN/~SB%s~SN)' %
(self.feed, feed_scores['negative'], feed_scores['neutral'],
feed_scores['positive']))
return self
def switch_feed(self, new_feed, old_feed):
# Rewrite feed in subscription folders
try:
user_sub_folders = UserSubscriptionFolders.objects.get(
user=self.user)
except Exception, e:
logging.info(" *** ---> UserSubscriptionFolders error: %s" % e)
return
# Switch to original feed for the user subscription
logging.info(" ===> %s " % self.user)
self.feed = new_feed
self.needs_unread_recalc = True
try:
self.save()
user_sub_folders.rewrite_feed(new_feed, old_feed)
except (IntegrityError, OperationError):
logging.info(" !!!!> %s already subscribed" % self.user)
self.delete()
return
# Switch read stories
user_stories = MUserStory.objects(user_id=self.user_id,
feed_id=old_feed.pk)
if user_stories.count() > 0:
logging.info(" ---> %s read stories" % user_stories.count())
for user_story in user_stories:
user_story.feed_id = new_feed.pk
duplicate_story = user_story.story
if duplicate_story:
story_guid = duplicate_story.story_guid if hasattr(
duplicate_story, 'story_guid') else duplicate_story.id
original_story, _ = MStory.find_story(
story_feed_id=new_feed.pk,
story_id=story_guid,
original_only=True)
if original_story:
user_story.story = original_story
try:
user_story.save()
except OperationError:
# User read the story in the original feed, too. Ugh, just ignore it.
pass
else:
user_story.delete()
else:
user_story.delete()
def switch_feed_for_classifier(model):
duplicates = model.objects(feed_id=old_feed.pk,
user_id=self.user_id)
if duplicates.count():
logging.info(" ---> Switching %s %s" %
(duplicates.count(), model))
for duplicate in duplicates:
duplicate.feed_id = new_feed.pk
if duplicate.social_user_id is None:
duplicate.social_user_id = 0
try:
duplicate.save()
pass
except (IntegrityError, OperationError):
logging.info(" !!!!> %s already exists" % duplicate)
duplicate.delete()
switch_feed_for_classifier(MClassifierTitle)
switch_feed_for_classifier(MClassifierAuthor)
switch_feed_for_classifier(MClassifierFeed)
switch_feed_for_classifier(MClassifierTag)
