def mark_read(cls, user_id, story_feed_id, story_hash, r=None, r2=None):
if not r:
r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
if not r2:
r2 = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL2)
story_hash = MStory.ensure_story_hash(story_hash, story_feed_id=story_feed_id)
if not story_hash: return
now = int(time.time())
all_read_stories_key = 'RS:%s' % (user_id)
r.sadd(all_read_stories_key, story_hash)
r2.sadd(all_read_stories_key, story_hash)
r2.zadd('z' + all_read_stories_key, story_hash, now)
r.expire(all_read_stories_key, settings.DAYS_OF_UNREAD*24*60*60)
r2.expire(all_read_stories_key, settings.DAYS_OF_UNREAD*24*60*60)
r2.expire('z' + all_read_stories_key, settings.DAYS_OF_UNREAD*24*60*60)
read_story_key = 'RS:%s:%s' % (user_id, story_feed_id)
r.sadd(read_story_key, story_hash)
r2.sadd(read_story_key, story_hash)
r2.zadd('z' + read_story_key, story_hash, now)
r.expire(read_story_key, settings.DAYS_OF_UNREAD*24*60*60)
r2.expire(read_story_key, settings.DAYS_OF_UNREAD*24*60*60)
r2.expire('z' + read_story_key, settings.DAYS_OF_UNREAD*24*60*60)
def mark_read(cls, user_id, story_feed_id, story_hash, social_user_ids=None, r=None):
if not r:
r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
# if not r2:
# r2 = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL2)
story_hash = MStory.ensure_story_hash(story_hash, story_feed_id=story_feed_id)
if not story_hash: return
def redis_commands(key):
r.sadd(key, story_hash)
# r2.sadd(key, story_hash)
r.expire(key, settings.DAYS_OF_STORY_HASHES*24*60*60)
# r2.expire(key, settings.DAYS_OF_STORY_HASHES*24*60*60)
all_read_stories_key = 'RS:%s' % (user_id)
redis_commands(all_read_stories_key)
read_story_key = 'RS:%s:%s' % (user_id, story_feed_id)
redis_commands(read_story_key)
if social_user_ids:
for social_user_id in social_user_ids:
social_read_story_key = 'RS:%s:B:%s' % (user_id, social_user_id)
redis_commands(social_read_story_key)
def receive_newsletter(self, params):
user = self.user_from_email(params['recipient'])
if not user:
return
sender_name, sender_username, sender_domain = self.split_sender(params['from'])
feed_address = self.feed_address(user, "%s@%s" % (sender_username, sender_domain))
usf = UserSubscriptionFolders.objects.get(user=user)
usf.add_folder('', 'Newsletters')
try:
feed = Feed.objects.get(feed_address=feed_address)
except Feed.DoesNotExist:
feed = Feed.objects.create(feed_address=feed_address,
feed_link='http://' + sender_domain,
feed_title=sender_name,
fetched_once=True,
known_good=True)
feed.update()
logging.user(user, "~FCCreating newsletter feed: ~SB%s" % (feed))
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(user.username, 'reload:%s' % feed.pk)
try:
usersub = UserSubscription.objects.get(user=user, feed=feed)
except UserSubscription.DoesNotExist:
_, _, usersub = UserSubscription.add_subscription(
user=user,
feed_address=feed_address,
folder='Newsletters'
)
story_hash = MStory.ensure_story_hash(params['signature'], feed.pk)
story_params = {
"story_feed_id": feed.pk,
"story_date": datetime.datetime.fromtimestamp(int(params['timestamp'])),
"story_title": params['subject'],
"story_content": self.get_content(params),
"story_author_name": escape(params['from']),
"story_permalink": reverse('newsletter-story',
kwargs={'story_hash': story_hash}),
"story_guid": params['signature'],
}
try:
story = MStory.objects.get(story_hash=story_hash)
except MStory.DoesNotExist:
story = MStory(**story_params)
story.save()
usersub.needs_unread_recalc = True
usersub.save()
self.publish_to_subscribers(feed)
MFetchHistory.add(feed_id=feed.pk, fetch_type='push')
logging.user(user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" % (story.story_title, feed))
return story
def mark_read(cls, user_id, story_feed_id, story_hash, r=None):
if not r:
r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
story_hash = MStory.ensure_story_hash(story_hash, story_feed_id=story_feed_id)
if not story_hash: return
all_read_stories_key = 'RS:%s' % (user_id)
r.sadd(all_read_stories_key, story_hash)
r.expire(all_read_stories_key, settings.DAYS_OF_UNREAD*24*60*60)
read_story_key = 'RS:%s:%s' % (user_id, story_feed_id)
r.sadd(read_story_key, story_hash)
r.expire(read_story_key, settings.DAYS_OF_UNREAD*24*60*60)
def mark_read(cls, user_id, story_feed_id, story_hash, r=None):
if not r:
r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
story_hash = MStory.ensure_story_hash(story_hash,
story_feed_id=story_feed_id)
if not story_hash: return
all_read_stories_key = 'RS:%s' % (user_id)
r.sadd(all_read_stories_key, story_hash)
r.expire(all_read_stories_key, settings.DAYS_OF_UNREAD * 24 * 60 * 60)
read_story_key = 'RS:%s:%s' % (user_id, story_feed_id)
r.sadd(read_story_key, story_hash)
r.expire(read_story_key, settings.DAYS_OF_UNREAD * 24 * 60 * 60)
def receive_newsletter(self, params):
user = self._user_from_email(params['recipient'])
if not user:
return
sender_name, sender_username, sender_domain = self._split_sender(
params['from'])
feed_address = self._feed_address(
user, "%s@%s" % (sender_username, sender_domain))
usf = UserSubscriptionFolders.objects.get(user=user)
usf.add_folder('', 'Newsletters')
try:
feed = Feed.objects.get(feed_address=feed_address)
except Feed.DoesNotExist:
feed = Feed.objects.create(feed_address=feed_address,
feed_link='http://' + sender_domain,
feed_title=sender_name,
fetched_once=True,
known_good=True)
feed.update()
logging.user(user, "~FCCreating newsletter feed: ~SB%s" % (feed))
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(user.username, 'reload:%s' % feed.pk)
self._check_if_first_newsletter(user)
if feed.feed_title != sender_name:
feed.feed_title = sender_name
feed.save()
try:
usersub = UserSubscription.objects.get(user=user, feed=feed)
except UserSubscription.DoesNotExist:
_, _, usersub = UserSubscription.add_subscription(
user=user, feed_address=feed_address, folder='Newsletters')
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(user.username, 'reload:feeds')
story_hash = MStory.ensure_story_hash(params['signature'], feed.pk)
story_content = self._get_content(params)
plain_story_content = self._get_content(params, force_plain=True)
if len(plain_story_content) > len(story_content):
story_content = plain_story_content
story_content = self._clean_content(story_content)
story_params = {
"story_feed_id":
feed.pk,
"story_date":
datetime.datetime.fromtimestamp(int(params['timestamp'])),
"story_title":
params['subject'],
"story_content":
story_content,
"story_author_name":
params['from'],
"story_permalink":
"https://%s%s" %
(Site.objects.get_current().domain,
reverse('newsletter-story', kwargs={'story_hash': story_hash})),
"story_guid":
params['signature'],
}
print story_params
try:
story = MStory.objects.get(story_hash=story_hash)
except MStory.DoesNotExist:
story = MStory(**story_params)
story.save()
usersub.needs_unread_recalc = True
usersub.save()
self._publish_to_subscribers(feed)
MFetchHistory.add(feed_id=feed.pk, fetch_type='push')
logging.user(
user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" %
(story.story_title, feed))
return story
def receive_newsletter(self, params):
user = self._user_from_email(params['recipient'])
if not user:
return
sender_name, sender_username, sender_domain = self._split_sender(
params['from'])
feed_address = self._feed_address(
user, "%s@%s" % (sender_username, sender_domain))
try:
usf = UserSubscriptionFolders.objects.get(user=user)
except UserSubscriptionFolders.DoesNotExist:
logging.user(user,
"~FRUser does not have a USF, ignoring newsletter.")
return
usf.add_folder('', 'Newsletters')
# First look for the email address
try:
feed = Feed.objects.get(feed_address=feed_address)
except Feed.MultipleObjectsReturned:
feeds = Feed.objects.filter(feed_address=feed_address)[:1]
if feeds.count():
feed = feeds[0]
except Feed.DoesNotExist:
feed = None
# If not found, check among titles user has subscribed to
if not feed:
newsletter_subs = UserSubscription.objects.filter(
user=user,
feed__feed_address__contains="newsletter:").only('feed')
newsletter_feed_ids = [us.feed.pk for us in newsletter_subs]
feeds = Feed.objects.filter(feed_title__iexact=sender_name,
pk__in=newsletter_feed_ids)
if feeds.count():
feed = feeds[0]
# Create a new feed if it doesn't exist by sender name or email
if not feed:
feed = Feed.objects.create(feed_address=feed_address,
feed_link='http://' + sender_domain,
feed_title=sender_name,
fetched_once=True,
known_good=True)
feed.update()
logging.user(user, "~FCCreating newsletter feed: ~SB%s" % (feed))
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(user.username, 'reload:%s' % feed.pk)
self._check_if_first_newsletter(user)
feed.last_update = datetime.datetime.now()
feed.last_story_date = datetime.datetime.now()
feed.save()
if feed.feed_title != sender_name:
feed.feed_title = sender_name
feed.save()
try:
usersub = UserSubscription.objects.get(user=user, feed=feed)
except UserSubscription.DoesNotExist:
_, _, usersub = UserSubscription.add_subscription(
user=user, feed_address=feed_address, folder='Newsletters')
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(user.username, 'reload:feeds')
story_hash = MStory.ensure_story_hash(params['signature'], feed.pk)
story_content = self._get_content(params)
plain_story_content = self._get_content(params, force_plain=True)
if len(plain_story_content) > len(story_content):
story_content = plain_story_content
story_content = self._clean_content(story_content)
story_params = {
"story_feed_id":
feed.pk,
"story_date":
datetime.datetime.fromtimestamp(int(params['timestamp'])),
"story_title":
params['subject'],
"story_content":
story_content,
"story_author_name":
params['from'],
"story_permalink":
"https://%s%s" %
(Site.objects.get_current().domain,
reverse('newsletter-story', kwargs={'story_hash': story_hash})),
"story_guid":
params['signature'],
}
try:
story = MStory.objects.get(story_hash=story_hash)
except MStory.DoesNotExist:
story = MStory(**story_params)
story.save()
usersub.needs_unread_recalc = True
usersub.save()
self._publish_to_subscribers(feed, story.story_hash)
MFetchHistory.add(feed_id=feed.pk, fetch_type='push')
logging.user(
user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" %
(story.story_title, feed))
return story
def receive_newsletter(self, params):
user = self._user_from_email(params['recipient'])
if not user:
return
sender_name, sender_username, sender_domain = self._split_sender(params['from'])
feed_address = self._feed_address(user, "%s@%s" % (sender_username, sender_domain))
try:
usf = UserSubscriptionFolders.objects.get(user=user)
except UserSubscriptionFolders.DoesNotExist:
logging.user(user, "~FRUser does not have a USF, ignoring newsletter.")
return
usf.add_folder('', 'Newsletters')
try:
feed = Feed.objects.get(feed_address=feed_address)
except Feed.DoesNotExist:
feed = Feed.objects.create(feed_address=feed_address,
feed_link='http://' + sender_domain,
feed_title=sender_name,
fetched_once=True,
known_good=True)
feed.update()
logging.user(user, "~FCCreating newsletter feed: ~SB%s" % (feed))
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(user.username, 'reload:%s' % feed.pk)
self._check_if_first_newsletter(user)
feed.last_update = datetime.datetime.now()
feed.last_story_date = datetime.datetime.now()
feed.save()
if feed.feed_title != sender_name:
feed.feed_title = sender_name
feed.save()
try:
usersub = UserSubscription.objects.get(user=user, feed=feed)
except UserSubscription.DoesNotExist:
_, _, usersub = UserSubscription.add_subscription(
user=user,
feed_address=feed_address,
folder='Newsletters'
)
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(user.username, 'reload:feeds')
story_hash = MStory.ensure_story_hash(params['signature'], feed.pk)
story_content = self._get_content(params)
plain_story_content = self._get_content(params, force_plain=True)
if len(plain_story_content) > len(story_content):
story_content = plain_story_content
story_content = self._clean_content(story_content)
story_params = {
"story_feed_id": feed.pk,
"story_date": datetime.datetime.fromtimestamp(int(params['timestamp'])),
"story_title": params['subject'],
"story_content": story_content,
"story_author_name": params['from'],
"story_permalink": "https://%s%s" % (
Site.objects.get_current().domain,
reverse('newsletter-story',
kwargs={'story_hash': story_hash})),
"story_guid": params['signature'],
}
try:
story = MStory.objects.get(story_hash=story_hash)
except MStory.DoesNotExist:
story = MStory(**story_params)
story.save()
usersub.needs_unread_recalc = True
usersub.save()
self._publish_to_subscribers(feed, story.story_hash)
MFetchHistory.add(feed_id=feed.pk, fetch_type='push')
logging.user(user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" % (story.story_title, feed))
return story
def receive_newsletter(self, params):
user = self.user_from_email(params["recipient"])
if not user:
return
sender_name, sender_username, sender_domain = self.split_sender(params["from"])
feed_address = self.feed_address(user, "%s@%s" % (sender_username, sender_domain))
usf = UserSubscriptionFolders.objects.get(user=user)
usf.add_folder("", "Newsletters")
try:
feed = Feed.objects.get(feed_address=feed_address)
except Feed.DoesNotExist:
feed = Feed.objects.create(
feed_address=feed_address,
feed_link="http://" + sender_domain,
feed_title=sender_name,
fetched_once=True,
known_good=True,
)
feed.update()
logging.user(user, "~FCCreating newsletter feed: ~SB%s" % (feed))
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(user.username, "reload:%s" % feed.pk)
if feed.feed_title != sender_name:
feed.feed_title = sender_name
feed.save()
try:
usersub = UserSubscription.objects.get(user=user, feed=feed)
except UserSubscription.DoesNotExist:
_, _, usersub = UserSubscription.add_subscription(
user=user, feed_address=feed_address, folder="Newsletters"
)
story_hash = MStory.ensure_story_hash(params["signature"], feed.pk)
story_content = self.get_content(params)
story_content = self.clean_content(story_content)
story_params = {
"story_feed_id": feed.pk,
"story_date": datetime.datetime.fromtimestamp(int(params["timestamp"])),
"story_title": params["subject"],
"story_content": story_content,
"story_author_name": params["from"],
"story_permalink": "https://%s%s"
% (Site.objects.get_current().domain, reverse("newsletter-story", kwargs={"story_hash": story_hash})),
"story_guid": params["signature"],
}
try:
story = MStory.objects.get(story_hash=story_hash)
except MStory.DoesNotExist:
story = MStory(**story_params)
story.save()
usersub.needs_unread_recalc = True
usersub.save()
self.publish_to_subscribers(feed)
MFetchHistory.add(feed_id=feed.pk, fetch_type="push")
logging.user(user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" % (story.story_title, feed))
return story
