def receive_newsletter(self, params):
user = self.user_from_email(params['recipient'])
if not user:
return
sender_name, sender_username, sender_domain = self.split_sender(params['from'])
feed_address = self.feed_address(user, "%s@%s" % (sender_username, sender_domain))
usf = UserSubscriptionFolders.objects.get(user=user)
usf.add_folder('', 'Newsletters')
try:
feed = Feed.objects.get(feed_address=feed_address)
except Feed.DoesNotExist:
feed = Feed.objects.create(feed_address=feed_address,
feed_link='http://' + sender_domain,
feed_title=sender_name,
fetched_once=True,
known_good=True)
feed.update()
logging.user(user, "~FCCreating newsletter feed: ~SB%s" % (feed))
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(user.username, 'reload:%s' % feed.pk)
try:
usersub = UserSubscription.objects.get(user=user, feed=feed)
except UserSubscription.DoesNotExist:
_, _, usersub = UserSubscription.add_subscription(
user=user,
feed_address=feed_address,
folder='Newsletters'
)
story_hash = MStory.ensure_story_hash(params['signature'], feed.pk)
story_params = {
"story_feed_id": feed.pk,
"story_date": datetime.datetime.fromtimestamp(int(params['timestamp'])),
"story_title": params['subject'],
"story_content": self.get_content(params),
"story_author_name": escape(params['from']),
"story_permalink": reverse('newsletter-story',
kwargs={'story_hash': story_hash}),
"story_guid": params['signature'],
}
try:
story = MStory.objects.get(story_hash=story_hash)
except MStory.DoesNotExist:
story = MStory(**story_params)
story.save()
usersub.needs_unread_recalc = True
usersub.save()
self.publish_to_subscribers(feed)
MFetchHistory.add(feed_id=feed.pk, fetch_type='push')
logging.user(user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" % (story.story_title, feed))
return story
def push_callback(request, push_id):
if request.method == 'GET':
mode = request.GET['hub.mode']
topic = request.GET['hub.topic']
challenge = request.GET['hub.challenge']
lease_seconds = request.GET.get('hub.lease_seconds')
verify_token = request.GET.get('hub.verify_token', '')
if mode == 'subscribe':
if not verify_token.startswith('subscribe'):
raise Http404
subscription = get_object_or_404(PushSubscription,
pk=push_id,
topic=topic,
verify_token=verify_token)
subscription.verified = True
subscription.set_expiration(int(lease_seconds))
subscription.save()
subscription.feed.setup_push()
logging.debug(' ---> [%-30s] [%s] ~BBVerified PuSH' % (unicode(subscription.feed)[:30], subscription.feed_id))
verified.send(sender=subscription)
return HttpResponse(challenge, content_type='text/plain')
elif request.method == 'POST':
subscription = get_object_or_404(PushSubscription, pk=push_id)
fetch_history = MFetchHistory.feed(subscription.feed_id)
latest_push_date_delta = None
if fetch_history and fetch_history.get('push_history'):
latest_push = fetch_history['push_history'][0]['push_date']
latest_push_date = datetime.datetime.strptime(latest_push, '%Y-%m-%d %H:%M:%S')
latest_push_date_delta = datetime.datetime.now() - latest_push_date
if latest_push_date > datetime.datetime.now() - datetime.timedelta(minutes=1):
logging.debug(' ---> [%-30s] ~SN~FBSkipping feed fetch, pushed %s seconds ago' % (unicode(subscription.feed)[:30], latest_push_date_delta.seconds))
return HttpResponse('Slow down, you just pushed %s seconds ago...' % latest_push_date_delta.seconds, status=429)
# XXX TODO: Optimize this by removing feedparser. It just needs to find out
# the hub_url or topic has changed. ElementTree could do it.
if random.random() < 0.1:
parsed = feedparser.parse(request.raw_post_data)
subscription.check_urls_against_pushed_data(parsed)
# Don't give fat ping, just fetch.
# subscription.feed.queue_pushed_feed_xml(request.raw_post_data)
if subscription.feed.active_premium_subscribers >= 1:
subscription.feed.queue_pushed_feed_xml("Fetch me", latest_push_date_delta=latest_push_date_delta)
MFetchHistory.add(feed_id=subscription.feed_id,
fetch_type='push')
else:
logging.debug(' ---> [%-30s] ~FBSkipping feed fetch, no actives: %s' % (unicode(subscription.feed)[:30], subscription.feed))
return HttpResponse('OK')
return Http404
def push_callback(request, push_id):
if request.method == 'GET':
mode = request.GET['hub.mode']
topic = request.GET['hub.topic']
challenge = request.GET['hub.challenge']
lease_seconds = request.GET.get('hub.lease_seconds')
verify_token = request.GET.get('hub.verify_token', '')
if mode == 'subscribe':
if not verify_token.startswith('subscribe'):
raise Http404
subscription = get_object_or_404(PushSubscription,
pk=push_id,
topic=topic,
verify_token=verify_token)
subscription.verified = True
subscription.set_expiration(int(lease_seconds))
subscription.save()
subscription.feed.setup_push()
logging.debug(
' ---> [%-30s] [%s] ~BBVerified PuSH' %
(unicode(subscription.feed)[:30], subscription.feed_id))
verified.send(sender=subscription)
return HttpResponse(challenge, content_type='text/plain')
elif request.method == 'POST':
subscription = get_object_or_404(PushSubscription, pk=push_id)
# XXX TODO: Optimize this by removing feedparser. It just needs to find out
# the hub_url or topic has changed. ElementTree could do it.
if random.random() < 0.1:
parsed = feedparser.parse(request.raw_post_data)
subscription.check_urls_against_pushed_data(parsed)
# Don't give fat ping, just fetch.
# subscription.feed.queue_pushed_feed_xml(request.raw_post_data)
if subscription.feed.active_premium_subscribers >= 1:
subscription.feed.queue_pushed_feed_xml("Fetch me")
MFetchHistory.add(feed_id=subscription.feed_id, fetch_type='push')
else:
logging.debug(
' ---> [%-30s] ~FBSkipping feed fetch, no actives: %s' %
(unicode(subscription.feed)[:30], subscription.feed))
return HttpResponse('')
return Http404
def push_callback(request, push_id):
if request.method == 'GET':
mode = request.GET['hub.mode']
topic = request.GET['hub.topic']
challenge = request.GET['hub.challenge']
lease_seconds = request.GET.get('hub.lease_seconds')
verify_token = request.GET.get('hub.verify_token', '')
if mode == 'subscribe':
if not verify_token.startswith('subscribe'):
raise Http404
subscription = get_object_or_404(PushSubscription,
pk=push_id,
topic=topic,
verify_token=verify_token)
subscription.verified = True
subscription.set_expiration(int(lease_seconds))
subscription.save()
subscription.feed.setup_push()
logging.debug(' ---> [%-30s] [%s] ~BBVerified PuSH' % (unicode(subscription.feed)[:30], subscription.feed_id))
verified.send(sender=subscription)
return HttpResponse(challenge, content_type='text/plain')
elif request.method == 'POST':
subscription = get_object_or_404(PushSubscription, pk=push_id)
# XXX TODO: Optimize this by removing feedparser. It just needs to find out
# the hub_url or topic has changed. ElementTree could do it.
if random.random() < 0.1:
parsed = feedparser.parse(request.raw_post_data)
subscription.check_urls_against_pushed_data(parsed)
# Don't give fat ping, just fetch.
# subscription.feed.queue_pushed_feed_xml(request.raw_post_data)
if subscription.feed.active_premium_subscribers >= 1:
subscription.feed.queue_pushed_feed_xml("Fetch me")
MFetchHistory.add(feed_id=subscription.feed_id,
fetch_type='push')
else:
logging.debug(' ---> [%-30s] ~FBSkipping feed fetch, no actives: %s' % (unicode(subscription.feed)[:30], subscription.feed))
return HttpResponse('')
return Http404
def receive_newsletter(self, params):
user = self._user_from_email(params['recipient'])
if not user:
return
sender_name, sender_username, sender_domain = self._split_sender(
params['from'])
feed_address = self._feed_address(
user, "%s@%s" % (sender_username, sender_domain))
usf = UserSubscriptionFolders.objects.get(user=user)
usf.add_folder('', 'Newsletters')
try:
feed = Feed.objects.get(feed_address=feed_address)
except Feed.DoesNotExist:
feed = Feed.objects.create(feed_address=feed_address,
feed_link='http://' + sender_domain,
feed_title=sender_name,
fetched_once=True,
known_good=True)
feed.update()
logging.user(user, "~FCCreating newsletter feed: ~SB%s" % (feed))
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(user.username, 'reload:%s' % feed.pk)
self._check_if_first_newsletter(user)
if feed.feed_title != sender_name:
feed.feed_title = sender_name
feed.save()
try:
usersub = UserSubscription.objects.get(user=user, feed=feed)
except UserSubscription.DoesNotExist:
_, _, usersub = UserSubscription.add_subscription(
user=user, feed_address=feed_address, folder='Newsletters')
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(user.username, 'reload:feeds')
story_hash = MStory.ensure_story_hash(params['signature'], feed.pk)
story_content = self._get_content(params)
plain_story_content = self._get_content(params, force_plain=True)
if len(plain_story_content) > len(story_content):
story_content = plain_story_content
story_content = self._clean_content(story_content)
story_params = {
"story_feed_id":
feed.pk,
"story_date":
datetime.datetime.fromtimestamp(int(params['timestamp'])),
"story_title":
params['subject'],
"story_content":
story_content,
"story_author_name":
params['from'],
"story_permalink":
"https://%s%s" %
(Site.objects.get_current().domain,
reverse('newsletter-story', kwargs={'story_hash': story_hash})),
"story_guid":
params['signature'],
}
print story_params
try:
story = MStory.objects.get(story_hash=story_hash)
except MStory.DoesNotExist:
story = MStory(**story_params)
story.save()
usersub.needs_unread_recalc = True
usersub.save()
self._publish_to_subscribers(feed)
MFetchHistory.add(feed_id=feed.pk, fetch_type='push')
logging.user(
user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" %
(story.story_title, feed))
return story
def receive_newsletter(self, params):
user = self._user_from_email(params['recipient'])
if not user:
return
sender_name, sender_username, sender_domain = self._split_sender(
params['from'])
feed_address = self._feed_address(
user, "%s@%s" % (sender_username, sender_domain))
try:
usf = UserSubscriptionFolders.objects.get(user=user)
except UserSubscriptionFolders.DoesNotExist:
logging.user(user,
"~FRUser does not have a USF, ignoring newsletter.")
return
usf.add_folder('', 'Newsletters')
# First look for the email address
try:
feed = Feed.objects.get(feed_address=feed_address)
except Feed.MultipleObjectsReturned:
feeds = Feed.objects.filter(feed_address=feed_address)[:1]
if feeds.count():
feed = feeds[0]
except Feed.DoesNotExist:
feed = None
# If not found, check among titles user has subscribed to
if not feed:
newsletter_subs = UserSubscription.objects.filter(
user=user,
feed__feed_address__contains="newsletter:").only('feed')
newsletter_feed_ids = [us.feed.pk for us in newsletter_subs]
feeds = Feed.objects.filter(feed_title__iexact=sender_name,
pk__in=newsletter_feed_ids)
if feeds.count():
feed = feeds[0]
# Create a new feed if it doesn't exist by sender name or email
if not feed:
feed = Feed.objects.create(feed_address=feed_address,
feed_link='http://' + sender_domain,
feed_title=sender_name,
fetched_once=True,
known_good=True)
feed.update()
logging.user(user, "~FCCreating newsletter feed: ~SB%s" % (feed))
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(user.username, 'reload:%s' % feed.pk)
self._check_if_first_newsletter(user)
feed.last_update = datetime.datetime.now()
feed.last_story_date = datetime.datetime.now()
feed.save()
if feed.feed_title != sender_name:
feed.feed_title = sender_name
feed.save()
try:
usersub = UserSubscription.objects.get(user=user, feed=feed)
except UserSubscription.DoesNotExist:
_, _, usersub = UserSubscription.add_subscription(
user=user, feed_address=feed_address, folder='Newsletters')
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(user.username, 'reload:feeds')
story_hash = MStory.ensure_story_hash(params['signature'], feed.pk)
story_content = self._get_content(params)
plain_story_content = self._get_content(params, force_plain=True)
if len(plain_story_content) > len(story_content):
story_content = plain_story_content
story_content = self._clean_content(story_content)
story_params = {
"story_feed_id":
feed.pk,
"story_date":
datetime.datetime.fromtimestamp(int(params['timestamp'])),
"story_title":
params['subject'],
"story_content":
story_content,
"story_author_name":
params['from'],
"story_permalink":
"https://%s%s" %
(Site.objects.get_current().domain,
reverse('newsletter-story', kwargs={'story_hash': story_hash})),
"story_guid":
params['signature'],
}
try:
story = MStory.objects.get(story_hash=story_hash)
except MStory.DoesNotExist:
story = MStory(**story_params)
story.save()
usersub.needs_unread_recalc = True
usersub.save()
self._publish_to_subscribers(feed, story.story_hash)
MFetchHistory.add(feed_id=feed.pk, fetch_type='push')
logging.user(
user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" %
(story.story_title, feed))
return story
def receive_newsletter(self, params):
user = self._user_from_email(params['recipient'])
if not user:
return
sender_name, sender_username, sender_domain = self._split_sender(params['from'])
feed_address = self._feed_address(user, "%s@%s" % (sender_username, sender_domain))
try:
usf = UserSubscriptionFolders.objects.get(user=user)
except UserSubscriptionFolders.DoesNotExist:
logging.user(user, "~FRUser does not have a USF, ignoring newsletter.")
return
usf.add_folder('', 'Newsletters')
try:
feed = Feed.objects.get(feed_address=feed_address)
except Feed.DoesNotExist:
feed = Feed.objects.create(feed_address=feed_address,
feed_link='http://' + sender_domain,
feed_title=sender_name,
fetched_once=True,
known_good=True)
feed.update()
logging.user(user, "~FCCreating newsletter feed: ~SB%s" % (feed))
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(user.username, 'reload:%s' % feed.pk)
self._check_if_first_newsletter(user)
feed.last_update = datetime.datetime.now()
feed.last_story_date = datetime.datetime.now()
feed.save()
if feed.feed_title != sender_name:
feed.feed_title = sender_name
feed.save()
try:
usersub = UserSubscription.objects.get(user=user, feed=feed)
except UserSubscription.DoesNotExist:
_, _, usersub = UserSubscription.add_subscription(
user=user,
feed_address=feed_address,
folder='Newsletters'
)
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(user.username, 'reload:feeds')
story_hash = MStory.ensure_story_hash(params['signature'], feed.pk)
story_content = self._get_content(params)
plain_story_content = self._get_content(params, force_plain=True)
if len(plain_story_content) > len(story_content):
story_content = plain_story_content
story_content = self._clean_content(story_content)
story_params = {
"story_feed_id": feed.pk,
"story_date": datetime.datetime.fromtimestamp(int(params['timestamp'])),
"story_title": params['subject'],
"story_content": story_content,
"story_author_name": params['from'],
"story_permalink": "https://%s%s" % (
Site.objects.get_current().domain,
reverse('newsletter-story',
kwargs={'story_hash': story_hash})),
"story_guid": params['signature'],
}
try:
story = MStory.objects.get(story_hash=story_hash)
except MStory.DoesNotExist:
story = MStory(**story_params)
story.save()
usersub.needs_unread_recalc = True
usersub.save()
self._publish_to_subscribers(feed, story.story_hash)
MFetchHistory.add(feed_id=feed.pk, fetch_type='push')
logging.user(user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" % (story.story_title, feed))
return story
def receive_newsletter(self, params):
user = self.user_from_email(params["recipient"])
if not user:
return
sender_name, sender_username, sender_domain = self.split_sender(params["from"])
feed_address = self.feed_address(user, "%s@%s" % (sender_username, sender_domain))
usf = UserSubscriptionFolders.objects.get(user=user)
usf.add_folder("", "Newsletters")
try:
feed = Feed.objects.get(feed_address=feed_address)
except Feed.DoesNotExist:
feed = Feed.objects.create(
feed_address=feed_address,
feed_link="http://" + sender_domain,
feed_title=sender_name,
fetched_once=True,
known_good=True,
)
feed.update()
logging.user(user, "~FCCreating newsletter feed: ~SB%s" % (feed))
r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL)
r.publish(user.username, "reload:%s" % feed.pk)
if feed.feed_title != sender_name:
feed.feed_title = sender_name
feed.save()
try:
usersub = UserSubscription.objects.get(user=user, feed=feed)
except UserSubscription.DoesNotExist:
_, _, usersub = UserSubscription.add_subscription(
user=user, feed_address=feed_address, folder="Newsletters"
)
story_hash = MStory.ensure_story_hash(params["signature"], feed.pk)
story_content = self.get_content(params)
story_content = self.clean_content(story_content)
story_params = {
"story_feed_id": feed.pk,
"story_date": datetime.datetime.fromtimestamp(int(params["timestamp"])),
"story_title": params["subject"],
"story_content": story_content,
"story_author_name": params["from"],
"story_permalink": "https://%s%s"
% (Site.objects.get_current().domain, reverse("newsletter-story", kwargs={"story_hash": story_hash})),
"story_guid": params["signature"],
}
try:
story = MStory.objects.get(story_hash=story_hash)
except MStory.DoesNotExist:
story = MStory(**story_params)
story.save()
usersub.needs_unread_recalc = True
usersub.save()
self.publish_to_subscribers(feed)
MFetchHistory.add(feed_id=feed.pk, fetch_type="push")
logging.user(user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" % (story.story_title, feed))
return story
