def receive_newsletter(self, params): user = self.user_from_email(params['recipient']) if not user: return sender_name, sender_username, sender_domain = self.split_sender(params['from']) feed_address = self.feed_address(user, "%s@%s" % (sender_username, sender_domain)) usf = UserSubscriptionFolders.objects.get(user=user) usf.add_folder('', 'Newsletters') try: feed = Feed.objects.get(feed_address=feed_address) except Feed.DoesNotExist: feed = Feed.objects.create(feed_address=feed_address, feed_link='http://' + sender_domain, feed_title=sender_name, fetched_once=True, known_good=True) feed.update() logging.user(user, "~FCCreating newsletter feed: ~SB%s" % (feed)) r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL) r.publish(user.username, 'reload:%s' % feed.pk) try: usersub = UserSubscription.objects.get(user=user, feed=feed) except UserSubscription.DoesNotExist: _, _, usersub = UserSubscription.add_subscription( user=user, feed_address=feed_address, folder='Newsletters' ) story_hash = MStory.ensure_story_hash(params['signature'], feed.pk) story_params = { "story_feed_id": feed.pk, "story_date": datetime.datetime.fromtimestamp(int(params['timestamp'])), "story_title": params['subject'], "story_content": self.get_content(params), "story_author_name": escape(params['from']), "story_permalink": reverse('newsletter-story', kwargs={'story_hash': story_hash}), "story_guid": params['signature'], } try: story = MStory.objects.get(story_hash=story_hash) except MStory.DoesNotExist: story = MStory(**story_params) story.save() usersub.needs_unread_recalc = True usersub.save() self.publish_to_subscribers(feed) MFetchHistory.add(feed_id=feed.pk, fetch_type='push') logging.user(user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" % (story.story_title, feed)) return story
def push_callback(request, push_id): if request.method == 'GET': mode = request.GET['hub.mode'] topic = request.GET['hub.topic'] challenge = request.GET['hub.challenge'] lease_seconds = request.GET.get('hub.lease_seconds') verify_token = request.GET.get('hub.verify_token', '') if mode == 'subscribe': if not verify_token.startswith('subscribe'): raise Http404 subscription = get_object_or_404(PushSubscription, pk=push_id, topic=topic, verify_token=verify_token) subscription.verified = True subscription.set_expiration(int(lease_seconds)) subscription.save() subscription.feed.setup_push() logging.debug(' ---> [%-30s] [%s] ~BBVerified PuSH' % (unicode(subscription.feed)[:30], subscription.feed_id)) verified.send(sender=subscription) return HttpResponse(challenge, content_type='text/plain') elif request.method == 'POST': subscription = get_object_or_404(PushSubscription, pk=push_id) fetch_history = MFetchHistory.feed(subscription.feed_id) latest_push_date_delta = None if fetch_history and fetch_history.get('push_history'): latest_push = fetch_history['push_history'][0]['push_date'] latest_push_date = datetime.datetime.strptime(latest_push, '%Y-%m-%d %H:%M:%S') latest_push_date_delta = datetime.datetime.now() - latest_push_date if latest_push_date > datetime.datetime.now() - datetime.timedelta(minutes=1): logging.debug(' ---> [%-30s] ~SN~FBSkipping feed fetch, pushed %s seconds ago' % (unicode(subscription.feed)[:30], latest_push_date_delta.seconds)) return HttpResponse('Slow down, you just pushed %s seconds ago...' % latest_push_date_delta.seconds, status=429) # XXX TODO: Optimize this by removing feedparser. It just needs to find out # the hub_url or topic has changed. ElementTree could do it. if random.random() < 0.1: parsed = feedparser.parse(request.raw_post_data) subscription.check_urls_against_pushed_data(parsed) # Don't give fat ping, just fetch. # subscription.feed.queue_pushed_feed_xml(request.raw_post_data) if subscription.feed.active_premium_subscribers >= 1: subscription.feed.queue_pushed_feed_xml("Fetch me", latest_push_date_delta=latest_push_date_delta) MFetchHistory.add(feed_id=subscription.feed_id, fetch_type='push') else: logging.debug(' ---> [%-30s] ~FBSkipping feed fetch, no actives: %s' % (unicode(subscription.feed)[:30], subscription.feed)) return HttpResponse('OK') return Http404
def push_callback(request, push_id): if request.method == 'GET': mode = request.GET['hub.mode'] topic = request.GET['hub.topic'] challenge = request.GET['hub.challenge'] lease_seconds = request.GET.get('hub.lease_seconds') verify_token = request.GET.get('hub.verify_token', '') if mode == 'subscribe': if not verify_token.startswith('subscribe'): raise Http404 subscription = get_object_or_404(PushSubscription, pk=push_id, topic=topic, verify_token=verify_token) subscription.verified = True subscription.set_expiration(int(lease_seconds)) subscription.save() subscription.feed.setup_push() logging.debug( ' ---> [%-30s] [%s] ~BBVerified PuSH' % (unicode(subscription.feed)[:30], subscription.feed_id)) verified.send(sender=subscription) return HttpResponse(challenge, content_type='text/plain') elif request.method == 'POST': subscription = get_object_or_404(PushSubscription, pk=push_id) # XXX TODO: Optimize this by removing feedparser. It just needs to find out # the hub_url or topic has changed. ElementTree could do it. if random.random() < 0.1: parsed = feedparser.parse(request.raw_post_data) subscription.check_urls_against_pushed_data(parsed) # Don't give fat ping, just fetch. # subscription.feed.queue_pushed_feed_xml(request.raw_post_data) if subscription.feed.active_premium_subscribers >= 1: subscription.feed.queue_pushed_feed_xml("Fetch me") MFetchHistory.add(feed_id=subscription.feed_id, fetch_type='push') else: logging.debug( ' ---> [%-30s] ~FBSkipping feed fetch, no actives: %s' % (unicode(subscription.feed)[:30], subscription.feed)) return HttpResponse('') return Http404
def push_callback(request, push_id): if request.method == 'GET': mode = request.GET['hub.mode'] topic = request.GET['hub.topic'] challenge = request.GET['hub.challenge'] lease_seconds = request.GET.get('hub.lease_seconds') verify_token = request.GET.get('hub.verify_token', '') if mode == 'subscribe': if not verify_token.startswith('subscribe'): raise Http404 subscription = get_object_or_404(PushSubscription, pk=push_id, topic=topic, verify_token=verify_token) subscription.verified = True subscription.set_expiration(int(lease_seconds)) subscription.save() subscription.feed.setup_push() logging.debug(' ---> [%-30s] [%s] ~BBVerified PuSH' % (unicode(subscription.feed)[:30], subscription.feed_id)) verified.send(sender=subscription) return HttpResponse(challenge, content_type='text/plain') elif request.method == 'POST': subscription = get_object_or_404(PushSubscription, pk=push_id) # XXX TODO: Optimize this by removing feedparser. It just needs to find out # the hub_url or topic has changed. ElementTree could do it. if random.random() < 0.1: parsed = feedparser.parse(request.raw_post_data) subscription.check_urls_against_pushed_data(parsed) # Don't give fat ping, just fetch. # subscription.feed.queue_pushed_feed_xml(request.raw_post_data) if subscription.feed.active_premium_subscribers >= 1: subscription.feed.queue_pushed_feed_xml("Fetch me") MFetchHistory.add(feed_id=subscription.feed_id, fetch_type='push') else: logging.debug(' ---> [%-30s] ~FBSkipping feed fetch, no actives: %s' % (unicode(subscription.feed)[:30], subscription.feed)) return HttpResponse('') return Http404
def receive_newsletter(self, params): user = self._user_from_email(params['recipient']) if not user: return sender_name, sender_username, sender_domain = self._split_sender( params['from']) feed_address = self._feed_address( user, "%s@%s" % (sender_username, sender_domain)) usf = UserSubscriptionFolders.objects.get(user=user) usf.add_folder('', 'Newsletters') try: feed = Feed.objects.get(feed_address=feed_address) except Feed.DoesNotExist: feed = Feed.objects.create(feed_address=feed_address, feed_link='http://' + sender_domain, feed_title=sender_name, fetched_once=True, known_good=True) feed.update() logging.user(user, "~FCCreating newsletter feed: ~SB%s" % (feed)) r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL) r.publish(user.username, 'reload:%s' % feed.pk) self._check_if_first_newsletter(user) if feed.feed_title != sender_name: feed.feed_title = sender_name feed.save() try: usersub = UserSubscription.objects.get(user=user, feed=feed) except UserSubscription.DoesNotExist: _, _, usersub = UserSubscription.add_subscription( user=user, feed_address=feed_address, folder='Newsletters') r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL) r.publish(user.username, 'reload:feeds') story_hash = MStory.ensure_story_hash(params['signature'], feed.pk) story_content = self._get_content(params) plain_story_content = self._get_content(params, force_plain=True) if len(plain_story_content) > len(story_content): story_content = plain_story_content story_content = self._clean_content(story_content) story_params = { "story_feed_id": feed.pk, "story_date": datetime.datetime.fromtimestamp(int(params['timestamp'])), "story_title": params['subject'], "story_content": story_content, "story_author_name": params['from'], "story_permalink": "https://%s%s" % (Site.objects.get_current().domain, reverse('newsletter-story', kwargs={'story_hash': story_hash})), "story_guid": params['signature'], } print story_params try: story = MStory.objects.get(story_hash=story_hash) except MStory.DoesNotExist: story = MStory(**story_params) story.save() usersub.needs_unread_recalc = True usersub.save() self._publish_to_subscribers(feed) MFetchHistory.add(feed_id=feed.pk, fetch_type='push') logging.user( user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" % (story.story_title, feed)) return story
def receive_newsletter(self, params): user = self._user_from_email(params['recipient']) if not user: return sender_name, sender_username, sender_domain = self._split_sender( params['from']) feed_address = self._feed_address( user, "%s@%s" % (sender_username, sender_domain)) try: usf = UserSubscriptionFolders.objects.get(user=user) except UserSubscriptionFolders.DoesNotExist: logging.user(user, "~FRUser does not have a USF, ignoring newsletter.") return usf.add_folder('', 'Newsletters') # First look for the email address try: feed = Feed.objects.get(feed_address=feed_address) except Feed.MultipleObjectsReturned: feeds = Feed.objects.filter(feed_address=feed_address)[:1] if feeds.count(): feed = feeds[0] except Feed.DoesNotExist: feed = None # If not found, check among titles user has subscribed to if not feed: newsletter_subs = UserSubscription.objects.filter( user=user, feed__feed_address__contains="newsletter:").only('feed') newsletter_feed_ids = [us.feed.pk for us in newsletter_subs] feeds = Feed.objects.filter(feed_title__iexact=sender_name, pk__in=newsletter_feed_ids) if feeds.count(): feed = feeds[0] # Create a new feed if it doesn't exist by sender name or email if not feed: feed = Feed.objects.create(feed_address=feed_address, feed_link='http://' + sender_domain, feed_title=sender_name, fetched_once=True, known_good=True) feed.update() logging.user(user, "~FCCreating newsletter feed: ~SB%s" % (feed)) r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL) r.publish(user.username, 'reload:%s' % feed.pk) self._check_if_first_newsletter(user) feed.last_update = datetime.datetime.now() feed.last_story_date = datetime.datetime.now() feed.save() if feed.feed_title != sender_name: feed.feed_title = sender_name feed.save() try: usersub = UserSubscription.objects.get(user=user, feed=feed) except UserSubscription.DoesNotExist: _, _, usersub = UserSubscription.add_subscription( user=user, feed_address=feed_address, folder='Newsletters') r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL) r.publish(user.username, 'reload:feeds') story_hash = MStory.ensure_story_hash(params['signature'], feed.pk) story_content = self._get_content(params) plain_story_content = self._get_content(params, force_plain=True) if len(plain_story_content) > len(story_content): story_content = plain_story_content story_content = self._clean_content(story_content) story_params = { "story_feed_id": feed.pk, "story_date": datetime.datetime.fromtimestamp(int(params['timestamp'])), "story_title": params['subject'], "story_content": story_content, "story_author_name": params['from'], "story_permalink": "https://%s%s" % (Site.objects.get_current().domain, reverse('newsletter-story', kwargs={'story_hash': story_hash})), "story_guid": params['signature'], } try: story = MStory.objects.get(story_hash=story_hash) except MStory.DoesNotExist: story = MStory(**story_params) story.save() usersub.needs_unread_recalc = True usersub.save() self._publish_to_subscribers(feed, story.story_hash) MFetchHistory.add(feed_id=feed.pk, fetch_type='push') logging.user( user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" % (story.story_title, feed)) return story
def receive_newsletter(self, params): user = self._user_from_email(params['recipient']) if not user: return sender_name, sender_username, sender_domain = self._split_sender(params['from']) feed_address = self._feed_address(user, "%s@%s" % (sender_username, sender_domain)) try: usf = UserSubscriptionFolders.objects.get(user=user) except UserSubscriptionFolders.DoesNotExist: logging.user(user, "~FRUser does not have a USF, ignoring newsletter.") return usf.add_folder('', 'Newsletters') try: feed = Feed.objects.get(feed_address=feed_address) except Feed.DoesNotExist: feed = Feed.objects.create(feed_address=feed_address, feed_link='http://' + sender_domain, feed_title=sender_name, fetched_once=True, known_good=True) feed.update() logging.user(user, "~FCCreating newsletter feed: ~SB%s" % (feed)) r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL) r.publish(user.username, 'reload:%s' % feed.pk) self._check_if_first_newsletter(user) feed.last_update = datetime.datetime.now() feed.last_story_date = datetime.datetime.now() feed.save() if feed.feed_title != sender_name: feed.feed_title = sender_name feed.save() try: usersub = UserSubscription.objects.get(user=user, feed=feed) except UserSubscription.DoesNotExist: _, _, usersub = UserSubscription.add_subscription( user=user, feed_address=feed_address, folder='Newsletters' ) r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL) r.publish(user.username, 'reload:feeds') story_hash = MStory.ensure_story_hash(params['signature'], feed.pk) story_content = self._get_content(params) plain_story_content = self._get_content(params, force_plain=True) if len(plain_story_content) > len(story_content): story_content = plain_story_content story_content = self._clean_content(story_content) story_params = { "story_feed_id": feed.pk, "story_date": datetime.datetime.fromtimestamp(int(params['timestamp'])), "story_title": params['subject'], "story_content": story_content, "story_author_name": params['from'], "story_permalink": "https://%s%s" % ( Site.objects.get_current().domain, reverse('newsletter-story', kwargs={'story_hash': story_hash})), "story_guid": params['signature'], } try: story = MStory.objects.get(story_hash=story_hash) except MStory.DoesNotExist: story = MStory(**story_params) story.save() usersub.needs_unread_recalc = True usersub.save() self._publish_to_subscribers(feed, story.story_hash) MFetchHistory.add(feed_id=feed.pk, fetch_type='push') logging.user(user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" % (story.story_title, feed)) return story
def receive_newsletter(self, params): user = self.user_from_email(params["recipient"]) if not user: return sender_name, sender_username, sender_domain = self.split_sender(params["from"]) feed_address = self.feed_address(user, "%s@%s" % (sender_username, sender_domain)) usf = UserSubscriptionFolders.objects.get(user=user) usf.add_folder("", "Newsletters") try: feed = Feed.objects.get(feed_address=feed_address) except Feed.DoesNotExist: feed = Feed.objects.create( feed_address=feed_address, feed_link="http://" + sender_domain, feed_title=sender_name, fetched_once=True, known_good=True, ) feed.update() logging.user(user, "~FCCreating newsletter feed: ~SB%s" % (feed)) r = redis.Redis(connection_pool=settings.REDIS_PUBSUB_POOL) r.publish(user.username, "reload:%s" % feed.pk) if feed.feed_title != sender_name: feed.feed_title = sender_name feed.save() try: usersub = UserSubscription.objects.get(user=user, feed=feed) except UserSubscription.DoesNotExist: _, _, usersub = UserSubscription.add_subscription( user=user, feed_address=feed_address, folder="Newsletters" ) story_hash = MStory.ensure_story_hash(params["signature"], feed.pk) story_content = self.get_content(params) story_content = self.clean_content(story_content) story_params = { "story_feed_id": feed.pk, "story_date": datetime.datetime.fromtimestamp(int(params["timestamp"])), "story_title": params["subject"], "story_content": story_content, "story_author_name": params["from"], "story_permalink": "https://%s%s" % (Site.objects.get_current().domain, reverse("newsletter-story", kwargs={"story_hash": story_hash})), "story_guid": params["signature"], } try: story = MStory.objects.get(story_hash=story_hash) except MStory.DoesNotExist: story = MStory(**story_params) story.save() usersub.needs_unread_recalc = True usersub.save() self.publish_to_subscribers(feed) MFetchHistory.add(feed_id=feed.pk, fetch_type="push") logging.user(user, "~FCNewsletter feed story: ~SB%s~SN / ~SB%s" % (story.story_title, feed)) return story