def process(self): """ Downloads and parses a feed. """ start = time.time() self.refresh_feed() ret_values = dict(new=0, updated=0, same=0, error=0) if hasattr(self.fpf, 'status'): if self.options['verbose']: if self.fpf.bozo and self.fpf.status != 304: logging.debug(u' ---> [%-30s] ~FRBOZO exception: %s ~SB(%s entries)' % ( self.feed.title[:30], self.fpf.bozo_exception, len(self.fpf.entries))) if self.fpf.status == 304: self.feed = self.feed.save() self.feed.save_feed_history(304, "Not modified") return FEED_SAME, ret_values # 302: Temporary redirect: ignore # 301: Permanent redirect: save it (after 20 tries) if self.fpf.status == 301: if self.fpf.href.endswith('feedburner.com/atom.xml'): return FEED_ERRHTTP, ret_values redirects, non_redirects = self.feed.count_redirects_in_history('feed') self.feed.save_feed_history(self.fpf.status, "HTTP Redirect (%d to go)" % (20-len(redirects))) if len(redirects) >= 20 or len(non_redirects) == 0: self.feed.feed_address = self.fpf.href if not self.feed.known_good: self.feed.fetched_once = True logging.debug(" ---> [%-30s] ~SB~SK~FRFeed is %s'ing. Refetching..." % (self.feed.title[:30], self.fpf.status)) self.feed = self.feed.schedule_feed_fetch_immediately() if not self.fpf.entries: self.feed = self.feed.save() self.feed.save_feed_history(self.fpf.status, "HTTP Redirect") return FEED_ERRHTTP, ret_values if self.fpf.status >= 400: logging.debug(" ---> [%-30s] ~SB~FRHTTP Status code: %s. Checking address..." % (self.feed.title[:30], self.fpf.status)) fixed_feed = None if not self.feed.known_good: fixed_feed, feed = self.feed.check_feed_link_for_feed_address() if not fixed_feed: self.feed.save_feed_history(self.fpf.status, "HTTP Error") else: self.feed = feed self.feed = self.feed.save() return FEED_ERRHTTP, ret_values if not self.fpf.entries: if self.fpf.bozo and isinstance(self.fpf.bozo_exception, feedparser.NonXMLContentType): logging.debug(" ---> [%-30s] ~SB~FRFeed is Non-XML. %s entries. Checking address..." % (self.feed.title[:30], len(self.fpf.entries))) fixed_feed = None if not self.feed.known_good: fixed_feed, feed = self.feed.check_feed_link_for_feed_address() if not fixed_feed: self.feed.save_feed_history(552, 'Non-xml feed', self.fpf.bozo_exception) else: self.feed = feed self.feed = self.feed.save() return FEED_ERRPARSE, ret_values elif self.fpf.bozo and isinstance(self.fpf.bozo_exception, xml.sax._exceptions.SAXException): logging.debug(" ---> [%-30s] ~SB~FRFeed has SAX/XML parsing issues. %s entries. Checking address..." % (self.feed.title[:30], len(self.fpf.entries))) fixed_feed = None if not self.feed.known_good: fixed_feed, feed = self.feed.check_feed_link_for_feed_address() if not fixed_feed: self.feed.save_feed_history(553, 'SAX Exception', self.fpf.bozo_exception) else: self.feed = feed self.feed = self.feed.save() return FEED_ERRPARSE, ret_values # the feed has changed (or it is the first time we parse it) # saving the etag and last_modified fields original_etag = self.feed.etag self.feed.etag = self.fpf.get('etag') if self.feed.etag: self.feed.etag = self.feed.etag[:255] # some times this is None (it never should) *sigh* if self.feed.etag is None: self.feed.etag = '' if self.feed.etag != original_etag: self.feed.save(update_fields=['etag']) original_last_modified = self.feed.last_modified try: self.feed.last_modified = mtime(self.fpf.modified) except: self.feed.last_modified = None pass if self.feed.last_modified != original_last_modified: self.feed.save(update_fields=['last_modified']) self.fpf.entries = self.fpf.entries[:100] original_title = self.feed.feed_title if self.fpf.feed.get('title'): self.feed.feed_title = strip_tags(self.fpf.feed.get('title')) if self.feed.feed_title != original_title: self.feed.save(update_fields=['feed_title']) tagline = self.fpf.feed.get('tagline', self.feed.data.feed_tagline) if tagline: original_tagline = self.feed.data.feed_tagline self.feed.data.feed_tagline = smart_unicode(tagline) if self.feed.data.feed_tagline != original_tagline: self.feed.data.save(update_fields=['feed_tagline']) if not self.feed.feed_link_locked: new_feed_link = self.fpf.feed.get('link') or self.fpf.feed.get('id') or self.feed.feed_link if new_feed_link != self.feed.feed_link: logging.debug(" ---> [%-30s] ~SB~FRFeed's page is different: %s to %s" % (self.feed.title[:30], self.feed.feed_link, new_feed_link)) redirects, non_redirects = self.feed.count_redirects_in_history('page') self.feed.save_page_history(301, "HTTP Redirect (%s to go)" % (20-len(redirects))) if len(redirects) >= 20 or len(non_redirects) == 0: self.feed.feed_link = new_feed_link self.feed.save(update_fields=['feed_link']) # Determine if stories aren't valid and replace broken guids guids_seen = set() permalinks_seen = set() for entry in self.fpf.entries: guids_seen.add(entry.get('guid')) permalinks_seen.add(Feed.get_permalink(entry)) guid_difference = len(guids_seen) != len(self.fpf.entries) single_guid = len(guids_seen) == 1 replace_guids = single_guid and guid_difference permalink_difference = len(permalinks_seen) != len(self.fpf.entries) single_permalink = len(permalinks_seen) == 1 replace_permalinks = single_permalink and permalink_difference # Compare new stories to existing stories, adding and updating start_date = datetime.datetime.utcnow() story_hashes = [] stories = [] for entry in self.fpf.entries: story = pre_process_story(entry) if story.get('published') < start_date: start_date = story.get('published') if replace_guids: if replace_permalinks: new_story_guid = unicode(story.get('published')) if self.options['verbose']: logging.debug(u' ---> [%-30s] ~FBReplacing guid (%s) with timestamp: %s' % ( self.feed.title[:30], story.get('guid'), new_story_guid)) story['guid'] = new_story_guid else: new_story_guid = Feed.get_permalink(story) if self.options['verbose']: logging.debug(u' ---> [%-30s] ~FBReplacing guid (%s) with permalink: %s' % ( self.feed.title[:30], story.get('guid'), new_story_guid)) story['guid'] = new_story_guid story['story_hash'] = MStory.feed_guid_hash_unsaved(self.feed.pk, story.get('guid')) stories.append(story) story_hashes.append(story.get('story_hash')) existing_stories = dict((s.story_hash, s) for s in MStory.objects( story_hash__in=story_hashes, # story_date__gte=start_date, # story_feed_id=self.feed.pk )) ret_values = self.feed.add_update_stories(stories, existing_stories, verbose=self.options['verbose'], updates_off=self.options['updates_off']) if (hasattr(self.fpf, 'feed') and hasattr(self.fpf.feed, 'links') and self.fpf.feed.links): hub_url = None self_url = self.feed.feed_address for link in self.fpf.feed.links: if link['rel'] == 'hub' and not hub_url: hub_url = link['href'] elif link['rel'] == 'self': self_url = link['href'] push_expired = False if self.feed.is_push: try: push_expired = self.feed.push.lease_expires < datetime.datetime.now() except PushSubscription.DoesNotExist: self.feed.is_push = False if (hub_url and self_url and not settings.DEBUG and self.feed.active_subscribers > 0 and (push_expired or not self.feed.is_push or self.options.get('force'))): logging.debug(u' ---> [%-30s] ~BB~FW%sSubscribing to PuSH hub: %s' % ( self.feed.title[:30], "~SKRe-~SN" if push_expired else "", hub_url)) try: PushSubscription.objects.subscribe(self_url, feed=self.feed, hub=hub_url) except TimeoutError: logging.debug(u' ---> [%-30s] ~BB~FW~FRTimed out~FW subscribing to PuSH hub: %s' % ( self.feed.title[:30], hub_url)) elif (self.feed.is_push and (self.feed.active_subscribers <= 0 or not hub_url)): logging.debug(u' ---> [%-30s] ~BB~FWTurning off PuSH, no hub found' % ( self.feed.title[:30])) self.feed.is_push = False self.feed = self.feed.save() logging.debug(u' ---> [%-30s] ~FYParsed Feed: %snew=%s~SN~FY %sup=%s~SN same=%s%s~SN %serr=%s~SN~FY total=~SB%s' % ( self.feed.title[:30], '~FG~SB' if ret_values['new'] else '', ret_values['new'], '~FY~SB' if ret_values['updated'] else '', ret_values['updated'], '~SB' if ret_values['same'] else '', ret_values['same'], '~FR~SB' if ret_values['error'] else '', ret_values['error'], len(self.fpf.entries))) self.feed.update_all_statistics(has_new_stories=bool(ret_values['new']), force=self.options['force']) if ret_values['new']: self.feed.trim_feed() self.feed.expire_redis() self.feed.save_feed_history(200, "OK") if self.options['verbose']: logging.debug(u' ---> [%-30s] ~FBTIME: feed parse in ~FM%.4ss' % ( self.feed.title[:30], time.time() - start)) return FEED_OK, ret_values
def process(self): """ Downloads and parses a feed. """ start = time.time() self.refresh_feed() ret_values = dict(new=0, updated=0, same=0, error=0) if hasattr(self.fpf, 'status'): if self.options['verbose']: if self.fpf.bozo and self.fpf.status != 304: logging.debug(u' ---> [%-30s] ~FRBOZO exception: %s ~SB(%s entries)' % ( self.feed.title[:30], self.fpf.bozo_exception, len(self.fpf.entries))) if self.fpf.status == 304: # 304 stands for resource not modified self.feed = self.feed.save() self.feed.save_feed_history(304, "Not modified") return FEED_SAME, ret_values # 302: Temporary redirect: ignore # 301: Permanent redirect: save it if self.fpf.status == 301: if not self.fpf.href.endswith('feedburner.com/atom.xml'): self.feed.feed_address = self.fpf.href if not self.feed.known_good: self.feed.fetched_once = True logging.debug(" ---> [%-30s] ~SB~SK~FRFeed is %s'ing. Refetching..." % (self.feed.title[:30], self.fpf.status)) self.feed = self.feed.schedule_feed_fetch_immediately() if not self.fpf.entries: self.feed = self.feed.save() self.feed.save_feed_history(self.fpf.status, "HTTP Redirect") return FEED_ERRHTTP, ret_values if self.fpf.status >= 400: logging.debug(" ---> [%-30s] ~SB~FRHTTP Status code: %s. Checking address..." % (self.feed.title[:30], self.fpf.status)) fixed_feed = None if not self.feed.known_good: fixed_feed, feed = self.feed.check_feed_link_for_feed_address() if not fixed_feed: self.feed.save_feed_history(self.fpf.status, "HTTP Error") else: self.feed = feed self.feed = self.feed.save() return FEED_ERRHTTP, ret_values if not self.fpf.entries: if self.fpf.bozo and isinstance(self.fpf.bozo_exception, feedparser.NonXMLContentType): logging.debug(" ---> [%-30s] ~SB~FRFeed is Non-XML. %s entries. Checking address..." % (self.feed.title[:30], len(self.fpf.entries))) fixed_feed = None if not self.feed.known_good: fixed_feed, feed = self.feed.check_feed_link_for_feed_address() if not fixed_feed: self.feed.save_feed_history(552, 'Non-xml feed', self.fpf.bozo_exception) else: self.feed = feed self.feed = self.feed.save() return FEED_ERRPARSE, ret_values elif self.fpf.bozo and isinstance(self.fpf.bozo_exception, xml.sax._exceptions.SAXException): logging.debug(" ---> [%-30s] ~SB~FRFeed has SAX/XML parsing issues. %s entries. Checking address..." % (self.feed.title[:30], len(self.fpf.entries))) fixed_feed = None if not self.feed.known_good: fixed_feed, feed = self.feed.check_feed_link_for_feed_address() if not fixed_feed: self.feed.save_feed_history(553, 'SAX Exception', self.fpf.bozo_exception) else: self.feed = feed self.feed = self.feed.save() return FEED_ERRPARSE, ret_values # the feed has changed (or it is the first time we parse it) # saving the etag and last_modified fields self.feed.etag = self.fpf.get('etag') if self.feed.etag: self.feed.etag = self.feed.etag[:255] # some times this is None (it never should) *sigh* if self.feed.etag is None: self.feed.etag = '' try: self.feed.last_modified = mtime(self.fpf.modified) except: self.feed.last_modified = None pass self.fpf.entries = self.fpf.entries[:100] if self.fpf.feed.get('title'): self.feed.feed_title = strip_tags(self.fpf.feed.get('title')) self.feed.feed_link = self.fpf.feed.get('link') or self.fpf.feed.get('id') or self.feed.feed_link self.feed = self.feed.save() # Determine if stories aren't valid and replace broken guids # if guid is single among many entries: # if permalink also is single among many entries: # replace the guid with published # else if permalink is not: # replace the guid with permalink guids_seen = set() permalinks_seen = set() for entry in self.fpf.entries: guids_seen.add(entry.get('guid')) permalinks_seen.add(Feed.get_permalink(entry)) guid_difference = len(guids_seen) != len(self.fpf.entries) # means guid is duplicated. single_guid = len(guids_seen) == 1 replace_guids = single_guid and guid_difference # means guid is single but entries not. permalink_difference = len(permalinks_seen) != len(self.fpf.entries) single_permalink = len(permalinks_seen) == 1 replace_permalinks = single_permalink and permalink_difference # Compare new stories to existing stories, adding and updating start_date = datetime.datetime.utcnow() story_hashes = [] stories = [] for entry in self.fpf.entries: story = pre_process_story(entry) if story.get('published') < start_date: start_date = story.get('published') if replace_guids: if replace_permalinks: new_story_guid = unicode(story.get('published')) if self.options['verbose']: logging.debug(u' ---> [%-30s] ~FBReplacing guid (%s) with timestamp: %s' % ( self.feed.title[:30], story.get('guid'), new_story_guid)) story['guid'] = new_story_guid else: new_story_guid = Feed.get_permalink(story) if self.options['verbose']: logging.debug(u' ---> [%-30s] ~FBReplacing guid (%s) with permalink: %s' % ( self.feed.title[:30], story.get('guid'), new_story_guid)) story['guid'] = new_story_guid story['story_hash'] = MStory.feed_guid_hash_unsaved(self.feed.pk, story.get('guid')) stories.append(story) story_hashes.append(story.get('story_hash')) # find the existing_stories with story_hash in story_hashes. existing_stories = dict((s.story_hash, s) for s in MStory.objects( story_hash__in=story_hashes, # story_date__gte=start_date, # story_feed_id=self.feed.pk )) ret_values = self.feed.add_update_stories(stories, existing_stories, verbose=self.options['verbose'],) logging.debug(u' ---> [%-30s] ~FYParsed Feed: %snew=%s~SN~FY %sup=%s~SN same=%s%s~SN %serr=%s~SN~FY total=~SB%s' % ( self.feed.title[:30], '~FG~SB' if ret_values['new'] else '', ret_values['new'], '~FY~SB' if ret_values['updated'] else '', ret_values['updated'], '~SB' if ret_values['same'] else '', ret_values['same'], '~FR~SB' if ret_values['error'] else '', ret_values['error'], len(self.fpf.entries))) # If there is new story, update all statistics self.feed.update_all_statistics(full=bool(ret_values['new'])) self.feed.save_feed_history(200, "OK") if self.options['verbose']: logging.debug(u' ---> [%-30s] ~FBTIME: feed parse in ~FM%.4ss' % ( self.feed.title[:30], time.time() - start)) return FEED_OK, ret_values
def process(self): """ Downloads and parses a feed. """ start = time.time() self.refresh_feed() ret_values = dict(new=0, updated=0, same=0, error=0) if hasattr(self.fpf, 'status'): if self.options['verbose']: if self.fpf.bozo and self.fpf.status != 304: logging.debug( u' ---> [%-30s] ~FRBOZO exception: %s ~SB(%s entries)' % (self.feed.title[:30], self.fpf.bozo_exception, len(self.fpf.entries))) if self.fpf.status == 304: self.feed = self.feed.save() self.feed.save_feed_history(304, "Not modified") return FEED_SAME, ret_values # 302: Temporary redirect: ignore # 301: Permanent redirect: save it if self.fpf.status == 301: if not self.fpf.href.endswith('feedburner.com/atom.xml'): self.feed.feed_address = self.fpf.href if not self.feed.known_good: self.feed.fetched_once = True logging.debug( " ---> [%-30s] ~SB~SK~FRFeed is %s'ing. Refetching..." % (self.feed.title[:30], self.fpf.status)) self.feed = self.feed.schedule_feed_fetch_immediately() if not self.fpf.entries: self.feed = self.feed.save() self.feed.save_feed_history(self.fpf.status, "HTTP Redirect") return FEED_ERRHTTP, ret_values if self.fpf.status >= 400: logging.debug( " ---> [%-30s] ~SB~FRHTTP Status code: %s. Checking address..." % (self.feed.title[:30], self.fpf.status)) fixed_feed = None if not self.feed.known_good: fixed_feed, feed = self.feed.check_feed_link_for_feed_address( ) if not fixed_feed: self.feed.save_feed_history(self.fpf.status, "HTTP Error") else: self.feed = feed self.feed = self.feed.save() return FEED_ERRHTTP, ret_values if not self.fpf.entries: if self.fpf.bozo and isinstance(self.fpf.bozo_exception, feedparser.NonXMLContentType): logging.debug( " ---> [%-30s] ~SB~FRFeed is Non-XML. %s entries. Checking address..." % (self.feed.title[:30], len(self.fpf.entries))) fixed_feed = None if not self.feed.known_good: fixed_feed, feed = self.feed.check_feed_link_for_feed_address( ) if not fixed_feed: self.feed.save_feed_history(552, 'Non-xml feed', self.fpf.bozo_exception) else: self.feed = feed self.feed = self.feed.save() return FEED_ERRPARSE, ret_values elif self.fpf.bozo and isinstance( self.fpf.bozo_exception, xml.sax._exceptions.SAXException): logging.debug( " ---> [%-30s] ~SB~FRFeed has SAX/XML parsing issues. %s entries. Checking address..." % (self.feed.title[:30], len(self.fpf.entries))) fixed_feed = None if not self.feed.known_good: fixed_feed, feed = self.feed.check_feed_link_for_feed_address( ) if not fixed_feed: self.feed.save_feed_history(553, 'SAX Exception', self.fpf.bozo_exception) else: self.feed = feed self.feed = self.feed.save() return FEED_ERRPARSE, ret_values # the feed has changed (or it is the first time we parse it) # saving the etag and last_modified fields self.feed.etag = self.fpf.get('etag') if self.feed.etag: self.feed.etag = self.feed.etag[:255] # some times this is None (it never should) *sigh* if self.feed.etag is None: self.feed.etag = '' try: self.feed.last_modified = mtime(self.fpf.modified) except: self.feed.last_modified = None pass self.fpf.entries = self.fpf.entries[:100] if self.fpf.feed.get('title'): self.feed.feed_title = strip_tags(self.fpf.feed.get('title')) tagline = self.fpf.feed.get('tagline', self.feed.data.feed_tagline) if tagline: self.feed.data.feed_tagline = utf8encode(tagline) self.feed.data.save() if not self.feed.feed_link_locked: self.feed.feed_link = self.fpf.feed.get( 'link') or self.fpf.feed.get('id') or self.feed.feed_link self.feed = self.feed.save() # Compare new stories to existing stories, adding and updating start_date = datetime.datetime.utcnow() story_guids = [] stories = [] for entry in self.fpf.entries: story = pre_process_story(entry) if story.get('published') < start_date: start_date = story.get('published') stories.append(story) story_guids.append(story.get('guid')) existing_stories = dict((s.story_guid, s) for s in MStory.objects( # story_guid__in=story_guids, story_date__gte=start_date, story_feed_id=self.feed.pk).limit( max(int(len(story_guids) * 1.5), 10))) ret_values = self.feed.add_update_stories( stories, existing_stories, verbose=self.options['verbose']) if (hasattr(self.fpf, 'feed') and hasattr(self.fpf.feed, 'links') and self.fpf.feed.links): hub_url = None self_url = self.feed.feed_address for link in self.fpf.feed.links: if link['rel'] == 'hub' and not hub_url: hub_url = link['href'] elif link['rel'] == 'self': self_url = link['href'] push_expired = False if self.feed.is_push: try: push_expired = self.feed.push.lease_expires < datetime.datetime.now( ) except PushSubscription.DoesNotExist: self.feed.is_push = False if (hub_url and self_url and not settings.DEBUG and self.feed.active_subscribers > 0 and (push_expired or not self.feed.is_push or self.options.get('force'))): logging.debug( u' ---> [%-30s] ~BB~FW%sSubscribing to PuSH hub: %s' % (self.feed.title[:30], "~SKRe-~SN" if push_expired else "", hub_url)) try: PushSubscription.objects.subscribe(self_url, feed=self.feed, hub=hub_url) except TimeoutError: logging.debug( u' ---> [%-30s] ~BB~FW~FRTimed out~FW subscribing to PuSH hub: %s' % (self.feed.title[:30], hub_url)) elif (self.feed.is_push and (self.feed.active_subscribers <= 0 or not hub_url)): logging.debug( u' ---> [%-30s] ~BB~FWTurning off PuSH, no hub found' % (self.feed.title[:30])) self.feed.is_push = False self.feed = self.feed.save() logging.debug( u' ---> [%-30s] ~FYParsed Feed: %snew=%s~SN~FY %sup=%s~SN same=%s%s~SN %serr=%s~SN~FY total=~SB%s' % (self.feed.title[:30], '~FG~SB' if ret_values['new'] else '', ret_values['new'], '~FY~SB' if ret_values['updated'] else '', ret_values['updated'], '~SB' if ret_values['same'] else '', ret_values['same'], '~FR~SB' if ret_values['error'] else '', ret_values['error'], len(self.fpf.entries))) self.feed.update_all_statistics(full=bool(ret_values['new']), force=self.options['force']) if ret_values['new']: self.feed.trim_feed() self.feed.expire_redis() self.feed.save_feed_history(200, "OK") if self.options['verbose']: logging.debug(u' ---> [%-30s] ~FBTIME: feed parse in ~FM%.4ss' % (self.feed.title[:30], time.time() - start)) return FEED_OK, ret_values
def load_social_stories(request, user_id, username=None): start = time.time() user = get_user(request) social_user_id = int(user_id) social_user = get_object_or_404(User, pk=social_user_id) offset = int(request.REQUEST.get('offset', 0)) limit = int(request.REQUEST.get('limit', 6)) page = request.REQUEST.get('page') order = request.REQUEST.get('order', 'newest') read_filter = request.REQUEST.get('read_filter', 'all') stories = [] if page: offset = limit * (int(page) - 1) now = localtime_for_timezone(datetime.datetime.now(), user.profile.timezone) UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD) social_profile = MSocialProfile.get_user(social_user.pk) try: socialsub = MSocialSubscription.objects.get(user_id=user.pk, subscription_user_id=social_user_id) except MSocialSubscription.DoesNotExist: socialsub = None mstories = MSharedStory.objects(user_id=social_user.pk).order_by('-shared_date')[offset:offset+limit] stories = Feed.format_stories(mstories) if socialsub and (read_filter == 'unread' or order == 'oldest'): story_ids = socialsub.get_stories(order=order, read_filter=read_filter, offset=offset, limit=limit) story_date_order = "%sshared_date" % ('' if order == 'oldest' else '-') if story_ids: mstories = MSharedStory.objects(user_id=social_user.pk, story_db_id__in=story_ids).order_by(story_date_order) stories = Feed.format_stories(mstories) else: mstories = MSharedStory.objects(user_id=social_user.pk).order_by('-shared_date')[offset:offset+limit] stories = Feed.format_stories(mstories) if not stories: return dict(stories=[]) checkpoint1 = time.time() stories, user_profiles = MSharedStory.stories_with_comments_and_profiles(stories, user.pk, check_all=True) story_feed_ids = list(set(s['story_feed_id'] for s in stories)) usersubs = UserSubscription.objects.filter(user__pk=user.pk, feed__pk__in=story_feed_ids) usersubs_map = dict((sub.feed_id, sub) for sub in usersubs) unsub_feed_ids = list(set(story_feed_ids).difference(set(usersubs_map.keys()))) unsub_feeds = Feed.objects.filter(pk__in=unsub_feed_ids) unsub_feeds = [feed.canonical(include_favicon=False) for feed in unsub_feeds] date_delta = UNREAD_CUTOFF if socialsub and date_delta < socialsub.mark_read_date: date_delta = socialsub.mark_read_date # Get intelligence classifier for user classifier_feeds = list(MClassifierFeed.objects(user_id=user.pk, social_user_id=social_user_id)) classifier_authors = list(MClassifierAuthor.objects(user_id=user.pk, social_user_id=social_user_id)) classifier_titles = list(MClassifierTitle.objects(user_id=user.pk, social_user_id=social_user_id)) classifier_tags = list(MClassifierTag.objects(user_id=user.pk, social_user_id=social_user_id)) # Merge with feed specific classifiers classifier_feeds = classifier_feeds + list(MClassifierFeed.objects(user_id=user.pk, feed_id__in=story_feed_ids)) classifier_authors = classifier_authors + list(MClassifierAuthor.objects(user_id=user.pk, feed_id__in=story_feed_ids)) classifier_titles = classifier_titles + list(MClassifierTitle.objects(user_id=user.pk, feed_id__in=story_feed_ids)) classifier_tags = classifier_tags + list(MClassifierTag.objects(user_id=user.pk, feed_id__in=story_feed_ids)) checkpoint2 = time.time() story_ids = [story['id'] for story in stories] userstories_db = MUserStory.objects(user_id=user.pk, feed_id__in=story_feed_ids, story_id__in=story_ids).only('story_id') userstories = set(us.story_id for us in userstories_db) starred_stories = MStarredStory.objects(user_id=user.pk, story_feed_id__in=story_feed_ids, story_guid__in=story_ids).only('story_guid', 'starred_date') shared_stories = MSharedStory.objects(user_id=user.pk, story_feed_id__in=story_feed_ids, story_guid__in=story_ids)\ .only('story_guid', 'shared_date', 'comments') starred_stories = dict([(story.story_guid, story.starred_date) for story in starred_stories]) shared_stories = dict([(story.story_guid, dict(shared_date=story.shared_date, comments=story.comments)) for story in shared_stories]) for story in stories: story['social_user_id'] = social_user_id story_feed_id = story['story_feed_id'] # story_date = localtime_for_timezone(story['story_date'], user.profile.timezone) shared_date = localtime_for_timezone(story['shared_date'], user.profile.timezone) story['short_parsed_date'] = format_story_link_date__short(shared_date, now) story['long_parsed_date'] = format_story_link_date__long(shared_date, now) if not socialsub: story['read_status'] = 1 elif story['id'] in userstories: story['read_status'] = 1 elif story['shared_date'] < date_delta: story['read_status'] = 1 elif not usersubs_map.get(story_feed_id): story['read_status'] = 0 elif not story.get('read_status') and story['story_date'] < usersubs_map[story_feed_id].mark_read_date: story['read_status'] = 1 elif not story.get('read_status') and story['shared_date'] < date_delta: story['read_status'] = 1 # elif not story.get('read_status') and socialsub and story['shared_date'] > socialsub.last_read_date: # story['read_status'] = 0 else: story['read_status'] = 0 if story['id'] in starred_stories: story['starred'] = True starred_date = localtime_for_timezone(starred_stories[story['id']], user.profile.timezone) story['starred_date'] = format_story_link_date__long(starred_date, now) if story['id'] in shared_stories: story['shared'] = True shared_date = localtime_for_timezone(shared_stories[story['id']]['shared_date'], user.profile.timezone) story['shared_date'] = format_story_link_date__long(shared_date, now) story['shared_comments'] = strip_tags(shared_stories[story['id']]['comments']) story['intelligence'] = { 'feed': apply_classifier_feeds(classifier_feeds, story['story_feed_id'], social_user_id=social_user_id), 'author': apply_classifier_authors(classifier_authors, story), 'tags': apply_classifier_tags(classifier_tags, story), 'title': apply_classifier_titles(classifier_titles, story), } classifiers = sort_classifiers_by_feed(user=user, feed_ids=story_feed_ids, classifier_feeds=classifier_feeds, classifier_authors=classifier_authors, classifier_titles=classifier_titles, classifier_tags=classifier_tags) if socialsub: socialsub.feed_opens += 1 socialsub.save() diff1 = checkpoint1-start diff2 = checkpoint2-start logging.user(request, "~FYLoading ~FMshared stories~FY: ~SB%s%s ~SN(~SB%.4ss/%.4ss~SN)" % ( social_profile.title[:22], ('~SN/p%s' % page) if page > 1 else '', diff1, diff2)) return { "stories": stories, "user_profiles": user_profiles, "feeds": unsub_feeds, "classifiers": classifiers, }
def process(self): """ Downloads and parses a feed. """ start = time.time() self.refresh_feed() ret_values = dict(new=0, updated=0, same=0, error=0) if hasattr(self.fpf, 'status'): if self.options['verbose']: if self.fpf.bozo and self.fpf.status != 304: logging.debug(u' ---> [%-30s] ~FRBOZO exception: %s ~SB(%s entries)' % ( self.feed.title[:30], self.fpf.bozo_exception, len(self.fpf.entries))) if self.fpf.status == 304: self.feed = self.feed.save() self.feed.save_feed_history(304, "Not modified") return FEED_SAME, ret_values # 302: Temporary redirect: ignore # 301: Permanent redirect: save it if self.fpf.status == 301: if not self.fpf.href.endswith('feedburner.com/atom.xml'): self.feed.feed_address = self.fpf.href if not self.feed.known_good: self.feed.fetched_once = True logging.debug(" ---> [%-30s] ~SB~SK~FRFeed is %s'ing. Refetching..." % (self.feed.title[:30], self.fpf.status)) self.feed = self.feed.schedule_feed_fetch_immediately() if not self.fpf.entries: self.feed = self.feed.save() self.feed.save_feed_history(self.fpf.status, "HTTP Redirect") return FEED_ERRHTTP, ret_values if self.fpf.status >= 400: logging.debug(" ---> [%-30s] ~SB~FRHTTP Status code: %s. Checking address..." % (self.feed.title[:30], self.fpf.status)) fixed_feed = None if not self.feed.known_good: fixed_feed, feed = self.feed.check_feed_link_for_feed_address() if not fixed_feed: self.feed.save_feed_history(self.fpf.status, "HTTP Error") else: self.feed = feed self.feed = self.feed.save() return FEED_ERRHTTP, ret_values if not self.fpf.entries: if self.fpf.bozo and isinstance(self.fpf.bozo_exception, feedparser.NonXMLContentType): logging.debug(" ---> [%-30s] ~SB~FRFeed is Non-XML. %s entries. Checking address..." % (self.feed.title[:30], len(self.fpf.entries))) fixed_feed = None if not self.feed.known_good: fixed_feed, feed = self.feed.check_feed_link_for_feed_address() if not fixed_feed: self.feed.save_feed_history(552, 'Non-xml feed', self.fpf.bozo_exception) else: self.feed = feed self.feed = self.feed.save() return FEED_ERRPARSE, ret_values elif self.fpf.bozo and isinstance(self.fpf.bozo_exception, xml.sax._exceptions.SAXException): logging.debug(" ---> [%-30s] ~SB~FRFeed has SAX/XML parsing issues. %s entries. Checking address..." % (self.feed.title[:30], len(self.fpf.entries))) fixed_feed = None if not self.feed.known_good: fixed_feed, feed = self.feed.check_feed_link_for_feed_address() if not fixed_feed: self.feed.save_feed_history(553, 'SAX Exception', self.fpf.bozo_exception) else: self.feed = feed self.feed = self.feed.save() return FEED_ERRPARSE, ret_values # the feed has changed (or it is the first time we parse it) # saving the etag and last_modified fields self.feed.etag = self.fpf.get('etag') if self.feed.etag: self.feed.etag = self.feed.etag[:255] # some times this is None (it never should) *sigh* if self.feed.etag is None: self.feed.etag = '' try: self.feed.last_modified = mtime(self.fpf.modified) except: self.feed.last_modified = None pass self.fpf.entries = self.fpf.entries[:100] if self.fpf.feed.get('title'): self.feed.feed_title = strip_tags(self.fpf.feed.get('title')) tagline = self.fpf.feed.get('tagline', self.feed.data.feed_tagline) if tagline: self.feed.data.feed_tagline = utf8encode(tagline) self.feed.data.save() if not self.feed.feed_link_locked: self.feed.feed_link = self.fpf.feed.get('link') or self.fpf.feed.get('id') or self.feed.feed_link self.feed = self.feed.save() # Compare new stories to existing stories, adding and updating start_date = datetime.datetime.utcnow() story_guids = [] stories = [] for entry in self.fpf.entries: story = pre_process_story(entry) if story.get('published') < start_date: start_date = story.get('published') stories.append(story) story_guids.append(story.get('guid')) existing_stories = dict((s.story_guid, s) for s in MStory.objects( # story_guid__in=story_guids, story_date__gte=start_date, story_feed_id=self.feed.pk ).limit(max(int(len(story_guids)*1.5), 10))) ret_values = self.feed.add_update_stories(stories, existing_stories, verbose=self.options['verbose']) if (hasattr(self.fpf, 'feed') and hasattr(self.fpf.feed, 'links') and self.fpf.feed.links): hub_url = None self_url = self.feed.feed_address for link in self.fpf.feed.links: if link['rel'] == 'hub' and not hub_url: hub_url = link['href'] elif link['rel'] == 'self': self_url = link['href'] push_expired = False if self.feed.is_push: try: push_expired = self.feed.push.lease_expires < datetime.datetime.now() except PushSubscription.DoesNotExist: self.feed.is_push = False if (hub_url and self_url and not settings.DEBUG and self.feed.active_subscribers > 0 and (push_expired or not self.feed.is_push or self.options.get('force'))): logging.debug(u' ---> [%-30s] ~BB~FW%sSubscribing to PuSH hub: %s' % ( self.feed.title[:30], "~SKRe-~SN" if push_expired else "", hub_url)) try: PushSubscription.objects.subscribe(self_url, feed=self.feed, hub=hub_url) except TimeoutError: logging.debug(u' ---> [%-30s] ~BB~FW~FRTimed out~FW subscribing to PuSH hub: %s' % ( self.feed.title[:30], hub_url)) elif (self.feed.is_push and (self.feed.active_subscribers <= 0 or not hub_url)): logging.debug(u' ---> [%-30s] ~BB~FWTurning off PuSH, no hub found' % ( self.feed.title[:30])) self.feed.is_push = False self.feed = self.feed.save() logging.debug(u' ---> [%-30s] ~FYParsed Feed: %snew=%s~SN~FY %sup=%s~SN same=%s%s~SN %serr=%s~SN~FY total=~SB%s' % ( self.feed.title[:30], '~FG~SB' if ret_values['new'] else '', ret_values['new'], '~FY~SB' if ret_values['updated'] else '', ret_values['updated'], '~SB' if ret_values['same'] else '', ret_values['same'], '~FR~SB' if ret_values['error'] else '', ret_values['error'], len(self.fpf.entries))) self.feed.update_all_statistics(full=bool(ret_values['new']), force=self.options['force']) if ret_values['new']: self.feed.trim_feed() self.feed.expire_redis() self.feed.save_feed_history(200, "OK") if self.options['verbose']: logging.debug(u' ---> [%-30s] ~FBTIME: feed parse in ~FM%.4ss' % ( self.feed.title[:30], time.time() - start)) return FEED_OK, ret_values
def mark_story_as_shared(request): code = 1 feed_id = int(request.POST['feed_id']) story_id = request.POST['story_id'] comments = request.POST.get('comments', '') source_user_id = request.POST.get('source_user_id') post_to_services = request.POST.getlist('post_to_services') format = request.REQUEST.get('format', 'json') MSocialProfile.get_user(request.user.pk) story, original_story_found = MStory.find_story(feed_id, story_id) if not story: return json.json_response(request, { 'code': -1, 'message': 'Could not find the original story and no copies could be found.' }) shared_story = MSharedStory.objects.filter(user_id=request.user.pk, story_feed_id=feed_id, story_guid=story_id).limit(1).first() if not shared_story: story_db = dict([(k, v) for k, v in story._data.items() if k is not None and v is not None]) story_values = dict(user_id=request.user.pk, comments=comments, has_comments=bool(comments), story_db_id=story.id) story_db.update(story_values) shared_story = MSharedStory.objects.create(**story_db) if source_user_id: shared_story.set_source_user_id(int(source_user_id)) socialsubs = MSocialSubscription.objects.filter(subscription_user_id=request.user.pk) for socialsub in socialsubs: socialsub.needs_unread_recalc = True socialsub.save() logging.user(request, "~FCSharing ~FM%s: ~SB~FB%s" % (story.story_title[:20], comments[:30])) else: shared_story.comments = comments shared_story.has_comments = bool(comments) shared_story.save() logging.user(request, "~FCUpdating shared story ~FM%s: ~SB~FB%s" % ( story.story_title[:20], comments[:30])) if original_story_found: story.count_comments() shared_story.publish_update_to_subscribers() story = Feed.format_story(story) check_all = not original_story_found stories, profiles = MSharedStory.stories_with_comments_and_profiles([story], request.user.pk, check_all=check_all) story = stories[0] story['shared_comments'] = strip_tags(shared_story['comments'] or "") if post_to_services: for service in post_to_services: if service not in shared_story.posted_to_services: PostToService.delay(shared_story_id=shared_story.id, service=service) if shared_story.source_user_id and shared_story.comments: EmailStoryReshares.apply_async(kwargs=dict(shared_story_id=shared_story.id), countdown=60) if format == 'html': stories = MSharedStory.attach_users_to_stories(stories, profiles) return render_to_response('social/story_share.xhtml', { 'story': story, }, context_instance=RequestContext(request)) else: return json.json_response(request, { 'code': code, 'story': story, 'user_profiles': profiles, })
def mark_story_as_shared(request): code = 1 feed_id = int(request.POST["feed_id"]) story_id = request.POST["story_id"] comments = request.POST.get("comments", "") source_user_id = request.POST.get("source_user_id") relative_user_id = request.POST.get("relative_user_id") or request.user.pk post_to_services = request.POST.getlist("post_to_services") format = request.REQUEST.get("format", "json") MSocialProfile.get_user(request.user.pk) story, original_story_found = MStory.find_story(feed_id, story_id) if not story: return json.json_response( request, {"code": -1, "message": "Could not find the original story and no copies could be found."} ) shared_story = ( MSharedStory.objects.filter(user_id=request.user.pk, story_feed_id=feed_id, story_guid=story_id) .limit(1) .first() ) if not shared_story: story_db = dict([(k, v) for k, v in story._data.items() if k is not None and v is not None]) story_values = dict( user_id=request.user.pk, comments=comments, has_comments=bool(comments), story_db_id=story.id ) story_db.update(story_values) shared_story = MSharedStory.objects.create(**story_db) if source_user_id: shared_story.set_source_user_id(int(source_user_id)) socialsubs = MSocialSubscription.objects.filter(subscription_user_id=request.user.pk) for socialsub in socialsubs: socialsub.needs_unread_recalc = True socialsub.save() logging.user(request, "~FCSharing ~FM%s: ~SB~FB%s" % (story.story_title[:20], comments[:30])) else: shared_story.comments = comments shared_story.has_comments = bool(comments) shared_story.save() logging.user(request, "~FCUpdating shared story ~FM%s: ~SB~FB%s" % (story.story_title[:20], comments[:30])) if original_story_found: story.count_comments() shared_story.publish_update_to_subscribers() story = Feed.format_story(story) check_all = not original_story_found stories, profiles = MSharedStory.stories_with_comments_and_profiles([story], relative_user_id, check_all=check_all) story = stories[0] story["shared_comments"] = strip_tags(shared_story["comments"] or "") story["shared_by_user"] = True if post_to_services: for service in post_to_services: if service not in shared_story.posted_to_services: PostToService.delay(shared_story_id=shared_story.id, service=service) if shared_story.source_user_id and shared_story.comments: EmailStoryReshares.apply_async( kwargs=dict(shared_story_id=shared_story.id), countdown=settings.SECONDS_TO_DELAY_CELERY_EMAILS ) if format == "html": stories = MSharedStory.attach_users_to_stories(stories, profiles) return render_to_response( "social/social_story.xhtml", {"story": story}, context_instance=RequestContext(request) ) else: return json.json_response(request, {"code": code, "story": story, "user_profiles": profiles})
def load_river_blurblog(request): limit = 10 start = time.time() user = get_user(request) social_user_ids = [int(uid) for uid in request.REQUEST.getlist("social_user_ids") if uid] original_user_ids = list(social_user_ids) page = int(request.REQUEST.get("page", 1)) order = request.REQUEST.get("order", "newest") read_filter = request.REQUEST.get("read_filter", "unread") relative_user_id = request.REQUEST.get("relative_user_id", None) now = localtime_for_timezone(datetime.datetime.now(), user.profile.timezone) UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD) if not relative_user_id: relative_user_id = get_user(request).pk if not social_user_ids: socialsubs = MSocialSubscription.objects.filter(user_id=user.pk) social_user_ids = [s.subscription_user_id for s in socialsubs] offset = (page - 1) * limit limit = page * limit - 1 story_ids, story_dates = MSocialSubscription.feed_stories( user.pk, social_user_ids, offset=offset, limit=limit, order=order, read_filter=read_filter ) mstories = MStory.objects(id__in=story_ids) story_id_to_dates = dict(zip(story_ids, story_dates)) def sort_stories_by_id(a, b): return int(story_id_to_dates[str(b.id)]) - int(story_id_to_dates[str(a.id)]) sorted_mstories = sorted(mstories, cmp=sort_stories_by_id) stories = Feed.format_stories(sorted_mstories) for s, story in enumerate(stories): story["story_date"] = datetime.datetime.fromtimestamp(story_dates[s]) stories, user_profiles = MSharedStory.stories_with_comments_and_profiles(stories, relative_user_id, check_all=True) story_feed_ids = list(set(s["story_feed_id"] for s in stories)) usersubs = UserSubscription.objects.filter(user__pk=user.pk, feed__pk__in=story_feed_ids) usersubs_map = dict((sub.feed_id, sub) for sub in usersubs) unsub_feed_ids = list(set(story_feed_ids).difference(set(usersubs_map.keys()))) unsub_feeds = Feed.objects.filter(pk__in=unsub_feed_ids) unsub_feeds = [feed.canonical(include_favicon=False) for feed in unsub_feeds] # Find starred stories if story_feed_ids: story_ids = [story["id"] for story in stories] starred_stories = MStarredStory.objects(user_id=user.pk, story_guid__in=story_ids).only( "story_guid", "starred_date" ) starred_stories = dict([(story.story_guid, story.starred_date) for story in starred_stories]) shared_stories = MSharedStory.objects(user_id=user.pk, story_guid__in=story_ids).only( "story_guid", "shared_date", "comments" ) shared_stories = dict( [ (story.story_guid, dict(shared_date=story.shared_date, comments=story.comments)) for story in shared_stories ] ) userstories_db = MUserStory.objects(user_id=user.pk, feed_id__in=story_feed_ids, story_id__in=story_ids).only( "story_id" ) userstories = set(us.story_id for us in userstories_db) else: starred_stories = {} shared_stories = {} userstories = [] # Intelligence classifiers for all feeds involved if story_feed_ids: classifier_feeds = list(MClassifierFeed.objects(user_id=user.pk, feed_id__in=story_feed_ids)) classifier_authors = list(MClassifierAuthor.objects(user_id=user.pk, feed_id__in=story_feed_ids)) classifier_titles = list(MClassifierTitle.objects(user_id=user.pk, feed_id__in=story_feed_ids)) classifier_tags = list(MClassifierTag.objects(user_id=user.pk, feed_id__in=story_feed_ids)) else: classifier_feeds = [] classifier_authors = [] classifier_titles = [] classifier_tags = [] classifiers = sort_classifiers_by_feed( user=user, feed_ids=story_feed_ids, classifier_feeds=classifier_feeds, classifier_authors=classifier_authors, classifier_titles=classifier_titles, classifier_tags=classifier_tags, ) # Just need to format stories for story in stories: if story["id"] in userstories: story["read_status"] = 1 elif story["story_date"] < UNREAD_CUTOFF: story["read_status"] = 1 else: story["read_status"] = 0 story_date = localtime_for_timezone(story["story_date"], user.profile.timezone) story["short_parsed_date"] = format_story_link_date__short(story_date, now) story["long_parsed_date"] = format_story_link_date__long(story_date, now) if story["id"] in starred_stories: story["starred"] = True starred_date = localtime_for_timezone(starred_stories[story["id"]], user.profile.timezone) story["starred_date"] = format_story_link_date__long(starred_date, now) story["intelligence"] = { "feed": apply_classifier_feeds(classifier_feeds, story["story_feed_id"]), "author": apply_classifier_authors(classifier_authors, story), "tags": apply_classifier_tags(classifier_tags, story), "title": apply_classifier_titles(classifier_titles, story), } if story["id"] in shared_stories: story["shared"] = True shared_date = localtime_for_timezone(shared_stories[story["id"]]["shared_date"], user.profile.timezone) story["shared_date"] = format_story_link_date__long(shared_date, now) story["shared_comments"] = strip_tags(shared_stories[story["id"]]["comments"]) diff = time.time() - start timediff = round(float(diff), 2) logging.user( request, "~FYLoading ~FCriver blurblogs stories~FY: ~SBp%s~SN (%s/%s " "stories, ~SN%s/%s/%s feeds)" % (page, len(stories), len(mstories), len(story_feed_ids), len(social_user_ids), len(original_user_ids)), ) return { "stories": stories, "user_profiles": user_profiles, "feeds": unsub_feeds, "classifiers": classifiers, "elapsed_time": timediff, }
def share_story(request, token): code = 0 story_url = request.POST['story_url'] comments = request.POST['comments'] title = request.POST['title'] content = request.POST['content'] rss_url = request.POST.get('rss_url') feed_id = request.POST.get('feed_id') or 0 feed = None message = None if not story_url: code = -1 else: try: profile = Profile.objects.get(secret_token=token) except Profile.DoesNotExist: code = -1 if feed_id: feed = Feed.objects.get(pk=feed_id) else: if rss_url: feed = Feed.get_feed_from_url(rss_url, create=True, fetch=True) if not feed: feed = Feed.get_feed_from_url(story_url, create=True, fetch=True) if feed: feed_id = feed.pk parsed_url = urlparse.urlparse(story_url) base_url = "%s://%s%s" % (parsed_url.scheme, parsed_url.hostname, parsed_url.path) scrubber = SelectiveScriptScrubber(base_url=base_url) content = scrubber.scrub(content) title = strip_tags(title) shared_story = MSharedStory.objects.filter( user_id=profile.user.pk, story_feed_id=feed_id, story_guid=story_url).limit(1).first() if not shared_story: story_db = { "story_guid": story_url, "story_permalink": story_url, "story_title": title, "story_feed_id": feed_id, "story_content": content, "story_date": datetime.datetime.now(), "user_id": profile.user.pk, "comments": comments, "has_comments": bool(comments), } shared_story = MSharedStory.objects.create(**story_db) socialsubs = MSocialSubscription.objects.filter( subscription_user_id=profile.user.pk) for socialsub in socialsubs: socialsub.needs_unread_recalc = True socialsub.save() logging.user( profile.user, "~BM~FYSharing story from site: ~SB%s: %s" % (story_url, comments)) else: shared_story.story_content = content shared_story.story_title = title shared_story.comments = comments shared_story.story_permalink = story_url shared_story.story_guid = story_url shared_story.has_comments = bool(comments) shared_story.story_feed_id = feed_id shared_story.save() logging.user( profile.user, "~BM~FY~SBUpdating~SN shared story from site: ~SB%s: %s" % (story_url, comments)) shared_story.publish_update_to_subscribers() response = HttpResponse(json.encode({ 'code': code, 'message': message, 'story': None, }), mimetype='text/plain') response['Access-Control-Allow-Origin'] = '*' response['Access-Control-Allow-Methods'] = 'POST' return response
def share_story(request, token): code = 0 story_url = request.POST['story_url'] comments = request.POST['comments'] title = request.POST['title'] content = request.POST['content'] rss_url = request.POST.get('rss_url') feed_id = request.POST.get('feed_id') or 0 feed = None message = None if not story_url: code = -1 else: try: profile = Profile.objects.get(secret_token=token) except Profile.DoesNotExist: code = -1 if feed_id: feed = Feed.objects.get(pk=feed_id) else: if rss_url: feed = Feed.get_feed_from_url(rss_url, create=True, fetch=True) if not feed: feed = Feed.get_feed_from_url(story_url, create=True, fetch=True) if feed: feed_id = feed.pk parsed_url = urlparse.urlparse(story_url) base_url = "%s://%s%s" % (parsed_url.scheme, parsed_url.hostname, parsed_url.path) scrubber = SelectiveScriptScrubber(base_url=base_url) content = scrubber.scrub(content) title = strip_tags(title) shared_story = MSharedStory.objects.filter(user_id=profile.user.pk, story_feed_id=feed_id, story_guid=story_url).limit(1).first() if not shared_story: story_db = { "story_guid": story_url, "story_permalink": story_url, "story_title": title, "story_feed_id": feed_id, "story_content": content, "story_date": datetime.datetime.now(), "user_id": profile.user.pk, "comments": comments, "has_comments": bool(comments), } shared_story = MSharedStory.objects.create(**story_db) socialsubs = MSocialSubscription.objects.filter(subscription_user_id=profile.user.pk) for socialsub in socialsubs: socialsub.needs_unread_recalc = True socialsub.save() logging.user(profile.user, "~BM~FYSharing story from site: ~SB%s: %s" % (story_url, comments)) else: shared_story.story_content = content shared_story.story_title = title shared_story.comments = comments shared_story.story_permalink = story_url shared_story.story_guid = story_url shared_story.has_comments = bool(comments) shared_story.story_feed_id = feed_id shared_story.save() logging.user(profile.user, "~BM~FY~SBUpdating~SN shared story from site: ~SB%s: %s" % (story_url, comments)) shared_story.publish_update_to_subscribers() response = HttpResponse(json.encode({ 'code': code, 'message': message, 'story': None, }), mimetype='text/plain') response['Access-Control-Allow-Origin'] = '*' response['Access-Control-Allow-Methods'] = 'POST' return response