def process_item(self, item, folders): feed_title = item.xpath('./string[@name="title"]') and \ item.xpath('./string[@name="title"]')[0].text feed_address = item.xpath('./string[@name="id"]') and \ item.xpath('./string[@name="id"]')[0].text.replace('feed/', '') feed_link = item.xpath('./string[@name="htmlUrl"]') and \ item.xpath('./string[@name="htmlUrl"]')[0].text category = item.xpath('./list[@name="categories"]/object/string[@name="label"]') and \ item.xpath('./list[@name="categories"]/object/string[@name="label"]')[0].text if not feed_address: feed_address = feed_link try: feed_link = urlnorm.normalize(feed_link) feed_address = urlnorm.normalize(feed_address) if len(feed_address) > Feed._meta.get_field( 'feed_address').max_length: return folders # See if it exists as a duplicate first duplicate_feed = DuplicateFeed.objects.filter( duplicate_address=feed_address) if duplicate_feed: feed_db = duplicate_feed[0].feed else: feed_data = dict(feed_title=feed_title) feed_data['active_subscribers'] = 1 feed_data['num_subscribers'] = 1 feed_db, _ = Feed.find_or_create(feed_address=feed_address, feed_link=feed_link, defaults=dict(**feed_data)) us, _ = UserSubscription.objects.get_or_create( feed=feed_db, user=self.user, defaults={ 'needs_unread_recalc': True, 'mark_read_date': datetime.datetime.utcnow() - datetime.timedelta(days=1), 'active': self.user.profile.is_premium or self.auto_active, }) if not us.needs_unread_recalc: us.needs_unread_recalc = True us.save() if not category: category = "" if category: obj = {category: []} folders = add_object_to_folder(obj, '', folders) folders = add_object_to_folder(feed_db.pk, category, folders) # if feed_db.pk not in folders[category]: # folders[category].append(feed_db.pk) except Exception, e: logging.info(' *** -> Exception: %s: %s' % (e, item))
def process_item(self, item, folders): feed_title = item.xpath('./string[@name="title"]') and item.xpath('./string[@name="title"]')[0].text feed_address = item.xpath('./string[@name="id"]') and item.xpath('./string[@name="id"]')[0].text.replace( "feed/", "" ) feed_link = item.xpath('./string[@name="htmlUrl"]') and item.xpath('./string[@name="htmlUrl"]')[0].text category = ( item.xpath('./list[@name="categories"]/object/string[@name="label"]') and item.xpath('./list[@name="categories"]/object/string[@name="label"]')[0].text ) if not feed_address: feed_address = feed_link try: feed_link = urlnorm.normalize(feed_link) feed_address = urlnorm.normalize(feed_address) if len(feed_address) > Feed._meta.get_field("feed_address").max_length: return folders # See if it exists as a duplicate first duplicate_feed = DuplicateFeed.objects.filter(duplicate_address=feed_address) if duplicate_feed: feed_db = duplicate_feed[0].feed else: feed_data = dict(feed_title=feed_title) feed_data["active_subscribers"] = 1 feed_data["num_subscribers"] = 1 feed_db, _ = Feed.find_or_create( feed_address=feed_address, feed_link=feed_link, defaults=dict(**feed_data) ) us, _ = UserSubscription.objects.get_or_create( feed=feed_db, user=self.user, defaults={ "needs_unread_recalc": True, "mark_read_date": datetime.datetime.utcnow() - datetime.timedelta(days=1), "active": self.user.profile.is_premium or self.auto_active, }, ) if not us.needs_unread_recalc: us.needs_unread_recalc = True us.save() if not category: category = "" if category: obj = {category: []} folders = add_object_to_folder(obj, "", folders) folders = add_object_to_folder(feed_db.pk, category, folders) # if feed_db.pk not in folders[category]: # folders[category].append(feed_db.pk) except Exception, e: logging.info(" *** -> Exception: %s: %s" % (e, item))
def add_missing_feeds(self): all_feeds = self.flat() subs = [us.feed_id for us in UserSubscription.objects.filter(user=self.user).only('feed')] missing_subs = set(all_feeds) - set(subs) if missing_subs: logging.debug(" ---> %s is missing %s subs. Adding %s..." % ( self.user, len(missing_subs), missing_subs)) for feed_id in missing_subs: feed = Feed.get_by_id(feed_id) if feed: us, _ = UserSubscription.objects.get_or_create(user=self.user, feed=feed, defaults={ 'needs_unread_recalc': True }) if not us.needs_unread_recalc: us.needs_unread_recalc = True us.save() missing_folder_feeds = set(subs) - set(all_feeds) if missing_folder_feeds: user_sub_folders = json.decode(self.folders) logging.debug(" ---> %s is missing %s folder feeds. Adding %s..." % ( self.user, len(missing_folder_feeds), missing_folder_feeds)) for feed_id in missing_folder_feeds: feed = Feed.get_by_id(feed_id) if feed and feed.pk == feed_id: user_sub_folders = add_object_to_folder(feed_id, "", user_sub_folders) self.folders = json.encode(user_sub_folders) self.save()
def add_subscription(cls, user, feed_address, folder=None, bookmarklet=False, auto_active=True, skip_fetch=False): feed = None us = None logging.user( user, "~FRAdding URL: ~SB%s (in %s) %s" % (feed_address, folder, "~FCAUTO-ADD" if not auto_active else "")) feed = Feed.get_feed_from_url(feed_address) if not feed: code = -1 if bookmarklet: message = "This site does not have an RSS feed. Nothing is linked to from this page." else: message = "This address does not point to an RSS feed or a website with an RSS feed." else: us, subscription_created = cls.objects.get_or_create( feed=feed, user=user, defaults={ 'needs_unread_recalc': True, 'active': auto_active, }) code = 1 message = "" if us: user_sub_folders_object, created = UserSubscriptionFolders.objects.get_or_create( user=user, defaults={'folders': '[]'}) if created: user_sub_folders = [] else: user_sub_folders = json.decode(user_sub_folders_object.folders) user_sub_folders = add_object_to_folder(feed.pk, folder, user_sub_folders) user_sub_folders_object.folders = json.encode(user_sub_folders) user_sub_folders_object.save() if auto_active or user.profile.is_premium: us.active = True us.save() if not skip_fetch and feed.last_update < datetime.datetime.utcnow( ) - datetime.timedelta(days=1): feed = feed.update() from apps.social.models import MActivity MActivity.new_feed_subscription(user_id=user.pk, feed_id=feed.pk, feed_title=feed.title) feed.setup_feed_for_premium_subscribers() return code, message, us
def add_folder(self, parent_folder, folder): if self.folders: user_sub_folders = json.decode(self.folders) else: user_sub_folders = [] obj = {folder: []} user_sub_folders = add_object_to_folder(obj, parent_folder, user_sub_folders) self.folders = json.encode(user_sub_folders) self.save()
def move_folder_to_folder(self, folder_name, in_folder=None, to_folder=None): user_sub_folders = json.decode(self.folders) deleted_folder = self.delete_folder(folder_name, in_folder, [], commit_delete=False) user_sub_folders = json.decode(self.folders) user_sub_folders = add_object_to_folder(deleted_folder, to_folder, user_sub_folders) self.folders = json.encode(user_sub_folders) self.save() return self
def move_feed_to_folder(self, feed_id, in_folder=None, to_folder=None): user_sub_folders = json.decode(self.folders) self.delete_feed(feed_id, in_folder, commit_delete=False) user_sub_folders = json.decode(self.folders) user_sub_folders = add_object_to_folder(int(feed_id), to_folder, user_sub_folders) self.folders = json.encode(user_sub_folders) self.save() return self
def move_folder_to_folder(self, folder_name, in_folder=None, to_folder=None): logging.user(self.user, "~FBMoving folder '~SB%s~SN' in '%s' to: ~SB%s" % (folder_name, in_folder, to_folder)) user_sub_folders = json.decode(self.folders) deleted_folder = self.delete_folder(folder_name, in_folder, [], commit_delete=False) user_sub_folders = json.decode(self.folders) user_sub_folders = add_object_to_folder(deleted_folder, to_folder, user_sub_folders) self.folders = json.encode(user_sub_folders) self.save() return self
def move_feed_to_folder(self, feed_id, in_folder=None, to_folder=None): logging.user(self.user, "~FBMoving feed '~SB%s~SN' in '%s' to: ~SB%s" % (feed_id, in_folder, to_folder)) user_sub_folders = json.decode(self.folders) self.delete_feed(feed_id, in_folder, commit_delete=False) user_sub_folders = json.decode(self.folders) user_sub_folders = add_object_to_folder(int(feed_id), to_folder, user_sub_folders) self.folders = json.encode(user_sub_folders) self.save() return self
def move_feed_to_folder(self, feed_id, in_folder=None, to_folder=None): logging.user(self.user, "~FBMoving feed '~SB%s~SN' in '%s' to: ~SB%s" % ( feed_id, in_folder, to_folder)) user_sub_folders = json.decode(self.folders) self.delete_feed(feed_id, in_folder, commit_delete=False) user_sub_folders = json.decode(self.folders) user_sub_folders = add_object_to_folder(int(feed_id), to_folder, user_sub_folders) self.folders = json.encode(user_sub_folders) self.save() return self
def move_folder_to_folder(self, folder_name, in_folder=None, to_folder=None): logging.user(self.user, "~FBMoving folder '~SB%s~SN' in '%s' to: ~SB%s" % ( folder_name, in_folder, to_folder)) user_sub_folders = json.decode(self.folders) deleted_folder = self.delete_folder(folder_name, in_folder, [], commit_delete=False) user_sub_folders = json.decode(self.folders) user_sub_folders = add_object_to_folder(deleted_folder, to_folder, user_sub_folders) self.folders = json.encode(user_sub_folders) self.save() return self
def add_subscription(cls, user, feed_address, folder=None, bookmarklet=False, auto_active=True, skip_fetch=False): feed = None us = None logging.user(user, "~FRAdding URL: ~SB%s (in %s) %s" % (feed_address, folder, "~FCAUTO-ADD" if not auto_active else "")) feed = Feed.get_feed_from_url(feed_address) if not feed: code = -1 if bookmarklet: message = "This site does not have an RSS feed. Nothing is linked to from this page." else: message = "This address does not point to an RSS feed or a website with an RSS feed." else: us, subscription_created = cls.objects.get_or_create( feed=feed, user=user, defaults={ 'needs_unread_recalc': True, 'active': auto_active, } ) code = 1 message = "" if us: user_sub_folders_object, created = UserSubscriptionFolders.objects.get_or_create( user=user, defaults={'folders': '[]'} ) if created: user_sub_folders = [] else: user_sub_folders = json.decode(user_sub_folders_object.folders) user_sub_folders = add_object_to_folder(feed.pk, folder, user_sub_folders) user_sub_folders_object.folders = json.encode(user_sub_folders) user_sub_folders_object.save() if auto_active or user.profile.is_premium: us.active = True us.save() if not skip_fetch and feed.last_update < datetime.datetime.utcnow() - datetime.timedelta(days=1): feed = feed.update() from apps.social.models import MActivity MActivity.new_feed_subscription(user_id=user.pk, feed_id=feed.pk, feed_title=feed.title) feed.setup_feed_for_premium_subscribers() return code, message, us
def add_subscription(cls, user, feed_address, folder=None, bookmarklet=False): feed = None us = None logging.user(user, "~FRAdding URL: ~SB%s (in %s)" % (feed_address, folder)) feed = Feed.get_feed_from_url(feed_address) if not feed: code = -1 if bookmarklet: message = "This site does not have an RSS feed. Nothing is linked to from this page." else: message = "This address does not point to an RSS feed or a website with an RSS feed." else: us, subscription_created = cls.objects.get_or_create( feed=feed, user=user, defaults={ 'needs_unread_recalc': True, 'active': True, }) code = 1 message = "" if us: user_sub_folders_object, created = UserSubscriptionFolders.objects.get_or_create( user=user, defaults={'folders': '[]'}) if created: user_sub_folders = [] else: user_sub_folders = json.decode(user_sub_folders_object.folders) user_sub_folders = add_object_to_folder(feed.pk, folder, user_sub_folders) user_sub_folders_object.folders = json.encode(user_sub_folders) user_sub_folders_object.save() feed.setup_feed_for_premium_subscribers() if feed.last_update < datetime.datetime.utcnow( ) - datetime.timedelta(days=1): feed.update() return code, message, us
def subscribe(cls, user_id, category_title): category = cls.objects.get(title=category_title) for feed_id in category.feed_ids: us, _ = UserSubscription.objects.get_or_create( feed_id=feed_id, user_id=user_id, defaults={"needs_unread_recalc": True, "active": True} ) usf, created = UserSubscriptionFolders.objects.get_or_create(user_id=user_id, defaults={"folders": "[]"}) usf.add_folder("", category.title) folders = json.decode(usf.folders) for feed_id in category.feed_ids: folders = add_object_to_folder(feed_id, category.title, folders) usf.folders = json.encode(folders) usf.save()
def add_subscription(cls, user, feed_address, folder=None, bookmarklet=False): feed = None us = None logging.user(user, "~FRAdding URL: ~SB%s (in %s)" % (feed_address, folder)) feed = Feed.get_feed_from_url(feed_address) if not feed: code = -1 if bookmarklet: message = "This site does not have an RSS feed. Nothing is linked to from this page." else: message = "This address does not point to an RSS feed or a website with an RSS feed." else: us, subscription_created = cls.objects.get_or_create( feed=feed, user=user, defaults={ 'needs_unread_recalc': True, 'active': True, } ) code = 1 message = "" if us: user_sub_folders_object, created = UserSubscriptionFolders.objects.get_or_create( user=user, defaults={'folders': '[]'} ) if created: user_sub_folders = [] else: user_sub_folders = json.decode(user_sub_folders_object.folders) user_sub_folders = add_object_to_folder(feed.pk, folder, user_sub_folders) user_sub_folders_object.folders = json.encode(user_sub_folders) user_sub_folders_object.save() feed.setup_feed_for_premium_subscribers() if feed.last_update < datetime.datetime.utcnow() - datetime.timedelta(days=1): feed.update() return code, message, us
def subscribe(cls, user_id, category_title): category = cls.objects.get(title=category_title) for feed_id in category.feed_ids: us, _ = UserSubscription.objects.get_or_create( feed_id=feed_id, user_id=user_id, defaults={ 'needs_unread_recalc': True, 'active': True, }) usf, created = UserSubscriptionFolders.objects.get_or_create( user_id=user_id, defaults={'folders': '[]'}) usf.add_folder('', category.title) folders = json.decode(usf.folders) for feed_id in category.feed_ids: folders = add_object_to_folder(feed_id, category.title, folders) usf.folders = json.encode(folders) usf.save()
def add_folder(request): folder = request.POST['folder'] parent_folder = request.POST['parent_folder'] logging.info(" ---> [%s] ~FRAdding Folder: ~SB%s (in %s)" % (request.user, folder, parent_folder)) if folder: code = 1 message = "" user_sub_folders_object, _ = UserSubscriptionFolders.objects.get_or_create(user=request.user) if user_sub_folders_object.folders: user_sub_folders = json.decode(user_sub_folders_object.folders) else: user_sub_folders = [] obj = {folder: []} user_sub_folders = add_object_to_folder(obj, parent_folder, user_sub_folders) user_sub_folders_object.folders = json.encode(user_sub_folders) user_sub_folders_object.save() else: code = -1 message = "Gotta write in a folder name." return dict(code=code, message=message)
def process_outline(self, outline, folders, in_folder=""): for item in outline: if not hasattr(item, "xmlUrl") and (hasattr(item, "text") or hasattr(item, "title")): folder = item title = getattr(item, "text", None) or getattr(item, "title", None) # if hasattr(folder, 'text'): # logging.info(' ---> [%s] ~FRNew Folder: %s' % (self.user, folder.text)) obj = {title: []} folders = add_object_to_folder(obj, in_folder, folders) folders = self.process_outline(folder, folders, title) elif hasattr(item, "xmlUrl"): feed = item if not hasattr(feed, "htmlUrl"): setattr(feed, "htmlUrl", None) # If feed title matches what's in the DB, don't override it on subscription. feed_title = getattr(feed, "title", None) or getattr(feed, "text", None) if not feed_title: setattr(feed, "title", feed.htmlUrl or feed.xmlUrl) user_feed_title = None else: setattr(feed, "title", feed_title) user_feed_title = feed.title feed_address = urlnorm.normalize(feed.xmlUrl) feed_link = urlnorm.normalize(feed.htmlUrl) if len(feed_address) > Feed._meta.get_field("feed_address").max_length: continue if feed_link and len(feed_link) > Feed._meta.get_field("feed_link").max_length: continue # logging.info(' ---> \t~FR%s - %s - %s' % (feed.title, feed_link, feed_address,)) feed_data = dict(feed_address=feed_address, feed_link=feed_link, feed_title=feed.title) # feeds.append(feed_data) # See if it exists as a duplicate first duplicate_feed = DuplicateFeed.objects.filter(duplicate_address=feed_address) if duplicate_feed: feed_db = duplicate_feed[0].feed else: feed_data["active_subscribers"] = 1 feed_data["num_subscribers"] = 1 feed_db, _ = Feed.find_or_create( feed_address=feed_address, feed_link=feed_link, defaults=dict(**feed_data) ) if user_feed_title == feed_db.feed_title: user_feed_title = None us, _ = UserSubscription.objects.get_or_create( feed=feed_db, user=self.user, defaults={ "needs_unread_recalc": True, "mark_read_date": datetime.datetime.utcnow() - datetime.timedelta(days=1), "active": self.user.profile.is_premium, "user_title": user_feed_title, }, ) if self.user.profile.is_premium and not us.active: us.active = True us.save() if not us.needs_unread_recalc: us.needs_unread_recalc = True us.save() folders = add_object_to_folder(feed_db.pk, in_folder, folders) return folders
def process_outline(self, outline, folders, in_folder=''): for item in outline: if (not hasattr(item, 'xmlUrl') and (hasattr(item, 'text') or hasattr(item, 'title'))): folder = item title = getattr(item, 'text', None) or getattr(item, 'title', None) # if hasattr(folder, 'text'): # logging.info(' ---> [%s] ~FRNew Folder: %s' % (self.user, folder.text)) obj = {title: []} folders = add_object_to_folder(obj, in_folder, folders) folders = self.process_outline(folder, folders, title) elif hasattr(item, 'xmlUrl'): feed = item if not hasattr(feed, 'htmlUrl'): setattr(feed, 'htmlUrl', None) # If feed title matches what's in the DB, don't override it on subscription. feed_title = getattr(feed, 'title', None) or getattr(feed, 'text', None) if not feed_title: setattr(feed, 'title', feed.htmlUrl or feed.xmlUrl) user_feed_title = None else: setattr(feed, 'title', feed_title) user_feed_title = feed.title feed_address = urlnorm.normalize(feed.xmlUrl) feed_link = urlnorm.normalize(feed.htmlUrl) if len(feed_address) > Feed._meta.get_field('feed_address').max_length: continue if feed_link and len(feed_link) > Feed._meta.get_field('feed_link').max_length: continue # logging.info(' ---> \t~FR%s - %s - %s' % (feed.title, feed_link, feed_address,)) feed_data = dict(feed_address=feed_address, feed_link=feed_link, feed_title=feed.title) # feeds.append(feed_data) # See if it exists as a duplicate first duplicate_feed = DuplicateFeed.objects.filter(duplicate_address=feed_address) if duplicate_feed: feed_db = duplicate_feed[0].feed else: feed_data['active_subscribers'] = 1 feed_data['num_subscribers'] = 1 feed_db, _ = Feed.find_or_create(feed_address=feed_address, feed_link=feed_link, defaults=dict(**feed_data)) if user_feed_title == feed_db.feed_title: user_feed_title = None us, _ = UserSubscription.objects.get_or_create( feed=feed_db, user=self.user, defaults={ 'needs_unread_recalc': True, 'mark_read_date': datetime.datetime.utcnow() - datetime.timedelta(days=1), 'active': self.user.profile.is_premium, 'user_title': user_feed_title } ) if self.user.profile.is_premium and not us.active: us.active = True us.save() if not us.needs_unread_recalc: us.needs_unread_recalc = True us.save() folders = add_object_to_folder(feed_db.pk, in_folder, folders) return folders
def add_subscription(cls, user, feed_address, folder=None, bookmarklet=False): feed = None us = None logging.info(" ---> [%s] ~FRAdding URL: ~SB%s (in %s)" % (user, feed_address, folder)) if feed_address: feed_address = urlnorm.normalize(feed_address) # See if it exists as a duplicate first duplicate_feed = DuplicateFeed.objects.filter( duplicate_address=feed_address).order_by('pk') if duplicate_feed: feed = [duplicate_feed[0].feed] else: feed = Feed.objects.filter( feed_address=feed_address).order_by('pk') if feed: feed = feed[0] else: try: feed = fetch_address_from_page(feed_address) except: code = -2 message = "This feed has been added, but something went wrong"\ " when downloading it. Maybe the server's busy." if not feed: code = -1 if bookmarklet: message = "This site does not have an RSS feed. Nothing is linked to from this page." else: message = "This site does not point to an RSS feed or a website with an RSS feed." else: us, subscription_created = cls.objects.get_or_create( feed=feed, user=user, defaults={ 'needs_unread_recalc': True, 'active': True, }) code = 1 message = "" if us and not subscription_created: code = -3 message = "You are already subscribed to this site." elif us: user_sub_folders_object, created = UserSubscriptionFolders.objects.get_or_create( user=user, defaults={'folders': '[]'}) if created: user_sub_folders = [] else: user_sub_folders = json.decode(user_sub_folders_object.folders) user_sub_folders = add_object_to_folder(feed.pk, folder, user_sub_folders) user_sub_folders_object.folders = json.encode(user_sub_folders) user_sub_folders_object.save() feed.setup_feed_for_premium_subscribers() if feed.last_update < datetime.datetime.utcnow( ) - datetime.timedelta(days=1): feed.update() print code, message, us return code, message, us
def add_subscription(cls, user, feed_address, folder=None, bookmarklet=False): feed = None us = None logging.info(" ---> [%s] ~FRAdding URL: ~SB%s (in %s)" % (user, feed_address, folder)) if feed_address: feed_address = urlnorm.normalize(feed_address) # See if it exists as a duplicate first duplicate_feed = DuplicateFeed.objects.filter(duplicate_address=feed_address).order_by('pk') if duplicate_feed: feed = [duplicate_feed[0].feed] else: feed = Feed.objects.filter(feed_address=feed_address).order_by('pk') if feed: feed = feed[0] else: try: feed = fetch_address_from_page(feed_address) except: code = -2 message = "This feed has been added, but something went wrong"\ " when downloading it. Maybe the server's busy." if not feed: code = -1 if bookmarklet: message = "This site does not have an RSS feed. Nothing is linked to from this page." else: message = "This site does not point to an RSS feed or a website with an RSS feed." else: us, subscription_created = cls.objects.get_or_create( feed=feed, user=user, defaults={ 'needs_unread_recalc': True, 'active': True, } ) code = 1 message = "" if us and not subscription_created: code = -3 message = "You are already subscribed to this site." elif us: user_sub_folders_object, created = UserSubscriptionFolders.objects.get_or_create( user=user, defaults={'folders': '[]'} ) if created: user_sub_folders = [] else: user_sub_folders = json.decode(user_sub_folders_object.folders) user_sub_folders = add_object_to_folder(feed.pk, folder, user_sub_folders) user_sub_folders_object.folders = json.encode(user_sub_folders) user_sub_folders_object.save() feed.setup_feed_for_premium_subscribers() if feed.last_update < datetime.datetime.utcnow() - datetime.timedelta(days=1): feed.update() print code, message, us return code, message, us