def fetch(self): import BlogPost # don't fetch internally hosted blogs if not self.from_feed: return events = [] # parse and iterate the feed entries = feedparser.parse(self.rss).entries for post in entries: try: date = dateutil.parser.parse(post.date).replace(tzinfo=None) except: date = datetime.datetime.utcnow() # don't re-add old posts if self.most_recent_date >= date: continue try: content = post.content[0].value except: content = post.description try: author_name = post.author_detail["name"] except: author_name = None # sanitize the post's content content = sanitize(content, [ "h1", "h2", "h3", "h4", "h5", "h6", "a:href", "p", "ul", "ol", "li", "br", "div", "img:src:alt:title", "b", "i", "u", "strong", "em", "table", "tbody", "td", "th", "thead", "tfoot", "pre", "tt", "code" ]) # format a summary for the post summary = sanitize(content, [], strip_tags=[ "h1", "h2", "h3", "h4", "h5", "h6", "p", "ul", "ol", "li", "br", "div", 'a', "b", "i", "u", "strong", "em", "pre", "tt", "code" ]) if len(summary) > 500: summary = summary[0:500] + u"..." summary = "<p>" + summary + "</p>" events.append( self.add_event(BlogPost.BlogPost, title=post.title, summary=summary, from_feed=True, author_name=author_name, date=date, extra_args={ "external_link": post.link, "content": content, "blog_id": self.id })) # find the new most recent date dates = [event.date for event in events if event is not None] dates.append(self.most_recent_date) self.most_recent_date = max(dates) self.save()
def fetch(self): import BlogPost # don't fetch internally hosted blogs if not self.from_feed: return events = [] # parse and iterate the feed entries = feedparser.parse(self.rss).entries for post in entries: try: date = dateutil.parser.parse(post.date).replace(tzinfo=None) except: date = datetime.datetime.utcnow() # don't re-add old posts if self.most_recent_date >= date: continue try: content = post.content[0].value except: content = post.description try: author_name = post.author_detail["name"] except: author_name = None # sanitize the post's content content = sanitize(content, [ "h1", "h2", "h3", "h4", "h5", "h6", "a:href", "p", "ul", "ol", "li", "br", "div", "img:src:alt:title", "b", "i", "u", "strong", "em", "table", "tbody", "td", "th", "thead", "tfoot", "pre", "tt", "code" ]) # format a summary for the post summary = sanitize(content, [], strip_tags = [ "h1", "h2", "h3", "h4", "h5", "h6", "p", "ul", "ol", "li", "br", "div", 'a', "b", "i", "u", "strong", "em", "pre", "tt", "code" ]) if len(summary) > 500: summary = summary[0:500] + u"..." summary = "<p>" + summary + "</p>" events.append(self.add_event(BlogPost.BlogPost, title = post.title, summary = summary, from_feed = True, author_name = author_name, date = date, extra_args = { "external_link": post.link, "content": content, "blog_id": self.id } )) # find the new most recent date dates = [event.date for event in events if event is not None] dates.append(self.most_recent_date) self.most_recent_date = max(dates) self.save()
def add_event(self, klass, title = None, summary = None, date = None, author_name = None, from_feed = None, append_unsanitized = "", extra_args = {}): # convert to UTC secs = time.mktime(date.timetuple()) date = datetime.datetime.utcfromtimestamp(secs) # don't re-add old events if self.most_recent_date >= date: return # can we find an author for this event? from dashboard.models import Blog if self.__class__ is not Blog or self.user is None: if author_name is not None: author, author_name, author_email = find_author(author_name) else: author = None author_email = None else: author = self.user author_email = None # sanitize the summary summary = append_unsanitized + sanitize(summary, [ "h1", "h2", "h3", "h4", "h5", "h6", "a:href", "p", "ul", "ol", "li", "br", "b", "i", "u", "strong", "em", "div", "pre", "tt", "code" ]) # create and save the event object event = klass(author_name = author_name, title = title, summary = summary, from_feed = from_feed, date = date, author_email = author_email, **extra_args) if author is not None: event.author = author event.save() # if this is a personal blog, we're all done if self.__class__ == Blog: if self.user is not None: print "Personal blog found by {0}".format(self.user.get_full_name()) return event # set the project event.project = self.project event.save() # add a contributor for the author if they are not a project author if author is not None or author_name is not None: from Contributor import Contributor cont = None # if there is no "author", create a contributor associated with a # name and an email. this can be upgraded to a User if that person # joins observatory. if author is None: try: if author_email is not None: cont = Contributor.objects.get(email = author_email) else: raise Contributor.DoesNotExist() except Contributor.DoesNotExist: try: cont = Contributor.objects.get(name = author_name) except Contributor.DoesNotExist: cont = Contributor(name = author_name, email = author_email) # otherwise, associate with the user model else: try: cont = Contributor.objects.get(user = author) except Contributor.DoesNotExist: cont = Contributor(user = author) # save the contributor and add it to the project cont.save() cont.projects.add(event.project) # print out results print "{0} by {1}{2} in {3} at {4}".format( klass.__name__, author_name, " (found)" if author != None else "", self.project.title, date ) return event
def add_event(self, klass, title=None, summary=None, date=None, author_name=None, from_feed=None, append_unsanitized="", extra_args={}): # convert to UTC secs = time.mktime(date.timetuple()) date = datetime.datetime.utcfromtimestamp(secs) # don't re-add old events if self.most_recent_date >= date: return # can we find an author for this event? from dashboard.models import Blog if self.__class__ is not Blog or self.user is None: if author_name is not None: author, author_name, author_email = find_author(author_name) else: author = None author_email = None else: author = self.user author_email = None # sanitize the summary summary = append_unsanitized + sanitize(summary, [ "h1", "h2", "h3", "h4", "h5", "h6", "a:href", "p", "ul", "ol", "li", "br", "b", "i", "u", "strong", "em", "div", "pre", "tt", "code" ]) # create and save the event object event = klass(author_name=author_name, title=title, summary=summary, from_feed=from_feed, date=date, author_email=author_email, **extra_args) if author is not None: event.author = author event.save() # if this is a personal blog, we're all done if self.__class__ == Blog: if self.user is not None: print "Personal blog found by {0}".format( self.user.get_full_name()) return event # set the project event.project = self.project event.save() # add a contributor for the author if they are not a project author if author is not None or author_name is not None: from Contributor import Contributor cont = None # if there is no "author", create a contributor associated with a # name and an email. this can be upgraded to a User if that person # joins observatory. if author is None: try: if author_email is not None: cont = Contributor.objects.get(email=author_email) else: raise Contributor.DoesNotExist() except Contributor.DoesNotExist: try: cont = Contributor.objects.get(name=author_name) except Contributor.DoesNotExist: cont = Contributor(name=author_name, email=author_email) # otherwise, associate with the user model else: try: cont = Contributor.objects.get(user=author) except Contributor.DoesNotExist: cont = Contributor(user=author) # save the contributor and add it to the project cont.save() cont.projects.add(event.project) # print out results print "{0} by {1}{2} in {3} at {4}".format( klass.__name__, author_name, " (found)" if author != None else "", self.project.title, date) return event