def fetch_single_feed(blog, callback_filter=None):
num_with_tags = 0
from nebula.models import AggregatedBlog, AggregatedPost
logging.debug('Fetching feed %s from blog %r' % (blog.feed, blog.name))
assert isinstance(blog, AggregatedBlog)
FeedPostClass = getattr(blog, '_post_class', AggregatedPost)
assert issubclass(FeedPostClass, AggregatedPost), "blog._post_class is %s class instead of subclass of AggregatedBlog" % FeedPostClass.__name__
if not blog.feed:
logging.info('Blog %r have no feed' % blog.name)
return
try:
d = feedparser.parse(blog.feed, agent=USER_AGENT, etag=blog.etag)
except Exception, e:
logging.error("Fail to fetch feed %s from blog %r: %s" % (blog.feed, blog.name, e))
def fetch_single_feed(blog, callback_filter=None):
num_with_tags = 0
from nebula.models import AggregatedBlog, AggregatedPost
logging.debug('Fetching feed %s from blog %r' % (blog.feed, blog.name))
assert isinstance(blog, AggregatedBlog)
FeedPostClass = getattr(blog, '_post_class', AggregatedPost)
assert issubclass(FeedPostClass, AggregatedPost), "blog._post_class is %s class instead of subclass of AggregatedBlog" % FeedPostClass.__name__
if not blog.feed:
logging.info('Blog %r have no feed' % blog.name)
return
try:
d = feedparser.parse(blog.feed, agent=USER_AGENT, etag=blog.etag)
except Exception, e:
logging.error("Fail to fetch feed %s from blog %r: %s" % (blog.feed, blog.name, e))
FeedPostClass = getattr(blog, '_post_class', AggregatedPost)
assert issubclass(FeedPostClass, AggregatedPost), "blog._post_class is %s class instead of subclass of AggregatedBlog" % FeedPostClass.__name__
if not blog.feed:
logging.info('Blog %r have no feed' % blog.name)
return
try:
d = feedparser.parse(blog.feed, agent=USER_AGENT, etag=blog.etag)
except Exception, e:
logging.error("Fail to fetch feed %s from blog %r: %s" % (blog.feed, blog.name, e))
status = d.get('status')
if status:
if status == 304:
logging.debug('Feed %s has not changed since our last attempt' % blog.feed)
elif status >= 400:
logging.error('HTTP error while trying to grab the feed %s: %s' % (blog.feed, status))
return
blog.etag = d.get('etag') or ''
for entry in d.entries:
created = False
active = True
guid = entry.get('guid', entry.get('link'))
if not guid:
logging.warning('Entry %r from feed have %s no guid' % (entry.title, blog.feed))
continue
try:
existing_post = FeedPostClass.objects.get(guid__iexact=guid)
continue
FeedPostClass = getattr(blog, '_post_class', AggregatedPost)
assert issubclass(FeedPostClass, AggregatedPost), "blog._post_class is %s class instead of subclass of AggregatedBlog" % FeedPostClass.__name__
if not blog.feed:
logging.info('Blog %r have no feed' % blog.name)
return
try:
d = feedparser.parse(blog.feed, agent=USER_AGENT, etag=blog.etag)
except Exception, e:
logging.error("Fail to fetch feed %s from blog %r: %s" % (blog.feed, blog.name, e))
status = d.get('status')
if status:
if status == 304:
logging.debug('Feed %s has not changed since our last attempt' % blog.feed)
elif status >= 400:
logging.error('HTTP error while trying to grab the feed %s: %s' % (blog.feed, status))
return
blog.etag = d.get('etag') or ''
for entry in d.entries:
created = False
active = True
guid = entry.get('guid', entry.get('link'))
if not guid:
logging.warning('Entry %r from feed have %s no guid' % (entry.title, blog.feed))
continue
try:
existing_post = FeedPostClass.objects.get(guid__iexact=guid)
continue