def handle(self, *args, **options): for feed in Feed.objects.all(): try: source = feedparser.parse(feed.url) except Exception as e: logger.debug('Failed to fetch feed {} because'.format(feed.name, str(e))) for entry in source.entries: try: created_date = entry.published if entry.published else entry.created # if none of these criteria applies what the f**k fix your invalid garbage rss feed created_date = parse(created_date) except: created_date = timezone.now() try: last_updated = parse(entry.updated) except: last_updated = timezone.now() try: created = False item = Item.objects.get(feed=feed, url=entry.link) if item.last_updated < last_updated: item.content = escape(entry.summary_detail.value) item.last_updated = last_updated item.save() except Item.DoesNotExist: created = True item = Item(feed=feed, url=entry.link, created=created_date, last_updated=last_updated, title=escape(entry.title), content=escape(entry.summary_detail.value)) item.save() logger.debug('{} item with link {} on {}'.format('Created' if created else 'Updated', item.url, item.last_updated))
def test_saving_and_retrieving_items(self): list_ = List() list_.save() first_item = Item() first_item.text = 'The first (ever) list item' first_item.list = list_ first_item.save() second_item = Item() second_item.text = 'Item the second' second_item.list = list_ second_item.save() saved_list = List.objects.first() self.assertEqual(saved_list, list_) saved_items = Item.objects.all() self.assertEqual(saved_items.count(), 2) first_saved_item = saved_items[0] second_saved_item = saved_items[1] self.assertEqual(first_saved_item.text, 'The first (ever) list item') self.assertEqual(first_saved_item.list, list_) self.assertEqual(second_saved_item.text, 'Item the second') self.assertEqual(second_saved_item.list, list_)
def update_feeds(): sources = Source.objects.all() for source in sources: feed = feedparser.parse(source.url) new_entries = [entry for entry in feed.entries if (pytz.utc.localize(datetime(*(entry.published_parsed[0:6]))) > source.updated)] for entry in new_entries: item = Item(source = source, url = entry.link) item.save()
def add_source(request): if request.user.is_authenticated(): if request.META['REQUEST_METHOD'] == 'GET': return render(request, 'feed/add_source.html', {}) else: url = urllib2.unquote(request.POST['url']).decode("utf8") feed = feedparser.parse(url) source = Source(owner=request.user, name = feed.feed.title, url = request.POST['url']) source.save() for i in range(0,9): item = Item(source = source, url = feed.entries[i].link) item.save() return HttpResponseRedirect(reverse('feed:add_source')) else: return HttpResponseRedirect(reverse('archives:index'))
def test_cannot_save_empty_list_items(self): list_ = List.objects.create() item = Item(list=list_, text='') with self.assertRaises(ValidationError): item.save() item.full_clean()