def test_slugify_on_save(self): item = NewsItem(title="My News Item", publish_date=now(), create_user_id=1, feed_id=1) item.save() self.assertEquals(item.slug, 'my-news-item')
def add(request): if request.method == "POST": feed_id = request.POST.get("feed") title = request.POST.get("title") link = request.POST.get("link") summary = request.POST.get("summary") date = request.POST.get("date") if not feed_id or not title or not summary: return HttpResponseBadRequest( "Bad Request: must include feed, title, and summary\n") if date: try: date = dateutil.parser.parse(date) except: return HttpResponseBadRequest( "Bad Request: invalid date format\n") else: date = datetime.datetime.now() if not link: link = "" try: feed = NewsFeed.objects.get(id=feed_id) except NewsFeed.DoesNotExist: return HttpResponseNotFound("Not Found: no such feed id\n") item = NewsItem(feed=feed, title=title, link=link, date=date, summary=summary) item.save() # get previous item id prev_id = None try: prev_items = NewsItem.objects.filter( feed=feed, id__lt=item.id).order_by("-id")[:1] if len(prev_items) > 0: prev_id = prev_items[0].id except: pass out = dict() if prev_id: out["prev_id"] = str(prev_id) out["items"] = [item.to_json()] hr = dict() hr["body"] = json.dumps(out) + "\n" fanout_publish("feed-" + str(feed.id), str(item.id), str(prev_id), hr) return HttpResponse("Posted\n") else: return HttpResponseNotAllowed(["POST"])
def create_news_items(is_sticky=False, amount=1): for i in range(0, amount): item = NewsItem(title='silly news item name', publish_date=now(), published=True, create_user_id=1, feed_id=randint(1, 2), sticky=is_sticky) item.save()
def add(request): if request.method == "POST": feed_id = request.POST.get("feed") title = request.POST.get("title") link = request.POST.get("link") summary = request.POST.get("summary") date = request.POST.get("date") if not feed_id or not title or not summary: return HttpResponseBadRequest("Bad Request: must include feed, title, and summary\n") if date: try: date = dateutil.parser.parse(date) except: return HttpResponseBadRequest("Bad Request: invalid date format\n") else: date = datetime.datetime.now() if not link: link = "" try: feed = NewsFeed.objects.get(id=feed_id) except NewsFeed.DoesNotExist: return HttpResponseNotFound("Not Found: no such feed id\n") item = NewsItem(feed=feed, title=title, link=link, date=date, summary=summary) item.save() # get previous item id prev_id = None try: prev_items = NewsItem.objects.filter(feed=feed, id__lt=item.id).order_by("-id")[:1] if len(prev_items) > 0: prev_id = prev_items[0].id except: pass out = dict() if prev_id: out["prev_id"] = str(prev_id) out["items"] = [item.to_json()] hr = dict() hr["body"] = json.dumps(out) + "\n" fanout_publish("feed-" + str(feed.id), str(item.id), str(prev_id), hr) return HttpResponse("Posted\n") else: return HttpResponseNotAllowed(["POST"])
link = entry.link url_hash = hashlib.md5(link).hexdigest() date = entry.published_parsed published_date = arrow.get(date).to('US/Pacific').date().strftime( '%Y-%m-%d') published_ts = arrow.get(date).to('US/Pacific').to('UTC').timestamp # See if we already have this story try: NewsItem.get(NewsItem.url_hash == url_hash) print 'Item exists. Skipping.' continue except peewee.DoesNotExist: print 'Creating item.' item = NewsItem() headline = entry.title summary = entry.summary item.url_hash = url_hash item.link = link item.source = 'Seattle Times' item.title = headline item.summary = summary item.published_date = published_date item.published_ts = published_ts item.inserted_ts = arrow.utcnow().timestamp item.save()
link = entry.link url_hash = hashlib.md5(link).hexdigest() date = entry.published_parsed published_date = arrow.get(date).to('US/Pacific').date().strftime('%Y-%m-%d') published_ts = arrow.get(date).to('US/Pacific').to('UTC').timestamp # See if we already have this story try: NewsItem.get(NewsItem.url_hash==url_hash) print 'Item exists. Skipping.' continue except peewee.DoesNotExist: print 'Creating item.' item = NewsItem() headline = entry.title summary = entry.summary item.url_hash = url_hash item.link = link item.source = 'Seattle Times' item.title = headline item.summary = summary item.published_date = published_date item.published_ts = published_ts item.inserted_ts = arrow.utcnow().timestamp item.save()
from models import NewsItem from datetime import datetime scrapper = Scrap() news_items = scrapper.get_news_items() for item in news_items: ## let's insert stuff # first check that it does not already exist! to_db = NewsItem.select().where(NewsItem.slug == item['slug']) if len(to_db) == 0: # item has not been in the db before to_db = NewsItem(news_hash=item['news_hash'], slug=item['slug'], news_title=item['title'], news_link=item['link'], date_updated=item['date_updated'], intro_text=item['intro_text'], scrapped_at=datetime.now()) to_db.save() else: to_db = to_db.get() # check if the hash has changed if to_db.news_hash != item['news_hash']: to_db.news_hash = item['news_hash'] to_db.news_title = item['news_title'] to_db.news_link = item['link'] to_db.date_updated = item['date_updated'] to_db.intro_text = item['intro_text'] to_db.save()