def test_slugify_on_save(self):
item = NewsItem(title="My News Item",
publish_date=now(),
create_user_id=1,
feed_id=1)
item.save()
self.assertEquals(item.slug, 'my-news-item')
def test_slugify_on_save(self):
item = NewsItem(title="My News Item",
publish_date=now(),
create_user_id=1,
feed_id=1)
item.save()
self.assertEquals(item.slug, 'my-news-item')
def add(request):
if request.method == "POST":
feed_id = request.POST.get("feed")
title = request.POST.get("title")
link = request.POST.get("link")
summary = request.POST.get("summary")
date = request.POST.get("date")
if not feed_id or not title or not summary:
return HttpResponseBadRequest(
"Bad Request: must include feed, title, and summary\n")
if date:
try:
date = dateutil.parser.parse(date)
except:
return HttpResponseBadRequest(
"Bad Request: invalid date format\n")
else:
date = datetime.datetime.now()
if not link:
link = ""
try:
feed = NewsFeed.objects.get(id=feed_id)
except NewsFeed.DoesNotExist:
return HttpResponseNotFound("Not Found: no such feed id\n")
item = NewsItem(feed=feed,
title=title,
link=link,
date=date,
summary=summary)
item.save()
# get previous item id
prev_id = None
try:
prev_items = NewsItem.objects.filter(
feed=feed, id__lt=item.id).order_by("-id")[:1]
if len(prev_items) > 0:
prev_id = prev_items[0].id
except:
pass
out = dict()
if prev_id:
out["prev_id"] = str(prev_id)
out["items"] = [item.to_json()]
hr = dict()
hr["body"] = json.dumps(out) + "\n"
fanout_publish("feed-" + str(feed.id), str(item.id), str(prev_id), hr)
return HttpResponse("Posted\n")
else:
return HttpResponseNotAllowed(["POST"])
def create_news_items(is_sticky=False, amount=1):
for i in range(0, amount):
item = NewsItem(title='silly news item name',
publish_date=now(),
published=True,
create_user_id=1,
feed_id=randint(1, 2),
sticky=is_sticky)
item.save()
def add(request):
if request.method == "POST":
feed_id = request.POST.get("feed")
title = request.POST.get("title")
link = request.POST.get("link")
summary = request.POST.get("summary")
date = request.POST.get("date")
if not feed_id or not title or not summary:
return HttpResponseBadRequest("Bad Request: must include feed, title, and summary\n")
if date:
try:
date = dateutil.parser.parse(date)
except:
return HttpResponseBadRequest("Bad Request: invalid date format\n")
else:
date = datetime.datetime.now()
if not link:
link = ""
try:
feed = NewsFeed.objects.get(id=feed_id)
except NewsFeed.DoesNotExist:
return HttpResponseNotFound("Not Found: no such feed id\n")
item = NewsItem(feed=feed, title=title, link=link, date=date, summary=summary)
item.save()
# get previous item id
prev_id = None
try:
prev_items = NewsItem.objects.filter(feed=feed, id__lt=item.id).order_by("-id")[:1]
if len(prev_items) > 0:
prev_id = prev_items[0].id
except:
pass
out = dict()
if prev_id:
out["prev_id"] = str(prev_id)
out["items"] = [item.to_json()]
hr = dict()
hr["body"] = json.dumps(out) + "\n"
fanout_publish("feed-" + str(feed.id), str(item.id), str(prev_id), hr)
return HttpResponse("Posted\n")
else:
return HttpResponseNotAllowed(["POST"])
link = entry.link
url_hash = hashlib.md5(link).hexdigest()
date = entry.published_parsed
published_date = arrow.get(date).to('US/Pacific').date().strftime(
'%Y-%m-%d')
published_ts = arrow.get(date).to('US/Pacific').to('UTC').timestamp
# See if we already have this story
try:
NewsItem.get(NewsItem.url_hash == url_hash)
print 'Item exists. Skipping.'
continue
except peewee.DoesNotExist:
print 'Creating item.'
item = NewsItem()
headline = entry.title
summary = entry.summary
item.url_hash = url_hash
item.link = link
item.source = 'Seattle Times'
item.title = headline
item.summary = summary
item.published_date = published_date
item.published_ts = published_ts
item.inserted_ts = arrow.utcnow().timestamp
item.save()
link = entry.link
url_hash = hashlib.md5(link).hexdigest()
date = entry.published_parsed
published_date = arrow.get(date).to('US/Pacific').date().strftime('%Y-%m-%d')
published_ts = arrow.get(date).to('US/Pacific').to('UTC').timestamp
# See if we already have this story
try:
NewsItem.get(NewsItem.url_hash==url_hash)
print 'Item exists. Skipping.'
continue
except peewee.DoesNotExist:
print 'Creating item.'
item = NewsItem()
headline = entry.title
summary = entry.summary
item.url_hash = url_hash
item.link = link
item.source = 'Seattle Times'
item.title = headline
item.summary = summary
item.published_date = published_date
item.published_ts = published_ts
item.inserted_ts = arrow.utcnow().timestamp
item.save()
from models import NewsItem
from datetime import datetime
scrapper = Scrap()
news_items = scrapper.get_news_items()
for item in news_items:
## let's insert stuff
# first check that it does not already exist!
to_db = NewsItem.select().where(NewsItem.slug == item['slug'])
if len(to_db) == 0:
# item has not been in the db before
to_db = NewsItem(news_hash=item['news_hash'],
slug=item['slug'],
news_title=item['title'],
news_link=item['link'],
date_updated=item['date_updated'],
intro_text=item['intro_text'],
scrapped_at=datetime.now())
to_db.save()
else:
to_db = to_db.get()
# check if the hash has changed
if to_db.news_hash != item['news_hash']:
to_db.news_hash = item['news_hash']
to_db.news_title = item['news_title']
to_db.news_link = item['link']
to_db.date_updated = item['date_updated']
to_db.intro_text = item['intro_text']
to_db.save()
