link = entry.link
url_hash = hashlib.md5(link).hexdigest()
date = entry.published_parsed
published_date = arrow.get(date).to('US/Pacific').date().strftime(
'%Y-%m-%d')
published_ts = arrow.get(date).to('US/Pacific').to('UTC').timestamp
# See if we already have this story
try:
NewsItem.get(NewsItem.url_hash == url_hash)
print 'Item exists. Skipping.'
continue
except peewee.DoesNotExist:
print 'Creating item.'
item = NewsItem()
headline = entry.title
summary = entry.summary
item.url_hash = url_hash
item.link = link
item.source = 'Seattle Times'
item.title = headline
item.summary = summary
item.published_date = published_date
item.published_ts = published_ts
item.inserted_ts = arrow.utcnow().timestamp
item.save()
link = entry.link
url_hash = hashlib.md5(link).hexdigest()
date = entry.published_parsed
published_date = arrow.get(date).to('US/Pacific').date().strftime('%Y-%m-%d')
published_ts = arrow.get(date).to('US/Pacific').to('UTC').timestamp
# See if we already have this story
try:
NewsItem.get(NewsItem.url_hash==url_hash)
print 'Item exists. Skipping.'
continue
except peewee.DoesNotExist:
print 'Creating item.'
item = NewsItem()
headline = entry.title
summary = entry.summary
item.url_hash = url_hash
item.link = link
item.source = 'Seattle Times'
item.title = headline
item.summary = summary
item.published_date = published_date
item.published_ts = published_ts
item.inserted_ts = arrow.utcnow().timestamp
item.save()
item.link = link
item.url_hash = url_hash
item.title = headline
item.summary = summary
item.source = 'Weather Underground'
# The author and date are in the same text string
parts = tr_el.em.text.strip().split('\n\t\t')
if len(parts) == 1:
dt = dateutil.parser.parse(parts[0])
dt = dt.replace(tzinfo=pytz.timezone('US/Eastern')).strftime(
'%Y-%m-%d') + 'T00:00:00-05:00'
timestamp = arrow.get(dt).to('UTC').timestamp
published_date = arrow.get(dt).date().strftime('%Y-%m-%d')
item.published_ts = timestamp
else:
dt = dateutil.parser.parse(parts[1])
dt = dt.replace(tzinfo=pytz.timezone('US/Eastern')).strftime(
'%Y-%m-%d') + 'T00:00:00-05:00'
timestamp = arrow.get(dt).to('UTC').timestamp
published_date = arrow.get(dt).date().strftime('%Y-%m-%d')
item.published_ts = timestamp
# Stories are posted without times, so we just assign the current time to the story
# since we check every 30 minutes. It's better than showing every story as midnight
item.published_ts = arrow.utcnow().timestamp
item.published_date = published_date
item.inserted_ts = arrow.utcnow().timestamp
item.save()
item.link = link
item.url_hash = url_hash
item.title = headline
item.summary = summary
item.source = 'Weather Underground'
# The author and date are in the same text string
parts = tr_el.em.text.strip().split('\n\t\t')
if len(parts) == 1:
dt = dateutil.parser.parse(parts[0])
dt = dt.replace(tzinfo=pytz.timezone('US/Eastern')).strftime('%Y-%m-%d') + 'T00:00:00-05:00'
timestamp = arrow.get(dt).to('UTC').timestamp
published_date = arrow.get(dt).date().strftime('%Y-%m-%d')
item.published_ts = timestamp
else:
dt = dateutil.parser.parse(parts[1])
dt = dt.replace(tzinfo=pytz.timezone('US/Eastern')).strftime('%Y-%m-%d') + 'T00:00:00-05:00'
timestamp = arrow.get(dt).to('UTC').timestamp
published_date = arrow.get(dt).date().strftime('%Y-%m-%d')
item.published_ts = timestamp
# Stories are posted without times, so we just assign the current time to the story
# since we check every 30 minutes. It's better than showing every story as midnight
item.published_ts = arrow.utcnow().timestamp
item.published_date = published_date
item.inserted_ts = arrow.utcnow().timestamp
item.save()
meta_og_title_el = links_soup.find('meta', {'property': 'og:title'})
meta_og_desc_el = links_soup.find('meta', {'property': 'og:description'})
meta_og_url_el = links_soup.find('meta', {'property': 'og:url'})
except Exception, e:
meta_og_title_el = None
meta_og_desc_el = None
meta_og_url_el = None
if meta_og_title_el is not None:
headline = meta_og_title_el['content'].strip()
if meta_og_desc_el is not None:
description = meta_og_desc_el['content'].strip()
if meta_og_url_el is not None:
link = meta_og_url_el['content']
item.link = link
item.url_hash = url_hash
item.title = headline
item.summary = description
item.source = "Accuweather"
item.published_date = published_date
item.published_ts = utc_dt.timestamp
item.inserted_ts = arrow.utcnow().timestamp
item.save()
# Sleep between requests
time.sleep(1)
