def save_twitter_data(tweet, location=None, print_test=False):
# https://www.ewg.org/tapwater/index.php#results-by-state-map
# ref: https://dev.twitter.com/overview/api/tweets
if models.alert.objects.filter(sourceId=tweet.id_str).exists():
print "%s - exists" % (tweet.text)
#log(tweet.text, 'success')
return
tw = models.alert()
tw.text = tweet.text
tw.text_wo_stopwords = remove_stopwords(tweet.text.lower())
tw.sourceId = tweet.id_str
tw.source = 'twitter'
tw.status = status(tweet.text.lower())
tw.published = tweet.created_at
tw.save()
if location:
tw.location = location
location.status = tw.status
location.save()
if tweet.entities.get('urls'):
for item in tweet.entities['urls']:
url = models.url()
url.alert = tw
url.link = item['url']
url.save()
print tweet.text
def save_feed_data(item, location=None):
sourceId = item['id']
if models.alert.objects.filter(sourceId=sourceId).exists():
print "%s - exists" % (sourceId)
#log(sourceId, 'success')
return
title = cleanhtml(item['title'])
summary = cleanhtml(item['summary'])
published = datetime.strptime(item['published'], '%Y-%m-%dT%H:%M:%SZ')
link = item['link']
concat_text = "%s :: %s" % (title, summary)
alert = models.alert()
alert.source = 'goog'
alert.text = concat_text
alert.text_wo_stopwords = remove_stopwords(concat_text)
alert.sourceId = sourceId
alert.status = status(concat_text)
alert.save()
url = models.url()
url.alert = alert
url.link = link
url.save()
print sourceId
def save_twitter_data(tweet, location ):
# https://www.ewg.org/tapwater/index.php#results-by-state-map
# ref: https://dev.twitter.com/overview/api/tweets
tw = models.tweet()
tw.location = location
tw.text = tweet.text
tw.sourceId = tweet.id_str
#tw.url =
tw.created = tweet.created_at
tw.save()
if tweet.entities.get('urls'):
for item in tweet.entities['urls']:
url = models.url()
url.tweet = tw
url.link = item['url']
url.save()
print tweet.text