def parse_trending_topic_data_from_text_file(self, text_file):
tt_imported = tt_found = tt_failure_format = tt_failure_other = 0
for text_tt in text_file:
if not text_tt:
continue
tt_array = text_tt.split('\t')
# Check for needed fields
if len(tt_array) != 4:
tt_failure_format += 1
continue
# tt_foo = tt_array[0]
tt_unicode_text = tt_array[1].decode('utf-8')
# tt_time = tt_array[2]
# tt_url = tt_array[3]
# Check for existing tweet or create new one
try:
_, created = models.TrendingTopic.objects.get_or_create(text=tt_unicode_text)
if created:
tt_imported += 1
else:
tt_found += 1
except Exception, e:
send_manually_exception_email(self.request, e)
tt_failure_other += 1
def parse_tweet_data_from_json_data_array(self, json_data_array):
tweet_imported = tweet_found = tweet_failure_format = tweet_failure_other = 0
for json_tweet in json_data_array:
tweet_text = json_tweet.get('text', None)
tweet_id = json_tweet.get('id', None)
# Check for needed fields
if tweet_text is None or tweet_id is None:
tweet_failure_format += 1
continue
# Check for existing tweet or create new one
try:
_ = models.Tweet.objects.get(id_twitter=int(json_tweet['id']))
tweet_found += 1
except models.Tweet.DoesNotExist:
try:
new_tweet = models.Tweet(id_twitter=int(tweet_id), text=unicode(tweet_text))
new_tweet.save()
tweet_imported += 1
except Exception, e:
send_manually_exception_email(self.request, e)
tweet_failure_other += 1
