def main():
password = urllib.quote_plus(examplepassword)
client = MongoClient(exampledatabase)
db = client[exampleclient]
collection = db[examplecollection]
json = import_simplejson()
def storeTweet(tweet):
data = {}
data['id'] = tweet.id
data['permalink'] = tweet.permalink
data['username'] = tweet.username
data['text'] = tweet.text
data['date'] = tweet.date
data['retweets'] = tweet.retweets
data['favorites'] = tweet.favorites
data['mentions'] = tweet.mentions
data['hashtags'] = tweet.hashtags
data['geo'] = tweet.geo
collection.insert(data)
tweetCriteria = got.manager.TweetCriteria().setQuerySearch(
'transgender').setSince(exampleStart).setUntil(exampleEnd)
tweets = got.manager.TweetManager.getTweets(tweetCriteria)
for tweet in tweets:
storeTweet(tweet)
class StreamListener(tweepy.StreamListener):
# mongocon = MongoClient('mongodb://localhost:27017/')
# db = mongocon.tstream
# col = db.tweets
json = import_simplejson()
def on_status(self, tweet):
print 'Ran on_status'
def on_error(self, status_code):
return False
def on_data(self, data):
global k
if data[0].isdigit():
pass
else:
k += 1
d = json.loads(data)
if d.get("geo") and d.get("text"):
tweet_id = uuid.uuid1().hex
d['tweet_id'] = tweet_id
col.insert(d)
print("Inserted new tweet in database")
m = Message()
body = {'tweet_id': tweet_id, 'text': d['text']}
m.set_body(json.dumps(body))
tweets_queue.write(m)
print("Inserted new tweet in Queue")
def tweet_analyses(tweet):
try:
json = import_simplejson()
result = {}
# status info
result['id_str'] = tweet._json['id_str']
result['text'] = tweet._json['text']
result['sentiment'] = liveSentiment.judge_sentiment(
tweet._json['text'])
result['created_at'] = tweet._json['created_at']
result['source'] = tweet._json['source']
result['in_reply_to_status_id_str'] = tweet._json[
'in_reply_to_status_id_str']
result['in_reply_to_user_id_str'] = tweet._json[
'in_reply_to_user_id_str']
result['in_reply_to_screen_name'] = tweet._json[
'in_reply_to_screen_name']
result['coordinates'] = tweet._json['coordinates']
result['place'] = tweet._json['place']
# user info
result['user'] = tweet._json['user']
str = json.dumps(result)
return str
except:
traceback.print_exc()
def write_tweets(tweets):
''' Function that appends tweets to couchdb. '''
json = import_simplejson()
_db_handle = CouchDBConnect.connect_CouchDB()
db = _db_handle["tweets_all"]
dtweet = {}
for tweet in tweets:
dtweet = json.dumps(tweet._json)
store_tweet(db, dtweet)
class StreamListener(tweepy.StreamListener):
json = import_simplejson()
def on_status(self, tweet):
print 'Ran on_status'
def on_error(self, status_code):
return False
def on_data(self, data):
print data
with open('fetched_tweets.txt', 'a') as tf:
tf.write(data)
return True
def __init__(self, api=None, **options):
StreamListener.__init__(self, api)
self.json = import_simplejson()
self.print_data = options.get("print_data", False)
self.print_data = options.get("created_at", False)
self.print_data = options.get("user", False)
self.print_data = options.get("id", False)
self.print_data = options.get("favorite_count", False)
self.print_data = options.get("retweet_count", False)
self.print_data = options.get("text", False)
self.print_data = options.get("source_url", False)
self.print_data = options.get("print_data", False)
self.print_data = options.get("print_data", False)
self.delay = int(options.get("delay", 0))
class TwitterStreamListener(StreamListener):
#input host = here
mongoc = MongoClient(host='localhost')
db = mongoc.natweet
col = db.tweets
json = import_simplejson()
def on_data(self, data):
if data[0].isdigit():
pass
else:
col.insert(json.loads(data))
print(json.loads(data))
def on_error(self, status):
return False
def __init__(self, api=None, **options):
QueuedListener.__init__(self, api, **options)
self.json = import_simplejson()
self.follow_list = options.get("follow_list", [])
self.fav_list = options.get("fav_list", [])
self.ignore_list = options.get("ignore_list", [])
self.block_users = options.get("block_users", [])
self.enable_retweet = options.get("enable_retweet", True)
self.enable_fav = options.get("enable_fav", True)
self.enable_follow = options.get("enable_follow", True)
self.at = options.get("at", False)
self.original = options.get("original", False)
self.quoted = options.get("quoted", False)
self.favs = options.get("favs", 3)
self.retweets = options.get("retweets", 3)
class StreamListener(tweepy.StreamListener):
mongocon = Connection()
db = mongocon.tstream
col = db.tweets
json = import_simplejson()
def on_status(self, tweet):
print 'Ran on_status'
def on_error(self, status_code):
return False
def on_data(self, data):
if data[0].isdigit():
pass
else:
col.insert(json.loads(data))
print(json.loads(data))
def media_metadata_create(self, media_id, alt_text):
"""
:reference: https://dev.twitter.com/rest/reference/post/media/metadata/create
:allowed_param: 'media_id', 'alt_text'
"""
post_data = {
"media_id": media_id,
"alt_text": {
"text": alt_text
}
}
json = import_simplejson()
return bind_api(
api=self,
path='/media/metadata/create.json',
method='POST',
allowed_param=[],
require_auth=True,
upload_api=True
)(post_data=json.dumps(post_data))
class StreamListener(tweepy.StreamListener):
json = import_simplejson()
def on_status(self, tweet):
pass
def on_timeout(self):
log_msg("twitterLog.txt","***timeout:sleeping for a minute***")
time.sleep(60)
return True #don't kill the stream
def on_error(self, status_code):
log_msg("twitterLog.txt","-----error----" + str(status_code))
return True #don't kill the stream
def on_data(self, data):
if data[0].isdigit():
pass
else:
data_md5 = hashlib.md5(json.dumps(data, sort_keys=True)).hexdigest()
json_tweet = json.loads(data)
try:
if "text" in json_tweet:
text = json_tweet["text"]
if "lang" in json_tweet:
clean_text = process(text)
language = json_tweet["lang"]
if language == 'en':
if not is_swearing_en(clean_text):
json_to_keep = fix_json_en(json_tweet, clean_text)
cbucket.set(data_md5,json_to_keep)
write_text_to_file(script_settings.sentimentQ+"files/",data_md5,clean_text)
elif language == 'es':
if not is_swearing_es(clean_text):
json_to_keep = fix_json_es(json_tweet, clean_text)
cbucket.set(data_md5,json_to_keep)
write_text_to_file(script_settings.sentimentQ+"files_spanish/",data_md5,clean_text)
except Exception,e:
print e
# Code by Artur Pimentel de Oliveira from tweepy.streaming import StreamListener from tweepy import OAuthHandler from tweepy import Stream from tweepy.utils import import_simplejson json = import_simplejson() # Please assign your keys strings to the variables below # Consumer keys consumer_key = "" consumer_secret = "" # Access keys access_token="" access_token_secret="" lon = 0 lat = 1 longitude_left_point = -86.33 latitude_left_point = 41.63 longitude_right_point = -86.20 latitude_right_point = 41.74 class StdOutListener(StreamListener): n_good_tweets = 0 def on_data(self, raw_data): data = json.loads(raw_data)
def __init__(self):
self.json_lib = import_simplejson()
def __init__(self):
self.json_lib = import_simplejson()
def tweet_save(search):
for tweet in search:
json = import_simplejson()
str = tweet_analyses(tweet)
tweetsSearchDB.save(json.loads(str))
# coding=utf-8
DEBUG = True
METRES = 0.000009
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler, Stream, API
from tweepy.utils import import_simplejson, parse_datetime
from tweepy.models import Model
import_simplejson()
import argparse
import os
import itertools
import datetime, time
from django.core.management import setup_environ
import nearbysources.settings as settings
import sys, json
if __name__ == "__main__":
setup_environ(settings)
from nearbysources.questions.models import *
questionnaire = Questionnaire.objects.get(name=sys.argv[1])
tweet = QuestionnaireTweet.objects.get(questionnaire=questionnaire, language=Language.objects.get(code="en")).text
betriebe = []
for loi in LocationOfInterest.objects.filter(campaign=questionnaire.campaign):
betriebe.append({"name": loi.name, "geometry": {"coordinates": [loi.lng, loi.lat]}, "id": loi.id, "loi": loi})
#with open("gwb.json", 'r') as f:
# betriebe = json.load(f)['features']
def on_status(self, status):
""" you can set DB store part here. ot outside"""
json = import_simplejson()
str = tweet_analyses(status)
tweetsDB.save(json.loads(str))
