def start():
most_used_words = pickle.load(open(MOST_USED_WORDS_DB, "rb"))
most_used_400_words = [".", ",", "?", "!", ";"] + most_used_words[:395]
# This handles Twitter authetification and the connection to Twitter Streaming API
accounts = get_accounts()
index = 1
while True:
try:
auth = OAuthHandler(accounts[index].consumer_key,
accounts[index].consumer_secret)
auth.set_access_token(accounts[index].access_token,
accounts[index].access_token_secret)
listener = TurkishTweetsListener()
stream = Stream(auth, listener)
if STREAMING_MODE == "track":
stream.filter(track=most_used_400_words, languages=["tr"])
elif STREAMING_MODE == "follow":
stream.filter(follow=FOLLOWING)
elif STREAMING_MODE == "mixed":
stream.filter(track=most_used_400_words, follow=FOLLOWING)
else:
print("STREAMING MODE track ya da folllow olabilir.")
break
except KeyboardInterrupt:
stream.disconnect()
break
except Exception as e:
print(e)
index += 1
index %= len(accounts)
# switch alternative account
continue
def main_old():
pubnub = Pubnub.Pubnub(settings.PUBNUB_PUBLISH_KEY,
settings.PUBNUB_SUBSCRIBE_KEY,
settings.PUBNUB_SECRET, False)
hashtags = set()
auth = OAuthHandler(settings.TWITTER_CONSUMER_KEY,
settings.TWITTER_CONSUMER_SECRET)
auth.set_access_token(settings.TWITTER_ACCESS_TOKEN,
settings.TWITTER_ACCESS_TOKEN_SECRET)
stream = None
print "Started"
while True:
try:
#current_hashtags = set(w.hashtag for w in Wall.objects.filter(last_ping__gt=datetime.datetime.now(utc) - datetime.timedelta(minutes=settings.WALL_EXPIRATION)))
current_hashtags = set(w.hashtag for w in Wall.objects.all())
if len(current_hashtags - hashtags) > 0:
if stream is not None:
stream.disconnect()
stream = Stream(auth, PubnubListener(pubnub))
hashtags = current_hashtags
print("Now filtering " + ", ".join(list(hashtags)))
stream.filter(track=list(hashtags), async=True)
time.sleep(5)
except KeyboardInterrupt:
stream.disconnect()
break
except Exception as e:
print e
time.sleep(5)
def main():
queue = Queue.Queue()
storage = Storage()
metadata = storage.get_metadata()
hashtags = metadata['options'] + [metadata['project_hashtag']]
auth = OAuthHandler(metadata['consumer_key'], metadata['consumer_secret'])
auth.set_access_token(metadata['access_token_key'],
metadata['access_token_secret'])
listener = StreamQueueListener(queue)
stream = Stream(auth, listener)
poll = Poll(storage, stream, queue, metadata)
try:
logger.info('Started the collector')
stream.filter(track=hashtags, async=True)
poll.run()
except (SystemExit, KeyboardInterrupt):
stream.disconnect()
logger.info('Interrupted')
except:
stream.disconnect()
logger.critical('Unexpected error', exc_info=True)
finally:
if hasattr(stream, '_thread'):
stream._thread.join()
logger.info('Exiting...')
def main():
# {path}は自分の環境にあわせる
db_path = str(Path.home()) + "{path}/Conversation." + date.today(
).strftime("20%y.%m.%d") + ".db"
parser = argparse.ArgumentParser()
parser.add_argument('--new',
type=int,
default=-1,
help='-1 indicates new database...')
args = parser.parse_args()
if args.new == -1:
make_db(db_path)
listener = QueueListener(db_path)
stream = Stream(listener.auth, listener)
print("Listening...\n")
delay = 0.25
try:
while True:
try:
stream.sample()
except KeyboardInterrupt:
print('Stopped')
return
except urllib3.exceptions.ProtocolError as e:
print("Incomplete read", e)
except urllib3.exceptions.ReadTimeoutError as e:
print("Read Timeout", e)
except (socket.error, http.client.HTTPException):
print("HTTP error waiting for a few seconds")
time.sleep(delay)
delay += 0.25
finally:
stream.disconnect()
def KeywordUpdateModule():
global KeywordsToFilter
print("Authenticating Keyword Updater")
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.secure = True
auth.set_access_token(access_token, access_token_secret)
myApi = tweepy.API(auth)
myKeywordStream = Stream(auth, tweepy.StreamListener())
print("Authentication successfull!")
keywords = list()
timeline = myApi.home_timeline()
for raw_data in timeline:
message = raw_data.text
kw = indicoKeywords(message)
for keyword in kw:
keywords.append(unicode(keyword.decode('utf-8')))
print("Keywords collected, sleeping...")
myKeywordStream.disconnect()
print(keywords)
KeywordsToFilter = keywords
class StreamConsumerThreadClass(threading.Thread):
def __init__(self,term='',oauthfile=''):
threading.Thread.__init__(self)
self.searchterm = term
self.name = term
self.consume = True
oauth = json.loads(open(oauthfile,'r').read())
listener = MongoDBListener()
auth = OAuthHandler(oauth['consumer_key'], oauth['consumer_secret'])
auth.set_access_token(oauth['access_token'], oauth['access_token_secret'])
self.stream = Stream(auth, listener,timeout=60)
def stopConsume(self):
self.stream.disconnect()
def run(self):
now = datetime.datetime.now()
print "Twitter Stream with terms: %s started at: %s" % (self.getName(), now)
connected = False
while True:
try:
if not connected:
connected = True
self.stream.filter(track=[self.searchterm])
except SSLError, e:
print e
connected = False
def clicked():
stock_quote = txt.get()
w2.configure(text=stock_quote + " is the value entered")
webbrowser.open(
'file:///home/aibot/Desktop/GIT_folder/Sentiment_Analysis_using_Python_Twitter_Kibana/index.html',
new=2)
listener = TweetStreamListener()
# set twitter keys/tokens
auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_SECRET)
# The most exception break up the kernel in my test is ImcompleteRead. This exception handler ensures
# the stream to resume when breaking up by ImcompleteRead
while True:
try:
# create instance of the tweepy stream
stream = Stream(auth, listener)
# search twitter for keyword "facebook"
stream.filter(track=[stock_quote])
#except IncompleteRead:
# continue
except KeyboardInterrupt:
# or however you want to exit this loop
stream.disconnect()
break
def main():
global defaultbg
win = Tk()
win.title("TwiSound")
win.geometry("1000x300")
for i,k in enumerate(inst.keys()):
box = ttk.LabelFrame(win,text=k,height=300,width=300)
box.grid(column=i,row=0,padx=10)
box.pack_propagate(0)
txt = Label(box,wraplength=290,font=('Segoe UI',14))
txt.pack()
pannels[k] = (box,txt)
defaultbg = txt['bg']
auth = OAuthHandler(keys['consumer_key'], keys['consumer_secret'])
auth.secure = True
auth.set_access_token(keys['access_token'], keys['access_token_secret'])
api = API(auth)
# If the authentication was successful, you should
# see the name of the account print out
print(api.me().name)
stream = Stream(auth, StdOutListener())
stream.filter(track=inst.keys(),is_async=True)
win.mainloop()
stream.disconnect()
class TwitterObservable(Observable):
def __init__(self):
super(TwitterObservable, self).__init__()
customer_key = "Twitter customer key"
customer_secret = "Twitter customer secret"
access_token = "Twitter access token"
access_secret = "Twitter access secret"
self.auth = OAuthHandler(customer_key, customer_secret)
self.auth.set_access_token(access_token, access_secret)
self.twitter_stream = None
self.listener = _TweetsStreamListener()
self.listener.set_observer(self)
def start_stream(self):
import requests
requests.packages.urllib3.disable_warnings()
try:
self.twitter_stream = Stream(self.auth, self.listener)
self.twitter_stream.filter(locations=[-180, -90, 180, 90])
except ProtocolError:
pass
def stop_stream(self):
if self.twitter_stream is not None:
self.twitter_stream.disconnect()
self.twitter_stream = None
class TweetsListener(StreamListener):
def __init__(self, keywords):
auth = OAuthHandler(TWITTER_APP_KEY, TWITTER_APP_SECRET)
auth.set_access_token(TWITTER_KEY, TWITTER_SECRET)
self.__stream = Stream(auth, listener=self)
self.__stream.filter(track=keywords, async=True)
self.count = 0
def on_data(self, data):
try:
tweet = data.split(',"text":"')[1].split('","source"')[0]
createdAt = data.split('"created_at":"')[1].split('","id"')[0]
print(createdAt + '::' + tweet)
saveTweet = createdAt + ',' + tweet
saveTwitterData = open('twitterStreamingData.csv', "a")
saveTwitterData.write(saveTweet)
saveTwitterData.write('\n')
#self.count = self.count + 1
saveTwitterData.close()
return True
except BaseException as e:
print('Failed on data', str(e))
time.sleep(5)
def on_error(self, status):
print(status)
def on_disconnect(self, notice):
self.__stream.disconnect()
def disconnect(self):
self.__stream.disconnect()
def dump_stream_feeds(company_name):
f = open(f"../data/twitter_{company_name}_stream.txt", "w")
class MyListener(StreamListener):
"""
A listener that will write the feeds in the file.
The default time limit is 1 minute, i.e., 60 seconds.
"""
def __init__(self, time_limit=60):
self.start_time = time.time()
self.limit = time_limit
def on_data(self, data):
if (time.time() - self.start_time) < self.limit:
f.write(data)
return True
else:
f.close()
return False
def on_error(self, status):
print(status)
listener = MyListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, listener)
stream.filter(track=[company_name])
stream.disconnect()
class TwitterIngestor():
"""
Ingests data from Twitter
"""
def __init__(self, twitter_config, callback):
"""
Creates an OAuth handler and a Twitter stream
:param twitter_config:
:param callback:
"""
self.callback = callback
self.auth = OAuthHandler(twitter_config['customer_key'],
twitter_config['customer_secret'])
self.auth.set_access_token(twitter_config['access_token'],
twitter_config['access_secret'])
self.twitter_stream = Stream(self.auth, TwitterListener(self.callback))
logger.info('Initialized Twitter Ingestor')
def start(self):
"""
Starts Twitter streaming
:return:
"""
logger.info('Starting Twitter feed ingestion')
self.twitter_stream.sample(is_async=True, languages=['en'])
logger.info('Started Twitter feed ingestion')
def stop(self):
"""
Requests Twitter stream to start receiving messages
:return:
"""
logger.info('Stopping Twitter feed ingestion')
self.twitter_stream.disconnect()
logger.info('Stopped Twitter feed ingestion')
def main():
# open stream
listener = QueueListener()
stream = Stream(listener.auth, listener, language='ja')
# [stream filter]
stream.filter(languages=["ja"],
track=[
'私', 'あなた', '俺', 'ー', 'する', 'です', 'ます', 'けど', '何', '@',
'#', '＃', '。', '，', '！', '？', '…', '.', '!', '?', ',',
':', '：', '』', ')', '）', '...'
])
#stream.filter(languages=["ja"], track=['私','あなた','俺','ー','する','です','ます','けど','何','I', 'you', 'http', 'www', 'co', '@', '#', '＃', '。', '，', '！','？','…', '.', '!','?', ',', ':', '：', '』', ')', '）', '...'])
# Default Script# stream.filter(locations=[-122.75,36.8,-121.75,37.8]) # San Francisco
# stream.filter(locations=[-74,40,-73,41]) # New York City
# stream.filter(languages=["en"], track=['python', 'obama', 'trump'])
try:
while True:
try:
stream.sample() # blocking!
except KeyboardInterrupt:
print('KEYBOARD INTERRUPT')
return
except (socket.error, http.client.HTTPException):
global tcpip_delay
print('TCP/IP Error: Restarting after %.2f seconds.' %
tcpip_delay)
time.sleep(min(tcpip_delay, MAX_TCPIP_TIMEOUT))
tcpip_delay += 0.25
finally:
stream.disconnect()
print('Exit successful, corpus dumped in %s' % (listener.dumpfile))
class TweetController:
"""docstring for Controller"""
def __init__(self):
self.settings = Settings()
# self.auth = OAuthHandler(Listener.api_data["consumer_key"], Listener.api_data["consumer_secret"])
# self.auth.set_access_token(Listener.api_data["access_token"], Listener.api_data["access_token_secret"])
self.api = tweepy.API(self.settings.auth, parser=tweepy.parsers.JSONParser())
self.db = DataBase()
# self.tweet_gui = tweet_gui
self.default_keyword = ['Obama', 'hillary ', 'Trump']
self.db.create_table_if_not_exist()
def start_stream(self):
self.tweet_listener = Listener()
self.stream = Stream(auth=self.settings.auth, listener=self.tweet_listener)
self.stream.filter(track=self.default_keyword, async=True)
def stop_stream(self):
self.stream.disconnect()
def set_keyword(self, default_keyword):
self.default_keyword = default_keyword
print(default_keyword)
def main():
while True:
try:
with utils.connect('ng') as conn:
with conn.cursor() as cur:
cur.execute("""CREATE TABLE IF NOT EXISTS {tablename}(
id_str text PRIMARY KEY,
source text,
user_id text,
created_at timestamp,
text text)""".format(tablename=TABLENAME))
data_streamer = tweet.PostgresStreamer(conn=conn, tablename=TABLENAME)
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN_KEY, ACCESS_TOKEN_SECRET)
stream = Stream(auth, data_streamer)
stream.sample(languages=LANGUAGES)
conn.commit()
except KeyboardInterrupt:
stream.disconnect()
break
except (IndexError, ConnectionError, ProtocolError, ReadTimeoutError):
#logger.exception(e)
stream.disconnect()
time.sleep(90)
continue
def gatherTwitterData():
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
twitterStream = Stream(auth, listener())
twitterStream.filter(locations=[-125,25,-65,48], async=False)
time.sleep(10000)
twitterStream.disconnect()
class MapConsumer(WebsocketConsumer):
twitter_stream = None
auth = None
def connect(self):
self.accept()
self.auth = OAuthHandler(settings.CONSUMER_KEY,
settings.CONSUMER_SECRET)
self.auth.set_access_token(settings.ACCESS_TOKEN,
settings.ACCESS_SECRET)
def disconnect(self, close_code):
if self.twitter_stream is not None:
self.twitter_stream.disconnect()
def receive(self, text_data):
text_data_json = json.loads(text_data)
if text_data_json['type'] == 'listen':
message = text_data_json['message']
self.twitter_stream = Stream(self.auth, MyListener(self))
# Create a new stream and filter by bounding box
# A new thread (async) is required not to block this HTTP transaction
# [SWlongitude, SWLatitude, NElongitude, NELatitude]
self.twitter_stream.filter(locations=message, async=True)
elif text_data_json['type'] == 'stop':
self.twitter_stream.disconnect()
def main_old():
pubnub = Pubnub.Pubnub(settings.PUBNUB_PUBLISH_KEY, settings.PUBNUB_SUBSCRIBE_KEY, settings.PUBNUB_SECRET, False)
hashtags = set()
auth = OAuthHandler(settings.TWITTER_CONSUMER_KEY, settings.TWITTER_CONSUMER_SECRET)
auth.set_access_token(settings.TWITTER_ACCESS_TOKEN, settings.TWITTER_ACCESS_TOKEN_SECRET)
stream = None
print "Started"
while True:
try:
#current_hashtags = set(w.hashtag for w in Wall.objects.filter(last_ping__gt=datetime.datetime.now(utc) - datetime.timedelta(minutes=settings.WALL_EXPIRATION)))
current_hashtags = set(w.hashtag for w in Wall.objects.all())
if len(current_hashtags - hashtags) > 0:
if stream is not None:
stream.disconnect()
stream = Stream(auth, PubnubListener(pubnub))
hashtags = current_hashtags
print("Now filtering " + ", ".join(list(hashtags)))
stream.filter(track=list(hashtags), async=True)
time.sleep(5)
except KeyboardInterrupt:
stream.disconnect()
break
except Exception as e:
print e
time.sleep(5)
def on_data(self, data):
# write file
try:
# add to tweet count (lets you control how many tweets are scraped)
global max_num_tweets
global tweet_count
# update tweet count
tweet_count += 1
# stream as long as max tweet count isn't exceeded
if tweet_count <= max_num_tweets:
# every thousand tweets
if tweet_count % 1000 == 0:
print("{a} tweets scraped (out of maximum of {b} indicated).".format(a = tweet_count, b = max_num_tweets))
else:
print("Specified maximum number of tweets ({max})reached. Streaming halted.".format(max = max_num_tweets))
Stream.disconnect(self) # stop collecting tweets after max limit is reached
# write data to a new file
with open("new_tweets.json", 'a') as f:
f.write(data)
return True
except BaseException as e:
print('Error on data: %s' % str(e))
t.sleep(5)
return True
class TwitterListener(object):
def __init__(self):
pubnub = Pubnub.Pubnub(settings.PUBNUB_PUBLISH_KEY, settings.PUBNUB_SUBSCRIBE_KEY, settings.PUBNUB_SECRET, False)
self.pubnub_listener = PubnubListener(pubnub)
self.stream = None
self.hashtags = set()
self.auth = OAuthHandler(settings.TWITTER_CONSUMER_KEY, settings.TWITTER_CONSUMER_SECRET)
self.auth.set_access_token(settings.TWITTER_ACCESS_TOKEN, settings.TWITTER_ACCESS_TOKEN_SECRET)
def update_hashtags(self):
current_hashtags = set(w.hashtag for w in Wall.objects.all())
if len(current_hashtags - self.hashtags) > 0:
self.hashtags = current_hashtags
return True
def filter(self):
if self.stream is not None:
self.stream.disconnect()
self.stream = Stream(self.auth, self.pubnub_listener)
print("Now filtering " + ", ".join(list(self.hashtags)))
self.stream.filter(track=list(self.hashtags), async=True)
def update(self):
if self.update_hashtags():
self.filter()
def exit(self):
if self.stream is not None:
self.stream.disconnect()
def start_stream():
while True:
try:
listener = Streamlistener()
twitterStream = Stream(
auth,
listener) #initialize Stream object with a time out limit
# twitterStream.filter(track=keyword_list) #call the filter method to run the Stream Object
#https://dev.twitter.com/streaming/reference/get/statuses/sample
twitterStream.sample()
#https://github.com/azurro/country-bounding-boxes/blob/master/dataset/ph.json
#twitterStream.filter(locations=[112.16672,4.3833541,127.0737203,21.5296298]) #call the filter method to run the Stream Object
#twitterStream.filter(locations=[12.865353, -168.344469, 67.929266, -42.133528]) #call the filter method to run the Stream Object
#twitterStream.filter(locations=[-56.286063, -29.301501,74.250793, 174.253191,-63.874556, -170.629625,70.718096, -29.828842 ]) #call the filter method to run the Stream Object
#twitterStream.filter(locations=[-69.624350, -23.9022753, -57.353627, -7.164010 ]) #call the filter method to run the Stream Object
except Exception, e:
# Oh well, reconnect and keep trucking
print 'Generic exception happened. Continuing..', str(e)
pass
except KeyboardInterrupt:
# Or however you want to exit this loop
twitterStream.disconnect()
break
def main():
"""Connects to the stream and starts threads to write them to a file."""
listener = QueueListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
writer_thread = threading.Thread(target=worker, args=(listener, ))
writer_thread.start()
stream = Stream(auth, listener)
print_status(listener)
while True:
try:
stream.sample() # blocking!
except KeyboardInterrupt:
print 'KEYBOARD INTERRUPT:'
return
except (socket.error, httplib.HTTPException):
global tcpip_delay
print('TCP/IP Error: Restarting after '
'{} seconds.'.format(tcpip_delay))
time.sleep(min(tcpip_delay, MAX_TCPIP_TIMEOUT))
tcpip_delay += 0.25
finally:
print 'Disconnecting stream'
stream.disconnect()
print 'Waiting for last tweets to finish processing'
# Send poison pill to writer thread and wait for it to exit
listener.queue.put(None)
listener.queue.join()
print 'Waiting for writer thread to finish'
writer_thread.join()
print 'Exit successful'
def main():
while True:
try:
with utils.connect('ng') as conn:
with conn.cursor() as cur:
cur.execute("""CREATE TABLE IF NOT EXISTS {tablename}(
id_str text PRIMARY KEY,
source text,
user_id text,
created_at timestamp,
text text)""".format(tablename=TABLENAME))
data_streamer = tweet.PostgresStreamer(conn=conn,
tablename=TABLENAME)
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN_KEY, ACCESS_TOKEN_SECRET)
stream = Stream(auth, data_streamer)
stream.sample(languages=LANGUAGES)
conn.commit()
except KeyboardInterrupt:
stream.disconnect()
break
except (IndexError, ConnectionError, ProtocolError, ReadTimeoutError):
#logger.exception(e)
stream.disconnect()
time.sleep(90)
continue
class StreamConsumerThreadClass(threading.Thread):
def __init__(self, term='', oauthfile=''):
threading.Thread.__init__(self)
self.searchterm = term
self.name = term
self.consume = True
oauth = json.loads(open(oauthfile, 'r').read())
listener = MongoDBListener()
auth = OAuthHandler(oauth['consumer_key'], oauth['consumer_secret'])
auth.set_access_token(oauth['access_token'],
oauth['access_token_secret'])
self.stream = Stream(auth, listener, timeout=60)
def stopConsume(self):
self.stream.disconnect()
def run(self):
now = datetime.datetime.now()
print "Twitter Stream with terms: %s started at: %s" % (self.getName(),
now)
connected = False
while True:
try:
if not connected:
connected = True
self.stream.filter(track=[self.searchterm])
except SSLError, e:
print e
connected = False
def stream_tweets(self, postive_words, negative_words, email, username):
# This handles Twitter authetification and the connection to Twitter Streaming API
global isclicked
listener = StdOutListener(postive_words, negative_words, email)
auth = OAuthHandler(twitter_credentials.CONSUMER_KEY,
twitter_credentials.CONSUMER_SECRET)
auth.set_access_token(twitter_credentials.ACCESS_TOKEN,
twitter_credentials.ACCESS_TOKEN_SECRET)
stream = Stream(auth, listener, wait_on_rate_limit=True)
update_stream(stream)
condition = True
# This line filter Twitter Streams to capture data by the keywords:
while condition:
condition = isclicked
if stream.running is True:
stream.disconnect()
else:
isclicked = False
time.sleep(5)
isclicked = True
IDS = []
for element in getuserdata(username):
IDS.append(str(element))
stream.filter(
follow=IDS, encoding='utf-8', is_async=True
) # Open the stream to work on asynchronously on a different thread
time.sleep(7200)
stream.disconnect()
def main_twitter(termo_consulta, ckey, csecret, atoken, asecret):
while True:
try:
l = listener()
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
# Connect/reconnect the stream
stream = Stream(auth, l)
# DON'T run this approach async or you'll just create a ton of streams!
stream.filter(track=[termo_consulta])
except IncompleteRead:
# Oh well, reconnect and keep trucking
continue
except KeyboardInterrupt:
# Or however you want to exit this loop
stream.disconnect()
break
except Exception as e:
print(e)
l = listener()
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
# Connect/reconnect the stream
stream = Stream(auth, l)
# DON'T run this approach async or you'll just create a ton of streams!
stream.filter(track=[termo_consulta])
class MapConsumer(WebsocketConsumer):
twitter_stream = None
auth = None
def connect(self):
self.accept()
self.auth = OAuthHandler(settings.CONSUMER_KEY,
settings.CONSUMER_SECRET)
self.auth.set_access_token(settings.ACCESS_TOKEN,
settings.ACCESS_SECRET)
def disconnect(self, close_code):
if self.twitter_stream is not None:
self.twitter_stream.disconnect()
def receive(self, text_data):
text_data_json = json.loads(text_data)
if text_data_json['type'] == 'listen':
message = text_data_json['message']
self.twitter_stream = Stream(auth=self.auth,
listener=MyListener(self))
# Create a new stream and filter by text
self.twitter_stream.filter(track=[message], is_async=True)
elif text_data_json[
'type'] == 'stop' and self.twitter_stream is not None:
self.twitter_stream.disconnect()
def main():
#listen = SListener(api, 'myprefix')
#stream = tweepy.Stream(auth, listen)
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
stream = Stream(auth, TweetListener())
stream.filter(track=['nba'])
api = tweepy.API(auth)
auth.set_access_token(access_token, access_secret)
twitterStream = Stream(auth, TweetListener())
#hadoop fs -put twitterStream /tweepy/hdfs.txt
#os.system('echo "%s" | hadoop fs -put twitterStream /tweepy/hdfs.txt' %(json.dump(twitterStream)))
print("Streaming started...")
try:
stream.filter(track=track)
except:
print("error!")
stream.disconnect()
def get_twitter_corpus(config, api_config):
"""
ツイッターよりコーパスを作成
@param config 設定ファイル情報
@param api_config Twitter APIの情報
"""
while True:
try:
# キューリスナー作成
listener = QueueListener(config, api_config)
# ストリームを開く
stream = Stream(listener.auth, listener)
# ストリームフィルタ（どれかにヒットすれば拾う）
stream.filter(languages=["ja"],
track=[
'。', '，', '！', '.', '!', ',', '?', '？', '、', '私',
'俺', '(', ')', '君', 'あなた'
])
except KeyboardInterrupt:
listener.log()
listener.save_tmp()
stream.disconnect()
break
except:
global tcpip_delay
time.sleep(min(tcpip_delay, MAX_TCPIP_TIMEOUT))
tcpip_delay += 0.25
class StreamConsumerThreadClass(threading.Thread):
def __init__(self, term="", oauthfile="", follow=False, user=False, track=False):
threading.Thread.__init__(self)
self.searchterm = term
self.name = term
self.consume = True
self.follow = follow
self.user = user
self.track = track
listener = MongoDBListener()
try:
oauth = json.loads(open(oauthfile, "r").read())
if "consumer_key" in oauth:
auth = OAuthHandler(oauth["consumer_key"], oauth["consumer_secret"])
auth.set_access_token(oauth["access_token"], oauth["access_token_secret"])
self.api = API(auth)
if not self.api.verify_credentials():
raise Exception("Invalid credentials")
self.stream = Stream(auth, listener, timeout=60)
except:
print "Error logging to Twitter"
raise
def stop_consume(self):
self.stream.disconnect()
def run(self):
now = datetime.datetime.now()
if self.user:
print "Twitter Stream of the OAuth user: started at: %s" % (now)
else:
print "Twitter Stream with terms: %s started at: %s" % (self.getName(), now)
connected = False
while True:
try:
if not connected:
connected = True
if self.user:
self.stream.userstream(_with="followings")
elif self.follow:
user_ids = []
for user in self.api.lookup_users([], self.searchterm.split(","), False):
user_ids.append(user.id)
self.stream.filter(follow=[",".join("{0}".format(n) for n in user_ids)])
elif self.track:
self.stream.filter(track=[self.searchterm])
except SSLError, sse:
print sse
connected = False
except Exception, e:
print "Stream error"
raise e
class StreamConsumerThreadClass(threading.Thread):
def __init__(self, term='', oauthfile='', follow=False):
threading.Thread.__init__(self)
self.searchterm = term
self.name = term
self.consume = True
self.follow = follow
listener = MongoDBListener()
try:
oauth = json.loads(open(oauthfile, 'r').read())
if 'consumer_key' in oauth:
auth = OAuthHandler(oauth['consumer_key'],
oauth['consumer_secret'])
auth.set_access_token(oauth['access_token'],
oauth['access_token_secret'])
self.api = API(auth)
if not self.api.verify_credentials():
raise Exception("Invalid credentials")
self.stream = Stream(auth, listener, timeout=60)
except:
print "Error logging to Twitter"
raise
def stop_consume(self):
self.stream.disconnect()
def run(self):
now = datetime.datetime.now()
print "Twitter Stream with terms: %s started at: %s" % (self.getName(),
now)
connected = False
while True:
try:
if not connected:
connected = True
if self.follow:
user_ids = []
for user in self.api.lookup_users(
[], self.searchterm.split(','), False):
user_ids.append(user.id)
self.stream.filter(follow=[
",".join("{0}".format(n) for n in user_ids)
])
else:
self.stream.filter(track=[self.searchterm])
except SSLError, sse:
print sse
connected = False
except Exception, e:
print "Stream error"
raise e
def get_hashtag_info(auth, hashtag):
global hash_tag
hash_tag = hashtag
twitter_stream = Stream(auth, MyListener())
twitter_stream.filter(track=hashtag, async=True)
time.sleep(10)
twitter_stream.disconnect()
def main():
try:
twitterStream = Stream(auth, listener())
twitterStream.filter(track=["@realDonaldTrump"], languages=["en"])
twitterStream.disconnect()
except Exception as e:
print(str(e))
time.sleep(5)
def startStreaming():
# if __name__ == '__main__':
runtime = 300 # Tracking for 600 seconds
twitterStream = Stream(auth, TweeterStreamListener())
twitterStream.filter(languages=["en"],
track=['hiring', 'hire', 'looking for', 'job'],
async=True)
time.sleep(runtime)
twitterStream.disconnect()
def main():
print("Streaming started.... Ctrl+C to abort")
try:
twitterStream = Stream(auth,listener())
twitterStream.filter(locations = GEOBOX_BHAM)
except Exception as e:
print (e)
print("Error or execution finished. Program exiting... ")
twitterStream.disconnect()
def listenTweets(waitTime=60):
l = myStreamListener()
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
stream = Stream(auth, l)
stream.filter(track=BRAND_NAMES, async= True)
time.sleep(waitTime)
print "Disconnecting"
stream.disconnect()
class Streamer():
def __init__(self):
self.stream = Stream(Utlity().get_authentication_token(), Listener())
def start_tweet_streaming(self, filter_list):
self.stream.filter(track=filter_list)
def stop_tweet_streaming(self):
self.stream.disconnect()
def start_streaming(track=[""], locations=[-180,-90,180,90], languages=["en"]):
l = StdOutListener()
stream = Stream(auth, l)
while True:
try:
stream.disconnect()
stream.filter(track=track, locations=locations, languages=languages)
except Exception, e:
print "Exception: ", e
class TweetDownloader(safethread.SafeThread, StreamListener):
def __init__(self,
destpath,
consumer_key,
consumer_secret,
access_token,
access_secret,
window = 10000,
verbose = False):
super(TweetDownloader, self).__init__(name="TweetDownloader")
self.destpath = destpath
self.consumer_key = consumer_key
self.consumer_secret = consumer_secret
self.access_token = access_token
self.access_secret = access_secret
self.prefix = 'tweets'
self.suffix = 'txt'
self.window = window
self.buf = collections.deque()
self.stopped = True
# Write the tweet text to the current file. May throw an error if the file
# is currently being switched out (i.e. writing at the end of a window).
def write(self, vals):
self.buf.appendleft(json.dumps(vals))
def action(self):
if len(self.buf) > 0:
self.f.write(self.buf.pop() + '\n')
if ((time.time() * 1000) - self.begin > self.window):
self.f.close()
fname = self.destpath + self.prefix + '-' + str(self.begin) + \
'.' + self.suffix
os.rename(self.destpath + 'tmp', fname)
self.begin = int(time.time() * 1000)
self.f = open(self.destpath + 'tmp', 'w')
def start(self):
# Setup the stream
auth = OAuthHandler(self.consumer_key, self.consumer_secret)
auth.set_access_token(self.access_token, self.access_secret)
self.stream = Stream(auth, TweetListener(self))
# Create the first file
self.begin = int(time.time() * 1000)
self.f = open(self.destpath + 'tmp', 'w')
# Start the threads
self.stream.sample(async=True)
super(TweetDownloader, self).start()
def stop(self):
self.stream.disconnect()
super(TweetDownloader, self).stop()
def init_read_api(self, term, limit):
receive_my_listener = MyListener(term, limit)
twitter_stream = Stream(self.auth, receive_my_listener)
twitter_stream.filter(track=[term], languages=["en"])
twitter_stream.disconnect()
self.ppt.initialize_process()
self.ppt.read_datas_to_generate_tokens()
self.result = self.ppt.selected_tweets
self.geo_location = self.ppt.geo_location
class streamer :
##accepts initial hashtag (without #) as parameter
def __init__(self,) :
self.tstreamer = None
self.data = None
self.inject_data = None
##does what the name implies, stream must first be stopped however
# returns true if successful
def set_hashtag(self, hashtag) :
if (not self.tstreamer) :
self.data = dataset.dataset(hashtag)
self.inject_data = self.data.get_injection_method()
return True
else :
return False
##returns current hashtag that is set
# returns none if there is none
def get_hashtag(self) :
if (self.data) :
return self.data.hashtag
else :
return None
##return true if the streamer is currently running, false if otherwise
def is_running(self) :
if (self.tstreamer) :
return True;
else :
return False
##starts the streamer, returns true if successful, false if otherwise
# if it is not successful it means it is already running, or else you did not set
# the hashtag with the set_hashtag method
def start_streamer(self) :
if (not self.tstreamer and self.data) :
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
self.tstreamer = Stream(auth, listener(lambda x : self.inject_data((x,)))) #my silly inject data method takes a list not a string . . .
self.tstreamer.filter(track=[self.data.hashtag], async=True)
return True;
else :
return False;
##stops the streamer, if it returns false it means that it was not running
# in the first place
def stop_streamer(self) :
if (self.tstreamer) :
self.tstreamer.disconnect()
self.tstreamer = None
return True
else :
return False
def stream_tweets_hashTag(self, hashTag):
output = open('tweets_hashTag.txt', 'w')
twitterStream = Stream(self.auth, listener(output_file=output))
try:
twitterStream.filter(track=[hashTag])
except KeyboardInterrupt:
print("Stopped.")
finally:
twitterStream.disconnect()
output.close()
class SparkStreamListener(StreamListener):
""" Use twitter streaming API to stream to PySpark. """
def __init__(self):
config = ConfigParser()
config.read(os.path.join(CONFIG_DIR, 'prod.cfg'))
self.sockets = []
auth = OAuthHandler(config.get('twitter', 'consumer_key'),
config.get('twitter', 'consumer_secret'))
auth.set_access_token(config.get('twitter', 'access_token'),
config.get('twitter', 'access_token_secret'))
self.stream = Stream(auth, self)
def add_socket(self, ws):
self.sockets.append(ws)
print(self.sockets)
def run(self):
try:
self.stream.filter(track=['python'])
except Exception as e:
print(e)
self.stream.disconnect()
def start(self):
""" Start GEvent """
gevent.spawn(self.run)
def send(self, status):
""" Send status to socket """
print(self.sockets)
if len(self.sockets) > 1:
ws = choice(self.sockets)
else:
ws = self.sockets[0]
try:
ws.send(status.encode('utf-8'))
except ValueError:
print(e)
# the web socket die..
self.sockets.remove(ws)
def on_data(self, data):
decoded = json.loads(data)
gevent.spawn(self.send, decoded.get('text') + '\n')
return True
def on_error(self, status):
print("Error: %s", status)
def on_timeout(self):
print("tweepy timeout.. wait 30 seconds")
gevent.sleep(30)
def startlistneruser():
global CONSUMER_KEY,CONSUMER_SECRET,ACCESS_KEY,ACCESS_SECRET
l = MyMessageListener()
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)
stream = Stream(auth, l)
try:
">>>>> start listening to stream <<<<<"
stream.filter(track=['yolo'])
api.update_status(status = Message)
except:
print ">>>>> disconect this stream <<<<<"
stream.disconnect()
def starthashlistener():
global CONSUMER_KEY,CONSUMER_SECRET,ACCESS_KEY,ACCESS_SECRET
a = MyHashListener()
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)
stream = Stream(auth, a)
try:
print ">>>>> start catching hash <<<<<"
stream.filter(follow=['4091042235'])
api.update_status(status = Message)
except:
print ">>>>> disconect this stream <<<<<"
stream.disconnect()
def main():
logger = support.getLogger('tsv_writer')
while True:
try:
data_streamer = tweet_access.JsonStreamer(filename=FILENAME)
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN_KEY, ACCESS_TOKEN_SECRET)
stream = Stream(auth, data_streamer)
stream.sample(languages=LANGUAGES)
#stream.filter(track=TAGS, languages=LANGUAGES, async=True)
except Exception, e:
logger.exception(e)
stream.disconnect()
time.sleep(60)
class StreamConsumerThreadClass(threading.Thread):
def __init__(self, term='', oauthfile=''):
threading.Thread.__init__(self)
self.searchterm = term
self.name = term
self.consume = True
listener = MongoDBListener()
try:
oauth = json.loads(open(oauthfile, 'r').read())
if 'consumer_key' in oauth:
auth = OAuthHandler(oauth['consumer_key'], oauth['consumer_secret'])
auth.set_access_token(oauth['access_token'], oauth['access_token_secret'])
api = API(auth)
if not api.verify_credentials():
raise Exception("Invalid credentials")
else:
auth = BasicAuthHandler(oauth['username'], oauth['password'])
except:
print "Error logging to Twitter"
raise
self.stream = Stream(auth, listener, timeout=60)
def stopConsume(self):
self.stream.disconnect()
def run(self):
now = datetime.datetime.now()
print "Twitter Stream with terms: %s started at: %s" % (self.getName(), now)
connected = False
while True:
try:
if not connected:
connected = True
self.stream.filter(track=[self.searchterm])
except SSLError, e:
print e
connected = False
except Exception, e:
print "Stream error"
raise e
def main():
"""Connects to the stream and starts threads to write them to a file."""
staticconf.YamlConfiguration(CONFIG_FILE)
listener = QueueListener()
auth = OAuthHandler(
staticconf.read_string('twitter.consumer_key'),
staticconf.read_string('twitter.consumer_secret'),
)
auth.set_access_token(
staticconf.read_string('twitter.access_token'),
staticconf.read_string('twitter.access_token_secret'),
)
writer_thread = threading.Thread(target=worker, args=(listener,))
writer_thread.start()
stream = Stream(auth, listener)
print_status(listener)
try:
while True:
try:
stream.sample(languages=['en']) # blocking!
except KeyboardInterrupt:
print('KEYBOARD INTERRUPT', file=sys.stderr)
return
except (socket.error, httplib.HTTPException):
global tcpip_delay
print(
'TCP/IP Error: Restarting after {delay} seconds.'.format(
delay=tcpip_delay,
),
file=sys.stderr,
)
time.sleep(min(tcpip_delay, MAX_TCPIP_TIMEOUT))
tcpip_delay += 0.25
finally:
print('Disconnecting stream', file=sys.stderr)
stream.disconnect()
print('Waiting for last tweets to finish processing', file=sys.stderr)
# Send poison pill to writer thread and wait for it to exit
listener.queue.put(None)
listener.queue.join()
print('Waiting for writer thread to finish', file=sys.stderr)
writer_thread.join()
print('Exit successful', file=sys.stderr)
class StreamThread(threading.Thread):
def __init__(self, listener):
super(StreamThread, self).__init__()
self.listener = listener
self.exit = 0
def run(self):
while not self.exit:
logger.info("connecting to twitter...")
self.stream = Stream(self.listener.auth, self.listener)
self.stream.filter(follow=self.listener.follow_ids)
if not self.exit:
time.sleep(5)
def stop(self):
self.exit = 1
self.stream.disconnect()
class TwitterTrends(StreamListener):
def __init__(self):
#initiate the reuqired authentication
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
self.stream = Stream(auth, self)
self.subscribers = []
def on_data(self, data):
for entry in data.split("\r\n"):
processed = json.loads(entry)
# Only include tweets that has a "topic", in this case one or more hashtags
if "entities" in processed:
if len(processed['entities']['hashtags']) > 0:
tags = [u"#" + t['text'].lower() for t in processed['entities']['hashtags']]
user = processed['user']['screen_name']
text = processed['text']
time = processed['created_at']
tweet = Tweet(text, tags, time, user)
for subscriber in self.subscribers:
subscriber.on_tweet(tweet)
if len(self.subscribers) == 0:
self.stop()
def on_error(self, error):
print u"Error occurred: %s" % error
def add_subscriber(self, subscriber):
self.subscribers.append(subscriber)
def remove_subscriber(self, subscriber):
self.subscribers.remove(subscriber)
def start(self):
self.stream.sample()
def stop(self):
print 'Stopped'
self.stream.disconnect()
def main():
if os.path.exists('data') == False:
os.mkdir('data')
hashtags = ['#isu', '#illinoisstate', '#illinoisstateu', '@IllinoisStateU' ]
creds = getCreds('creds.txt')
auth = OAuthHandler(creds[0], creds[1])
auth.set_access_token(creds[2], creds[3])
l = SListener()
stream = Stream(auth, l)
try:
stream.filter(track=hashtags)
except:
print('error')
stream.disconnect()
def main():
if os.path.exists('data') == False:
os.mkdir('data')
hashtags = ['#kysen', '#iasen', '#ncsen', '#cosen', '#lasen', '#arsen' ]
creds = getCreds('creds.txt')
auth = OAuthHandler(creds[0], creds[1])
auth.set_access_token(creds[2], creds[3])
l = SListener()
stream = Stream(auth, l)
try:
stream.filter(track=hashtags)
except:
print('error')
stream.disconnect()
def scrap_twitter():
""" this function scraps twitter stream of India for Cricket related URLs"""
listener = UrlListener()
auth = utils.get_twitter_auth()
tweetstream = Stream(auth, listener)
while True:
# right before midnight close the stream so that next run can take
# another files
if time.localtime().tm_hour == 23:
if tweetstream.running is True:
tweetstream.disconnect()
else:
if tweetstream.running is False:
# filtering for India
tweetstream.filter(track=['cricket'])
#tweetstream.filter(
# locations=[70.04, 8.99, 93.0, 34.52], async=True)
# wake up every 10 minutes to check the stream status
time.sleep(600)
def streamer(consumer_key, consumer_secret, access_token, access_token_secret):
#This handles Twitter authetification and the connection to Twitter Streaming AP
if __name__ == '__main__':
l = StdOutListener()
#Twitter API access with streaming class
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
try:
# streaming API search terms
stream.filter(track=['MyCensus', 'Census Australia', 'ABSCensus', 'Get online on August 9', 'making sense of the census'])
except Exception as e:
print(e)
stream.disconnect()
class _MyStreamFilter(object):
""" Wrapper used to run and to close a stream. """
def __init__(self, runner):
self._listener = TimedRotatingStreamListener(runner.log_dir,
runner.prefix,
runner.when_interval)
self._stream = Stream(runner.auth, self._listener)
self._track = runner.track
self._locations = runner.locations
def run(self):
""" Run the collector. """
self._stream.filter(track=self._track, locations=self._locations)
def close(self):
""" Close the collector. """
self._stream.disconnect()
self._listener.flush()
self._listener.close()
class TwitterStream(TwitterBase):
""" A TwitterStream class that will continuously capture data from a
Twitter timeline and store in our backend model.
"""
stream = None
listener = None
def __init__(self, listener):
TwitterBase.__init__(self)
self.listener = listener
def start(self):
logging.info('start:')
self.stream = Stream(self.auth, self.listener)
self.stream.userstream(_with='user',
stall_warnings=True,
async=False)
def stop(self):
logging.info('stop:')
self.stream.disconnect()
def handle(self, *args, **options):
#Negotiates with twitter to authorize access to the stream
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
battle_c = BattleConnection()
twitterStream = Stream(auth, StdOutListener())
#Start first stream
current_hashtags = battle_c.get_hashtags_to_evaluate()
# Open the stream to work on asynchronously on a different thread
twitterStream.filter(track=current_hashtags, async=True)
time.sleep(60) # sleep for 10 min
while True:
#actualizes the hashtags depending on the ongoing battles
new_hashtags = battle_c.get_hashtags_to_evaluate()
#if there is diferent hashtags to search stops the strem and searches with new hashtags
if set(current_hashtags) != set(new_hashtags):
current_hashtags = new_hashtags
if twitterStream.running is True:
twitterStream.disconnect()
twitterStream.filter(track=current_hashtags, async=True)
time.sleep(60)
def run(args):
pid = current_process().pid
round = 1
while True:
try:
l = StdOutListener(args['output_file'])
auth = OAuthHandler(args['consumer_key'], args['consumer_secret'])
auth.set_access_token(args['token_key'], args['token_secret'])
print "%i's %i authentication success. (420 means rate limited)" % (pid, round)
stream = Stream(auth, l)
#l.setStream(stream)
#stream.filter(track=['#programming','#google','#android'])
stream.sample()
#locations=[-129.19,23.96,-64.68,50.68]
except KeyboardInterrupt:
print "KeyboardInterrupt sensed, %i exiting..." % pid
stream.disconnect()
return True
except Exception as e:
print e
finally:
stream.disconnect()
round += 1
def handle(self, **options):
try:
b = Bidomatic()
b.start()
d = Dispatcher()
d.start()
# c = UpdateCache()
# c.start()
#threads.extend([k, c, b])
l = StdOutListener()
auth = OAuthHandler(settings.TWITTER_CONSUMER_KEY,
settings.TWITTER_CONSUMER_SECRET)
auth.set_access_token(settings.TWITTER_ACCESS_TOKEN,
settings.TWITTER_TOKEN_SECRET)
stream = Stream(auth, l)
stream.filter(track=settings.TWITTER_HASH_TAGS, async=True)
threads.extend([b, d])
while True:
raw_input()
except (KeyboardInterrupt, SystemExit):
for t in threads:
t.kill_received = True
stream.disconnect()
def main():
"""Connects to the stream and starts threads to write them to a file."""
listener = QueueListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
writer_thread = threading.Thread(target=worker, args=(listener,))
writer_thread.start()
stream = Stream(auth, listener)
print_status(listener)
try:
while True:
try:
stream.sample() # blocking!
except KeyboardInterrupt:
print 'KEYBOARD INTERRUPT:'
return
except (socket.error, httplib.HTTPException):
global tcpip_delay
print ('TCP/IP Error: Restarting after '
'{0} seconds.'.format(tcpip_delay))
time.sleep(min(tcpip_delay, MAX_TCPIP_TIMEOUT))
tcpip_delay += 0.25
finally:
print 'Disconnecting stream'
stream.disconnect()
print 'Waiting for last tweets to finish processing'
# Send poison pill to writer thread and wait for it to exit
listener.queue.put(None)
listener.queue.join()
print 'Waiting for writer thread to finish'
writer_thread.join()
print 'Exit successful'
