def run(self):
"""
"""
twitter_config = Pit.get(
"twitter.com", {"require": {"user": "******", "password": "******"}}
)
stream = TwitterStream(twitter_config["user"], twitter_config["password"])
print "start stream"
while True:
for status in stream.search("ACV_DEFPS3"):
msg = status["text"]
print msg, type(msg)
self._bot.connection.notice(self._bot.channel, msg.encode("utf-8"))
self._bot.connection.privmsg(self._bot.channel, u"twitter から切断されました。再接続します".encode("utf-8"))
print "end stream"
sleep(60)
def main():
# Set up Twitter api use
print('User default API and authentications...')
api = TwitterStream(config.api_key, config.api_secret,
config.access_token_key, config.access_token_secret)
# Get user desired search
query = UserInteraction.userInput()
# Save tweets to file
tweet_file = api.search(query, config.twitterFile)
# Load data
tweets = ParsingTwitterJson(tweet_file)
# Parse Input, convert Unicode, etc.
tweets.clean_data()
def stream_crawl(keyword):
loop = True
while (loop):
track = keyword
listen = TwitterStream(api)
stream = tweepy.Stream(auth, listen)
# Starting a Stream
# stream.filter(track=['python'])
print("Twitter streaming started... ")
try:
# stream.filer( track = track )
stream.filter(track=['python'])
loop = False
except:
print("Error!...And Retry after 60 sec")
loop = True
stream.disconnect()
sleep(5)
continue
from TwitterStream import TwitterStream
#Driver code for the project
print "Main"
twitter = TwitterStream()
twitter.get_stream("Ivanka")
#!/usr/bin/python
# -*- coding: utf-8 -*-
from TwitterStream import TwitterStream
oauth_keys = {'consumer_key': <Consumer key>,
'consumer_secret': <Consumer secret>,
'access_token_key': <Token key>,
'access_token_secret': <Token secret>}
post_params = {'include_entities': 0,
'stall_warning': 'true',
'track': 'brasil'
}
def handle_tweet(message):
if message.get('limit'):
print 'Rate limiting caused us to miss %s tweets' % (message['limit'].get('track'))
elif message.get('disconnect'):
raise Exception('Got disconnect: %s' % message['disconnect'].get('reason'))
elif message.get('warning'):
print 'Got warning: %s' % message['warning'].get('message')
else:
print 'Got tweet with text: %s' % message.get('text')
if __name__ == '__main__':
ts = TwitterStream(handle_tweet, oauth_keys, post_params)
ts.setup_connection()
ts.start()
(1 - a) * (self.stats.get(term, (0, 0))[1] + a * delta ** 2))
def find_emerging_terms(self):
with open(os.path.join(self.result_dir,'burst_terms'), 'a', encoding = 'utf-8') as f:
f.write('Interval %d:\n' % self.ii)
for term, count in self.curr_term_count.items():
stats = self.stats.get(term,(0,0))
ewma = stats[0]
ewmvar = stats[1]
ewma = max(ewma, self.beta)
ratio = (count / self.curr_tweets_count - ewma) / (math.sqrt(ewmvar) + self.beta) #THE FORMULA
if ratio > self.s:
f.write('%s %f\r\n' % (term, ratio)) #\r\n under windows
f.write('\n')
f.flush()
if __name__ == '__main__':
ts = TwitterStream()
#datapath = '/Users/Adward/Github/Automatic-Rumor-Detection/TwitterEventDetection/TestData/original'
datapath = '/Volumes/Adward_Backup/SRTP/data'
dirlist = os.listdir(datapath)
for path in dirlist:
if path.startswith('201') and os.path.isdir(os.path.join(datapath,path)):
ts.source(os.path.join(datapath,path))
ts.sort()
temp = TemporalProcessor(3600,0.3,0.0002,8,
#result_dir='/Users/Adward/Github/Automatic-Rumor-Detection/TwitterEventDetection/TestData/serialized')
result_dir = '/Volumes/Adward_Backup/SRTP/serialized_test')
ed = EventDetector(TextProcessor(),temp,ts.generator())
ed.process_stream()
from TwitterStream import TwitterStream
api_key = "Put the api key here"
api_secret = "Put the api secret here"
access_token_key = "Put the token key here"
access_token_secret = "Put the token secret here"
url = "https://stream.twitter.com/1.1/statuses/filter.json?locations=-122.995004,32.323198,-67.799695,49.893813 & language=en"
mytweet = TwitterStream(api_key=api_key,
api_secret=api_secret,
access_token_key=access_token_key,
access_token_secret=access_token_secret,
debug=0,
http_method="GET")
mytweet.fetchsamples(url=url, numberoftweets=25000)
output = mytweet.clean_tweets()
proxy = os.environ["http_proxy"]
except KeyError:
print "Not using a proxy. If you want to use a proxy, you need to do something like this"
print "export http_proxy=http://www-cache.your.site.com:3128/"
proxy = None
print username, password, proxy
pids = ["b00001", "b00002", "b00003"]
# keywords = [ "Sarah Jane", "CBBC"] #trending topics earlier
trends_url = "http://api.twitter.com/1/trends/current.json"
raw_trending = get_url(trends_url)
trending_cooked = cjson.decode(raw_trending)
print "Trending Topics", trending_cooked
trending_topics = [X["query"] for X in trending_cooked["trends"].values()[0]]
keywords = trending_topics
# request = [ pids, keywords ]
request = [keywords, pids] # docstring wrong, should be this way round
Pipeline(
DataSource([request]),
TwitterStream(username=username, password=password, proxy=proxy),
PureTransformer(lambda x: repr(x) + "\n"),
ConsoleEchoer(),
).run()
10),
linkages={
("self", "inbox"): ("LINKRESOLVE", "inbox"),
("LINKRESOLVE", "outbox"): ("self", "outbox"),
("LINKRESOLVE", "urlrequests"):
("LINKREQUESTER", "inbox"),
("LINKREQUESTER", "outbox"):
("LINKRESOLVE", "responses")
}).activate()
system = Graphline(
CURRENTPROG=WhatsOn(proxy),
REQUESTER=Requester(
"all", dbuser, dbpass
), # Can set this for specific channels to limit Twitter requests whilst doing dev
FIREHOSE=TwitterStream(
username, password, proxy, True, 40
), # Twitter API sends blank lines every 30 secs so timeout of 40 should be fine
SEARCH=PeopleSearch(consumerkeypair, keypair, proxy),
COLLECTOR=DataCollector(dbuser, dbpass),
RAWCOLLECTOR=RawDataCollector(dbuser, dbpass),
HTTPGETTER=HTTPGetter(proxy, "BBC R&D Grabber", 10),
HTTPGETTERRDF=HTTPGetter(proxy, "BBC R&D Grabber", 10),
TWOWAY=TwoWaySplitter(),
ANALYSIS=LiveAnalysis(dbuser, dbpass),
NLTKANALYSIS=LiveAnalysisNLTK(dbuser, dbpass),
TWEETCLEANER=Pipeline(
LINKER, RetweetFixer(), RetweetCorrector(dbuser, dbpass),
TweetCleaner(['user_mentions', 'urls', 'hashtags'])),
NLTKANALYSISFINAL=FinalAnalysisNLTK(dbuser, dbpass),
TWEETCLEANERFINAL=Pipeline(
LINKERFINAL, RetweetFixer(), RetweetCorrector(dbuser, dbpass),
