def filter_track(follow):
auth = OAuthHandler(consumer_key2, consumer_secret2)
auth.set_access_token(access_token2, access_token_secret2)
stream = Stream(auth, MyStreamListener())
api = API(auth)
#print 'start filter track ', ','.join(track)
stream.filter(track=follow)
def test_follow_encoding(self):
s = Stream(None, None)
s._start = lambda async: None
s.filter(follow=[u'Caf\xe9'])
# Should be UTF-8 encoded
self.assertEqual(u'Caf\xe9'.encode('utf8'), s.session.params['follow'])
def main():
logging.info("bot starting")
bot = Bot()
logging.info("markov data loading")
if load_csv == True:
markov = MarkovGenerator()
logging.info("load_csv == True")
csv_tweets = Bot.load_csv_tweets("./tweets.csv")
#newest_tweet_idを保存
newest_tweet = csv_tweets.next()
save_data(newest_tweet[0], "newest_tweet_id")
markov.add_from_sentence(newest_tweet[7])
#読み込み
for tweet in csv_tweets:
markov.add_from_sentence(tweet)
markov.save_dictionary()
else:
markov = MarkovGenerator(markov_dictionary=load_data("markov_dictionary"), reversed_markov_dictionary=load_data("reversed_markov_dictionary"))
# 20分ごとの定期ツイート
#multiprocess間での値のやり取り用変数
logging.info("regular tweet starting")
p = Process(target=regular_tweet, args=(bot, markov))
p.start()
# userstreamによるTL監視（リプライ・ふぁぼ・パクツイ・RT）
logging.info("start timeline watching")
stream = Stream(bot.auth, AbstractedlyListener(), secure=True)
user_stream = stream.userstream()
user_stream.start()
def test_track_encoding(self):
s = Stream(None, None)
s._start = lambda is_async: None
s.filter(track=['Caf\xe9'])
# Should be UTF-8 encoded
self.assertEqual('Caf\xe9'.encode('utf8'), s.body['track'])
def test_track_encoding(self):
s = Stream(None, None)
s._start = lambda is_async: None
s.filter(track=[u'Caf\xe9'])
# Should be UTF-8 encoded
self.assertEqual(u'Caf\xe9'.encode('utf8'), s.body['track'])
class KeywordStreamer:
'''
Streams in tweets matching a set of terms/from a set of user streams.
'''
def __init__(self, kw_list):
self.kws = kw_list
self.api = get_api()
self.stream = Stream(auth=self.api.auth, listener=MyStreamListener())
def streamTweets(self):
goForIt = True
while goForIt:
try:
goForIt = False
self.stream.filter(track=self.kws)
except IncompleteRead as ex:
try:
tb = traceback.format_exc()
except:
tb = ''
print("%s: %s\n%s\n\n" % (getTimeString(),
str(sys.exc_info()) + "\nStill ticking!", tb))
time.sleep(15 * 60)
goForIt = True
except:
try:
tb = traceback.format_exc()
except:
tb = ''
print('%s: %s\n%s\n\n' % (getTimeString(),
str(sys.exc_info()), tb))
time.sleep(15 * 60)
goForIt = True
def startBot():
print("")
print("Initializing....")
getBanAccounts()
print("getting banned accounts....check")
getBanWords()
print("getting banned words....check")
getTrackWords()
print("getting track words....check")
getFollowAccounts()
print("getting follow accounts....check")
getWhiteListAccounts()
print("getting whitelist accounts....check")
print("")
print("""\
_____ _ _ _ _____ _____ _____
|_ _|_ _ _|_| |_| |_ ___ ___| __ | |_ _|
| | | | | | | _| _| -_| _| __ -| | | | |
|_| |_____|_|_| |_| |___|_| |_____|_____| |_|
created by vishwenga
Running.... :) """)
try:
twt = Stream(auths, listener())
twt.filter(track=track_words)
except Exception as e:
print(str(e))
pass
def init_stream(self):
self.q = Queue()
self.keywords = []
self.listener = TwitterStreamListener(self.keywords, self.q)
auth = OAuthHandler(config.con_secret, config.con_secret_key)
auth.set_access_token(config.token, config.token_key)
self.stream = Stream(auth, self.listener)
def test_follow_encoding(self):
s = Stream(None, None)
s._start = lambda is_async: None
s.filter(follow=[u'Caf\xe9'])
# Should be UTF-8 encoded
self.assertEqual(u'Caf\xe9'.encode('utf8'), s.body['follow'])
def test_track_encoding(self):
s = Stream(None, None)
s._start = lambda async: None
s.filter(track=[u'Caf\xe9'])
# Should be UTF-8 encoded
self.assertEqual(u'Caf\xe9'.encode('utf8'), s.session.params['track'])
class IRCPlugin(BasePlugin):
name = "twitter_stream"
enabled = True
def _validate(self, event):
return False
def stop(self):
if self._stream:
self._stream.disconnect()
def run(self):
try:
auth = tweepy.OAuthHandler(config.TWITTER_CONSUMER_KEY, config.TWITTER_CONSUMER_SECRET)
auth.set_access_token(config.TWITTER_ACCESS_KEY, config.TWITTER_ACCESS_SECRET)
self._stream = Stream(auth, StreamerToIrc(self))
LOG.info("Twitter stream successfully initializing")
except Exception as e:
LOG.error("Twitter stream authorization failed: %s" % e)
return
followers = []
api = API(auth)
for f in config.TWITTER_FOLLOW_IDS:
if isinstance(f, (str, unicode)):
try:
user_id = api.get_user(f).id
followers.append(str(user_id))
except Exception:
LOG.debug("Can't get ID for %s" % user_id)
continue
else:
followers.append(str(f))
self._stream.filter(followers)
def main(api: tweepy.API, main_user):
"""
Authentication and calling the helper function
"""
listner = MyListner(api)
stream = Stream(auth, listner)
ans = stream.filter(follow=[f"{main_user}"], is_async=True)
def main():
inifile = configparser.ConfigParser()
inifile.read('./user.conf')
init = {
'consumer_key': inifile.get('twitter', 'consumer_key'),
'consumer_secret': inifile.get('twitter', 'consumer_secret'),
'access_key': inifile.get('twitter', 'access_key'),
'access_secret': inifile.get('twitter', 'access_secret'),
}
github_user = {
'repo_owner': inifile.get('github', 'repo_owner'),
'repo_name': inifile.get('github', 'repo_name'),
'username': inifile.get('github', 'username'),
'password': inifile.get('github', 'password'),
}
auth = get_oauth(init)
stream = Stream(auth,
AbstractedlyListener(github_user=github_user),
secure=True)
error = 0
while True:
try:
stream.userstream()
except:
error += 1
print(error, traceback.format_exc())
class TweepyStreamTests(unittest.TestCase):
def setUp(self):
self.auth = create_auth()
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener, timeout=3.0)
def tearDown(self):
self.stream.disconnect()
def test_userstream(self):
# Generate random tweet which should show up in the stream.
def on_connect():
API(self.auth).update_status(mock_tweet())
self.listener.connect_cb = on_connect
self.listener.status_stop_count = 1
self.stream.userstream()
self.assertEqual(self.listener.status_count, 1)
def test_userstream_with_params(self):
# Generate random tweet which should show up in the stream.
def on_connect():
API(self.auth).update_status(mock_tweet())
self.listener.connect_cb = on_connect
self.listener.status_stop_count = 1
self.stream.userstream(_with='user',
replies='all',
stall_warnings=True)
self.assertEqual(self.listener.status_count, 1)
def test_sample(self):
self.listener.status_stop_count = 10
self.stream.sample()
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
def test_filter_track(self):
self.listener.status_stop_count = 5
phrases = ['twitter']
self.stream.filter(track=phrases)
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
def test_track_encoding(self):
s = Stream(None, None)
s._start = lambda async: None
s.filter(track=[u'Caf\xe9'])
# Should be UTF-8 encoded
self.assertEqual(u'Caf\xe9'.encode('utf8'), s.session.params['track'])
def test_follow_encoding(self):
s = Stream(None, None)
s._start = lambda async: None
s.filter(follow=[u'Caf\xe9'])
# Should be UTF-8 encoded
self.assertEqual(u'Caf\xe9'.encode('utf8'), s.session.params['follow'])
def filter_track():
track = ["python"]
stream_auth = BasicAuthHandler('<USERNAME>', '<PASSWORD>')
api = API()
stream = Stream(stream_auth, MyStreamListener(api))
print 'start filter track ', ','.join(track)
stream.filter(track=track)
def get_tweets(keyword):
init_tweepy()
auth = tweepy.OAuthHandler('QxpECDyg9eM6inD0jo7Q','a0A8OE2ZN7imCg6WR5ygkrGvKG6tNtoZIChQXQ8NIf4')
auth.set_access_token('18752311-FXmc9zaPGcszH1bdDNJQa0MY2XRYfYzT3nBRnMqgB','tzXURgYPAbsD1VgmchkoKH9QOJ80qGgSSL13K5A3rY')
api = tweepy.API(auth)
listener = Listener()
stream = Stream(auth, listener)
stream.filter(track=[keyword])
def filter_track():
q = Scrapers.all().filter('','')
follow=[s.uid for s in q]
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, MyStreamListener())
api = API(auth)
#print 'start filter track ', ','.join(track)
stream.filter(follow=follow)
class TweepyStreamTests(unittest.TestCase):
def setUp(self):
self.auth = create_auth()
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener, timeout=3.0)
def tearDown(self):
self.stream.disconnect()
def test_userstream(self):
# Generate random tweet which should show up in the stream.
def on_connect():
API(self.auth).update_status(mock_tweet())
self.listener.connect_cb = on_connect
self.listener.status_stop_count = 1
self.stream.userstream()
self.assertEqual(self.listener.status_count, 1)
def test_userstream_with_params(self):
# Generate random tweet which should show up in the stream.
def on_connect():
API(self.auth).update_status(mock_tweet())
self.listener.connect_cb = on_connect
self.listener.status_stop_count = 1
self.stream.userstream(_with='user', replies='all', stall_warnings=True)
self.assertEqual(self.listener.status_count, 1)
def test_sample(self):
self.listener.status_stop_count = 10
self.stream.sample()
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
def test_filter_track(self):
self.listener.status_stop_count = 5
phrases = ['twitter']
self.stream.filter(track=phrases)
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
def test_track_encoding(self):
s = Stream(None, None)
s._start = lambda async: None
s.filter(track=[u'Caf\xe9'])
# Should be UTF-8 encoded
self.assertEqual(u'Caf\xe9'.encode('utf8'), s.parameters['track'])
def test_follow_encoding(self):
s = Stream(None, None)
s._start = lambda async: None
s.filter(follow=[u'Caf\xe9'])
# Should be UTF-8 encoded
self.assertEqual(u'Caf\xe9'.encode('utf8'), s.parameters['follow'])
def main_logger(basename=None):
l = RotatingLogListener(basename=basename)
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, l)
# This fetches ANY geotagged tweet:
# https://dev.twitter.com/docs/streaming-apis/parameters#locations
stream.filter(locations=[-180,-90,180,90])
def my_main():
args = get_parser().parse_args()
auth = OAuthHandler("HqQlPfDvsu6oOVKcbSaC3dwy6",
"Nr801nZ9gJoq5D6x1cQewFJeHDBnyI2IqUsYVePv7sQLLLjwxU")
auth.set_access_token("2344102761-3D84t8gRyGEE2N9tBM99n8oTjGMdDJb12lIAWJ9",
"MEZtBsCwbFjrku0JOnggvlGmqB2O8x8gBC7s65wkjkJGx")
print("Streaming...")
twitter_stream = Stream(auth, Streamer(args.data_dir))
twitter_stream.filter(track=[search_word])
def filter_track():
q = Scrapers.all().filter('', '')
follow = [s.uid for s in q]
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, MyStreamListener())
api = API(auth)
#print 'start filter track ', ','.join(track)
stream.filter(follow=follow)
def test_exp_backoff_cap(self):
self.stream = Stream(self.auth,
self.listener,
timeout=3.0,
retry_count=1,
retry_time=1.0,
retry_time_cap=3.0)
self.stream.sample()
# 1 retry, but 4x the retry_time exceeds the cap, so should be capped
self.assertEqual(self.stream.retry_time, 3.0)
def test_exp_backoff(self):
self.stream = Stream(self.auth,
self.listener,
timeout=3.0,
retry_count=1,
retry_time=1.0,
retry_time_cap=100.0)
self.stream.sample()
# 1 retry, should be 4x the retry_time
self.assertEqual(self.stream.retry_time, 4.0)
def test_420(self):
self.stream = Stream(self.auth,
self.listener,
timeout=3.0,
retry_count=0,
retry_time=1.0,
retry_420=1.5,
retry_time_cap=20.0)
self.stream.sample()
# no retries, but error 420, should be double the retry_420, not double the retry_time
self.assertEqual(self.stream.retry_time, 3.0)
def __init__(self, termPath):
try:
os.mkdir(OUT_DIR)
except:
pass
kwFile = open(termPath, 'r')
self.kws = [line.strip() for line in kwFile if line.strip()]
kwFile.close()
self.stream = Stream(getKeyAndAuth(), TermListener(NUM_TO_CACHE))
def __init__(self):
listener = TweetStreamListener()
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
stream = Stream(auth, listener)
stream.filter(track=[
"soccer transfer", "transfer news", "premier league", "liverpool"
],
is_async=True,
languages=["en"])
logging.basicConfig(filename="tweet_stream.log", level=logging.DEBUG)
logging.info("Streaming Soccer Related Tweets\n")
def train_tweets():
tokyo = LocationFence("Tokyo", 138.946381, 35.523285, 139.953232, 35.906849)
hokkaido = LocationFence("Hokkaido", 139.546509, 41.393294, 145.742798, 45.729191)
kyusyu = LocationFence("Kyusyu", 129.538879, 31.147006, 131.856995, 33.934245)
locations = [tokyo, hokkaido, kyusyu]
request_coordinates = []
for l in locations:
request_coordinates += l.get_coordinates()
stream = Stream(oauth(), Trainer(locations), secure=True)
stream.filter(locations=request_coordinates)
def train_tweets():
tokyo = LocationFence("Tokyo", 138.946381, 35.523285, 139.953232, 35.906849)
hokkaido = LocationFence("Hokkaido", 139.546509, 41.393294, 145.742798, 45.729191)
kyusyu = LocationFence("Kyusyu", 129.538879, 31.147006, 131.856995, 33.934245)
locations = [tokyo, hokkaido, kyusyu]
request_coordinates = []
for l in locations:
request_coordinates += l.get_coordinates()
stream = Stream(oauth(), Trainer(locations), secure=True)
stream.filter(locations=request_coordinates)
def _stream_init(self):
api1 = API()
headers = {}
headers["Accept-Encoding"] = "deflate, gzip"
stream_auth = BasicAuthHandler(social_keys.TWITTER_HTTP_AUTH_U,
social_keys.TWITTER_HTTP_AUTH_P)
l = TestStreamListener(api=api1)
self._stream = Stream(auth=stream_auth,
listener=l,
secure=True,
headers=headers)
def run():
# Authenticates with Twitter
print('Auth')
auth = authenticate()
api = connect_to_api(auth)
print('Handling')
listener = TweepyListener(api)
listener.add_handler(handler=PrintTweetHandler())
print('Streaming')
stream = Stream(auth=auth, listener=listener)
stream.sample()
class TweepyStreamBackoffTests(unittest.TestCase):
def setUp(self):
#bad auth causes twitter to return 401 errors
self.auth = OAuthHandler("bad-key", "bad-secret")
self.auth.set_access_token("bad-token", "bad-token-secret")
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener)
def tearDown(self):
self.stream.disconnect()
def test_exp_backoff(self):
self.stream = Stream(self.auth, self.listener, timeout=3.0,
retry_count=1, retry_time=1.0, retry_time_cap=100.0)
self.stream.sample()
# 1 retry, should be 4x the retry_time
self.assertEqual(self.stream.retry_time, 4.0)
def test_exp_backoff_cap(self):
self.stream = Stream(self.auth, self.listener, timeout=3.0,
retry_count=1, retry_time=1.0, retry_time_cap=3.0)
self.stream.sample()
# 1 retry, but 4x the retry_time exceeds the cap, so should be capped
self.assertEqual(self.stream.retry_time, 3.0)
mock_resp = MagicMock()
mock_resp.return_value.status_code = 420
@patch('requests.Session.request', mock_resp)
def test_420(self):
self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=0,
retry_time=1.0, retry_420=1.5, retry_time_cap=20.0)
self.stream.sample()
# no retries, but error 420, should be double the retry_420, not double the retry_time
self.assertEqual(self.stream.retry_time, 3.0)
class TweepyStreamTests(unittest.TestCase):
def setUp(self):
self.auth = create_auth()
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener, timeout=3.0)
def tearDown(self):
self.stream.disconnect()
def test_userstream(self):
# Generate random tweet which should show up in the stream.
def on_connect():
API(self.auth).update_status(mock_tweet())
self.listener.connect_cb = on_connect
self.listener.status_stop_count = 1
self.stream.userstream()
self.assertEqual(self.listener.status_count, 1)
def test_sample(self):
self.listener.status_stop_count = 10
self.stream.sample()
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
def test_filter_track(self):
self.listener.status_stop_count = 5
phrases = ['twitter']
self.stream.filter(track=phrases)
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
class TweepyStreamTests(unittest.TestCase):
def setUp(self):
self.auth = create_auth()
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener, timeout=3.0)
def tearDown(self):
self.stream.disconnect()
def test_userstream(self):
# Generate random tweet which should show up in the stream.
def on_connect():
API(self.auth).update_status(mock_tweet())
self.listener.connect_cb = on_connect
self.listener.status_stop_count = 1
self.stream.userstream()
self.assertEqual(self.listener.status_count, 1)
def test_sample(self):
self.listener.status_stop_count = 10
self.stream.sample()
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
def test_filter_track(self):
self.listener.status_stop_count = 5
phrases = ['twitter']
self.stream.filter(track=phrases)
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
class TweepyStreamBackoffTests(unittest.TestCase):
def setUp(self):
#bad auth causes twitter to return 401 errors
self.auth = OAuthHandler("bad-key", "bad-secret")
self.auth.set_access_token("bad-token", "bad-token-secret")
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener)
def tearDown(self):
self.stream.disconnect()
def test_exp_backoff(self):
self.stream = Stream(self.auth, self.listener, timeout=3.0,
retry_count=1, retry_time=1.0, retry_time_cap=100.0)
self.stream.sample()
# 1 retry, should be 4x the retry_time
self.assertEqual(self.stream.retry_time, 4.0)
def test_exp_backoff_cap(self):
self.stream = Stream(self.auth, self.listener, timeout=3.0,
retry_count=1, retry_time=1.0, retry_time_cap=3.0)
self.stream.sample()
# 1 retry, but 4x the retry_time exceeds the cap, so should be capped
self.assertEqual(self.stream.retry_time, 3.0)
mock_resp = MagicMock()
mock_resp.return_value.status = 420
@patch(getresponse_location, mock_resp)
def test_420(self):
self.stream = Stream(self.auth, self.listener, timeout=3.0, retry_count=0,
retry_time=1.0, retry_420=1.5, retry_time_cap=20.0)
self.stream.sample()
# no retries, but error 420, should be double the retry_420, not double the retry_time
self.assertEqual(self.stream.retry_time, 3.0)
def main():
try:
config = get_twitter_config(CONFIG_FILE, TWITTER_SCREEN_NAME)
auth = tweepy.OAuthHandler(config['consumer_key'], config['consumer_secret'])
auth.set_access_token(config['access_key'], config['access_secret'])
# auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
# auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)
listener = Listener()
stream = Stream(auth, listener, timeout=None,secure=1)
#stream = Stream(USERNAME, PASSWORD, listener, timeout=None)
# sample returns all tweets
stream.filter(track = KEYWORDS)
except KeyboardInterrupt:
print '\nGoodbye!'
def do_stream(self):
listener = twitter_listen()
auth = tweepy.OAuthHandler(credentials.CONSUMER_KEY,
credentials.CONSUMER_SECRET)
api = tweepy.API(auth)
auth.set_access_token(credentials.ACCESS_TOKEN,
credentials.ACCESS_TOKEN_SECRET)
stream = Stream(auth, listener)
#filter tweets
stream.filter(languages=["en"],
track=[
'Storm', 'Winter', 'Canada', 'Temprature', 'Flu',
'Snow', 'Indoor', 'Safety'
])
def _start_stream(self):
self.listener = Streamer(self)
self._auth()
self.stream = Stream(self.auth, self.listener)
self.stream.userstream()
def __init__(self,
stream_ref,
auth_params,
stream_manager,
db=None,
bot_user_agent=None):
"""
:param username: twitter username
:param auth: tuple of auth credentials representing twitter user
:param mstream: multi stream container owning this stream
`mstream` should implement listener protocol to handle user events
"""
# self.me = self.username = stream_id
self.stream_ref = stream_ref
self.stream_id = stream_ref.id
self.auth_params = auth_params
self.auth = TwitterApiWrapper.auth_from_settings(*auth_params)
self.api = TwitterApiWrapper.make_api(self.auth)
self.listener = self.ListenerCls()(self, stream_manager, db=db)
headers = {'X-User-Agent': bot_user_agent} if bot_user_agent else {}
self.stream = Stream(self.auth, self.listener, headers=headers)
self.listener.log_event('initialized')
if on_test():
self.test_stream = TestStream(self.listener)
def __listen(self):
listener = TweetStreamListener(self.api, self.sentiment, self.error)
stream = Stream(self.auth, listener)
print 'Starting stream...'
try:
stream.filter(track=self.track)
except:
print 'Encountered error!'
print 'Exiting application'
item = {
'status' : 'stream down',
'timestamp' : datetime.utcnow()
}
self.error.save(item)
stream.disconnect()
def stream(self):
self.botname = self.api.me().screen_name
print "Bot started: @" + self.botname
self.taghandler = TagHandler(self.api, self.botname)
self.customhandler = CustomHandler(self.api)
self.listener = Listener(self.botname, self.taghandler, self.customhandler)
self.stream = Stream(self.auth, self.listener)
self.stream.filter(track=(self.botname, ))
class KeywordStreamer:
'''
Streams in tweets matching a set of terms/from a set of user streams.
'''
def __init__(self, termPath):
try:
os.mkdir(OUT_DIR)
except:
pass
kwFile = open(termPath, 'r')
self.kws = [line.strip() for line in kwFile if line.strip()]
kwFile.close()
self.stream = Stream(getKeyAndAuth(), TermListener(NUM_TO_CACHE))
def streamTweets(self, searchForTerms=True):
goForIt = True
while goForIt:
try:
goForIt = False
if searchForTerms:
self.stream.filter(track=self.kws, locations=LOCATIONS)
else:
self.stream.filter(follow=[int(k) for k in self.kws],
locations=LOCATIONS)
#except IncompleteRead as ex:
# try:
# tb = traceback.format_exc()
# except:
# tb = ''
# emailAlert("%s: %s\n%s\n\n" % (getTimeString(),
# str(sys.exc_info()) + "\nStill ticking!", tb))
# zzz(SNOOZE)
# goForIt = True
except:
try:
tb = traceback.format_exc()
except:
tb = ''
emailAlert('%s: %s\n%s\n\n' % (getTimeString(),
str(sys.exc_info()), tb))
zzz(SNOOZE)
goForIt = True
def _stream_init(self):
api1 = API()
headers = {}
headers["Accept-Encoding"] = "deflate, gzip"
stream_auth = BasicAuthHandler(social_keys.TWITTER_HTTP_AUTH_U,social_keys.TWITTER_HTTP_AUTH_P)
l = TestStreamListener(api=api1)
self._stream = Stream(auth=stream_auth,listener=l,secure=True,headers=headers)
class Crawler():
''' The global crawler manager '''
def __init__(self, name):
self.name = name
self.regexp = re.compile(r'@%s\s+' % (self.name),
re.IGNORECASE)
self._start_stream()
def _auth(self):
self.auth = OAuthHandler(Ukey, Usecret)
self.auth.set_access_token(Akey, Asecret)
self.api = API(self.auth)
def _start_stream(self):
self.listener = Streamer(self)
self._auth()
self.stream = Stream(self.auth, self.listener)
self.stream.userstream()
def on_request(self, text, author):
text = re.sub(self.regexp, '', text)
text = re.sub(r'\s+$', '', text)
lang = get_language(text, key=u'startword')
if lang is not None:
core = Core(lang)
self.send_message(core.send_message(), author)
def send_message(self, text, target=None):
if(target is not None):
text = u'@%s %s' % (target, text)
print(text)
print '>>>', len(text)
self.api.update_status(text)
def __init__(self, settings, publishers):
"""Constructor."""
if "consumer_key" not in settings:
raise ValueError("consumer_key must not be None")
if "consumer_secret" not in settings:
raise ValueError("consumer_secret must not be None")
if "access_token" not in settings:
raise ValueError("access_token must not be None")
if "access_secret" not in settings:
raise ValueError("access_secret must not be None")
if "fields" not in settings:
raise ValueError("access_secret must not be None")
self.auth = OAuthHandler(settings["consumer_key"],
settings["consumer_secret"])
self.auth.set_access_token(settings["access_token"],
settings["access_secret"])
twitter_stream = Stream(self.auth,
StdOutListener(settings["fields"], publishers))
twitter_stream.filter(track=settings["post_strings"])
def main():
""" Main run function """
set_logging()
logger = logging.getLogger(f"{__name__}.main")
auth = tweepy.OAuthHandler(os.getenv("CONSUMER_KEY"),
os.getenv("CONSUMER_KEY_SECRET"))
auth.set_access_token(os.getenv("ACCESS_TOKEN"),
os.getenv("ACCESS_TOKEN_SECRET"))
api = tweepy.API(auth,
wait_on_rate_limit=True,
wait_on_rate_limit_notify=True)
following = get_following_users(api)
following.extend(get_default_users(api))
tracks = get_following_searches()
logger.info(f"Following {len(following)} users and {len(tracks)} searches")
while True:
listener = Listener()
stream = Stream(auth=api.auth, listener=listener)
try:
logger.info("Started streaming")
stream.filter(follow=following, track=tracks)
except KeyboardInterrupt:
logger.info("Stopped")
break
except ReadTimeoutError as exc:
logger.error("Handled exception:", str(exc), exc_info=True)
finally:
logger.info("Done")
stream.disconnect()
def main():
try:
parser = ArgumentParser()
parser.add_argument("oauth_keys_file",
help="The location of a file containing the application " +
"oath consumer key, consumer secret, " +
"access token, and access token secret, " +
"each on its own line, in that order. " +
"See the tweepy example on oauth to figure " +
"out how to get these keys."
)
parser.add_argument("mail_settings_file", help="The automail settings file for sending emails.")
args = parser.parse_args()
(consumer_key, consumer_secret, access_token, access_token_secret) = \
parse_oauth_file(args.oauth_keys_file)
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
pl = PaxListener(args.mail_settings_file, auth)
#open a stream, with ourself as the listener
stream = Stream(auth, pl)
#stream.filter(track=["the"])
pax_user_id = '26281970' #follow requires userid, found at mytwitterid.com
daemon_user_id = '1954653840'
stream.filter(follow=[pax_user_id]) #track ignores follow, pulls from firehose regardless (this is testing acct)
except BaseException as e:
subject = "Exception from paxtix"
exc_type, exc_value, exc_traceback = sys.exc_info()
message = '\n'.join(["Pax tix listener has hit an exception!",] +
traceback.format_exception(exc_type, exc_value, exc_traceback),
)
send_email(parse_settings(args.mail_settings_file), subject, message)
traceback.print_exception(exc_type, exc_value, exc_traceback)
class Bot:
consumer_key='FSWJUlCHsAaiLBDY1MMEA'
consumer_secret='aCFv6Tslb038cdUAIv7m4QDplBn076wvLd4YKmC3yU'
key = "135845084-FONlvU4wGXyl9o4RkfyPdv4grclbqr4Pi2UH2w3j"
secret = "DC6I1xp2uONTIYOL2DtEIljhl46LEkUwlPB5NUEBds"
def __init__(self):
self.auth = tweepy.OAuthHandler(consumer_key = self.consumer_key,
consumer_secret = self.consumer_secret)
self.auth.set_access_token(self.key, self.secret)
self.api = tweepy.API(self.auth)
def auth(self):
try:
redirect_url = self.auth.get_authorization_url()
except tweepy.TweepError:
print 'Error! Failed to get request token.'
verifier = raw_input('Verifier:')
try:
self.auth.get_access_token(verifier)
except tweepy.TweepError:
print 'Error! Failed to get access token.'
self.key, self.secret = self.auth.access_token.key, self.auth.access_token.secret
print self.key, self.secret
def stream(self):
self.botname = self.api.me().screen_name
print "Bot started: @" + self.botname
self.taghandler = TagHandler(self.api, self.botname)
self.customhandler = CustomHandler(self.api)
self.listener = Listener(self.botname, self.taghandler, self.customhandler)
self.stream = Stream(self.auth, self.listener)
self.stream.filter(track=(self.botname, ))
def handle(self, *args, **options):
if not args or len(args) < 3:
self.stdout.write('No hashtags and/or options provided.\n')
else:
# Fetch info from DB
hashtag_name = args.pop(0)
hashtag = HashTag.objects.get(name=hashtag_name)
options = VoteOption.objects.filter(hashtag=hashtag)
self.stdout.write('Following: %s\n' % hashtag_name)
self.stdout.write('Options: %s\n' % ', '.join(options))
self.stdout.flush()
# OAuth connect
consumer_key = settings.TWITTERVOTER['consumer_key']
consumer_secret = settings.TWITTERVOTER['consumer_secret']
access_token = settings.TWITTERVOTER['access_token']
access_token_secret = settings.TWITTERVOTER['access_token_secret']
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
# Connect too the streaming API.
mylisten = MyStreamListener(self.stdout)
mystream = Stream(auth, mylisten, timeout=30)
mystream.filter(track=args)
class TweepyStreamTests(unittest.TestCase):
def setUp(self):
self.auth = create_auth()
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener, timeout=3.0)
def tearDown(self):
self.stream.disconnect()
def test_userstream(self):
# Generate random tweet which should show up in the stream.
def on_connect():
API(self.auth).update_status(mock_tweet())
self.listener.connect_cb = on_connect
self.listener.status_stop_count = 1
self.stream.userstream()
self.assertEqual(self.listener.status_count, 1)
def test_sample(self):
self.listener.status_stop_count = 10
self.stream.sample()
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
def test_filter_track(self):
self.listener.status_stop_count = 5
phrases = ['twitter']
self.stream.filter(track=phrases)
self.assertEquals(self.listener.status_count,
self.listener.status_stop_count)
def test_on_data(self):
test_wrong_data = [
'{"disc', # this is actual data read from twitter
'600', # this is actual data read from twitter
'41\n', # this is actual data read from twitter
'obviously non-json',
'"json but not dict"',
'{"json dict":"but not a twitter message"}'
]
for raw_data in test_wrong_data:
# should log errors but not raise / not return False
self.assertEquals(self.listener.on_data(raw_data), None)
self.assertEquals(self.listener.status_count, 0)
def doStreaming(self):
class StdOutListener(StreamListener):
def __init__(self, twitterObject):
super(StdOutListener, self).__init__()
self.twitterObject = twitterObject
def on_status(self, status):
logger.info("adds new tweeter to queue with id %s: %s" % (status.id, status.text))
self.twitterObject.tweetsQueue.put(status)
return True
def on_error(self, status):
logger.warning("Error %s while reading twitter" % status)
listener = StdOutListener(self)
self.stream = Stream(self.auth, listener)
self.stream.filter(track=self.filter, async=True)
class StreamsTestsPlain(object):
def __init__(self):
self._stream = None
self._oauth = None
self._oauth_api = None
self._stream_init()
self._oauth_init()
def _oauth_init(self):
self._oauth = OAuthHandler(social_keys.TWITTER_CONSUMER_KEY,
social_keys.TWITTER_CONSUMER_SECRET)
self._oauth.set_access_token(
social_keys.TWITTER_APP_ACCESS_TOKEN,
social_keys.TWITTER_APP_ACCESS_TOKEN_SECRET)
self._oauth_api = API(self._oauth)
def _stream_init(self):
api1 = API()
headers = {}
headers["Accept-Encoding"] = "deflate, gzip"
stream_auth = BasicAuthHandler(social_keys.TWITTER_HTTP_AUTH_U,
social_keys.TWITTER_HTTP_AUTH_P)
l = TestStreamListener(api=api1)
self._stream = Stream(auth=stream_auth,
listener=l,
secure=True,
headers=headers)
def sample(self):
self._stream.sample()
def filter_follow(self):
follow_names = [
'hnfirehose', 'StockTwits', 'YahooFinance', 'Street_Insider',
'TheStreet', 'SquawkCNBC', 'CNBC', 'AP_PersonalFin',
'themotleyfool', 'MarketWatch', 'Reuters_Biz'
]
follow_usr_objs = self._oauth_api.lookup_users(
screen_names=follow_names)
follow_ids = []
for follow_usr in follow_usr_objs:
follow_ids.append(follow_usr.id)
print follow_ids
self._stream.filter(follow=follow_ids)
def filter(self):
self._stream.filter(track=["@newsdotme", "@twixmit", "@app_engine"])
def run(self):
try:
auth = tweepy.OAuthHandler(config.TWITTER_CONSUMER_KEY, config.TWITTER_CONSUMER_SECRET)
auth.set_access_token(config.TWITTER_ACCESS_KEY, config.TWITTER_ACCESS_SECRET)
self._stream = Stream(auth, StreamerToIrc(self))
LOG.info("Twitter stream successfully initializing")
except Exception as e:
LOG.error("Twitter stream authorization failed: %s" % e)
return
followers = []
api = API(auth)
for f in config.TWITTER_FOLLOW_IDS:
if isinstance(f, (str, unicode)):
try:
user_id = api.get_user(f).id
followers.append(user_id)
except Exception:
LOG.debug("Can't get ID for %s" % user_id)
continue
else:
followers.append(f)
self._stream.filter(followers)
class StreamsTestsPlain(object):
def __init__(self):
self._stream = None
self._oauth = None
self._oauth_api = None
self._stream_init()
self._oauth_init()
def _oauth_init(self):
self._oauth = OAuthHandler(social_keys.TWITTER_CONSUMER_KEY, social_keys.TWITTER_CONSUMER_SECRET)
self._oauth.set_access_token(social_keys.TWITTER_APP_ACCESS_TOKEN,social_keys.TWITTER_APP_ACCESS_TOKEN_SECRET)
self._oauth_api = API(self._oauth)
def _stream_init(self):
api1 = API()
headers = {}
headers["Accept-Encoding"] = "deflate, gzip"
stream_auth = BasicAuthHandler(social_keys.TWITTER_HTTP_AUTH_U,social_keys.TWITTER_HTTP_AUTH_P)
l = TestStreamListener(api=api1)
self._stream = Stream(auth=stream_auth,listener=l,secure=True,headers=headers)
def sample(self):
self._stream.sample()
def filter_follow(self):
follow_names = ['hnfirehose','StockTwits','YahooFinance','Street_Insider','TheStreet','SquawkCNBC','CNBC','AP_PersonalFin','themotleyfool','MarketWatch','Reuters_Biz']
follow_usr_objs = self._oauth_api.lookup_users(screen_names=follow_names)
follow_ids = []
for follow_usr in follow_usr_objs:
follow_ids.append(follow_usr.id)
print follow_ids
self._stream.filter(follow=follow_ids)
def filter(self):
self._stream.filter(track=["@newsdotme","@twixmit","@app_engine"])
class TweepyStreamTests(unittest.TestCase):
def setUp(self):
self.auth = create_auth()
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener, timeout=3.0)
def tearDown(self):
self.stream.disconnect()
def on_connect(self):
API(self.auth).update_status(mock_tweet())
def test_sample(self):
self.listener.status_stop_count = 10
self.stream.sample()
self.assertEqual(self.listener.status_count,
self.listener.status_stop_count)
def test_filter_track(self):
self.listener.status_stop_count = 5
phrases = ['twitter']
self.stream.filter(track=phrases)
self.assertEqual(self.listener.status_count,
self.listener.status_stop_count)
def test_track_encoding(self):
s = Stream(None, None)
s._start = lambda is_async: None
s.filter(track=['Caf\xe9'])
# Should be UTF-8 encoded
self.assertEqual('Caf\xe9'.encode('utf8'), s.body['track'])
def test_follow_encoding(self):
s = Stream(None, None)
s._start = lambda is_async: None
s.filter(follow=['Caf\xe9'])
# Should be UTF-8 encoded
self.assertEqual('Caf\xe9'.encode('utf8'), s.body['follow'])
def setUp(self):
self.auth = create_auth()
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener, timeout=3.0)
def retweet_by_all(status):
for auth in get_client_auths():
api = tweepy.API(auth)
api.retweet(status["id"])
print "i will retweet", status["id"], status["text"]
class StdOutListener(StreamListener):
# listens to stream and will start retweet_by_all if condition
def on_data(self, data):
status = json.loads(data)
if condition(status):
retweet_by_all(status)
def on_error(self, status):
print status
if __name__ == "__main__":
# starts main loop
auth = get_main_identity_auth()
api = tweepy.API(auth)
stream = Stream(auth, StdOutListener(StreamListener))
stream.filter(config["listen"])
def setUp(self):
self.auth = create_auth()
self.listener = MockStreamListener(self)
self.stream = Stream(self.auth, self.listener, timeout=3.0)
self.output_filenames = output_filenames
self.OpenOutputFiles()
def ReloadOutputFiles(self):
self.CloseOutputFiles()
self.OpenOutputFiles()
""" Let's stare abstractedly at the User Streams ! """
def on_status(self, status):
dump_text = json.dumps(status._json) + "\n"
if self.output_files is None:
print dump_text
sys.stdout.flush()
return
for file in self.output_files:
file.write(dump_text)
file.flush()
listener = Listener()
def SigHupHandler(signum, frame):
listener.ReloadOutputFiles()
if __name__ == '__main__':
signal.signal(signal.SIGHUP, SigHupHandler)
auth = get_oauth()
listener.SetOutputFilenames(sys.argv[1:])
stream = Stream(auth, listener, secure=True)
daemon.daemonize(PID_FILE_NAME)
stream.userstream()
class TweepyLib:
CONFIG_FILENAME = "twitterConfig.ini" # name of the config file defining e.g. the Twitter API key
CONFIG_SECTION_KEYS = "keys" # name of the section of the config file holding the various Twitter API keys/tokens/etc.
def __init__(self):
config = RawConfigParser()
configFilePath = os.path.join(os.path.dirname(__file__), self.__class__.CONFIG_FILENAME)
logger.info("HELLO WORLD")
logger.info(configFilePath)
config.read(configFilePath)
# Read API keys/tokens from config file
self.consumer_key = config.get(self.__class__.CONFIG_SECTION_KEYS, "consumer_key")
self.consumer_secret = config.get(self.__class__.CONFIG_SECTION_KEYS, "consumer_secret")
self.access_token = config.get(self.__class__.CONFIG_SECTION_KEYS, "access_token")
self.access_token_secret = config.get(self.__class__.CONFIG_SECTION_KEYS, "access_token_secret")
self.auth = OAuthHandler(self.consumer_key, self.consumer_secret)
self.isStreaming=False
self.connect()
self.tweetsQueue = Queue.Queue()
self.filter=[]
def connect(self):
#todo: recheck needed auth
self.auth.set_access_token(self.access_token, self.access_token_secret)
logger.info("Connected with Twitter")
def doStreaming(self):
class StdOutListener(StreamListener):
def __init__(self, twitterObject):
super(StdOutListener, self).__init__()
self.twitterObject = twitterObject
def on_status(self, status):
logger.info("adds new tweeter to queue with id %s: %s" % (status.id, status.text))
self.twitterObject.tweetsQueue.put(status)
return True
def on_error(self, status):
logger.warning("Error %s while reading twitter" % status)
listener = StdOutListener(self)
self.stream = Stream(self.auth, listener)
self.stream.filter(track=self.filter, async=True)
def startStreamer(self):
if(not self.filter):
logger.warning("Trying to start stream, but the filter is empty")
return
self.doStreaming()
logger.info("Streamer was started")
self.isStreaming = True
def stopStreamer(self):
if(self.isStreaming):
self.isStreaming = False
self.stream.disconnect()
logger.info("Streamer was stopped")
def restartStreamer(self):
self.stopStreamer()
self.startStreamer()
def addHashtagToFilter(self, newHashtag):
if(newHashtag[:1] != '#'):
newHashtag = '#' + newHashtag
if(newHashtag in self.filter):
return
self.filter.append(newHashtag.encode())
logger.info("%s added to the filter" % newHashtag)
self.restartStreamer()
def removeHashtagFromFilter(self, existingHashtag):
if(existingHashtag[:1] != '#'):
existingHashtag = '#' + existingHashtag
if(existingHashtag in self.filter):
return
self.filter.remove(existingHashtag)
logger.info("%s removed from the filter" % existingHashtag)
self.restartStreamer()
def removeAllHashtagsFromFilter(self):
changes = 0
for i in self.filter:
# we only want to rmeove hashtags, not other filters
if(i[:1] == '#'):
changes += 1
self.filter.remove(i)
logger.info("All hashtags removed from the filter")
if(changes != 0):
self.restartStreamer()
