def __init__(self, username, password, listener, timeout=5.0, retry_count = None,
retry_time = 10.0, snooze_time = 5.0, buffer_size=1500):
self.auth = BasicAuthHandler(username, password)
self.running = False
self.timeout = timeout
self.retry_count = retry_count
self.retry_time = retry_time
self.snooze_time = snooze_time
self.buffer_size = buffer_size
self.listener = listener
self.api = API()
self.headers = {}
self.body = None
def filter_track():
track = ["python"]
stream_auth = BasicAuthHandler('<USERNAME>', '<PASSWORD>')
api = API()
stream = Stream(stream_auth, MyStreamListener(api))
print 'start filter track ', ','.join(track)
stream.filter(track=track)
def build_stream(self):
if self.stream != None:
self.stop_query()
time.sleep(.01) # make sure old stream has time to disconnect
self.stream = Stream(BasicAuthHandler(self.username, self.password),
self, # this object implements StreamListener
timeout = 600, # reconnect if no messages in 600s
retry_count = 20, # try reconnecting 20 times
retry_time = 10.0, # wait 10s if no HTTP 200
snooze_time = 1.0) # wait 1s if timeout in 600s
def __init__(self, username, password, listener, timeout=5.0, retry_count = None,
retry_time = 10.0, snooze_time = 5.0, buffer_size=1500):
self.auth = BasicAuthHandler(username, password)
self.running = False
self.timeout = timeout
self.retry_count = retry_count
self.retry_time = retry_time
self.snooze_time = snooze_time
self.buffer_size = buffer_size
self.listener = listener
self.api = API()
def _stream_init(self):
api1 = API()
headers = {}
headers["Accept-Encoding"] = "deflate, gzip"
stream_auth = BasicAuthHandler(social_keys.TWITTER_HTTP_AUTH_U,
social_keys.TWITTER_HTTP_AUTH_P)
l = TestStreamListener(api=api1)
self._stream = Stream(auth=stream_auth,
listener=l,
secure=True,
headers=headers)
def basic_authenticate(self, file_name):
"""
Creates an Authhandler using Basic method
"""
try:
with open(file_name) as f:
cred = f.readline().split(',')
except IOError:
print "Error : Authentication file not found"
if len(cred) != 2:
print "Error : Expecting to retrieve 2 values"
else:
#username, password
self.auth = BasicAuthHandler(cred[0], cred[1])
def __init__(self, username, password, listener, timeout=5.0, retry_count = None,
retry_time = 10.0, snooze_time = 5.0, buffer_size=1500, headers=None,
proxy_host=None,proxy_port=80):
self.auth = BasicAuthHandler(username, password)
self.running = False
self.timeout = timeout
self.retry_count = retry_count
self.retry_time = retry_time
self.snooze_time = snooze_time
self.buffer_size = buffer_size
self.listener = listener
self.api = API()
self.headers = headers or {}
self.body = None
self.proxy_host = proxy_host
self.proxy_port = proxy_port
def main():
global follow_list
infoModule.info.site['dblink'] = mysql_tools.db_connect()
auth = BasicAuthHandler(twitterAccount, tiwtterPassword)
stream = newStream(auth, StreamWatcherListener(), timeout=None)
follow_list = getUserList()
# Demo Mode
#follow_list = getUserList(True)
infoModule.info.site['dblink'].close()
if len(follow_list) == 0:
file_put_contents('Could not get a list of users to follow')
sys.exit()
stream.filter(follow=follow_list)
def train_tweets():
tokyo = LocationFence("Tokyo", 138.946381, 35.523285, 139.953232,
35.906849)
hokkaido = LocationFence("Hokkaido", 139.546509, 41.393294, 145.742798,
45.729191)
kyusyu = LocationFence("Kyusyu", 129.538879, 31.147006, 131.856995,
33.934245)
locations = [tokyo, hokkaido, kyusyu]
request_coordinates = []
for l in locations:
request_coordinates += l.get_coordinates()
stream = Stream(BasicAuthHandler(tw_username, tw_password),
Trainer(locations),
secure=True)
stream.filter(locations=request_coordinates)
class Stream(object):
host = 'stream.twitter.com'
def __init__(self, username, password, listener, timeout=5.0, retry_count = None,
retry_time = 10.0, snooze_time = 5.0, buffer_size=1500):
self.auth = BasicAuthHandler(username, password)
self.running = False
self.timeout = timeout
self.retry_count = retry_count
self.retry_time = retry_time
self.snooze_time = snooze_time
self.buffer_size = buffer_size
self.listener = listener
self.api = API()
self.headers = {}
self.body = None
def _run(self):
# setup
self.auth.apply_auth(None, None, self.headers, None)
# enter loop
error_counter = 0
conn = None
while self.running:
if self.retry_count and error_counter > self.retry_count:
# quit if error count greater than retry count
break
try:
conn = httplib.HTTPConnection(self.host)
conn.connect()
conn.sock.settimeout(self.timeout)
conn.request('POST', self.url, self.body, headers=self.headers)
resp = conn.getresponse()
if resp.status != 200:
if self.listener.on_error(resp.status) is False:
break
error_counter += 1
sleep(self.retry_time)
else:
error_counter = 0
self._read_loop(resp)
except timeout:
if self.listener.on_timeout() == False:
break
if self.running is False:
break
conn.close()
sleep(self.snooze_time)
except Exception:
# any other exception is fatal, so kill loop
break
# cleanup
self.running = False
if conn:
conn.close()
def _read_loop(self, resp):
data = ''
while self.running:
if resp.isclosed():
break
# read length
length = ''
while True:
c = resp.read(1)
if c == '\n':
break
length += c
length = length.strip()
if length.isdigit():
length = int(length)
else:
continue
# read data and pass into listener
data = resp.read(length)
if self.listener.on_data(data) is False:
self.running = False
def _start(self, async):
self.running = True
if async:
Thread(target=self._run).start()
else:
self._run()
def firehose(self, count=None, async=False):
if self.running:
raise TweepError('Stream object already connected!')
self.url = '/%i/statuses/firehose.json?delimited=length' % STREAM_VERSION
if count:
self.url += '&count=%s' % count
self._start(async)
conn = connect(opts.dsn)
cursor = conn.cursor()
# Don't know how this happens
if status.geo == None:
print 'No geo information'
return
if status.geo['type'] != 'Point':
print 'Geo is of type ' + status.geo['type']
return
# Insert the status into the DB.
wkt = 'SRID=4326;POINT(%s %s)' % (status.geo['coordinates'][1],
status.geo['coordinates'][0])
cursor.execute(
'INSERT INTO ' + opts.table +
' (screenname, tweet, the_geom) VALUES ' +
'(%s, %s, ST_GeomFromEWKT(%s))',
(status.user.screen_name, status.text, wkt))
conn.commit()
listener = PGSQLListener()
auth = BasicAuthHandler(args[0], args[1])
s = Stream(auth, listener, secure=True)
#s.filter(track="foothill college")
s.filter(locations=[float(i) for i in opts.geo.split(',')])
class Stream(object):
host = 'stream.twitter.com'
def __init__(self, username, password, listener, timeout=5.0, retry_count = None,
retry_time = 10.0, snooze_time = 5.0, buffer_size=1500):
self.auth = BasicAuthHandler(username, password)
self.running = False
self.timeout = timeout
self.retry_count = retry_count
self.retry_time = retry_time
self.snooze_time = snooze_time
self.buffer_size = buffer_size
self.listener = listener
self.api = API()
self.headers = {}
self.body = None
def _run(self):
print 'run....'
# setup
self.auth.apply_auth(None, None, self.headers, None)
# enter loop
error_counter = 0
conn = None
while self.running:
if self.retry_count and error_counter > self.retry_count:
# quit if error count greater than retry count
break
try:
conn = httplib.HTTPConnection(self.host)
conn.connect()
conn.sock.settimeout(self.timeout)
conn.request('POST', self.url, self.body, headers=self.headers)
resp = conn.getresponse()
if resp.status != 200:
if self.listener.on_error(resp.status) is False:
break
error_counter += 1
sleep(self.retry_time)
else:
error_counter = 0
self._read_loop(resp)
except timeout:
if self.listener.on_timeout() == False:
break
if self.running is False:
break
conn.close()
sleep(self.snooze_time)
except Exception:
# any other exception is fatal, so kill loop
break
# cleanup
self.running = False
if conn:
conn.close()
def _read_loop(self, resp):
data = ''
while self.running:
if resp.isclosed():
break
# read length
length = ''
while True:
c = resp.read(1)
if c == '\n':
break
length += c
length = length.strip()
if length.isdigit():
length = int(length)
else:
continue
# read data and pass into listener
data = resp.read(length)
if self.listener.on_data(data) is False:
self.running = False
def _start(self, async):
self.running = True
if async:
Thread(target=self._run).start()
else:
self._run()
def firehose(self, count=None, async=False):
if self.running:
raise TweepError('Stream object already connected!')
self.url = '/%i/statuses/firehose.json?delimited=length' % STREAM_VERSION
if count:
self.url += '&count=%s' % count
self._start(async)
class Stream(object):
host = 'stream.twitter.com'
def __init__(self, username, password, listener, timeout=5.0, retry_count = None,
retry_time = 10.0, snooze_time = 5.0, buffer_size=1500, headers=None,
proxy_host=None,proxy_port=80):
self.auth = BasicAuthHandler(username, password)
self.running = False
self.timeout = timeout
self.retry_count = retry_count
self.retry_time = retry_time
self.snooze_time = snooze_time
self.buffer_size = buffer_size
self.listener = listener
self.api = API()
self.headers = headers or {}
self.body = None
self.proxy_host = proxy_host
self.proxy_port = proxy_port
def _run(self):
# setup
self.auth.apply_auth(None, None, self.headers, None)
# enter loop
error_counter = 0
conn = None
exception = None
while self.running:
if self.retry_count and error_counter > self.retry_count:
# quit if error count greater than retry count
break
try:
if not self.proxy_host:
conn = httplib.HTTPConnection(self.host)
else:
conn = httplib.HTTPConnection(self.proxy_host, self.proxy_port)
conn.connect()
conn.sock.settimeout(self.timeout)
_url=self.url
if self.proxy_host:
_url="http://" + self.host + self.url
conn.request('POST', _url, self.body, headers=self.headers)
resp = conn.getresponse()
if resp.status != 200:
if self.listener.on_error(resp.status) is False:
break
error_counter += 1
sleep(self.retry_time)
else:
error_counter = 0
self._read_loop(resp)
except timeout:
if self.listener.on_timeout() == False:
break
if self.running is False:
break
conn.close()
sleep(self.snooze_time)
except Exception, exception:
# any other exception is fatal, so kill loop
break
# cleanup
self.running = False
if conn:
conn.close()
if exception:
raise exception
class Stream(object):
host = 'stream.twitter.com'
def __init__(self, username, password, listener, timeout=5.0, retry_count = None,
retry_time = 10.0, snooze_time = 5.0, buffer_size=1500):
self.auth = BasicAuthHandler(username, password)
self.running = False
self.timeout = timeout
self.retry_count = retry_count
self.retry_time = retry_time
self.snooze_time = snooze_time
self.buffer_size = buffer_size
self.listener = listener
self.api = API()
def _run(self):
# setup
headers = {}
self.auth.apply_auth(None, None, headers, None)
# enter loop
error_counter = 0
conn = None
while self.running:
if self.retry_count and error_counter > self.retry_count:
# quit if error count greater than retry count
break
try:
conn = httplib.HTTPConnection(self.host)
conn.connect()
conn.sock.settimeout(self.timeout)
conn.request('POST', self.url, headers=headers)
resp = conn.getresponse()
if resp.status != 200:
if self.listener.on_error(resp.status) is False:
break
error_counter += 1
sleep(self.retry_time)
else:
error_counter = 0
self._read_loop(resp)
except timeout:
if self.listener.on_timeout() == False:
break
if self.running is False:
break
conn.close()
sleep(self.snooze_time)
except Exception:
# any other exception is fatal, so kill loop
break
# cleanup
self.running = False
if conn:
conn.close()
def _read_loop(self, resp):
data = ''
while self.running:
if resp.isclosed():
break
# read length
length = ''
while True:
c = resp.read(1)
if c == '\n':
break
length += c
length = length.strip()
if length.isdigit():
length = int(length)
else:
continue
# read data
data = resp.read(length)
# turn json data into status object
if 'in_reply_to_status_id' in data:
status = parse_status(json.loads(data), self.api)
if self.listener.on_status(status) == False:
self.running = False
elif 'delete' in data:
delete = json.loads(data)['delete']['status']
if self.listener.on_delete(delete['id'], delete['user_id']) == False:
self.running = False
elif 'limit' in data:
if self.listener.on_limit(json.loads(data)['limit']['track']) == False:
self.running = False
def firehose(self, count=None):
if self.running:
raise TweepError('Stream object already connected!')
self.url = '/%i/statuses/firehose.json?delimited=length' % STREAM_VERSION
if count:
self.url += '&count=%s' % count
self.running = True
Thread(target=self._run).start()
def retweet(self):
if self.running:
raise TweepError('Stream object already connected!')
self.url = '/%i/statuses/retweet.json?delimited=length' % STREAM_VERSION
self.running = True
Thread(target=self._run).start()
def sample(self, count=None):
if self.running:
raise TweepError('Stream object already connected!')
self.url = '/%i/statuses/sample.json?delimited=length' % STREAM_VERSION
if count:
self.url += '&count=%s' % count
self.running = True
Thread(target=self._run).start()
def filter(self, follow=None, track=None):
if self.running:
raise TweepError('Stream object already connected!')
self.url = '/%i/statuses/filter.json?delimited=length' % STREAM_VERSION
if follow:
self.url += '&follow=%s' % ','.join(follow)
if track:
self.url += '&track=%s' % ','.join(track)
self.running = True
Thread(target=self._run).start()
def disconnect(self):
if self.running is False:
return
self.running = False
class Stream(object):
host = 'stream.twitter.com'
def __init__(self,
username,
password,
listener,
timeout=5.0,
retry_count=None,
retry_time=10.0,
snooze_time=5.0,
buffer_size=1500,
headers=None):
self.auth = BasicAuthHandler(username, password)
self.running = False
self.timeout = timeout
self.retry_count = retry_count
self.retry_time = retry_time
self.snooze_time = snooze_time
self.buffer_size = buffer_size
self.listener = listener
self.api = API()
self.headers = headers or {}
self.body = None
def _run(self):
# setup
self.auth.apply_auth(None, None, self.headers, None)
# enter loop
error_counter = 0
conn = None
exception = None
while self.running:
if self.retry_count and error_counter > self.retry_count:
# quit if error count greater than retry count
break
try:
conn = httplib.HTTPConnection(self.host)
conn.connect()
conn.sock.settimeout(self.timeout)
conn.request('POST', self.url, self.body, headers=self.headers)
resp = conn.getresponse()
if resp.status != 200:
if self.listener.on_error(resp.status) is False:
break
error_counter += 1
sleep(self.retry_time)
else:
error_counter = 0
self._read_loop(resp)
except timeout:
if self.listener.on_timeout() == False:
break
if self.running is False:
break
conn.close()
sleep(self.snooze_time)
except Exception, exception:
# any other exception is fatal, so kill loop
break
# cleanup
self.running = False
if conn:
conn.close()
if exception:
raise exception
