def __init__(self, username, password, listener, timeout=5.0, retry_count = None, retry_time = 10.0, snooze_time = 5.0, buffer_size=1500): self.auth = BasicAuthHandler(username, password) self.running = False self.timeout = timeout self.retry_count = retry_count self.retry_time = retry_time self.snooze_time = snooze_time self.buffer_size = buffer_size self.listener = listener self.api = API() self.headers = {} self.body = None
def filter_track(): track = ["python"] stream_auth = BasicAuthHandler('<USERNAME>', '<PASSWORD>') api = API() stream = Stream(stream_auth, MyStreamListener(api)) print 'start filter track ', ','.join(track) stream.filter(track=track)
def build_stream(self): if self.stream != None: self.stop_query() time.sleep(.01) # make sure old stream has time to disconnect self.stream = Stream(BasicAuthHandler(self.username, self.password), self, # this object implements StreamListener timeout = 600, # reconnect if no messages in 600s retry_count = 20, # try reconnecting 20 times retry_time = 10.0, # wait 10s if no HTTP 200 snooze_time = 1.0) # wait 1s if timeout in 600s
def __init__(self, username, password, listener, timeout=5.0, retry_count = None, retry_time = 10.0, snooze_time = 5.0, buffer_size=1500): self.auth = BasicAuthHandler(username, password) self.running = False self.timeout = timeout self.retry_count = retry_count self.retry_time = retry_time self.snooze_time = snooze_time self.buffer_size = buffer_size self.listener = listener self.api = API()
def _stream_init(self): api1 = API() headers = {} headers["Accept-Encoding"] = "deflate, gzip" stream_auth = BasicAuthHandler(social_keys.TWITTER_HTTP_AUTH_U, social_keys.TWITTER_HTTP_AUTH_P) l = TestStreamListener(api=api1) self._stream = Stream(auth=stream_auth, listener=l, secure=True, headers=headers)
def basic_authenticate(self, file_name): """ Creates an Authhandler using Basic method """ try: with open(file_name) as f: cred = f.readline().split(',') except IOError: print "Error : Authentication file not found" if len(cred) != 2: print "Error : Expecting to retrieve 2 values" else: #username, password self.auth = BasicAuthHandler(cred[0], cred[1])
def __init__(self, username, password, listener, timeout=5.0, retry_count = None, retry_time = 10.0, snooze_time = 5.0, buffer_size=1500, headers=None, proxy_host=None,proxy_port=80): self.auth = BasicAuthHandler(username, password) self.running = False self.timeout = timeout self.retry_count = retry_count self.retry_time = retry_time self.snooze_time = snooze_time self.buffer_size = buffer_size self.listener = listener self.api = API() self.headers = headers or {} self.body = None self.proxy_host = proxy_host self.proxy_port = proxy_port
def main(): global follow_list infoModule.info.site['dblink'] = mysql_tools.db_connect() auth = BasicAuthHandler(twitterAccount, tiwtterPassword) stream = newStream(auth, StreamWatcherListener(), timeout=None) follow_list = getUserList() # Demo Mode #follow_list = getUserList(True) infoModule.info.site['dblink'].close() if len(follow_list) == 0: file_put_contents('Could not get a list of users to follow') sys.exit() stream.filter(follow=follow_list)
def train_tweets(): tokyo = LocationFence("Tokyo", 138.946381, 35.523285, 139.953232, 35.906849) hokkaido = LocationFence("Hokkaido", 139.546509, 41.393294, 145.742798, 45.729191) kyusyu = LocationFence("Kyusyu", 129.538879, 31.147006, 131.856995, 33.934245) locations = [tokyo, hokkaido, kyusyu] request_coordinates = [] for l in locations: request_coordinates += l.get_coordinates() stream = Stream(BasicAuthHandler(tw_username, tw_password), Trainer(locations), secure=True) stream.filter(locations=request_coordinates)
class Stream(object): host = 'stream.twitter.com' def __init__(self, username, password, listener, timeout=5.0, retry_count = None, retry_time = 10.0, snooze_time = 5.0, buffer_size=1500): self.auth = BasicAuthHandler(username, password) self.running = False self.timeout = timeout self.retry_count = retry_count self.retry_time = retry_time self.snooze_time = snooze_time self.buffer_size = buffer_size self.listener = listener self.api = API() self.headers = {} self.body = None def _run(self): # setup self.auth.apply_auth(None, None, self.headers, None) # enter loop error_counter = 0 conn = None while self.running: if self.retry_count and error_counter > self.retry_count: # quit if error count greater than retry count break try: conn = httplib.HTTPConnection(self.host) conn.connect() conn.sock.settimeout(self.timeout) conn.request('POST', self.url, self.body, headers=self.headers) resp = conn.getresponse() if resp.status != 200: if self.listener.on_error(resp.status) is False: break error_counter += 1 sleep(self.retry_time) else: error_counter = 0 self._read_loop(resp) except timeout: if self.listener.on_timeout() == False: break if self.running is False: break conn.close() sleep(self.snooze_time) except Exception: # any other exception is fatal, so kill loop break # cleanup self.running = False if conn: conn.close() def _read_loop(self, resp): data = '' while self.running: if resp.isclosed(): break # read length length = '' while True: c = resp.read(1) if c == '\n': break length += c length = length.strip() if length.isdigit(): length = int(length) else: continue # read data and pass into listener data = resp.read(length) if self.listener.on_data(data) is False: self.running = False def _start(self, async): self.running = True if async: Thread(target=self._run).start() else: self._run() def firehose(self, count=None, async=False): if self.running: raise TweepError('Stream object already connected!') self.url = '/%i/statuses/firehose.json?delimited=length' % STREAM_VERSION if count: self.url += '&count=%s' % count self._start(async)
conn = connect(opts.dsn) cursor = conn.cursor() # Don't know how this happens if status.geo == None: print 'No geo information' return if status.geo['type'] != 'Point': print 'Geo is of type ' + status.geo['type'] return # Insert the status into the DB. wkt = 'SRID=4326;POINT(%s %s)' % (status.geo['coordinates'][1], status.geo['coordinates'][0]) cursor.execute( 'INSERT INTO ' + opts.table + ' (screenname, tweet, the_geom) VALUES ' + '(%s, %s, ST_GeomFromEWKT(%s))', (status.user.screen_name, status.text, wkt)) conn.commit() listener = PGSQLListener() auth = BasicAuthHandler(args[0], args[1]) s = Stream(auth, listener, secure=True) #s.filter(track="foothill college") s.filter(locations=[float(i) for i in opts.geo.split(',')])
class Stream(object): host = 'stream.twitter.com' def __init__(self, username, password, listener, timeout=5.0, retry_count = None, retry_time = 10.0, snooze_time = 5.0, buffer_size=1500): self.auth = BasicAuthHandler(username, password) self.running = False self.timeout = timeout self.retry_count = retry_count self.retry_time = retry_time self.snooze_time = snooze_time self.buffer_size = buffer_size self.listener = listener self.api = API() self.headers = {} self.body = None def _run(self): print 'run....' # setup self.auth.apply_auth(None, None, self.headers, None) # enter loop error_counter = 0 conn = None while self.running: if self.retry_count and error_counter > self.retry_count: # quit if error count greater than retry count break try: conn = httplib.HTTPConnection(self.host) conn.connect() conn.sock.settimeout(self.timeout) conn.request('POST', self.url, self.body, headers=self.headers) resp = conn.getresponse() if resp.status != 200: if self.listener.on_error(resp.status) is False: break error_counter += 1 sleep(self.retry_time) else: error_counter = 0 self._read_loop(resp) except timeout: if self.listener.on_timeout() == False: break if self.running is False: break conn.close() sleep(self.snooze_time) except Exception: # any other exception is fatal, so kill loop break # cleanup self.running = False if conn: conn.close() def _read_loop(self, resp): data = '' while self.running: if resp.isclosed(): break # read length length = '' while True: c = resp.read(1) if c == '\n': break length += c length = length.strip() if length.isdigit(): length = int(length) else: continue # read data and pass into listener data = resp.read(length) if self.listener.on_data(data) is False: self.running = False def _start(self, async): self.running = True if async: Thread(target=self._run).start() else: self._run() def firehose(self, count=None, async=False): if self.running: raise TweepError('Stream object already connected!') self.url = '/%i/statuses/firehose.json?delimited=length' % STREAM_VERSION if count: self.url += '&count=%s' % count self._start(async)
class Stream(object): host = 'stream.twitter.com' def __init__(self, username, password, listener, timeout=5.0, retry_count = None, retry_time = 10.0, snooze_time = 5.0, buffer_size=1500, headers=None, proxy_host=None,proxy_port=80): self.auth = BasicAuthHandler(username, password) self.running = False self.timeout = timeout self.retry_count = retry_count self.retry_time = retry_time self.snooze_time = snooze_time self.buffer_size = buffer_size self.listener = listener self.api = API() self.headers = headers or {} self.body = None self.proxy_host = proxy_host self.proxy_port = proxy_port def _run(self): # setup self.auth.apply_auth(None, None, self.headers, None) # enter loop error_counter = 0 conn = None exception = None while self.running: if self.retry_count and error_counter > self.retry_count: # quit if error count greater than retry count break try: if not self.proxy_host: conn = httplib.HTTPConnection(self.host) else: conn = httplib.HTTPConnection(self.proxy_host, self.proxy_port) conn.connect() conn.sock.settimeout(self.timeout) _url=self.url if self.proxy_host: _url="http://" + self.host + self.url conn.request('POST', _url, self.body, headers=self.headers) resp = conn.getresponse() if resp.status != 200: if self.listener.on_error(resp.status) is False: break error_counter += 1 sleep(self.retry_time) else: error_counter = 0 self._read_loop(resp) except timeout: if self.listener.on_timeout() == False: break if self.running is False: break conn.close() sleep(self.snooze_time) except Exception, exception: # any other exception is fatal, so kill loop break # cleanup self.running = False if conn: conn.close() if exception: raise exception
class Stream(object): host = 'stream.twitter.com' def __init__(self, username, password, listener, timeout=5.0, retry_count = None, retry_time = 10.0, snooze_time = 5.0, buffer_size=1500): self.auth = BasicAuthHandler(username, password) self.running = False self.timeout = timeout self.retry_count = retry_count self.retry_time = retry_time self.snooze_time = snooze_time self.buffer_size = buffer_size self.listener = listener self.api = API() def _run(self): # setup headers = {} self.auth.apply_auth(None, None, headers, None) # enter loop error_counter = 0 conn = None while self.running: if self.retry_count and error_counter > self.retry_count: # quit if error count greater than retry count break try: conn = httplib.HTTPConnection(self.host) conn.connect() conn.sock.settimeout(self.timeout) conn.request('POST', self.url, headers=headers) resp = conn.getresponse() if resp.status != 200: if self.listener.on_error(resp.status) is False: break error_counter += 1 sleep(self.retry_time) else: error_counter = 0 self._read_loop(resp) except timeout: if self.listener.on_timeout() == False: break if self.running is False: break conn.close() sleep(self.snooze_time) except Exception: # any other exception is fatal, so kill loop break # cleanup self.running = False if conn: conn.close() def _read_loop(self, resp): data = '' while self.running: if resp.isclosed(): break # read length length = '' while True: c = resp.read(1) if c == '\n': break length += c length = length.strip() if length.isdigit(): length = int(length) else: continue # read data data = resp.read(length) # turn json data into status object if 'in_reply_to_status_id' in data: status = parse_status(json.loads(data), self.api) if self.listener.on_status(status) == False: self.running = False elif 'delete' in data: delete = json.loads(data)['delete']['status'] if self.listener.on_delete(delete['id'], delete['user_id']) == False: self.running = False elif 'limit' in data: if self.listener.on_limit(json.loads(data)['limit']['track']) == False: self.running = False def firehose(self, count=None): if self.running: raise TweepError('Stream object already connected!') self.url = '/%i/statuses/firehose.json?delimited=length' % STREAM_VERSION if count: self.url += '&count=%s' % count self.running = True Thread(target=self._run).start() def retweet(self): if self.running: raise TweepError('Stream object already connected!') self.url = '/%i/statuses/retweet.json?delimited=length' % STREAM_VERSION self.running = True Thread(target=self._run).start() def sample(self, count=None): if self.running: raise TweepError('Stream object already connected!') self.url = '/%i/statuses/sample.json?delimited=length' % STREAM_VERSION if count: self.url += '&count=%s' % count self.running = True Thread(target=self._run).start() def filter(self, follow=None, track=None): if self.running: raise TweepError('Stream object already connected!') self.url = '/%i/statuses/filter.json?delimited=length' % STREAM_VERSION if follow: self.url += '&follow=%s' % ','.join(follow) if track: self.url += '&track=%s' % ','.join(track) self.running = True Thread(target=self._run).start() def disconnect(self): if self.running is False: return self.running = False
class Stream(object): host = 'stream.twitter.com' def __init__(self, username, password, listener, timeout=5.0, retry_count=None, retry_time=10.0, snooze_time=5.0, buffer_size=1500, headers=None): self.auth = BasicAuthHandler(username, password) self.running = False self.timeout = timeout self.retry_count = retry_count self.retry_time = retry_time self.snooze_time = snooze_time self.buffer_size = buffer_size self.listener = listener self.api = API() self.headers = headers or {} self.body = None def _run(self): # setup self.auth.apply_auth(None, None, self.headers, None) # enter loop error_counter = 0 conn = None exception = None while self.running: if self.retry_count and error_counter > self.retry_count: # quit if error count greater than retry count break try: conn = httplib.HTTPConnection(self.host) conn.connect() conn.sock.settimeout(self.timeout) conn.request('POST', self.url, self.body, headers=self.headers) resp = conn.getresponse() if resp.status != 200: if self.listener.on_error(resp.status) is False: break error_counter += 1 sleep(self.retry_time) else: error_counter = 0 self._read_loop(resp) except timeout: if self.listener.on_timeout() == False: break if self.running is False: break conn.close() sleep(self.snooze_time) except Exception, exception: # any other exception is fatal, so kill loop break # cleanup self.running = False if conn: conn.close() if exception: raise exception